From a1b36cd530b9d3d996a4a96065e2bdf554be7e85 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 8 Mar 2015 13:46:07 +0000 Subject: [PATCH] added method in ftServer to allow retrieving data from shared/downloaded files (patch from electron, modified) git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@7999 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/ft/ftchunkmap.cc | 18 ++++++++++--- libretroshare/src/ft/ftchunkmap.h | 4 +++ libretroshare/src/ft/ftcontroller.cc | 3 +-- libretroshare/src/ft/ftcontroller.h | 2 +- libretroshare/src/ft/ftdatamultiplex.cc | 36 +++++++++++++++++++++++++ libretroshare/src/ft/ftdatamultiplex.h | 10 +++++++ libretroshare/src/ft/ftfilecreator.cc | 33 ++++++++++++++++++++++- libretroshare/src/ft/ftfilecreator.h | 2 +- libretroshare/src/ft/ftfileprovider.cc | 2 +- libretroshare/src/ft/ftfileprovider.h | 11 +++++++- libretroshare/src/ft/ftserver.cc | 9 +++++-- libretroshare/src/ft/ftserver.h | 7 ++++- libretroshare/src/retroshare/rsfiles.h | 21 ++++++++++++--- 13 files changed, 140 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/ft/ftchunkmap.cc b/libretroshare/src/ft/ftchunkmap.cc index 1fff8a3b7..49550077b 100644 --- a/libretroshare/src/ft/ftchunkmap.cc +++ b/libretroshare/src/ft/ftchunkmap.cc @@ -402,7 +402,17 @@ void ChunkMap::removeInactiveChunks(std::vector& to_remove) ++it ; } -bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const +bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const +{ + return hasChunkState(offset, chunk_size, FileChunksInfo::CHUNK_DONE); +} + +bool ChunkMap::isChunkOutstanding(uint64_t offset, uint32_t chunk_size) const +{ + return hasChunkState(offset, chunk_size, FileChunksInfo::CHUNK_OUTSTANDING); +} + +bool ChunkMap::hasChunkState(uint64_t offset, uint32_t chunk_size, FileChunksInfo::ChunkState state) const { uint32_t chunk_number_start = offset/(uint64_t)_chunk_size ; uint32_t chunk_number_end = (offset+(uint64_t)chunk_size)/(uint64_t)_chunk_size ; @@ -414,16 +424,16 @@ bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const // chunk_size=0, and offset%_chunk_size=0, so the response "true" is still valid. // for(uint32_t i=chunk_number_start;i& to_remove) ; @@ -206,6 +208,8 @@ class ChunkMap uint32_t getAvailableChunk(const RsPeerId& peer_id,bool& chunk_map_too_old) ; private: + bool hasChunkState(uint64_t offset, uint32_t chunk_size, FileChunksInfo::ChunkState state) const; + uint64_t _file_size ; //! total size of the file in bytes. uint32_t _chunk_size ; //! Size of chunks. Common to all chunks. FileChunksInfo::ChunkStrategy _strategy ; //! how do we allocate new chunks diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index 8a3fde866..6dee42a26 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -1524,7 +1524,7 @@ bool ftController::FileClearCompleted() } /* get Details of File Transfers */ -bool ftController::FileDownloads(std::list &hashs) +void ftController::FileDownloads(std::list &hashs) { RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ @@ -1537,7 +1537,6 @@ bool ftController::FileDownloads(std::list &hashs) { hashs.push_back(it->second->mHash); } - return true; } diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index b19836b86..f78b66d86 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -167,7 +167,7 @@ class ftController: public CacheTransfer, public RsThread, public pqiServiceMoni uint32_t getMinPrioritizedTransfers() ; /* get Details of File Transfers */ - bool FileDownloads(std::list &hashs); + void FileDownloads(std::list &hashs); /* Directory Handling */ bool setDownloadDirectory(std::string path); diff --git a/libretroshare/src/ft/ftdatamultiplex.cc b/libretroshare/src/ft/ftdatamultiplex.cc index 46f2fba44..05c8265ef 100644 --- a/libretroshare/src/ft/ftdatamultiplex.cc +++ b/libretroshare/src/ft/ftdatamultiplex.cc @@ -77,6 +77,42 @@ ftDataMultiplex::ftDataMultiplex(const RsPeerId& ownId, ftDataSend *server, ftSe return; } +bool ftDataMultiplex::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size, uint8_t *data) +{ + RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ + ftFileProvider* provider = 0; + + std::map::iterator cit; + std::map::iterator sit; + + // check if file is currently downloading + if (mClients.end() != (cit = mClients.find(hash))) + provider = (cit->second).mCreator; + + // else check if its already uploading + else if (mServers.end() != (sit = mServers.find(hash))) + provider = sit->second; + + // else create a new provider + else + { + FileInfo info; + FileSearchFlags hintflags = RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY | RS_FILE_HINTS_NETWORK_WIDE; + if(mSearch->search(hash, hintflags, info)) + { + provider = new ftFileProvider(info.path, info.size, hash); + mServers[hash] = provider; + } + } + + if(!provider || ! provider->getFileData(mOwnId, offset, requested_size, data, true)) + { + requested_size = 0 ; + return false ; + } + return true ; +} + bool ftDataMultiplex::addTransferModule(ftTransferModule *mod, ftFileCreator *f) { RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ diff --git a/libretroshare/src/ft/ftdatamultiplex.h b/libretroshare/src/ft/ftdatamultiplex.h index 0a670134d..1679e7832 100644 --- a/libretroshare/src/ft/ftdatamultiplex.h +++ b/libretroshare/src/ft/ftdatamultiplex.h @@ -95,6 +95,16 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread ftDataMultiplex(const RsPeerId& ownId, ftDataSend *server, ftSearch *search); + /** + * @see RsFiles::getFileData + * + * data should be pre-allocated by the client with size sufficient gfor requested_size bytes. + * requested_size will be changed so as to contain the actual number of bytes copied from the file, + * in case where the full size wasn't available. + * False is returned if no data can be obtained from that file. + */ + bool getFileData(const RsFileHash& hash, uint64_t offset,uint32_t& requested_size, uint8_t *data); + /* ftController Interface */ bool addTransferModule(ftTransferModule *mod, ftFileCreator *f); bool removeTransferModule(const RsFileHash& hash); diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index 80a0e90e6..ec6d0099f 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -65,7 +65,7 @@ ftFileCreator::ftFileCreator(const std::string& path, uint64_t size, const RsFil #endif } -bool ftFileCreator::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data) +bool ftFileCreator::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool allow_unverified) { // Only send the data if we actually have it. // @@ -77,6 +77,37 @@ bool ftFileCreator::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ have_it = chunkMap.isChunkAvailable(offset, chunk_size) ; + +#define ENABLE_SLICES +#ifdef ENABLE_SLICES + // try if we have data from an incomplete or not veryfied chunk + if(!have_it && allow_unverified) + { + std::map::iterator it; + have_it = true; + // this map contains chunks which are currently being downloaded + for(std::map::iterator it=mChunks.begin(); it!=mChunks.end(); ++it) + { + ftChunk chunk = it->second; + // begin of slice is in requested range + if(chunk.offset >= offset && chunk.offset < (offset+chunk_size)) + { + // reduce the requested size + chunk_size = chunk.offset - offset; + } + // end of slice is in requested range + if((chunk.offset+chunk.size) >= offset && (chunk.offset+chunk.size) < (offset+chunk_size)) + { + // can do nothing about this + have_it = false; + } + } + // check if the chunk was already started to download + // if not, we don't have it + if(chunkMap.isChunkOutstanding(offset, chunk_size)) + have_it = false; + } +#endif } #ifdef FILE_DEBUG if(have_it) diff --git a/libretroshare/src/ft/ftfilecreator.h b/libretroshare/src/ft/ftfilecreator.h index b331991e5..912b74814 100644 --- a/libretroshare/src/ft/ftfilecreator.h +++ b/libretroshare/src/ft/ftfilecreator.h @@ -52,7 +52,7 @@ class ftFileCreator: public ftFileProvider ~ftFileCreator(); /* overloaded from FileProvider */ - virtual bool getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data); + virtual bool getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool allow_unverified = false); bool finished() ; uint64_t getRecvd(); diff --git a/libretroshare/src/ft/ftfileprovider.cc b/libretroshare/src/ft/ftfileprovider.cc index 584bdd379..50e1f8bdb 100644 --- a/libretroshare/src/ft/ftfileprovider.cc +++ b/libretroshare/src/ft/ftfileprovider.cc @@ -139,7 +139,7 @@ void ftFileProvider::getAvailabilityMap(CompressedChunkMap& cmap) } -bool ftFileProvider::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data) +bool ftFileProvider::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool /*allow_unverified*/) { /* dodgey checking outside of mutex... * much check again inside FileAttrs(). diff --git a/libretroshare/src/ft/ftfileprovider.h b/libretroshare/src/ft/ftfileprovider.h index 9f48f54b7..8b6acf9f6 100644 --- a/libretroshare/src/ft/ftfileprovider.h +++ b/libretroshare/src/ft/ftfileprovider.h @@ -41,7 +41,16 @@ class ftFileProvider ftFileProvider(const std::string& path, uint64_t size, const RsFileHash& hash); virtual ~ftFileProvider(); - virtual bool getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data); + /** + * read a block of data from the file + * @param peer_id for the uploading stats: to which peer the data will be send + * @param offset begin of the requested data range + * @param chunk_size how many bytes to read. Will be set to the number of valid bytes in data on return. + * @param data pointer to a buffer of size chunk_size. Contains the data on success. + * @param allow_unverified for ftFileCreator: set to true to return data from unverified chunks. Use it if you want data asap. + * @return true if data was read + */ + virtual bool getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool allow_unverified = false); virtual bool FileDetails(FileInfo &info); RsFileHash getHash(); uint64_t getFileSize(); diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 99ca19595..5113c8340 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -262,6 +262,11 @@ bool ftServer::checkHash(const RsFileHash& hash,std::string& error_string) return true ; } +bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data) +{ + return mFtDataplex->getFileData(hash, offset, requested_size,data); +} + bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) { return mFtController->alreadyHaveFile(hash, info); @@ -403,9 +408,9 @@ std::string ftServer::getPartialsDirectory() /************************* Other Access ************************/ /***************************************************************/ -bool ftServer::FileDownloads(std::list &hashs) +void ftServer::FileDownloads(std::list &hashs) { - return mFtController->FileDownloads(hashs); + mFtController->FileDownloads(hashs); /* this only contains downloads.... not completed */ //return mFtDataplex->FileDownloads(hashs); } diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 1f8a9af19..0c5b12aa7 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -124,6 +124,11 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; } ftController *getController() const { return mFtController ; } + /** + * @see RsFiles::getFileData + */ + bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data); + /*** * Control of Downloads ***/ @@ -157,7 +162,7 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu /*** * Download/Upload Details ***/ - virtual bool FileDownloads(std::list &hashs); + virtual void FileDownloads(std::list &hashs); virtual bool FileUploads(std::list &hashs); virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info); virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ; diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index d52c30453..c6480b540 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -109,9 +109,22 @@ class RsFiles RsFiles() { return; } virtual ~RsFiles() { return; } - /****************************************/ - /* download */ - + /** + * Provides file data for the gui: media streaming or rpc clients. + * It may return unverified chunks. This allows streaming without having to wait for hashes or completion of the file. + * This function returns an unspecified amount of bytes. Either as much data as available or a sensible maximum. Expect a block size of around 1MiB. + * To get more data, call this function repeatedly with different offsets. + * Returns false in case + * - the files is not available on the local node + * - not downloading + * - the requested data was not downloaded yet + * - end of file was reached + * @param hash hash of a file. The file has to be available on this node or it has to be in downloading state. + * @param offset where the desired block starts + * @param requested_size size of pre-allocated data. Will be updated by the function. + * @param data pre-allocated memory chunk of size 'requested_size' by the client + */ + virtual bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data)=0; /*** * Control of Downloads. @@ -147,7 +160,7 @@ class RsFiles /*** * Download / Upload Details. ***/ - virtual bool FileDownloads(std::list &hashs) = 0; + virtual void FileDownloads(std::list &hashs) = 0; virtual bool FileUploads(std::list &hashs) = 0; virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) = 0;