From 13a41309a7b716d2b4e5d5dc16bda1be527d1472 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Aug 2014 14:22:30 +0000 Subject: [PATCH 001/174] branched for new file list sharing, and getting rid of cache system git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.6-FileLists@7493 b45a01b8-16f6-495d-af2f-9b41ad6348cc From 89af650f7447b2038dd8902b0cfe45c0d6742c55 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 14 Apr 2016 18:25:12 -0400 Subject: [PATCH 002/174] basic bits of new file list sharing system --- libretroshare/src/file_sharing/README.txt | 41 +++++++++++++++++++ .../src/file_sharing/directory_list.h | 13 ++++++ .../src/file_sharing/directory_updater.h | 25 +++++++++++ libretroshare/src/file_sharing/p3filelists.h | 29 +++++++++++++ libretroshare/src/libretroshare.pro | 4 ++ 5 files changed, 112 insertions(+) create mode 100644 libretroshare/src/file_sharing/directory_list.h create mode 100644 libretroshare/src/file_sharing/directory_updater.h create mode 100644 libretroshare/src/file_sharing/p3filelists.h diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index f5e298afc..19a24db43 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -87,6 +87,20 @@ Generating sync events * Server side - after a change, broadcast a "directory changed" packet to all connected friends + + DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy) + | | + | | + | | + +-----------------------+------------------+ + | | + Shared File Service | + | | + | | + +----------- own file list -------+---------- Encrypted/compressed save to disk + | | | + +----------- friend file lists ---+ + Roadmap ------- @@ -99,3 +113,30 @@ Roadmap - optionally - change the saving system of FileIndex to make it locally encrypted and compact +TODO +==== + [ ] implement directory updater + [ ] local directory updater + [ ] remote directory updater + + [ ] implement directory handler + [ ] implement p3FileLists with minimal functonality: no exchange. Only storage of own + + + + + + + + + + + + + + + + + + + diff --git a/libretroshare/src/file_sharing/directory_list.h b/libretroshare/src/file_sharing/directory_list.h new file mode 100644 index 000000000..c643b6153 --- /dev/null +++ b/libretroshare/src/file_sharing/directory_list.h @@ -0,0 +1,13 @@ +// This class keeps a shared directory. It's quite the equivalent of the old "FileIndex" class +// The main difference is that it is +// - extensible +// - fast to search (at least for hashes). Should provide possibly multiple search handles for +// the same file, e.g. if connexion is encrypted. +// - abstracts the browsing in a same manner. +// +class SharedDirectoryList +{ + public: + + DirEntry mRoot ; +}; diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h new file mode 100644 index 000000000..be59eb987 --- /dev/null +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -0,0 +1,25 @@ +// This class crawls the given directry hierarchy and updates it. It does so by calling the +// shared file list source. This source may be of two types: +// - local: directories are crawled n disk and files are hashed / requested from a cache +// - remote: directories are requested remotely to a providing client +// +class DirectoryUpdater +{ + public: + DirectoryUpdater() ; + + // Does some updating job. Crawls the existing directories and checks wether it has been updated + // recently enough. If not, calls the directry source. + // + void tick() ; + + // +}; + +class LocalDirectoryUpdater: public DirectoryUpdater +{ +}; + +class RemoteDirectoryUpdater: public DirectoryUpdater +{ +}; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h new file mode 100644 index 000000000..7bddccc53 --- /dev/null +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -0,0 +1,29 @@ +// This class is responsible for +// - maintaining a list of shared file hierarchies for each known friends +// - talking to the GUI +// - providing handles for the directory tree listing GUI +// - providing search handles for FT +// - keeping these lists up to date +// - sending our own file list to friends depending on the defined access rights +// - serving file search requests from other services such as file transfer +// +// p3FileList does the following micro-tasks: +// - tick the watchers +// - get incoming info from the service layer, which can be: +// - directory content request => the directory content is shared to the friend +// - directory content => the directory watcher is notified +// - keep two queues of update requests: +// - fast queue that is handled in highest priority. This one is used for e.g. updating while browsing. +// - slow queue that is handled slowly. Used in background update of shared directories. +// +// The file lists are not directry updated. A FileListWatcher class is responsible for this +// in every case. +// +class p3FileLists: public p3Service +{ + public: + p3FileLists() ; + + +}; + diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index d01d4c9dd..13b315713 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -354,6 +354,10 @@ HEADERS += ft/ftchunkmap.h \ ft/fttransfermodule.h \ ft/ftturtlefiletransferitem.h +HEADERS += directory_updater.h \ + directory_list.h \ + p3filelists.h + HEADERS += chat/distantchat.h \ chat/p3chatservice.h \ chat/distributedchat.h \ From a6d467d13851aaad75bd1773309b9be80ea47402 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 18 Jul 2016 21:33:54 -0400 Subject: [PATCH 003/174] added interface for p3FileLists --- libretroshare/src/file_sharing/README.txt | 89 ++++++++++++++++++-- libretroshare/src/file_sharing/p3filelists.h | 61 +++++++++++++- 2 files changed, 140 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index 19a24db43..9cd98b84c 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -40,9 +40,35 @@ Big picture - the same file should be able to be held by two different directories. Hash search will return a single hit. - the previously existing PersonEntry had no field and was overloading DirEntry, with overwritten file names, hashes etc. Super bad! +Directory storage file format +----------------------------- + * should be extensible (xml or binary format? Binary, because it's going to be encrypted anyway) + => works with binary fields + => + + [= version =] + [= peer id =] + [= num entries =] + [= some information =] + + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ... + ... + 2 1-5 v 2 1-5 v + * entry content + Tag | Content | Size + ----------------+--------------------------------------+------ + 01 | sha1 hash | 20 + 01 | sha1^2 hash | 20 + 02 | file name | < 512 + 03 | file size | 8 + 04 | dir name | < 512 + 05 | last modif time local | 4 + 06 | last modif time including sub-dirs | 4 + Classes ------- - Rs p3ShareManager - tick() @@ -75,6 +101,29 @@ Classes - parent groups - group flags +Best data structure for file index +---------------------------------- + + | Hash map map list + ----------------+-----------------+------------+-------------- + Adding | Constant | log(n) | O(n) + Hash search | Constant | log(n) | O(n) + Name/exp search | O(n) | O(n) | O(n) + Recursive browse| Constant | log(n) | O(n) + + Should we use the same struct for files and directories? + + Sol 1: + DirClass + PersonClass + FileEntry class + - each has pointers to elements list of the same type + - lists are handled for Files (all file entries), + + Directories are represented by the hash of the full path + + Sol 2: + Same class for all elements, in a single hash map. Each element is + defined by its type (Dir, Person, File) which all have a hash. + Syncing between peers --------------------- @@ -83,10 +132,18 @@ Generating sync events - for each directory, in breadth first order - if directory has changed, or last update is old => push a sync request + - store the peer's last up time. Compare with peer uptimes recursively. * Server side - after a change, broadcast a "directory changed" packet to all connected friends - + + * directoy updater + - crawl through directories + - compare TS of files, missing files, new files + - feed a queue of files to hash + - directory whatcher gets notified when files are hashed + + - a separate component hashes files (FileHashingProcess) DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy) | | @@ -100,7 +157,6 @@ Generating sync events +----------- own file list -------+---------- Encrypted/compressed save to disk | | | +----------- friend file lists ---+ - Roadmap ------- @@ -115,12 +171,29 @@ Roadmap TODO ==== - [ ] implement directory updater - [ ] local directory updater - [ ] remote directory updater - [ ] implement directory handler - [ ] implement p3FileLists with minimal functonality: no exchange. Only storage of own + [ ] directory handler + [ ] abstract functions to keep a directory and get updates to it. + [ ] hierarchical storage representation. + [ ] allow add/delete entries + [ ] auto-cleanup + + [ ] directory updater + [ ] abstract layer + [ ] crawls the directory and ask updates + + [ ] derive local directory updater + [ ] crawl local files, and asks updates to storage class + [ ] derive remote directory updater + [ ] crawl stored files, and request updates to storage class + + [ ] load/save of directory content. Should be extensible + [ ] p3FileLists with minimal functonality: no exchange. Only storage of own file lists + [ ] service (items) for p3FileLists + [ ] connect RemoteDirModel to new system + [ ] test GUI functions + [ ] test update between peers + diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 7bddccc53..bca69ca6f 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -19,11 +19,68 @@ // The file lists are not directry updated. A FileListWatcher class is responsible for this // in every case. // -class p3FileLists: public p3Service +#include +#include + +class p3FileLists: public p3Service, public p3Config, public RsSharedFileService { public: - p3FileLists() ; + typedef uint64_t EntryIndex ; // this should probably be defined elsewhere + p3FileLists(mPeerMgr *mpeers) ; + /* + */ + + // derived from p3Service + // + virtual int tick() ; + + // access to own/remote shared files + // + virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; + + virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; + virtual int SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; + + // Interface for browsing dir hierarchy + // + int RequestDirDetails(EntryIndex, DirDetails&, FileSearchFlags) const ; + uint32_t getType(const EntryIndex&) const ; + int RequestDirDetails(const std::string& path, DirDetails &details) const ; + + // set/update shared directories + void setSharedDirectories(const std::list& dirs); + void getSharedDirectories(std::list& dirs); + void updateShareFlags(const SharedDirInfo& info) ; + + void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed + bool inDirectoryCheck(); + + // Derived from p3Config + // + private: + p3PeerMgr *mPeers ; + + // File sync request queues. The fast one is used for online browsing when friends are connected. + // The slow one is used for background update of file lists. + // + std::list mFastRequestQueue ; + std::list mSlowRequestQueue ; + + // Directory storage hierarchies + // + std::map mRemoteDirectories ; + + LocalSharedDirectoryMap *mLocalSharedDirs ; + + RemoteSharedDirectoryWatcher *mRemoteDirWatcher ; + LocalSharedDirectoryWatcher *mLocalDirWatcher ; + + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. + // + HashCache *mHashCache ; + + RsMutex mFLSMtx ; }; From b31e396c2bb577811d66a26f1d496845997c52c6 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 18 Jul 2016 21:52:44 -0400 Subject: [PATCH 004/174] added skeleton code for p3filelists, and interface files for directory storage and hash cache --- .../src/file_sharing/directory_storage.h | 65 ++++++++++ libretroshare/src/file_sharing/hash_cache.h | 29 +++++ libretroshare/src/file_sharing/p3filelists.cc | 113 ++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 libretroshare/src/file_sharing/directory_storage.h create mode 100644 libretroshare/src/file_sharing/hash_cache.h create mode 100644 libretroshare/src/file_sharing/p3filelists.cc diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h new file mode 100644 index 000000000..1ba46a062 --- /dev/null +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -0,0 +1,65 @@ +#include + +static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; + +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; +static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; +static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; +static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; +static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; + +class DirectoryStorage +{ + public: + DirectoryStorage(const std::string& local_file_name) ; + + void save() const ; + + virtual int searchTerms(const std::list& terms, std::list &results) const; + virtual int searchHash(const RsFileHash& hash, std::list &results) const; + virtual int searchBoolExp(Expression * exp, std::list &results) const; + + void getFileDetails(EntryIndex i) ; + + // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. + // It gives access to sub-files and sub-directories below. + // + class DirIterator + { + public: + DirIterator(const DirectoryStorage& d) ; + + DirIterator& operator++() ; + EntryIndex operator*() const ; // current directory entry + + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + }; + class FileIterator + { + public: + FileIterator(const DirectoryStorage& d) ; + + FileIterator& operator++() ; + EntryIndex operator*() const ; // current file entry + + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + }; + + private: + void load(const std::string& local_file_name) ; + void save(const std::string& local_file_name) ; + + void loadNextTag(const void *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) ; + void saveNextTag(void *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; +}; + +class RemoteDirectoryStorage: public DirectoryStorage +{ +}; + +class LocalDirectoryStorage: public DirectoryStorage +{ +}; + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h new file mode 100644 index 000000000..144e6d8ee --- /dev/null +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -0,0 +1,29 @@ +class HashCache +{ + public: + HashCache(const std::string& save_file_name) ; + + void save() ; + void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; + bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; + void clean() ; + + typedef struct + { + uint64_t size ; + uint64_t time_stamp ; + uint64_t modf_stamp ; + RsFileHash hash ; + } HashCacheInfo ; + + void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } + uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } + void clear() { _files.clear(); } + bool empty() const { return _files.empty() ; } + private: + uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries + std::map _files ; + std::string _path ; + bool _changed ; +}; + diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc new file mode 100644 index 000000000..baf87dbf3 --- /dev/null +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -0,0 +1,113 @@ +#include +#include + +#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " ; + +static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; +static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; +static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; +static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; + +p3FileLists::p3FileLists(mPeerPgr *mpeers) + : mPeers(mpeers) +{ + // loads existing indexes for friends. Some might be already present here. + // + + // init the directory watchers + + // local variables/structures + // + mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; +} +void p3FileLists::~p3FileLists() +{ + // delete all pointers +} + +void p3FileLists::tick() +{ + // tick the watchers, possibly create new ones if additional friends do connect. + // + tickWatchers(); + + // tick the input/output list of update items and process them + // + tickRecv() ; + tickSend() ; + + // cleanup + // - remove/delete shared file lists for people who are not friend anymore + // - + + cleanup(); +} + +void p3FileLists::tickRecv() +{ +} +void p3FileLists::tickSend() +{ + // go through the list of out requests and send them to the corresponding friends, if they are online. +} + +void p3FileLists::loadList(std::list& items) +{ + // This loads + // + // - list of locally shared directories, and the permissions that go with them +} + +void p3FileLists::saveList(const std::list& items) +{ +} + +void p3FileLists::cleanup() +{ + // look through the list of friend directories. Remove those who are not our friends anymore. + // + P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; + + std::set friend_set ; + { +#warning we should use a std::set in the first place. Modify rsPeers? + std::list friend_list ; + mPeers->getFriendList(friend_list) ; + + for(std::list::const_iterator it(friend_list.begin());it!=friend_list.end();++it) + friend_set.insert(*it) ; + } + + for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) + if(friend_set.find(it->first) == friend_set.end()) + { + P3FILELISTS_DEBUG() << " removing file list of non friend " << it->first << std::endl; + + delete it->second ; + std::map::iterator tmp(it) ; + ++tmp ; + mRemoteDirectories.erase(it) ; + it=tmp ; + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + } + else + ++it ; + + // look through the list of friends, and add a directory storage when it's missing + // + for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) + if(mRemoteDirectories.find(*it) == mRemoteDirectories.end()) + { + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; + + mRemoteDirectories[*it] = new RemoteDirectoryStorage(*it); + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + } + + if(mUpdateFlags) + IndicateConfigChanged(); +} + + From 3e20b9ddf74ad3f292c30b941c12b9ed0592fe9e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 20 Jul 2016 15:28:55 -0400 Subject: [PATCH 005/174] added p3FileLists service --- libretroshare/src/file_sharing/p3filelists.cc | 4 ++-- libretroshare/src/file_sharing/p3filelists.h | 4 +++- libretroshare/src/libretroshare.pro | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index baf87dbf3..20fd14db6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,5 +1,5 @@ -#include -#include +#include "file_sharing/p3filelists.h" +#include "retroshare/rsids.h" #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index bca69ca6f..f0a6e2a1d 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -19,10 +19,12 @@ // The file lists are not directry updated. A FileListWatcher class is responsible for this // in every case. // +#pragma once + #include #include -class p3FileLists: public p3Service, public p3Config, public RsSharedFileService +class p3FileLists: public p3Service, public p3Config, public RsSharedFileService { public: typedef uint64_t EntryIndex ; // this should probably be defined elsewhere diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 94c527b73..b7a113fd7 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -33,6 +33,20 @@ debug { QMAKE_CXXFLAGS *= -g -fno-omit-frame-pointer } +CONFIG += file_lists + +file_lists { + HEADERS *= file_sharing/p3filelists.h \ + file_sharing/hash_cache.h \ + file_sharing/directory_storage.h \ + file_sharing/directory_updater.h + + SOURCES *= file_sharing/p3filelists.cc \ + file_sharing/hash_cache.cc \ + file_sharing/directory_storage.cc \ + file_sharing/directory_updater.cc +} + dsdv { DEFINES *= SERVICES_DSDV HEADERS += services/p3dsdv.h \ From 63063102bb5e6076e14d2a3554087dc355f54c1d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 20 Jul 2016 16:10:51 -0400 Subject: [PATCH 006/174] fixed compilation of file lists code base --- .gitignore | 3 +- .../src/file_sharing/directory_storage.cc | 0 .../src/file_sharing/directory_storage.h | 15 ++++++- .../src/file_sharing/directory_updater.h | 6 ++- libretroshare/src/file_sharing/hash_cache.cc | 0 libretroshare/src/file_sharing/p3filelists.cc | 35 +++++++++++----- libretroshare/src/file_sharing/p3filelists.h | 40 ++++++++++++++++--- 7 files changed, 80 insertions(+), 19 deletions(-) create mode 100644 libretroshare/src/file_sharing/directory_storage.cc create mode 100644 libretroshare/src/file_sharing/hash_cache.cc diff --git a/.gitignore b/.gitignore index eb9b055b3..b799c1eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ moc_*.cpp qrc_*.cpp ui_*.h -Makefile.* +Makefile* +*.sw? *~ Thumbs.db diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc new file mode 100644 index 000000000..e69de29bb diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 1ba46a062..6106e4bb6 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -1,8 +1,12 @@ #include +#include +#include + +#include "retroshare/rsids.h" +#include "retroshare/rsfiles.h" static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; @@ -14,6 +18,9 @@ class DirectoryStorage { public: DirectoryStorage(const std::string& local_file_name) ; + virtual ~DirectoryStorage() {} + + typedef uint32_t EntryIndex ; void save() const ; @@ -57,9 +64,15 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { +public: + RemoteDirectoryStorage(const RsPeerId& pid) ; + virtual ~RemoteDirectoryStorage() {} }; class LocalDirectoryStorage: public DirectoryStorage { +public: + LocalDirectoryStorage() ; + virtual ~LocalDirectoryStorage() {} }; diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index be59eb987..3571247ab 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -11,15 +11,19 @@ class DirectoryUpdater // Does some updating job. Crawls the existing directories and checks wether it has been updated // recently enough. If not, calls the directry source. // - void tick() ; + virtual void tick() =0; // }; class LocalDirectoryUpdater: public DirectoryUpdater { + public: + virtual void tick() ; }; class RemoteDirectoryUpdater: public DirectoryUpdater { + public: + virtual void tick() ; }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc new file mode 100644 index 000000000..e69de29bb diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 20fd14db6..7c35fe332 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,15 +1,16 @@ #include "file_sharing/p3filelists.h" +#include "file_sharing/directory_storage.h" #include "retroshare/rsids.h" -#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " ; +#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -p3FileLists::p3FileLists(mPeerPgr *mpeers) - : mPeers(mpeers) +p3FileLists::p3FileLists(p3LinkMgr *mpeers) + : mLinkMgr(mpeers), mFLSMtx("p3FileLists") { // loads existing indexes for friends. Some might be already present here. // @@ -20,12 +21,19 @@ p3FileLists::p3FileLists(mPeerPgr *mpeers) // mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } -void p3FileLists::~p3FileLists() +p3FileLists::~p3FileLists() { + RS_STACK_MUTEX(mFLSMtx) ; + + for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();++it) + delete it->second ; + + mRemoteDirectories.clear(); // just a precaution, not to leave deleted pointers around. + // delete all pointers } -void p3FileLists::tick() +int p3FileLists::tick() { // tick the watchers, possibly create new ones if additional friends do connect. // @@ -41,6 +49,8 @@ void p3FileLists::tick() // - cleanup(); + + return 0; } void p3FileLists::tickRecv() @@ -51,20 +61,25 @@ void p3FileLists::tickSend() // go through the list of out requests and send them to the corresponding friends, if they are online. } -void p3FileLists::loadList(std::list& items) +bool p3FileLists::loadList(std::list& items) { // This loads // // - list of locally shared directories, and the permissions that go with them + + return true ; } -void p3FileLists::saveList(const std::list& items) +bool p3FileLists::saveList(const std::list& items) { + return true ; } void p3FileLists::cleanup() { - // look through the list of friend directories. Remove those who are not our friends anymore. + RS_STACK_MUTEX(mFLSMtx) ; + + // look through the list of friend directories. Remove those who are not our friends anymore. // P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; @@ -72,13 +87,13 @@ void p3FileLists::cleanup() { #warning we should use a std::set in the first place. Modify rsPeers? std::list friend_list ; - mPeers->getFriendList(friend_list) ; + mLinkMgr->getFriendList(friend_list) ; for(std::list::const_iterator it(friend_list.begin());it!=friend_list.end();++it) friend_set.insert(*it) ; } - for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) + for(std::map::iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) if(friend_set.find(it->first) == friend_set.end()) { P3FILELISTS_DEBUG() << " removing file list of non friend " << it->first << std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index f0a6e2a1d..e2c9d2016 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -21,15 +21,32 @@ // #pragma once -#include -#include +#include "retroshare/rsfiles.h" +#include "services/p3service.h" -class p3FileLists: public p3Service, public p3Config, public RsSharedFileService +#include "pqi/p3cfgmgr.h" +#include "pqi/p3linkmgr.h" + +class RemoteDirectoryStorage ; +class RemoteSharedDirectoryWatcher ; +class LocalSharedDirectoryWatcher ; +class LocalSharedDirectoryMap ; +class HashCache ; + +class p3FileLists: public p3Service, public p3Config //, public RsSharedFileService { public: typedef uint64_t EntryIndex ; // this should probably be defined elsewhere - p3FileLists(mPeerMgr *mpeers) ; + struct RsFileListSyncRequest + { + RsPeerId peerId ; + EntryIndex index ; + // [...] more to add here + }; + + p3FileLists(p3LinkMgr *mpeers) ; + ~p3FileLists(); /* */ @@ -61,8 +78,16 @@ class p3FileLists: public p3Service, public p3Config, public RsSharedFileService // Derived from p3Config // - private: - p3PeerMgr *mPeers ; + protected: + virtual bool loadList(std::list& items); + virtual bool saveList(const std::list& items); + void cleanup(); + void tickRecv(); + void tickSend(); + void tickWatchers(); + + private: + p3LinkMgr *mLinkMgr ; // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. @@ -83,6 +108,9 @@ class p3FileLists: public p3Service, public p3Config, public RsSharedFileService // HashCache *mHashCache ; + // Local flags and mutexes + RsMutex mFLSMtx ; + uint32_t mUpdateFlags ; }; From 3bf3d0c360848e78b48fd64a6ddd970ce909ac48 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 20 Jul 2016 16:11:26 -0400 Subject: [PATCH 007/174] added blank cc file --- .../src/file_sharing/directory_updater.cc | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 libretroshare/src/file_sharing/directory_updater.cc diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc new file mode 100644 index 000000000..72917248b --- /dev/null +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -0,0 +1,35 @@ +#include "directory_storage.h" +#include "directory_updater.h" + +void RemoteDirectoryUpdater::tick() +{ + // use the stored iterator +} + +void LocalDirectoryUpdater::tick() +{ +#ifdef TODO + // recursive update algorithm works that way: + // - the external loop starts on the shared directory list and goes through sub-directories + // - at the same time, it updates the list of shared directories + // - the information that is costly to compute (the hash) is store externally into a separate structure. + // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. + // + for(LocalSharedDirectoryMap::iterator it(mLocalSharedDirs);it;++it) + { + librs::util::FolderIterator dirIt(realpath); + if(!dirIt.isValid()) + mLocalSharedDirs.removeDirectory(it) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + + while(dirIt.readdir()) + { + + } + + if(mHashCache.getHash(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash)) + { + ; + } + } +#endif +} From 3c976bb7ee181d4183d43355c7f530f26cc7318d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 21 Jul 2016 00:16:12 -0400 Subject: [PATCH 008/174] added squeleton code for own directory update --- libretroshare/src/file_sharing/README.txt | 16 +-- .../src/file_sharing/directory_storage.cc | 60 +++++++++++ .../src/file_sharing/directory_storage.h | 48 +++++++-- .../src/file_sharing/directory_updater.cc | 101 +++++++++++++++--- .../src/file_sharing/directory_updater.h | 12 ++- libretroshare/src/file_sharing/hash_cache.h | 65 +++++++---- 6 files changed, 248 insertions(+), 54 deletions(-) diff --git a/libretroshare/src/file_sharing/README.txt b/libretroshare/src/file_sharing/README.txt index 9cd98b84c..1b6cccb2b 100644 --- a/libretroshare/src/file_sharing/README.txt +++ b/libretroshare/src/file_sharing/README.txt @@ -160,14 +160,14 @@ Generating sync events Roadmap ------- -- complete this file until a proper description of the whole thing is achieved. -- create a new directory and implement the .h for the basic functionality -- look into existing code in ftServer for the integration, but don't change anything yet -- setup class hierarchy -- merge hash cache into file lists. - -- optionally - - change the saving system of FileIndex to make it locally encrypted and compact +[X] complete this file until a proper description of the whole thing is achieved. +[X] create a new directory and implement the .h for the basic functionality +[ ] look into existing code in ftServer for the integration, but don't change anything yet +[X] setup class hierarchy +[ ] merge hash cache into file lists. +[ ] new format for saving of FileIndex to make it locally encrypted, compact and extensible +[ ] create basic directory functionality with own files: re-hash, and store +[ ] display own files in GUI, with proper update and working sort TODO ==== diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e69de29bb..acfdcbf25 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -0,0 +1,60 @@ +#include "directory_storage.h" + +class InternalFileHierarchyStorage +{ + class FileStorageNode + { + public: + virtual uint32_t type() const =0; + }; + class FileEntry: public FileStorageNode + { + public: + virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + + }; + + class DirEntry: public FileStorageNode + { + public: + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } + + std::set subdirs ; + std::set subfiles ; + }; + + // file/dir access and modification + bool findSubDirectory(EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + + uint32_t root ; + + std::vector mNodes;// uses pointers to keep information about valid/invalid objects. + + void compress() ; // use empty space in the vector, mostly due to deleted entries. + + friend class DirectoryStorage ; // only class that can use this. +}; + +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; +// static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; + +void DirectoryStorage::loadNextTag(const unsigned char *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) +{ + entry_tag = data[offset++] ; +} +void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_t entry_tag, uint32_t entry_size) +{ +} + +void DirectoryStorage::load(const std::string& local_file_name) +{ + // first load the header, than all fields. +} +void DirectoryStorage::save(const std::string& local_file_name) +{ + // first write the header, than all fields. +} diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 6106e4bb6..0399d9d9e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -14,6 +14,8 @@ static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; +class InternalFileHierarchyStorage ; + class DirectoryStorage { public: @@ -36,30 +38,37 @@ class DirectoryStorage class DirIterator { public: - DirIterator(const DirectoryStorage& d) ; + DirIterator(const DirIterator& d) ; DirIterator& operator++() ; EntryIndex operator*() const ; // current directory entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. }; class FileIterator { public: - FileIterator(const DirectoryStorage& d) ; + FileIterator(DirIterator& d); // crawls all files in specified directory + uint32_t size() const ; // number of files in this directory FileIterator& operator++() ; EntryIndex operator*() const ; // current file entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. + bool operator()() const ; // used in for loops. Returns true when the iterator is valid. }; + virtual DirIterator root() ; // returns the index of the root directory entry. + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; - void loadNextTag(const void *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) ; - void saveNextTag(void *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; + void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + + // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. + + InternalFileHierarchyStorage *mFileHierarchy ; }; class RemoteDirectoryStorage: public DirectoryStorage @@ -74,5 +83,32 @@ class LocalDirectoryStorage: public DirectoryStorage public: LocalDirectoryStorage() ; virtual ~LocalDirectoryStorage() {} + + void setSharedDirectoryList(const std::list& lst) ; + void getSharedDirectoryList(std::list& lst) ; + + /*! + * \brief addFile + * \param dir + * \param hash + * \param modf_time + */ + void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + +private: + std::list mLocalDirs ; + + std::map mHashes ; // used for fast search access }; + + + + + + + + + + diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 72917248b..c20c3703b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -6,30 +6,99 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } +LocalDirectoryUpdater::LocalDirectoryUpdater() +{ + // tell the hash cache that we're the client + mHashCache->setClient(this) ; +} + void LocalDirectoryUpdater::tick() { -#ifdef TODO // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories - // - at the same time, it updates the list of shared directories + // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. // - the information that is costly to compute (the hash) is store externally into a separate structure. // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // - for(LocalSharedDirectoryMap::iterator it(mLocalSharedDirs);it;++it) - { - librs::util::FolderIterator dirIt(realpath); - if(!dirIt.isValid()) - mLocalSharedDirs.removeDirectory(it) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + std::list shared_directory_list ; - while(dirIt.readdir()) - { + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it) + sub_dir_list.push_back( (*it).filename ) ; - } + // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - if(mHashCache.getHash(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash)) - { - ; - } - } -#endif + mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + + // now for each of them, go recursively and match both files and dirs + + DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ; + + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it) + recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } + +LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx) +{ + // make sure list of subdirs is the same + + // make sure list of subfiles is the same + + // request all hashes to the hashcache + + librs::util::FolderIterator dirIt(cumulated_path); + + if(!dirIt.isValid()) + { + mSharedDirectories->removeDirectory(indx) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. + return ; + } + + // collect subdirs and subfiles + + std::list subfiles ; + std::list subdirs ; + + while(dirIt.readdir()) + { + std::string name ; + uint64_t size ; + uint8_t type ; + time_t mod_time ; + + dirIt.readEntryInformation(type,name,size,mod_time) ; + + switch(type) + { + case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ; + break; + case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(name) ; + break; + + default: + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl; + } + } + // update file and dir lists for current directory. + + mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; + mSharedDirectories->updateSubFilesList(indx,subfiles) ; + + // now go through list of subfiles and request the hash to hashcache + + for(DirectoryStorage::FileIterator dit(indx);dit;++dit) + { + // ask about the hash. If not present, ask HashCache. If not present, the callback will update it. + + if() + if(mHashCache.requestHash(realpath,name,size,modtime,hash),this) + mSharedDirectories->updateFileHash(*dit,hash) ; + } + + // go through the list of sub-dirs and recursively update + + DirectoryStorage::DirIterator stored_dir_it(indx) ; + + for(std::list::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it) + recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; +} + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 3571247ab..54b70f30c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -3,6 +3,9 @@ // - local: directories are crawled n disk and files are hashed / requested from a cache // - remote: directories are requested remotely to a providing client // +class HashCache ; +class LocalDirectoryStorage ; + class DirectoryUpdater { public: @@ -18,8 +21,13 @@ class DirectoryUpdater class LocalDirectoryUpdater: public DirectoryUpdater { - public: - virtual void tick() ; +public: + LocalDirectoryUpdater(HashCache *hash_cache) ; + virtual void tick() ; + +private: + HashCache *mHashCache ; + LocalDirectoryStorage *mSharedDirectories ; }; class RemoteDirectoryUpdater: public DirectoryUpdater diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 144e6d8ee..6734310fe 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -1,29 +1,50 @@ +#pragma once + +#include +#include "util/rsthreads.h" + class HashCache { - public: - HashCache(const std::string& save_file_name) ; +public: + HashCache(const std::string& save_file_name) ; - void save() ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - void clean() ; + void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; + bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - typedef struct - { - uint64_t size ; - uint64_t time_stamp ; - uint64_t modf_stamp ; - RsFileHash hash ; - } HashCacheInfo ; + struct HashCacheInfo + { + std::string filename ; + uint64_t size ; + uint32_t time_stamp ; + uint32_t modf_stamp ; + RsFileHash hash ; + } ; - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } - private: - uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries - std::map _files ; - std::string _path ; - bool _changed ; + // interaction with GUI, called from p3FileLists + void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } + uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } + void clear() { _files.clear(); } + bool empty() const { return _files.empty() ; } + +private: + void clean() ; + + void save() ; + void load() ; + + // threaded stuff + RsTickingThread mHashingThread ; + RsMutex mHashMtx ; + + // Local configuration and storage + + uint32_t mMaxCacheDurationDays ; // maximum duration of un-requested cache entries + std::map mFiles ; + std::string mFilePath ; + bool mChanged ; + + // current work + + std::map mFilesToHash ; }; From 3e48b0fd9515f80fddafa23ddc9f664ae53a0133 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 23 Jul 2016 22:14:43 -0400 Subject: [PATCH 009/174] improved FolderIterator class --- libretroshare/src/dbase/fimonitor.cc | 2 +- .../src/file_sharing/directory_storage.cc | 9 ++- .../src/file_sharing/directory_storage.h | 29 +++++-- .../src/file_sharing/directory_updater.cc | 58 +++++++------- .../src/file_sharing/directory_updater.h | 9 ++- libretroshare/src/file_sharing/hash_cache.h | 30 +++++-- libretroshare/src/plugins/pluginmanager.cc | 2 +- libretroshare/src/rsserver/rsaccounts.cc | 2 +- libretroshare/src/util/folderiterator.cc | 79 +++++++++++++++++-- libretroshare/src/util/folderiterator.h | 23 +++++- retroshare-nogui/src/introserver.cc | 2 +- 11 files changed, 183 insertions(+), 62 deletions(-) diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index c4f60ee81..56b1e7bc2 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -789,7 +789,7 @@ void FileIndexMonitor::updateCycle() to_hash.back().realpath = realpath ; to_hash.back().dirpath = dirpath ; - while(isRunning() && dirIt.readdir()) + for(;dirIt.isValid() && isRunning();dirIt.next()) { /* check entry type */ std::string fname; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index acfdcbf25..0b7e100e4 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -5,6 +5,9 @@ class InternalFileHierarchyStorage class FileStorageNode { public: + static const uint32_t TYPE_FILE = 0x0001 ; + static const uint32_t TYPE_DIR = 0x0002 ; + virtual uint32_t type() const =0; }; class FileEntry: public FileStorageNode @@ -19,12 +22,12 @@ class InternalFileHierarchyStorage public: virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - std::set subdirs ; - std::set subfiles ; + std::set subdirs ; + std::set subfiles ; }; // file/dir access and modification - bool findSubDirectory(EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e uint32_t root ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0399d9d9e..0fb94037a 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -39,27 +39,44 @@ class DirectoryStorage { public: DirIterator(const DirIterator& d) ; + DirIterator(const EntryIndex& d) ; DirIterator& operator++() ; EntryIndex operator*() const ; // current directory entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. - }; + operator bool() const ; // used in for loops. Returns true when the iterator is valid. + + // info about the directory that is pointed by the iterator + + const std::string& name() const ; + }; class FileIterator { public: FileIterator(DirIterator& d); // crawls all files in specified directory + FileIterator(EntryIndex& e); // crawls all files in specified directory - uint32_t size() const ; // number of files in this directory FileIterator& operator++() ; EntryIndex operator*() const ; // current file entry - bool operator()() const ; // used in for loops. Returns true when the iterator is valid. - }; + operator bool() const ; // used in for loops. Returns true when the iterator is valid. + + // info about the file that is pointed by the iterator + + std::string name() const ; + std::string fullpath() const ; + uint64_t size() const ; + RsFileHash hash() const ; + time_t modtime() const ; + }; virtual DirIterator root() ; // returns the index of the root directory entry. - private: + void updateSubDirectoryList(const EntryIndex& indx,const std::list& subdirs) ; + void updateSubFilesList(const EntryIndex& indx,const std::list& subfiles) ; + void removeDirectory(const EntryIndex& indx) ; + + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c20c3703b..c88820fb0 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -1,19 +1,24 @@ +#include "util/folderiterator.h" + #include "directory_storage.h" #include "directory_updater.h" +#define DEBUG_LOCAL_DIR_UPDATER 1 + void RemoteDirectoryUpdater::tick() { // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater() +LocalDirectoryUpdater::LocalDirectoryUpdater(HashCache *hc) + : mHashCache(hc) { - // tell the hash cache that we're the client - mHashCache->setClient(this) ; } void LocalDirectoryUpdater::tick() { + std::cerr << "LocalDirectoryUpdater::tick()" << std::endl; + // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. @@ -21,28 +26,29 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; + std::list sub_dir_list ; - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it) - sub_dir_list.push_back( (*it).filename ) ; + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) + sub_dir_list.push_back( (*real_dir_it).filename ) ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + mSharedDirectories->updateSubDirectoryList(*mSharedDirectories->root(),sub_dir_list) ; // now for each of them, go recursively and match both files and dirs - DirectoryStorage::DirIterator stored_dir_it(mLocalSharedDirs.root()) ; + DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories->root()) ; - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());it!=shared_directory_list.end();++it, ++stored_dir_it) - recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; + for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; } -LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,const DirectoryStorage::EntryIndex indx) +void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) { + std::cerr << " parsing directory " << cumulated_path << ", index=" << indx << std::endl; + // make sure list of subdirs is the same - // make sure list of subfiles is the same - // request all hashes to the hashcache librs::util::FolderIterator dirIt(cumulated_path); @@ -60,22 +66,18 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c while(dirIt.readdir()) { - std::string name ; - uint64_t size ; - uint8_t type ; - time_t mod_time ; - - dirIt.readEntryInformation(type,name,size,mod_time) ; - - switch(type) + switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(name) ; - break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(name) ; + case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ; + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + break; + + case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; default: - std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << name << "\"" << std::endl; + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } } // update file and dir lists for current directory. @@ -87,18 +89,16 @@ LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path,c for(DirectoryStorage::FileIterator dit(indx);dit;++dit) { - // ask about the hash. If not present, ask HashCache. If not present, the callback will update it. + // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. - if() - if(mHashCache.requestHash(realpath,name,size,modtime,hash),this) - mSharedDirectories->updateFileHash(*dit,hash) ; + mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),dit.hash(),this) ; } // go through the list of sub-dirs and recursively update DirectoryStorage::DirIterator stored_dir_it(indx) ; - for(std::list::const_iterator real_dir_it(subdirs.begin());it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::list::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 54b70f30c..35f069b3c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -3,7 +3,8 @@ // - local: directories are crawled n disk and files are hashed / requested from a cache // - remote: directories are requested remotely to a providing client // -class HashCache ; +#include "file_sharing/hash_cache.h" + class LocalDirectoryStorage ; class DirectoryUpdater @@ -19,12 +20,16 @@ class DirectoryUpdater // }; -class LocalDirectoryUpdater: public DirectoryUpdater +class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient { public: LocalDirectoryUpdater(HashCache *hash_cache) ; virtual void tick() ; +protected: + virtual void hash_callback(const std::string& full_name,const RsFileHash& hash) ; + void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); + private: HashCache *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 6734310fe..f867f6109 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -2,14 +2,28 @@ #include #include "util/rsthreads.h" +#include "retroshare/rsfiles.h" + +class HashCacheClient +{ +public: + virtual void hash_callback(const std::string& full_path,const RsFileHash& hash) ; +}; + +class FileHashingThread: public RsTickingThread +{ +public: + FileHashingThread() {} + + virtual void data_tick() ; +}; class HashCache { public: HashCache(const std::string& save_file_name) ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; + bool requestHash(const std::string& full_path,uint64_t size,time_t mod_time,const RsFileHash& known_hash,HashCacheClient *c) ; struct HashCacheInfo { @@ -21,10 +35,10 @@ public: } ; // interaction with GUI, called from p3FileLists - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } + void setRememberHashFilesDuration(uint32_t days) { mMaxCacheDurationDays = days ; } + uint32_t rememberHashFilesDuration() const { return mMaxCacheDurationDays ; } + void clear() { mFiles.clear(); } + bool empty() const { return mFiles.empty() ; } private: void clean() ; @@ -33,7 +47,7 @@ private: void load() ; // threaded stuff - RsTickingThread mHashingThread ; + FileHashingThread mHashingThread ; RsMutex mHashMtx ; // Local configuration and storage @@ -45,6 +59,6 @@ private: // current work - std::map mFilesToHash ; + std::map mFilesToHash ; }; diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 7d101fdf3..31ee2a39f 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -146,7 +146,7 @@ void RsPluginManager::loadPlugins(const std::vector& plugin_directo continue ; } - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { std::string fname; dirIt.d_name(fname); diff --git a/libretroshare/src/rsserver/rsaccounts.cc b/libretroshare/src/rsserver/rsaccounts.cc index ad9e1306e..9bb8b49ed 100644 --- a/libretroshare/src/rsserver/rsaccounts.cc +++ b/libretroshare/src/rsserver/rsaccounts.cc @@ -533,7 +533,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map & struct stat64 buf; - while (dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { /* check entry type */ std::string fname; diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 01ff94461..747ec93d9 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -1,11 +1,17 @@ +#include +#include +#include +#include +#include + #include "folderiterator.h" #include "rsstring.h" - namespace librs { namespace util { FolderIterator::FolderIterator(const std::string& folderName) + : mFolderName(folderName) { #ifdef WINDOWS_SYS std::wstring utf16Name; @@ -18,10 +24,10 @@ FolderIterator::FolderIterator(const std::string& folderName) handle = FindFirstFileW(utf16Name.c_str(), &fileInfo); validity = handle != INVALID_HANDLE_VALUE; - isFirstCall = true; #else handle = opendir(folderName.c_str()); validity = handle != NULL; + next(); #endif } @@ -30,19 +36,71 @@ FolderIterator::~FolderIterator() closedir(); } -bool FolderIterator::readdir() { +void FolderIterator::next() +{ + do { + if(!readdir()) + { + validity = false ; + break ; + } + + d_name(mFileName); + } while(mFileName == "." || mFileName == "..") ; + + mFullPath = mFolderName + "/" + mFileName ; + + struct stat64 buf ; + +#ifdef WINDOWS_SYS + std::wstring wfullname; + librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname); + if ( 0 == _wstati64(wfullname.c_str(), &buf)) +#else + if ( 0 == stat64(mFullPath.c_str(), &buf)) +#endif + { + mFileModTime = buf.st_mtime ; + mStatInfoOk = true; + + if (S_ISDIR(buf.st_mode)) + { + mType = TYPE_DIR ; + mFileSize = 0 ; + mFileModTime = buf.st_mtime; + } + else if (S_ISREG(buf.st_mode)) + { + mType = TYPE_FILE ; + mFileSize = buf.st_size; + mFileModTime = buf.st_mtime; + } + else + { + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; + } + } + else + { + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; + validity = false ; + } +} + +bool FolderIterator::readdir() +{ if(!validity) return false; #ifdef WINDOWS_SYS - if(isFirstCall) { - isFirstCall = false; - return true; - } return FindNextFileW(handle, &fileInfo) != 0; #else ent = ::readdir(handle); - return ent != 0; + return ent != NULL; #endif } @@ -65,6 +123,11 @@ bool FolderIterator::d_name(std::string& dest) return true; } +const std::string& FolderIterator::file_fullpath() { return mFullPath ; } +const std::string& FolderIterator::file_name() { return mFileName ; } +uint64_t FolderIterator::file_size() { return mFileSize ; } +time_t FolderIterator::file_modtime() { return mFileModTime ; } + bool FolderIterator::closedir() { if(!validity) diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 8007db1a1..823aa9a9a 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -2,6 +2,7 @@ #define FOLDERITERATOR_H +#include #include #include @@ -24,14 +25,24 @@ public: FolderIterator(const std::string& folderName); ~FolderIterator(); - bool isValid() const { return validity; } + enum { TYPE_UNKNOWN = 0x00, + TYPE_FILE = 0x01, + TYPE_DIR = 0x02 + }; + bool isValid() const { return validity; } bool readdir(); + void next(); bool d_name(std::string& dest); - bool closedir(); + const std::string& file_name() ; + const std::string& file_fullpath() ; + uint64_t file_size() ; + uint64_t file_type() ; + time_t file_modtime() ; + private: bool validity; @@ -43,7 +54,15 @@ private: DIR* handle; struct dirent* ent; #endif + void updateStatsInfo() ; + bool mStatInfoOk ; + time_t mFileModTime ; + uint64_t mFileSize ; + uint8_t mType ; + std::string mFileName ; + std::string mFullPath ; + std::string mFolderName ; }; diff --git a/retroshare-nogui/src/introserver.cc b/retroshare-nogui/src/introserver.cc index 5445dc814..3734154b2 100644 --- a/retroshare-nogui/src/introserver.cc +++ b/retroshare-nogui/src/introserver.cc @@ -290,7 +290,7 @@ int RsIntroServer::checkForNewCerts() mCertCheckTime = now; struct stat64 buf; - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { /* check entry type */ std::string fname; From 25c824fd5f05c9a1b9d9b66b8945fc2a766419c6 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 24 Jul 2016 23:48:22 -0400 Subject: [PATCH 010/174] added iterators for directory storage class --- .../src/file_sharing/directory_storage.cc | 268 +++++++++++++++++- .../src/file_sharing/directory_storage.h | 50 ++-- .../src/file_sharing/directory_updater.cc | 20 +- .../src/file_sharing/directory_updater.h | 11 +- libretroshare/src/util/folderiterator.cc | 7 +- libretroshare/src/util/folderiterator.h | 2 +- 6 files changed, 320 insertions(+), 38 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 0b7e100e4..a955df832 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,5 +1,22 @@ +#include #include "directory_storage.h" +/******************************************************************************************************************/ +/* Internal File Hierarchy Storage */ +/******************************************************************************************************************/ + +template typename std::set::iterator erase_from_set(typename std::set& s,const typename std::set::iterator& it) +{ + typename std::set::iterator tmp(it); + ++tmp; + s.erase(it) ; + return tmp; +} + +// This class handles the file hierarchy +// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed, +// so that the hierarchy stays completely coherent between calls. + class InternalFileHierarchyStorage { class FileStorageNode @@ -8,36 +25,279 @@ class InternalFileHierarchyStorage static const uint32_t TYPE_FILE = 0x0001 ; static const uint32_t TYPE_DIR = 0x0002 ; + virtual ~FileStorageNode() {} virtual uint32_t type() const =0; }; class FileEntry: public FileStorageNode { public: + FileEntry(const std::string& name) : file_name(name) {} virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + virtual ~FileEntry() {} + // local stuff + std::string file_name ; + uint64_t file_size ; + RsFileHash file_hash ; + time_t file_modtime ; }; class DirEntry: public FileStorageNode { public: + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} + virtual ~DirEntry() {} + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - std::set subdirs ; - std::set subfiles ; + // local stuff + std::string dir_name ; + DirectoryStorage::EntryIndex parent_index; + + std::vector subdirs ; + std::vector subfiles ; }; + // class stuff + InternalFileHierarchyStorage() : mRoot(0) + { + mNodes.push_back(new DirEntry("",0)) ; + } + + // high level modification routines + + bool isIndexValid(DirectoryStorage::EntryIndex e) + { + return e < mNodes.size() && mNodes[e] != NULL ; + } + + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set& subdirs) + { + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("updateSubDirectoryList(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("updateSubDirectoryList(): Node is not a directory") ; + + DirEntry& d(*static_cast(mNodes[indx])) ; + + std::set should_create(subdirs); + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) + removeDirectory(d.subdirs[i]) ; + else + { + should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; + ++i; + } + + for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) + { + d.subdirs.push_back(mNodes.size()) ; + mNodes.push_back(new DirEntry(*it,indx)); + } + + return true; + } + bool removeDirectory(const DirectoryStorage::EntryIndex& indx) + { + // check that it's a directory + + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("removeDirectory(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("removeDirectory(): Node is not a directory") ; + + if(indx == 0) + return nodeAccessError("removeDirectory(): Cannot remove top level directory") ; + + // remove from parent + + DirEntry& d(*static_cast(mNodes[indx])) ; + DirEntry& parent_dir(*static_cast(mNodes[d.parent_index])); + + for(uint32_t i=0;i& subfiles,std::set& new_files) + { + if(indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("updateSubFilesList(): Node does not exist") ; + + if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) + return nodeAccessError("updateSubFilesList(): Node is not a directory") ; + + DirEntry& d(*static_cast(mNodes[indx])) ; + new_files = subfiles ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name) == subfiles.end()) + { + d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; + d.subfiles.pop_back(); + } + else + { + new_files.erase(static_cast(mNodes[d.subfiles[i]])->file_name) ; + ++i; + } + + for(std::set::const_iterator it(new_files.begin());it!=new_files.end();++it) + { + d.subfiles.push_back(mNodes.size()) ; + mNodes.push_back(new FileEntry(*it)); + } + return true; + } + void updateFile(const DirectoryStorage::EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const DirectoryStorage::EntryIndex& parent_dir,const std::string& dname) ; + // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e - uint32_t root ; + uint32_t mRoot ; std::vector mNodes;// uses pointers to keep information about valid/invalid objects. - void compress() ; // use empty space in the vector, mostly due to deleted entries. + void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to + // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that + // are not referenced. friend class DirectoryStorage ; // only class that can use this. + + // low level stuff. Should normally not be used externally. + + DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) + { + if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[file_tab_index]; + } + DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) + { + if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[dir_tab_index]; + } + + bool check() // checks consistency of storage. + { + return true; + } + + void print() + { + } +private: + bool nodeAccessError(const std::string& s) + { + std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; + return false ; + } + + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning + // phase. That allows to keep file information when moving them around. + + bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir) + { + DirEntry& d(*static_cast(mNodes[dir])) ; + + for(uint32_t i=0;imFileHierarchy ; + mParentIndex = i; + mDirTabIndex = 0; +} + +DirectoryStorage::FileIterator::FileIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i) +{ + mStorage = s->mFileHierarchy ; + mParentIndex = i; + mFileTabIndex = 0; +} + +DirectoryStorage::DirIterator& DirectoryStorage::DirIterator::operator++() +{ + ++mDirTabIndex ; + + return *this; +} +DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++() +{ + ++mFileTabIndex ; + + return *this; +} +DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const +{ + return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; +} +DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const +{ + return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; +} +DirectoryStorage::DirIterator::operator bool() const +{ + return **this != DirectoryStorage::NO_INDEX; +} +DirectoryStorage::FileIterator::operator bool() const +{ + return **this != DirectoryStorage::NO_INDEX; +} + +/******************************************************************************************************************/ +/* Directory Storage */ +/******************************************************************************************************************/ + +DirectoryStorage::DirIterator DirectoryStorage::root() +{ + return DirIterator(this,EntryIndex(0)) ; +} + +void DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; +} +void DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; +} +void DirectoryStorage::removeDirectory(const EntryIndex& indx) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->removeDirectory(indx); +} + // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0fb94037a..90a0a889e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -23,12 +25,13 @@ class DirectoryStorage virtual ~DirectoryStorage() {} typedef uint32_t EntryIndex ; + static const EntryIndex NO_INDEX = 0xffffffff; void save() const ; - virtual int searchTerms(const std::list& terms, std::list &results) const; - virtual int searchHash(const RsFileHash& hash, std::list &results) const; - virtual int searchBoolExp(Expression * exp, std::list &results) const; + virtual int searchTerms(const std::list& terms, std::list &results) const { return 0;} + virtual int searchHash(const RsFileHash& hash, std::list &results) const { return 0;} + virtual int searchBoolExp(Expression * exp, std::list &results) const { return 0; } void getFileDetails(EntryIndex i) ; @@ -39,25 +42,31 @@ class DirectoryStorage { public: DirIterator(const DirIterator& d) ; - DirIterator(const EntryIndex& d) ; + DirIterator(DirectoryStorage *d,EntryIndex i) ; DirIterator& operator++() ; - EntryIndex operator*() const ; // current directory entry + EntryIndex operator*() const ; operator bool() const ; // used in for loops. Returns true when the iterator is valid. // info about the directory that is pointed by the iterator const std::string& name() const ; + private: + EntryIndex mParentIndex ; // index of the parent dir. + uint32_t mDirTabIndex ; // index in the vector of subdirs. + InternalFileHierarchyStorage *mStorage ; + + friend class DirectoryStorage ; }; class FileIterator { public: - FileIterator(DirIterator& d); // crawls all files in specified directory - FileIterator(EntryIndex& e); // crawls all files in specified directory + FileIterator(DirIterator& d); // crawls all files in specified directory + FileIterator(DirectoryStorage *d,EntryIndex e); // crawls all files in specified directory FileIterator& operator++() ; - EntryIndex operator*() const ; // current file entry + EntryIndex operator*() const ; // current file entry operator bool() const ; // used in for loops. Returns true when the iterator is valid. @@ -68,14 +77,24 @@ class DirectoryStorage uint64_t size() const ; RsFileHash hash() const ; time_t modtime() const ; + + private: + EntryIndex mParentIndex ; // index of the parent dir. + uint32_t mFileTabIndex ; // index in the vector of subdirs. + InternalFileHierarchyStorage *mStorage ; }; virtual DirIterator root() ; // returns the index of the root directory entry. - void updateSubDirectoryList(const EntryIndex& indx,const std::list& subdirs) ; - void updateSubFilesList(const EntryIndex& indx,const std::list& subfiles) ; + void updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; + void updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; void removeDirectory(const EntryIndex& indx) ; + void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; + void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + + void cleanup(); + private: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; @@ -86,6 +105,8 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. InternalFileHierarchyStorage *mFileHierarchy ; + + RsMutex mDirStorageMtx ; }; class RemoteDirectoryStorage: public DirectoryStorage @@ -104,15 +125,6 @@ public: void setSharedDirectoryList(const std::list& lst) ; void getSharedDirectoryList(std::list& lst) ; - /*! - * \brief addFile - * \param dir - * \param hash - * \param modf_time - */ - void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; - void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; - private: std::list mLocalDirs ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c88820fb0..c80968d8b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -26,10 +26,12 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; - std::list sub_dir_list ; + + + std::set sub_dir_list ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) - sub_dir_list.push_back( (*real_dir_it).filename ) ; + sub_dir_list.insert( (*real_dir_it).filename ) ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. @@ -61,18 +63,18 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // collect subdirs and subfiles - std::list subfiles ; - std::list subdirs ; + std::set subfiles ; + std::set subdirs ; while(dirIt.readdir()) { switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.push_back(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_FILE: subfiles.insert(dirIt.file_name()) ; std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.push_back(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; @@ -83,7 +85,9 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // update file and dir lists for current directory. mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; - mSharedDirectories->updateSubFilesList(indx,subfiles) ; + + std::set new_files ; + mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; // now go through list of subfiles and request the hash to hashcache @@ -98,7 +102,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(indx) ; - for(std::list::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 35f069b3c..74795ae14 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -4,13 +4,13 @@ // - remote: directories are requested remotely to a providing client // #include "file_sharing/hash_cache.h" - -class LocalDirectoryStorage ; +#include "file_sharing/directory_storage.h" class DirectoryUpdater { public: - DirectoryUpdater() ; + DirectoryUpdater() {} + virtual ~DirectoryUpdater(){} // Does some updating job. Crawls the existing directories and checks wether it has been updated // recently enough. If not, calls the directry source. @@ -24,6 +24,8 @@ class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient { public: LocalDirectoryUpdater(HashCache *hash_cache) ; + virtual ~LocalDirectoryUpdater() {} + virtual void tick() ; protected: @@ -38,5 +40,8 @@ private: class RemoteDirectoryUpdater: public DirectoryUpdater { public: + RemoteDirectoryUpdater() {} + virtual ~RemoteDirectoryUpdater() {} + virtual void tick() ; }; diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 747ec93d9..105c7776c 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -124,9 +124,10 @@ bool FolderIterator::d_name(std::string& dest) } const std::string& FolderIterator::file_fullpath() { return mFullPath ; } -const std::string& FolderIterator::file_name() { return mFileName ; } -uint64_t FolderIterator::file_size() { return mFileSize ; } -time_t FolderIterator::file_modtime() { return mFileModTime ; } +const std::string& FolderIterator::file_name() { return mFileName ; } +uint64_t FolderIterator::file_size() { return mFileSize ; } +time_t FolderIterator::file_modtime() { return mFileModTime ; } +uint8_t FolderIterator::file_type() { return mType ; } bool FolderIterator::closedir() { diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 823aa9a9a..dcf8ed3f3 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -40,7 +40,7 @@ public: const std::string& file_name() ; const std::string& file_fullpath() ; uint64_t file_size() ; - uint64_t file_type() ; + uint8_t file_type() ; time_t file_modtime() ; private: From d86b89b15acd2428603ad259e7b25ff6ea722360 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 27 Jul 2016 21:22:59 +0200 Subject: [PATCH 011/174] fixed compilation/running of directory updater --- .../src/file_sharing/directory_storage.cc | 166 ++++++++++++------ .../src/file_sharing/directory_storage.h | 16 +- .../src/file_sharing/directory_updater.cc | 24 ++- .../src/file_sharing/directory_updater.h | 8 +- libretroshare/src/file_sharing/hash_cache.cc | 124 +++++++++++++ libretroshare/src/file_sharing/hash_cache.h | 65 ++++--- 6 files changed, 309 insertions(+), 94 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index a955df832..813b0c38d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -22,8 +22,9 @@ class InternalFileHierarchyStorage class FileStorageNode { public: - static const uint32_t TYPE_FILE = 0x0001 ; - static const uint32_t TYPE_DIR = 0x0002 ; + static const uint32_t TYPE_UNKNOWN = 0x0000 ; + static const uint32_t TYPE_FILE = 0x0001 ; + static const uint32_t TYPE_DIR = 0x0002 ; virtual ~FileStorageNode() {} virtual uint32_t type() const =0; @@ -45,7 +46,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(DirectoryStorage::NO_INDEX) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -66,18 +67,15 @@ class InternalFileHierarchyStorage // high level modification routines - bool isIndexValid(DirectoryStorage::EntryIndex e) + bool isIndexValid(DirectoryStorage::EntryIndex e) const { return e < mNodes.size() && mNodes[e] != NULL ; } bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set& subdirs) { - if(indx >= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("updateSubDirectoryList(): Node does not exist") ; - - if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) - return nodeAccessError("updateSubDirectoryList(): Node is not a directory") ; + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; DirEntry& d(*static_cast(mNodes[indx])) ; @@ -104,14 +102,11 @@ class InternalFileHierarchyStorage { // check that it's a directory - if(indx >= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("removeDirectory(): Node does not exist") ; - - if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) - return nodeAccessError("removeDirectory(): Node is not a directory") ; + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; if(indx == 0) - return nodeAccessError("removeDirectory(): Cannot remove top level directory") ; + return nodeAccessError("checkIndex(): Cannot remove top level directory") ; // remove from parent @@ -125,13 +120,21 @@ class InternalFileHierarchyStorage return nodeAccessError("removeDirectory(): inconsistency!!") ; } + bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const + { + if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("checkIndex(): Node does not exist") ; + + if(mNodes[indx]->type() != type) + return nodeAccessError("checkIndex(): Node is of wrong type") ; + + return true; + } + bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::set& subfiles,std::set& new_files) { - if(indx >= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("updateSubFilesList(): Node does not exist") ; - - if(mNodes[indx]->type() != FileStorageNode::TYPE_DIR) - return nodeAccessError("updateSubFilesList(): Node is not a directory") ; + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; DirEntry& d(*static_cast(mNodes[indx])) ; new_files = subfiles ; @@ -155,14 +158,32 @@ class InternalFileHierarchyStorage } return true; } - void updateFile(const DirectoryStorage::EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; - void updateDirectory(const DirectoryStorage::EntryIndex& parent_dir,const std::string& dname) ; + bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) + { + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + return false; + + static_cast(mNodes[file_index])->file_hash = hash ; + return true; + } + bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const uint32_t modf_time) + { + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + return false; + + FileEntry& fe(*static_cast(mNodes[file_index])) ; + + fe.file_hash = hash; + fe.file_size = size; + fe.file_modtime = modf_time; + + return true; + } // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e uint32_t mRoot ; - std::vector mNodes;// uses pointers to keep information about valid/invalid objects. void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to @@ -171,11 +192,25 @@ class InternalFileHierarchyStorage friend class DirectoryStorage ; // only class that can use this. - // low level stuff. Should normally not be used externally. + // Low level stuff. Should normally not be used externally. + const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const + { + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return NULL ; + + return static_cast(mNodes[indx]) ; + } + const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const + { + if(!checkIndex(indx,FileStorageNode::TYPE_FILE)) + return NULL ; + + return static_cast(mNodes[indx]) ; + } DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) { - if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) return DirectoryStorage::NO_INDEX; if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) @@ -185,7 +220,7 @@ class InternalFileHierarchyStorage } DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) { - if(parent_index >= mNodes.size() || mNodes[parent_index] == NULL || mNodes[parent_index]->type() != FileStorageNode::TYPE_DIR) + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) return DirectoryStorage::NO_INDEX; if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) @@ -203,7 +238,7 @@ class InternalFileHierarchyStorage { } private: - bool nodeAccessError(const std::string& s) + static bool nodeAccessError(const std::string& s) { std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; return false ; @@ -256,46 +291,58 @@ DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++() return *this; } -DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const -{ - return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; -} -DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const -{ - return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; -} -DirectoryStorage::DirIterator::operator bool() const -{ - return **this != DirectoryStorage::NO_INDEX; -} -DirectoryStorage::FileIterator::operator bool() const -{ - return **this != DirectoryStorage::NO_INDEX; -} +DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; } +DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; } + +DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; } +DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; } + +RsFileHash DirectoryStorage::FileIterator::hash() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_hash):RsFileHash(); } +uint64_t DirectoryStorage::FileIterator::size() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_size):0; } +std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } +std::string DirectoryStorage::FileIterator::fullpath() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } +time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; } /******************************************************************************************************************/ /* Directory Storage */ /******************************************************************************************************************/ -DirectoryStorage::DirIterator DirectoryStorage::root() +DirectoryStorage::DirectoryStorage(const std::string& local_file_name) + : mFileName(local_file_name), mDirStorageMtx("Directory storage "+local_file_name) { - return DirIterator(this,EntryIndex(0)) ; + mFileHierarchy = new InternalFileHierarchyStorage(); } -void DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) +DirectoryStorage::EntryIndex DirectoryStorage::root() const { - RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + return EntryIndex(0) ; } -void DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) + +bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) { RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + return mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; } -void DirectoryStorage::removeDirectory(const EntryIndex& indx) +bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy->removeDirectory(indx); + return mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; +} +bool DirectoryStorage::removeDirectory(const EntryIndex& indx) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->removeDirectory(indx); +} + +bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->updateFile(index,hash,fname,size,modf_time); +} +bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->updateHash(index,hash); } // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; @@ -321,3 +368,18 @@ void DirectoryStorage::save(const std::string& local_file_name) { // first write the header, than all fields. } + +/******************************************************************************************************************/ +/* Local Directory Storage */ +/******************************************************************************************************************/ + +void LocalDirectoryStorage::setSharedDirectoryList(const std::list& lst) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mLocalDirs = lst ; +} +void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + lst = mLocalDirs ; +} diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 90a0a889e..fe7307d6d 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -84,14 +84,14 @@ class DirectoryStorage InternalFileHierarchyStorage *mStorage ; }; - virtual DirIterator root() ; // returns the index of the root directory entry. + EntryIndex root() const ; // returns the index of the root directory entry. - void updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; - void updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; - void removeDirectory(const EntryIndex& indx) ; + bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; + bool updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; + bool removeDirectory(const EntryIndex& indx) ; - void updateFile(const EntryIndex& parent_dir,const RsFileHash& hash, const std::string& fname, const uint32_t modf_time) ; - void updateDirectory(const EntryIndex& parent_dir,const std::string& dname) ; + bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; + bool updateHash(const EntryIndex& index,const RsFileHash& hash); void cleanup(); @@ -105,7 +105,9 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. InternalFileHierarchyStorage *mFileHierarchy ; + std::string mFileName; + protected: RsMutex mDirStorageMtx ; }; @@ -119,7 +121,7 @@ public: class LocalDirectoryStorage: public DirectoryStorage { public: - LocalDirectoryStorage() ; + LocalDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} virtual ~LocalDirectoryStorage() {} void setSharedDirectoryList(const std::list& lst) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c80968d8b..3d6c5bb74 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,9 +10,10 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater(HashCache *hc) +LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc) : mHashCache(hc) { + mSharedDirectories = new LocalDirectoryStorage("local_storage.txt"); } void LocalDirectoryUpdater::tick() @@ -35,11 +36,11 @@ void LocalDirectoryUpdater::tick() // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - mSharedDirectories->updateSubDirectoryList(*mSharedDirectories->root(),sub_dir_list) ; + mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; // now for each of them, go recursively and match both files and dirs - DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories->root()) ; + DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; @@ -66,7 +67,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p std::set subfiles ; std::set subdirs ; - while(dirIt.readdir()) + for(;dirIt.isValid();dirIt.next()) { switch(dirIt.file_type()) { @@ -91,18 +92,27 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // now go through list of subfiles and request the hash to hashcache - for(DirectoryStorage::FileIterator dit(indx);dit;++dit) + for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit) { // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. - mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),dit.hash(),this) ; + RsFileHash hash ; + + if(mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) + mSharedDirectories->updateHash(*dit,hash); } // go through the list of sub-dirs and recursively update - DirectoryStorage::DirIterator stored_dir_it(indx) ; + DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; } +void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) +{ + if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) + std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; +} + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 74795ae14..b70164bdd 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -20,20 +20,20 @@ class DirectoryUpdater // }; -class LocalDirectoryUpdater: public DirectoryUpdater, public HashCacheClient +class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient { public: - LocalDirectoryUpdater(HashCache *hash_cache) ; + LocalDirectoryUpdater(HashStorage *hash_cache) ; virtual ~LocalDirectoryUpdater() {} virtual void tick() ; protected: - virtual void hash_callback(const std::string& full_name,const RsFileHash& hash) ; + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); private: - HashCache *mHashCache ; + HashStorage *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index e69de29bb..25af17984 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -0,0 +1,124 @@ +#include "util/rsdir.h" +#include "hash_cache.h" + +#define HASHSTORAGE_DEBUG 1 + +HashStorage::HashStorage(const std::string& save_file_name) + : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") +{ +} + +void HashStorage::data_tick() +{ + std::cerr << "Ticking hash thread." << std::endl; + + FileHashJob job; + + { + RS_STACK_MUTEX(mHashMtx) ; + + if(mFilesToHash.empty()) + return ; + + job = mFilesToHash.begin()->second ; + } + + std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); + + RsFileHash hash; + uint64_t size ; + + if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) + { + std::cerr << "ERROR" << std::endl; + return; + } + // update the hash storage + + + // call the client + + job.client->hash_callback(job.client_param, job.full_path, hash, size); + + std::cerr << "done."<< std::endl; + + if(mFilesToHash.empty()) + { + std::cerr << "Starting hashing thread." << std::endl; + fullstop(); + std::cerr << "done." << std::endl; + } +} + +bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) +{ + // check if the hash is up to date w.r.t. cache. + + RS_STACK_MUTEX(mHashMtx) ; + + time_t now = time(NULL) ; + std::map::iterator it = mFiles.find(full_path) ; + + if(it != mFiles.end() && (uint64_t)mod_time == it->second.modf_stamp && size == it->second.size) + { + it->second.time_stamp = now ; +#ifdef HASHCACHE_DEBUG + std::cerr << "Found in cache." << std::endl ; +#endif + return true ; + } + + // we need to schedule a re-hashing + + if(mFilesToHash.find(full_path) != mFilesToHash.end()) + return false ; + + FileHashJob job ; + + job.client = c ; + job.client_param = client_param ; + job.full_path = full_path ; + + mFilesToHash[full_path] = job; + + if(!isRunning()) + { + std::cerr << "Starting hashing thread." << std::endl; + start() ; + } + + return false; +} + +void HashStorage::clean() +{ +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Cleaning HashStorage..." << std::endl ; +#endif + time_t now = time(NULL) ; + time_t duration = mMaxStorageDurationDays * 24 * 3600 ; // seconds + +#ifdef HASHSTORAGE_DEBUG + std::cerr << "cleaning hash cache." << std::endl ; +#endif + + for(std::map::iterator it(mFiles.begin());it!=mFiles.end();) + if(it->second.time_stamp + duration < (uint64_t)now) + { +#ifdef HASHSTORAGE_DEBUG + std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ; +#endif + std::map::iterator tmp(it) ; + ++tmp ; + mFiles.erase(it) ; + it=tmp ; + mChanged = true ; + } + else + ++it ; + +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Done." << std::endl; +#endif +} + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index f867f6109..fa73797c2 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -4,61 +4,78 @@ #include "util/rsthreads.h" #include "retroshare/rsfiles.h" -class HashCacheClient +class HashStorageClient { public: - virtual void hash_callback(const std::string& full_path,const RsFileHash& hash) ; + HashStorageClient() {} + virtual ~HashStorageClient() {} + + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)=0; }; -class FileHashingThread: public RsTickingThread +class HashStorage: public RsTickingThread { public: - FileHashingThread() {} + HashStorage(const std::string& save_file_name) ; - virtual void data_tick() ; -}; + /*! + * \brief requestHash Requests the hash for the given file, assuming size and mod_time are the same. + * + * \param full_path Full path to reach the file + * \param size Actual file size + * \param mod_time Actual file modification time + * \param known_hash Returned hash for the file. + * \param c Hash cache client to which the hash should be sent once calculated + * \param client_param Param to be passed to the client callback + * + * \return true if the supplied hash info is up to date. + */ + bool requestHash(const std::string& full_path, uint64_t size, time_t mod_time, RsFileHash& known_hash, HashStorageClient *c, uint32_t client_param) ; -class HashCache -{ -public: - HashCache(const std::string& save_file_name) ; - - bool requestHash(const std::string& full_path,uint64_t size,time_t mod_time,const RsFileHash& known_hash,HashCacheClient *c) ; - - struct HashCacheInfo + struct HashStorageInfo { std::string filename ; uint64_t size ; - uint32_t time_stamp ; + uint32_t time_stamp ; // last time the hash was tested/requested uint32_t modf_stamp ; RsFileHash hash ; } ; // interaction with GUI, called from p3FileLists - void setRememberHashFilesDuration(uint32_t days) { mMaxCacheDurationDays = days ; } - uint32_t rememberHashFilesDuration() const { return mMaxCacheDurationDays ; } + void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } + uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; } void clear() { mFiles.clear(); } bool empty() const { return mFiles.empty() ; } + // Functions called by the thread + + virtual void data_tick() ; private: void clean() ; void save() ; void load() ; - // threaded stuff - FileHashingThread mHashingThread ; - RsMutex mHashMtx ; - // Local configuration and storage - uint32_t mMaxCacheDurationDays ; // maximum duration of un-requested cache entries - std::map mFiles ; + uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries + std::map mFiles ; // stored as (full_path, hash_info) std::string mFilePath ; bool mChanged ; + struct FileHashJob + { + std::string full_path; + HashStorageClient *client; + uint32_t client_param ; + }; + // current work - std::map mFilesToHash ; + std::map mFilesToHash ; + + // thread/mutex stuff + + RsMutex mHashMtx ; }; From 463f21c85c0ea1a1fc9f2e72074913573f9decc8 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 28 Jul 2016 00:48:28 +0200 Subject: [PATCH 012/174] fixed hashing thread --- .../src/file_sharing/directory_storage.cc | 61 +++++++++++++++++-- .../src/file_sharing/directory_storage.h | 2 +- .../src/file_sharing/directory_updater.cc | 14 +++-- .../src/file_sharing/directory_updater.h | 2 +- libretroshare/src/file_sharing/hash_cache.cc | 52 ++++++++-------- libretroshare/src/file_sharing/hash_cache.h | 1 + 6 files changed, 92 insertions(+), 40 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 813b0c38d..ee89d572c 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -46,7 +46,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(DirectoryStorage::NO_INDEX) {} + DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -226,7 +226,7 @@ class InternalFileHierarchyStorage if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) return DirectoryStorage::NO_INDEX; - return static_cast(mNodes[parent_index])->subfiles[dir_tab_index]; + return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; } bool check() // checks consistency of storage. @@ -234,10 +234,57 @@ class InternalFileHierarchyStorage return true; } - void print() + void print() const { + int nfiles = 0 ; + int ndirs = 0 ; + int nempty = 0 ; + int nunknown = 0; + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++ndirs; + } + else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++nfiles; + } + else + { + ++nunknown; + std::cerr << "(EE) Error: unknown type node found!" << std::endl; + } + + std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; + + recursPrint(0,DirectoryStorage::EntryIndex(0)); } private: + void recursPrint(int depth,DirectoryStorage::EntryIndex node) const + { + std::string indent(2*depth,' '); + DirEntry& d(*static_cast(mNodes[node])); + + std::cerr << indent << d.dir_name << std::endl; + + for(int i=0;i(mNodes[i])); + std::cerr << indent << " " << f.file_hash << " " << f.file_modtime << " " << f.file_name << std::endl; + } + } + static bool nodeAccessError(const std::string& s) { std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; @@ -300,7 +347,6 @@ DirectoryStorage::DirIterator ::operator bool() const { return **this != Directo RsFileHash DirectoryStorage::FileIterator::hash() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_hash):RsFileHash(); } uint64_t DirectoryStorage::FileIterator::size() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_size):0; } std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } -std::string DirectoryStorage::FileIterator::fullpath() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; } /******************************************************************************************************************/ @@ -368,7 +414,12 @@ void DirectoryStorage::save(const std::string& local_file_name) { // first write the header, than all fields. } - +void DirectoryStorage::print() +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + std::cerr << "LocalDirectoryStorage:" << std::endl; + mFileHierarchy->print(); +} /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index fe7307d6d..9f8293963 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -73,7 +73,6 @@ class DirectoryStorage // info about the file that is pointed by the iterator std::string name() const ; - std::string fullpath() const ; uint64_t size() const ; RsFileHash hash() const ; time_t modtime() const ; @@ -93,6 +92,7 @@ class DirectoryStorage bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); + void print(); void cleanup(); private: diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 3d6c5bb74..9367e826e 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,10 +10,9 @@ void RemoteDirectoryUpdater::tick() // use the stored iterator } -LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc) - : mHashCache(hc) +LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) + : mHashCache(hc),mSharedDirectories(lds) { - mSharedDirectories = new LocalDirectoryStorage("local_storage.txt"); } void LocalDirectoryUpdater::tick() @@ -27,7 +26,7 @@ void LocalDirectoryUpdater::tick() // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. // std::list shared_directory_list ; - + mSharedDirectories->getSharedDirectoryList(shared_directory_list); std::set sub_dir_list ; @@ -43,7 +42,10 @@ void LocalDirectoryUpdater::tick() DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + { + std::cerr << " recursing into " << real_dir_it->filename << std::endl; recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; + } } void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) @@ -98,7 +100,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p RsFileHash hash ; - if(mHashCache->requestHash(dit.fullpath(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) + if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) mSharedDirectories->updateHash(*dit,hash); } @@ -107,7 +109,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) - recursUpdateSharedDir(*real_dir_it, *stored_dir_it) ; + recursUpdateSharedDir(cumulated_path + "/" + *real_dir_it, *stored_dir_it) ; } void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index b70164bdd..f0317b36d 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -23,7 +23,7 @@ class DirectoryUpdater class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient { public: - LocalDirectoryUpdater(HashStorage *hash_cache) ; + LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} virtual void tick() ; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 25af17984..0acd323c2 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -6,13 +6,14 @@ HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { + mRunning = false ; } void HashStorage::data_tick() { - std::cerr << "Ticking hash thread." << std::endl; - FileHashJob job; + RsFileHash hash; + uint64_t size ; { RS_STACK_MUTEX(mHashMtx) ; @@ -21,33 +22,29 @@ void HashStorage::data_tick() return ; job = mFilesToHash.begin()->second ; + + std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); + + + if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) + std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; + else + std::cerr << "done."<< std::endl; + + mFilesToHash.erase(mFilesToHash.begin()) ; + + if(mFilesToHash.empty()) + { + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + } } - - std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); - - RsFileHash hash; - uint64_t size ; - - if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) - { - std::cerr << "ERROR" << std::endl; - return; - } - // update the hash storage - - // call the client - job.client->hash_callback(job.client_param, job.full_path, hash, size); - - std::cerr << "done."<< std::endl; - - if(mFilesToHash.empty()) - { - std::cerr << "Starting hashing thread." << std::endl; - fullstop(); - std::cerr << "done." << std::endl; - } + if(!hash.isNull()) + job.client->hash_callback(job.client_param, job.full_path, hash, size); } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -81,8 +78,9 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mFilesToHash[full_path] = job; - if(!isRunning()) + if(!mRunning) { + mRunning = true ; std::cerr << "Starting hashing thread." << std::endl; start() ; } diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index fa73797c2..88b4206f9 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -77,5 +77,6 @@ private: // thread/mutex stuff RsMutex mHashMtx ; + bool mRunning; }; From 2aac88464df96de1a73bf4f9079be2ad8b09ea7e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 28 Jul 2016 09:29:15 +0200 Subject: [PATCH 013/174] fixed a few bugs in directory parsing --- .../src/file_sharing/directory_storage.cc | 41 ++++++++++++------- .../src/file_sharing/directory_storage.h | 8 +++- .../src/file_sharing/directory_updater.cc | 16 ++++---- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index ee89d572c..b178daf85 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -32,7 +32,7 @@ class InternalFileHierarchyStorage class FileEntry: public FileStorageNode { public: - FileEntry(const std::string& name) : file_name(name) {} + FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } virtual ~FileEntry() {} @@ -131,7 +131,7 @@ class InternalFileHierarchyStorage return true; } - bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::set& subfiles,std::set& new_files) + bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return false; @@ -139,22 +139,33 @@ class InternalFileHierarchyStorage DirEntry& d(*static_cast(mNodes[indx])) ; new_files = subfiles ; + // remove from new_files the ones that already exist and have a modf time that is not older. + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name) == subfiles.end()) + { + FileEntry& f(*static_cast(mNodes[d.subfiles[i]])) ; + std::map::const_iterator it = subfiles.find(f.file_name) ; + + if(it == subfiles.end()) // file does not exist anymore => delete { d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; d.subfiles.pop_back(); - } - else - { - new_files.erase(static_cast(mNodes[d.subfiles[i]])->file_name) ; - ++i; + + continue; } - for(std::set::const_iterator it(new_files.begin());it!=new_files.end();++it) + if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size + f.file_hash.clear(); // hash needs recomputing + else + new_files.erase(f.file_name) ; + + ++i; + } + + for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) { d.subfiles.push_back(mNodes.size()) ; - mNodes.push_back(new FileEntry(*it)); + mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); } return true; } @@ -166,7 +177,7 @@ class InternalFileHierarchyStorage static_cast(mNodes[file_index])->file_hash = hash ; return true; } - bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const uint32_t modf_time) + bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) { if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) return false; @@ -273,15 +284,15 @@ private: std::string indent(2*depth,' '); DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << d.dir_name << std::endl; + std::cerr << indent << "dir:" << d.dir_name << std::endl; for(int i=0;i(mNodes[i])); - std::cerr << indent << " " << f.file_hash << " " << f.file_modtime << " " << f.file_name << std::endl; + FileEntry& f(*static_cast(mNodes[d.subfiles[i]])); + std::cerr << indent << " hash:" << f.file_hash << " ts:" << std::fill(8) << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; } } @@ -369,7 +380,7 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std:: RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; } -bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) +bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map& subfiles,std::map& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 9f8293963..bd0f5b8ce 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -83,10 +83,16 @@ class DirectoryStorage InternalFileHierarchyStorage *mStorage ; }; + struct FileTS + { + uint64_t size ; + time_t modtime; + }; + EntryIndex root() const ; // returns the index of the root directory entry. bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; - bool updateSubFilesList(const EntryIndex& indx,const std::set& subfiles,std::set& new_files) ; + bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 9367e826e..f9a228e1b 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -66,21 +66,21 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // collect subdirs and subfiles - std::set subfiles ; + std::map subfiles ; std::set subdirs ; for(;dirIt.isValid();dirIt.next()) { switch(dirIt.file_type()) { - case librs::util::FolderIterator::TYPE_FILE: subfiles.insert(dirIt.file_name()) ; - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; - break; + case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; + subfiles[dirIt.file_name()].size = dirIt.file_size(); + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; - std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; - default: std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } @@ -89,7 +89,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; - std::set new_files ; + std::map new_files ; mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; // now go through list of subfiles and request the hash to hashcache From aeb05953015d786997303abe55686f3255dc8cd0 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 28 Jul 2016 10:49:49 +0200 Subject: [PATCH 014/174] fixed updated ts in hash cache files --- .../src/file_sharing/directory_storage.cc | 6 +++++- .../src/file_sharing/directory_updater.cc | 18 +++++++++++++++-- .../src/file_sharing/directory_updater.h | 14 +++++-------- libretroshare/src/file_sharing/hash_cache.cc | 20 ++++++++++++++++++- libretroshare/src/file_sharing/hash_cache.h | 1 + 5 files changed, 46 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index b178daf85..d94cb64f0 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -155,7 +155,11 @@ class InternalFileHierarchyStorage } if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size + { f.file_hash.clear(); // hash needs recomputing + f.file_modtime = it->second.modtime; + f.file_size = it->second.size; + } else new_files.erase(f.file_name) ; @@ -292,7 +296,7 @@ private: for(int i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << std::fill(8) << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; } } diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index f9a228e1b..b4ebfb0f5 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -5,6 +5,8 @@ #define DEBUG_LOCAL_DIR_UPDATER 1 +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 10 ; // 10 seconds for testing. Should be much more!! + void RemoteDirectoryUpdater::tick() { // use the stored iterator @@ -13,11 +15,23 @@ void RemoteDirectoryUpdater::tick() LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) { + mLastSweepTime = 0; } -void LocalDirectoryUpdater::tick() +void LocalDirectoryUpdater::data_tick() { - std::cerr << "LocalDirectoryUpdater::tick()" << std::endl; + time_t now = time(NULL) ; + + if(now > DELAY_BETWEEN_DIRECTORY_UPDATES + mLastSweepTime) + { + sweepSharedDirectories() ; + mLastSweepTime = now; + } +} + +void LocalDirectoryUpdater::sweepSharedDirectories() +{ + std::cerr << "LocalDirectoryUpdater::sweep()" << std::endl; // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index f0317b36d..2d30793cd 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -11,30 +11,26 @@ class DirectoryUpdater public: DirectoryUpdater() {} virtual ~DirectoryUpdater(){} - - // Does some updating job. Crawls the existing directories and checks wether it has been updated - // recently enough. If not, calls the directry source. - // - virtual void tick() =0; - - // }; -class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient +class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread { public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} - virtual void tick() ; + virtual void data_tick() ; protected: virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); + void sweepSharedDirectories(); private: HashStorage *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; + + time_t mLastSweepTime; }; class RemoteDirectoryUpdater: public DirectoryUpdater diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 0acd323c2..eb86813b3 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -40,6 +40,16 @@ void HashStorage::data_tick() mRunning = false ; std::cerr << "done." << std::endl; } + + // store the result + + HashStorageInfo& info(mFiles[job.full_path]); + + info.filename = job.full_path ; + info.size = size ; + info.modf_stamp = job.ts ; + info.time_stamp = time(NULL); + info.hash = hash; } // call the client @@ -51,6 +61,9 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t { // check if the hash is up to date w.r.t. cache. +#ifdef HASHSTORAGE_DEBUG + std::cerr << "HASH Requested for file " << full_path << ": "; +#endif RS_STACK_MUTEX(mHashMtx) ; time_t now = time(NULL) ; @@ -59,11 +72,15 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t if(it != mFiles.end() && (uint64_t)mod_time == it->second.modf_stamp && size == it->second.size) { it->second.time_stamp = now ; -#ifdef HASHCACHE_DEBUG + known_hash = it->second.hash; +#ifdef HASHSTORAGE_DEBUG std::cerr << "Found in cache." << std::endl ; #endif return true ; } +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Not in cache. Sceduling for re-hash." << std::endl ; +#endif // we need to schedule a re-hashing @@ -75,6 +92,7 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t job.client = c ; job.client_param = client_param ; job.full_path = full_path ; + job.ts = mod_time ; mFilesToHash[full_path] = job; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 88b4206f9..2d10c09ca 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -68,6 +68,7 @@ private: std::string full_path; HashStorageClient *client; uint32_t client_param ; + time_t ts; }; // current work From 5b9fd74d85b018a00fe7f16c6905e8577cc288c4 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 30 Jul 2016 21:52:42 +0200 Subject: [PATCH 015/174] removed most of CacheStrapper stuff from ftServer, replaced fimonitor by p3FileDatabase --- libretroshare/src/dbase/fimonitor.cc | 6 +- libretroshare/src/dbase/fimonitor.h | 4 +- libretroshare/src/file_sharing/p3filelists.cc | 149 ++++++-- libretroshare/src/file_sharing/p3filelists.h | 92 +++-- libretroshare/src/ft/ftcontroller.cc | 226 ++--------- libretroshare/src/ft/ftcontroller.h | 8 +- libretroshare/src/ft/ftdatamultiplex.cc | 2 - libretroshare/src/ft/ftserver.cc | 358 ++++-------------- libretroshare/src/ft/ftserver.h | 335 ++++++++-------- libretroshare/src/plugins/pluginmanager.cc | 3 +- libretroshare/src/retroshare/rsfiles.h | 25 +- libretroshare/src/rsserver/rsinit.cc | 8 - libretroshare/src/serialiser/rsserviceids.h | 1 + .../src/gui/FileTransfer/TransfersDialog.cpp | 15 - 14 files changed, 463 insertions(+), 769 deletions(-) diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index 56b1e7bc2..ec8140892 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -1594,9 +1594,9 @@ std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const int FileIndexMonitor::RequestDirDetails(const std::string& path, DirDetails& details) const { - /* lock it up */ - RsStackMutex mutex(fiMutex) ; - return fi.extractData(path,details) ; + /* lock it up */ + RsStackMutex mutex(fiMutex) ; + return fi.extractData(path,details) ; } uint32_t FileIndexMonitor::getType(void *ref) const diff --git a/libretroshare/src/dbase/fimonitor.h b/libretroshare/src/dbase/fimonitor.h index dc616625d..1ba0c327c 100644 --- a/libretroshare/src/dbase/fimonitor.h +++ b/libretroshare/src/dbase/fimonitor.h @@ -137,7 +137,7 @@ class FileIndexMonitor: public CacheSource, public RsTickingThread // Interface for browsing dir hirarchy int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ; uint32_t getType(void*) const ; - int RequestDirDetails(const std::string& path, DirDetails &details) const ; + int RequestDirDetails(const std::string& path, DirDetails &details) const ; // set/update shared directories virtual void setSharedDirectories(const std::list& dirs); @@ -146,7 +146,7 @@ class FileIndexMonitor: public CacheSource, public RsTickingThread void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files. - bool inDirectoryCheck(); + bool inDirectoryCheck(); /* util fns */ diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 7c35fe332..0fd3a830c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,5 +1,9 @@ +#include "serialiser/rsserviceids.h" + #include "file_sharing/p3filelists.h" #include "file_sharing/directory_storage.h" +#include "file_sharing/directory_updater.h" + #include "retroshare/rsids.h" #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " @@ -9,19 +13,36 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -p3FileLists::p3FileLists(p3LinkMgr *mpeers) - : mLinkMgr(mpeers), mFLSMtx("p3FileLists") +#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not implemented!" << std::endl; } + +p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) + : mServCtrl(mpeers), mFLSMtx("p3FileLists") { // loads existing indexes for friends. Some might be already present here. // - - // init the directory watchers - - // local variables/structures - // + mRemoteDirectories.clear() ; // we should load them! + + mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin") ; + mHashCache = new HashStorage("hash_cache.bin") ; + + mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; + mRemoteDirWatcher = NULL; // not used yet + + mLocalDirWatcher->start(); + mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } -p3FileLists::~p3FileLists() + +void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) +{ + mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; +} +void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) +{ + mLocalSharedDirs->getSharedDirectoryList(shared_dirs) ; +} + +p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; @@ -30,10 +51,28 @@ p3FileLists::~p3FileLists() mRemoteDirectories.clear(); // just a precaution, not to leave deleted pointers around. - // delete all pointers + delete mLocalSharedDirs ; + delete mLocalDirWatcher ; + delete mRemoteDirWatcher ; + delete mHashCache ; } -int p3FileLists::tick() +const std::string FILE_DB_APP_NAME = "file_database"; +const uint16_t FILE_DB_APP_MAJOR_VERSION = 1; +const uint16_t FILE_DB_APP_MINOR_VERSION = 0; +const uint16_t FILE_DB_MIN_MAJOR_VERSION = 1; +const uint16_t FILE_DB_MIN_MINOR_VERSION = 0; + +RsServiceInfo p3FileDatabase::getServiceInfo() +{ + return RsServiceInfo(RS_SERVICE_TYPE_FILE_DATABASE, + FILE_DB_APP_NAME, + FILE_DB_APP_MAJOR_VERSION, + FILE_DB_APP_MINOR_VERSION, + FILE_DB_MIN_MAJOR_VERSION, + FILE_DB_MIN_MINOR_VERSION); +} +int p3FileDatabase::tick() { // tick the watchers, possibly create new ones if additional friends do connect. // @@ -53,29 +92,37 @@ int p3FileLists::tick() return 0; } -void p3FileLists::tickRecv() +void p3FileDatabase::tickWatchers() { -} -void p3FileLists::tickSend() -{ - // go through the list of out requests and send them to the corresponding friends, if they are online. + NOT_IMPLEMENTED(); } -bool p3FileLists::loadList(std::list& items) +void p3FileDatabase::tickRecv() +{ +} +void p3FileDatabase::tickSend() +{ + // go through the list of out requests and send them to the corresponding friends, if they are online. + NOT_IMPLEMENTED(); +} + +bool p3FileDatabase::loadList(std::list& items) { // This loads // // - list of locally shared directories, and the permissions that go with them + NOT_IMPLEMENTED(); return true ; } -bool p3FileLists::saveList(const std::list& items) +bool p3FileDatabase::saveList(bool &cleanup, std::list&) { + NOT_IMPLEMENTED(); return true ; } -void p3FileLists::cleanup() +void p3FileDatabase::cleanup() { RS_STACK_MUTEX(mFLSMtx) ; @@ -83,15 +130,8 @@ void p3FileLists::cleanup() // P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; - std::set friend_set ; - { -#warning we should use a std::set in the first place. Modify rsPeers? - std::list friend_list ; - mLinkMgr->getFriendList(friend_list) ; - - for(std::list::const_iterator it(friend_list.begin());it!=friend_list.end();++it) - friend_set.insert(*it) ; - } + std::set friend_set ; + mServCtrl->getPeersConnected(getServiceInfo().mServiceType, friend_set); for(std::map::iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) if(friend_set.find(it->first) == friend_set.end()) @@ -125,4 +165,59 @@ void p3FileLists::cleanup() IndicateConfigChanged(); } +int p3FileDatabase::RequestDirDetails(void *ref, DirDetails&, FileSearchFlags) const +{ + NOT_IMPLEMENTED(); + return 0; +} +int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const +{ + NOT_IMPLEMENTED(); + return 0; +} +uint32_t p3FileDatabase::getType(void *ref) const +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed +{ + NOT_IMPLEMENTED(); +} +bool p3FileDatabase::inDirectoryCheck() +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::setWatchPeriod(uint32_t seconds) +{ + NOT_IMPLEMENTED(); +} +uint32_t p3FileDatabase::watchPeriod() +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::setRememberHashCacheDuration(uint32_t days) +{ + NOT_IMPLEMENTED(); +} +uint32_t p3FileDatabase::rememberHashCacheDuration() +{ + NOT_IMPLEMENTED(); + return 0; +} +void p3FileDatabase::clearHashCache() +{ + NOT_IMPLEMENTED(); +} +bool p3FileDatabase::rememberHashCache() +{ + NOT_IMPLEMENTED(); + return false; +} +void p3FileDatabase::setRememberHashCache(bool) +{ + NOT_IMPLEMENTED(); +} diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index e2c9d2016..7c0060183 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -21,23 +21,30 @@ // #pragma once +#include "ft/ftsearch.h" #include "retroshare/rsfiles.h" #include "services/p3service.h" +#include "file_sharing/hash_cache.h" + #include "pqi/p3cfgmgr.h" #include "pqi/p3linkmgr.h" -class RemoteDirectoryStorage ; -class RemoteSharedDirectoryWatcher ; -class LocalSharedDirectoryWatcher ; -class LocalSharedDirectoryMap ; -class HashCache ; +class RemoteDirectoryUpdater ; +class LocalDirectoryUpdater ; -class p3FileLists: public p3Service, public p3Config //, public RsSharedFileService +class RemoteDirectoryStorage ; +class LocalDirectoryStorage ; + +class HashStorage ; + +class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService { public: typedef uint64_t EntryIndex ; // this should probably be defined elsewhere + virtual RsServiceInfo getServiceInfo(); + struct RsFileListSyncRequest { RsPeerId peerId ; @@ -45,12 +52,17 @@ class p3FileLists: public p3Service, public p3Config //, public RsSharedFileServ // [...] more to add here }; - p3FileLists(p3LinkMgr *mpeers) ; - ~p3FileLists(); + p3FileDatabase(p3ServiceControl *mpeers) ; + ~p3FileDatabase(); + + /*! + * \brief forceSyncWithPeers + * + * Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new gorup of friend is created, or when + * a friend was added/removed from a group. + */ + void forceSyncWithPeers(); - /* - */ - // derived from p3Service // virtual int tick() ; @@ -62,32 +74,61 @@ class p3FileLists: public p3Service, public p3Config //, public RsSharedFileServ virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; virtual int SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; + // ftSearch + virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const; + // Interface for browsing dir hierarchy // - int RequestDirDetails(EntryIndex, DirDetails&, FileSearchFlags) const ; - uint32_t getType(const EntryIndex&) const ; + + void stopThreads() ; + void startThreads() ; + + int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); int RequestDirDetails(const std::string& path, DirDetails &details) const ; - // set/update shared directories - void setSharedDirectories(const std::list& dirs); - void getSharedDirectories(std::list& dirs); - void updateShareFlags(const SharedDirInfo& info) ; + // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. + + int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; + uint32_t getType(void *) const ; + + // set/update shared directories + void setSharedDirectories(const std::list& dirs); + void getSharedDirectories(std::list& dirs); + void updateShareFlags(const SharedDirInfo& info) ; + bool convertSharedFilePath(const std::string& path,std::string& fullpath); + + // interface for hash caching + + void setWatchPeriod(uint32_t seconds); + uint32_t watchPeriod() ; + void setRememberHashCacheDuration(uint32_t days) ; + uint32_t rememberHashCacheDuration() ; + void clearHashCache() ; + bool rememberHashCache() ; + void setRememberHashCache(bool) ; + + // interfact for directory parsing void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed bool inDirectoryCheck(); - // Derived from p3Config - // + // debug + void full_print(); + protected: + // Derived from p3Config + // virtual bool loadList(std::list& items); - virtual bool saveList(const std::list& items); + virtual bool saveList(bool &cleanup, std::list&); + virtual RsSerialiser *setupSerialiser() ; + void cleanup(); void tickRecv(); void tickSend(); void tickWatchers(); private: - p3LinkMgr *mLinkMgr ; + p3ServiceControl *mServCtrl ; // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. @@ -98,15 +139,14 @@ class p3FileLists: public p3Service, public p3Config //, public RsSharedFileServ // Directory storage hierarchies // std::map mRemoteDirectories ; - - LocalSharedDirectoryMap *mLocalSharedDirs ; + LocalDirectoryStorage *mLocalSharedDirs ; - RemoteSharedDirectoryWatcher *mRemoteDirWatcher ; - LocalSharedDirectoryWatcher *mLocalDirWatcher ; + RemoteDirectoryUpdater *mRemoteDirWatcher ; + LocalDirectoryUpdater *mLocalDirWatcher ; // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // - HashCache *mHashCache ; + HashStorage *mHashCache ; // Local flags and mutexes diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index fc66c7f2a..ae90dfe50 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -73,10 +73,8 @@ static const int32_t SAVE_TRANSFERS_DELAY = 301 ; // save transfer progress every 301 seconds. static const int32_t INACTIVE_CHUNKS_CHECK_DELAY = 240 ; // time after which an inactive chunk is released static const int32_t MAX_TIME_INACTIVE_REQUEUED = 120 ; // time after which an inactive ftFileControl is bt-queued -static const int32_t TIMOUT_CACHE_FILE_TRANSFER = 800 ; // time after which cache transfer gets cancelled if inactive. static const int32_t FT_FILECONTROL_QUEUE_ADD_END = 0 ; -static const int32_t FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE = 1 ; const uint32_t FT_CNTRL_STANDARD_RATE = 10 * 1024 * 1024; const uint32_t FT_CNTRL_SLOW_RATE = 100 * 1024; @@ -99,8 +97,8 @@ ftFileControl::ftFileControl(std::string fname, return; } -ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId) - :CacheTransfer(cs), p3Config(), +ftController::ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId) + : p3Config(), last_save_time(0), last_clean_time(0), mDataplex(dm), @@ -231,7 +229,6 @@ void ftController::data_tick() time_t now = time(NULL) ; if(now > last_save_time + SAVE_TRANSFERS_DELAY) { - cleanCacheDownloads() ; searchForDirectSources() ; IndicateConfigChanged() ; @@ -282,16 +279,15 @@ void ftController::searchForDirectSources() for(std::map::iterator it(mDownloads.begin()); it != mDownloads.end(); ++it) if(it->second->mState != ftFileControl::QUEUED && it->second->mState != ftFileControl::PAUSED) - if(! (it->second->mFlags & RS_FILE_REQ_CACHE)) - { - FileInfo info ; // info needs to be re-allocated each time, to start with a clear list of peers (it's not cleared down there) + { + FileInfo info ; // info needs to be re-allocated each time, to start with a clear list of peers (it's not cleared down there) - if(mSearch->search(it->first, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info)) - for(std::list::const_iterator pit = info.peers.begin(); pit != info.peers.end(); ++pit) - if(rsPeers->servicePermissionFlags(pit->peerId) & RS_NODE_PERM_DIRECT_DL) - if(it->second->mTransfer->addFileSource(pit->peerId)) /* if the sources don't exist already - add in */ - setPeerState(it->second->mTransfer, pit->peerId, FT_CNTRL_STANDARD_RATE, mServiceCtrl->isPeerConnected(mFtServiceId, pit->peerId)); - } + if(mSearch->search(it->first, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info)) + for(std::list::const_iterator pit = info.peers.begin(); pit != info.peers.end(); ++pit) + if(rsPeers->servicePermissionFlags(pit->peerId) & RS_NODE_PERM_DIRECT_DL) + if(it->second->mTransfer->addFileSource(pit->peerId)) /* if the sources don't exist already - add in */ + setPeerState(it->second->mTransfer, pit->peerId, FT_CNTRL_STANDARD_RATE, mServiceCtrl->isPeerConnected(mFtServiceId, pit->peerId)); + } } void ftController::tickTransfers() @@ -335,37 +331,6 @@ void ftController::setPriority(const RsFileHash& hash,DwlSpeed p) it->second->mTransfer->setDownloadPriority(p) ; } -void ftController::cleanCacheDownloads() -{ - std::vector toCancel ; - time_t now = time(NULL) ; - - { - RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ - - for(std::map::iterator it(mDownloads.begin());it!=mDownloads.end();++it) - if (((it->second)->mFlags & RS_FILE_REQ_CACHE) && it->second->mState != ftFileControl::DOWNLOADING) - // check if a cache file is downloaded, if the case, timeout the transfer after TIMOUT_CACHE_FILE_TRANSFER - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::run() cache transfer found. age of this tranfer is :" << (int)(time(NULL) - (it->second)->mCreateTime); - std::cerr << std::endl; -#endif - if (now > (it->second)->mCreator->creationTimeStamp() + TIMOUT_CACHE_FILE_TRANSFER) - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::run() cache transfer to old. Cancelling transfer. Hash :" << (it->second)->mHash << ", time=" << (it->second)->mCreateTime << ", now = " << time(NULL) ; - std::cerr << std::endl; -#endif - toCancel.push_back((it->second)->mHash); - } - } - } - - for(uint32_t i=0;imFlags & RS_FILE_REQ_CACHE) // cache file. add to potential move list - to_move_before.push_back(p) ; - else ++user_transfers ; // count one more user file in the prioritized range. else { if(to_move_after.size() + user_transfers >= _min_prioritized_transfers) // we caught enough transfers to move back to the top of the queue. break ; - if(!(_queue[p]->mFlags & RS_FILE_REQ_CACHE)) // non cache file. add to potential move list - to_move_after.push_back(p) ; + to_move_after.push_back(p) ; } } uint32_t to_move = (uint32_t)std::max(0,(int)_min_prioritized_transfers - (int)user_transfers) ; // we move as many transfers as needed to get _min_prioritized_transfers user transfers. @@ -482,27 +443,6 @@ void ftController::locked_addToQueue(ftFileControl* ftfc,int add_strategy) case FT_FILECONTROL_QUEUE_ADD_END: _queue.push_back(ftfc) ; locked_checkQueueElement(_queue.size()-1) ; break ; - case FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE: - { - // We add the transfer just before the first non cache transfer. - // This is costly, so only use this in case we really need it. - // - uint32_t pos =0; - while(pos < _queue.size() && (pos < _min_prioritized_transfers || (_queue[pos]->mFlags & RS_FILE_REQ_CACHE)>0) ) - ++pos ; - - _queue.push_back(NULL) ; - - for(int i=int(_queue.size())-1;i>(int)pos;--i) - { - _queue[i] = _queue[i-1] ; - locked_checkQueueElement(i) ; - } - - _queue[pos] = ftfc ; - locked_checkQueueElement(pos) ; - } - break ; } } @@ -787,8 +727,8 @@ bool ftController::completeFile(const RsFileHash& hash) std::string path; std::string name; uint64_t size = 0; - uint32_t state = 0; - uint32_t period = 0; + uint32_t state = 0; + uint32_t period = 0; TransferRequestFlags flags ; TransferRequestFlags extraflags ; uint32_t completeCount = 0; @@ -889,12 +829,8 @@ bool ftController::completeFile(const RsFileHash& hash) locked_queueRemove(it->second->mQueuePosition) ; /* switch map */ - if (!(fc->mFlags & RS_FILE_REQ_CACHE)) /* clean up completed cache files automatically */ - { - mCompleted[fc->mHash] = fc; - completeCount = mCompleted.size(); - } else - delete fc ; + mCompleted[fc->mHash] = fc; + completeCount = mCompleted.size(); mDownloads.erase(it); @@ -910,51 +846,14 @@ bool ftController::completeFile(const RsFileHash& hash) /* If it has a callback - do it now */ - if(flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA))// | RS_FILE_HINTS_MEDIA)) + if(flags & RS_FILE_REQ_EXTRA)// | RS_FILE_HINTS_MEDIA)) { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::completeFile() doing Callback, callbackflags:" << (flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA ));//| RS_FILE_HINTS_MEDIA)) ; - std::cerr << std::endl; -#endif - if(flags & RS_FILE_REQ_CACHE) - { - /* callback */ - if (state == ftFileControl::COMPLETED) - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::completeFile() doing Callback : Success"; - std::cerr << std::endl; -#endif - - CompletedCache(hash); - } - else - { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::completeFile() Cache Callback : Failed"; - std::cerr << std::endl; -#endif - FailedCache(hash); - } - } - - if(flags & RS_FILE_REQ_EXTRA) - { #ifdef CONTROL_DEBUG std::cerr << "ftController::completeFile() adding to ExtraList"; std::cerr << std::endl; #endif mExtraList->addExtraFile(path, hash, size, period, extraflags); - } - -// if(flags & RS_FILE_HINTS_MEDIA) -// { -//#ifdef CONTROL_DEBUG -// std::cerr << "ftController::completeFile() NULL MEDIA callback"; -// std::cerr << std::endl; -//#endif -// } } else { @@ -965,14 +864,12 @@ bool ftController::completeFile(const RsFileHash& hash) } /* Notify GUI */ - if ((flags & RS_FILE_REQ_CACHE) == 0) { - RsServer::notify()->AddPopupMessage(RS_POPUP_DOWNLOAD, hash.toStdString(), name, ""); + RsServer::notify()->AddPopupMessage(RS_POPUP_DOWNLOAD, hash.toStdString(), name, ""); - RsServer::notify()->notifyDownloadComplete(hash.toStdString()); - RsServer::notify()->notifyDownloadCompleteCount(completeCount); + RsServer::notify()->notifyDownloadComplete(hash.toStdString()); + RsServer::notify()->notifyDownloadCompleteCount(completeCount); - rsFiles->ForceDirectoryCheck() ; - } + rsFiles->ForceDirectoryCheck() ; IndicateConfigChanged(); /* completed transfer -> save */ return true; @@ -1134,17 +1031,16 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash // remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files. // - if(!(flags & RS_FILE_REQ_CACHE)) - for(std::list::iterator it = srcIds.begin(); it != srcIds.end(); ) - if(!(rsPeers->servicePermissionFlags(*it) & RS_NODE_PERM_DIRECT_DL)) - { - std::list::iterator tmp(it) ; - ++tmp ; - srcIds.erase(it) ; - it = tmp ; - } - else - ++it ; + for(std::list::iterator it = srcIds.begin(); it != srcIds.end(); ) + if(!(rsPeers->servicePermissionFlags(*it) & RS_NODE_PERM_DIRECT_DL)) + { + std::list::iterator tmp(it) ; + ++tmp ; + srcIds.erase(it) ; + it = tmp ; + } + else + ++it ; std::list::const_iterator it; std::list::const_iterator pit; @@ -1280,7 +1176,7 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash if(flags & RS_FILE_REQ_ANONYMOUS_ROUTING) mTurtle->monitorTunnels(hash,mFtServer,true) ; - bool assume_availability = flags & RS_FILE_REQ_CACHE ; // assume availability for cache files + bool assume_availability = false; ftFileCreator *fc = new ftFileCreator(savepath, size, hash,assume_availability); ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this); @@ -1911,58 +1807,6 @@ void ftController::statusChange(const std::list &plist) } } - /* Cache Interface */ -bool ftController::RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size) -{ -#ifdef CONTROL_DEBUG - std::cerr << "ftController::RequestCacheFile(" << id << ","; - std::cerr << path << "," << hash << "," << size << ")"; - std::cerr << std::endl; -#endif - - /* Request File */ - std::list ids; - ids.push_back(id); - - FileInfo info ; - if(mSearch->search(hash, RS_FILE_HINTS_CACHE, info)) - { -#ifdef CONTROL_DEBUG - std::cerr << "I already have this file:" << std::endl ; - std::cerr << " path: " << info.path << std::endl ; - std::cerr << " fname: " << info.fname << std::endl ; - std::cerr << " hash: " << info.hash << std::endl ; - - std::cerr << "Copying it !!" << std::endl ; -#endif - - if(info.size > 0 && copyFile(info.path,path+"/"+hash.toStdString())) - { - CompletedCache(hash); - return true ; - } - else - return false ; - } - - FileRequest(hash.toStdString(), hash, size, path, RS_FILE_REQ_CACHE | RS_FILE_REQ_NO_SEARCH, ids); - - return true; -} - - -bool ftController::CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size) -{ - std::cerr << "ftController::CancelCacheFile(" << id << ","; - std::cerr << path << "," << hash << "," << size << ")"; - std::cerr << std::endl; -#ifdef CONTROL_DEBUG -#endif - - return FileCancel(hash); - -} - const std::string active_downloads_size_ss("MAX_ACTIVE_DOWNLOADS"); const std::string min_prioritized_downl_ss("MIN_PRORITIZED_DOWNLOADS"); const std::string download_dir_ss("DOWN_DIR"); @@ -2056,14 +1900,6 @@ bool ftController::saveList(bool &cleanup, std::list& saveData) /* ignore cache files. As this is small files, better download them again from scratch at restart.*/ - if (fit->second->mFlags & RS_FILE_REQ_CACHE) - { -#ifdef CONTROL_DEBUG - std::cerr << "ftcontroller::saveList(): Not saving (callback) file entry " << fit->second->mName << ", " << fit->second->mHash << ", " << fit->second->mSize << std::endl ; -#endif - continue; - } - // Node: We still save finished transfers. This keeps transfers that are // in checking mode. Finished or checked transfers will restart and // immediately terminate/recheck at next startup. diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index fe20a7845..5ee84623b 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -113,12 +113,12 @@ class ftPendingRequest }; -class ftController: public CacheTransfer, public RsTickingThread, public pqiServiceMonitor, public p3Config +class ftController: public RsTickingThread, public pqiServiceMonitor, public p3Config { public: /* Setup */ - ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId); + ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId); void setFtSearchNExtra(ftSearch *, ftExtraList *); void setTurtleRouter(p3turtle *) ; @@ -195,10 +195,6 @@ class ftController: public CacheTransfer, public RsTickingThread, public pqiServ protected: - virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - - void cleanCacheDownloads() ; void searchForDirectSources() ; void tickTransfers() ; diff --git a/libretroshare/src/ft/ftdatamultiplex.cc b/libretroshare/src/ft/ftdatamultiplex.cc index a30545b38..ad16e0e4c 100644 --- a/libretroshare/src/ft/ftdatamultiplex.cc +++ b/libretroshare/src/ft/ftdatamultiplex.cc @@ -1104,8 +1104,6 @@ bool ftDataMultiplex::handleSearchRequest(const RsPeerId& peerId, const RsFileHa if(rsTurtle->isTurtlePeer(peerId)) hintflags |= RS_FILE_HINTS_NETWORK_WIDE ; - else - hintflags |= RS_FILE_HINTS_CACHE ; if(mSearch->search(hash, hintflags, info)) { diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index a861d79b7..f78784bee 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -31,6 +31,7 @@ #include "retroshare/rspeers.h" const int ftserverzone = 29539; +#include "file_sharing/p3filelists.h" #include "ft/ftturtlefiletransferitem.h" #include "ft/ftserver.h" #include "ft/ftextralist.h" @@ -64,15 +65,11 @@ static const time_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priorit /* Setup */ ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc) : p3Service(), - mPeerMgr(pm), - mServiceCtrl(sc), - mCacheStrapper(NULL), - mFiStore(NULL), mFiMon(NULL), + mPeerMgr(pm), mServiceCtrl(sc), + mFileDatabase(NULL), mFtController(NULL), mFtExtra(NULL), mFtDataplex(NULL), mFtSearch(NULL), srvMutex("ftServer") { - mCacheStrapper = new ftCacheStrapper(sc, getServiceInfo().mServiceType); - addSerialType(new RsFileTransferSerialiser()) ; } @@ -145,7 +142,7 @@ void ftServer::SetupFtServer() mFtDataplex = new ftDataMultiplex(ownId, this, mFtSearch); /* make Controller */ - mFtController = new ftController(mCacheStrapper, mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType); + mFtController = new ftController(mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType); mFtController -> setFtSearchNExtra(mFtSearch, mFtExtra); std::string tmppath = "."; mFtController->setPartialsDirectory(tmppath); @@ -153,21 +150,13 @@ void ftServer::SetupFtServer() /* Make Cache Source/Store */ - mFiStore = new ftFiStore(mCacheStrapper, mFtController, mPeerMgr, ownId, remotecachedir); - mFiMon = new ftFiMonitor(mCacheStrapper,localcachedir, ownId,mConfigPath); - - /* now add the set to the cachestrapper */ - CachePair cp(mFiMon, mFiStore, CacheId(RS_SERVICE_TYPE_FILE_INDEX, 0)); - mCacheStrapper -> addCachePair(cp); + mFileDatabase = new p3FileDatabase(mServiceCtrl) ; /* complete search setup */ - mFtSearch->addSearchMode(mCacheStrapper, RS_FILE_HINTS_CACHE); mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA); - mFtSearch->addSearchMode(mFiMon, RS_FILE_HINTS_LOCAL); - mFtSearch->addSearchMode(mFiStore, RS_FILE_HINTS_REMOTE); + mFtSearch->addSearchMode(mFileDatabase, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE); mServiceCtrl->registerServiceMonitor(mFtController, getServiceInfo().mServiceType); - mServiceCtrl->registerServiceMonitor(mCacheStrapper, getServiceInfo().mServiceType); return; } @@ -193,7 +182,7 @@ void ftServer::StartupThreads() /* startup Monitor Thread */ /* startup the FileMonitor (after cache load) */ /* start it up */ - mFiMon->start("ft monitor"); + mFileDatabase->startThreads(); /* Controller thread */ mFtController->start("ft ctrl"); @@ -210,9 +199,6 @@ void ftServer::StopThreads() /* stop Controller thread */ mFtController->join(); - /* stop Monitor Thread */ - mFiMon->join(); - /* self contained threads */ /* stop ExtraList Thread */ mFtExtra->join(); @@ -223,24 +209,15 @@ void ftServer::StopThreads() delete (mFtController); mFtController = NULL; - delete (mFiMon); - mFiMon = NULL; - delete (mFtExtra); mFtExtra = NULL; -} -CacheStrapper *ftServer::getCacheStrapper() -{ - return mCacheStrapper; + /* stop Monitor Thread */ + mFileDatabase->stopThreads(); + delete mFileDatabase; + mFileDatabase = NULL ; } -CacheTransfer *ftServer::getCacheTransfer() -{ - return mFtController; -} - - /***************************************************************/ /********************** RsFiles Interface **********************/ /***************************************************************/ @@ -256,11 +233,6 @@ bool ftServer::ResumeTransfers() return true; } -bool ftServer::checkHash(const RsFileHash& /*hash*/, std::string& /*error_string*/) -{ - return true ; -} - bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data) { return mFtDataplex->getFileData(hash, offset, requested_size,data); @@ -273,15 +245,7 @@ bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds) { - std::string error_string ; - - if(!checkHash(hash,error_string)) - { - RsServer::notify()->notifyErrorMsg(0,0,"Error handling hash \""+hash.toStdString()+"\". This hash appears to be invalid(Error string=\""+error_string+"\"). This is probably due an bad handling of strings.") ; - return false ; - } - - std::cerr << "Requesting " << fname << std::endl ; + std::cerr << "Requesting " << fname << std::endl ; if(!mFtController->FileRequest(fname, hash, size, dest, flags, srcIds)) return false ; @@ -402,16 +366,18 @@ std::string ftServer::getPartialsDirectory() return mFtController->getPartialsDirectory(); } +/***************************************************************/ +/************************* Other Access ************************/ +/***************************************************************/ - /***************************************************************/ - /************************* Other Access ************************/ - /***************************************************************/ +bool ftServer::copyFile(const std::string& source, const std::string& dest) +{ + return mFtController->copyFile(source, dest); +} void ftServer::FileDownloads(std::list &hashs) { mFtController->FileDownloads(hashs); - /* this only contains downloads.... not completed */ - //return mFtDataplex->FileDownloads(hashs); } bool ftServer::FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& cmap) @@ -438,7 +404,7 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi // file, we skip the call to fileDetails() for efficiency reasons. // FileInfo info2 ; - if( (!(info.transfer_info_flags & RS_FILE_REQ_CACHE)) && mFtController->FileDetails(hash, info2)) + if(mFtController->FileDetails(hash, info2)) info.fname = info2.fname ; return true ; @@ -539,12 +505,11 @@ bool ftServer::handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_i return res ; } - /***************************************************************/ - /******************* ExtraFileList Access **********************/ - /***************************************************************/ +/***************************************************************/ +/******************* ExtraFileList Access **********************/ +/***************************************************************/ -bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, - uint32_t period, TransferRequestFlags flags) +bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags) { return mFtExtra->addExtraFile(fname, hash, size, period, flags); } @@ -564,73 +529,27 @@ bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info) return mFtExtra->hashExtraFileDone(localpath, info); } -bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, - std::string destpath) +bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath) { return mFtExtra->moveExtraFile(fname, hash, size, destpath); } - - /***************************************************************/ - /******************** Directory Listing ************************/ - /***************************************************************/ +/***************************************************************/ +/******************** Directory Listing ************************/ +/***************************************************************/ int ftServer::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::RequestDirDetails(uid:" << uid; - std::cerr << ", path:" << path << ", ...) -> mFiStore"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } -#endif - if(uid == mServiceCtrl->getOwnId()) - return mFiMon->RequestDirDetails(path, details); - else - return mFiStore->RequestDirDetails(uid, path, details); + return mFileDatabase->RequestDirDetails(uid, path, details); } int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::RequestDirDetails(ref:" << ref; - std::cerr << ", flags:" << flags << ", ...) -> mFiStore"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } - -#endif - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->RequestDirDetails(ref, details, flags); - else - return mFiStore->RequestDirDetails(ref, details, flags); + return mFileDatabase->RequestDirDetails(ref,details,flags) ; } uint32_t ftServer::getType(void *ref, FileSearchFlags flags) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::RequestDirDetails(ref:" << ref; - std::cerr << ", flags:" << flags << ", ...) -> mFiStore"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } - -#endif - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->getType(ref); - else - return mFiStore->getType(ref); + return mFileDatabase->getType(ref) ; } /***************************************************************/ /******************** Search Interface *************************/ @@ -639,86 +558,55 @@ uint32_t ftServer::getType(void *ref, FileSearchFlags flags) int ftServer::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) { - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchKeywords(keywords, results,flags,RsPeerId()); - else - return mFiStore->SearchKeywords(keywords, results,flags); - return 0 ; + return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId()); } int ftServer::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::SearchKeywords()"; - std::cerr << std::endl; - - if (!mFiStore) - { - std::cerr << "mFiStore not SET yet = FAIL"; - std::cerr << std::endl; - } - -#endif - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchKeywords(keywords, results,flags,peer_id); - else - return mFiStore->SearchKeywords(keywords, results,flags); + return mFileDatabase->SearchKeywords(keywords, results,flags,peer_id); } int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags) { - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchBoolExp(exp,results,flags,RsPeerId()) ; - else - return mFiStore->searchBoolExp(exp, results); - return 0 ; + return mFileDatabase->SearchBoolExp(exp, results,flags,RsPeerId()); } int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) { - if(flags & RS_FILE_HINTS_LOCAL) - return mFiMon->SearchBoolExp(exp,results,flags,peer_id) ; - else - return mFiStore->searchBoolExp(exp, results); + return mFileDatabase->SearchBoolExp(exp,results,flags,peer_id) ; } - /***************************************************************/ /*************** Local Shared Dir Interface ********************/ /***************************************************************/ -bool ftServer::ConvertSharedFilePath(std::string path, std::string &fullpath) +bool ftServer::ConvertSharedFilePath(std::string path, std::string &fullpath) { - return mFiMon->convertSharedFilePath(path, fullpath); + return mFileDatabase->convertSharedFilePath(path, fullpath); } void ftServer::updateSinceGroupPermissionsChanged() { - mFiMon->forceDirListsRebuildAndSend(); + mFileDatabase->forceSyncWithPeers(); } void ftServer::ForceDirectoryCheck() { - mFiMon->forceDirectoryCheck(); + mFileDatabase->forceDirectoryCheck(); return; } bool ftServer::InDirectoryCheck() { - return mFiMon->inDirectoryCheck(); -} - -bool ftServer::copyFile(const std::string& source, const std::string& dest) -{ - return mFtController->copyFile(source, dest); + return mFileDatabase->inDirectoryCheck(); } bool ftServer::getSharedDirectories(std::list &dirs) { - mFiMon->getSharedDirectories(dirs); + mFileDatabase->getSharedDirectories(dirs); return true; } bool ftServer::setSharedDirectories(std::list &dirs) { - mFiMon->setSharedDirectories(dirs); + mFileDatabase->setSharedDirectories(dirs); return true; } @@ -728,7 +616,7 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir) _dir.filename = RsDirUtil::convertPathToUnix(_dir.filename); std::list dirList; - mFiMon->getSharedDirectories(dirList); + mFileDatabase->getSharedDirectories(dirList); // check that the directory is not already in the list. for(std::list::const_iterator it(dirList.begin());it!=dirList.end();++it) @@ -738,13 +626,13 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir) // ok then, add the shared directory. dirList.push_back(_dir); - mFiMon->setSharedDirectories(dirList); + mFileDatabase->setSharedDirectories(dirList); return true; } bool ftServer::updateShareFlags(const SharedDirInfo& info) { - mFiMon->updateShareFlags(info); + mFileDatabase->updateShareFlags(info); return true ; } @@ -761,7 +649,7 @@ bool ftServer::removeSharedDirectory(std::string dir) std::cerr << std::endl; #endif - mFiMon->getSharedDirectories(dirList); + mFileDatabase->getSharedDirectories(dirList); #ifdef SERVER_DEBUG for(it = dirList.begin(); it != dirList.end(); ++it) @@ -776,12 +664,7 @@ bool ftServer::removeSharedDirectory(std::string dir) if(it == dirList.end()) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::removeSharedDirectory()"; - std::cerr << " Cannot Find Directory... Fail"; - std::cerr << std::endl; -#endif - + std::cerr << "(EE) ftServer::removeSharedDirectory(): Cannot Find Directory... Fail" << std::endl; return false; } @@ -793,44 +676,44 @@ bool ftServer::removeSharedDirectory(std::string dir) #endif dirList.erase(it); - mFiMon->setSharedDirectories(dirList); + mFileDatabase->setSharedDirectories(dirList); return true; } void ftServer::setWatchPeriod(int minutes) { - mFiMon->setWatchPeriod(minutes*60) ; + mFileDatabase->setWatchPeriod(minutes*60) ; } int ftServer::watchPeriod() const { - return mFiMon->watchPeriod()/60 ; + return mFileDatabase->watchPeriod()/60 ; } void ftServer::setRememberHashFiles(bool b) { - mFiMon->setRememberHashCache(b) ; + mFileDatabase->setRememberHashCache(b) ; } bool ftServer::rememberHashFiles() const { - return mFiMon->rememberHashCache() ; + return mFileDatabase->rememberHashCache() ; } void ftServer::setRememberHashFilesDuration(uint32_t days) { - mFiMon->setRememberHashCacheDuration(days) ; + mFileDatabase->setRememberHashCacheDuration(days) ; } uint32_t ftServer::rememberHashFilesDuration() const { - return mFiMon->rememberHashCacheDuration() ; + return mFileDatabase->rememberHashCacheDuration() ; } void ftServer::clearHashCache() { - mFiMon->clearHashCache() ; + mFileDatabase->clearHashCache() ; } bool ftServer::getShareDownloadDirectory() { std::list dirList; - mFiMon->getSharedDirectories(dirList); + mFileDatabase->getSharedDirectories(dirList); std::string dir = mFtController->getDownloadDirectory(); @@ -844,7 +727,8 @@ bool ftServer::getShareDownloadDirectory() bool ftServer::shareDownloadDirectory(bool share) { - if (share) { + if (share) + { /* Share */ SharedDirInfo inf ; inf.filename = mFtController->getDownloadDirectory(); @@ -852,48 +736,22 @@ bool ftServer::shareDownloadDirectory(bool share) return addSharedDirectory(inf); } - - /* Unshare */ - std::string dir = mFtController->getDownloadDirectory(); - return removeSharedDirectory(dir); + else + { + /* Unshare */ + std::string dir = mFtController->getDownloadDirectory(); + return removeSharedDirectory(dir); + } } /***************************************************************/ /****************** End of RsFiles Interface *******************/ /***************************************************************/ - - /***************************************************************/ - /**************** Config Interface *****************************/ - /***************************************************************/ - -/* Key Functions to be overloaded for Full Configuration */ -RsSerialiser *ftServer::setupSerialiser() -{ - RsSerialiser *rss = new RsSerialiser ; - rss->addSerialType(new RsFileTransferSerialiser) ; - - //rss->addSerialType(new RsGeneralConfigSerialiser()); - - return rss ; -} - -bool ftServer::saveList(bool &/*cleanup*/, std::list& /*list*/) -{ - - return true; -} - -bool ftServer::loadList(std::list& /*load*/) -{ - return true; -} - -bool ftServer::loadConfigMap(std::map &/*configMap*/) -{ - return true; -} - +//bool ftServer::loadConfigMap(std::map &/*configMap*/) +//{ +// return true; +//} /***************************************************************/ /********************** Data Flow **********************/ @@ -1257,9 +1115,6 @@ int ftServer::tick() if(handleIncoming()) moreToTick = true; - if(handleCacheData()) - moreToTick = true; - static time_t last_law_priority_tasks_handling_time = 0 ; time_t now = time(NULL) ; @@ -1275,45 +1130,6 @@ int ftServer::tick() return moreToTick; } -bool ftServer::handleCacheData() -{ - std::list > cacheUpdates; - std::list >::iterator it; - int i=0 ; - -#ifdef SERVER_DEBUG_CACHE - std::cerr << "handleCacheData() " << std::endl; -#endif - mCacheStrapper->getCacheUpdates(cacheUpdates); - for(it = cacheUpdates.begin(); it != cacheUpdates.end(); ++it) - { - /* construct reply */ - RsFileTransferCacheItem *ci = new RsFileTransferCacheItem(); - - /* id from incoming */ - ci -> PeerId(it->first); - - ci -> file.hash = (it->second).hash; - ci -> file.name = (it->second).name; - ci -> file.path = ""; // (it->second).path; - ci -> file.filesize = (it->second).size; - ci -> cacheType = (it->second).cid.type; - ci -> cacheSubId = (it->second).cid.subid; - -#ifdef SERVER_DEBUG_CACHE - std::string out2 = "Outgoing CacheStrapper Update -> RsCacheItem:\n"; - ci -> print_string(out2); - std::cerr << out2 << std::endl; -#endif - - sendItem(ci); - - i++; - } - - return i>0 ; -} - int ftServer::handleIncoming() { // now File Input. @@ -1411,42 +1227,6 @@ int ftServer::handleIncoming() } } break ; - - case RS_PKT_SUBTYPE_FT_CACHE_ITEM: - { - RsFileTransferCacheItem *ci = dynamic_cast(item) ; - if (ci) - { -#ifdef SERVER_DEBUG_CACHE - std::cerr << "ftServer::handleIncoming: received cache item hash=" << ci->file.hash << ". from peer " << ci->PeerId() << std::endl; -#endif - /* these go to the CacheStrapper! */ - RsCacheData data; - data.pid = ci->PeerId(); - data.cid = CacheId(ci->cacheType, ci->cacheSubId); - data.path = ci->file.path; - data.name = ci->file.name; - data.hash = ci->file.hash; - data.size = ci->file.filesize; - data.recvd = time(NULL) ; - - mCacheStrapper->recvCacheResponse(data, time(NULL)); - } - } - break ; - -// case RS_PKT_SUBTYPE_FT_CACHE_REQUEST: -// { -// // do nothing -// RsFileTransferCacheRequestItem *cr = dynamic_cast(item) ; -// if (cr) -// { -//#ifdef SERVER_DEBUG_CACHE -// std::cerr << "ftServer::handleIncoming: received cache request from peer " << cr->PeerId() << std::endl; -//#endif -// } -// } -// break ; } delete item ; @@ -1465,7 +1245,7 @@ int ftServer::handleIncoming() bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr) { /* add all the subbits to config mgr */ - cfgmgr->addConfiguration("ft_shared.cfg", mFiMon); + cfgmgr->addConfiguration("ft_database.cfg", mFileDatabase); cfgmgr->addConfiguration("ft_extra.cfg", mFtExtra); cfgmgr->addConfiguration("ft_transfers.cfg", mFtController); diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 0c5b12aa7..1b66db710 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -70,230 +70,203 @@ class p3turtle; class p3PeerMgr; class p3ServiceControl; +class p3FileDatabase; class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService { - public: +public: - /***************************************************************/ - /******************** Setup ************************************/ - /***************************************************************/ + /***************************************************************/ + /******************** Setup ************************************/ + /***************************************************************/ - ftServer(p3PeerMgr *peerMgr, p3ServiceControl *serviceCtrl); - virtual RsServiceInfo getServiceInfo(); + ftServer(p3PeerMgr *peerMgr, p3ServiceControl *serviceCtrl); + virtual RsServiceInfo getServiceInfo(); - /* Assign important variables */ - void setConfigDirectory(std::string path); + /* Assign important variables */ + void setConfigDirectory(std::string path); - /* add Config Items (Extra, Controller) */ - void addConfigComponents(p3ConfigMgr *mgr); + /* add Config Items (Extra, Controller) */ + void addConfigComponents(p3ConfigMgr *mgr); - virtual CacheStrapper *getCacheStrapper(); - virtual CacheTransfer *getCacheTransfer(); + const RsPeerId& OwnId(); - const RsPeerId& OwnId(); + /* Final Setup (once everything is assigned) */ + void SetupFtServer() ; + virtual void connectToTurtleRouter(p3turtle *p) ; - /* Final Setup (once everything is assigned) */ - void SetupFtServer() ; - virtual void connectToTurtleRouter(p3turtle *p) ; + // Implements RsTurtleClientService + // + virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; + virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; + virtual RsTurtleGenericTunnelItem *deserialiseItem(void *data,uint32_t size) const ; - // Checks that the given hash is well formed. Used to chase - // string bugs. - static bool checkHash(const RsFileHash& hash,std::string& error_string) ; + void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; + void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; - // Implements RsTurtleClientService - // - virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; - virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; - virtual RsTurtleGenericTunnelItem *deserialiseItem(void *data,uint32_t size) const ; + /***************************************************************/ + /*************** Control Interface *****************************/ + /************** (Implements RsFiles) ***************************/ + /***************************************************************/ - void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; - void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; + void StartupThreads(); + void StopThreads(); - /***************************************************************/ - /*************** Control Interface *****************************/ - /************** (Implements RsFiles) ***************************/ - /***************************************************************/ + // member access - void StartupThreads(); - void StopThreads(); + ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; } + ftController *getController() const { return mFtController ; } - // member access - - ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; } - ftController *getController() const { return mFtController ; } - - /** + /** * @see RsFiles::getFileData */ - bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data); + bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data); - /*** - * Control of Downloads - ***/ - virtual bool alreadyHaveFile(const RsFileHash& hash, FileInfo &info); - virtual bool FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds); - virtual bool FileCancel(const RsFileHash& hash); - virtual bool FileControl(const RsFileHash& hash, uint32_t flags); - virtual bool FileClearCompleted(); - virtual bool setDestinationDirectory(const RsFileHash& hash,const std::string& new_path) ; - virtual bool setDestinationName(const RsFileHash& hash,const std::string& new_name) ; - virtual bool setChunkStrategy(const RsFileHash& hash,FileChunksInfo::ChunkStrategy s) ; - virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ; - virtual FileChunksInfo::ChunkStrategy defaultChunkStrategy() ; - virtual uint32_t freeDiskSpaceLimit() const ; - virtual void setFreeDiskSpaceLimit(uint32_t size_in_mb) ; + /*** + * Control of Downloads + ***/ + virtual bool alreadyHaveFile(const RsFileHash& hash, FileInfo &info); + virtual bool FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds); + virtual bool FileCancel(const RsFileHash& hash); + virtual bool FileControl(const RsFileHash& hash, uint32_t flags); + virtual bool FileClearCompleted(); + virtual bool setDestinationDirectory(const RsFileHash& hash,const std::string& new_path) ; + virtual bool setDestinationName(const RsFileHash& hash,const std::string& new_name) ; + virtual bool setChunkStrategy(const RsFileHash& hash,FileChunksInfo::ChunkStrategy s) ; + virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ; + virtual FileChunksInfo::ChunkStrategy defaultChunkStrategy() ; + virtual uint32_t freeDiskSpaceLimit() const ; + virtual void setFreeDiskSpaceLimit(uint32_t size_in_mb) ; - /*** - * Control of Downloads Priority. - ***/ - virtual uint32_t getMinPrioritizedTransfers() ; - virtual void setMinPrioritizedTransfers(uint32_t s) ; - virtual uint32_t getQueueSize() ; - virtual void setQueueSize(uint32_t s) ; - virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv); - virtual bool changeDownloadSpeed(const RsFileHash& hash, int speed); - virtual bool getDownloadSpeed(const RsFileHash& hash, int & speed); - virtual bool clearDownload(const RsFileHash& hash); - //virtual void getDwlDetails(std::list & details); + /*** + * Control of Downloads Priority. + ***/ + virtual uint32_t getMinPrioritizedTransfers() ; + virtual void setMinPrioritizedTransfers(uint32_t s) ; + virtual uint32_t getQueueSize() ; + virtual void setQueueSize(uint32_t s) ; + virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv); + virtual bool changeDownloadSpeed(const RsFileHash& hash, int speed); + virtual bool getDownloadSpeed(const RsFileHash& hash, int & speed); + virtual bool clearDownload(const RsFileHash& hash); + //virtual void getDwlDetails(std::list & details); - /*** - * Download/Upload Details - ***/ - virtual void FileDownloads(std::list &hashs); - virtual bool FileUploads(std::list &hashs); - virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info); - virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ; - virtual bool FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& map) ; + /*** + * Download/Upload Details + ***/ + virtual void FileDownloads(std::list &hashs); + virtual bool FileUploads(std::list &hashs); + virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info); + virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ; + virtual bool FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& map) ; - /*** - * Extra List Access - ***/ - virtual bool ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags); - virtual bool ExtraFileRemove(const RsFileHash& hash, TransferRequestFlags flags); - virtual bool ExtraFileHash(std::string localpath, uint32_t period, TransferRequestFlags flags); - virtual bool ExtraFileStatus(std::string localpath, FileInfo &info); - virtual bool ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath); + /*** + * Extra List Access + ***/ + virtual bool ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags); + virtual bool ExtraFileRemove(const RsFileHash& hash, TransferRequestFlags flags); + virtual bool ExtraFileHash(std::string localpath, uint32_t period, TransferRequestFlags flags); + virtual bool ExtraFileStatus(std::string localpath, FileInfo &info); + virtual bool ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath); - /*** - * Directory Listing / Search Interface - ***/ - virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); - virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags); - virtual uint32_t getType(void *ref,FileSearchFlags flags) ; + /*** + * Directory Listing / Search Interface + ***/ + virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); + virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags); + virtual uint32_t getType(void *ref,FileSearchFlags flags) ; - virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); - virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); + virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); + virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); + virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags); + virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); - /*** - * Utility Functions - ***/ - virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath); - virtual void ForceDirectoryCheck(); - virtual void updateSinceGroupPermissionsChanged() ; - virtual bool InDirectoryCheck(); - virtual bool copyFile(const std::string& source, const std::string& dest); + /*** + * Utility Functions + ***/ + virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath); + virtual void ForceDirectoryCheck(); + virtual void updateSinceGroupPermissionsChanged() ; + virtual bool InDirectoryCheck(); + virtual bool copyFile(const std::string& source, const std::string& dest); - /*** - * Directory Handling - ***/ - virtual void setDownloadDirectory(std::string path); - virtual void setPartialsDirectory(std::string path); - virtual std::string getDownloadDirectory(); - virtual std::string getPartialsDirectory(); + /*** + * Directory Handling + ***/ + virtual void setDownloadDirectory(std::string path); + virtual void setPartialsDirectory(std::string path); + virtual std::string getDownloadDirectory(); + virtual std::string getPartialsDirectory(); - virtual bool getSharedDirectories(std::list &dirs); - virtual bool setSharedDirectories(std::list &dirs); - virtual bool addSharedDirectory(const SharedDirInfo& dir); - virtual bool updateShareFlags(const SharedDirInfo& dir); // updates the flags. The directory should already exist ! - virtual bool removeSharedDirectory(std::string dir); + virtual bool getSharedDirectories(std::list &dirs); + virtual bool setSharedDirectories(std::list &dirs); + virtual bool addSharedDirectory(const SharedDirInfo& dir); + virtual bool updateShareFlags(const SharedDirInfo& dir); // updates the flags. The directory should already exist ! + virtual bool removeSharedDirectory(std::string dir); - virtual bool getShareDownloadDirectory(); - virtual bool shareDownloadDirectory(bool share); + virtual bool getShareDownloadDirectory(); + virtual bool shareDownloadDirectory(bool share); - virtual void setRememberHashFilesDuration(uint32_t days) ; - virtual uint32_t rememberHashFilesDuration() const ; - virtual bool rememberHashFiles() const ; - virtual void setRememberHashFiles(bool) ; - virtual void clearHashCache() ; - virtual void setWatchPeriod(int minutes) ; - virtual int watchPeriod() const ; + virtual void setRememberHashFilesDuration(uint32_t days) ; + virtual uint32_t rememberHashFilesDuration() const ; + virtual bool rememberHashFiles() const ; + virtual void setRememberHashFiles(bool) ; + virtual void clearHashCache() ; + virtual void setWatchPeriod(int minutes) ; + virtual int watchPeriod() const ; - /***************************************************************/ - /*************** Control Interface *****************************/ - /***************************************************************/ + /***************************************************************/ + /*************** Data Transfer Interface ***********************/ + /***************************************************************/ +public: + virtual bool sendData(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data); + virtual bool sendDataRequest(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize); + virtual bool sendChunkMapRequest(const RsPeerId& peer_id,const RsFileHash& hash,bool is_client) ; + virtual bool sendChunkMap(const RsPeerId& peer_id,const RsFileHash& hash,const CompressedChunkMap& cmap,bool is_client) ; + virtual bool sendSingleChunkCRCRequest(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number) ; + virtual bool sendSingleChunkCRC(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ; - /***************************************************************/ - /*************** Data Transfer Interface ***********************/ - /***************************************************************/ - public: - virtual bool sendData(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data); - virtual bool sendDataRequest(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize); - virtual bool sendChunkMapRequest(const RsPeerId& peer_id,const RsFileHash& hash,bool is_client) ; - virtual bool sendChunkMap(const RsPeerId& peer_id,const RsFileHash& hash,const CompressedChunkMap& cmap,bool is_client) ; - virtual bool sendSingleChunkCRCRequest(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number) ; - virtual bool sendSingleChunkCRC(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ; + /*************** Internal Transfer Fns *************************/ + virtual int tick(); - /*************** Internal Transfer Fns *************************/ - virtual int tick(); + /* Configuration */ + bool addConfiguration(p3ConfigMgr *cfgmgr); + bool ResumeTransfers(); - /* Configuration */ - bool addConfiguration(p3ConfigMgr *cfgmgr); - bool ResumeTransfers(); + /*************************** p3 Config Overload ********************/ - /******************* p3 Config Overload ************************/ - protected: - /* Key Functions to be overloaded for Full Configuration */ - virtual RsSerialiser *setupSerialiser(); - virtual bool saveList(bool &cleanup, std::list&); - virtual bool loadList(std::list& load); +protected: + int handleIncoming() ; + bool handleCacheData() ; - private: - bool loadConfigMap(std::map &configMap); - /******************* p3 Config Overload ************************/ +private: - /*************************** p3 Config Overload ********************/ + /**** INTERNAL FUNCTIONS ***/ + //virtual int reScanDirs(); + //virtual int check_dBUpdate(); - protected: - int handleIncoming() ; - bool handleCacheData() ; +private: - private: + /* no need for Mutex protection - + * as each component is protected independently. + */ + p3PeerMgr *mPeerMgr; + p3ServiceControl *mServiceCtrl; + p3FileDatabase *mFileDatabase ; + ftController *mFtController; + ftExtraList *mFtExtra; + ftDataMultiplex *mFtDataplex; + p3turtle *mTurtleRouter ; + ftFileSearch *mFtSearch; - /**** INTERNAL FUNCTIONS ***/ - //virtual int reScanDirs(); - //virtual int check_dBUpdate(); - - private: - - /* no need for Mutex protection - - * as each component is protected independently. - */ - p3PeerMgr *mPeerMgr; - p3ServiceControl *mServiceCtrl; - - ftCacheStrapper *mCacheStrapper; - ftFiStore *mFiStore; - ftFiMonitor *mFiMon; - - ftController *mFtController; - ftExtraList *mFtExtra; - - ftDataMultiplex *mFtDataplex; - p3turtle *mTurtleRouter ; - - ftFileSearch *mFtSearch; - - RsMutex srvMutex; - std::string mConfigPath; - std::string mDownloadPath; - std::string mPartialsPath; + RsMutex srvMutex; + std::string mConfigPath; + std::string mDownloadPath; + std::string mPartialsPath; }; diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 31ee2a39f..af1bfbb65 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -410,12 +410,13 @@ void RsPluginManager::slowTickPlugins(time_t seconds) void RsPluginManager::registerCacheServices() { +#warning this code should go std::cerr << " Registering cache services." << std::endl; for(uint32_t i=0;i<_plugins.size();++i) if(_plugins[i].plugin != NULL && _plugins[i].plugin->rs_cache_service() != NULL) { - rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ; + //rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ; std::cerr << " adding new cache pair for plugin " << _plugins[i].plugin->getPluginName() << ", with RS_ID " << _plugins[i].plugin->rs_service_id() << std::endl ; } } diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index c6480b540..6fd1967d7 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -65,26 +65,26 @@ const uint32_t RS_FILE_PEER_OFFLINE = 0x00002000; // Flags used when requesting info about transfers, mostly to filter out the result. // -const FileSearchFlags RS_FILE_HINTS_CACHE ( 0x00000001 ); -const FileSearchFlags RS_FILE_HINTS_EXTRA ( 0x00000002 ); -const FileSearchFlags RS_FILE_HINTS_LOCAL ( 0x00000004 ); -const FileSearchFlags RS_FILE_HINTS_REMOTE ( 0x00000008 ); +const FileSearchFlags RS_FILE_HINTS_CACHE_deprecated ( 0x00000001 ); +const FileSearchFlags RS_FILE_HINTS_EXTRA ( 0x00000002 ); +const FileSearchFlags RS_FILE_HINTS_LOCAL ( 0x00000004 ); +const FileSearchFlags RS_FILE_HINTS_REMOTE ( 0x00000008 ); const FileSearchFlags RS_FILE_HINTS_DOWNLOAD ( 0x00000010 ); -const FileSearchFlags RS_FILE_HINTS_UPLOAD ( 0x00000020 ); -const FileSearchFlags RS_FILE_HINTS_SPEC_ONLY ( 0x01000000 ); +const FileSearchFlags RS_FILE_HINTS_UPLOAD ( 0x00000020 ); +const FileSearchFlags RS_FILE_HINTS_SPEC_ONLY ( 0x01000000 ); -const FileSearchFlags RS_FILE_HINTS_NETWORK_WIDE ( 0x00000080 );// anonymously shared over network -const FileSearchFlags RS_FILE_HINTS_BROWSABLE ( 0x00000100 );// browsable by friends -const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000180 );// OR of the last two flags. Used to filter out. +const FileSearchFlags RS_FILE_HINTS_NETWORK_WIDE ( 0x00000080 );// anonymously shared over network +const FileSearchFlags RS_FILE_HINTS_BROWSABLE ( 0x00000100 );// browsable by friends +const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000180 );// OR of the last two flags. Used to filter out. // Flags used when requesting a transfer // const TransferRequestFlags RS_FILE_REQ_ANONYMOUS_ROUTING ( 0x00000040 ); // Use to ask turtle router to download the file. const TransferRequestFlags RS_FILE_REQ_ASSUME_AVAILABILITY ( 0x00000200 ); // Assume full source availability. Used for cache files. -const TransferRequestFlags RS_FILE_REQ_CACHE ( 0x00000400 ); // Assume full source availability. Used for cache files. +const TransferRequestFlags RS_FILE_REQ_CACHE_deprecated ( 0x00000400 ); // Assume full source availability. Used for cache files. const TransferRequestFlags RS_FILE_REQ_EXTRA ( 0x00000800 ); const TransferRequestFlags RS_FILE_REQ_MEDIA ( 0x00001000 ); -const TransferRequestFlags RS_FILE_REQ_BACKGROUND ( 0x00002000 ); // To download slowly. +const TransferRequestFlags RS_FILE_REQ_BACKGROUND ( 0x00002000 ); // To download slowly. const TransferRequestFlags RS_FILE_REQ_NO_SEARCH ( 0x02000000 ); // disable searching for potential direct sources. // const uint32_t RS_FILE_HINTS_SHARE_FLAGS_MASK = RS_FILE_HINTS_NETWORK_WIDE_OTHERS | RS_FILE_HINTS_BROWSABLE_OTHERS @@ -222,9 +222,6 @@ class RsFiles virtual void setWatchPeriod(int minutes) =0; virtual int watchPeriod() const =0; - virtual CacheStrapper *getCacheStrapper() =0; - virtual CacheTransfer *getCacheTransfer() =0; - virtual bool getShareDownloadDirectory() = 0; virtual bool shareDownloadDirectory(bool share) = 0; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 733009932..40c3417ff 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1228,8 +1228,6 @@ int RsServer::StartupRetroShare() ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory()); ftserver->SetupFtServer() ; - CacheStrapper *mCacheStrapper = ftserver->getCacheStrapper(); - //CacheTransfer *mCacheTransfer = ftserver->getCacheTransfer(); /* setup any extra bits (Default Paths) */ ftserver->setPartialsDirectory(emergencyPartialsDir); @@ -1273,7 +1271,6 @@ int RsServer::StartupRetroShare() // These are needed to load plugins: plugin devs might want to know the place of // cache directories, get pointers to cache strapper, or access ownId() // - mPluginsManager->setCacheDirectories(localcachedir,remotecachedir) ; mPluginsManager->setServiceControl(serviceCtrl) ; // std::cerr << "rsinitconf (core 1) = " << (void*)rsInitConfig<addMonitor(serviceCtrl); mLinkMgr->addMonitor(serviceInfo); - // NOTE these were added in ftServer (was added twice). - //mLinkMgr->addMonitor(mCacheStrapper); - //mLinkMgr->addMonitor(((ftController *) mCacheTransfer)); - // Services that have been changed to pqiServiceMonitor serviceCtrl->registerServiceMonitor(msgSrv, msgSrv->getServiceInfo().mServiceType); serviceCtrl->registerServiceMonitor(mDisc, mDisc->getServiceInfo().mServiceType); @@ -1668,7 +1661,6 @@ int RsServer::StartupRetroShare() * after all the other configurations have happened. */ - mConfigMgr->addConfiguration("cache.cfg", mCacheStrapper); mConfigMgr->loadConfiguration(); /**************************************************************************/ diff --git a/libretroshare/src/serialiser/rsserviceids.h b/libretroshare/src/serialiser/rsserviceids.h index 7aadd3dea..475d92595 100644 --- a/libretroshare/src/serialiser/rsserviceids.h +++ b/libretroshare/src/serialiser/rsserviceids.h @@ -49,6 +49,7 @@ const uint16_t RS_SERVICE_TYPE_TUNNEL = 0x0015; const uint16_t RS_SERVICE_TYPE_HEARTBEAT = 0x0016; const uint16_t RS_SERVICE_TYPE_FILE_TRANSFER = 0x0017; const uint16_t RS_SERVICE_TYPE_GROUTER = 0x0018; +const uint16_t RS_SERVICE_TYPE_FILE_DATABASE = 0x0019; const uint16_t RS_SERVICE_TYPE_SERVICEINFO = 0x0020; /* Bandwidth Control */ diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index 3a7f5cf61..895b082c7 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -1269,13 +1269,6 @@ void TransfersDialog::insertTransfers() continue; } - if ((fileInfo.transfer_info_flags & RS_FILE_REQ_CACHE) && !_show_cache_transfers) { - // if file transfer is a cache file index file, don't show it - DLListModel->removeRow(row); - rowCount = DLListModel->rowCount(); - continue; - } - hashs.erase(hashIt); if (addItem(row, fileInfo) < 0) { @@ -1295,11 +1288,6 @@ void TransfersDialog::insertTransfers() continue; } - if ((fileInfo.transfer_info_flags & RS_FILE_REQ_CACHE) && !_show_cache_transfers) { - //if file transfer is a cache file index file, don't show it - continue; - } - addItem(-1, fileInfo); } @@ -1322,9 +1310,6 @@ void TransfersDialog::insertTransfers() if (!rsFiles->FileDetails(*it, RS_FILE_HINTS_UPLOAD, info)) continue; - if((info.transfer_info_flags & RS_FILE_REQ_CACHE) && _show_cache_transfers) - continue ; - std::list::iterator pit; for(pit = info.peers.begin(); pit != info.peers.end(); ++pit) { From 538a11fd14c3686aa6cad6a5d72c53381bb263ba Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 31 Jul 2016 15:59:58 +0200 Subject: [PATCH 016/174] fixed compilation. Added missing functions, left blank --- .../src/file_sharing/directory_storage.cc | 68 +++++++++- .../src/file_sharing/directory_storage.h | 27 ++-- libretroshare/src/file_sharing/p3filelists.cc | 128 +++++++++++++++++- libretroshare/src/file_sharing/p3filelists.h | 14 +- libretroshare/src/ft/ftserver.cc | 7 +- 5 files changed, 221 insertions(+), 23 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index d94cb64f0..8663dddda 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,4 +1,5 @@ #include +#include "util/rsdir.h" #include "directory_storage.h" /******************************************************************************************************************/ @@ -321,6 +322,8 @@ private: return true ; } + + std::map mHashes ; // used for fast search access }; /******************************************************************************************************************/ @@ -442,10 +445,71 @@ void DirectoryStorage::print() void LocalDirectoryStorage::setSharedDirectoryList(const std::list& lst) { RS_STACK_MUTEX(mDirStorageMtx) ; - mLocalDirs = lst ; + + for(std::list::const_iterator it(lst.begin());it!= lst.end();++it) + mLocalDirs[(*it).filename] = *it ; } void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) { RS_STACK_MUTEX(mDirStorageMtx) ; - lst = mLocalDirs ; + + lst.clear(); + + for(std::map::iterator it(mLocalDirs.begin());it!=mLocalDirs.end();++it) + lst.push_back(it->second) ; } + +void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + std::map::iterator it = mLocalDirs.find(info.virtualname) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; + return ; + } + it->second = info; +} + +bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) +{ + std::string shpath = RsDirUtil::removeRootDir(path); + std::string basedir = RsDirUtil::getRootDir(path); + std::string realroot ; + { + RS_STACK_MUTEX(mDirStorageMtx) ; + realroot = locked_findRealRootFromVirtualFilename(basedir); + } + + if (realroot.empty()) + return false; + + /* construct full name */ + fullpath = realroot + "/"; + fullpath += shpath; + + return true; +} + +std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const +{ + /**** MUST ALREADY BE LOCKED ****/ + + std::map::const_iterator cit = mLocalDirs.find(virtual_rootdir) ; + + if (cit == mLocalDirs.end()) + { + std::cerr << "FileIndexMonitor::locked_findRealRoot() Invalid RootDir: " << virtual_rootdir << std::endl; + return std::string(); + } + return cit->second.filename; +} + +bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +{ + NOT_IMPLEMENTED() ; + return false; +} + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index bd0f5b8ce..28ba00401 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -16,6 +16,8 @@ static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; +#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } + class InternalFileHierarchyStorage ; class DirectoryStorage @@ -29,9 +31,9 @@ class DirectoryStorage void save() const ; - virtual int searchTerms(const std::list& terms, std::list &results) const { return 0;} - virtual int searchHash(const RsFileHash& hash, std::list &results) const { return 0;} - virtual int searchBoolExp(Expression * exp, std::list &results) const { return 0; } + virtual int searchTerms(const std::list& terms, std::list &results) const { NOT_IMPLEMENTED() ; return 0;} + virtual int searchHash(const RsFileHash& hash, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } + virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } void getFileDetails(EntryIndex i) ; @@ -112,7 +114,6 @@ class DirectoryStorage InternalFileHierarchyStorage *mFileHierarchy ; std::string mFileName; - protected: RsMutex mDirStorageMtx ; }; @@ -120,7 +121,7 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const RsPeerId& pid) ; + RemoteDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} virtual ~RemoteDirectoryStorage() {} }; @@ -133,10 +134,20 @@ public: void setSharedDirectoryList(const std::list& lst) ; void getSharedDirectoryList(std::list& lst) ; -private: - std::list mLocalDirs ; + void updateShareFlags(const SharedDirInfo& info) ; + bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ; - std::map mHashes ; // used for fast search access + /*! + * \brief getFileInfo Converts an index info a full file info structure. + * \param i index in the directory structure + * \param info structure to be filled in + * \return false if the file does not exist, or is a directory,... + */ + bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ; +private: + std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; + + std::map mLocalDirs ; // map is better for search. it->first=it->second.filename }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0fd3a830c..ffc3d401f 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -5,6 +5,7 @@ #include "file_sharing/directory_updater.h" #include "retroshare/rsids.h" +#include "retroshare/rspeers.h" #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " @@ -13,8 +14,6 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not implemented!" << std::endl; } - p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") { @@ -41,6 +40,10 @@ void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) { mLocalSharedDirs->getSharedDirectoryList(shared_dirs) ; } +void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) +{ + mLocalSharedDirs->updateShareFlags(info) ; +} p3FileDatabase::~p3FileDatabase() { @@ -86,12 +89,26 @@ int p3FileDatabase::tick() // cleanup // - remove/delete shared file lists for people who are not friend anymore // - - cleanup(); return 0; } +void p3FileDatabase::startThreads() +{ + std::cerr << "Starting hash cache thread..." ; + + mHashCache->start(); +} +void p3FileDatabase::stopThreads() +{ + std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; + + mHashCache->fullstop(); + + std::cerr << "Done." << std::endl; +} + void p3FileDatabase::tickWatchers() { NOT_IMPLEMENTED(); @@ -156,7 +173,7 @@ void p3FileDatabase::cleanup() { P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; - mRemoteDirectories[*it] = new RemoteDirectoryStorage(*it); + mRemoteDirectories[*it] = new RemoteDirectoryStorage(makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } @@ -165,11 +182,22 @@ void p3FileDatabase::cleanup() IndicateConfigChanged(); } +std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const +{ +#warning we should use the default paths here. Ask p3config + return "dirlist_"+pid.toStdString()+".txt" ; +} + int p3FileDatabase::RequestDirDetails(void *ref, DirDetails&, FileSearchFlags) const { NOT_IMPLEMENTED(); return 0; } +int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) +{ + NOT_IMPLEMENTED(); + return 0; +} int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const { NOT_IMPLEMENTED(); @@ -189,7 +217,6 @@ bool p3FileDatabase::inDirectoryCheck() NOT_IMPLEMENTED(); return 0; } - void p3FileDatabase::setWatchPeriod(uint32_t seconds) { NOT_IMPLEMENTED(); @@ -221,3 +248,94 @@ void p3FileDatabase::setRememberHashCache(bool) { NOT_IMPLEMENTED(); } + +bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const +{ + std::list firesults; + mLocalSharedDirs->searchHash(hash,firesults) ; + + NOT_IMPLEMENTED(); + return false; +} + +int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) +{ + std::list firesults; + mLocalSharedDirs->searchTerms(keywords,firesults) ; + + return filterResults(firesults,results,flags,client_peer_id) ; +} +int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const +{ + std::list firesults; + mLocalSharedDirs->searchBoolExp(exp,firesults) ; + + return filterResults(firesults,results,flags,client_peer_id) ; +} +bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const +{ + if(hintflags & RS_FILE_HINTS_LOCAL) + { + std::list res; + mLocalSharedDirs->searchHash(hash,res) ; + + if(res.empty()) + return false; + + EntryIndex indx = *res.begin() ; // no need to report dupicates + + mLocalSharedDirs->getFileInfo(indx,info) ; + } + else + { + NOT_IMPLEMENTED(); + return 0; + } +} + +int p3FileDatabase::filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const +{ + results.clear(); + +#ifdef P3FILELISTS_DEBUG + if((flags & ~RS_FILE_HINTS_PERMISSION_MASK) > 0) + std::cerr << "(EE) ***** FileIndexMonitor:: Flags ERROR in filterResults!!" << std::endl; +#endif + /* translate/filter results */ + + for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) + { + DirDetails cdetails ; + RequestDirDetails ((void*)*rit,cdetails,FileSearchFlags(0u)); +#ifdef P3FILELISTS_DEBUG + std::cerr << "Filtering candidate " << (*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; +#endif + + if(!peer_id.isNull()) + { + FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ; + + if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags )) + { + cdetails.id.clear() ; + results.push_back(cdetails); +#ifdef P3FILELISTS_DEBUG + std::cerr << ": kept" << std::endl ; +#endif + } +#ifdef P3FILELISTS_DEBUG + else + std::cerr << ": discarded" << std::endl ; +#endif + } + else + results.push_back(cdetails); + } + + return !results.empty() ; +} + +bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath) +{ + return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ; +} diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 7c0060183..0d631cdbd 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -26,6 +26,7 @@ #include "services/p3service.h" #include "file_sharing/hash_cache.h" +#include "file_sharing/directory_storage.h" #include "pqi/p3cfgmgr.h" #include "pqi/p3linkmgr.h" @@ -41,7 +42,7 @@ class HashStorage ; class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService { public: - typedef uint64_t EntryIndex ; // this should probably be defined elsewhere + typedef DirectoryStorage::EntryIndex EntryIndex; // this should probably be defined elsewhere virtual RsServiceInfo getServiceInfo(); @@ -58,10 +59,10 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub /*! * \brief forceSyncWithPeers * - * Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new gorup of friend is created, or when + * Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new group of friend is created, or when * a friend was added/removed from a group. */ - void forceSyncWithPeers(); + void forceSyncWithPeers() { NOT_IMPLEMENTED() ; } // derived from p3Service // @@ -92,6 +93,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub uint32_t getType(void *) const ; // set/update shared directories + void setSharedDirectories(const std::list& dirs); void getSharedDirectories(std::list& dirs); void updateShareFlags(const SharedDirInfo& info) ; @@ -116,11 +118,15 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void full_print(); protected: + + int filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const; + std::string makeRemoteFileName(const RsPeerId& pid) const; + // Derived from p3Config // virtual bool loadList(std::list& items); virtual bool saveList(bool &cleanup, std::list&); - virtual RsSerialiser *setupSerialiser() ; + virtual RsSerialiser *setupSerialiser() { return NULL;} void cleanup(); void tickRecv(); diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index f78784bee..44b361384 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -551,10 +551,9 @@ uint32_t ftServer::getType(void *ref, FileSearchFlags flags) { return mFileDatabase->getType(ref) ; } - /***************************************************************/ - /******************** Search Interface *************************/ - /***************************************************************/ - +/***************************************************************/ +/******************** Search Interface *************************/ +/***************************************************************/ int ftServer::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) { From 067d4231ae9eaf410f6c89c33e88e69728600e7a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 5 Aug 2016 22:00:25 +0200 Subject: [PATCH 017/174] fixed compilation --- .../src/file_sharing/directory_storage.h | 7 +- libretroshare/src/file_sharing/p3filelists.cc | 151 +++++++++++++++--- libretroshare/src/file_sharing/p3filelists.h | 19 ++- 3 files changed, 149 insertions(+), 28 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 28ba00401..a3a084612 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -121,8 +121,13 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} + RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname),mPeerId(pid) {} virtual ~RemoteDirectoryStorage() {} + + const RsPeerId& peerId() const { return mPeerId ; } + +private: + RsPeerId mPeerId; }; class LocalDirectoryStorage: public DirectoryStorage diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ffc3d401f..597c9292e 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -14,6 +14,10 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; +static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; +static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; +static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. + p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") { @@ -49,8 +53,8 @@ p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; - for(std::map::const_iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();++it) - delete it->second ; + for(uint32_t i=0;i friend_set ; - mServCtrl->getPeersConnected(getServiceInfo().mServiceType, friend_set); + { + std::list friend_lst ; - for(std::map::iterator it(mRemoteDirectories.begin());it!=mRemoteDirectories.end();) - if(friend_set.find(it->first) == friend_set.end()) + rsPeers->getFriendList(friend_lst); + + for(std::list::const_iterator it(friend_lst.begin());it!=friend_lst.end();++it) + friend_set.insert(*it) ; + } + + for(uint32_t i=0;ipeerId()) == friend_set.end()) { - P3FILELISTS_DEBUG() << " removing file list of non friend " << it->first << std::endl; + P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; - delete it->second ; - std::map::iterator tmp(it) ; - ++tmp ; - mRemoteDirectories.erase(it) ; - it=tmp ; + delete mRemoteDirectories[i]; + mRemoteDirectories[i] = NULL ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - } - else - ++it ; - // look through the list of friends, and add a directory storage when it's missing + friend_set.erase(mRemoteDirectories[i]->peerId()); + + mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); + mFriendIndexTab[i].clear(); + } + + // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. // for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) - if(mRemoteDirectories.find(*it) == mRemoteDirectories.end()) - { - P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; + { + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; - mRemoteDirectories[*it] = new RemoteDirectoryStorage(makeRemoteFileName(*it)); + uint32_t i; + for(i=0;i::const_iterator it = mFriendIndexMap.find(pid) ; + + if(it == mFriendIndexMap.end()) + { + // allocate a new index for that friend, and tell that we should save. + uint32_t found = 0 ; + for(uint32_t i=1;isecond; +} + +const RsPeerId& p3FileDatabase::getFriendFromIndex(uint32_t indx) const +{ + static const RsPeerId null_id ; + + if(indx >= mFriendIndexTab.size()) + return null_id ; + + if(mFriendIndexTab[indx].isNull()) + { + std::cerr << "(EE) null friend id requested from index " << indx << ": this is a bug, most likely" << std::endl; + return null_id ; + } + + return mFriendIndexTab[indx]; +} +bool p3FileDatabase::convertPointerToEntryIndex(void *p, EntryIndex& e, uint32_t& friend_index) +{ + // trust me, I can do this ;-) + + e = EntryIndex( *reinterpret_cast(&p) & ENTRY_INDEX_BIT_MASK ) ; + friend_index = (*reinterpret_cast(&p)) >> NB_ENTRY_INDEX_BITS ; + + return true; +} +bool p3FileDatabase::convertEntryIndexToPointer(EntryIndex& e, uint32_t fi, void *& p) +{ + // the pointer is formed the following way: + // + // [ 10 bits | 22 bits ] + // + // This means that the whoel software has the following build-in limitation: + // * 1024 friends + // * 4M shared files. + + uint32_t fe = (uint32_t)e ; + + if(fi >= (1<= (1<< NB_ENTRY_INDEX_BITS)) + { + std::cerr << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; + return false ; + } + + p = reinterpret_cast( (fi << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ; + + return true; +} int p3FileDatabase::RequestDirDetails(void *ref, DirDetails&, FileSearchFlags) const { - NOT_IMPLEMENTED(); + uint32_t fi; + EntryIndex e ; + + convertPointerToEntryIndex(ref,e,fi) ; +#warning code needed here + return 0; } -int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) +int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) const { NOT_IMPLEMENTED(); return 0; @@ -285,11 +386,13 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F EntryIndex indx = *res.begin() ; // no need to report dupicates mLocalSharedDirs->getFileInfo(indx,info) ; + + return true; } else { NOT_IMPLEMENTED(); - return 0; + return false; } } diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 0d631cdbd..d7c8e2250 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -84,7 +84,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void stopThreads() ; void startThreads() ; - int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); + int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; int RequestDirDetails(const std::string& path, DirDetails &details) const ; // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. @@ -144,13 +144,26 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // Directory storage hierarchies // - std::map mRemoteDirectories ; + // The remote one is the reference for the PeerId index below: + // RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid) + + std::vector mRemoteDirectories ; LocalDirectoryStorage *mLocalSharedDirs ; RemoteDirectoryUpdater *mRemoteDirWatcher ; LocalDirectoryUpdater *mLocalDirWatcher ; - // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. + // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc + + static bool convertEntryIndexToPointer(EntryIndex& e,uint32_t friend_index,void *& p); + static bool convertPointerToEntryIndex(void *p, EntryIndex& e, uint32_t& friend_index) ; + uint32_t getFriendIndex(const RsPeerId& pid); + const RsPeerId& getFriendFromIndex(uint32_t indx) const; + + std::map mFriendIndexMap ; + std::vector mFriendIndexTab; + + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // HashStorage *mHashCache ; From 43f4f5d2d9ed928b5a5b082ddcdf0f57d0cfaa15 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 5 Aug 2016 22:37:40 +0200 Subject: [PATCH 018/174] fixed threading issues in directory updater --- .../src/file_sharing/directory_updater.cc | 4 +++- libretroshare/src/file_sharing/hash_cache.cc | 16 ++++++++-------- libretroshare/src/file_sharing/p3filelists.cc | 12 ++++++++---- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index b4ebfb0f5..bccae26cf 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -5,7 +5,7 @@ #define DEBUG_LOCAL_DIR_UPDATER 1 -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! void RemoteDirectoryUpdater::tick() { @@ -27,6 +27,8 @@ void LocalDirectoryUpdater::data_tick() sweepSharedDirectories() ; mLastSweepTime = now; } + else + usleep(10*1000*1000); } void LocalDirectoryUpdater::sweepSharedDirectories() diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index eb86813b3..58ca5d80a 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -19,7 +19,15 @@ void HashStorage::data_tick() RS_STACK_MUTEX(mHashMtx) ; if(mFilesToHash.empty()) + { + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + + usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. return ; + } job = mFilesToHash.begin()->second ; @@ -33,14 +41,6 @@ void HashStorage::data_tick() mFilesToHash.erase(mFilesToHash.begin()) ; - if(mFilesToHash.empty()) - { - std::cerr << "Stopping hashing thread." << std::endl; - shutdown(); - mRunning = false ; - std::cerr << "done." << std::endl; - } - // store the result HashStorageInfo& info(mFiles[job.full_path]); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 597c9292e..302bc778d 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -31,8 +31,6 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; mRemoteDirWatcher = NULL; // not used yet - mLocalDirWatcher->start(); - mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } @@ -101,15 +99,21 @@ int p3FileDatabase::tick() void p3FileDatabase::startThreads() { std::cerr << "Starting hash cache thread..." ; - mHashCache->start(); + std::cerr << "Done." << std::endl; + + std::cerr << "Starting directory watcher thread..." ; + mLocalDirWatcher->start(); + std::cerr << "Done." << std::endl; } void p3FileDatabase::stopThreads() { std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; - mHashCache->fullstop(); + std::cerr << "Done." << std::endl; + std::cerr << "Stopping directory watcher thread..." ; std::cerr.flush() ; + mLocalDirWatcher->fullstop(); std::cerr << "Done." << std::endl; } From 2d1824b01d2b091a68802b4d850a1b3800df67f7 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 6 Aug 2016 19:04:54 +0200 Subject: [PATCH 019/174] fixed virtual names --- .../src/file_sharing/directory_storage.cc | 30 ++++++++++++++++++- .../src/file_sharing/directory_updater.cc | 9 ++++++ .../src/file_sharing/directory_updater.h | 4 ++- libretroshare/src/file_sharing/hash_cache.cc | 7 +++++ libretroshare/src/file_sharing/p3filelists.cc | 7 +++++ 5 files changed, 55 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 8663dddda..b91bbe4cb 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,5 +1,6 @@ #include #include "util/rsdir.h" +#include "util/rsstring.h" #include "directory_storage.h" /******************************************************************************************************************/ @@ -374,6 +375,7 @@ time_t DirectoryStorage::FileIterator::modtime() const { const InternalFil DirectoryStorage::DirectoryStorage(const std::string& local_file_name) : mFileName(local_file_name), mDirStorageMtx("Directory storage "+local_file_name) { + RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy = new InternalFileHierarchyStorage(); } @@ -446,8 +448,33 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list virtual_names ; // maps virtual to real name + std::list processed_list ; + for(std::list::const_iterator it(lst.begin());it!= lst.end();++it) - mLocalDirs[(*it).filename] = *it ; + { + int i=0; + std::string candidate_virtual_name = it->virtualname ; + + if(candidate_virtual_name.empty()) + candidate_virtual_name = RsDirUtil::getTopDir(it->filename); + + while(virtual_names.find(candidate_virtual_name) != virtual_names.end()) + rs_sprintf_append(candidate_virtual_name, "-%d", ++i); + + SharedDirInfo d(*it); + d.virtualname = candidate_virtual_name ; + processed_list.push_back(d) ; + + virtual_names.insert(candidate_virtual_name) ; + } + + mLocalDirs.clear(); + + for(std::list::const_iterator it(processed_list.begin());it!=processed_list.end();++it) + mLocalDirs[it->filename] = *it; } void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) { @@ -509,6 +536,7 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) { + RS_STACK_MUTEX(mDirStorageMtx) ; NOT_IMPLEMENTED() ; return false; } diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index bccae26cf..149937901 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -1,4 +1,5 @@ #include "util/folderiterator.h" +#include "rsserver/p3face.h" #include "directory_storage.h" #include "directory_updater.h" @@ -31,8 +32,15 @@ void LocalDirectoryUpdater::data_tick() usleep(10*1000*1000); } +void LocalDirectoryUpdater::forceUpdate() +{ + mLastSweepTime = 0; +} + void LocalDirectoryUpdater::sweepSharedDirectories() { + RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); + std::cerr << "LocalDirectoryUpdater::sweep()" << std::endl; // recursive update algorithm works that way: @@ -62,6 +70,7 @@ void LocalDirectoryUpdater::sweepSharedDirectories() std::cerr << " recursing into " << real_dir_it->filename << std::endl; recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; } + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 2d30793cd..d146a61b5 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -19,9 +19,11 @@ public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} - virtual void data_tick() ; + virtual void forceUpdate(); protected: + virtual void data_tick() ; + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); void sweepSharedDirectories(); diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 58ca5d80a..32401ca4b 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,4 +1,5 @@ #include "util/rsdir.h" +#include "rsserver/p3face.h" #include "hash_cache.h" #define HASHSTORAGE_DEBUG 1 @@ -25,6 +26,8 @@ void HashStorage::data_tick() mRunning = false ; std::cerr << "done." << std::endl; + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; + usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. return ; } @@ -33,6 +36,10 @@ void HashStorage::data_tick() std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); + std::string tmpout; + //rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", cnt+1, n_files, friendlyUnit(size).c_str(), int(size/double(total_size)*100.0), fe.name.c_str()) ; + + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, job.full_path) ; if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 302bc778d..299d90029 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -7,6 +7,8 @@ #include "retroshare/rsids.h" #include "retroshare/rspeers.h" +#include "rsserver/p3face.h" + #define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; @@ -36,14 +38,19 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) { + RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; + mLocalDirWatcher->forceUpdate(); } void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) { + RS_STACK_MUTEX(mFLSMtx) ; mLocalSharedDirs->getSharedDirectoryList(shared_dirs) ; } void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) { + RS_STACK_MUTEX(mFLSMtx) ; mLocalSharedDirs->updateShareFlags(info) ; } From 82c18bbb868093da319d51b4a1ab2346ed70a770 Mon Sep 17 00:00:00 2001 From: MrAlice Date: Tue, 9 Aug 2016 15:07:02 +0200 Subject: [PATCH 020/174] added load/save for file hash cache --- .../src/file_sharing/directory_storage.h | 9 - libretroshare/src/file_sharing/filelist_io.cc | 2 + libretroshare/src/file_sharing/filelist_io.h | 58 ++++++ libretroshare/src/file_sharing/hash_cache.cc | 175 +++++++++++++++++- libretroshare/src/file_sharing/hash_cache.h | 8 +- libretroshare/src/libretroshare.pro | 6 +- 6 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 libretroshare/src/file_sharing/filelist_io.cc create mode 100644 libretroshare/src/file_sharing/filelist_io.h diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index a3a084612..9e7872611 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -7,15 +7,6 @@ #include "retroshare/rsids.h" #include "retroshare/rsfiles.h" -static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; - -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; -static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; -static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; -static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; - #define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } class InternalFileHierarchyStorage ; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc new file mode 100644 index 000000000..e8e1207f7 --- /dev/null +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -0,0 +1,2 @@ +#include "filelist_io.h" + diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h new file mode 100644 index 000000000..d8f5f0d10 --- /dev/null +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +// This file implements load/save of various fields used for file lists and directory content. +// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. + +static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; + +static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; +static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; +static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; +static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; +static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; +static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; + +class FileListIO +{ +public: + template + static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val) + { + if(!checkSectionSize(buff,buff_size,offset,sizeof(T))) + return false; + + if(!writeSectionHeader(buff,buff_size,offset,section_tag,sizeof(T))) + return false; + + memcpy(&buff[offset],reinterpret_cast(&val),sizeof(T)) ; + offset += sizeof(T) ; + + return true; + } + +private: + static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S) + { + if(offset + S > buff_size) + { + buff = (unsigned char *)realloc(buff,offset + S) ; + buff_size = offset + S ; + + if(!buff) + return false ; + } + return true ; + } + + static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint8_t section_tag,uint32_t S) + { + buff[offset++] = section_tag ; + if(!write125Size(buff,offset,buff_size,S)) return false ; + + return true; + } +}; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 32401ca4b..c86e6e8cc 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,13 +1,19 @@ #include "util/rsdir.h" #include "rsserver/p3face.h" +#include "pqi/authssl.h" #include "hash_cache.h" +#include "filelist_io.h" #define HASHSTORAGE_DEBUG 1 +static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000; +static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000; + HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { mRunning = false ; + load() ; } void HashStorage::data_tick() @@ -28,9 +34,15 @@ void HashStorage::data_tick() RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; - usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + usleep(mInactivitySleepTime); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + mInactivitySleepTime *= 2; + + if(mInactivitySleepTime > MAX_INACTIVITY_SLEEP_TIME) + mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; + return ; } + mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; job = mFilesToHash.begin()->second ; @@ -145,3 +157,164 @@ void HashStorage::clean() #endif } +void HashStorage::load() +{ + uint64_t file_size ; + + if(!RsDirUtil::checkFile( mFilePath,file_size,false ) ) + { + std::cerr << "Encrypted hash cache file not present." << std::endl; + return ; + } + + // read the binary stream into memory. + // + void *buffer = rs_malloc(file_size) ; + + if(buffer == NULL) + return ; + + FILE *F = fopen( mFilePath.c_str(),"rb") ; + if (!F) + { + std::cerr << "Cannot open file for reading encrypted file cache, filename " << mFilePath << std::endl; + free(buffer); + return; + } + if(fread(buffer,1,file_size,F) != file_size) + { + std::cerr << "Cannot read from file " + mFilePath << ": something's wrong." << std::endl; + free(buffer) ; + fclose(F) ; + return ; + } + fclose(F) ; + + // now decrypt + void *decrypted_data =NULL; + int decrypted_data_size =0; + + if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) + { + std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; + free(buffer) ; + return ; + } + free(buffer) ; + +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Loading HashCache from file " << mFilePath << std::endl ; + int n=0 ; +#endif + unsigned char *data = (unsigned char*)decrypted_data ; + uint32_t offset = 0 ; + HashStorageInfo info ; + + while(offset < decrypted_data_size) + if(readHashStorageInfo(data,decrypted_data_size,offset,info)) + { +#ifdef HASHSTORAGE_DEBUG + std::cerr << info << std::endl; + ++n ; +#endif + mFiles[info.filename] = info ; + } + + free(decrypted_data) ; +#ifdef HASHSTORAGE_DEBUG + std::cerr << n << " entries loaded." << std::endl ; +#endif +} + +void HashStorage::save() +{ +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ; +#endif + + unsigned char *data = NULL ; + uint32_t offset = 0 ; + uint32_t total_size = 0; + + for(std::map::const_iterator it(mFiles.begin());it!=mFiles.end();++it) + writeHashStorageInfo(it->second,data,total_size,offset) ; + + void *encryptedData = NULL ; + int encDataLen = 0 ; + + if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,offset, AuthSSL::getAuthSSL()->OwnId())) + { + std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; + return; + } + + FILE *F = fopen( (mFilePath+".tmp").c_str(),"wb" ) ; + + if(!F) + { + std::cerr << "Cannot open encrypted file cache for writing: " << mFilePath+".tmp" << std::endl; + goto save_free; + } + if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) + { + std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; + fclose(F) ; + goto save_free; + } + + fclose(F) ; + + RsDirUtil::renameFile(mFilePath+".tmp",mFilePath) ; +#ifdef FIM_DEBUG + std::cerr << "done." << std::endl ; +#endif + +save_free: + free(encryptedData); +} + +bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const +{ + unsigned char *section_data = NULL ; + uint32_t section_size = 0; + + if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size)) + return false; + + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ; + + free(section_data); + return true; +} + +bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const +{ + unsigned char *section_data = NULL ; + uint32_t section_offset = 0 ; + uint32_t section_size = 0; + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ; + + // now write the whole string into a single section in the file + + if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ; + + free(section_data) ; + + return true; +} + +std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& info) +{ + return o << info.hash << " " << info.size << " " << info.filename ; +} + + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 2d10c09ca..cf6b12a9f 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -34,7 +34,7 @@ public: struct HashStorageInfo { - std::string filename ; + std::string filename ; // full path of the file uint64_t size ; uint32_t time_stamp ; // last time the hash was tested/requested uint32_t modf_stamp ; @@ -50,12 +50,17 @@ public: // Functions called by the thread virtual void data_tick() ; + + friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ; private: void clean() ; void save() ; void load() ; + bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const; + bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const; + // Local configuration and storage uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries @@ -79,5 +84,6 @@ private: RsMutex mHashMtx ; bool mRunning; + uint32_t mInactivitySleepTime = 50*1000 ; }; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index b7a113fd7..70994b8c7 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -38,12 +38,14 @@ CONFIG += file_lists file_lists { HEADERS *= file_sharing/p3filelists.h \ file_sharing/hash_cache.h \ - file_sharing/directory_storage.h \ + file_sharing/filelist_io.h \ + file_sharing/directory_storage.h \ file_sharing/directory_updater.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ - file_sharing/directory_storage.cc \ + file_sharing/filelist_io.cc \ + file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc } From d8dfce4fff4e5a5e14d867eeb4dd85ed61cef509 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 11 Aug 2016 14:07:45 +0200 Subject: [PATCH 021/174] added parent row and friend status to shared dirs --- .../src/file_sharing/directory_storage.cc | 154 ++++++++++++++++-- .../src/file_sharing/directory_storage.h | 32 ++-- libretroshare/src/file_sharing/p3filelists.cc | 109 ++++++++++--- libretroshare/src/file_sharing/p3filelists.h | 8 +- libretroshare/src/retroshare/rstypes.h | 29 ++-- 5 files changed, 267 insertions(+), 65 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index b91bbe4cb..5230ec2fd 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -30,6 +30,9 @@ class InternalFileHierarchyStorage virtual ~FileStorageNode() {} virtual uint32_t type() const =0; + + DirectoryStorage::EntryIndex parent_index; + uint32_t row ; }; class FileEntry: public FileStorageNode { @@ -48,23 +51,35 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name,DirectoryStorage::EntryIndex parent) : dir_name(name),parent_index(parent) {} + DirEntry(const std::string& name) : dir_name(name) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } // local stuff std::string dir_name ; - DirectoryStorage::EntryIndex parent_index; + std::string dir_parent_path ; std::vector subdirs ; std::vector subfiles ; + + time_t most_recent_time; }; // class stuff InternalFileHierarchyStorage() : mRoot(0) { - mNodes.push_back(new DirEntry("",0)) ; + mNodes.push_back(new DirEntry("")) ; + mNodes.back()->row=0; + mNodes.back()->parent_index=-1; + } + + int parentRow(DirectoryStorage::EntryIndex e) + { + if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) + return -1 ; + + return mNodes[mNodes[e]->parent_index]->row; } // high level modification routines @@ -95,7 +110,9 @@ class InternalFileHierarchyStorage for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) { d.subdirs.push_back(mNodes.size()) ; - mNodes.push_back(new DirEntry(*it,indx)); + mNodes.push_back(new DirEntry(*it)); + mNodes.back()->row = mNodes.size()-1; + mNodes.back()->parent_index = indx; } return true; @@ -127,7 +144,7 @@ class InternalFileHierarchyStorage if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL) return nodeAccessError("checkIndex(): Node does not exist") ; - if(mNodes[indx]->type() != type) + if(! mNodes[indx]->type() & type) return nodeAccessError("checkIndex(): Node is of wrong type") ; return true; @@ -172,6 +189,8 @@ class InternalFileHierarchyStorage { d.subfiles.push_back(mNodes.size()) ; mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); + mNodes.back()->row = mNodes.size()-1; + mNodes.back()->parent_index = indx; } return true; } @@ -208,6 +227,7 @@ class InternalFileHierarchyStorage // are not referenced. friend class DirectoryStorage ; // only class that can use this. + friend class LocalDirectoryStorage ; // only class that can use this. // Low level stuff. Should normally not be used externally. @@ -225,6 +245,16 @@ class InternalFileHierarchyStorage return static_cast(mNodes[indx]) ; } + uint32_t getType(DirectoryStorage::EntryIndex indx) const + { + if(checkIndex(indx,FileStorageNode::TYPE_FILE)) + return FileStorageNode::TYPE_FILE; + else if(checkIndex(indx,FileStorageNode::TYPE_DIR)) + return FileStorageNode::TYPE_DIR; + else + return FileStorageNode::TYPE_UNKNOWN; + } + DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) { if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) @@ -368,12 +398,14 @@ uint64_t DirectoryStorage::FileIterator::size() const { const InternalFil std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); } time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; } +std::string DirectoryStorage::DirIterator::name() const { const InternalFileHierarchyStorage::DirEntry *d = mStorage->getDirEntry(**this) ; return d?(d->dir_name):std::string(); } + /******************************************************************************************************************/ /* Directory Storage */ /******************************************************************************************************************/ -DirectoryStorage::DirectoryStorage(const std::string& local_file_name) - : mFileName(local_file_name), mDirStorageMtx("Directory storage "+local_file_name) +DirectoryStorage::DirectoryStorage(const std::string& local_file_name, const RsPeerId &pid) + : mFileName(local_file_name),mPeerId(pid), mDirStorageMtx("Directory storage "+local_file_name) { RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy = new InternalFileHierarchyStorage(); @@ -383,6 +415,25 @@ DirectoryStorage::EntryIndex DirectoryStorage::root() const { return EntryIndex(0) ; } +int DirectoryStorage::parentRow(EntryIndex e) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + return mFileHierarchy->parentRow(e) ; +} + +uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + switch(mFileHierarchy->getType(indx)) + { + case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR: return DIR_TYPE_DIR ; + case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE: return DIR_TYPE_FILE ; + default: + return DIR_TYPE_UNKNOWN; + } +} bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) { @@ -440,6 +491,15 @@ void DirectoryStorage::print() std::cerr << "LocalDirectoryStorage:" << std::endl; mFileHierarchy->print(); } + +bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; +#warning todo + + return true; +} + /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ @@ -534,10 +594,80 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const return cit->second.filename; } -bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) { - RS_STACK_MUTEX(mDirStorageMtx) ; - NOT_IMPLEMENTED() ; - return false; -} + d.children.clear() ; + time_t now = time(NULL) ; + const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + + if (dir_entry != NULL) /* has children --- fill */ + { +#ifdef FI_DEBUG + std::cerr << "FileIndex::extractData() ref=dir" << std::endl; +#endif + /* extract all the entries */ + + for(DirectoryStorage::DirIterator it(this,indx);it;++it) + { + DirStub stub; + stub.type = DIR_TYPE_DIR; + stub.name = it.name(); + stub.ref = (void*)(intptr_t)*it; // this is updated by the caller, who knows which friend we're dealing with + + d.children.push_back(stub); + } + + for(DirectoryStorage::FileIterator it(this,indx);it;++it) + { + DirStub stub; + stub.type = DIR_TYPE_FILE; + stub.name = it.name(); + stub.ref = (void*)(intptr_t)*it; + + d.children.push_back(stub); + } + + if(dir_entry->parent_index == 0) + d.type = DIR_TYPE_PERSON ; + else + d.type = DIR_TYPE_DIR; + + d.hash.clear() ; + d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size(); + d.min_age = now - dir_entry->most_recent_time ; + d.name = dir_entry->dir_name; + d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; + d.parent = (void*)(intptr_t)dir_entry->parent_index ; + } + else + { + const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; +#ifdef FI_DEBUG + std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; +#endif + d.type = DIR_TYPE_FILE; + d.count = file_entry->file_size; + d.min_age = now - file_entry->file_modtime ; + d.name = file_entry->file_name; + d.hash = file_entry->file_hash; + d.age = now - file_entry->file_modtime; + d.parent = (void*)(intptr_t)file_entry->parent_index ; + + const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index); + + if(parent_dir_entry != NULL) + d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ; + else + d.path = "" ; + } + +#ifdef FI_DEBUG + std::cerr << "FileIndexStore::extractData() name: " << file->name << std::endl; +#endif + d.flags.clear() ; + + /* find parent pointer, and row */ + + return true; +} diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 9e7872611..0bdbb9dd6 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -14,10 +14,10 @@ class InternalFileHierarchyStorage ; class DirectoryStorage { public: - DirectoryStorage(const std::string& local_file_name) ; + DirectoryStorage(const std::string& local_file_name,const RsPeerId& pid) ; virtual ~DirectoryStorage() {} - typedef uint32_t EntryIndex ; + typedef int32_t EntryIndex ; static const EntryIndex NO_INDEX = 0xffffffff; void save() const ; @@ -27,6 +27,8 @@ class DirectoryStorage virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } void getFileDetails(EntryIndex i) ; + uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. + virtual bool extractData(const EntryIndex& indx,DirDetails& d) =0; // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. // It gives access to sub-files and sub-directories below. @@ -44,7 +46,7 @@ class DirectoryStorage // info about the directory that is pointed by the iterator - const std::string& name() const ; + std::string name() const ; private: EntryIndex mParentIndex ; // index of the parent dir. uint32_t mDirTabIndex ; // index in the vector of subdirs. @@ -83,6 +85,8 @@ class DirectoryStorage }; EntryIndex root() const ; // returns the index of the root directory entry. + const RsPeerId& peerId() const { return mPeerId ; } + int parentRow(EntryIndex e) const ; bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; @@ -103,28 +107,31 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. - InternalFileHierarchyStorage *mFileHierarchy ; std::string mFileName; + RsPeerId mPeerId; + protected: - RsMutex mDirStorageMtx ; + mutable RsMutex mDirStorageMtx ; + + InternalFileHierarchyStorage *mFileHierarchy ; }; class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname),mPeerId(pid) {} + RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} virtual ~RemoteDirectoryStorage() {} - const RsPeerId& peerId() const { return mPeerId ; } - -private: - RsPeerId mPeerId; + virtual bool extractData(const EntryIndex& indx,DirDetails& d) + { + NOT_IMPLEMENTED(); + } }; class LocalDirectoryStorage: public DirectoryStorage { public: - LocalDirectoryStorage(const std::string& fname) : DirectoryStorage(fname) {} + LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(fname,own_id) {} virtual ~LocalDirectoryStorage() {} void setSharedDirectoryList(const std::list& lst) ; @@ -140,6 +147,9 @@ public: * \return false if the file does not exist, or is a directory,... */ bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ; + + virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; + private: std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 299d90029..1c6f50802 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -25,9 +25,11 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) { // loads existing indexes for friends. Some might be already present here. // - mRemoteDirectories.clear() ; // we should load them! + mDirectories.clear() ; // we should load them! + + mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mpeers->getOwnId()) ; + mDirectories.push_back(mLocalSharedDirs) ; - mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin") ; mHashCache = new HashStorage("hash_cache.bin") ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; @@ -58,10 +60,10 @@ p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; - for(uint32_t i=0;ipeerId()) == friend_set.end()) + for(uint32_t i=1;ipeerId()) == friend_set.end()) { - P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; + P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; - delete mRemoteDirectories[i]; - mRemoteDirectories[i] = NULL ; + delete mDirectories[i]; + mDirectories[i] = NULL ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - friend_set.erase(mRemoteDirectories[i]->peerId()); + friend_set.erase(mDirectories[i]->peerId()); - mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); + mFriendIndexMap.erase(mDirectories[i]->peerId()); mFriendIndexTab[i].clear(); } @@ -195,12 +197,12 @@ void p3FileDatabase::cleanup() P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; uint32_t i; - for(i=0;iroot(),i,p); + + DirStub stub; + stub.type = DIR_TYPE_PERSON; + stub.name = mDirectories[i]->peerId().toStdString(); + stub.ref = p; + d.children.push_back(stub); + } + d.count = mDirectories.size(); + + d.parent = NULL; + d.prow = -1; + d.ref = NULL; + d.type = DIR_TYPE_ROOT; + d.name = "root"; + d.hash.clear() ; + d.path = "root"; + d.age = 0; + d.flags.clear() ; + d.min_age = 0 ; + + return true ; + } + uint32_t fi; EntryIndex e ; - convertPointerToEntryIndex(ref,e,fi) ; -#warning code needed here + convertPointerToEntryIndex(ref,e,fi); - return 0; + // Case where the index is the top of a single person. Can be us, or a friend. + + bool res = mDirectories[fi]->extractData(e,d) ; + + // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert + // a pointer into an int. + + convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ; + convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; + + for(std::list::iterator it(d.children.begin());it!=d.children.end();++it) + convertEntryIndexToPointer((intptr_t)it->ref,fi,it->ref); + + d.prow = mDirectories[fi]->parentRow(e) ; + d.id = mDirectories[fi]->peerId(); } + int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) const { NOT_IMPLEMENTED(); @@ -317,9 +367,20 @@ int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &detai } uint32_t p3FileDatabase::getType(void *ref) const { - NOT_IMPLEMENTED(); - return 0; + EntryIndex e ; + uint32_t fi; + + if(ref == NULL) + return DIR_TYPE_ROOT ; + + convertPointerToEntryIndex(ref,e,fi); + + if(e == 0) + return DIR_TYPE_PERSON ; + + return mDirectories[fi]->getEntryType(e) ; } + void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed { NOT_IMPLEMENTED(); diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index d7c8e2250..cd090af61 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -147,16 +147,16 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // The remote one is the reference for the PeerId index below: // RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid) - std::vector mRemoteDirectories ; + std::vector mDirectories ; // mDirectories[0]=mLocalSharedDirs LocalDirectoryStorage *mLocalSharedDirs ; - RemoteDirectoryUpdater *mRemoteDirWatcher ; + RemoteDirectoryUpdater *mRemoteDirWatcher ; // not used yet. LocalDirectoryUpdater *mLocalDirWatcher ; // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc - static bool convertEntryIndexToPointer(EntryIndex& e,uint32_t friend_index,void *& p); - static bool convertPointerToEntryIndex(void *p, EntryIndex& e, uint32_t& friend_index) ; + static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p); + static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ; uint32_t getFriendIndex(const RsPeerId& pid); const RsPeerId& getFriendFromIndex(uint32_t indx) const; diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 37975d605..d30b156eb 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -138,6 +138,7 @@ public: /********************** For FileCache Interface *****************/ +#define DIR_TYPE_UNKNOWN 0x00 #define DIR_TYPE_ROOT 0x01 #define DIR_TYPE_PERSON 0x02 #define DIR_TYPE_DIR 0x04 @@ -159,7 +160,7 @@ const FileStorageFlags DIR_FLAGS_BROWSABLE_OTHERS ( 0x0100 ); // one should const FileStorageFlags DIR_FLAGS_NETWORK_WIDE_GROUPS ( 0x0200 ); const FileStorageFlags DIR_FLAGS_BROWSABLE_GROUPS ( 0x0400 ); const FileStorageFlags DIR_FLAGS_PERMISSIONS_MASK ( DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_OTHERS - | DIR_FLAGS_NETWORK_WIDE_GROUPS | DIR_FLAGS_BROWSABLE_GROUPS ); + | DIR_FLAGS_NETWORK_WIDE_GROUPS | DIR_FLAGS_BROWSABLE_GROUPS ); const FileStorageFlags DIR_FLAGS_LOCAL ( 0x1000 ); const FileStorageFlags DIR_FLAGS_REMOTE ( 0x2000 ); @@ -227,23 +228,23 @@ class DirStub class DirDetails { - public: - void *parent; - int prow; /* parent row */ +public: + void *parent; + int prow; /* parent row */ - void *ref; - uint8_t type; + void *ref; + uint8_t type; RsPeerId id; - std::string name; + std::string name; RsFileHash hash; - std::string path; - uint64_t count; - uint32_t age; - FileStorageFlags flags; - uint32_t min_age ; // minimum age of files in this subtree + std::string path; + uint64_t count; + uint32_t age; + FileStorageFlags flags; + uint32_t min_age ; // minimum age of files in this subtree - std::list children; - std::list parent_groups; // parent groups for the shared directory + std::list children; + std::list parent_groups; // parent groups for the shared directory }; class FileDetail From 733ad438fbfc9cc8a04e75174ab0c190e8532a40 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 12 Aug 2016 15:20:23 +0200 Subject: [PATCH 022/174] added IO for hash cache --- libretroshare/src/file_sharing/filelist_io.cc | 99 +++++++++++++++++++ libretroshare/src/file_sharing/filelist_io.h | 45 ++++++++- libretroshare/src/file_sharing/hash_cache.cc | 5 +- 3 files changed, 144 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index e8e1207f7..3cd153ae8 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,2 +1,101 @@ #include "filelist_io.h" +bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) +{ + if(!checkSectionSize(buff,buff_size,offset,size)) + return false; + + if(!writeSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + return false; + + memcpy(&buff[offset],val,size) ; + offset += size ; + + return true; +} + +bool FileListIO::readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) +{ + if(!readSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + return false; + + val = (unsigned char *)rs_malloc(size) ; + + if(!val) + return false; + + memcpy(val,&buff[offset],size); + offset += size ; + + return true ; +} + +bool FileListIO::write125Size(unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t S) +{ + if(S < 192) + { + if(offset+1 > data_size) + return false; + + data[offset++] = (uint8_t)S ; + return true; + } + else if(S < 8384) + { + if(offset+2 > data_size) + return false; + + data[offset++] = (uint8_t)((S >> 8) + 192) ; + data[offset++] = (uint8_t)((S & 255) - 192) ; + + return true; + } + else + { + if(offset+5 > data_size) + return false; + + data[offset++] = 0xff ; + data[offset++] = (uint8_t)((S >> 24) & 255) ; + data[offset++] = (uint8_t)((S >> 16) & 255) ; + data[offset++] = (uint8_t)((S >> 8) & 255) ; + data[offset++] = (uint8_t)((S ) & 255) ; + + return true ; + } +} + +bool FileListIO::read125Size(const unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t& S) +{ + if(offset + 1 >= data_size) return false; + + uint8_t b1 = data[offset++] ; + + if(b1 < 192) + { + S = b1; + return true ; + } + if(offset + 1 >= data_size) return false; + + uint8_t b2 = data[offset++] ; + + if(b1 < 224) + { + S = ((b1-192) << 8) + b2 + 192 ; + return true; + } + + if(b1 != 0xff) + return false; + + if(offset + 3 >= data_size) return false; + + uint8_t b3 = data[offset++]; + uint8_t b4 = data[offset++]; + uint8_t b5 = data[offset++]; + + S = (b2 << 24) | (b3 << 16) | (b4 << 8) | b5 ; + return true; +} + diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index d8f5f0d10..9c5818025 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -1,6 +1,10 @@ #pragma once +#include #include +#include + +#include "util/rsmemory.h" // This file implements load/save of various fields used for file lists and directory content. // WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. @@ -15,6 +19,7 @@ static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; +static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; class FileListIO { @@ -34,7 +39,30 @@ public: return true; } + template + static bool readField(const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,T& val) + { + uint32_t section_size ; + + if(!readSectionHeader(buff,buff_size,offset,check_section_tag,section_size)) + return false; + + if(section_size != sizeof(T)) + return false ; + + memcpy(reinterpret_cast(&val),&buff[offset],sizeof(T)) ; + offset += sizeof(T) ; + + return true; + } + + static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ; + static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ; + private: + static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ; + static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ; + static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S) { if(offset + S > buff_size) @@ -48,11 +76,24 @@ private: return true ; } - static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint8_t section_tag,uint32_t S) + static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,uint32_t S) { buff[offset++] = section_tag ; - if(!write125Size(buff,offset,buff_size,S)) return false ; + if(!write125Size(buff,buff_size,offset,S)) return false ; return true; } + + static bool readSectionHeader(const unsigned char *& buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,uint32_t& S) + { + if(offset + 1 > buff_size) + return false ; + + uint8_t section_tag = buff[offset++] ; + + if(section_tag != check_section_tag) + return false; + + return read125Size(buff,buff_size,offset,S) ; + } }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index c86e6e8cc..743e959dd 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -237,7 +237,7 @@ void HashStorage::save() uint32_t total_size = 0; for(std::map::const_iterator it(mFiles.begin());it!=mFiles.end();++it) - writeHashStorageInfo(it->second,data,total_size,offset) ; + writeHashStorageInfo(data,total_size,offset,it->second) ; void *encryptedData = NULL ; int encDataLen = 0 ; @@ -277,6 +277,7 @@ bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_s { unsigned char *section_data = NULL ; uint32_t section_size = 0; + uint32_t section_offset = 0; if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size)) return false; @@ -316,5 +317,3 @@ std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& inf { return o << info.hash << " " << info.size << " " << info.filename ; } - - From df14b41ba6663dc472cfa0848fa481743c66d032 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 12 Aug 2016 19:30:19 +0200 Subject: [PATCH 023/174] fixed sleeping when hashing strategy --- libretroshare/src/file_sharing/hash_cache.cc | 44 ++++++++++++++----- libretroshare/src/file_sharing/p3filelists.cc | 11 +++++ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 743e959dd..60eec0cd9 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -23,22 +23,41 @@ void HashStorage::data_tick() uint64_t size ; { - RS_STACK_MUTEX(mHashMtx) ; + bool empty ; + uint32_t st ; - if(mFilesToHash.empty()) { - std::cerr << "Stopping hashing thread." << std::endl; - shutdown(); - mRunning = false ; - std::cerr << "done." << std::endl; + RS_STACK_MUTEX(mHashMtx) ; - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; + empty = mFilesToHash.empty(); + st = mInactivitySleepTime ; + } - usleep(mInactivitySleepTime); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. - mInactivitySleepTime *= 2; + // sleep off mutex! + if(empty) + { + std::cerr << "nothing to hash. Sleeping for " << st << " us" << std::endl; - if(mInactivitySleepTime > MAX_INACTIVITY_SLEEP_TIME) - mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; + usleep(st); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + + if(st > MAX_INACTIVITY_SLEEP_TIME) + { + RS_STACK_MUTEX(mHashMtx) ; + + mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; + + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; + } + else + { + RS_STACK_MUTEX(mHashMtx) ; + mInactivitySleepTime = 2*st ; + } return ; } @@ -279,6 +298,9 @@ bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_s uint32_t section_size = 0; uint32_t section_offset = 0; + // This way, the entire section is either read or skipped. That avoids the risk of being stuck somewhere in the middle + // of a section because of some unknown field, etc. + if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size)) return false; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 1c6f50802..0db5ad01a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -102,6 +102,15 @@ int p3FileDatabase::tick() // - cleanup(); + static time_t last_print_time = 0; + time_t now = time(NULL) ; + + if(last_print_time + 60 < now) + { + mLocalSharedDirs->print(); + last_print_time = now ; + } + return 0; } @@ -353,6 +362,8 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.prow = mDirectories[fi]->parentRow(e) ; d.id = mDirectories[fi]->peerId(); + + return true; } int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) const From 2668b6da1cd90e37cb2a8f2fb779afdecf00de1e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 13 Aug 2016 10:15:02 +0200 Subject: [PATCH 024/174] added p3fileDatabase as a service in rsinit --- .../src/file_sharing/directory_storage.cc | 5 ++++- libretroshare/src/file_sharing/directory_storage.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 6 +++++- libretroshare/src/ft/ftserver.cc | 10 +++++----- libretroshare/src/ft/ftserver.h | 4 +++- libretroshare/src/rsserver/rsinit.cc | 14 +++++++++----- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 5230ec2fd..e112d6651 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -100,7 +100,10 @@ class InternalFileHierarchyStorage for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) - removeDirectory(d.subdirs[i]) ; + { + if( !removeDirectory(d.subdirs[i])) + i++ ; + } else { should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0bdbb9dd6..a96790684 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -125,6 +125,7 @@ public: virtual bool extractData(const EntryIndex& indx,DirDetails& d) { NOT_IMPLEMENTED(); + return false; } }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0db5ad01a..2d27c9788 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -105,7 +105,7 @@ int p3FileDatabase::tick() static time_t last_print_time = 0; time_t now = time(NULL) ; - if(last_print_time + 60 < now) + if(last_print_time + 20 < now) { mLocalSharedDirs->print(); last_print_time = now ; @@ -212,6 +212,10 @@ void p3FileDatabase::cleanup() mDirectories.push_back(NULL) ; mDirectories[i] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + + if(mFriendIndexTab.size() <= i) + mFriendIndexTab.resize(i+1) ; + mFriendIndexTab[i] = *it ; mFriendIndexMap[*it] = i; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 44b361384..8a0424a5c 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -148,19 +148,19 @@ void ftServer::SetupFtServer() mFtController->setPartialsDirectory(tmppath); mFtController->setDownloadDirectory(tmppath); - - /* Make Cache Source/Store */ - mFileDatabase = new p3FileDatabase(mServiceCtrl) ; - /* complete search setup */ mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA); - mFtSearch->addSearchMode(mFileDatabase, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE); mServiceCtrl->registerServiceMonitor(mFtController, getServiceInfo().mServiceType); return; } +void ftServer::connectToFileDatabase(p3FileDatabase *fdb) +{ + mFileDatabase = fdb ; + mFtSearch->addSearchMode(fdb, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE); +} void ftServer::connectToTurtleRouter(p3turtle *fts) { mTurtleRouter = fts ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 1b66db710..4e5ae3a61 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -53,6 +53,7 @@ #include "pqi/p3cfgmgr.h" class p3ConnectMgr; +class p3FileDatabase; class CacheStrapper; class CacheTransfer; @@ -94,7 +95,8 @@ public: /* Final Setup (once everything is assigned) */ void SetupFtServer() ; - virtual void connectToTurtleRouter(p3turtle *p) ; + virtual void connectToTurtleRouter(p3turtle *p) ; + virtual void connectToFileDatabase(p3FileDatabase *b); // Implements RsTurtleClientService // diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 40c3417ff..0f484fae3 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -56,7 +56,8 @@ #include #include -#include +#include "gxstunnel/p3gxstunnel.h" +#include "file_sharing/p3filelists.h" #define ENABLE_GROUTER @@ -1481,22 +1482,25 @@ int RsServer::StartupRetroShare() pqih->addService(gr,true) ; #endif - p3turtle *tr = new p3turtle(serviceCtrl,mLinkMgr) ; + p3FileDatabase *fdb = new p3FileDatabase(serviceCtrl) ; + p3turtle *tr = new p3turtle(serviceCtrl,mLinkMgr) ; rsTurtle = tr ; pqih -> addService(tr,true); - pqih -> addService(ftserver,true); + pqih -> addService(fdb,true); + pqih -> addService(ftserver,true); mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ; mGxsTunnels->connectToTurtleRouter(tr) ; rsGxsTunnel = mGxsTunnels; - + rsDisc = mDisc; rsMsgs = new p3Msgs(msgSrv, chatSrv); // connect components to turtle router. ftserver->connectToTurtleRouter(tr) ; - chatSrv->connectToGxsTunnelService(mGxsTunnels) ; + ftserver->connectToFileDatabase(fdb) ; + chatSrv->connectToGxsTunnelService(mGxsTunnels) ; gr->connectToTurtleRouter(tr) ; #ifdef ENABLE_GROUTER msgSrv->connectToGlobalRouter(gr) ; From ca8f95c7f74b16c208c25f2f1a25037987c69978 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 16 Aug 2016 13:46:55 +0200 Subject: [PATCH 025/174] fixed a few bugs in internal directory storage structure --- .../src/file_sharing/directory_storage.cc | 30 +++++++++++++++- .../src/file_sharing/directory_updater.cc | 6 ---- libretroshare/src/file_sharing/hash_cache.cc | 10 ++++-- libretroshare/src/file_sharing/hash_cache.h | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 35 ++++++++++++------- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e112d6651..7b913b4c3 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -101,17 +101,23 @@ class InternalFileHierarchyStorage for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) { + std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + if( !removeDirectory(d.subdirs[i])) i++ ; } else { + std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; ++i; } for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) { + std::cerr << "[directory storage] adding new subdirectory " << *it << " at index " << mNodes.size() << std::endl; + d.subdirs.push_back(mNodes.size()) ; mNodes.push_back(new DirEntry(*it)); mNodes.back()->row = mNodes.size()-1; @@ -170,9 +176,13 @@ class InternalFileHierarchyStorage if(it == subfiles.end()) // file does not exist anymore => delete { + std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; + + delete mNodes[d.subfiles[i]] ; + mNodes[d.subfiles[i]] = NULL ; + d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; d.subfiles.pop_back(); - continue; } @@ -190,6 +200,8 @@ class InternalFileHierarchyStorage for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) { + std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; + d.subfiles.push_back(mNodes.size()) ; mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); mNodes.back()->row = mNodes.size()-1; @@ -200,7 +212,12 @@ class InternalFileHierarchyStorage bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) { if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; return false; + } + + std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; static_cast(mNodes[file_index])->file_hash = hash ; return true; @@ -208,10 +225,15 @@ class InternalFileHierarchyStorage bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) { if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; return false; + } FileEntry& fe(*static_cast(mNodes[file_index])) ; + std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; + fe.file_hash = hash; fe.file_size = size; fe.file_modtime = modf_time; @@ -321,6 +343,12 @@ private: void recursPrint(int depth,DirectoryStorage::EntryIndex node) const { std::string indent(2*depth,' '); + + if(mNodes[node] == NULL) + { + std::cerr << "EMPTY NODE !!" << std::endl; + return ; + } DirEntry& d(*static_cast(mNodes[node])); std::cerr << indent << "dir:" << d.dir_name << std::endl; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 149937901..e0f0438a0 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -83,12 +83,6 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p librs::util::FolderIterator dirIt(cumulated_path); - if(!dirIt.isValid()) - { - mSharedDirectories->removeDirectory(indx) ; // this is a complex operation since it needs to *update* it so that it is kept consistent. - return ; - } - // collect subdirs and subfiles std::map subfiles ; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 60eec0cd9..1e7cf0fe7 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -12,7 +12,9 @@ static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000; HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { + mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; mRunning = false ; + load() ; } @@ -63,7 +65,11 @@ void HashStorage::data_tick() } mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; - job = mFilesToHash.begin()->second ; + { + RS_STACK_MUTEX(mHashMtx) ; + job = mFilesToHash.begin()->second ; + mFilesToHash.erase(mFilesToHash.begin()) ; + } std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); @@ -77,8 +83,6 @@ void HashStorage::data_tick() else std::cerr << "done."<< std::endl; - mFilesToHash.erase(mFilesToHash.begin()) ; - // store the result HashStorageInfo& info(mFiles[job.full_path]); diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index cf6b12a9f..6b2dbf8f3 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -84,6 +84,6 @@ private: RsMutex mHashMtx ; bool mRunning; - uint32_t mInactivitySleepTime = 50*1000 ; + uint32_t mInactivitySleepTime ; }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 2d27c9788..09cf31e53 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -203,21 +203,16 @@ void p3FileDatabase::cleanup() // for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) { - P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << std::endl; + // Check if a remote directory exists for that friend, possibly creating the index. - uint32_t i; - for(i=0;i friend_index && mDirectories[friend_index] != NULL) + continue ; - mDirectories[i] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; - if(mFriendIndexTab.size() <= i) - mFriendIndexTab.resize(i+1) ; - - mFriendIndexTab[i] = *it ; - mFriendIndexMap[*it] = i; + mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } @@ -249,19 +244,33 @@ uint32_t p3FileDatabase::getFriendIndex(const RsPeerId& pid) if(!found) { - std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than 1024 friends??" << std::endl; - return 1024 ; + found = mFriendIndexTab.size(); + mFriendIndexTab.push_back(pid); + } + + if(mFriendIndexTab.size() >= (1 << NB_FRIEND_INDEX_BITS) ) + { + std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than " << (1<second) + mDirectories.resize(it->second+1,NULL) ; + return it->second; + } } const RsPeerId& p3FileDatabase::getFriendFromIndex(uint32_t indx) const From ac242ce58bde9bfe22eb49b323968ff771a5fd5d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 16 Aug 2016 23:44:48 +0200 Subject: [PATCH 026/174] fixed display of own files --- .../src/file_sharing/directory_storage.cc | 160 ++++++++++-------- .../src/file_sharing/directory_storage.h | 8 +- libretroshare/src/file_sharing/p3filelists.cc | 57 +++++-- libretroshare/src/ft/ftserver.cc | 2 +- libretroshare/src/retroshare/rstypes.h | 5 +- libretroshare/src/rsserver/rstypes.cc | 21 ++- 6 files changed, 155 insertions(+), 98 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 7b913b4c3..77b9e70c9 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -3,6 +3,8 @@ #include "util/rsstring.h" #include "directory_storage.h" +#define DEBUG_DIRECTORY_STORAGE 1 + /******************************************************************************************************************/ /* Internal File Hierarchy Storage */ /******************************************************************************************************************/ @@ -71,7 +73,7 @@ class InternalFileHierarchyStorage { mNodes.push_back(new DirEntry("")) ; mNodes.back()->row=0; - mNodes.back()->parent_index=-1; + mNodes.back()->parent_index=0; } int parentRow(DirectoryStorage::EntryIndex e) @@ -153,7 +155,7 @@ class InternalFileHierarchyStorage if(mNodes.empty() || indx==DirectoryStorage::NO_INDEX || indx >= mNodes.size() || mNodes[indx] == NULL) return nodeAccessError("checkIndex(): Node does not exist") ; - if(! mNodes[indx]->type() & type) + if(! (mNodes[indx]->type() & type)) return nodeAccessError("checkIndex(): Node is of wrong type") ; return true; @@ -351,7 +353,10 @@ private: } DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << std::endl; + std::cerr << indent << "dir:" << d.dir_name << ", subdirs: " ; + for(int i=0;isubdirs.size() + dir_entry->subfiles.size(); + d.min_age = now - dir_entry->most_recent_time ; + d.name = dir_entry->dir_name; + d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; + d.parent = (void*)(intptr_t)dir_entry->parent_index ; + + if(indx == 0) + { + d.type = DIR_TYPE_PERSON ; + d.name = mPeerId.toStdString(); + } + } + else + { + const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; +#endif + d.type = DIR_TYPE_FILE; + d.count = file_entry->file_size; + d.min_age = now - file_entry->file_modtime ; + d.name = file_entry->file_name; + d.hash = file_entry->file_hash; + d.age = now - file_entry->file_modtime; + d.parent = (void*)(intptr_t)file_entry->parent_index ; + + const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index); + + if(parent_dir_entry != NULL) + d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ; + else + d.path = "" ; + } + + d.flags.clear() ; + + return true; +} /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ @@ -627,78 +708,19 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) { - d.children.clear() ; - time_t now = time(NULL) ; + bool res = DirectoryStorage::extractData(indx,d) ; - const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + if(!res) + return false; - if (dir_entry != NULL) /* has children --- fill */ - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() ref=dir" << std::endl; -#endif - /* extract all the entries */ + // here we should update the file sharing flags - for(DirectoryStorage::DirIterator it(this,indx);it;++it) - { - DirStub stub; - stub.type = DIR_TYPE_DIR; - stub.name = it.name(); - stub.ref = (void*)(intptr_t)*it; // this is updated by the caller, who knows which friend we're dealing with - - d.children.push_back(stub); - } - - for(DirectoryStorage::FileIterator it(this,indx);it;++it) - { - DirStub stub; - stub.type = DIR_TYPE_FILE; - stub.name = it.name(); - stub.ref = (void*)(intptr_t)*it; - - d.children.push_back(stub); - } - - if(dir_entry->parent_index == 0) - d.type = DIR_TYPE_PERSON ; - else - d.type = DIR_TYPE_DIR; - - d.hash.clear() ; - d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size(); - d.min_age = now - dir_entry->most_recent_time ; - d.name = dir_entry->dir_name; - d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; - d.parent = (void*)(intptr_t)dir_entry->parent_index ; - } - else - { - const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; -#endif - d.type = DIR_TYPE_FILE; - d.count = file_entry->file_size; - d.min_age = now - file_entry->file_modtime ; - d.name = file_entry->file_name; - d.hash = file_entry->file_hash; - d.age = now - file_entry->file_modtime; - d.parent = (void*)(intptr_t)file_entry->parent_index ; - - const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index); - - if(parent_dir_entry != NULL) - d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ; - else - d.path = "" ; - } - -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() name: " << file->name << std::endl; -#endif d.flags.clear() ; /* find parent pointer, and row */ + std::cerr << "LocalDirectoryStorage::extractData(): Returning this:" << std::endl; + std::cerr << d << std::endl; + return true; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index a96790684..9f2b63be9 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -28,7 +28,7 @@ class DirectoryStorage void getFileDetails(EntryIndex i) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. - virtual bool extractData(const EntryIndex& indx,DirDetails& d) =0; + virtual bool extractData(const EntryIndex& indx,DirDetails& d); // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. // It gives access to sub-files and sub-directories below. @@ -121,12 +121,6 @@ class RemoteDirectoryStorage: public DirectoryStorage public: RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} virtual ~RemoteDirectoryStorage() {} - - virtual bool extractData(const EntryIndex& indx,DirDetails& d) - { - NOT_IMPLEMENTED(); - return false; - } }; class LocalDirectoryStorage: public DirectoryStorage diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 09cf31e53..20e2c5c6c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -137,7 +137,6 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { - NOT_IMPLEMENTED(); } void p3FileDatabase::tickRecv() @@ -146,7 +145,6 @@ void p3FileDatabase::tickRecv() void p3FileDatabase::tickSend() { // go through the list of out requests and send them to the corresponding friends, if they are online. - NOT_IMPLEMENTED(); } bool p3FileDatabase::loadList(std::list& items) @@ -295,6 +293,13 @@ bool p3FileDatabase::convertPointerToEntryIndex(const void *p, EntryIndex& e, ui e = EntryIndex( *reinterpret_cast(&p) & ENTRY_INDEX_BIT_MASK ) ; friend_index = (*reinterpret_cast(&p)) >> NB_ENTRY_INDEX_BITS ; + if(friend_index == 0) + { + std::cerr << "(EE) Cannot find friend index in pointer. Encoded value is zero!" << std::endl; + return false; + } + friend_index--; + return true; } bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi, void *& p) @@ -309,13 +314,13 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi uint32_t fe = (uint32_t)e ; - if(fi >= (1<= (1<< NB_ENTRY_INDEX_BITS)) + if(fi+1 >= (1<= (1<< NB_ENTRY_INDEX_BITS)) { std::cerr << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; return false ; } - p = reinterpret_cast( (fi << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ; + p = reinterpret_cast( ( (1+fi) << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ; return true; } @@ -325,10 +330,36 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const { // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us) + // or at the top of our own list of shred directories, depending on the flags. if (ref == NULL) { - for(uint32_t i=0;igetOwnId().toStdString(); + stub.ref = p; + d.children.push_back(stub); + } + else for(uint32_t i=1;iroot(),i,p); @@ -339,24 +370,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags stub.ref = p; d.children.push_back(stub); } - d.count = mDirectories.size(); - d.parent = NULL; - d.prow = -1; - d.ref = NULL; - d.type = DIR_TYPE_ROOT; - d.name = "root"; - d.hash.clear() ; - d.path = "root"; - d.age = 0; - d.flags.clear() ; - d.min_age = 0 ; + d.count = d.children.size(); return true ; } uint32_t fi; - EntryIndex e ; + DirectoryStorage::EntryIndex e ; convertPointerToEntryIndex(ref,e,fi); @@ -505,7 +526,7 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) { DirDetails cdetails ; - RequestDirDetails ((void*)*rit,cdetails,FileSearchFlags(0u)); + RequestDirDetails ((void*)(intptr_t)*rit,cdetails,FileSearchFlags(0u)); #ifdef P3FILELISTS_DEBUG std::cerr << "Filtering candidate " << (*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; #endif diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 8a0424a5c..1268ea831 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -240,7 +240,7 @@ bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& re bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) { - return mFtController->alreadyHaveFile(hash, info); + return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info); } bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds) diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index d30b156eb..9ab56c70b 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -215,8 +215,7 @@ class FileInfo std::list parent_groups ; }; -std::ostream &operator<<(std::ostream &out, const FileInfo &info); - +std::ostream &operator<<(std::ostream &out, const FileInfo& info); class DirStub { @@ -247,6 +246,8 @@ public: std::list parent_groups; // parent groups for the shared directory }; +std::ostream &operator<<(std::ostream &out, const DirDetails& details); + class FileDetail { public: diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index c5c3c8b02..128f08005 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -44,7 +44,26 @@ * **********************************************************************/ - +std::ostream &operator<<(std::ostream &out, const DirDetails& d) +{ + std::cerr << "====DIR DETAILS====" << std::endl; + std::cerr << " parent pointer: " << (intptr_t)d.parent << std::endl; + std::cerr << " current pointer: " << (intptr_t)d.ref << std::endl; + std::cerr << " parent row : " << d.prow << std::endl; + std::cerr << " type : " << (int)d.type << std::endl; + std::cerr << " PeerId : " << d.id << std::endl; + std::cerr << " Name : " << d.name << std::endl; + std::cerr << " Hash : " << d.hash << std::endl; + std::cerr << " Path : " << d.path << std::endl; + std::cerr << " Count : " << d.count << std::endl; + std::cerr << " Age : " << d.age << std::endl; + std::cerr << " Min age : " << d.min_age << std::endl; + std::cerr << " Flags : " << d.flags << std::endl; + std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; + std::cerr << " Children : " ; for(std::list::const_iterator it(d.children.begin());it!=d.children.end();++it) std::cerr << (intptr_t)(*it).ref << " "; std::cerr << std::endl; + std::cerr << "===================" << std::endl; + return out; +} std::ostream &operator<<(std::ostream &out, const FileInfo &info) { out << "FileInfo: path: " << info.path; From 78b87441838a4a16de9082136a0983156f020a24 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 17 Aug 2016 14:48:54 +0200 Subject: [PATCH 027/174] fixed display of remote directories --- .../src/file_sharing/directory_storage.cc | 2 +- .../src/file_sharing/directory_updater.cc | 15 +- libretroshare/src/file_sharing/p3filelists.cc | 133 +++++++++++------- libretroshare/src/file_sharing/p3filelists.h | 6 +- libretroshare/src/rsserver/rstypes.cc | 4 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 2 +- 6 files changed, 100 insertions(+), 62 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 77b9e70c9..d0bd333b6 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -719,7 +719,7 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) /* find parent pointer, and row */ - std::cerr << "LocalDirectoryStorage::extractData(): Returning this:" << std::endl; + std::cerr << "LocalDirectoryStorage::extractData(): indx=" << indx << " Returning this:" << std::endl; std::cerr << d << std::endl; return true; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index e0f0438a0..7f427ff7d 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -63,12 +63,12 @@ void LocalDirectoryUpdater::sweepSharedDirectories() // now for each of them, go recursively and match both files and dirs - DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; - - for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it, ++stored_dir_it) + for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it) { - std::cerr << " recursing into " << real_dir_it->filename << std::endl; - recursUpdateSharedDir(real_dir_it->filename, *stored_dir_it) ; + std::cerr << " recursing into " << stored_dir_it.name() << std::endl; + + recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it) ; // here we need to use the list that was stored, instead of the shared dir list, because the two + // are not necessarily in the same order. } RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } @@ -128,7 +128,10 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) - recursUpdateSharedDir(cumulated_path + "/" + *real_dir_it, *stored_dir_it) ; + { + std::cerr << " recursing into " << stored_dir_it.name() << std::endl; + recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ; + } } void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 20e2c5c6c..64bc38924 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -107,25 +107,39 @@ int p3FileDatabase::tick() if(last_print_time + 20 < now) { + RS_STACK_MUTEX(mFLSMtx) ; mLocalSharedDirs->print(); last_print_time = now ; + +#warning this should be removed, but it's necessary atm for updating the GUI + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } + if(mUpdateFlags) + { + IndicateConfigChanged(); + + if(mUpdateFlags & P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED) + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); + + if(mUpdateFlags & P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED) + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + + mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; + } return 0; } void p3FileDatabase::startThreads() { - std::cerr << "Starting hash cache thread..." ; - mHashCache->start(); - std::cerr << "Done." << std::endl; - + RS_STACK_MUTEX(mFLSMtx) ; std::cerr << "Starting directory watcher thread..." ; mLocalDirWatcher->start(); std::cerr << "Done." << std::endl; } void p3FileDatabase::stopThreads() { + RS_STACK_MUTEX(mFLSMtx) ; std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; mHashCache->fullstop(); std::cerr << "Done." << std::endl; @@ -138,7 +152,6 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { } - void p3FileDatabase::tickRecv() { } @@ -165,58 +178,55 @@ bool p3FileDatabase::saveList(bool &cleanup, std::list&) void p3FileDatabase::cleanup() { - RS_STACK_MUTEX(mFLSMtx) ; - - // look through the list of friend directories. Remove those who are not our friends anymore. - // - P3FILELISTS_DEBUG() << "Cleanup pass." << std::endl; - - std::set friend_set ; { - std::list friend_lst ; + RS_STACK_MUTEX(mFLSMtx) ; - rsPeers->getFriendList(friend_lst); + // look through the list of friend directories. Remove those who are not our friends anymore. + // + std::set friend_set ; + { + std::list friend_lst ; - for(std::list::const_iterator it(friend_lst.begin());it!=friend_lst.end();++it) - friend_set.insert(*it) ; - } + rsPeers->getFriendList(friend_lst); - for(uint32_t i=1;ipeerId()) == friend_set.end()) - { - P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; - - delete mDirectories[i]; - mDirectories[i] = NULL ; - - mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - - friend_set.erase(mDirectories[i]->peerId()); - - mFriendIndexMap.erase(mDirectories[i]->peerId()); - mFriendIndexTab[i].clear(); + for(std::list::const_iterator it(friend_lst.begin());it!=friend_lst.end();++it) + friend_set.insert(*it) ; } - // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. - // - for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) - { - // Check if a remote directory exists for that friend, possibly creating the index. + for(uint32_t i=1;ipeerId()) == friend_set.end()) + { + P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; - uint32_t friend_index = getFriendIndex(*it) ; + delete mDirectories[i]; + mDirectories[i] = NULL ; - if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL) - continue ; + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; + friend_set.erase(mDirectories[i]->peerId()); - mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + mFriendIndexMap.erase(mDirectories[i]->peerId()); + mFriendIndexTab[i].clear(); + } - mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. + // + for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) + { + // Check if a remote directory exists for that friend, possibly creating the index. + + uint32_t friend_index = locked_getFriendIndex(*it) ; + + if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL) + continue ; + + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; + + mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + } } - - if(mUpdateFlags) - IndicateConfigChanged(); } std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const @@ -225,7 +235,7 @@ std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const return "dirlist_"+pid.toStdString()+".txt" ; } -uint32_t p3FileDatabase::getFriendIndex(const RsPeerId& pid) +uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) { std::map::const_iterator it = mFriendIndexMap.find(pid) ; @@ -271,7 +281,7 @@ uint32_t p3FileDatabase::getFriendIndex(const RsPeerId& pid) } } -const RsPeerId& p3FileDatabase::getFriendFromIndex(uint32_t indx) const +const RsPeerId& p3FileDatabase::locked_getFriendFromIndex(uint32_t indx) const { static const RsPeerId null_id ; @@ -329,6 +339,8 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const { + RS_STACK_MUTEX(mFLSMtx) ; + // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us) // or at the top of our own list of shred directories, depending on the flags. @@ -389,14 +401,26 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // a pointer into an int. convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ; - convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; for(std::list::iterator it(d.children.begin());it!=d.children.end();++it) convertEntryIndexToPointer((intptr_t)it->ref,fi,it->ref); - d.prow = mDirectories[fi]->parentRow(e) ; + if(e == 0) + { + d.prow = fi-1 ; + d.parent = NULL ; + } + else + { + d.prow = mDirectories[fi]->parentRow(e) ; + convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; + } + d.id = mDirectories[fi]->peerId(); + std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + std::cerr << d << std::endl; + return true; } @@ -412,6 +436,8 @@ int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &detai } uint32_t p3FileDatabase::getType(void *ref) const { + RS_STACK_MUTEX(mFLSMtx) ; + EntryIndex e ; uint32_t fi; @@ -469,6 +495,8 @@ void p3FileDatabase::setRememberHashCache(bool) bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const { + RS_STACK_MUTEX(mFLSMtx) ; + std::list firesults; mLocalSharedDirs->searchHash(hash,firesults) ; @@ -478,6 +506,8 @@ bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags, int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) { + RS_STACK_MUTEX(mFLSMtx) ; + std::list firesults; mLocalSharedDirs->searchTerms(keywords,firesults) ; @@ -485,6 +515,8 @@ int p3FileDatabase::SearchKeywords(const std::list& keywords, std:: } int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const { + RS_STACK_MUTEX(mFLSMtx) ; + std::list firesults; mLocalSharedDirs->searchBoolExp(exp,firesults) ; @@ -492,6 +524,8 @@ int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& result } bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const { + RS_STACK_MUTEX(mFLSMtx) ; + if(hintflags & RS_FILE_HINTS_LOCAL) { std::list res; @@ -557,5 +591,6 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath) { + RS_STACK_MUTEX(mFLSMtx) ; return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ; } diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index cd090af61..0fa36088e 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -157,8 +157,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p); static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ; - uint32_t getFriendIndex(const RsPeerId& pid); - const RsPeerId& getFriendFromIndex(uint32_t indx) const; + uint32_t locked_getFriendIndex(const RsPeerId& pid); + const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; std::map mFriendIndexMap ; std::vector mFriendIndexTab; @@ -169,7 +169,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // Local flags and mutexes - RsMutex mFLSMtx ; + mutable RsMutex mFLSMtx ; uint32_t mUpdateFlags ; }; diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index 128f08005..0c88cb97a 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -47,8 +47,8 @@ std::ostream &operator<<(std::ostream &out, const DirDetails& d) { std::cerr << "====DIR DETAILS====" << std::endl; - std::cerr << " parent pointer: " << (intptr_t)d.parent << std::endl; - std::cerr << " current pointer: " << (intptr_t)d.ref << std::endl; + std::cerr << " parent pointer: " << d.parent << std::endl; + std::cerr << " current pointer: " << d.ref << std::endl; std::cerr << " parent row : " << d.prow << std::endl; std::cerr << " type : " << (int)d.type << std::endl; std::cerr << " PeerId : " << d.id << std::endl; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index c3bf94d5a..fcaec5d09 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -825,7 +825,7 @@ void SharedFilesDialog::preModDirectories(bool local) void SharedFilesDialog::postModDirectories(bool local) { - if (isRemote() == local) { + if (isRemote() == local) { return; } From f8ed1d3fb707bc578f2842ed8c0ed85143865659 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 19 Aug 2016 18:49:42 +0200 Subject: [PATCH 028/174] changed DirDetails::children to be a vector, removed costly caching of DirDetailVector --- .../src/file_sharing/directory_storage.cc | 2 + libretroshare/src/file_sharing/p3filelists.cc | 12 +- libretroshare/src/retroshare/rstypes.h | 2 +- libretroshare/src/rsserver/rstypes.cc | 2 +- .../src/gui/FileTransfer/SearchDialog.cpp | 5 +- retroshare-gui/src/gui/RemoteDirModel.cpp | 203 +++++++++--------- retroshare-gui/src/gui/RemoteDirModel.h | 2 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 4 +- .../src/gui/common/RsCollectionFile.cpp | 6 +- retroshare-gui/src/main.cpp | 2 +- 10 files changed, 124 insertions(+), 116 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index d0bd333b6..72aaf6f52 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -545,6 +545,8 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + d.ref = (void*)(intptr_t)indx ; + if (dir_entry != NULL) /* has children --- fill */ { #ifdef DEBUG_DIRECTORY_STORAGE diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 64bc38924..9a8496470 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -108,11 +108,12 @@ int p3FileDatabase::tick() if(last_print_time + 20 < now) { RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->print(); last_print_time = now ; -#warning this should be removed, but it's necessary atm for updating the GUI - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); +//#warning this should be removed, but it's necessary atm for updating the GUI +// RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } if(mUpdateFlags) @@ -402,12 +403,13 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ; - for(std::list::iterator it(d.children.begin());it!=d.children.end();++it) - convertEntryIndexToPointer((intptr_t)it->ref,fi,it->ref); + for(uint32_t i=0;i children; + std::vector children; std::list parent_groups; // parent groups for the shared directory }; diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index 0c88cb97a..be83f7d7a 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -60,7 +60,7 @@ std::ostream &operator<<(std::ostream &out, const DirDetails& d) std::cerr << " Min age : " << d.min_age << std::endl; std::cerr << " Flags : " << d.flags << std::endl; std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; - std::cerr << " Children : " ; for(std::list::const_iterator it(d.children.begin());it!=d.children.end();++it) std::cerr << (intptr_t)(*it).ref << " "; std::cerr << std::endl; + std::cerr << " Children : " ; for(uint32_t i=0;i::const_iterator it(dir.children.begin()); it != dir.children.end(); ++it) { + for (uint32_t i=0;iRequestDirDetails(it->ref, details, FileSearchFlags(0u)); + rsFiles->RequestDirDetails(dir.children[i].ref, details, FileSearchFlags(0u)); insertDirectory(txt, searchId, details, child); } } diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index a2220f2ff..295ebbf83 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -99,10 +99,9 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const void *ref = parent.internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); - - if (!details) - { + DirDetails details ; + if (!requestDirDetails(ref, RemoteMode,details)) + { /* error */ #ifdef RDM_DEBUG std::cerr << "lookup failed -> false"; @@ -111,7 +110,7 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const return false; } - if (details->type == DIR_TYPE_FILE) + if (details.type == DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "lookup FILE -> false"; @@ -124,7 +123,7 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const std::cerr << "lookup PER/DIR #" << details->count; std::cerr << std::endl; #endif - return (details->count > 0); /* do we have children? */ + return (details.count > 0); /* do we have children? */ } bool FlatStyle_RDM::hasChildren(const QModelIndex &parent) const { @@ -155,9 +154,9 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const void *ref = (parent.isValid())? parent.internalPointer() : NULL ; - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) + if (! requestDirDetails(ref, RemoteMode,details)) { #ifdef RDM_DEBUG std::cerr << "lookup failed -> 0"; @@ -165,7 +164,7 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const #endif return 0; } - if (details->type == DIR_TYPE_FILE) + if (details.type == DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "lookup FILE: 0"; @@ -179,7 +178,7 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const std::cerr << "lookup PER/DIR #" << details->count; std::cerr << std::endl; #endif - return details->count; + return details.count; } int FlatStyle_RDM::rowCount(const QModelIndex &parent) const @@ -381,16 +380,16 @@ QVariant TreeStyle_RDM::displayRole(const DirDetails& details,int coln) const QString FlatStyle_RDM::computeDirectoryPath(const DirDetails& details) const { QString dir ; - const DirDetails *det = requestDirDetails(details.parent,RemoteMode); + DirDetails det ; - if(!det) - return QString(); + if(!requestDirDetails(details.parent,RemoteMode,det)) + return QString(); #ifdef SHOW_TOTAL_PATH do { #endif - dir = QString::fromUtf8(det->name.c_str())+"/"+dir ; + dir = QString::fromUtf8(det.name.c_str())+"/"+dir ; #ifdef SHOW_TOTAL_PATH if(!requestDirDetails(det.parent,det,flags)) @@ -521,28 +520,28 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const void *ref = index.internalPointer(); int coln = index.column(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) - return QVariant(); + if (!requestDirDetails(ref, RemoteMode,details)) + return QVariant(); if (role == RetroshareDirModel::FileNameRole) /* end of FileNameRole */ - return QString::fromUtf8(details->name.c_str()); + return QString::fromUtf8(details.name.c_str()); if (role == Qt::TextColorRole) { - if(details->min_age > ageIndicator) + if(details.min_age > ageIndicator) return QVariant(QColor(Qt::gray)) ; else if(RemoteMode) { FileInfo info; QVariant local_file_color = QVariant(QColor(Qt::red)); - if(rsFiles->alreadyHaveFile(details->hash, info)) + if(rsFiles->alreadyHaveFile(details.hash, info)) return local_file_color; std::list downloads; rsFiles->FileDownloads(downloads); - if(std::find(downloads.begin(), downloads.end(), details->hash) != downloads.end()) + if(std::find(downloads.begin(), downloads.end(), details.hash) != downloads.end()) return local_file_color; else return QVariant(); @@ -553,7 +552,7 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const if(role == Qt::DecorationRole) - return decorationRole(*details,coln) ; + return decorationRole(details,coln) ; /***************** Qt::EditRole @@ -578,10 +577,10 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const } /* end of TextAlignmentRole */ if (role == Qt::DisplayRole) - return displayRole(*details,coln) ; + return displayRole(details,coln) ; if (role == SortRole) - return sortRole(index,*details,coln) ; + return sortRole(index,details,coln) ; return QVariant(); } @@ -712,9 +711,10 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent } ********/ - const DirDetailsVector *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) + + if (! requestDirDetails(ref, RemoteMode,details)) { #ifdef RDM_DEBUG std::cerr << "lookup failed -> invalid"; @@ -728,7 +728,7 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent * get the reference number */ - if (row >= (int) details->childrenVector.size()) + if (row >= (int) details.children.size()) { #ifdef RDM_DEBUG std::cerr << "wrong number of children -> invalid"; @@ -744,7 +744,7 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent /* we can just grab the reference now */ - return createIndex(row, column, details->childrenVector[row].ref); + return createIndex(row, column, details.children[row].ref); } QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const { @@ -786,10 +786,10 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const } void *ref = index.internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) - { + if (! requestDirDetails(ref, RemoteMode,details)) + { #ifdef RDM_DEBUG std::cerr << "Failed Lookup -> invalid"; std::cerr << std::endl; @@ -797,7 +797,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const return QModelIndex(); } - if (!(details->parent)) + if (!(details.parent)) { #ifdef RDM_DEBUG std::cerr << "success. parent is Root/NULL --> invalid"; @@ -811,7 +811,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const std::cerr << std::endl; #endif - return createIndex(details->prow, 0, details->parent); + return createIndex(details.prow, 0, details.parent); } QModelIndex FlatStyle_RDM::parent( const QModelIndex & index ) const { @@ -836,12 +836,12 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const void *ref = index.internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) - return Qt::ItemIsSelectable; // Error. + if (! requestDirDetails(ref, RemoteMode,details)) + return Qt::ItemIsSelectable; // Error. - switch(details->type) + switch(details.type) { case DIR_TYPE_PERSON: return Qt::ItemIsEnabled; case DIR_TYPE_DIR: return Qt::ItemIsSelectable | Qt::ItemIsEnabled; @@ -902,32 +902,36 @@ void RetroshareDirModel::postMods() emit layoutChanged(); } -const DirDetailsVector *RetroshareDirModel::requestDirDetails(void *ref, bool remote) const +bool RetroshareDirModel::requestDirDetails(void *ref, bool remote,DirDetails& d) const { - const QMap::const_iterator it = mCache.constFind(ref); - if (it != mCache.constEnd()) { - /* Details found in cache */ - return &it.value(); - } + FileSearchFlags flags = (remote) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL; - /* Get details from the lib */ - DirDetailsVector details; - FileSearchFlags flags = (remote) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL; - if (rsFiles->RequestDirDetails(ref, details, flags)) { - /* Convert std::list to std::vector for fast access with index */ - std::list::const_iterator childIt; - for (childIt = details.children.begin(); childIt != details.children.end(); ++childIt) { - details.childrenVector.push_back(*childIt); - } - - /* Add to cache, must cast to none const */ - const QMap::iterator it1 = ((QMap*) &mCache)->insert(ref, details); - return &it1.value(); - } - - /* No details found */ - return NULL; + return rsFiles->RequestDirDetails(ref, d, flags) ; } +// const QMap::const_iterator it = mCache.constFind(ref); +// if (it != mCache.constEnd()) { +// /* Details found in cache */ +// return &it.value(); +// } +// +//#warning this is terrible! A vector in a std::map will keep being copied and create a lot of CPU overload. Use a pointer instead. +// /* Get details from the lib */ +// +// if (rsFiles->RequestDirDetails(ref, details, flags)) { +// /* Convert std::list to std::vector for fast access with index */ +// std::list::const_iterator childIt; +// for (childIt = details.children.begin(); childIt != details.children.end(); ++childIt) { +// details.childrenVector.push_back(*childIt); +// } +// +// /* Add to cache, must cast to none const */ +// const QMap::iterator it1 = ((QMap*) &mCache)->insert(ref, details); +// return &it1.value(); +// } +// +// /* No details found */ +// return NULL; +//} void RetroshareDirModel::createCollectionFile(QWidget *parent, const QModelIndexList &list) { @@ -1010,15 +1014,16 @@ void RetroshareDirModel::downloadDirectory(const DirDetails & dirDetails, int pr if (!dwlDir.mkpath(cleanPath)) return; - for (it = dirDetails.children.begin(); it != dirDetails.children.end(); ++it) + for(uint32_t i=0;iref) continue; + if (!dirDetails.children[i].ref) continue; - const DirDetails *subDirDetails = requestDirDetails(it->ref, true); + DirDetails subDirDetails ; - if (!subDirDetails) continue; + if(!requestDirDetails(dirDetails.children[i].ref, true,subDirDetails)) + continue; - downloadDirectory(*subDirDetails, prefixLen); + downloadDirectory(subDirDetails, prefixLen); } } } @@ -1035,12 +1040,12 @@ void RetroshareDirModel::getDirDetailsFromSelect (const QModelIndexList &list, s { void *ref = it -> internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details ; - if (!details) + if(!requestDirDetails(ref, RemoteMode,details)) continue; - dirVec.push_back(*details); + dirVec.push_back(details); } } } @@ -1072,12 +1077,12 @@ void RetroshareDirModel::getFileInfoFromIndexList(const QModelIndexList& list, s { void *ref = it -> internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); + DirDetails details; - if (!details) + if (!requestDirDetails(ref, RemoteMode,details)) continue; - if(details->type == DIR_TYPE_PERSON) + if(details.type == DIR_TYPE_PERSON) continue ; #ifdef RDM_DEBUG @@ -1090,10 +1095,10 @@ void RetroshareDirModel::getFileInfoFromIndexList(const QModelIndexList& list, s // Note: for directories, the returned hash, is the peer id, so if we collect // dirs, we need to be a bit more conservative for the - if(already_in.find(details->hash.toStdString()+details->name) == already_in.end()) + if(already_in.find(details.hash.toStdString()+details.name) == already_in.end()) { - file_details.push_back(*details) ; - already_in.insert(details->hash.toStdString()+details->name) ; + file_details.push_back(details) ; + already_in.insert(details.hash.toStdString()+details.name) ; } } #ifdef RDM_DEBUG @@ -1167,17 +1172,17 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list internalPointer(); - const DirDetails *details = requestDirDetails(ref, false); + DirDetails details ; - if (!details) - { + if (!requestDirDetails(ref, false,details) ) + { #ifdef RDM_DEBUG std::cerr << "getFilePaths() Bad Request" << std::endl; #endif continue; } - if (details->type != DIR_TYPE_FILE) + if (details.type != DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "getFilePaths() Not File" << std::endl; @@ -1187,14 +1192,14 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::listname << std::endl; - std::cerr << "Hash: " << details->hash << std::endl; - std::cerr << "Size: " << details->count << std::endl; - std::cerr << "Path: " << details->path << std::endl; + std::cerr << "Name: " << details.name << std::endl; + std::cerr << "Hash: " << details.hash << std::endl; + std::cerr << "Size: " << details.count << std::endl; + std::cerr << "Path: " << details.path << std::endl; #endif - std::string filepath = details->path + "/"; - filepath += details->name; + std::string filepath = details.path + "/"; + filepath += details.name; #ifdef RDM_DEBUG std::cerr << "Constructed FilePath: " << filepath << std::endl; @@ -1222,12 +1227,9 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con { void *ref = it -> internalPointer(); - const DirDetails *details = requestDirDetails(ref, RemoteMode); - - if (!details) - { - continue; - } + DirDetails details ; + if (!requestDirDetails(ref, RemoteMode,details)) + continue; #ifdef RDM_DEBUG std::cerr << "::::::::::::FileDrag:::: " << std::endl; @@ -1237,7 +1239,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con std::cerr << "Path: " << details->path << std::endl; #endif - if (details->type != DIR_TYPE_FILE) + if (details.type != DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::mimeData() Not File" << std::endl; @@ -1245,7 +1247,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con continue; /* not file! */ } - if (drags.end() != (dit = drags.find(details->hash))) + if (drags.end() != (dit = drags.find(details.hash))) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::mimeData() Duplicate" << std::endl; @@ -1253,9 +1255,9 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con continue; /* duplicate */ } - drags[details->hash] = details->count; + drags[details.hash] = details.count; - QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details->name.c_str()), QString::fromStdString(details->hash.toStdString()), QString::number(details->count)); + QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details.name.c_str()), QString::fromStdString(details.hash.toStdString()), QString::number(details.count)); if (RemoteMode) { @@ -1348,17 +1350,18 @@ void FlatStyle_RDM::updateRefs() std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl; #endif _ref_stack.pop_back() ; - const DirDetails *details = requestDirDetails(ref, RemoteMode) ; - if (details) + DirDetails details ; + + if (requestDirDetails(ref, RemoteMode,details)) { - if(details->type == DIR_TYPE_FILE) // only push files, not directories nor persons. - _ref_entries.push_back(std::pair(ref,computeDirectoryPath(*details))); + if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons. + _ref_entries.push_back(std::pair(ref,computeDirectoryPath(details))); #ifdef RDM_DEBUG std::cerr << "FlatStyle_RDM::postMods(): adding ref " << ref << std::endl; #endif - for(std::list::const_iterator it = details->children.begin(); it != details->children.end(); ++it) - _ref_stack.push_back(it->ref) ; + for(uint32_t i=0;i MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to { // the user and setup a timer to finish the job later. diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index a16dcfc3f..dd81e2c84 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -74,7 +74,7 @@ class RetroshareDirModel : public QAbstractItemModel void getFilePaths(const QModelIndexList &list, std::list &fullpaths); void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; } - const DirDetailsVector *requestDirDetails(void *ref, bool remote) const; + bool requestDirDetails(void *ref, bool remote,DirDetails& d) const; virtual void update() {} public: diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index fcaec5d09..67a5e3d0e 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -544,9 +544,9 @@ void SharedFilesDialog::copyLink (const QModelIndexList& lst, bool remote) if (details.type == DIR_TYPE_DIR) { - for (std::list::const_iterator cit = details.children.begin();cit != details.children.end(); ++cit) + for(uint32_t j=0;j::const_iterator it = details.children.begin(); it != details.children.end(); ++it) + for(uint32_t i=0;iref) + if (!details.children[i].ref) continue; DirDetails subDirDetails; FileSearchFlags flags = RS_FILE_HINTS_LOCAL; - if (!rsFiles->RequestDirDetails(it->ref, subDirDetails, flags)) + if (!rsFiles->RequestDirDetails(details.children[i].ref, subDirDetails, flags)) continue; recursAddElements(doc,subDirDetails,d) ; diff --git a/retroshare-gui/src/main.cpp b/retroshare-gui/src/main.cpp index f7e7c835d..c64b6fa54 100644 --- a/retroshare-gui/src/main.cpp +++ b/retroshare-gui/src/main.cpp @@ -368,7 +368,7 @@ feenableexcept(FE_INVALID | FE_DIVBYZERO); QObject::connect(notify,SIGNAL(deferredSignatureHandlingRequested()),notify,SLOT(handleSignatureEvent()),Qt::QueuedConnection) ; QObject::connect(notify,SIGNAL(chatLobbyTimeShift(int)),notify,SLOT(handleChatLobbyTimeShift(int)),Qt::QueuedConnection) ; QObject::connect(notify,SIGNAL(diskFull(int,int)) ,w ,SLOT(displayDiskSpaceWarning(int,int))) ; - QObject::connect(notify,SIGNAL(filesPostModChanged(bool)) ,w ,SLOT(postModDirectories(bool) )) ; + QObject::connect(notify,SIGNAL(filesPostModChanged(bool)) ,w ,SLOT(postModDirectories(bool)) ,Qt::QueuedConnection ) ; QObject::connect(notify,SIGNAL(transfersChanged()) ,w->transfersDialog ,SLOT(insertTransfers() )) ; QObject::connect(notify,SIGNAL(publicChatChanged(int)) ,w->friendsDialog ,SLOT(publicChatChanged(int) )); QObject::connect(notify,SIGNAL(neighboursChanged()) ,w->friendsDialog->networkDialog ,SLOT(securedUpdateDisplay())) ; From edc602f68f67e5549af3288d5b4bb7a137a6f71b Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 20 Aug 2016 16:23:11 +0200 Subject: [PATCH 029/174] added infrastructure for synchronisation of file lists --- .../src/file_sharing/directory_storage.cc | 22 +- .../src/file_sharing/directory_storage.h | 6 +- .../src/file_sharing/directory_updater.cc | 11 +- .../src/file_sharing/directory_updater.h | 10 +- libretroshare/src/file_sharing/p3filelists.cc | 198 ++++++++--- libretroshare/src/file_sharing/p3filelists.h | 23 +- .../src/file_sharing/rsfilelistitems.cc | 336 ++++++++++++++++++ .../src/file_sharing/rsfilelistitems.h | 132 +++++++ libretroshare/src/libretroshare.pro | 18 +- libretroshare/src/serialiser/itempriorities.h | 5 + 10 files changed, 692 insertions(+), 69 deletions(-) create mode 100644 libretroshare/src/file_sharing/rsfilelistitems.cc create mode 100644 libretroshare/src/file_sharing/rsfilelistitems.h diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 72aaf6f52..7e8cea686 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -303,6 +303,20 @@ class InternalFileHierarchyStorage return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; } + bool searchHash(const RsFileHash& hash,std::list& results) + { + std::map::const_iterator it = mHashes.find(hash); + + if( it != mHashes.end() ) + { + results.clear(); + results.push_back(it->second) ; + return true ; + } + else + return false; + } + bool check() // checks consistency of storage. { return true; @@ -390,7 +404,7 @@ private: return true ; } - std::map mHashes ; // used for fast search access + std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? }; /******************************************************************************************************************/ @@ -498,6 +512,12 @@ bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash return mFileHierarchy->updateHash(index,hash); } +int DirectoryStorage::searchHash(const RsFileHash& hash, std::list &results) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->searchHash(hash,results); +} + // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; // static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 9f2b63be9..637a8e88e 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -23,10 +23,10 @@ class DirectoryStorage void save() const ; virtual int searchTerms(const std::list& terms, std::list &results) const { NOT_IMPLEMENTED() ; return 0;} - virtual int searchHash(const RsFileHash& hash, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } + virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - void getFileDetails(EntryIndex i) ; + bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& last_update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); @@ -47,6 +47,8 @@ class DirectoryStorage // info about the directory that is pointed by the iterator std::string name() const ; + time_t last_modif_time() const ; // last time a file in this directory or in the directories below has been modified. + time_t last_update_time() const ; // last time this directory was updated private: EntryIndex mParentIndex ; // index of the parent dir. uint32_t mDirTabIndex ; // index in the vector of subdirs. diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 7f427ff7d..731d92902 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -6,12 +6,12 @@ #define DEBUG_LOCAL_DIR_UPDATER 1 -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +//=============================================================================================================// +// Local Directory Updater // +//=============================================================================================================// -void RemoteDirectoryUpdater::tick() -{ - // use the stored iterator -} +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) @@ -139,4 +139,3 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; } - diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index d146a61b5..1248aeefb 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -13,6 +13,8 @@ class DirectoryUpdater virtual ~DirectoryUpdater(){} }; +#warning: simplify, if we don't keep the remote directory updater + class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread { public: @@ -35,11 +37,3 @@ private: time_t mLastSweepTime; }; -class RemoteDirectoryUpdater: public DirectoryUpdater -{ - public: - RemoteDirectoryUpdater() {} - virtual ~RemoteDirectoryUpdater() {} - - virtual void tick() ; -}; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9a8496470..7772af5c6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -3,6 +3,7 @@ #include "file_sharing/p3filelists.h" #include "file_sharing/directory_storage.h" #include "file_sharing/directory_updater.h" +#include "file_sharing/rsfilelistitems.h" #include "retroshare/rsids.h" #include "retroshare/rspeers.h" @@ -25,17 +26,16 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) { // loads existing indexes for friends. Some might be already present here. // - mDirectories.clear() ; // we should load them! - - mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mpeers->getOwnId()) ; - mDirectories.push_back(mLocalSharedDirs) ; + mRemoteDirectories.clear() ; // we should load them! + mOwnId = mpeers->getOwnId() ; + mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId); mHashCache = new HashStorage("hash_cache.bin") ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; - mRemoteDirWatcher = NULL; // not used yet mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; + mLastRemoteDirSweepTS = 0 ; } void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) @@ -60,14 +60,13 @@ p3FileDatabase::~p3FileDatabase() { RS_STACK_MUTEX(mFLSMtx) ; - for(uint32_t i=0;iroot()) ; + + mLastRemoteDirSweepTS = now; + } + return 0; } @@ -153,19 +163,12 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { } -void p3FileDatabase::tickRecv() -{ -} -void p3FileDatabase::tickSend() -{ - // go through the list of out requests and send them to the corresponding friends, if they are online. -} - bool p3FileDatabase::loadList(std::list& items) { // This loads // // - list of locally shared directories, and the permissions that go with them + NOT_IMPLEMENTED(); return true ; @@ -194,19 +197,19 @@ void p3FileDatabase::cleanup() friend_set.insert(*it) ; } - for(uint32_t i=1;ipeerId()) == friend_set.end()) + for(uint32_t i=0;ipeerId()) == friend_set.end()) { - P3FILELISTS_DEBUG() << " removing file list of non friend " << mDirectories[i]->peerId() << std::endl; + P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; - delete mDirectories[i]; - mDirectories[i] = NULL ; + delete mRemoteDirectories[i]; + mRemoteDirectories[i] = NULL ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; - friend_set.erase(mDirectories[i]->peerId()); + friend_set.erase(mRemoteDirectories[i]->peerId()); - mFriendIndexMap.erase(mDirectories[i]->peerId()); + mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); mFriendIndexTab[i].clear(); } @@ -218,12 +221,12 @@ void p3FileDatabase::cleanup() uint32_t friend_index = locked_getFriendIndex(*it) ; - if(mDirectories.size() > friend_index && mDirectories[friend_index] != NULL) + if(mRemoteDirectories.size() > friend_index && mRemoteDirectories[friend_index] != NULL) continue ; P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; - mDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } @@ -244,7 +247,7 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) { // allocate a new index for that friend, and tell that we should save. uint32_t found = 0 ; - for(uint32_t i=1;isecond) - mDirectories.resize(it->second+1,NULL) ; + if(mRemoteDirectories.size() <= it->second) + mRemoteDirectories.resize(it->second,NULL) ; return it->second; } @@ -372,14 +375,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags stub.ref = p; d.children.push_back(stub); } - else for(uint32_t i=1;iroot(),i,p); + convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i,p); DirStub stub; stub.type = DIR_TYPE_PERSON; - stub.name = mDirectories[i]->peerId().toStdString(); + stub.name = mRemoteDirectories[i]->peerId().toStdString(); stub.ref = p; d.children.push_back(stub); } @@ -396,7 +399,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // Case where the index is the top of a single person. Can be us, or a friend. - bool res = mDirectories[fi]->extractData(e,d) ; + bool res = (flags & RS_FILE_HINTS_LOCAL)? (mLocalSharedDirs->extractData(e,d)) : (mRemoteDirectories[fi]->extractData(e,d)) ; // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -414,11 +417,14 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags } else { - d.prow = mDirectories[fi]->parentRow(e) ; + d.prow = mRemoteDirectories[fi]->parentRow(e) ; convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } - d.id = mDirectories[fi]->peerId(); + if(flags & RS_FILE_HINTS_LOCAL) + d.id = mOwnId ; + else + d.id = mRemoteDirectories[fi]->peerId(); std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; @@ -451,7 +457,7 @@ uint32_t p3FileDatabase::getType(void *ref) const if(e == 0) return DIR_TYPE_PERSON ; - return mDirectories[fi]->getEntryType(e) ; + return mRemoteDirectories[fi]->getEntryType(e) ; } void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed @@ -536,13 +542,14 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F if(res.empty()) return false; - EntryIndex indx = *res.begin() ; // no need to report dupicates + EntryIndex indx = *res.begin() ; // no need to report duplicates mLocalSharedDirs->getFileInfo(indx,info) ; return true; } - else + + if(hintflags & RS_FILE_HINTS_REMOTE) { NOT_IMPLEMENTED(); return false; @@ -596,3 +603,114 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& RS_STACK_MUTEX(mFLSMtx) ; return mLocalSharedDirs->convertSharedFilePath(path,fullpath) ; } + +//==============================================================================================================================// +// Update of remote directories // +//==============================================================================================================================// + +void p3FileDatabase::tickRecv() +{ + RsItem *item ; + + while( NULL != (item = recvItem()) ) + { + switch(item->PacketSubType()) + { + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast(item) ) ; + break ; + case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: handleDirSyncContent( dynamic_cast(item) ) ; + break ; + default: + std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; + } + + delete item ; + } +} + +void p3FileDatabase::tickSend() +{ + // go through the list of out requests and send them to the corresponding friends, if they are online. +} + +void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) +{ + // look at item TS. If local is newer, send the full directory content. + + time_t recurs_max_modf_TS; + + if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl; + return ; + } + + if(item->known_recurs_last_modf_TS < recurs_last_modf_TS) + { + // send full update of directory content + } + else + { + // send last recurs update TS. + } +} + +void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item) +{ + // update the directory content for the specified friend. + + // set the update TS, and the remote modif TS accordingly + + // notify the GUI if the hierarchy has changed +} + +void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e) +{ + time_t now = time(NULL) ; + + // get the info for this entry + + // compare TS + + // if not up to date, request update, and return (content is not certified, so no need to recurs yet). + // if up to date, return, because TS is about the last modif TS below, so no need to recurs either. + + time_t recurs_max_modf_TS,last_update_TS ; + + if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl; + return ; + } + + if(now > last_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) + { + // check if a request already exists and is not too old either: no need to re-ask. + + DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),e) ; + + std::map::iterator it = mPendingSyncRequests.find(sync_req_id) ; + + if(it != mPendingSyncRequests.end()) + { + std::cerr << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; + return ; + } + + std::cerr << "Asking for sync of directory " << e << " because it's " << (now - last_update_TS) << " secs old since last check." << std::endl; + + DirSyncRequestData data ; + + data.request_TS = now ; + + mPendingSyncRequests[sync_req_id] = data ; + + RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + item->entry_index = e ; + item->known_TS = recurs_max_modf_TS ; + + sendItem(item) ; + + return ; + } +} diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 0fa36088e..8160095aa 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -37,6 +37,9 @@ class LocalDirectoryUpdater ; class RemoteDirectoryStorage ; class LocalDirectoryStorage ; +class RsFileListsSyncReqItem ; +class RsFileListsSyncDirItem ; + class HashStorage ; class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService @@ -135,22 +138,26 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub private: p3ServiceControl *mServCtrl ; + RsPeerId mOwnId ; + + typedef uint64_t DirSyncRequestId ; + + static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) ; // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. // - std::list mFastRequestQueue ; - std::list mSlowRequestQueue ; + std::map mFastRequestQueue ; + std::map mSlowRequestQueue ; // Directory storage hierarchies // // The remote one is the reference for the PeerId index below: // RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid) - std::vector mDirectories ; // mDirectories[0]=mLocalSharedDirs + std::vector mRemoteDirectories ; LocalDirectoryStorage *mLocalSharedDirs ; - RemoteDirectoryUpdater *mRemoteDirWatcher ; // not used yet. LocalDirectoryUpdater *mLocalDirWatcher ; // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc @@ -160,9 +167,17 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub uint32_t locked_getFriendIndex(const RsPeerId& pid); const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; + void handleDirSyncRequest(RsFileListsSyncReqItem *) ; + void handleDirSyncContent(RsFileListsSyncDirItem *) ; + std::map mFriendIndexMap ; std::vector mFriendIndexTab; + // TS for friend list update + time_t mLastRemoteDirSweepTS ; + + void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); + // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // HashStorage *mHashCache ; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc new file mode 100644 index 000000000..986b4284c --- /dev/null +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -0,0 +1,336 @@ +#include "file_sharing/rsfilelistitems.h" + +RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialise()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE)) + return NULL; /* wrong type */ + + switch(getRsItemSubType(rstype)) + { + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncReqItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: return deserialFileListsSyncDirItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM: return deserialFileListsConfigItem (data, size); + + default: + { + std::cerr << "(WW) RsFileListsSerialiser::deserialise() : unhandled item type " << getRsItemSubType(rstype) << std::endl; + return NULL; + + } + } +} + +uint32_t RsFileListsSerialiser::size(RsItem *item) +{ + RsFileListsItem *flst_item = dynamic_cast(item) ; + + if(flst_item != NULL) + return flst_item->serial_size() ; + else + { + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + return 0; + } +} + +bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size) +{ + RsFileListsItem *flst_item = dynamic_cast(item) ; + + if(flst_item != NULL) + return flst_item->serialise(data,*size) ; + else + { + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + return 0; + } +} + +bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const +{ + tlvsize = serial_size() ; + offset = 0; + + if (pktsize < tlvsize) + return false; /* not enough space */ + + pktsize = tlvsize; + + if(!setRsItemHeader(data, tlvsize, PacketId(), tlvsize)) + { + std::cerr << "RsFileTransferItem::serialise_header(): ERROR. Not enough size!" << std::endl; + return false ; + } +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl; +#endif + offset += 8; + + return true ; +} + +bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const +{ + uint32_t tlvsize,offset=0; + bool ok = true; + + if(!serialise_header(data,size,tlvsize,offset)) + return false ; + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl; +#endif + + /* RsFileListsSyncMsgItem */ + + ok &= setRawUInt32(data, size, &offset, entry_index); + ok &= setRawUInt32(data, size, &offset, flags ); + ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); + ok &= setRawUInt64(data, size, &offset, request_id); + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl; + } +#endif + + return ok; +} + +bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const +{ + uint32_t tlvsize,offset=0; + bool ok = true; + + if(!serialise_header(data,size,tlvsize,offset)) + return false ; + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl; +#endif + + /* RsFileListsSyncMsgItem */ + + ok &= setRawUInt32(data, size, &offset, entry_index); + ok &= setRawUInt32(data, size, &offset, flags ); + ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); + ok &= setRawUInt64(data, size, &offset, request_id); + ok &= directory_content_data.SetTlv(data,size,&offset) ; + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl; + } +#endif + + return ok; +} + +//============================================================================================================================// +// Deserialisation // +//============================================================================================================================// + +RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void *data, uint32_t *size) +{ + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + uint32_t offset = 8; + + RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + + ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= getRawUInt32(data, *size, &offset, &item->flags); + ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); + ok &= getRawUInt64(data, *size, &offset, &item->request_id); + + if (offset != *size) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} +RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +{ + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + uint32_t offset = 8; + + RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + + uint32_t entry_index ; // index of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash + + ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= getRawUInt32(data, *size, &offset, &item->flags); + ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); + ok &= getRawUInt64(data, *size, &offset, &item->request_id); + + ok &= item->directory_content_data.GetTlv(data,*size,&offset) ; + + if (offset != *size) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} +bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t subservice_type) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl; +#endif + return false; /* wrong type */ + } + + if (*size < rssize) /* check size */ + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong size" << std::endl; +#endif + return false; /* not enough data */ + } + + /* set the packet length */ + *size = rssize; + + return true ; +} + +//============================================================================================================================// +// Sizes // +//============================================================================================================================// + +uint32_t RsFileListsSyncReqItem::serial_size()const +{ + + uint32_t s = 8; //header size + + entry_index ; + flags; + last_known_recurs_modf_TS; + request_id; + + s += 4; // entry index + s += 4; // flags + s += 4; // last_known_recurs_modf_TS + s += 8; // request_id + + return s; +} + +uint32_t RsFileListsSyncReqItem::serial_size()const +{ + + uint32_t s = 8; //header size + + entry_index ; + flags; + last_known_recurs_modf_TS; + request_id; + + s += 4; // entry index + s += 4; // flags + s += 4; // last_known_recurs_modf_TS + s += 8; // request_id + s += directory_content_data.TlvSize(); + + return s; +} + +void RsFileListsSyncReqItem::clear() +{ +} +void RsFileListsSyncDirItem::clear() +{ + directory_content_data.TlvClear(); +} +std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsFileListsSyncReqItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; + printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; + printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; + + printRsItemEnd(out ,"RsFileListsSyncReqItem", indent); + + return out; +} + +std::ostream& RsFileListsSyncDirItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsFileListsSyncDirItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; + printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; + printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; + printIndent(out , int_Indent); out << "Data size: " << directory_content_data.bin_len << std::endl; + + printRsItemEnd(out ,"RsFileListsSyncDirItem", indent); + + return out; +} diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h new file mode 100644 index 000000000..6eb463b7d --- /dev/null +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -0,0 +1,132 @@ +#pragma once + +/* + * libretroshare/src/serialiser: rsnxssitems.h + * + * RetroShare Serialiser. + * + * Copyright 2012 Christopher Evi-Parker, Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include +#include + +#include "serialiser/rsserviceids.h" +#include "serialiser/rsserial.h" +#include "serialiser/rstlvbase.h" +#include "serialiser/rstlvitem.h" +#include "serialiser/rstlvkeys.h" +#include "gxs/rsgxsdata.h" + +// These items have "flag type" numbers, but this is not used. + +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM = 0x02; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03; + +/*! + * Base class for filelist sync items + */ +class RsFileListsItem : public RsItem +{ +public: + RsFileListsItem(uint8_t subtype) + : RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE, subtype) + { + setPriorityLevel(QOS_PRIORITY_RS_SLOW_SYNC_REQUEST); // this is the default. Someitems may be faster, on demand. + return; + } + virtual ~RsFileListsItem(){} + + virtual bool serialise(void *data,uint32_t& size) const = 0 ; + virtual uint32_t serial_size() const = 0 ; + virtual void clear() = 0; + virtual std::ostream &print(std::ostream &out, uint16_t indent = 0) = 0; + + bool serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const; + + static const uint32_t FLAGS_SYNC_REQUEST = 0x0001 ; + static const uint32_t FLAGS_SYNC_RESPONSE = 0x0002 ; + static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0004 ; + static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0008 ; +}; + +/*! + * Use to request synchronization on a specific directory. Also used to respond that the directory is up to date. + */ +class RsFileListsSyncReqItem : public RsFileListsItem +{ +public: + + RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; + + uint32_t entry_index ; // index of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash +}; + +class RsFileListsSyncDirItem : public RsFileListsItem +{ +public: + + RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; + + uint32_t entry_index ; // advises whether to use sync hash + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash + + RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. +}; + +class RsFileListsSerialiser : public RsSerialType +{ +public: + + RsFileListsSerialiser(uint16_t servtype) : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {} + + virtual ~RsFileListsSerialiser() {} + + virtual uint32_t size(RsItem *item); + virtual bool serialise(RsItem *item, void *data, uint32_t *size); + virtual RsItem* deserialise(void *data, uint32_t *size); + +private: + RsFileListsSyncReqItem *deserialFileListsSyncReqItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncDirItem *deserialFileListsSyncDirItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncDirItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + + bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type); +}; + + diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 70994b8c7..ea7344976 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -37,16 +37,18 @@ CONFIG += file_lists file_lists { HEADERS *= file_sharing/p3filelists.h \ - file_sharing/hash_cache.h \ - file_sharing/filelist_io.h \ - file_sharing/directory_storage.h \ - file_sharing/directory_updater.h + file_sharing/hash_cache.h \ + file_sharing/filelist_io.h \ + file_sharing/directory_storage.h \ + file_sharing/directory_updater.h \ + file_sharing/filelistitems.h SOURCES *= file_sharing/p3filelists.cc \ - file_sharing/hash_cache.cc \ - file_sharing/filelist_io.cc \ - file_sharing/directory_storage.cc \ - file_sharing/directory_updater.cc + file_sharing/hash_cache.cc \ + file_sharing/filelist_io.cc \ + file_sharing/directory_storage.cc \ + file_sharing/directory_updater.cc \ + file_sharing/filelistitems.cc } dsdv { diff --git a/libretroshare/src/serialiser/itempriorities.h b/libretroshare/src/serialiser/itempriorities.h index 8809754a9..f5008eef0 100644 --- a/libretroshare/src/serialiser/itempriorities.h +++ b/libretroshare/src/serialiser/itempriorities.h @@ -76,6 +76,11 @@ const uint8_t QOS_PRIORITY_RS_DISC_PGP_LIST = 2 ; // same priority. const uint8_t QOS_PRIORITY_RS_DISC_SERVICES = 2 ; const uint8_t QOS_PRIORITY_RS_DISC_PGP_CERT = 1 ; +// File database +// +const uint8_t QOS_PRIORITY_RS_FAST_SYNC_REQUEST = 7 ; +const uint8_t QOS_PRIORITY_RS_SLOW_SYNC_REQUEST = 3 ; + // Heartbeat. // const uint8_t QOS_PRIORITY_RS_HEARTBEAT_PULSE = 8 ; From 76831348e63452dcec59d8c57cc39d49d92adc5a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 22 Aug 2016 07:49:45 +0200 Subject: [PATCH 030/174] added update of last modif LS, and rsitems to update directories between friends --- .../src/file_sharing/directory_storage.cc | 40 +++++++++++++++-- .../src/file_sharing/directory_storage.h | 1 + .../src/file_sharing/directory_updater.cc | 31 +++++++++---- .../src/file_sharing/directory_updater.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 45 ++++++++++++------- libretroshare/src/file_sharing/p3filelists.h | 12 ++++- .../src/file_sharing/rsfilelistitems.cc | 40 +++++++---------- .../src/file_sharing/rsfilelistitems.h | 22 ++++----- libretroshare/src/libretroshare.pro | 4 +- 9 files changed, 129 insertions(+), 67 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 7e8cea686..8ea522b6f 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -53,7 +53,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -65,7 +65,8 @@ class InternalFileHierarchyStorage std::vector subdirs ; std::vector subfiles ; - time_t most_recent_time; + time_t dir_modtime ; // this accounts for deleted files, etc. + time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. }; // class stuff @@ -221,7 +222,10 @@ class InternalFileHierarchyStorage std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; - static_cast(mNodes[file_index])->file_hash = hash ; + RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; + + old_hash = hash ; + return true; } bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) @@ -243,6 +247,25 @@ class InternalFileHierarchyStorage return true; } + // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. + + time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index) + { + DirEntry& d(*static_cast(mNodes[dir_index])) ; + + time_t largest_modf_time = d.dir_modtime ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; + + for(uint32_t i=0;i(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", subdirs: " ; + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", subdirs: " ; for(int i=0;irecursUpdateLastModfTime(EntryIndex(0)) ; + + std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; +} + std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const { /**** MUST ALREADY BE LOCKED ****/ diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 637a8e88e..fbc6b3297 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -137,6 +137,7 @@ public: void updateShareFlags(const SharedDirInfo& info) ; bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ; + void updateTimeStamps(); /*! * \brief getFileInfo Converts an index info a full file info structure. * \param i index in the directory structure diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 731d92902..f9fada262 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -10,13 +10,15 @@ // Local Directory Updater // //=============================================================================================================// -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) { mLastSweepTime = 0; + mLastTSUpdateTime = 0; } void LocalDirectoryUpdater::data_tick() @@ -28,8 +30,13 @@ void LocalDirectoryUpdater::data_tick() sweepSharedDirectories() ; mLastSweepTime = now; } - else - usleep(10*1000*1000); + + if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime) + { + mSharedDirectories->updateTimeStamps() ; + mLastTSUpdateTime = now ; + } + usleep(10*1000*1000); } void LocalDirectoryUpdater::forceUpdate() @@ -93,13 +100,13 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p switch(dirIt.file_type()) { case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; - subfiles[dirIt.file_name()].size = dirIt.file_size(); - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; + subfiles[dirIt.file_name()].size = dirIt.file_size(); + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; - std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; - break; + case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; + break; default: std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } @@ -139,3 +146,9 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; } + + + + + + diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 1248aeefb..94b9a2080 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -35,5 +35,6 @@ private: LocalDirectoryStorage *mSharedDirectories ; time_t mLastSweepTime; + time_t mLastTSUpdateTime; }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 7772af5c6..5ef9af42a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -17,9 +17,10 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; -static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; -static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; -static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. +static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; +static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; +static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so. p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") @@ -608,6 +609,17 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // Update of remote directories // //==============================================================================================================================// +// Algorithm: +// +// Local dirs store the last modif time of the file, in local time +// - the max time is computed upward until the root of the hierarchy +// - because the hash is performed late, the last modf time upward is updated only when the hash is obtained. +// +// Remote dirs store the last modif time of the files/dir in the friend's time +// - local node sends the last known modf time to friends, +// - friends respond with either a full directory content, or an acknowledge that the time is right +// + void p3FileDatabase::tickRecv() { RsItem *item ; @@ -637,15 +649,17 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) { // look at item TS. If local is newer, send the full directory content. - time_t recurs_max_modf_TS; + time_t recurs_max_modf_TS, last_update_TS; - if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) - { - std::cerr << "(EE) Cannot get update TS for entry " << index << " in local dir, asked by " << item->PeerId() << std::endl; - return ; - } + P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - if(item->known_recurs_last_modf_TS < recurs_last_modf_TS) + if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) + { + std::cerr << "(EE) Cannot get update TS for entry " << item->entry_index << " in local dir, asked by " << item->PeerId() << std::endl; + return ; + } + + if(item->last_known_recurs_modf_TS < recurs_max_modf_TS) { // send full update of directory content } @@ -655,7 +669,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) } } -void p3FileDatabase::handleDirSyncContent(RsFileListsSyncContentItem *item) +void p3FileDatabase::handleDirSyncContent(RsFileListsSyncDirItem *item) { // update the directory content for the specified friend. @@ -677,9 +691,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r time_t recurs_max_modf_TS,last_update_TS ; - if(!rds->getUpdateTS(e,time_t& recurs_max_modf_TS,time_t& last_update_TS)) + if(!rds->getUpdateTS(e,recurs_max_modf_TS,last_update_TS)) { - std::cerr << "(EE) Cannot get update TS for entry " << index << " in remote dir from " << rds->peerId() << std::endl; + std::cerr << "(EE) Cannot get update TS for entry " << e << " in remote dir from " << rds->peerId() << std::endl; return ; } @@ -705,9 +719,10 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r mPendingSyncRequests[sync_req_id] = data ; - RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + RsFileListsSyncReqItem *item = new RsFileListsSyncReqItem ; item->entry_index = e ; - item->known_TS = recurs_max_modf_TS ; + item->last_known_recurs_modf_TS = recurs_max_modf_TS ; + item->PeerId(rds->peerId()) ; sendItem(item) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 8160095aa..58a635d8d 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -173,8 +173,16 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub std::map mFriendIndexMap ; std::vector mFriendIndexTab; - // TS for friend list update - time_t mLastRemoteDirSweepTS ; + // Directory synchronization + // + struct DirSyncRequestData + { + time_t request_TS ; + uint32_t flags ; + }; + + time_t mLastRemoteDirSweepTS ; // TS for friend list update + std::map mPendingSyncRequests ; // pending requests, waiting for an answer void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 986b4284c..198bcc9db 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -1,3 +1,5 @@ +#include "serialiser/rsbaseserial.h" + #include "file_sharing/rsfilelistitems.h" RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) @@ -8,8 +10,8 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) /* get the type and size */ uint32_t rstype = getRsItemId(data); - if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != RS_SERVICE_TYPE_FILE_DATABASE)) - return NULL; /* wrong type */ + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (getRsItemService(rstype) != RS_SERVICE_TYPE_FILE_DATABASE)) + return NULL; /* wrong type */ switch(getRsItemSubType(rstype)) { @@ -28,26 +30,26 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) uint32_t RsFileListsSerialiser::size(RsItem *item) { - RsFileListsItem *flst_item = dynamic_cast(item) ; + RsFileListsItem *flst_item = dynamic_cast(item) ; - if(flst_item != NULL) - return flst_item->serial_size() ; + if(flst_item != NULL) + return flst_item->serial_size() ; else { - std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; return 0; } } bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size) { - RsFileListsItem *flst_item = dynamic_cast(item) ; + RsFileListsItem *flst_item = dynamic_cast(item) ; - if(flst_item != NULL) - return flst_item->serialise(data,*size) ; + if(flst_item != NULL) + return flst_item->serialise(data,*size) ; else { - std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; + std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl; return 0; } } @@ -185,12 +187,12 @@ RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void return item; } -RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +RsFileListsSyncDirItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) { bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); uint32_t offset = 8; - RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + RsFileListsSyncDirItem* item = new RsFileListsSyncDirItem(); uint32_t entry_index ; // index of the directory to sync uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. @@ -234,7 +236,7 @@ bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t su uint32_t rstype = getRsItemId(data); uint32_t rssize = getRsItemSize(data); - if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_FILE_DATABASE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype))) { #ifdef RSSERIAL_DEBUG std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl; @@ -265,11 +267,6 @@ uint32_t RsFileListsSyncReqItem::serial_size()const uint32_t s = 8; //header size - entry_index ; - flags; - last_known_recurs_modf_TS; - request_id; - s += 4; // entry index s += 4; // flags s += 4; // last_known_recurs_modf_TS @@ -278,16 +275,11 @@ uint32_t RsFileListsSyncReqItem::serial_size()const return s; } -uint32_t RsFileListsSyncReqItem::serial_size()const +uint32_t RsFileListsSyncDirItem::serial_size()const { uint32_t s = 8; //header size - entry_index ; - flags; - last_known_recurs_modf_TS; - request_id; - s += 4; // entry index s += 4; // flags s += 4; // last_known_recurs_modf_TS diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 6eb463b7d..5ddd1f063 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -75,7 +75,7 @@ class RsFileListsSyncReqItem : public RsFileListsItem { public: - RsFileListsSyncReqItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + RsFileListsSyncReqItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -93,20 +93,20 @@ class RsFileListsSyncDirItem : public RsFileListsItem { public: - RsFileListsSyncDirItem(uint16_t servtype) : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + RsFileListsSyncDirItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} - virtual void clear(); - virtual std::ostream &print(std::ostream &out, uint16_t indent); + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); - virtual bool serialise(void *data,uint32_t& size) const; - virtual uint32_t serial_size() const ; + virtual bool serialise(void *data,uint32_t& size) const; + virtual uint32_t serial_size() const ; - uint32_t entry_index ; // advises whether to use sync hash - uint32_t flags; // is it a partial/final item (used for large items only) - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + uint32_t entry_index ; // advises whether to use sync hash + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash - RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. + RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. }; class RsFileListsSerialiser : public RsSerialType diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index ea7344976..82b0ea27b 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -41,14 +41,14 @@ file_lists { file_sharing/filelist_io.h \ file_sharing/directory_storage.h \ file_sharing/directory_updater.h \ - file_sharing/filelistitems.h + file_sharing/rsfilelistitems.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ file_sharing/filelist_io.cc \ file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ - file_sharing/filelistitems.cc + file_sharing/rsfilelistitems.cc } dsdv { From 99ea8af015df77f20d4aeb8e908478ba89605f7b Mon Sep 17 00:00:00 2001 From: MrAlice Date: Tue, 23 Aug 2016 21:23:58 +0200 Subject: [PATCH 031/174] added logic for file list item exchange requests --- .../src/file_sharing/directory_storage.h | 4 +- libretroshare/src/file_sharing/p3filelists.cc | 144 +++++++++++++----- libretroshare/src/file_sharing/p3filelists.h | 8 +- .../src/file_sharing/rsfilelistitems.cc | 28 ++-- .../src/file_sharing/rsfilelistitems.h | 21 +-- 5 files changed, 140 insertions(+), 65 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index fbc6b3297..309bff328 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -26,7 +26,9 @@ class DirectoryStorage virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& last_update_TS) ; + bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; + bool setUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; + uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 5ef9af42a..6fa89bd35 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -555,6 +555,7 @@ bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, F NOT_IMPLEMENTED(); return false; } + return false; } int p3FileDatabase::filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const @@ -628,9 +629,9 @@ void p3FileDatabase::tickRecv() { switch(item->PacketSubType()) { - case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast(item) ) ; + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast(item) ) ; break ; - case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: handleDirSyncContent( dynamic_cast(item) ) ; + case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: handleDirSyncResponse( dynamic_cast(item) ) ; break ; default: std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; @@ -645,59 +646,119 @@ void p3FileDatabase::tickSend() // go through the list of out requests and send them to the corresponding friends, if they are online. } -void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncReqItem *item) +void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) { + RsFileListsSyncResponseItem *ritem = new RsFileListsSyncResponseItem; + // look at item TS. If local is newer, send the full directory content. - - time_t recurs_max_modf_TS, last_update_TS; - - P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - - if(!mLocalSharedDirs->getUpdateTS(item->entry_index,recurs_max_modf_TS,last_update_TS)) { - std::cerr << "(EE) Cannot get update TS for entry " << item->entry_index << " in local dir, asked by " << item->PeerId() << std::endl; - return ; + RS_STACK_MUTEX(mFLSMtx) ; + + P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; + + uint32_t entry_type = mLocalSharedDirs->getEntryType(item->entry_index) ; + ritem->PeerId(item->PeerId()) ; + ritem->request_id = item->request_id; + ritem->entry_index = item->entry_index ; + + if(entry_type != DIR_TYPE_DIR) + { + P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory. Answering with proper flags." << std::endl; + + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; + } + else + { + time_t local_recurs_max_time,local_update_time; + mLocalSharedDirs->getUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + + if(item->last_known_recurs_modf_TS < local_recurs_max_time) + { + P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl; + + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT; + ritem->last_known_recurs_modf_TS = local_recurs_max_time; + + mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data) ; + } + else + { + P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl; + + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ; + ritem->last_known_recurs_modf_TS = local_update_time ; + } + } } - if(item->last_known_recurs_modf_TS < recurs_max_modf_TS) - { - // send full update of directory content - } - else - { - // send last recurs update TS. - } + // sends the response. + + sendItem(ritem); } -void p3FileDatabase::handleDirSyncContent(RsFileListsSyncDirItem *item) +void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { - // update the directory content for the specified friend. + // find the correct friend entry - // set the update TS, and the remote modif TS accordingly + uint32_t fi = 0 ; - // notify the GUI if the hierarchy has changed + P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; + { + RS_STACK_MUTEX(mFLSMtx) ; + fi = locked_getFriendIndex(item->PeerId()); + + // make sure we have a remote directory for that friend. + + if(mRemoteDirectories.size() <= fi) + mRemoteDirectories.resize(fi+1,NULL) ; + + if(mRemoteDirectories[fi] == NULL) + mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); + } + + if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) + { + P3FILELISTS_DEBUG() << " removing directory with index " << item->entry_index << " because it does not exist." << std::endl; + mRemoteDirectories[fi]->removeDirectory(item->entry_index); + } + else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) + { + P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; + + mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + } + else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) + { + P3FILELISTS_DEBUG() << " Item contains directory data. Uptating." << std::endl; + + mRemoteDirectories[fi]->deserialiseUpdateEntry(item->entry_index,item->directory_content_data) ; + mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + +#warning should notify the GUI here + // notify the GUI if the hierarchy has changed + } } void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e) { time_t now = time(NULL) ; - // get the info for this entry - - // compare TS - // if not up to date, request update, and return (content is not certified, so no need to recurs yet). // if up to date, return, because TS is about the last modif TS below, so no need to recurs either. - time_t recurs_max_modf_TS,last_update_TS ; + // get the info for this entry - if(!rds->getUpdateTS(e,recurs_max_modf_TS,last_update_TS)) + time_t recurs_max_modf_TS_remote_time,local_update_TS; + + if(!rds->getUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << "(EE) Cannot get update TS for entry " << e << " in remote dir from " << rds->peerId() << std::endl; - return ; + std::cerr << "(EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; + return; } - if(now > last_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) + // compare TS + + if(now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) // we need to compare local times only. We cannot compare local (now) with remote time. { // check if a request already exists and is not too old either: no need to re-ask. @@ -707,11 +768,11 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if(it != mPendingSyncRequests.end()) { - std::cerr << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; + P3FILELISTS_DEBUG() << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; return ; } - std::cerr << "Asking for sync of directory " << e << " because it's " << (now - last_update_TS) << " secs old since last check." << std::endl; + P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; DirSyncRequestData data ; @@ -719,13 +780,24 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r mPendingSyncRequests[sync_req_id] = data ; - RsFileListsSyncReqItem *item = new RsFileListsSyncReqItem ; + RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; item->entry_index = e ; - item->last_known_recurs_modf_TS = recurs_max_modf_TS ; + item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; + item->request_id = sync_req_id ; + item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; item->PeerId(rds->peerId()) ; sendItem(item) ; + // Dont recurs into sub-directories, since we dont know yet were to go. + return ; } + + for(DirectoryStorage::DirIterator it(rds,e);it;++it) + locked_recursSweepRemoteDirectory(rds,*it); } + + + + diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 58a635d8d..a15333ace 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -37,8 +37,8 @@ class LocalDirectoryUpdater ; class RemoteDirectoryStorage ; class LocalDirectoryStorage ; -class RsFileListsSyncReqItem ; -class RsFileListsSyncDirItem ; +class RsFileListsSyncRequestItem ; +class RsFileListsSyncResponseItem ; class HashStorage ; @@ -167,8 +167,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub uint32_t locked_getFriendIndex(const RsPeerId& pid); const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; - void handleDirSyncRequest(RsFileListsSyncReqItem *) ; - void handleDirSyncContent(RsFileListsSyncDirItem *) ; + void handleDirSyncRequest (RsFileListsSyncRequestItem *) ; + void handleDirSyncResponse (RsFileListsSyncResponseItem *) ; std::map mFriendIndexMap ; std::vector mFriendIndexTab; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 198bcc9db..ae2d1dc5f 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -15,8 +15,8 @@ RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size) switch(getRsItemSubType(rstype)) { - case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncReqItem(data, size); - case RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM: return deserialFileListsSyncDirItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncRequestItem(data, size); + case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: return deserialFileListsSyncResponseItem(data, size); case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM: return deserialFileListsConfigItem (data, size); default: @@ -77,7 +77,7 @@ bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tl return true ; } -bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const +bool RsFileListsSyncRequestItem::serialise(void *data, uint32_t& size) const { uint32_t tlvsize,offset=0; bool ok = true; @@ -113,7 +113,7 @@ bool RsFileListsSyncReqItem::serialise(void *data, uint32_t& size) const return ok; } -bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const +bool RsFileListsSyncResponseItem::serialise(void *data, uint32_t& size) const { uint32_t tlvsize,offset=0; bool ok = true; @@ -154,12 +154,12 @@ bool RsFileListsSyncDirItem::serialise(void *data, uint32_t& size) const // Deserialisation // //============================================================================================================================// -RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void *data, uint32_t *size) +RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestItem(void *data, uint32_t *size) { bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); uint32_t offset = 8; - RsFileListsSyncReqItem* item = new RsFileListsSyncReqItem(); + RsFileListsSyncRequestItem* item = new RsFileListsSyncRequestItem(); ok &= getRawUInt32(data, *size, &offset, &item->entry_index); ok &= getRawUInt32(data, *size, &offset, &item->flags); @@ -187,12 +187,12 @@ RsFileListsSyncReqItem* RsFileListsSerialiser::deserialFileListsSyncReqItem(void return item; } -RsFileListsSyncDirItem* RsFileListsSerialiser::deserialFileListsSyncDirItem(void *data, uint32_t *size) +RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncResponseItem(void *data, uint32_t *size) { bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); uint32_t offset = 8; - RsFileListsSyncDirItem* item = new RsFileListsSyncDirItem(); + RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem(); uint32_t entry_index ; // index of the directory to sync uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. @@ -262,7 +262,7 @@ bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t su // Sizes // //============================================================================================================================// -uint32_t RsFileListsSyncReqItem::serial_size()const +uint32_t RsFileListsSyncRequestItem::serial_size()const { uint32_t s = 8; //header size @@ -275,7 +275,7 @@ uint32_t RsFileListsSyncReqItem::serial_size()const return s; } -uint32_t RsFileListsSyncDirItem::serial_size()const +uint32_t RsFileListsSyncResponseItem::serial_size()const { uint32_t s = 8; //header size @@ -289,14 +289,14 @@ uint32_t RsFileListsSyncDirItem::serial_size()const return s; } -void RsFileListsSyncReqItem::clear() +void RsFileListsSyncRequestItem::clear() { } -void RsFileListsSyncDirItem::clear() +void RsFileListsSyncResponseItem::clear() { directory_content_data.TlvClear(); } -std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent) +std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t indent) { printRsItemBase(out, "RsFileListsSyncReqItem", indent); uint16_t int_Indent = indent + 2; @@ -311,7 +311,7 @@ std::ostream& RsFileListsSyncReqItem::print(std::ostream &out, uint16_t indent) return out; } -std::ostream& RsFileListsSyncDirItem::print(std::ostream &out, uint16_t indent) +std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t indent) { printRsItemBase(out, "RsFileListsSyncDirItem", indent); uint16_t int_Indent = indent + 2; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 5ddd1f063..a396574d1 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -38,7 +38,7 @@ // These items have "flag type" numbers, but this is not used. const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01; -const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM = 0x02; +const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM = 0x02; const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03; /*! @@ -64,18 +64,19 @@ public: static const uint32_t FLAGS_SYNC_REQUEST = 0x0001 ; static const uint32_t FLAGS_SYNC_RESPONSE = 0x0002 ; - static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0004 ; - static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0008 ; + static const uint32_t FLAGS_SYNC_DIR_CONTENT = 0x0004 ; + static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0008 ; + static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0010 ; }; /*! * Use to request synchronization on a specific directory. Also used to respond that the directory is up to date. */ -class RsFileListsSyncReqItem : public RsFileListsItem +class RsFileListsSyncRequestItem : public RsFileListsItem { public: - RsFileListsSyncReqItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} + RsFileListsSyncRequestItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -89,11 +90,11 @@ public: uint64_t request_id; // use to determine if changes that have occured since last hash }; -class RsFileListsSyncDirItem : public RsFileListsItem +class RsFileListsSyncResponseItem : public RsFileListsItem { public: - RsFileListsSyncDirItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_DIR_ITEM) {} + RsFileListsSyncResponseItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM) {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -122,9 +123,9 @@ public: virtual RsItem* deserialise(void *data, uint32_t *size); private: - RsFileListsSyncReqItem *deserialFileListsSyncReqItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ - RsFileListsSyncDirItem *deserialFileListsSyncDirItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ - RsFileListsSyncDirItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncRequestItem *deserialFileListsSyncRequestItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncResponseItem *deserialFileListsSyncResponseItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ + RsFileListsSyncResponseItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */ bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type); }; From 746f4d729260aa94bd8673a198e0d3bd98f13ba8 Mon Sep 17 00:00:00 2001 From: Gio Date: Thu, 25 Aug 2016 15:11:58 +0200 Subject: [PATCH 032/174] libretroshare remove unuseful code from RsRawItem --- libretroshare/src/serialiser/rsserial.h | 33 +++++++++---------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/libretroshare/src/serialiser/rsserial.h b/libretroshare/src/serialiser/rsserial.h index 47b0666fc..bb8b84d8c 100644 --- a/libretroshare/src/serialiser/rsserial.h +++ b/libretroshare/src/serialiser/rsserial.h @@ -178,32 +178,21 @@ std::ostream &printIndent(std::ostream &out, uint16_t indent); class RsRawItem: public RsItem { - public: - RsRawItem(uint32_t t, uint32_t size) - :RsItem(t), len(size) - { - data = rs_malloc(len); - } +public: + RsRawItem(uint32_t t, uint32_t size) : RsItem(t), len(size) + { data = rs_malloc(len); } + virtual ~RsRawItem() { free(data); } - virtual ~RsRawItem() - { - if (data) - free(data); - data = NULL; - len = 0; - } + uint32_t getRawLength() { return len; } + void * getRawData() { return data; } - uint32_t getRawLength() { return len; } - void *getRawData() { return data; } + virtual void clear() {} + virtual std::ostream &print(std::ostream &out, uint16_t indent = 0); - virtual void clear() { return; } /* what can it do? */ - virtual std::ostream &print(std::ostream &out, uint16_t indent = 0); - - private: - void *data; - uint32_t len; +private: + void *data; + uint32_t len; }; - #endif /* RS_BASE_SERIALISER_H */ From 5274b8aa97f42df53f0d907cc91cdb9e0b4fedcc Mon Sep 17 00:00:00 2001 From: Gio Date: Fri, 26 Aug 2016 13:10:09 +0200 Subject: [PATCH 033/174] Solve some compiler warnings: usage of memory after free (false positive) unused variable (stacked mutex) malloc parameter type mismatch --- libbitdht/src/udp/udplayer.cc | 68 ++++++++++++++++------------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/libbitdht/src/udp/udplayer.cc b/libbitdht/src/udp/udplayer.cc index db048b857..a0cc2324e 100644 --- a/libbitdht/src/udp/udplayer.cc +++ b/libbitdht/src/udp/udplayer.cc @@ -243,74 +243,66 @@ void UdpLayer::run() /* higher level interface */ void UdpLayer::recv_loop() { - int maxsize = 16000; + size_t maxsize = 16000; void *inbuf = malloc(maxsize); - if(inbuf == NULL) - { - std::cerr << "(EE) Error in memory allocation of size " << maxsize << " in " << __PRETTY_FUNCTION__ << std::endl; - return ; - } - - int status; - struct timeval timeout; + if(inbuf == NULL) + { + std::cerr << "(EE) Error in memory allocation of size " << maxsize + << " in " << __PRETTY_FUNCTION__ << std::endl; + return; + } + + int status; + struct timeval timeout; while(1) { - fd_set rset; - for(;;) + for(;;) { /* check if we need to stop */ bool toStop = false; { - bdStackMutex stack(sockMtx); /********** LOCK MUTEX *********/ + bdStackMutex stack(sockMtx); (void) stack; toStop = stopThread; } - + if (toStop) { #ifdef DEBUG_UDP_LAYER std::cerr << "UdpLayer::recv_loop() stopping thread" << std::endl; #endif - free(inbuf) ; + free(inbuf); stop(); + return; // Avoid compiler warning about usage of inbuf after free } - FD_ZERO(&rset); - FD_SET(sockfd, &rset); - timeout.tv_sec = 0; - timeout.tv_usec = 500000; /* 500 ms timeout */ - status = select(sockfd+1, &rset, NULL, NULL, &timeout); - if (status > 0) - { - break; /* data available, go read it */ - } - else if (status < 0) - { + fd_set rset; + FD_ZERO(&rset); + FD_SET(sockfd, &rset); + timeout.tv_sec = 0; + timeout.tv_usec = 500000; // 500 ms timeout + status = select(sockfd+1, &rset, NULL, NULL, &timeout); + if (status > 0) break; // data available, go read it #ifdef DEBUG_UDP_LAYER - std::cerr << "UdpLayer::recv_loop() Error: " << bdnet_errno() << std::endl; + else if (status < 0) std::cerr << "UdpLayer::recv_loop() Error: " + << bdnet_errno() << std::endl; #endif - } - }; + }; - int nsize = maxsize; + int nsize = static_cast(maxsize); struct sockaddr_in from; if (0 < receiveUdpPacket(inbuf, &nsize, from)) { #ifdef DEBUG_UDP_LAYER - std::cerr << "UdpLayer::readPkt() from : " << from << std::endl; - std::cerr << printPkt(inbuf, nsize); + std::cerr << "UdpLayer::readPkt() from : " << from << std::endl + << printPkt(inbuf, nsize); #endif - // send to reciever. - recv -> recvPkt(inbuf, nsize, from); + recv->recvPkt(inbuf, nsize, from); // pass to reciever. } - else - { #ifdef DEBUG_UDP_LAYER - std::cerr << "UdpLayer::readPkt() not ready" << from; - std::cerr << std::endl; + else std::cerr << "UdpLayer::readPkt() not ready" << from << std::endl; #endif - } } } From d8a24c8c3a5714923d62181c2e9a28bcdb488cf4 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 26 Aug 2016 16:29:02 +0200 Subject: [PATCH 034/174] added serialisation of dir entries for exchange with friends. Fixed a few bugs in row and ref calculation --- .../src/file_sharing/directory_storage.cc | 211 +++++++++++++++++- .../src/file_sharing/directory_storage.h | 30 ++- libretroshare/src/file_sharing/filelist_io.h | 3 + libretroshare/src/file_sharing/p3filelists.cc | 76 +++++-- .../src/file_sharing/rsfilelistitems.cc | 9 +- .../src/file_sharing/rsfilelistitems.h | 2 +- libretroshare/src/rsserver/rstypes.cc | 2 +- 7 files changed, 298 insertions(+), 35 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 8ea522b6f..f63f0df6a 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -2,6 +2,7 @@ #include "util/rsdir.h" #include "util/rsstring.h" #include "directory_storage.h" +#include "filelist_io.h" #define DEBUG_DIRECTORY_STORAGE 1 @@ -53,7 +54,7 @@ class InternalFileHierarchyStorage class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -67,6 +68,8 @@ class InternalFileHierarchyStorage time_t dir_modtime ; // this accounts for deleted files, etc. time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. + + time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. }; // class stuff @@ -246,6 +249,36 @@ class InternalFileHierarchyStorage return true; } + bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + { + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + recurs_max_modf_TS = d.most_recent_time ; + local_update_TS = d.dir_update_time ; + + return true; + } + bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + { + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + d.most_recent_time = recurs_max_modf_TS ; + d.dir_update_time = local_update_TS ; + + return true; + } // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. @@ -340,9 +373,58 @@ class InternalFileHierarchyStorage return false; } - bool check() // checks consistency of storage. + bool check(std::string& error_string) const // checks consistency of storage. { - return true; + // recurs go through all entries, check that all + + std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. + hits[0] = 1 ; // because 0 is never the child of anyone + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + // stamp the kids + const DirEntry& de = *static_cast(mNodes[i]) ; + + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subdirs[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subdirs[j]] = 1; + } + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subfiles[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subfiles[j]] = 1; + } + + } + + for(uint32_t i=0;igetDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; +} +bool DirectoryStorage::setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->setDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; +} bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + locked_check() ; + return res ; } bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map& subfiles,std::map& new_files) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ; + locked_check() ; + return res ; } bool DirectoryStorage::removeDirectory(const EntryIndex& indx) { RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->removeDirectory(indx); + bool res = mFileHierarchy->removeDirectory(indx); + + locked_check(); + return res ; +} + +void DirectoryStorage::locked_check() +{ + std::string error ; + if(!mFileHierarchy->check(error)) + std::cerr << "Check error: " << error << std::endl; } bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time) @@ -773,8 +879,95 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) /* find parent pointer, and row */ - std::cerr << "LocalDirectoryStorage::extractData(): indx=" << indx << " Returning this:" << std::endl; - std::cerr << d << std::endl; - return true; } + +bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); + + if(dir == NULL) + { + std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl; + return false; + } + + unsigned char *section_data = NULL; + uint32_t section_size = 0; + uint32_t section_offset = 0; + + // we need to send: + // - the name of the directory, its TS + // - the index entry for each subdir (the updte TS are exchanged at a higher level) + // - the file info for each subfile + // + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir->most_recent_time)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; + + // serialise number of subdirs and number of subfiles + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subdirs.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subfiles.size() )) return false ; + + // serialise subdirs entry indexes + + for(uint32_t i=0;isubdirs.size();++i) + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subdirs[i] )) return false ; + + // serialise directory subfiles, with info for each of them + + for(uint32_t i=0;isubfiles.size();++i) + { + unsigned char *file_section_data = NULL ; + uint32_t file_section_offset = 0 ; + uint32_t file_section_size = 0; + + const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ; + + if(file == NULL) + { + std::cerr << "(EE) cannot reach file entry " << dir->subfiles[i] << " to get/send file info." << std::endl; + continue ; + } + + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subfiles[i] )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file->file_modtime)) return false ; + + // now write the whole string into a single section in the file + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) return false ; + + free(file_section_data) ; + } + + std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl; + + return true ; +} + +/******************************************************************************************************************/ +/* Remote Directory Storage */ +/******************************************************************************************************************/ + +bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; + + NOT_IMPLEMENTED(); + + return true ; +} + + + + + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 309bff328..dd3353ad1 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -9,7 +9,9 @@ #define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } +class RsTlvBinaryData ; class InternalFileHierarchyStorage ; +class RsTlvBinaryData ; class DirectoryStorage { @@ -26,8 +28,8 @@ class DirectoryStorage virtual int searchHash(const RsFileHash& hash, std::list &results) const ; virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } - bool getUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; - bool setUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; + bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; + bool setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); @@ -109,6 +111,9 @@ class DirectoryStorage void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; + // debug + void locked_check(); + // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. std::string mFileName; @@ -125,6 +130,17 @@ class RemoteDirectoryStorage: public DirectoryStorage public: RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} virtual ~RemoteDirectoryStorage() {} + + /*! + * \brief deserialiseDirEntry + * Loads a serialised directory content coming from a friend. The directory entry needs to exist already, + * as it is created when updating the parent. + * + * \param indx index of the directory to update + * \param bindata binary data to deserialise from + * \return false when the directory cannot be found. + */ + bool deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; }; class LocalDirectoryStorage: public DirectoryStorage @@ -150,6 +166,16 @@ public: virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; + /*! + * \brief serialiseDirEntry + * Produced a serialised directory content listing suitable for export to friends. + * + * \param indx index of the directory to serialise + * \param bindata binary data created by serialisation + * \return false when the directory cannot be found. + */ + bool serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) ; + private: std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 9c5818025..a2ee26916 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -20,6 +20,9 @@ static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; +static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; +static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; +static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; class FileListIO { diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 6fa89bd35..768ce370c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -113,7 +113,7 @@ int p3FileDatabase::tick() last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI -// RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } if(mUpdateFlags) @@ -129,7 +129,7 @@ int p3FileDatabase::tick() mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } - if(mLastRemoteDirSweepTS + 5 < now) + if(mLastRemoteDirSweepTS + 30 < now) { RS_STACK_MUTEX(mFLSMtx) ; @@ -227,6 +227,9 @@ void p3FileDatabase::cleanup() P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; + if(mRemoteDirectories.size() <= friend_index) + mRemoteDirectories.resize(friend_index+1,NULL) ; + mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; @@ -324,7 +327,7 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi // [ 10 bits | 22 bits ] // // This means that the whoel software has the following build-in limitation: - // * 1024 friends + // * 1023 friends // * 4M shared files. uint32_t fe = (uint32_t)e ; @@ -346,8 +349,21 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags { RS_STACK_MUTEX(mFLSMtx) ; + d.children.clear(); + // Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us) - // or at the top of our own list of shred directories, depending on the flags. + // or at the top of our own list of shared directories, depending on the flags. + + // Friend index is used as follows: + // 0 : own id + // 1...n : other friends + // + // entry_index: starts at 0. + // + // The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL. + // So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories. + // + // We should also check the consistency between flags and the content of ref. if (ref == NULL) { @@ -363,7 +379,6 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.age = 0; d.flags.clear() ; d.min_age = 0 ; - d.children.clear(); if(flags & RS_FILE_HINTS_LOCAL) { @@ -379,7 +394,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags else for(uint32_t i=0;iroot(),i,p); + convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i+1,p); DirStub stub; stub.type = DIR_TYPE_PERSON; @@ -390,6 +405,9 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.count = d.children.size(); + std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + std::cerr << d << std::endl; + return true ; } @@ -398,9 +416,17 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags convertPointerToEntryIndex(ref,e,fi); + // check consistency + if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) + { + std::cerr << "remote request on local index or local request on remote index. This should not happen." << std::endl; + return false ; + } + DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); + // Case where the index is the top of a single person. Can be us, or a friend. - bool res = (flags & RS_FILE_HINTS_LOCAL)? (mLocalSharedDirs->extractData(e,d)) : (mRemoteDirectories[fi]->extractData(e,d)) ; + bool res = storage->extractData(e,d); // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -410,22 +436,18 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags for(uint32_t i=0;iparentRow(e) ; + d.prow = storage->parentRow(e) ; convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } - if(flags & RS_FILE_HINTS_LOCAL) - d.id = mOwnId ; - else - d.id = mRemoteDirectories[fi]->peerId(); + d.id = storage->peerId(); std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; @@ -670,7 +692,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) else { time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + mLocalSharedDirs->getDirUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); if(item->last_known_recurs_modf_TS < local_recurs_max_time) { @@ -686,7 +708,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ; - ritem->last_known_recurs_modf_TS = local_update_time ; + ritem->last_known_recurs_modf_TS = local_recurs_max_time ; } } } @@ -725,14 +747,14 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { - P3FILELISTS_DEBUG() << " Item contains directory data. Uptating." << std::endl; + P3FILELISTS_DEBUG() << " Item contains directory data. Updating." << std::endl; - mRemoteDirectories[fi]->deserialiseUpdateEntry(item->entry_index,item->directory_content_data) ; - mRemoteDirectories[fi]->setUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data) ; + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); #warning should notify the GUI here // notify the GUI if the hierarchy has changed @@ -750,7 +772,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r time_t recurs_max_modf_TS_remote_time,local_update_TS; - if(!rds->getUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) + if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { std::cerr << "(EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; @@ -798,6 +820,18 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r locked_recursSweepRemoteDirectory(rds,*it); } +p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) +{ + uint64_t r = e ; + + // This is kind of arbitrary. The important thing is that the same ID needs to be generated for a given (peer_id,e) pair. + + for(uint32_t i=0;i::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; - std::cerr << " Children : " ; for(uint32_t i=0;i Date: Sat, 27 Aug 2016 13:22:29 +0200 Subject: [PATCH 035/174] fixed SEGV due to calling layout change in the wrong place --- libretroshare/src/file_sharing/p3filelists.cc | 11 +++++++++-- retroshare-gui/src/gui/RemoteDirModel.cpp | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 768ce370c..a1b98af65 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -443,7 +443,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags } else { - d.prow = storage->parentRow(e) ; + if(d.parent == 0) // child of root node + d.prow = (flags & RS_FILE_HINTS_LOCAL)?0:(fi-1); + else + d.prow = storage->parentRow(e) ; + convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; } @@ -480,7 +484,10 @@ uint32_t p3FileDatabase::getType(void *ref) const if(e == 0) return DIR_TYPE_PERSON ; - return mRemoteDirectories[fi]->getEntryType(e) ; + if(fi == 0) + return mLocalSharedDirs->getEntryType(e) ; + else + return mRemoteDirectories[fi-1]->getEntryType(e) ; } void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 295ebbf83..9ccbc43ad 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -862,6 +862,7 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const /* Callback from */ void RetroshareDirModel::preMods() { + emit layoutAboutToBeChanged(); #if QT_VERSION < 0x050000 reset(); #else @@ -877,7 +878,7 @@ void RetroshareDirModel::preMods() /* Callback from */ void RetroshareDirModel::postMods() { - emit layoutAboutToBeChanged(); +// emit layoutAboutToBeChanged(); #if QT_VERSION >= 0x040600 beginResetModel(); #endif From 07c7d192a94e812cfc3c478b6cea72998839603a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 27 Aug 2016 14:09:26 +0200 Subject: [PATCH 036/174] added deserialisation and update of remote directories --- .../src/file_sharing/directory_storage.cc | 146 +++++++++++++++++- 1 file changed, 143 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f63f0df6a..228f1b208 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,4 +1,5 @@ #include +#include "serialiser/rstlvbinary.h" #include "util/rsdir.h" #include "util/rsstring.h" #include "directory_storage.h" @@ -24,6 +25,7 @@ template typename std::set::iterator erase_from_set(typename std::se class InternalFileHierarchyStorage { +public: class FileStorageNode { public: @@ -249,6 +251,65 @@ class InternalFileHierarchyStorage return true; } + + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) + { + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl; + return false; + } + DirEntry& d(*static_cast(mNodes[indx])) ; + + d.most_recent_time = most_recent_time; + d.dir_modtime = dir_modtime; + d.dir_update_time = time(NULL); + + d.subfiles = subfiles_array ; + d.subdirs = subdirs_array ; + + // check that all subdirs already exist. If not, create. + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subdirs_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subdirs_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new DirEntry(""); + + ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; + node->row = i ; + } + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subfiles_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subfiles_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new FileEntry("",0,0); + + node->row = subdirs_array.size()+i ; + } + + // we should also update the row of each subfile and each subdir + } + bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) @@ -906,7 +967,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir->most_recent_time)) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles @@ -958,11 +1019,90 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) { - RS_STACK_MUTEX(mDirStorageMtx) ; + const unsigned char *section_data = (unsigned char*)bindata.bin_data ; + uint32_t section_size = bindata.bin_len ; + uint32_t section_offset ; std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; - NOT_IMPLEMENTED(); + std::string dir_name ; + time_t most_recent_time ,dir_modtime ; + + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir_name )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) return false ; + + // serialise number of subdirs and number of subfiles + + uint32_t n_subdirs,n_subfiles ; + + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles )) return false ; + + // serialise subdirs entry indexes + + std::vector subdirs_array ; + uint32_t subdir_index ; + + for(uint32_t i=0;i subfiles_array ; + std::vector subfiles_name ; + std::vector subfiles_size ; + std::vector subfiles_hash ; + std::vector subfiles_modtime ; + + for(uint32_t i=0;iupdateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_array,subfiles_array)) + { + std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl; + return false ; + } + + // then update the subfiles + for(uint32_t i=0;iupdateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) + std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; return true ; } From eca9ca0e4dc0c102fb868a0b112c9dacb2b28ca3 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 27 Aug 2016 16:38:15 +0200 Subject: [PATCH 037/174] fixed a few bugs in serialisation of shared dir content --- .../src/file_sharing/directory_storage.cc | 25 +++--- libretroshare/src/file_sharing/filelist_io.h | 6 +- libretroshare/src/file_sharing/p3filelists.cc | 85 ++++++++++++++++--- libretroshare/src/file_sharing/p3filelists.h | 3 +- .../src/file_sharing/rsfilelistitems.cc | 2 +- .../src/file_sharing/rsfilelistitems.h | 2 +- 6 files changed, 89 insertions(+), 34 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 228f1b208..8c54d9e6f 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -391,10 +391,8 @@ public: } uint32_t getType(DirectoryStorage::EntryIndex indx) const { - if(checkIndex(indx,FileStorageNode::TYPE_FILE)) - return FileStorageNode::TYPE_FILE; - else if(checkIndex(indx,FileStorageNode::TYPE_DIR)) - return FileStorageNode::TYPE_DIR; + if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) + return mNodes[indx]->type() ; else return FileStorageNode::TYPE_UNKNOWN; } @@ -751,17 +749,14 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.children.clear() ; time_t now = time(NULL) ; - std::cerr << "LocalDirectoryStorage::extractData(). Index=" << indx << std::endl; - - const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + uint32_t type = mFileHierarchy->getType(indx) ; d.ref = (void*)(intptr_t)indx ; - if (dir_entry != NULL) /* has children --- fill */ + if (type == InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR) /* has children --- fill */ { -#ifdef DEBUG_DIRECTORY_STORAGE - std::cerr << "FileIndex::extractData() ref=dir" << std::endl; -#endif + const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ; + /* extract all the entries */ for(DirectoryStorage::DirIterator it(this,indx);it;++it) @@ -798,12 +793,10 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.name = mPeerId.toStdString(); } } - else + else if(type == InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE) { const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ; -#ifdef DEBUG_DIRECTORY_STORAGE - std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; -#endif + d.type = DIR_TYPE_FILE; d.count = file_entry->file_size; d.min_age = now - file_entry->file_modtime ; @@ -819,6 +812,8 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) else d.path = "" ; } + else + return false; d.flags.clear() ; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index a2ee26916..a984f2cd4 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -24,6 +24,8 @@ static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; +static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) + class FileListIO { public: @@ -68,9 +70,9 @@ private: static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S) { - if(offset + S > buff_size) + if(offset + S + SECTION_HEADER_MAX_SIZE > buff_size) { - buff = (unsigned char *)realloc(buff,offset + S) ; + buff = (unsigned char *)realloc(buff,offset + S + SECTION_HEADER_MAX_SIZE) ; buff_size = offset + S ; if(!buff) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index a1b98af65..218c6bed2 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -21,6 +21,7 @@ static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so. +static const uint32_t DELAY_BEFORE_DROP_REQUEST = 55 ; // every 55 secs, for debugging. Should be evey 10 minutes or so. p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") @@ -37,8 +38,18 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; + + addSerialType(new RsFileListsSerialiser()) ; } +RsSerialiser *p3FileDatabase::setupSerialiser() +{ + RsSerialiser *rss = new RsSerialiser ; + rss->addSerialType(new RsFileListsSerialiser()) ; + rss->addSerialType(new RsGeneralConfigSerialiser()); + + return rss ; +} void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) { RS_STACK_MUTEX(mFLSMtx) ; @@ -97,13 +108,22 @@ int p3FileDatabase::tick() tickRecv() ; tickSend() ; - // cleanup + time_t now = time(NULL) ; + + // cleanup // - remove/delete shared file lists for people who are not friend anymore // - - cleanup(); + + static time_t last_cleanup_time = 0; + +#warning we should use members here, not static + if(last_cleanup_time + 5 < now) + { + cleanup(); + last_cleanup_time = now ; + } static time_t last_print_time = 0; - time_t now = time(NULL) ; if(last_print_time + 20 < now) { @@ -133,8 +153,12 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; + std::set online_peers ; + mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ; + for(uint32_t i=0;iroot()) ; + if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) + locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; mLastRemoteDirSweepTS = now; } @@ -186,6 +210,8 @@ void p3FileDatabase::cleanup() { RS_STACK_MUTEX(mFLSMtx) ; + std::cerr << "p3FileDatabase::cleanup()" << std::endl; + // look through the list of friend directories. Remove those who are not our friends anymore. // std::set friend_set ; @@ -234,6 +260,29 @@ void p3FileDatabase::cleanup() mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } + + // cancel existing requests for which the peer is offline + + std::set online_peers ; + mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ; + + time_t now = time(NULL); + + for(std::map::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();) + if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now) + { + std::cerr << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; + + std::map::iterator tmp(it); + ++tmp; + mPendingSyncRequests.erase(it) ; + it = tmp; + } + else + { + std::cerr << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; + ++it ; + } } } @@ -405,8 +454,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.count = d.children.size(); - std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; - std::cerr << d << std::endl; +#ifdef DEBUG_FILE_HIERARCHY + std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + std::cerr << d << std::endl; +#endif return true ; } @@ -453,8 +504,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.id = storage->peerId(); +#ifdef DEBUG_FILE_HIERARCHY std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; std::cerr << d << std::endl; +#endif return true; } @@ -801,13 +854,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r return ; } - P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - - DirSyncRequestData data ; - - data.request_TS = now ; - - mPendingSyncRequests[sync_req_id] = data ; + P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; item->entry_index = e ; @@ -816,7 +863,17 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; item->PeerId(rds->peerId()) ; - sendItem(item) ; + DirSyncRequestData data ; + + data.request_TS = now ; + data.peer_id = item->PeerId(); + data.flags = item->flags; + + std::cerr << "Pushing req in pending list with peer id " << data.peer_id << std::endl; + + mPendingSyncRequests[sync_req_id] = data ; + + sendItem(item) ; // at end! Because item is destroyed by the process. // Dont recurs into sub-directories, since we dont know yet were to go. diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index a15333ace..b06335458 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -129,7 +129,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // virtual bool loadList(std::list& items); virtual bool saveList(bool &cleanup, std::list&); - virtual RsSerialiser *setupSerialiser() { return NULL;} + virtual RsSerialiser *setupSerialiser() ; void cleanup(); void tickRecv(); @@ -177,6 +177,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // struct DirSyncRequestData { + RsPeerId peer_id ; time_t request_TS ; uint32_t flags ; }; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 8441148dd..a4d409d44 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -196,7 +196,7 @@ RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestI } RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncResponseItem(void *data, uint32_t *size) { - bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM); + bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM); uint32_t offset = 8; RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem(); diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 24f8ab314..1ff00d57a 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -114,7 +114,7 @@ class RsFileListsSerialiser : public RsSerialType { public: - RsFileListsSerialiser(uint16_t servtype) : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {} + RsFileListsSerialiser() : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {} virtual ~RsFileListsSerialiser() {} From fc5176a652142154f57bab9c26a1ea1849688fdd Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 27 Aug 2016 23:56:23 +0200 Subject: [PATCH 038/174] fixed serialisation/deserialisation of dir content. Fixed exchange of dir data. First working prototype of file list exchange. --- .../src/file_sharing/directory_storage.cc | 34 ++++++++------ libretroshare/src/file_sharing/filelist_io.cc | 21 ++++++++- libretroshare/src/file_sharing/filelist_io.h | 25 +++++------ libretroshare/src/file_sharing/p3filelists.cc | 44 ++++++++++++++++--- 4 files changed, 88 insertions(+), 36 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 8c54d9e6f..20a093b2f 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -248,6 +248,7 @@ public: fe.file_hash = hash; fe.file_size = size; fe.file_modtime = modf_time; + fe.file_name = fname; return true; } @@ -264,6 +265,7 @@ public: d.most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; d.dir_update_time = time(NULL); + d.dir_name = dir_name; d.subfiles = subfiles_array ; d.subdirs = subdirs_array ; @@ -287,6 +289,7 @@ public: ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; node->row = i ; + node->parent_index = indx ; } for(uint32_t i=0;irow = subdirs_array.size()+i ; + node->parent_index = indx ; } - // we should also update the row of each subfile and each subdir + return true; } bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) @@ -531,7 +535,8 @@ private: } DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", subdirs: " ; + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + for(int i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << std::endl; + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; } } @@ -961,18 +966,18 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir->most_recent_time)) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir->dir_modtime )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->most_recent_time)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subdirs.size() )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,dir->subfiles.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subdirs.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ; // serialise subdirs entry indexes for(uint32_t i=0;isubdirs.size();++i) - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subdirs[i] )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)dir->subdirs[i] )) return false ; // serialise directory subfiles, with info for each of them @@ -990,11 +995,11 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary continue ; } - if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,dir->subfiles[i] )) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)dir->subfiles[i] )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ; - if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file->file_modtime)) return false ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)file->file_modtime)) return false ; // now write the whole string into a single section in the file @@ -1005,6 +1010,9 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl; + bindata.bin_data = section_data ; + bindata.bin_len = section_offset ; + return true ; } @@ -1016,12 +1024,12 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs { const unsigned char *section_data = (unsigned char*)bindata.bin_data ; uint32_t section_size = bindata.bin_len ; - uint32_t section_offset ; + uint32_t section_offset=0 ; std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; std::string dir_name ; - time_t most_recent_time ,dir_modtime ; + uint32_t most_recent_time ,dir_modtime ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir_name )) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ; @@ -1068,7 +1076,7 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs std::string entry_name ; uint64_t entry_size ; RsFileHash entry_hash ; - time_t entry_modtime ; + uint32_t entry_modtime ; if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,entry_index )) return false ; if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,entry_name )) return false ; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 3cd153ae8..770478c37 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,11 +1,28 @@ +#include "retroshare/rsids.h" +#include "serialiser/rsbaseserial.h" #include "filelist_io.h" +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const size_t & val) { return setRawUInt64(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const Sha1CheckSum & val) { return val.serialise(buff,size,offset) ; } + +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint32_t & val) { return getRawUInt32(const_cast(buff),size,&offset,&val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,size_t & val) { return getRawUInt64(const_cast(buff),size,&offset,&val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,std::string & val) { return getRawString(const_cast(buff),size,&offset,val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,Sha1CheckSum & val) { return val.deserialise(const_cast(buff),size,offset) ; } + +template<> uint32_t FileListIO::serial_size(const uint32_t & ) { return 4 ; } +template<> uint32_t FileListIO::serial_size(const size_t & ) { return 8 ; } +template<> uint32_t FileListIO::serial_size(const std::string & val) { return getRawStringSize(val) ; } +template<> uint32_t FileListIO::serial_size(const Sha1CheckSum & ) { return Sha1CheckSum::serial_size(); } + bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) { if(!checkSectionSize(buff,buff_size,offset,size)) return false; - if(!writeSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + if(!writeSectionHeader(buff,buff_size,offset,section_tag,size)) return false; memcpy(&buff[offset],val,size) ; @@ -16,7 +33,7 @@ bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32 bool FileListIO::readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) { - if(!readSectionHeader(buff,buff_size,offset,FILE_LIST_IO_TAG_BINARY_DATA,size)) + if(!readSectionHeader(buff,buff_size,offset,check_section_tag,size)) return false; val = (unsigned char *)rs_malloc(size) ; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index a984f2cd4..b22a57cc4 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -32,16 +32,15 @@ public: template static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val) { - if(!checkSectionSize(buff,buff_size,offset,sizeof(T))) + uint32_t s = serial_size(val) ; + + if(!checkSectionSize(buff,buff_size,offset,s)) return false; - if(!writeSectionHeader(buff,buff_size,offset,section_tag,sizeof(T))) + if(!writeSectionHeader(buff,buff_size,offset,section_tag,s)) return false; - memcpy(&buff[offset],reinterpret_cast(&val),sizeof(T)) ; - offset += sizeof(T) ; - - return true; + return serialise(buff,buff_size,offset,val) ; } template @@ -52,18 +51,16 @@ public: if(!readSectionHeader(buff,buff_size,offset,check_section_tag,section_size)) return false; - if(section_size != sizeof(T)) - return false ; - - memcpy(reinterpret_cast(&val),&buff[offset],sizeof(T)) ; - offset += sizeof(T) ; - - return true; + return deserialise(buff,buff_size,offset,val); } static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ; static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ; + template static bool serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const T& val) ; + template static bool deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,T& val) ; + template static uint32_t serial_size(const T& val) ; + private: static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ; static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ; @@ -73,7 +70,7 @@ private: if(offset + S + SECTION_HEADER_MAX_SIZE > buff_size) { buff = (unsigned char *)realloc(buff,offset + S + SECTION_HEADER_MAX_SIZE) ; - buff_size = offset + S ; + buff_size = offset + S + SECTION_HEADER_MAX_SIZE; if(!buff) return false ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 218c6bed2..403233183 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -148,8 +148,8 @@ int p3FileDatabase::tick() mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } - - if(mLastRemoteDirSweepTS + 30 < now) +#warning we need to make sure that one req per directory will not cause to keep re-asking the top level dirs. + if(mLastRemoteDirSweepTS + 5 < now) { RS_STACK_MUTEX(mFLSMtx) ; @@ -158,7 +158,12 @@ int p3FileDatabase::tick() for(uint32_t i=0;ipeerId()) != online_peers.end()) + { + std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; + mRemoteDirectories[i]->print(); + locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; + } mLastRemoteDirSweepTS = now; } @@ -780,15 +785,33 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { + P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; + + // remove the original request from pending list + + { + RS_STACK_MUTEX(mFLSMtx) ; + + std::map::iterator it = mPendingSyncRequests.find(item->request_id) ; + + if(it == mPendingSyncRequests.end()) + { + std::cerr << " request " << std::hex << item->request_id << std::dec << " cannot be found. ERROR!" << std::endl; + return ; + } + mPendingSyncRequests.erase(it) ; + } + // find the correct friend entry uint32_t fi = 0 ; - P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; { RS_STACK_MUTEX(mFLSMtx) ; fi = locked_getFriendIndex(item->PeerId()); + std::cerr << " friend index is " << fi ; + // make sure we have a remote directory for that friend. if(mRemoteDirectories.size() <= fi) @@ -813,11 +836,18 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Item contains directory data. Updating." << std::endl; - mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data) ; - mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + if(mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data)) + { + mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); -#warning should notify the GUI here - // notify the GUI if the hierarchy has changed + // notify the GUI if the hierarchy has changed + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + } + else + std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; + + std::cerr << " new content after update: " << std::endl; + mRemoteDirectories[fi]->print(); } } From b36d0fad4aff1ffa92775a93919d5d9da2c217a6 Mon Sep 17 00:00:00 2001 From: AsamK Date: Sun, 28 Aug 2016 17:50:40 +0200 Subject: [PATCH 039/174] Fix crash when running natively on Wayland --- retroshare-gui/src/idle/idle_platform.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/idle/idle_platform.cpp b/retroshare-gui/src/idle/idle_platform.cpp index 149908741..0e11effd4 100644 --- a/retroshare-gui/src/idle/idle_platform.cpp +++ b/retroshare-gui/src/idle/idle_platform.cpp @@ -74,7 +74,7 @@ bool IdlePlatform::init() int event_base, error_base; #if QT_VERSION >= QT_VERSION_CHECK(5,0,0) - if(XScreenSaverQueryExtension(QX11Info::display(), &event_base, &error_base)) { + if(QX11Info::isPlatformX11() && XScreenSaverQueryExtension(QX11Info::display(), &event_base, &error_base)) { #else if(XScreenSaverQueryExtension(QApplication::desktop()->screen()->x11Info().display(), &event_base, &error_base)) { #endif @@ -89,7 +89,7 @@ int IdlePlatform::secondsIdle() if(!d->ss_info) return 0; #if QT_VERSION >= QT_VERSION_CHECK(5,0,0) - if(!XScreenSaverQueryInfo(QX11Info::display(), QX11Info::appRootWindow(), d->ss_info)) + if(!QX11Info::isPlatformX11() || !XScreenSaverQueryInfo(QX11Info::display(), QX11Info::appRootWindow(), d->ss_info)) #else if(!XScreenSaverQueryInfo(QApplication::desktop()->screen()->x11Info().display(), QX11Info::appRootWindow(), d->ss_info)) #endif From 494d2071c95a4290a91ce6fc2efdc933b9593667 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 28 Aug 2016 18:42:52 +0200 Subject: [PATCH 040/174] fixed update of directory TS, also fixing update of empty dirs at friends --- .../src/file_sharing/directory_storage.cc | 48 +++++++++++++------ .../src/file_sharing/directory_storage.h | 2 +- .../src/file_sharing/directory_updater.cc | 10 ++-- libretroshare/src/file_sharing/p3filelists.cc | 9 +--- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 20a093b2f..e801b4ba8 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -47,10 +47,10 @@ public: virtual ~FileEntry() {} // local stuff + time_t file_modtime; std::string file_name ; uint64_t file_size ; RsFileHash file_hash ; - time_t file_modtime ; }; class DirEntry: public FileStorageNode @@ -68,18 +68,21 @@ public: std::vector subdirs ; std::vector subfiles ; - time_t dir_modtime ; // this accounts for deleted files, etc. + time_t dir_modtime; time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. - time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. }; // class stuff InternalFileHierarchyStorage() : mRoot(0) { - mNodes.push_back(new DirEntry("")) ; - mNodes.back()->row=0; - mNodes.back()->parent_index=0; + DirEntry *de = new DirEntry("") ; + + de->row=0; + de->parent_index=0; + de->dir_modtime=0; + + mNodes.push_back(de) ; } int parentRow(DirectoryStorage::EntryIndex e) @@ -97,14 +100,14 @@ public: return e < mNodes.size() && mNodes[e] != NULL ; } - bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::set& subdirs) + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return false; DirEntry& d(*static_cast(mNodes[indx])) ; - std::set should_create(subdirs); + std::map should_create(subdirs); for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) @@ -122,14 +125,18 @@ public: ++i; } - for(std::set::const_iterator it(should_create.begin());it!=should_create.end();++it) + for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) { - std::cerr << "[directory storage] adding new subdirectory " << *it << " at index " << mNodes.size() << std::endl; + std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; + + DirEntry *de = new DirEntry(it->first) ; + + de->row = mNodes.size(); + de->parent_index = indx; + de->dir_modtime = it->second; d.subdirs.push_back(mNodes.size()) ; - mNodes.push_back(new DirEntry(*it)); - mNodes.back()->row = mNodes.size()-1; - mNodes.back()->parent_index = indx; + mNodes.push_back(de) ; } return true; @@ -664,7 +671,7 @@ bool DirectoryStorage::setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_T return mFileHierarchy->setDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; } -bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) +bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs) { RS_STACK_MUTEX(mDirStorageMtx) ; bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; @@ -1035,6 +1042,10 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) return false ; + std::cerr << " dir name : \"" << dir_name << "\"" << std::endl; + std::cerr << " most recent time : " << most_recent_time << std::endl; + std::cerr << " modification time : " << dir_modtime << std::endl; + // serialise number of subdirs and number of subfiles uint32_t n_subdirs,n_subfiles ; @@ -1042,6 +1053,9 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs )) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles )) return false ; + std::cerr << " number of subdirs : " << n_subdirs << std::endl; + std::cerr << " number of files : " << n_subfiles << std::endl; + // serialise subdirs entry indexes std::vector subdirs_array ; @@ -1095,6 +1109,8 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs RS_STACK_MUTEX(mDirStorageMtx) ; + std::cerr << " updating dir entry..." << std::endl; + // first create the entries for each subdir and each subfile. if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_array,subfiles_array)) { @@ -1104,8 +1120,12 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs // then update the subfiles for(uint32_t i=0;iupdateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; + } return true ; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index dd3353ad1..c86477d19 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -94,7 +94,7 @@ class DirectoryStorage const RsPeerId& peerId() const { return mPeerId ; } int parentRow(EntryIndex e) const ; - bool updateSubDirectoryList(const EntryIndex& indx,const std::set& subdirs) ; + bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index f9fada262..2af4c036f 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -59,10 +59,10 @@ void LocalDirectoryUpdater::sweepSharedDirectories() std::list shared_directory_list ; mSharedDirectories->getSharedDirectoryList(shared_directory_list); - std::set sub_dir_list ; + std::map sub_dir_list ; for(std::list::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it) - sub_dir_list.insert( (*real_dir_it).filename ) ; + sub_dir_list[(*real_dir_it).filename] = 0 ; // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. @@ -93,7 +93,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // collect subdirs and subfiles std::map subfiles ; - std::set subdirs ; + std::map subdirs ; for(;dirIt.isValid();dirIt.next()) { @@ -104,7 +104,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; break; - case librs::util::FolderIterator::TYPE_DIR: subdirs.insert(dirIt.file_name()) ; + case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime(); std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; break; default: @@ -134,7 +134,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; - for(std::set::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(std::map::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) { std::cerr << " recursing into " << stored_dir_it.name() << std::endl; recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 403233183..ea6fce421 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -834,15 +834,10 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { - P3FILELISTS_DEBUG() << " Item contains directory data. Updating." << std::endl; + P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; if(mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data)) - { - mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); - - // notify the GUI if the hierarchy has changed - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - } + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; From 7c2ed3fca04f75ca092b2a3f4057a505cbfe2117 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 28 Aug 2016 23:14:39 +0200 Subject: [PATCH 041/174] save/restore expanded state in file list tree view so that view gets updated seamlessly --- libretroshare/src/file_sharing/p3filelists.cc | 6 ++ retroshare-gui/src/gui/RemoteDirModel.cpp | 73 ++++++++--------- retroshare-gui/src/gui/RemoteDirModel.h | 3 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 78 +++++++++++++++++-- retroshare-gui/src/gui/SharedFilesDialog.h | 7 +- 5 files changed, 118 insertions(+), 49 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ea6fce421..a3ec08e4d 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -129,7 +129,9 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_FILE_HIERARCHY mLocalSharedDirs->print(); +#endif last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI @@ -160,7 +162,9 @@ int p3FileDatabase::tick() if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) { std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; +#ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[i]->print(); +#endif locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; } @@ -842,7 +846,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; std::cerr << " new content after update: " << std::endl; +#ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[fi]->print(); +#endif } } diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 9ccbc43ad..1c18c5062 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -744,6 +744,9 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent /* we can just grab the reference now */ +#ifdef RDM_DEBUG + std::cerr << "Creating index 1 row=" << row << ", column=" << column << ", ref=" << (void*)details.children[row].ref << std::endl; +#endif return createIndex(row, column, details.children[row].ref); } QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const @@ -761,7 +764,10 @@ QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent { void *ref = _ref_entries[row].first ; - return createIndex(row, column, ref); +#ifdef RDM_DEBUG + std::cerr << "Creating index 2 row=" << row << ", column=" << column << ", ref=" << (void*)ref << std::endl; +#endif + return createIndex(row, column, ref); } else return QModelIndex(); @@ -810,6 +816,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const std::cerr << "success index(" << details->prow << ",0," << details->parent << ")"; std::cerr << std::endl; + std::cerr << "Creating index 3 row=" << details.prow << ", column=" << 0 << ", ref=" << (void*)details.parent << std::endl; #endif return createIndex(details.prow, 0, details.parent); } @@ -1155,11 +1162,29 @@ void RetroshareDirModel::openSelected(const QModelIndexList &qmil) #endif } +void RetroshareDirModel::getFilePath(const QModelIndex& index, std::string& fullpath) +{ + void *ref = index.sibling(index.row(),1).internalPointer(); + + DirDetails details ; + + if (!requestDirDetails(ref, false,details) ) + { +#ifdef RDM_DEBUG + std::cerr << "getFilePaths() Bad Request" << std::endl; +#endif + return; + } + + fullpath = details.path + "/" + details.name; +} + void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list &fullpaths) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::getFilePaths()" << std::endl; #endif +#warning make sure we atually output something here if (RemoteMode) { #ifdef RDM_DEBUG @@ -1168,48 +1193,18 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list internalPointer(); + std::string path ; - DirDetails details ; - - if (!requestDirDetails(ref, false,details) ) - { + getFilePath(*it,path) ; #ifdef RDM_DEBUG - std::cerr << "getFilePaths() Bad Request" << std::endl; + std::cerr << "Constructed FilePath: " << path << std::endl; #endif - continue; - } - - if (details.type != DIR_TYPE_FILE) - { -#ifdef RDM_DEBUG - std::cerr << "getFilePaths() Not File" << std::endl; -#endif - continue; /* not file! */ - } - -#ifdef RDM_DEBUG - std::cerr << "::::::::::::File Details:::: " << std::endl; - std::cerr << "Name: " << details.name << std::endl; - std::cerr << "Hash: " << details.hash << std::endl; - std::cerr << "Size: " << details.count << std::endl; - std::cerr << "Path: " << details.path << std::endl; -#endif - - std::string filepath = details.path + "/"; - filepath += details.name; - -#ifdef RDM_DEBUG - std::cerr << "Constructed FilePath: " << filepath << std::endl; -#endif - if (fullpaths.end() == std::find(fullpaths.begin(), fullpaths.end(), filepath)) - { - fullpaths.push_back(filepath); - } - } +#warning TERRIBLE COST here. Use a std::set! + if (fullpaths.end() == std::find(fullpaths.begin(), fullpaths.end(), path)) + fullpaths.push_back(path); + } #ifdef RDM_DEBUG std::cerr << "::::::::::::Done getFilePaths" << std::endl; #endif diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index dd81e2c84..6028b5785 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -72,7 +72,8 @@ class RetroshareDirModel : public QAbstractItemModel void getFileInfoFromIndexList(const QModelIndexList& list, std::list& files_info) ; void openSelected(const QModelIndexList &list); void getFilePaths(const QModelIndexList &list, std::list &fullpaths); - void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; } + void getFilePath(const QModelIndex& index, std::string& fullpath); + void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; } bool requestDirDetails(void *ref, bool remote,DirDetails& d) const; virtual void update() {} diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 67a5e3d0e..a74cfe690 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -19,6 +19,8 @@ * Boston, MA 02110-1301, USA. ****************************************************************/ +#include + #include #include #include @@ -327,8 +329,8 @@ void RemoteSharedFilesDialog::processSettings(bool bLoad) void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) { - //disconnect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), NULL, NULL ); - //disconnect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), NULL, NULL ); +// disconnect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), NULL, NULL ); +// disconnect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), NULL, NULL ); if(model!=NULL) model->setVisible(false) ; @@ -352,8 +354,8 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) model->update() ; } - //connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), model, SLOT( collapsed(const QModelIndex & ) ) ); - //connect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), model, SLOT( expanded(const QModelIndex & ) ) ); +// connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), this, SLOT( collapsed(const QModelIndex & ) ) ); +// connect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), this, SLOT( expanded(const QModelIndex & ) ) ); ui.dirTreeView->setModel(proxyModel); ui.dirTreeView->update(); @@ -363,6 +365,7 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) ui.dirTreeView->header()->headerDataChanged(Qt::Horizontal, COLUMN_NAME, COLUMN_DIR) ; +// recursRestoreExpandedItems(ui.dirTreeView->rootIndex(),expanded_indexes); FilterItems(); } @@ -823,20 +826,79 @@ void SharedFilesDialog::preModDirectories(bool local) flat_model->preMods(); } +void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexes) +{ + for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) + { + std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); + recursSaveExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); + } +} + +void SharedFilesDialog::restoreExpandedPaths(const std::set& expanded_indexes) +{ + std::cerr << "Restoring expanded items. " << std::endl; + for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) + { + std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); + recursRestoreExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); + } +} + +void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp) +{ + std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); + std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; + + if(ui.dirTreeView->isExpanded(index)) + { + std::cerr << "Index " << local_path << " is expanded." << std::endl; + if(index.isValid()) + exp.insert(local_path) ; + + for(int row=0;rowmodel()->rowCount(index);++row) + recursSaveExpandedItems(index.child(row,0),local_path,exp) ; + } + else + std::cerr << "Index is not expanded." << std::endl; +} + +void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set& exp) +{ + std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); + std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; + + if(exp.find(local_path) != exp.end()) + { + std::cerr << "re expanding index " << local_path << std::endl; + ui.dirTreeView->setExpanded(index,true) ; + + for(int row=0;rowmodel()->rowCount(index);++row) + recursRestoreExpandedItems(index.child(row,0),local_path,exp) ; + } +} + + void SharedFilesDialog::postModDirectories(bool local) { if (isRemote() == local) { - return; - } + return; + } + std::set expanded_indexes; + saveExpandedPaths(expanded_indexes) ; + std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; - /* Notify both models, only one is visible */ + /* Notify both models, only one is visible */ tree_model->postMods(); flat_model->postMods(); ui.dirTreeView->update() ; - if (ui.filterPatternLineEdit->text().isEmpty() == false) + restoreExpandedPaths(expanded_indexes) ; + + if (ui.filterPatternLineEdit->text().isEmpty() == false) FilterItems(); + std::cerr << "****** updated directories! ******" << std::endl; QCoreApplication::flush(); } diff --git a/retroshare-gui/src/gui/SharedFilesDialog.h b/retroshare-gui/src/gui/SharedFilesDialog.h index c0258c190..9b7387224 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.h +++ b/retroshare-gui/src/gui/SharedFilesDialog.h @@ -22,6 +22,7 @@ #ifndef _SHAREDFILESDIALOG_H #define _SHAREDFILESDIALOG_H +#include #include "RsAutoUpdatePage.h" #include "ui_SharedFilesDialog.h" @@ -51,7 +52,6 @@ protected slots: virtual void spawnCustomPopupMenu(QPoint point) = 0; private slots: - /* For handling the model updates */ void preModDirectories(bool local) ; void postModDirectories(bool local) ; @@ -94,6 +94,11 @@ protected: Ui::SharedFilesDialog ui; virtual void processSettings(bool bLoad) = 0; + void recursRestoreExpandedItems(const QModelIndex& index,const std::string& path,const std::set& exp); + void recursSaveExpandedItems(const QModelIndex& index, const std::string &path, std::set &exp); + void saveExpandedPaths(std::set& paths) ; + void restoreExpandedPaths(const std::set& paths) ; + protected: //now context menu are created again every time theu are called ( in some //slots.. Maybe it's not good... From 1290aa84031ec1e8b2a55815adaaa05e746f14ef Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 29 Aug 2016 21:30:56 +0200 Subject: [PATCH 042/174] fixed proper display of file sharing flags --- .../src/file_sharing/directory_storage.cc | 37 +++++++++++++++++-- libretroshare/src/file_sharing/hash_cache.cc | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 10 +++-- retroshare-gui/src/gui/SharedFilesDialog.cpp | 30 ++++++++++++--- 4 files changed, 65 insertions(+), 14 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index e801b4ba8..f5b90f8d6 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -207,8 +207,7 @@ public: f.file_modtime = it->second.modtime; f.file_size = it->second.size; } - else - new_files.erase(f.file_name) ; + new_files.erase(f.file_name) ; ++i; } @@ -881,7 +880,7 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) { RS_STACK_MUTEX(mDirStorageMtx) ; - std::map::iterator it = mLocalDirs.find(info.virtualname) ; + std::map::iterator it = mLocalDirs.find(info.filename) ; if(it == mLocalDirs.end()) { @@ -944,8 +943,38 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) // here we should update the file sharing flags d.flags.clear() ; + d.parent_groups.clear(); - /* find parent pointer, and row */ + std::string base_dir; + + for(DirectoryStorage::EntryIndex i=((d.type==DIR_TYPE_FILE)?((intptr_t)d.parent):((intptr_t)d.ref));;) + { + const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ; + + if(e == NULL) + break ; + + if(e->parent_index == 0) + { + base_dir = e->dir_name ; + break ; + } + i = e->parent_index ; + } + + if(!base_dir.empty()) + { + std::map::const_iterator it = mLocalDirs.find(base_dir) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl; + return false ; + } +#warning we should use a NodeGroupId here + d.flags = it->second.shareflags; + d.parent_groups = it->second.parent_groups; + } return true; } diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 1e7cf0fe7..d8e65f810 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -121,7 +121,7 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t return true ; } #ifdef HASHSTORAGE_DEBUG - std::cerr << "Not in cache. Sceduling for re-hash." << std::endl ; + std::cerr << "Not in cache. Scheduling for re-hash." << std::endl ; #endif // we need to schedule a re-hashing diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index a3ec08e4d..e164a87d0 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -64,8 +64,12 @@ void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) } void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) { - RS_STACK_MUTEX(mFLSMtx) ; - mLocalSharedDirs->updateShareFlags(info) ; + { + RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->updateShareFlags(info) ; + } + + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } p3FileDatabase::~p3FileDatabase() @@ -135,7 +139,7 @@ int p3FileDatabase::tick() last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); } if(mUpdateFlags) diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index a74cfe690..cc578cb8d 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -250,9 +250,14 @@ void SharedFilesDialog::showEvent(QShowEvent *) { if(model!=NULL) { - model->setVisible(true) ; + std::set expanded_indexes ; + saveExpandedPaths(expanded_indexes); + + model->setVisible(true) ; model->update() ; - } + + restoreExpandedPaths(expanded_indexes); + } } RemoteSharedFilesDialog::~RemoteSharedFilesDialog() { @@ -348,9 +353,13 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) showProperColumns() ; - if(isVisible()) + std::set expanded_indexes ; + saveExpandedPaths(expanded_indexes); + + if(isVisible()) { - model->setVisible(true) ; + model->setVisible(true) ; + model->update() ; } @@ -360,7 +369,9 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) ui.dirTreeView->setModel(proxyModel); ui.dirTreeView->update(); - QHeaderView * header = ui.dirTreeView->header () ; + restoreExpandedPaths(expanded_indexes); + + QHeaderView * header = ui.dirTreeView->header () ; QHeaderView_setSectionResizeModeColumn(header, COLUMN_NAME, QHeaderView::Interactive); ui.dirTreeView->header()->headerDataChanged(Qt::Horizontal, COLUMN_NAME, COLUMN_DIR) ; @@ -828,6 +839,10 @@ void SharedFilesDialog::preModDirectories(bool local) void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexes) { + if(ui.dirTreeView->model() == NULL) + return ; + + std::cerr << "Saving expanded items. " << std::endl; for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); @@ -837,6 +852,9 @@ void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexe void SharedFilesDialog::restoreExpandedPaths(const std::set& expanded_indexes) { + if(ui.dirTreeView->model() == NULL) + return ; + std::cerr << "Restoring expanded items. " << std::endl; for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { @@ -886,7 +904,7 @@ void SharedFilesDialog::postModDirectories(bool local) } std::set expanded_indexes; saveExpandedPaths(expanded_indexes) ; - std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; + std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; /* Notify both models, only one is visible */ tree_model->postMods(); From 4b00d47d3a5e31cf2e7b41b0ee67e751c08d8f3d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 30 Aug 2016 21:28:32 +0200 Subject: [PATCH 043/174] added permission check for friends to see shared files or not --- .../src/file_sharing/directory_storage.cc | 32 +++++++++++++++---- .../src/file_sharing/directory_storage.h | 12 +++++++ libretroshare/src/file_sharing/p3filelists.cc | 11 +++++-- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f5b90f8d6..60662e701 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -385,6 +385,14 @@ public: // Low level stuff. Should normally not be used externally. + const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const + { + if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) + return mNodes[indx] ; + else + return NULL ; + } + const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) @@ -450,7 +458,7 @@ public: hits[0] = 1 ; // because 0 is never the child of anyone for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR) { // stamp the kids const DirEntry& de = *static_cast(mNodes[i]) ; @@ -942,12 +950,24 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) // here we should update the file sharing flags - d.flags.clear() ; - d.parent_groups.clear(); + return getFileSharingPermissions(indx,d.flags,d.parent_groups) ; +} + +bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + flags.clear() ; + parent_groups.clear(); std::string base_dir; - for(DirectoryStorage::EntryIndex i=((d.type==DIR_TYPE_FILE)?((intptr_t)d.parent):((intptr_t)d.ref));;) + const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ; + + if(n == NULL) + return false ; + + for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;) { const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ; @@ -972,8 +992,8 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return false ; } #warning we should use a NodeGroupId here - d.flags = it->second.shareflags; - d.parent_groups = it->second.parent_groups; + flags = it->second.shareflags; + parent_groups = it->second.parent_groups; } return true; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index c86477d19..4f5a48f38 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -164,6 +164,18 @@ public: */ bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ; + /*! + * \brief getFileSharingPermissions + * Computes the flags and parent groups for any index. + * \param indx index of the entry to compute the flags for + * \param flags computed flags + * \param parent_groups computed parent groups + * \return + * false if the index is not valid + * false otherwise + */ + bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; /*! diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index e164a87d0..0d634854d 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -756,10 +756,17 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->request_id = item->request_id; ritem->entry_index = item->entry_index ; + std::list node_groups; + FileStorageFlags node_flags; + if(entry_type != DIR_TYPE_DIR) { - P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory. Answering with proper flags." << std::endl; - + P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl; + ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; + } + else if(item->entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(item->entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) + { + std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)item->entry_index << ", or permission denied." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else From 51c98fdf5020ca4f594c22660819e45a876ee349 Mon Sep 17 00:00:00 2001 From: sehraf Date: Tue, 30 Aug 2016 21:50:20 +0200 Subject: [PATCH 044/174] add stats to libresapi --- libresapi/src/api/ApiServer.cpp | 7 +++++- libresapi/src/api/StatsHandler.cpp | 39 ++++++++++++++++++++++++++++++ libresapi/src/api/StatsHandler.h | 25 +++++++++++++++++++ libresapi/src/libresapi.pro | 6 +++-- 4 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 libresapi/src/api/StatsHandler.cpp create mode 100644 libresapi/src/api/StatsHandler.h diff --git a/libresapi/src/api/ApiServer.cpp b/libresapi/src/api/ApiServer.cpp index 34d1cfc5c..b9606a003 100644 --- a/libresapi/src/api/ApiServer.cpp +++ b/libresapi/src/api/ApiServer.cpp @@ -15,6 +15,7 @@ #include "ApiPluginHandler.h" #include "ChannelsHandler.h" +#include "StatsHandler.h" /* data types in json http://json.org/ @@ -234,7 +235,8 @@ public: mTransfersHandler(sts, ifaces.mFiles), mChatHandler(sts, ifaces.mNotify, ifaces.mMsgs, ifaces.mPeers, ifaces.mIdentity, &mPeersHandler), mApiPluginHandler(sts, ifaces), - mChannelsHandler(ifaces.mGxsChannels) + mChannelsHandler(ifaces.mGxsChannels), + mStatsHandler() { // the dynamic cast is to not confuse the addResourceHandler template like this: // addResourceHandler(derived class, parent class) @@ -258,6 +260,8 @@ public: &ChatHandler::handleRequest); router.addResourceHandler("channels", dynamic_cast(&mChannelsHandler), &ChannelsHandler::handleRequest); + router.addResourceHandler("stats", dynamic_cast(&mStatsHandler), + &StatsHandler::handleRequest); } PeersHandler mPeersHandler; @@ -269,6 +273,7 @@ public: ChatHandler mChatHandler; ApiPluginHandler mApiPluginHandler; ChannelsHandler mChannelsHandler; + StatsHandler mStatsHandler; }; ApiServer::ApiServer(): diff --git a/libresapi/src/api/StatsHandler.cpp b/libresapi/src/api/StatsHandler.cpp new file mode 100644 index 000000000..cd904ffa6 --- /dev/null +++ b/libresapi/src/api/StatsHandler.cpp @@ -0,0 +1,39 @@ +#include "StatsHandler.h" +#include "Operators.h" + +#include +#include +#include + +namespace resource_api +{ + +StatsHandler::StatsHandler() +{ + addResourceHandler("*", this, &StatsHandler::handleStatsRequest); +} + +void StatsHandler::handleStatsRequest(Request &/*req*/, Response &resp) +{ + StreamBase& itemStream = resp.mDataStream.getStreamToMember(); + + // location info + itemStream << makeKeyValue("name", rsPeers->getGPGName(rsPeers->getGPGOwnId())); + itemStream << makeKeyValue("location", AuthSSL::getAuthSSL()->getOwnLocation()); + + // peer info + unsigned int all, online; + rsPeers->getPeerCount(&all, &online, false); + itemStream << makeKeyValue("peers_all", all); + itemStream << makeKeyValue("peers_connected", online); + + // bandwidth info + float downKb, upKb; + rsConfig->GetCurrentDataRates(downKb, upKb); + itemStream << makeKeyValue("bandwidth_up_kb", (double)upKb); + itemStream << makeKeyValue("bandwidth_down_kb", (double)downKb); + + resp.setOk(); +} + +} // namespace resource_api diff --git a/libresapi/src/api/StatsHandler.h b/libresapi/src/api/StatsHandler.h new file mode 100644 index 000000000..cbe8ae231 --- /dev/null +++ b/libresapi/src/api/StatsHandler.h @@ -0,0 +1,25 @@ +#ifndef STATSHANDLER_H +#define STATSHANDLER_H + +/* + * simple class to output some basic stats about RS + * like bandwidth, connected peers, ... + */ + +#include "ResourceRouter.h" + +namespace resource_api +{ + +class StatsHandler : public ResourceRouter +{ +public: + StatsHandler(); + +private: + void handleStatsRequest(Request& req, Response& resp); +}; + +} // namespace resource_api + +#endif // STATSHANDLER_H diff --git a/libresapi/src/libresapi.pro b/libresapi/src/libresapi.pro index 14faff570..b76b2a30b 100644 --- a/libresapi/src/libresapi.pro +++ b/libresapi/src/libresapi.pro @@ -146,7 +146,8 @@ SOURCES += \ api/TmpBlobStore.cpp \ util/ContentTypes.cpp \ api/ApiPluginHandler.cpp \ - api/ChannelsHandler.cpp + api/ChannelsHandler.cpp \ + api/StatsHandler.cpp HEADERS += \ api/ApiServer.h \ @@ -172,7 +173,8 @@ HEADERS += \ api/TmpBlobStore.h \ util/ContentTypes.h \ api/ApiPluginHandler.h \ - api/ChannelsHandler.h + api/ChannelsHandler.h \ + api/StatsHandler.h libresapilocalserver { CONFIG *= qt From e4bd7f66a0d0e4bdabc729f0b28968a9b0599d10 Mon Sep 17 00:00:00 2001 From: sehraf Date: Tue, 30 Aug 2016 21:58:07 +0200 Subject: [PATCH 045/174] add dht/nat to librespai stats --- libresapi/src/api/StatsHandler.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libresapi/src/api/StatsHandler.cpp b/libresapi/src/api/StatsHandler.cpp index cd904ffa6..46bf600d4 100644 --- a/libresapi/src/api/StatsHandler.cpp +++ b/libresapi/src/api/StatsHandler.cpp @@ -1,8 +1,8 @@ #include "StatsHandler.h" #include "Operators.h" -#include #include +#include #include namespace resource_api @@ -33,6 +33,17 @@ void StatsHandler::handleStatsRequest(Request &/*req*/, Response &resp) itemStream << makeKeyValue("bandwidth_up_kb", (double)upKb); itemStream << makeKeyValue("bandwidth_down_kb", (double)downKb); + // DHT/NAT info + RsConfigNetStatus config; + rsConfig->getConfigNetStatus(config); + itemStream << makeKeyValue("dht_active", config.DHTActive); + itemStream << makeKeyValue("dht_ok", config.netDhtOk); + itemStream << makeKeyValue("dht_size_all", config.netDhtNetSize); + itemStream << makeKeyValue("dht_size_rs", config.netDhtRsNetSize); + uint32_t netState = rsConfig -> getNetState(); + itemStream << makeKeyValue("nat_state", netState); + + // ok resp.setOk(); } From 6a30e44756933031b7b39f8d410c234d00c8492a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 30 Aug 2016 22:30:58 +0200 Subject: [PATCH 046/174] fixed bug in removal of directory --- .../src/file_sharing/directory_storage.cc | 26 ++++++++++++++++--- libretroshare/src/file_sharing/p3filelists.cc | 2 ++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 60662e701..518546c9d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -141,7 +141,7 @@ public: return true; } - bool removeDirectory(const DirectoryStorage::EntryIndex& indx) + bool removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! { // check that it's a directory @@ -151,6 +151,10 @@ public: if(indx == 0) return nodeAccessError("checkIndex(): Cannot remove top level directory") ; +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "(--) Removing directory " << indx << std::endl; + print(); +#endif // remove from parent DirEntry& d(*static_cast(mNodes[indx])) ; @@ -158,8 +162,19 @@ public: for(uint32_t i=0;ientry_index << " because it does not exist." << std::endl; mRemoteDirectories[fi]->removeDirectory(item->entry_index); + + mRemoteDirectories[fi]->print(); } else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) { From 3ec50837d07bb7a5dee725acf230d4e21dbd2fce Mon Sep 17 00:00:00 2001 From: Gio Date: Wed, 31 Aug 2016 07:41:22 +0200 Subject: [PATCH 047/174] Fix typos in qmake files --- libretroshare/src/libretroshare.pro | 2 +- retroshare.pri | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 6499243b8..39c9512e9 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -276,7 +276,7 @@ mac { DEFINES *= MINIUPNPC_VERSION=13 CONFIG += upnp_miniupnpc - CONFIG += c+11 + CONFIG += c++11 # zeroconf disabled at the end of libretroshare.pro (but need the code) #CONFIG += zeroconf diff --git a/retroshare.pri b/retroshare.pri index fe7d77df2..5d8572807 100644 --- a/retroshare.pri +++ b/retroshare.pri @@ -84,7 +84,7 @@ macx { message(***retroshare.pri: No SDK, set it to MacOS 10.10 ) QMAKE_MAC_SDK = macosx10.10 } - CONFIG += c+11 + CONFIG += c++11 } unfinished { From 1f3ea6fa0fec6d5078525e52e3a23d1d37d5dddf Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 31 Aug 2016 22:47:05 +0200 Subject: [PATCH 048/174] added selection of subdirs to be seen for friends depending on permissions. Enabled dynamic update at friends --- .../src/file_sharing/directory_storage.cc | 58 +++++++++++++++++-- .../src/file_sharing/directory_storage.h | 10 ++-- libretroshare/src/file_sharing/p3filelists.cc | 3 +- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 518546c9d..a3e0f15d7 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,5 +1,6 @@ #include #include "serialiser/rstlvbinary.h" +#include "retroshare/rspeers.h" #include "util/rsdir.h" #include "util/rsstring.h" #include "directory_storage.h" @@ -99,6 +100,14 @@ public: { return e < mNodes.size() && mNodes[e] != NULL ; } + bool stampDirectory(const DirectoryStorage::EntryIndex& indx) + { + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + static_cast(mNodes[indx])->dir_modtime = time(NULL) ; + return true; + } bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) { @@ -904,6 +913,18 @@ void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst lst.push_back(it->second) ; } +static bool sameLists(const std::list& l1,const std::list& l2) +{ + std::list::const_iterator it1(l1.begin()) ; + std::list::const_iterator it2(l2.begin()) ; + + for(; (it1!=l1.end() && it2!=l2.end());++it1,++it2) + if(*it1 != *it2) + return false ; + + return it1 == l1.end() && it2 == l2.end() ; +} + void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) { RS_STACK_MUTEX(mDirStorageMtx) ; @@ -915,7 +936,17 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; return ; } - it->second = info; + + // we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just + // want to make sure that if they are different, flags get updated. + + if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname) + { + it->second = info; + mFileHierarchy->stampDirectory(0) ; + + std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; + } } bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) @@ -976,7 +1007,11 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) { RS_STACK_MUTEX(mDirStorageMtx) ; + return locked_getFileSharingPermissions(indx,flags,parent_groups) ; +} +bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +{ flags.clear() ; parent_groups.clear(); @@ -1019,7 +1054,7 @@ bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,Fil return true; } -bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata) +bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id) { RS_STACK_MUTEX(mDirStorageMtx) ; @@ -1031,6 +1066,17 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary return false; } + // compute list of allowed subdirs + std::vector allowed_subdirs ; + FileStorageFlags node_flags ; + std::list node_groups ; + + // for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter + + for(uint32_t i=0;isubdirs.size();++i) + if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) + allowed_subdirs.push_back(dir->subdirs[i]) ; + unsigned char *section_data = NULL; uint32_t section_size = 0; uint32_t section_offset = 0; @@ -1047,13 +1093,13 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // serialise number of subdirs and number of subfiles - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subdirs.size() )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ; // serialise subdirs entry indexes - for(uint32_t i=0;isubdirs.size();++i) - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)dir->subdirs[i] )) return false ; + for(uint32_t i=0;i& parent_groups); std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; std::map mLocalDirs ; // map is better for search. it->first=it->second.filename diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 09049b5d0..8c948fb94 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -781,7 +781,8 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT; ritem->last_known_recurs_modf_TS = local_recurs_max_time; - mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data) ; + // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer. + mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data,item->PeerId()) ; } else { From f191fe675d756e8bee50a22e3d4c4ce2e048bed5 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 1 Sep 2016 20:02:47 +0200 Subject: [PATCH 049/174] fixed compilation with new RsNodeGroupId --- .../src/file_sharing/directory_storage.cc | 14 +++++++------- libretroshare/src/file_sharing/directory_storage.h | 4 ++-- libretroshare/src/file_sharing/p3filelists.cc | 2 +- libretroshare/src/retroshare/rstypes.h | 4 ++-- libretroshare/src/rsserver/rstypes.cc | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index a3e0f15d7..b490038f8 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -913,10 +913,10 @@ void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst lst.push_back(it->second) ; } -static bool sameLists(const std::list& l1,const std::list& l2) +static bool sameLists(const std::list& l1,const std::list& l2) { - std::list::const_iterator it1(l1.begin()) ; - std::list::const_iterator it2(l2.begin()) ; + std::list::const_iterator it1(l1.begin()) ; + std::list::const_iterator it2(l2.begin()) ; for(; (it1!=l1.end() && it2!=l2.end());++it1,++it2) if(*it1 != *it2) @@ -1004,13 +1004,13 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return getFileSharingPermissions(indx,d.flags,d.parent_groups) ; } -bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) { RS_STACK_MUTEX(mDirStorageMtx) ; return locked_getFileSharingPermissions(indx,flags,parent_groups) ; } -bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) +bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list &parent_groups) { flags.clear() ; parent_groups.clear(); @@ -1046,7 +1046,7 @@ bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& i std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl; return false ; } -#warning we should use a NodeGroupId here + flags = it->second.shareflags; parent_groups = it->second.parent_groups; } @@ -1069,7 +1069,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // compute list of allowed subdirs std::vector allowed_subdirs ; FileStorageFlags node_flags ; - std::list node_groups ; + std::list node_groups ; // for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index b82696b2c..0f5311944 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -174,7 +174,7 @@ public: * false if the index is not valid * false otherwise */ - bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list &parent_groups); virtual bool extractData(const EntryIndex& indx,DirDetails& d) ; @@ -190,7 +190,7 @@ public: bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ; private: - bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; std::map mLocalDirs ; // map is better for search. it->first=it->second.filename diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 8c948fb94..21beec912 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -756,7 +756,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->request_id = item->request_id; ritem->entry_index = item->entry_index ; - std::list node_groups; + std::list node_groups; FileStorageFlags node_flags; if(entry_type != DIR_TYPE_DIR) diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 56a9af280..8ca770a76 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -236,13 +236,13 @@ public: RsPeerId id; std::string name; RsFileHash hash; - std::string path; + std::string path; uint64_t count; uint32_t age; FileStorageFlags flags; uint32_t min_age ; // minimum age of files in this subtree - std::list children; + std::vector children; std::list parent_groups; // parent groups for the shared directory }; diff --git a/libretroshare/src/rsserver/rstypes.cc b/libretroshare/src/rsserver/rstypes.cc index f9b7a8471..4254ee258 100644 --- a/libretroshare/src/rsserver/rstypes.cc +++ b/libretroshare/src/rsserver/rstypes.cc @@ -59,7 +59,7 @@ std::ostream &operator<<(std::ostream &out, const DirDetails& d) std::cerr << " Age : " << d.age << std::endl; std::cerr << " Min age : " << d.min_age << std::endl; std::cerr << " Flags : " << d.flags << std::endl; - std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; + std::cerr << " Parent groups : " ; for(std::list::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl; std::cerr << " Children : " ; for(uint32_t i=0;i Date: Thu, 1 Sep 2016 21:04:48 +0200 Subject: [PATCH 050/174] added load/save of configuration and shared dirs list --- .../src/file_sharing/directory_updater.cc | 7 +- .../src/file_sharing/directory_updater.h | 5 + .../src/file_sharing/file_sharing_defaults.h | 9 + libretroshare/src/file_sharing/hash_cache.h | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 160 ++++++++++++++++-- libretroshare/src/libretroshare.pro | 3 +- 6 files changed, 163 insertions(+), 23 deletions(-) create mode 100644 libretroshare/src/file_sharing/file_sharing_defaults.h diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 2af4c036f..adc3e8fff 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -3,6 +3,7 @@ #include "directory_storage.h" #include "directory_updater.h" +#include "file_sharing_defaults.h" #define DEBUG_LOCAL_DIR_UPDATER 1 @@ -10,15 +11,13 @@ // Local Directory Updater // //=============================================================================================================// -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! - LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds) : mHashCache(hc),mSharedDirectories(lds) { mLastSweepTime = 0; mLastTSUpdateTime = 0; + + mDelayBetweenDirectoryUpdates = DELAY_BETWEEN_DIRECTORY_UPDATES; } void LocalDirectoryUpdater::data_tick() diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 94b9a2080..63d6ec260 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -23,6 +23,9 @@ public: virtual void forceUpdate(); + void setFileWatchPeriod(uint32_t seconds) { mDelayBetweenDirectoryUpdates = seconds ; } + uint32_t fileWatchPeriod() const { return mDelayBetweenDirectoryUpdates ; } + protected: virtual void data_tick() ; @@ -36,5 +39,7 @@ private: time_t mLastSweepTime; time_t mLastTSUpdateTime; + + uint32_t mDelayBetweenDirectoryUpdates; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h new file mode 100644 index 000000000..a36e311b0 --- /dev/null +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -0,0 +1,9 @@ +#pragma once + +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! + +static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time +static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 6b2dbf8f3..39d9767fe 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -44,7 +44,7 @@ public: // interaction with GUI, called from p3FileLists void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; } - void clear() { mFiles.clear(); } + void clear() { mFiles.clear(); mChanged=true; } bool empty() const { return mFiles.empty() ; } // Functions called by the thread diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 21beec912..597762958 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -4,6 +4,7 @@ #include "file_sharing/directory_storage.h" #include "file_sharing/directory_updater.h" #include "file_sharing/rsfilelistitems.h" +#include "file_sharing/file_sharing_defaults.h" #include "retroshare/rsids.h" #include "retroshare/rspeers.h" @@ -20,7 +21,6 @@ static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED = 0x0004 ; static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 60 ; // every minute, for debugging. Should be evey 10 minutes or so. static const uint32_t DELAY_BEFORE_DROP_REQUEST = 55 ; // every 55 secs, for debugging. Should be evey 10 minutes or so. p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) @@ -39,14 +39,19 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; + // This is for the transmission of data + addSerialType(new RsFileListsSerialiser()) ; } RsSerialiser *p3FileDatabase::setupSerialiser() { + // This one is for saveList/loadList + RsSerialiser *rss = new RsSerialiser ; rss->addSerialType(new RsFileListsSerialiser()) ; rss->addSerialType(new RsGeneralConfigSerialiser()); + rss->addSerialType(new RsFileConfigSerialiser()); return rss ; } @@ -201,21 +206,137 @@ void p3FileDatabase::stopThreads() void p3FileDatabase::tickWatchers() { } -bool p3FileDatabase::loadList(std::list& items) + +bool p3FileDatabase::saveList(bool &cleanup, std::list& sList) { - // This loads - // - // - list of locally shared directories, and the permissions that go with them +cleanup = true; - NOT_IMPLEMENTED(); +#ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << "Save list" << std::endl; +#endif - return true ; + /* get list of directories */ + std::list dirList; + { + RS_STACK_MUTEX(mFLSMtx) ; + mLocalSharedDirs->getSharedDirectoryList(dirList); + } + + for(std::list::iterator it = dirList.begin(); it != dirList.end(); ++it) + { + RsFileConfigItem *fi = new RsFileConfigItem(); + + fi->file.path = (*it).filename ; + fi->file.name = (*it).virtualname ; + fi->flags = (*it).shareflags.toUInt32() ; + + for(std::list::const_iterator it2( (*it).parent_groups.begin());it2!=(*it).parent_groups.end();++it2) + fi->parent_groups.ids.insert(*it2) ; + + sList.push_back(fi); + } + + RsConfigKeyValueSet *rskv = new RsConfigKeyValueSet(); + + /* basic control parameters */ + { + RS_STACK_MUTEX(mFLSMtx) ; + std::string s ; + rs_sprintf(s, "%lu", mHashCache->rememberHashFilesDuration()) ; + + RsTlvKeyValue kv; + + kv.key = HASH_CACHE_DURATION_SS; + kv.value = s ; + + rskv->tlvkvs.pairs.push_back(kv); + } + + { + std::string s ; + rs_sprintf(s, "%d", watchPeriod()) ; + + RsTlvKeyValue kv; + + kv.key = WATCH_FILE_DURATION_SS; + kv.value = s ; + + rskv->tlvkvs.pairs.push_back(kv); + } + + /* Add KeyValue to saveList */ + sList.push_back(rskv); + + return true; } -bool p3FileDatabase::saveList(bool &cleanup, std::list&) +bool p3FileDatabase::loadList(std::list& load) { - NOT_IMPLEMENTED(); - return true ; + /* for each item, check it exists .... + * - remove any that are dead (or flag?) + */ + static const FileStorageFlags PERMISSION_MASK = DIR_FLAGS_BROWSABLE_OTHERS | DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_GROUPS | DIR_FLAGS_NETWORK_WIDE_GROUPS ; + +#ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << "Load list" << std::endl; +#endif + + std::list dirList; + + for(std::list::iterator it = load.begin(); it != load.end(); ++it) + { + RsConfigKeyValueSet *rskv ; + + if (NULL != (rskv = dynamic_cast(*it))) + { + /* make into map */ + std::map configMap; + std::map::const_iterator mit ; + + for(std::list::const_iterator kit = rskv->tlvkvs.pairs.begin(); kit != rskv->tlvkvs.pairs.end(); ++kit) + if (kit->key == HASH_CACHE_DURATION_SS) + { + uint32_t t=0 ; + if(sscanf(kit->value.c_str(),"%d",&t) == 1) + mHashCache->setRememberHashFilesDuration(t); + } + else if(kit->key == WATCH_FILE_DURATION_SS) + { + int t=0 ; + if(sscanf(kit->value.c_str(),"%d",&t) == 1) + setWatchPeriod(t); + } + delete *it ; + continue ; + } + + RsFileConfigItem *fi = dynamic_cast(*it); + + if (fi) + { + /* ensure that it exists? */ + + SharedDirInfo info ; + info.filename = RsDirUtil::convertPathToUnix(fi->file.path); + info.virtualname = fi->file.name; + info.shareflags = FileStorageFlags(fi->flags) ; + info.shareflags &= PERMISSION_MASK ; + info.shareflags &= ~DIR_FLAGS_NETWORK_WIDE_GROUPS ; // disabling this flag for know, for consistency reasons + + for(std::set::const_iterator itt(fi->parent_groups.ids.begin());itt!=fi->parent_groups.ids.end();++itt) + info.parent_groups.push_back(*itt) ; + + dirList.push_back(info) ; + } + + delete *it ; + } + + /* set directories */ + mLocalSharedDirs->setSharedDirectoryList(dirList); + load.clear() ; + + return true; } void p3FileDatabase::cleanup() @@ -567,25 +688,30 @@ bool p3FileDatabase::inDirectoryCheck() } void p3FileDatabase::setWatchPeriod(uint32_t seconds) { - NOT_IMPLEMENTED(); + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalDirWatcher->setFileWatchPeriod(seconds); + IndicateConfigChanged(); } uint32_t p3FileDatabase::watchPeriod() { - NOT_IMPLEMENTED(); - return 0; + RS_STACK_MUTEX(mFLSMtx) ; + return mLocalDirWatcher->fileWatchPeriod(); } void p3FileDatabase::setRememberHashCacheDuration(uint32_t days) { - NOT_IMPLEMENTED(); + RS_STACK_MUTEX(mFLSMtx) ; + mHashCache->setRememberHashFilesDuration(days) ; } uint32_t p3FileDatabase::rememberHashCacheDuration() { - NOT_IMPLEMENTED(); - return 0; + RS_STACK_MUTEX(mFLSMtx) ; + return mHashCache->rememberHashFilesDuration() ; } void p3FileDatabase::clearHashCache() { - NOT_IMPLEMENTED(); + RS_STACK_MUTEX(mFLSMtx) ; + mHashCache->clear() ; } bool p3FileDatabase::rememberHashCache() { diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index e6d08a3c6..582f5c508 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -45,7 +45,8 @@ file_lists { file_sharing/filelist_io.h \ file_sharing/directory_storage.h \ file_sharing/directory_updater.h \ - file_sharing/rsfilelistitems.h + file_sharing/rsfilelistitems.h \ + file_sharing/file_sharing_defaults.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ From 52ef7adfae6cc7291040ed1103994792557f85bf Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 1 Sep 2016 21:41:12 +0200 Subject: [PATCH 051/174] fixed saving of config when diretory list is changed --- libretroshare/src/file_sharing/p3filelists.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 597762958..489af067b 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -57,10 +57,14 @@ RsSerialiser *p3FileDatabase::setupSerialiser() } void p3FileDatabase::setSharedDirectories(const std::list& shared_dirs) { - RS_STACK_MUTEX(mFLSMtx) ; + { + RS_STACK_MUTEX(mFLSMtx) ; - mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; - mLocalDirWatcher->forceUpdate(); + mLocalSharedDirs->setSharedDirectoryList(shared_dirs) ; + mLocalDirWatcher->forceUpdate(); + } + + IndicateConfigChanged(); } void p3FileDatabase::getSharedDirectories(std::list& shared_dirs) { @@ -75,6 +79,7 @@ void p3FileDatabase::updateShareFlags(const SharedDirInfo& info) } RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); + IndicateConfigChanged(); } p3FileDatabase::~p3FileDatabase() From 7b566f2c55a4f8f7fe2334254fa82604f9116773 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 2 Sep 2016 21:49:43 +0200 Subject: [PATCH 052/174] added load/save for hash cache in the proper place --- .../src/file_sharing/directory_updater.h | 11 +----- .../src/file_sharing/file_sharing_defaults.h | 4 +++ libretroshare/src/file_sharing/hash_cache.cc | 34 +++++++++++++++---- libretroshare/src/file_sharing/hash_cache.h | 5 +-- libretroshare/src/file_sharing/p3filelists.cc | 17 ++++++++-- libretroshare/src/file_sharing/p3filelists.h | 1 + 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 63d6ec260..3fb52c81e 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -6,16 +6,7 @@ #include "file_sharing/hash_cache.h" #include "file_sharing/directory_storage.h" -class DirectoryUpdater -{ - public: - DirectoryUpdater() {} - virtual ~DirectoryUpdater(){} -}; - -#warning: simplify, if we don't keep the remote directory updater - -class LocalDirectoryUpdater: public DirectoryUpdater, public HashStorageClient, public RsTickingThread +class LocalDirectoryUpdater: public HashStorageClient, public RsTickingThread { public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index a36e311b0..b40672478 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -7,3 +7,7 @@ static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 sec static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files +static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. +static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. + +static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 60 secs. diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index d8e65f810..39bbdc6f3 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -3,6 +3,7 @@ #include "pqi/authssl.h" #include "hash_cache.h" #include "filelist_io.h" +#include "file_sharing_defaults.h" #define HASHSTORAGE_DEBUG 1 @@ -14,8 +15,12 @@ HashStorage::HashStorage(const std::string& save_file_name) { mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; mRunning = false ; + mLastSaveTime = 0 ; - load() ; + { + RS_STACK_MUTEX(mHashMtx) ; + locked_load() ; + } } void HashStorage::data_tick() @@ -48,10 +53,13 @@ void HashStorage::data_tick() mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; - std::cerr << "Stopping hashing thread." << std::endl; - shutdown(); - mRunning = false ; - std::cerr << "done." << std::endl; + if(!mChanged) // otherwise it might prevent from saving the hash cache + { + std::cerr << "Stopping hashing thread." << std::endl; + shutdown(); + mRunning = false ; + std::cerr << "done." << std::endl; + } RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; } @@ -92,11 +100,23 @@ void HashStorage::data_tick() info.modf_stamp = job.ts ; info.time_stamp = time(NULL); info.hash = hash; + + mChanged = true ; } // call the client if(!hash.isNull()) job.client->hash_callback(job.client_param, job.full_path, hash, size); + + { + RS_STACK_MUTEX(mHashMtx) ; + if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL)) + { + locked_save(); + mLastSaveTime = time(NULL) ; + mChanged = false ; + } + } } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -180,7 +200,7 @@ void HashStorage::clean() #endif } -void HashStorage::load() +void HashStorage::locked_load() { uint64_t file_size ; @@ -249,7 +269,7 @@ void HashStorage::load() #endif } -void HashStorage::save() +void HashStorage::locked_save() { #ifdef HASHSTORAGE_DEBUG std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 39d9767fe..6413ce68c 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -55,8 +55,8 @@ public: private: void clean() ; - void save() ; - void load() ; + void locked_save() ; + void locked_load() ; bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const; bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const; @@ -85,5 +85,6 @@ private: RsMutex mHashMtx ; bool mRunning; uint32_t mInactivitySleepTime ; + time_t mLastSaveTime ; }; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 489af067b..f92e7be10 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -8,6 +8,7 @@ #include "retroshare/rsids.h" #include "retroshare/rspeers.h" +#include "rsserver/rsaccounts.h" #include "rsserver/p3face.h" @@ -26,13 +27,25 @@ static const uint32_t DELAY_BEFORE_DROP_REQUEST = 55 ; // every p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) : mServCtrl(mpeers), mFLSMtx("p3FileLists") { - // loads existing indexes for friends. Some might be already present here. + // make sure the base directory exists + + std::string base_dir = rsAccounts->PathAccountDirectory(); + + if(base_dir.empty()) + throw std::runtime_error("Cannot create base directory to store/access file sharing files.") ; + + mFileSharingDir = base_dir + "/" + FILE_SHARING_DIR_NAME ; + + if(!RsDirUtil::checkCreateDirectory(mFileSharingDir)) + throw std::runtime_error("Cannot create base directory to store/access file sharing files.") ; + + // loads existing indexes for friends. Some might be already present here. // mRemoteDirectories.clear() ; // we should load them! mOwnId = mpeers->getOwnId() ; mLocalSharedDirs = new LocalDirectoryStorage("local_file_store.bin",mOwnId); - mHashCache = new HashStorage("hash_cache.bin") ; + mHashCache = new HashStorage(mFileSharingDir + "/" + HASH_CACHE_FILE_NAME) ; mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index b06335458..d4504bb77 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -195,5 +195,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub mutable RsMutex mFLSMtx ; uint32_t mUpdateFlags ; + std::string mFileSharingDir ; }; From 153783acb5d16b45a94728fa10238558bbbd8ff0 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 2 Sep 2016 22:08:27 +0200 Subject: [PATCH 053/174] fixed bug in saving hash cache --- libretroshare/src/file_sharing/hash_cache.cc | 39 ++++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 39bbdc6f3..a0ea48c95 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -33,6 +33,16 @@ void HashStorage::data_tick() bool empty ; uint32_t st ; + { + RS_STACK_MUTEX(mHashMtx) ; + if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL)) + { + locked_save(); + mLastSaveTime = time(NULL) ; + mChanged = false ; + } + } + { RS_STACK_MUTEX(mHashMtx) ; @@ -93,30 +103,24 @@ void HashStorage::data_tick() // store the result - HashStorageInfo& info(mFiles[job.full_path]); + { + RS_STACK_MUTEX(mHashMtx) ; + HashStorageInfo& info(mFiles[job.full_path]); - info.filename = job.full_path ; - info.size = size ; - info.modf_stamp = job.ts ; - info.time_stamp = time(NULL); - info.hash = hash; + info.filename = job.full_path ; + info.size = size ; + info.modf_stamp = job.ts ; + info.time_stamp = time(NULL); + info.hash = hash; - mChanged = true ; + mChanged = true ; + } } // call the client if(!hash.isNull()) job.client->hash_callback(job.client_param, job.full_path, hash, size); - { - RS_STACK_MUTEX(mHashMtx) ; - if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL)) - { - locked_save(); - mLastSaveTime = time(NULL) ; - mChanged = false ; - } - } } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -170,6 +174,8 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t void HashStorage::clean() { + RS_STACK_MUTEX(mHashMtx) ; + #ifdef HASHSTORAGE_DEBUG std::cerr << "Cleaning HashStorage..." << std::endl ; #endif @@ -311,6 +317,7 @@ void HashStorage::locked_save() #ifdef FIM_DEBUG std::cerr << "done." << std::endl ; #endif + std::cerr << mFiles.size() << " Entries saved." << std::endl; save_free: free(encryptedData); From 371171fcbb66adecaadd7ffd5e33410595714b35 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 3 Sep 2016 13:29:23 +0200 Subject: [PATCH 054/174] moved InternalFileHierarchyStorage to separate file. Fixed loading/saving infrastructure (not working yet) --- .../src/file_sharing/dir_hierarchy.cc | 556 +++++++++++++++ .../src/file_sharing/dir_hierarchy.h | 119 ++++ .../src/file_sharing/directory_storage.cc | 651 +----------------- .../src/file_sharing/directory_storage.h | 27 +- .../src/file_sharing/file_sharing_defaults.h | 3 +- libretroshare/src/file_sharing/p3filelists.cc | 17 +- libretroshare/src/file_sharing/p3filelists.h | 1 + libretroshare/src/libretroshare.pro | 2 + 8 files changed, 743 insertions(+), 633 deletions(-) create mode 100644 libretroshare/src/file_sharing/dir_hierarchy.cc create mode 100644 libretroshare/src/file_sharing/dir_hierarchy.h diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc new file mode 100644 index 000000000..74db29e1b --- /dev/null +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -0,0 +1,556 @@ +#include "dir_hierarchy.h" + +#define DEBUG_DIRECTORY_STORAGE 1 + +/******************************************************************************************************************/ +/* Internal File Hierarchy Storage */ +/******************************************************************************************************************/ + +template typename std::set::iterator erase_from_set(typename std::set& s,const typename std::set::iterator& it) +{ + typename std::set::iterator tmp(it); + ++tmp; + s.erase(it) ; + return tmp; +} + +// This class handles the file hierarchy +// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed, +// so that the hierarchy stays completely coherent between calls. + +InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0) +{ + DirEntry *de = new DirEntry("") ; + + de->row=0; + de->parent_index=0; + de->dir_modtime=0; + + mNodes.push_back(de) ; +} + +int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) +{ + if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) + return -1 ; + + return mNodes[mNodes[e]->parent_index]->row; +} + +// high level modification routines + +bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e) const +{ + return e < mNodes.size() && mNodes[e] != NULL ; +} +bool InternalFileHierarchyStorage::stampDirectory(const DirectoryStorage::EntryIndex& indx) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + static_cast(mNodes[indx])->dir_modtime = time(NULL) ; + return true; +} + +bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + DirEntry& d(*static_cast(mNodes[indx])) ; + + std::map should_create(subdirs); + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) + { + std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + + if( !removeDirectory(d.subdirs[i])) + i++ ; + } + else + { + std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; + + should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; + ++i; + } + + for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) + { + std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; + + DirEntry *de = new DirEntry(it->first) ; + + de->row = mNodes.size(); + de->parent_index = indx; + de->dir_modtime = it->second; + + d.subdirs.push_back(mNodes.size()) ; + mNodes.push_back(de) ; + } + + return true; +} +bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! +{ + // check that it's a directory + + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + if(indx == 0) + return nodeAccessError("checkIndex(): Cannot remove top level directory") ; + +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "(--) Removing directory " << indx << std::endl; + print(); +#endif + // remove from parent + + DirEntry& d(*static_cast(mNodes[indx])) ; + DirEntry& parent_dir(*static_cast(mNodes[d.parent_index])); + + for(uint32_t i=0;i= mNodes.size() || mNodes[indx] == NULL) + return nodeAccessError("checkIndex(): Node does not exist") ; + + if(! (mNodes[indx]->type() & type)) + return nodeAccessError("checkIndex(): Node is of wrong type") ; + + return true; +} + +bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + return false; + + DirEntry& d(*static_cast(mNodes[indx])) ; + new_files = subfiles ; + + // remove from new_files the ones that already exist and have a modf time that is not older. + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])) ; + std::map::const_iterator it = subfiles.find(f.file_name) ; + + if(it == subfiles.end()) // file does not exist anymore => delete + { + std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; + + delete mNodes[d.subfiles[i]] ; + mNodes[d.subfiles[i]] = NULL ; + + d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; + d.subfiles.pop_back(); + continue; + } + + if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size + { + f.file_hash.clear(); // hash needs recomputing + f.file_modtime = it->second.modtime; + f.file_size = it->second.size; + } + new_files.erase(f.file_name) ; + + ++i; + } + + for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) + { + std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; + + d.subfiles.push_back(mNodes.size()) ; + mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); + mNodes.back()->row = mNodes.size()-1; + mNodes.back()->parent_index = indx; + } + return true; +} +bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) +{ + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; + return false; + } + + std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; + + RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; + + old_hash = hash ; + + return true; +} +bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) +{ + if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) + { + std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; + return false; + } + + FileEntry& fe(*static_cast(mNodes[file_index])) ; + + std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; + + fe.file_hash = hash; + fe.file_size = size; + fe.file_modtime = modf_time; + fe.file_name = fname; + + return true; +} + +bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) +{ + if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl; + return false; + } + DirEntry& d(*static_cast(mNodes[indx])) ; + + d.most_recent_time = most_recent_time; + d.dir_modtime = dir_modtime; + d.dir_update_time = time(NULL); + d.dir_name = dir_name; + + d.subfiles = subfiles_array ; + d.subdirs = subdirs_array ; + + // check that all subdirs already exist. If not, create. + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subdirs_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subdirs_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new DirEntry(""); + + ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; + node->row = i ; + node->parent_index = indx ; + } + for(uint32_t i=0;i= mNodes.size() ) + mNodes.resize(subfiles_array[i]+1,NULL); + + FileStorageNode *& node(mNodes[subfiles_array[i]]); + + if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) + { + delete node ; + node = NULL ; + } + + if(node == NULL) + node = new FileEntry("",0,0); + + node->row = subdirs_array.size()+i ; + node->parent_index = indx ; + } + + return true; +} + +bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) +{ + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + recurs_max_modf_TS = d.most_recent_time ; + local_update_TS = d.dir_update_time ; + + return true; +} +bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) +{ + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + { + std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + return false; + } + + DirEntry& d(*static_cast(mNodes[index])) ; + + d.most_recent_time = recurs_max_modf_TS ; + d.dir_update_time = local_update_TS ; + + return true; +} + +// Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. + +time_t InternalFileHierarchyStorage::recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index) +{ + DirEntry& d(*static_cast(mNodes[dir_index])) ; + + time_t largest_modf_time = d.dir_modtime ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; + + for(uint32_t i=0;i(mNodes[indx]) ; +} +const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const +{ + if(!checkIndex(indx,FileStorageNode::TYPE_FILE)) + return NULL ; + + return static_cast(mNodes[indx]) ; +} +uint32_t InternalFileHierarchyStorage::getType(DirectoryStorage::EntryIndex indx) const +{ + if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) + return mNodes[indx]->type() ; + else + return FileStorageNode::TYPE_UNKNOWN; +} + +DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) +{ + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subfiles[file_tab_index]; +} +DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) +{ + if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) + return DirectoryStorage::NO_INDEX; + + if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) + return DirectoryStorage::NO_INDEX; + + return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; +} + +bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list& results) +{ + std::map::const_iterator it = mHashes.find(hash); + + if( it != mHashes.end() ) + { + results.clear(); + results.push_back(it->second) ; + return true ; + } + else + return false; +} + +bool InternalFileHierarchyStorage::check(std::string& error_string) const // checks consistency of storage. +{ + // recurs go through all entries, check that all + + std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. + hits[0] = 1 ; // because 0 is never the child of anyone + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + // stamp the kids + const DirEntry& de = *static_cast(mNodes[i]) ; + + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subdirs[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subdirs[j]] = 1; + } + for(uint32_t j=0;j= mNodes.size()) + { + error_string = "Node child out of tab!" ; + return false ; + } + if(hits[de.subfiles[j]] != 0) + { + error_string = "Double hit on a single node" ; + return false; + } + hits[de.subfiles[j]] = 1; + } + + } + + for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++ndirs; + } + else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE) + { + std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; + ++nfiles; + } + else + { + ++nunknown; + std::cerr << "(EE) Error: unknown type node found!" << std::endl; + } + + std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; + + recursPrint(0,DirectoryStorage::EntryIndex(0)); +} +void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::EntryIndex node) const +{ + std::string indent(2*depth,' '); + + if(mNodes[node] == NULL) + { + std::cerr << "EMPTY NODE !!" << std::endl; + return ; + } + DirEntry& d(*static_cast(mNodes[node])); + + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])); + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; + } +} + +bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s) +{ + std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; + return false ; +} + +// Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning +// phase. That allows to keep file information when moving them around. + +bool InternalFileHierarchyStorage::recursRemoveDirectory(DirectoryStorage::EntryIndex dir) +{ + DirEntry& d(*static_cast(mNodes[dir])) ; + + for(uint32_t i=0;i +#include +#include + +#include "directory_storage.h" + +class InternalFileHierarchyStorage +{ +public: + class FileStorageNode + { + public: + static const uint32_t TYPE_UNKNOWN = 0x0000 ; + static const uint32_t TYPE_FILE = 0x0001 ; + static const uint32_t TYPE_DIR = 0x0002 ; + + virtual ~FileStorageNode() {} + virtual uint32_t type() const =0; + + DirectoryStorage::EntryIndex parent_index; + uint32_t row ; + }; + class FileEntry: public FileStorageNode + { + public: + FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} + virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } + virtual ~FileEntry() {} + + // local stuff + std::string file_name ; + uint64_t file_size ; + time_t file_modtime; + RsFileHash file_hash ; + }; + + class DirEntry: public FileStorageNode + { + public: + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} + virtual ~DirEntry() {} + + virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } + + // local stuff + std::string dir_name ; + std::string dir_parent_path ; + + std::vector subdirs ; + std::vector subfiles ; + + time_t dir_modtime; + time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. + time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. + }; + + // class stuff + InternalFileHierarchyStorage() ; + + bool load(const std::string& fname) ; + bool save(const std::string& fname) ; + + int parentRow(DirectoryStorage::EntryIndex e); + bool isIndexValid(DirectoryStorage::EntryIndex e) const; + bool stampDirectory(const DirectoryStorage::EntryIndex& indx); + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); + bool removeDirectory(DirectoryStorage::EntryIndex indx) ; + bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const; + bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); + bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); + bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array); + bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); + bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); + + // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. + + time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index); + + // file/dir access and modification + bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e + + uint32_t mRoot ; + std::vector mNodes;// uses pointers to keep information about valid/invalid objects. + + void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to + // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that + // are not referenced. + + friend class DirectoryStorage ; // only class that can use this. + friend class LocalDirectoryStorage ; // only class that can use this. + + // Low level stuff. Should normally not be used externally. + + const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const; + const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const; + const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const; + uint32_t getType(DirectoryStorage::EntryIndex indx) const; + DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index); + DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index); + bool searchHash(const RsFileHash& hash,std::list& results); + + bool check(std::string& error_string) const ;// checks consistency of storage. + + void print() const; + +private: + void recursPrint(int depth,DirectoryStorage::EntryIndex node) const; + static bool nodeAccessError(const std::string& s); + + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning + // phase. That allows to keep file information when moving them around. + + bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); + + std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? +}; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index b490038f8..f7e9b4775 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -3,622 +3,11 @@ #include "retroshare/rspeers.h" #include "util/rsdir.h" #include "util/rsstring.h" +#include "file_sharing_defaults.h" #include "directory_storage.h" +#include "dir_hierarchy.h" #include "filelist_io.h" -#define DEBUG_DIRECTORY_STORAGE 1 - -/******************************************************************************************************************/ -/* Internal File Hierarchy Storage */ -/******************************************************************************************************************/ - -template typename std::set::iterator erase_from_set(typename std::set& s,const typename std::set::iterator& it) -{ - typename std::set::iterator tmp(it); - ++tmp; - s.erase(it) ; - return tmp; -} - -// This class handles the file hierarchy -// A Mutex is used to ensure total coherence at this level. So only abstracted operations are allowed, -// so that the hierarchy stays completely coherent between calls. - -class InternalFileHierarchyStorage -{ -public: - class FileStorageNode - { - public: - static const uint32_t TYPE_UNKNOWN = 0x0000 ; - static const uint32_t TYPE_FILE = 0x0001 ; - static const uint32_t TYPE_DIR = 0x0002 ; - - virtual ~FileStorageNode() {} - virtual uint32_t type() const =0; - - DirectoryStorage::EntryIndex parent_index; - uint32_t row ; - }; - class FileEntry: public FileStorageNode - { - public: - FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} - virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } - virtual ~FileEntry() {} - - // local stuff - time_t file_modtime; - std::string file_name ; - uint64_t file_size ; - RsFileHash file_hash ; - }; - - class DirEntry: public FileStorageNode - { - public: - DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} - virtual ~DirEntry() {} - - virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } - - // local stuff - std::string dir_name ; - std::string dir_parent_path ; - - std::vector subdirs ; - std::vector subfiles ; - - time_t dir_modtime; - time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. - time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. - }; - - // class stuff - InternalFileHierarchyStorage() : mRoot(0) - { - DirEntry *de = new DirEntry("") ; - - de->row=0; - de->parent_index=0; - de->dir_modtime=0; - - mNodes.push_back(de) ; - } - - int parentRow(DirectoryStorage::EntryIndex e) - { - if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) - return -1 ; - - return mNodes[mNodes[e]->parent_index]->row; - } - - // high level modification routines - - bool isIndexValid(DirectoryStorage::EntryIndex e) const - { - return e < mNodes.size() && mNodes[e] != NULL ; - } - bool stampDirectory(const DirectoryStorage::EntryIndex& indx) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - static_cast(mNodes[indx])->dir_modtime = time(NULL) ; - return true; - } - - bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - DirEntry& d(*static_cast(mNodes[indx])) ; - - std::map should_create(subdirs); - - for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) - { - std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; - - if( !removeDirectory(d.subdirs[i])) - i++ ; - } - else - { - std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; - - should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; - ++i; - } - - for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) - { - std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; - - DirEntry *de = new DirEntry(it->first) ; - - de->row = mNodes.size(); - de->parent_index = indx; - de->dir_modtime = it->second; - - d.subdirs.push_back(mNodes.size()) ; - mNodes.push_back(de) ; - } - - return true; - } - bool removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! - { - // check that it's a directory - - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - if(indx == 0) - return nodeAccessError("checkIndex(): Cannot remove top level directory") ; - -#ifdef DEBUG_DIRECTORY_STORAGE - std::cerr << "(--) Removing directory " << indx << std::endl; - print(); -#endif - // remove from parent - - DirEntry& d(*static_cast(mNodes[indx])) ; - DirEntry& parent_dir(*static_cast(mNodes[d.parent_index])); - - for(uint32_t i=0;i= mNodes.size() || mNodes[indx] == NULL) - return nodeAccessError("checkIndex(): Node does not exist") ; - - if(! (mNodes[indx]->type() & type)) - return nodeAccessError("checkIndex(): Node is of wrong type") ; - - return true; - } - - bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - DirEntry& d(*static_cast(mNodes[indx])) ; - new_files = subfiles ; - - // remove from new_files the ones that already exist and have a modf time that is not older. - - for(uint32_t i=0;i(mNodes[d.subfiles[i]])) ; - std::map::const_iterator it = subfiles.find(f.file_name) ; - - if(it == subfiles.end()) // file does not exist anymore => delete - { - std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; - - delete mNodes[d.subfiles[i]] ; - mNodes[d.subfiles[i]] = NULL ; - - d.subfiles[i] = d.subfiles[d.subfiles.size()-1] ; - d.subfiles.pop_back(); - continue; - } - - if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size - { - f.file_hash.clear(); // hash needs recomputing - f.file_modtime = it->second.modtime; - f.file_size = it->second.size; - } - new_files.erase(f.file_name) ; - - ++i; - } - - for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) - { - std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; - - d.subfiles.push_back(mNodes.size()) ; - mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); - mNodes.back()->row = mNodes.size()-1; - mNodes.back()->parent_index = indx; - } - return true; - } - bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash) - { - if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) - { - std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; - return false; - } - - std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; - - RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; - - old_hash = hash ; - - return true; - } - bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time) - { - if(!checkIndex(file_index,FileStorageNode::TYPE_FILE)) - { - std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; - return false; - } - - FileEntry& fe(*static_cast(mNodes[file_index])) ; - - std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; - - fe.file_hash = hash; - fe.file_size = size; - fe.file_modtime = modf_time; - fe.file_name = fname; - - return true; - } - - bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - { - std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl; - return false; - } - DirEntry& d(*static_cast(mNodes[indx])) ; - - d.most_recent_time = most_recent_time; - d.dir_modtime = dir_modtime; - d.dir_update_time = time(NULL); - d.dir_name = dir_name; - - d.subfiles = subfiles_array ; - d.subdirs = subdirs_array ; - - // check that all subdirs already exist. If not, create. - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subdirs_array[i]+1,NULL); - - FileStorageNode *& node(mNodes[subdirs_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) - { - delete node ; - node = NULL ; - } - - if(node == NULL) - node = new DirEntry(""); - - ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; - node->row = i ; - node->parent_index = indx ; - } - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subfiles_array[i]+1,NULL); - - FileStorageNode *& node(mNodes[subfiles_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) - { - delete node ; - node = NULL ; - } - - if(node == NULL) - node = new FileEntry("",0,0); - - node->row = subdirs_array.size()+i ; - node->parent_index = indx ; - } - - return true; - } - - bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) - { - if(!checkIndex(index,FileStorageNode::TYPE_DIR)) - { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; - return false; - } - - DirEntry& d(*static_cast(mNodes[index])) ; - - recurs_max_modf_TS = d.most_recent_time ; - local_update_TS = d.dir_update_time ; - - return true; - } - bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) - { - if(!checkIndex(index,FileStorageNode::TYPE_DIR)) - { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; - return false; - } - - DirEntry& d(*static_cast(mNodes[index])) ; - - d.most_recent_time = recurs_max_modf_TS ; - d.dir_update_time = local_update_TS ; - - return true; - } - - // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. - - time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index) - { - DirEntry& d(*static_cast(mNodes[dir_index])) ; - - time_t largest_modf_time = d.dir_modtime ; - - for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; - - for(uint32_t i=0;i mNodes;// uses pointers to keep information about valid/invalid objects. - - void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to - // all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that - // are not referenced. - - friend class DirectoryStorage ; // only class that can use this. - friend class LocalDirectoryStorage ; // only class that can use this. - - // Low level stuff. Should normally not be used externally. - - const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const - { - if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) - return mNodes[indx] ; - else - return NULL ; - } - - const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const - { - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return NULL ; - - return static_cast(mNodes[indx]) ; - } - const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const - { - if(!checkIndex(indx,FileStorageNode::TYPE_FILE)) - return NULL ; - - return static_cast(mNodes[indx]) ; - } - uint32_t getType(DirectoryStorage::EntryIndex indx) const - { - if(checkIndex(indx,FileStorageNode::TYPE_FILE | FileStorageNode::TYPE_DIR)) - return mNodes[indx]->type() ; - else - return FileStorageNode::TYPE_UNKNOWN; - } - - DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index) - { - if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) - return DirectoryStorage::NO_INDEX; - - if(static_cast(mNodes[parent_index])->subfiles.size() <= file_tab_index) - return DirectoryStorage::NO_INDEX; - - return static_cast(mNodes[parent_index])->subfiles[file_tab_index]; - } - DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index) - { - if(!checkIndex(parent_index,FileStorageNode::TYPE_DIR)) - return DirectoryStorage::NO_INDEX; - - if(static_cast(mNodes[parent_index])->subdirs.size() <= dir_tab_index) - return DirectoryStorage::NO_INDEX; - - return static_cast(mNodes[parent_index])->subdirs[dir_tab_index]; - } - - bool searchHash(const RsFileHash& hash,std::list& results) - { - std::map::const_iterator it = mHashes.find(hash); - - if( it != mHashes.end() ) - { - results.clear(); - results.push_back(it->second) ; - return true ; - } - else - return false; - } - - bool check(std::string& error_string) const // checks consistency of storage. - { - // recurs go through all entries, check that all - - std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. - hits[0] = 1 ; // because 0 is never the child of anyone - - for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) - { - // stamp the kids - const DirEntry& de = *static_cast(mNodes[i]) ; - - for(uint32_t j=0;j= mNodes.size()) - { - error_string = "Node child out of tab!" ; - return false ; - } - if(hits[de.subdirs[j]] != 0) - { - error_string = "Double hit on a single node" ; - return false; - } - hits[de.subdirs[j]] = 1; - } - for(uint32_t j=0;j= mNodes.size()) - { - error_string = "Node child out of tab!" ; - return false ; - } - if(hits[de.subfiles[j]] != 0) - { - error_string = "Double hit on a single node" ; - return false; - } - hits[de.subfiles[j]] = 1; - } - - } - - for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) - { - std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; - ++ndirs; - } - else if(mNodes[i]->type() == FileStorageNode::TYPE_FILE) - { - std::cerr << " Node " << i << ": type=" << mNodes[i]->type() << std::endl; - ++nfiles; - } - else - { - ++nunknown; - std::cerr << "(EE) Error: unknown type node found!" << std::endl; - } - - std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; - - recursPrint(0,DirectoryStorage::EntryIndex(0)); - } -private: - void recursPrint(int depth,DirectoryStorage::EntryIndex node) const - { - std::string indent(2*depth,' '); - - if(mNodes[node] == NULL) - { - std::cerr << "EMPTY NODE !!" << std::endl; - return ; - } - DirEntry& d(*static_cast(mNodes[node])); - - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; - - for(int i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; - } - } - - static bool nodeAccessError(const std::string& s) - { - std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl; - return false ; - } - - // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning - // phase. That allows to keep file information when moving them around. - - bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir) - { - DirEntry& d(*static_cast(mNodes[dir])) ; - - for(uint32_t i=0;i mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? -}; - /******************************************************************************************************************/ /* Iterators */ /******************************************************************************************************************/ @@ -666,8 +55,8 @@ std::string DirectoryStorage::DirIterator::name() const { const InternalFil /* Directory Storage */ /******************************************************************************************************************/ -DirectoryStorage::DirectoryStorage(const std::string& local_file_name, const RsPeerId &pid) - : mFileName(local_file_name),mPeerId(pid), mDirStorageMtx("Directory storage "+local_file_name) +DirectoryStorage::DirectoryStorage(const RsPeerId &pid) + : mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString()) { RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy = new InternalFileHierarchyStorage(); @@ -771,10 +160,20 @@ void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_ void DirectoryStorage::load(const std::string& local_file_name) { + std::cerr << "DirectoryStorage::load()" << std::endl; + // first load the header, than all fields. + + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->load(local_file_name); } void DirectoryStorage::save(const std::string& local_file_name) { + std::cerr << "DirectoryStorage::Save()" << std::endl; + + RS_STACK_MUTEX(mDirStorageMtx) ; + mFileHierarchy->save(local_file_name); + // first write the header, than all fields. } void DirectoryStorage::print() @@ -787,8 +186,8 @@ void DirectoryStorage::print() bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) { RS_STACK_MUTEX(mDirStorageMtx) ; -#warning todo + NOT_IMPLEMENTED(); return true; } @@ -1142,7 +541,24 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary /* Remote Directory Storage */ /******************************************************************************************************************/ -bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) +RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) + : DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname) +{ + load(fname) ; +} + +void RemoteDirectoryStorage::checkSave() +{ + time_t now = time(NULL); + + if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now) + { + save(mFileName); + mLastSavedTime = now ; + } +} + +bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata) { const unsigned char *section_data = (unsigned char*)bindata.bin_data ; uint32_t section_size = bindata.bin_len ; @@ -1241,6 +657,7 @@ bool RemoteDirectoryStorage::deserialiseDirEntry(const EntryIndex& indx,const Rs if(!mFileHierarchy->updateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; } + mChanged = true ; return true ; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 0f5311944..302c24be5 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -16,10 +16,10 @@ class RsTlvBinaryData ; class DirectoryStorage { public: - DirectoryStorage(const std::string& local_file_name,const RsPeerId& pid) ; + DirectoryStorage(const RsPeerId& pid) ; virtual ~DirectoryStorage() {} - typedef int32_t EntryIndex ; + typedef uint32_t EntryIndex ; static const EntryIndex NO_INDEX = 0xffffffff; void save() const ; @@ -104,10 +104,12 @@ class DirectoryStorage void print(); void cleanup(); - private: + protected: void load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; + private: + void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; @@ -116,7 +118,6 @@ class DirectoryStorage // storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector. - std::string mFileName; RsPeerId mPeerId; protected: @@ -128,7 +129,7 @@ class DirectoryStorage class RemoteDirectoryStorage: public DirectoryStorage { public: - RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) : DirectoryStorage(fname,pid) {} + RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) ; virtual ~RemoteDirectoryStorage() {} /*! @@ -140,13 +141,24 @@ public: * \param bindata binary data to deserialise from * \return false when the directory cannot be found. */ - bool deserialiseDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; + bool deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ; + + /*! + * \brief checkSave + * Checks the time of last saving, last modification time, and saves if needed. + */ + void checkSave() ; + +private: + time_t mLastSavedTime ; + bool mChanged ; + std::string mFileName; }; class LocalDirectoryStorage: public DirectoryStorage { public: - LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(fname,own_id) {} + LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {} virtual ~LocalDirectoryStorage() {} void setSharedDirectoryList(const std::list& lst) ; @@ -194,6 +206,7 @@ private: std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; std::map mLocalDirs ; // map is better for search. it->first=it->second.filename + std::string mFileName; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index b40672478..5bc404c5f 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -10,4 +10,5 @@ static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. -static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 60 secs. +static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. +static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index f92e7be10..9e1b323e2 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -51,6 +51,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; + mLastCleanupTime = 0 ; // This is for the transmission of data @@ -141,13 +142,10 @@ int p3FileDatabase::tick() // - remove/delete shared file lists for people who are not friend anymore // - - static time_t last_cleanup_time = 0; - -#warning we should use members here, not static - if(last_cleanup_time + 5 < now) + if(mLastCleanupTime + 5 < now) { cleanup(); - last_cleanup_time = now ; + mLastCleanupTime = now ; } static time_t last_print_time = 0; @@ -186,6 +184,7 @@ int p3FileDatabase::tick() mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ; for(uint32_t i=0;ipeerId()) != online_peers.end()) { std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; @@ -196,6 +195,9 @@ int p3FileDatabase::tick() locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; } + mRemoteDirectories[i]->checkSave() ; + } + mLastRemoteDirSweepTS = now; } @@ -440,8 +442,7 @@ void p3FileDatabase::cleanup() std::string p3FileDatabase::makeRemoteFileName(const RsPeerId& pid) const { -#warning we should use the default paths here. Ask p3config - return "dirlist_"+pid.toStdString()+".txt" ; + return mFileSharingDir + "/" + "dirlist_"+pid.toStdString()+".bin" ; } uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) @@ -998,7 +999,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; - if(mRemoteDirectories[fi]->deserialiseDirEntry(item->entry_index,item->directory_content_data)) + if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(item->entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index d4504bb77..ec5f1b517 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -196,5 +196,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub mutable RsMutex mFLSMtx ; uint32_t mUpdateFlags ; std::string mFileSharingDir ; + time_t mLastCleanupTime; }; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 582f5c508..56d36469c 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -46,6 +46,7 @@ file_lists { file_sharing/directory_storage.h \ file_sharing/directory_updater.h \ file_sharing/rsfilelistitems.h \ + file_sharing/dir_hierarchy.h \ file_sharing/file_sharing_defaults.h SOURCES *= file_sharing/p3filelists.cc \ @@ -53,6 +54,7 @@ file_lists { file_sharing/filelist_io.cc \ file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ + file_sharing/dir_hierarchy.cc \ file_sharing/rsfilelistitems.cc } From 5b221c56dbc12f39bce2917974a232a30c7c3559 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 3 Sep 2016 18:46:03 +0200 Subject: [PATCH 055/174] moved encrypted data output/input to FileListIO --- .../src/file_sharing/dir_hierarchy.cc | 3 + libretroshare/src/file_sharing/filelist_io.cc | 88 +++++++++++++++++++ libretroshare/src/file_sharing/filelist_io.h | 3 + libretroshare/src/file_sharing/hash_cache.cc | 88 +++---------------- 4 files changed, 107 insertions(+), 75 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 74db29e1b..d2b3abc05 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -548,9 +548,12 @@ bool InternalFileHierarchyStorage::recursRemoveDirectory(DirectoryStorage::Entry bool InternalFileHierarchyStorage::save(const std::string& fname) { NOT_IMPLEMENTED(); + return true; } bool InternalFileHierarchyStorage::load(const std::string& fname) { NOT_IMPLEMENTED(); + return true; } + diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 770478c37..47d8899e3 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,4 +1,6 @@ #include "retroshare/rsids.h" +#include "pqi/authssl.h" +#include "util/rsdir.h" #include "serialiser/rsbaseserial.h" #include "filelist_io.h" @@ -116,3 +118,89 @@ bool FileListIO::read125Size(const unsigned char *data,uint32_t data_size,uint32 return true; } +bool FileListIO::saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size) +{ + void *encryptedData = NULL ; + int encDataLen = 0 ; + + if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,total_size, AuthSSL::getAuthSSL()->OwnId())) + { + std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; + return false; + } + + FILE *F = fopen( (fname+".tmp").c_str(),"wb" ) ; + + if(!F) + { + std::cerr << "Cannot open encrypted file cache for writing: " << fname+".tmp" << std::endl; + + free(encryptedData); + return false; + } + if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) + { + std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; + fclose(F) ; + + free(encryptedData); + return false; + } + + fclose(F) ; + + RsDirUtil::renameFile(fname+".tmp",fname) ; +#ifdef FIM_DEBUG + std::cerr << "done." << std::endl ; +#endif + + free(encryptedData); + return true; +} + +bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size) +{ + uint64_t file_size ; + + if(!RsDirUtil::checkFile( fname,file_size,false ) ) + { + std::cerr << "Encrypted hash cache file not present." << std::endl; + return false; + } + + // read the binary stream into memory. + // + RsTemporaryMemory buffer(file_size) ; + + if(buffer == NULL) + return false; + + FILE *F = fopen( fname.c_str(),"rb") ; + if (!F) + { + std::cerr << "Cannot open file for reading encrypted file cache, filename " << fname << std::endl; + return false; + } + if(fread(buffer,1,file_size,F) != file_size) + { + std::cerr << "Cannot read from file " + fname << ": something's wrong." << std::endl; + fclose(F) ; + return false; + } + fclose(F) ; + + // now decrypt + void *decrypted_data =NULL; + int decrypted_data_size =0; + + if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) + { + std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; + return false; + } + + data = (unsigned char*)decrypted_data ; + total_size = decrypted_data_size ; + + return true; +} diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index b22a57cc4..bb88fc577 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -61,6 +61,9 @@ public: template static bool deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,T& val) ; template static uint32_t serial_size(const T& val) ; + static bool saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size); + static bool loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size); + private: static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ; static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index a0ea48c95..a92842c54 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -208,59 +208,20 @@ void HashStorage::clean() void HashStorage::locked_load() { - uint64_t file_size ; + unsigned char *data = NULL ; + uint32_t data_size=0; - if(!RsDirUtil::checkFile( mFilePath,file_size,false ) ) + if(!FileListIO::loadEncryptedDataFromFile(mFilePath,data,data_size)) { - std::cerr << "Encrypted hash cache file not present." << std::endl; + std::cerr << "(EE) Cannot read hash cache." << std::endl; return ; } - - // read the binary stream into memory. - // - void *buffer = rs_malloc(file_size) ; - - if(buffer == NULL) - return ; - - FILE *F = fopen( mFilePath.c_str(),"rb") ; - if (!F) - { - std::cerr << "Cannot open file for reading encrypted file cache, filename " << mFilePath << std::endl; - free(buffer); - return; - } - if(fread(buffer,1,file_size,F) != file_size) - { - std::cerr << "Cannot read from file " + mFilePath << ": something's wrong." << std::endl; - free(buffer) ; - fclose(F) ; - return ; - } - fclose(F) ; - - // now decrypt - void *decrypted_data =NULL; - int decrypted_data_size =0; - - if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) - { - std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; - free(buffer) ; - return ; - } - free(buffer) ; - -#ifdef HASHSTORAGE_DEBUG - std::cerr << "Loading HashCache from file " << mFilePath << std::endl ; - int n=0 ; -#endif - unsigned char *data = (unsigned char*)decrypted_data ; uint32_t offset = 0 ; HashStorageInfo info ; + uint32_t n=0; - while(offset < decrypted_data_size) - if(readHashStorageInfo(data,decrypted_data_size,offset,info)) + while(offset < data_size) + if(readHashStorageInfo(data,data_size,offset,info)) { #ifdef HASHSTORAGE_DEBUG std::cerr << info << std::endl; @@ -269,7 +230,7 @@ void HashStorage::locked_load() mFiles[info.filename] = info ; } - free(decrypted_data) ; + free(data) ; #ifdef HASHSTORAGE_DEBUG std::cerr << n << " entries loaded." << std::endl ; #endif @@ -288,39 +249,16 @@ void HashStorage::locked_save() for(std::map::const_iterator it(mFiles.begin());it!=mFiles.end();++it) writeHashStorageInfo(data,total_size,offset,it->second) ; - void *encryptedData = NULL ; - int encDataLen = 0 ; - - if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,offset, AuthSSL::getAuthSSL()->OwnId())) + if(!FileListIO::saveEncryptedDataToFile(mFilePath,data,offset)) { - std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; - return; + std::cerr << "(EE) Cannot save hash cache data." << std::endl; + free(data) ; + return ; } - FILE *F = fopen( (mFilePath+".tmp").c_str(),"wb" ) ; - - if(!F) - { - std::cerr << "Cannot open encrypted file cache for writing: " << mFilePath+".tmp" << std::endl; - goto save_free; - } - if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) - { - std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; - fclose(F) ; - goto save_free; - } - - fclose(F) ; - - RsDirUtil::renameFile(mFilePath+".tmp",mFilePath) ; -#ifdef FIM_DEBUG - std::cerr << "done." << std::endl ; -#endif std::cerr << mFiles.size() << " Entries saved." << std::endl; -save_free: - free(encryptedData); + free(data) ; } bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const From ea6431e2769de217873131b67101f5e034583bdc Mon Sep 17 00:00:00 2001 From: thunder2 Date: Mon, 5 Sep 2016 14:03:39 +0200 Subject: [PATCH 056/174] Fixed Windows shadow build with Qt 5.6 or higher --- plugins/Common/retroshare_plugin.pri | 8 ++++---- plugins/VOIP/VOIP.pro | 4 ++-- retroshare-gui/src/retroshare-gui.pro | 20 ++++++++++---------- retroshare-nogui/src/retroshare-nogui.pro | 20 ++++++++++---------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/plugins/Common/retroshare_plugin.pri b/plugins/Common/retroshare_plugin.pri index 5030d6ee7..cbfec1b8d 100644 --- a/plugins/Common/retroshare_plugin.pri +++ b/plugins/Common/retroshare_plugin.pri @@ -3,8 +3,8 @@ TEMPLATE = lib CONFIG *= plugin -DEPENDPATH += ../../libretroshare/src/ ../../retroshare-gui/src/ -INCLUDEPATH += ../../libretroshare/src/ ../../retroshare-gui/src/ +DEPENDPATH += $$PWD/../../libretroshare/src/ $$PWD/../../retroshare-gui/src/ +INCLUDEPATH += $$PWD/../../libretroshare/src/ $$PWD/../../retroshare-gui/src/ unix { target.path = "$${PLUGIN_DIR}" @@ -67,8 +67,8 @@ win32 { DEPENDPATH += . $$INC_DIR INCLUDEPATH += . $$INC_DIR - PRE_TARGETDEPS += ../../retroshare-gui/src/lib/libretroshare-gui.a - LIBS += -L"../../retroshare-gui/src/lib" -lretroshare-gui + PRE_TARGETDEPS += $$OUT_PWD/../../retroshare-gui/src/lib/libretroshare-gui.a + LIBS += -L"$$OUT_PWD/../../retroshare-gui/src/lib" -lretroshare-gui for(lib, LIB_DIR):LIBS += -L"$$lib" for(bin, BIN_DIR):LIBS += -L"$$bin" diff --git a/plugins/VOIP/VOIP.pro b/plugins/VOIP/VOIP.pro index 72b2d1d5e..5fac6b780 100644 --- a/plugins/VOIP/VOIP.pro +++ b/plugins/VOIP/VOIP.pro @@ -13,8 +13,8 @@ exists($$[QMAKE_MKSPECS]/features/mobility.prf) { CONFIG += qt uic qrc resources MOBILITY = multimedia -DEPENDPATH += ../../retroshare-gui/src/temp/ui ../../libretroshare/src -INCLUDEPATH += ../../retroshare-gui/src/temp/ui ../../libretroshare/src +DEPENDPATH += $$PWD/../../retroshare-gui/src/temp/ui +INCLUDEPATH += $$PWD/../../retroshare-gui/src/temp/ui #################################### Linux ##################################### diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index d21d35eb8..4dcaf8dca 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -305,22 +305,22 @@ openbsd-* { # ########################################### -DEPENDPATH += . ../../libretroshare/src/ -INCLUDEPATH += ../../libretroshare/src/ +DEPENDPATH += . $$PWD/../../libretroshare/src/ +INCLUDEPATH += $$PWD/../../libretroshare/src/ -PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a -LIBS *= ../../libretroshare/src/lib/libretroshare.a +PRE_TARGETDEPS *= $$OUT_PWD/../../libretroshare/src/lib/libretroshare.a +LIBS *= $$OUT_PWD/../../libretroshare/src/lib/libretroshare.a wikipoos { - PRE_TARGETDEPS *= ../../supportlibs/pegmarkdown/lib/libpegmarkdown.a - LIBS *= ../../supportlibs/pegmarkdown/lib/libpegmarkdown.a + PRE_TARGETDEPS *= $$OUT_PWD/../../supportlibs/pegmarkdown/lib/libpegmarkdown.a + LIBS *= $$OUT_PWD/../../supportlibs/pegmarkdown/lib/libpegmarkdown.a } # webinterface -DEPENDPATH += ../../libresapi/src -INCLUDEPATH += ../../libresapi/src -PRE_TARGETDEPS *= ../../libresapi/src/lib/libresapi.a -LIBS += ../../libresapi/src/lib/libresapi.a +DEPENDPATH += $$PWD/../../libresapi/src +INCLUDEPATH += $$PWD/../../libresapi/src +PRE_TARGETDEPS *= $$OUT_PWD/../../libresapi/src/lib/libresapi.a +LIBS += $$OUT_PWD/../../libresapi/src/lib/libresapi.a # Input HEADERS += rshare.h \ diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index 03482fe6c..c66ae37cd 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -165,11 +165,11 @@ haiku-* { ############################## Common stuff ###################################### -DEPENDPATH += . ../../libretroshare/src -INCLUDEPATH += . ../../libretroshare/src +DEPENDPATH += . $$PWD/../../libretroshare/src +INCLUDEPATH += . $$PWD/../../libretroshare/src -PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a -LIBS *= ../../libretroshare/src/lib/libretroshare.a +PRE_TARGETDEPS *= $$OUT_PWD/../../libretroshare/src/lib/libretroshare.a +LIBS *= $$OUT_PWD/../../libretroshare/src/lib/libretroshare.a # Input HEADERS += notifytxt.h @@ -184,10 +184,10 @@ introserver { libresapihttpserver { DEFINES *= ENABLE_WEBUI - PRE_TARGETDEPS *= ../../libresapi/src/lib/libresapi.a - LIBS += ../../libresapi/src/lib/libresapi.a - DEPENDPATH += ../../libresapi/src - INCLUDEPATH += ../../libresapi/src + PRE_TARGETDEPS *= $$OUT_PWD/../../libresapi/src/lib/libresapi.a + LIBS += $$OUT_PWD/../../libresapi/src/lib/libresapi.a + DEPENDPATH += $$PWD/../../libresapi/src + INCLUDEPATH += $$PWD/../../libresapi/src HEADERS += \ TerminalApiClient.h SOURCES += \ @@ -200,8 +200,8 @@ sshserver { # Please use this path below. # (You can modify it locally if required - but dont commit it!) - #LIBSSH_DIR = ../../../lib/libssh-0.5.2 - LIBSSH_DIR = ../../../libssh-0.6.0rc1 + #LIBSSH_DIR = $PWD/../../../lib/libssh-0.5.2 + LIBSSH_DIR = $PWD/../../../libssh-0.6.0rc1 # # Use the following commend to generate a Server RSA Key. From 5f5ce4ab634154a3033d42f6ad8e25a8d74fce1e Mon Sep 17 00:00:00 2001 From: thunder2 Date: Tue, 17 May 2016 19:02:14 +0200 Subject: [PATCH 057/174] Fixed version_detail_bash_script with shadow build --- libretroshare/src/libretroshare.pro | 2 +- libretroshare/src/version_detail.sh | 5 +++++ retroshare-gui/src/retroshare-gui.pro | 2 +- retroshare-gui/src/version_detail.sh | 5 +++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 6499243b8..a1e8f984f 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -203,7 +203,7 @@ version_detail_bash_script { linux-* { QMAKE_EXTRA_TARGETS += write_version_detail PRE_TARGETDEPS = write_version_detail - write_version_detail.commands = ./version_detail.sh + write_version_detail.commands = $$PWD/version_detail.sh } win32 { QMAKE_EXTRA_TARGETS += write_version_detail diff --git a/libretroshare/src/version_detail.sh b/libretroshare/src/version_detail.sh index d6076b931..3f83ff8f4 100755 --- a/libretroshare/src/version_detail.sh +++ b/libretroshare/src/version_detail.sh @@ -3,6 +3,8 @@ #don't exit even if a command fails set +e +pushd $(dirname "$0") + SCRIPT_PATH=$(dirname "`readlink -f "${0}"`") OLDLANG=${LANG} @@ -32,5 +34,8 @@ if [[ ${version} != '' ]]; then fi export LANG=${OLDLANG} + +popd + echo "script version_detail.sh finished normally" exit 0 diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 4dcaf8dca..b548108ca 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -123,7 +123,7 @@ version_detail_bash_script { DEFINES += ADD_LIBRETROSHARE_VERSION_INFO QMAKE_EXTRA_TARGETS += write_version_detail PRE_TARGETDEPS = write_version_detail - write_version_detail.commands = ./version_detail.sh + write_version_detail.commands = $$PWD/version_detail.sh } win32 { QMAKE_EXTRA_TARGETS += write_version_detail diff --git a/retroshare-gui/src/version_detail.sh b/retroshare-gui/src/version_detail.sh index a6123e08b..1d4c87008 100755 --- a/retroshare-gui/src/version_detail.sh +++ b/retroshare-gui/src/version_detail.sh @@ -5,6 +5,8 @@ #don't exit even if a command fails set +e +pushd $(dirname "$0") + OLDLANG=${LANG} export LANG=C @@ -34,5 +36,8 @@ if (ls &> /dev/null); then fi export LANG=${OLDLANG} + +popd + echo "version_detail.sh scripts finished" exit 0 From 679c53b0b00aedc5b0906c7b7a8774cb97b8b803 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 5 Sep 2016 21:22:11 +0200 Subject: [PATCH 058/174] added load/save of directory hierarchies. Fixed bug in loading of hash cache (caused y non initialisation of dest string in GetRawString()) --- .../src/file_sharing/dir_hierarchy.cc | 205 +++++++++++++++++- .../src/file_sharing/dir_hierarchy.h | 6 +- .../src/file_sharing/directory_storage.cc | 4 +- libretroshare/src/file_sharing/filelist_io.h | 35 +-- libretroshare/src/file_sharing/hash_cache.cc | 3 + libretroshare/src/serialiser/rsbaseserial.cc | 3 + 6 files changed, 229 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d2b3abc05..b6713c9d6 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -1,4 +1,5 @@ #include "dir_hierarchy.h" +#include "filelist_io.h" #define DEBUG_DIRECTORY_STORAGE 1 @@ -235,7 +236,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } DirEntry& d(*static_cast(mNodes[indx])) ; - d.most_recent_time = most_recent_time; + d.dir_most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; d.dir_update_time = time(NULL); d.dir_name = dir_name; @@ -297,7 +298,7 @@ bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryI DirEntry& d(*static_cast(mNodes[index])) ; - recurs_max_modf_TS = d.most_recent_time ; + recurs_max_modf_TS = d.dir_most_recent_time ; local_update_TS = d.dir_update_time ; return true; @@ -312,7 +313,7 @@ bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryI DirEntry& d(*static_cast(mNodes[index])) ; - d.most_recent_time = recurs_max_modf_TS ; + d.dir_most_recent_time = recurs_max_modf_TS ; d.dir_update_time = local_update_TS ; return true; @@ -332,7 +333,7 @@ time_t InternalFileHierarchyStorage::recursUpdateLastModfTime(const DirectorySto for(uint32_t i=0;i(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.dir_most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; for(uint32_t i=0;itype() == FileStorageNode::TYPE_FILE) + { + const FileEntry& fe(*static_cast(mNodes[i])) ; + + unsigned char *file_section_data = NULL ; + uint32_t file_section_offset = 0 ; + uint32_t file_section_size = 0; + + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,fe.file_name )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,fe.file_size )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,fe.file_hash )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)fe.file_modtime)) throw std::runtime_error("Write error") ; + + if(!FileListIO::writeField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,file_section_data,file_section_offset)) throw std::runtime_error("Write error") ; + + free(file_section_data) ; + } + else if(mNodes[i] != NULL && mNodes[i]->type() == FileStorageNode::TYPE_DIR) + { + const DirEntry& de(*static_cast(mNodes[i])) ; + + unsigned char *dir_section_data = NULL ; + uint32_t dir_section_offset = 0 ; + uint32_t dir_section_size = 0; + + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,de.dir_name )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,de.dir_parent_path )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)de.dir_modtime )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,(uint32_t)de.dir_update_time )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)de.dir_most_recent_time )) throw std::runtime_error("Write error") ; + + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)de.subdirs.size())) throw std::runtime_error("Write error") ; + + for(uint32_t j=0;j= mNodes.size()) + mNodes.resize(node_index+1,NULL) ; + + mNodes[node_index] = new FileEntry(file_name,file_size,file_modtime,file_hash); + } + else if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size)) + { + uint32_t node_index ; + std::string dir_name ; + std::string dir_parent_path ; + uint32_t dir_modtime ; + uint32_t dir_update_time ; + uint64_t dir_most_recent_time ; + + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir_most_recent_time )) throw std::runtime_error("Read error") ; + + if(node_index >= mNodes.size()) + mNodes.resize(node_index+1,NULL) ; + + DirEntry *de = new DirEntry(dir_name) ; + de->dir_name = dir_name ; + de->dir_parent_path = dir_parent_path ; + de->dir_modtime = dir_modtime ; + de->dir_update_time = dir_update_time ; + de->dir_most_recent_time = dir_most_recent_time ; + + uint32_t n_subdirs = 0 ; + uint32_t n_subfiles = 0 ; + + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw std::runtime_error("Read error") ; + + for(uint32_t j=0;jsubdirs.push_back(di) ; + } + + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw std::runtime_error("Read error") ; + + for(uint32_t j=0;jsubfiles.push_back(fi) ; + } + } + else + throw std::runtime_error("Unknown node section.") ; + + free(node_section_data) ; + } + free(buffer) ; + return true ; + } + catch(std::exception& e) + { + std::cerr << "Error while writing: " << e.what() << std::endl; + + if(buffer != NULL) + free(buffer) ; + return false; + } } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index abf10a3ba..f0b109c7d 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -26,6 +26,8 @@ public: { public: FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} + FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {} + virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; } virtual ~FileEntry() {} @@ -39,7 +41,7 @@ public: class DirEntry: public FileStorageNode { public: - DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),most_recent_time(0),dir_update_time(0) {} + DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),dir_most_recent_time(0),dir_update_time(0) {} virtual ~DirEntry() {} virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; } @@ -52,7 +54,7 @@ public: std::vector subfiles ; time_t dir_modtime; - time_t most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. + time_t dir_most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below. time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info. }; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f7e9b4775..ba10db1a2 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -229,7 +229,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.type = DIR_TYPE_DIR; d.hash.clear() ; d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size(); - d.min_age = now - dir_entry->most_recent_time ; + d.min_age = now - dir_entry->dir_most_recent_time ; d.name = dir_entry->dir_name; d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; d.parent = (void*)(intptr_t)dir_entry->parent_index ; @@ -487,7 +487,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->most_recent_time)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) return false ; if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index bb88fc577..408ba26c8 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -9,20 +9,25 @@ // This file implements load/save of various fields used for file lists and directory content. // WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. -static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; +static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; -static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; -static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; -static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; -static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; -static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; -static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; -static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; -static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; -static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; +static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ; + +static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; +static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; +static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; +static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; +static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; +static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; +static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; +static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; +static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; +static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x0d ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x0e ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x0f ; static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) @@ -94,11 +99,13 @@ private: if(offset + 1 > buff_size) return false ; - uint8_t section_tag = buff[offset++] ; + uint8_t section_tag = buff[offset] ; // we do the offset++ after, only if the header can be read. Doing so, we can make multiple read attempts. if(section_tag != check_section_tag) return false; + offset++ ; + return read125Size(buff,buff_size,offset,S) ; } }; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index a92842c54..c2ed3ed4e 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,4 +1,5 @@ #include "util/rsdir.h" +#include "util/rsprint.h" #include "rsserver/p3face.h" #include "pqi/authssl.h" #include "hash_cache.h" @@ -299,6 +300,8 @@ bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_si if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ; + std::cerr << "Writing hash storage section " << RsUtil::BinToHex(section_data,section_offset) << std::endl; + std::cerr << "Info.filename = " << info.filename << std::endl; free(section_data) ; return true; diff --git a/libretroshare/src/serialiser/rsbaseserial.cc b/libretroshare/src/serialiser/rsbaseserial.cc index e38dcb8e0..283c4f50e 100644 --- a/libretroshare/src/serialiser/rsbaseserial.cc +++ b/libretroshare/src/serialiser/rsbaseserial.cc @@ -231,6 +231,9 @@ uint32_t getRawStringSize(const std::string &outStr) bool getRawString(void *data, uint32_t size, uint32_t *offset, std::string &outStr) { +#warning I had to change this. It seems like a bug to not clear the string. Should make sure it's not introducing any side effect. + outStr.clear(); + uint32_t len = 0; if (!getRawUInt32(data, size, offset, &len)) { From e9540d7b055976d2088aba7e244bb35f78b109f7 Mon Sep 17 00:00:00 2001 From: defnax Date: Tue, 6 Sep 2016 17:45:42 +0200 Subject: [PATCH 059/174] Improved look and feel layout of Profile Generation Dialog --- retroshare-gui/src/gui/GenCertDialog.cpp | 6 +- retroshare-gui/src/gui/GenCertDialog.ui | 1330 ++++++++--------- retroshare-gui/src/gui/icons.qrc | 1 + retroshare-gui/src/gui/icons/svg/profile.svg | 77 + .../src/gui/qss/stylesheet/Standard.qss | 19 + 5 files changed, 763 insertions(+), 670 deletions(-) create mode 100644 retroshare-gui/src/gui/icons/svg/profile.svg diff --git a/retroshare-gui/src/gui/GenCertDialog.cpp b/retroshare-gui/src/gui/GenCertDialog.cpp index 65780f954..a6f62a4ef 100644 --- a/retroshare-gui/src/gui/GenCertDialog.cpp +++ b/retroshare-gui/src/gui/GenCertDialog.cpp @@ -107,13 +107,13 @@ void GenCertDialog::grabMouse() if(ui.entropy_bar->value() < 20) { ui.genButton->setEnabled(false) ; - ui.genButton->setIcon(QIcon(":/images/delete.png")) ; + //ui.genButton->setIcon(QIcon(":/images/delete.png")) ; ui.genButton->setToolTip(tr("Currently disabled. Please move your mouse around until you reach at least 20%")) ; } else { ui.genButton->setEnabled(true) ; - ui.genButton->setIcon(QIcon(":/images/resume.png")) ; + //ui.genButton->setIcon(QIcon(":/images/resume.png")) ; ui.genButton->setToolTip(tr("Click to create your node and/or profile")) ; } @@ -133,7 +133,7 @@ GenCertDialog::GenCertDialog(bool onlyGenerateIdentity, QWidget *parent) /* Invoke Qt Designer generated QObject setup routine */ ui.setupUi(this); - ui.headerFrame->setHeaderImage(QPixmap(":/images/contact_new128.png")); + ui.headerFrame->setHeaderImage(QPixmap(":/icons/svg/profile.svg")); ui.headerFrame->setHeaderText(tr("Create a new profile")); connect(ui.new_gpg_key_checkbox, SIGNAL(clicked()), this, SLOT(newGPGKeyGenUiSetup())); diff --git a/retroshare-gui/src/gui/GenCertDialog.ui b/retroshare-gui/src/gui/GenCertDialog.ui index 48ca23803..0229cf023 100644 --- a/retroshare-gui/src/gui/GenCertDialog.ui +++ b/retroshare-gui/src/gui/GenCertDialog.ui @@ -6,8 +6,8 @@ 0 0 - 853 - 711 + 855 + 572 @@ -18,6 +18,21 @@ :/images/logo/logo_48.png:/images/logo/logo_48.png + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + @@ -38,674 +53,656 @@ - - - - - - - - 255 - 255 - 255 - - - - - - - 255 - 255 - 178 - - - - - - - - - 255 - 255 - 255 - - - - - - - 255 - 255 - 178 - - - - - - - - - 255 - 255 - 178 - - - - - - - 255 - 255 - 178 - - - - - - - - true - + - QFrame::Box + QFrame::StyledPanel - - You can create a new profile with this form. -Alternatively you can use an existing profile. Just uncheck "Create a new profile" + + QFrame::Raised - - true - - - - - - - - - - - - 255 - 255 - 255 - - - - - - - 255 - 255 - 178 - - - - - - - - - 255 - 255 - 255 - - - - - - - 255 - 255 - 178 - - - - - - - - - 255 - 255 - 178 - - - - - - - 255 - 255 - 178 - - - - - - - - true - - - QFrame::Box - - - You can create and run Retroshare nodes on different computers using the same profile. To do so just export the selected profile, import it on the other computer and create a new node with it. - - - true - - - - - - - It looks like no profile (PGP keys) exists. Please fill in the form below to create one, or import an existing profile. - - - true - - - - - - - No node exists for this profile. - - - true - - - - - - - - - - 16777215 - 32 - - - - Your profile is associated with a PGP key pair - - - Create a new profile - - - - :/images/contact_new128.png:/images/contact_new128.png - - - - 32 - 32 - - - - false - - - - - - - - 0 - 0 - - - - Import new profile - - - - - - - - 0 - 0 - - - - Export selected profile - - - - - - - - 16777215 - 32 - - - - Your profile is associated with a PGP key pair - - - Advanced options - - - - :/images/kcmsystem24.png:/images/kcmsystem24.png - - - - 24 - 24 - - - - false - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - - - - - - - - 6 - - - 0 - - - 0 - - - - - Create a hidden node - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - - - - - - - - - - Use profile - - - - - - - Your profile is associated with a PGP key pair. RetroShare currently ignores DSA keys. - - - - - - - Name - - - - - - - - 0 - 0 - - - - - 0 - 0 - - - - - 16777215 - 32 - - - - Enter your nickname here - - - 64 - - - - - - - - - - Chatrooms/Forums Nickname - - - - - - - You can have one or more identities. They are used when you write in chat lobbies, forums and channel comments. They act as the destination for distant chat and the Retroshare distant mail system. - - - - - - - Email - - - - - - - Be careful: this email will be visible to your friends and friends + + + + + No node exists for this profile. + + + true + + + + + + + It looks like no profile (PGP keys) exists. Please fill in the form below to create one, or import an existing profile. + + + true + + + + + + + + + + + + 255 + 255 + 255 + + + + + + + 255 + 255 + 178 + + + + + + + + + 255 + 255 + 255 + + + + + + + 255 + 255 + 178 + + + + + + + + + 255 + 255 + 178 + + + + + + + 255 + 255 + 178 + + + + + + + + true + + + QFrame::Box + + + You can create and run Retroshare nodes on different computers using the same profile. To do so just export the selected profile, import it on the other computer and create a new node with it. + + + true + + + + + + + + + Use profile + + + + + + + + 0 + 0 + + + + 64 + + + + + + + Node name + + + + + + + This password is for PGP + + + Confirm + + + + + + + + 0 + 0 + + + + Put a strong password here. This password protects your private PGP key. + + + + + + 1024 + + + QLineEdit::Password + + + + + + + Your profile is associated with a PGP key pair. RetroShare currently ignores DSA keys. + + + + + + + Name + + + + + + + + 0 + 0 + + + + + 0 + 0 + + + + + 16777215 + 32 + + + + Enter your nickname here + + + 64 + + + + + + + + + + Nickname + + + + + + + You can have one or more identities. They are used when you write in chat lobbies, forums and channel comments. They act as the destination for distant chat and the Retroshare distant mail system. + + + + + + + Email + + + + + + + Be careful: this email will be visible to your friends and friends of your friends. This information is required by PGP, but to stay anonymous, you can use a fake email. - - - 64 - - - - - - - This password is for PGP - - - Password - - - - - - - Put a strong password here. This password protects your private PGP key. - - - - - - 1024 - - - QLineEdit::Password - - - - - - - This password is for PGP - - - Password (check) - - - - - - - - 0 - 0 - - - - Put a strong password here. This password protects your private PGP key. - - - - - - 1024 - - - QLineEdit::Password - - - - - - - Node name - - - - - - - - 0 - 0 - - - - 64 - - - - - - - hidden address - - - - - - - 6 - - - - - - 0 - 0 - - - - - 16777215 - 32 - - - - 64 - - - - - - - Port - - - - - - - <html><head/><body><p>This is your connection port.</p><p>Any value between 1024 and 65535 </p><p>should be ok. You can change it later.</p></body></html> - - - 1 - - - 65535 - - - 7812 - - - - - - - - - <html><head/><body><p>This can be a Tor Onion address of the form: xa76giaf6ifda7ri63i263.onion <br/>or an I2P address in the form: [52 characters].b32.i2p </p><p>In order to get one, you must configure either Tor or I2P to create a new hidden service / server tunnel. If you do not yet have one, you can still go on, and make it right later in RetroShare's Options-&gt;Network-&gt;Hidden Service configuration panel.</p></body></html> - - - true - - - - - - - PGP key length - - - - - - - - - - - - - - - - - 255 - 255 - 255 - - - - - - - 255 - 255 - 178 - - - - - - - - - 255 - 255 - 255 - - - - - - - 255 - 255 - 178 - - - - - - - - - 255 - 255 - 178 - - - - - - - 255 - 255 - 178 - - - - - - - - true - - - QFrame::Box - - - <html><head/><body><p align="justify">Before proceeding, move your mouse around to help Retroshare collect as much randomness as possible. Filling the progressbar to 20% is needed, 100% is advised.</p></body></html> - - - true - - - - - - - 24 - - - - - - - - 16777215 - 26 - - - - - 50 - false - - - - - - - Create new profile - - - - :/images/resume.png:/images/resume.png - + + + 64 + + + + + + + This password is for PGP + + + Password + + + + + + + Put a strong password here. This password protects your private PGP key. + + + + + + 1024 + + + QLineEdit::Password + + + + + + + hidden address + + + + + + + 6 + + + + + + 0 + 0 + + + + + 16777215 + 32 + + + + 64 + + + + + + + Port + + + + + + + <html><head/><body><p>This is your connection port.</p><p>Any value between 1024 and 65535 </p><p>should be ok. You can change it later.</p></body></html> + + + 1 + + + 65535 + + + 7812 + + + + + + + + + <html><head/><body><p>This can be a Tor Onion address of the form: xa76giaf6ifda7ri63i263.onion <br/>or an I2P address in the form: [52 characters].b32.i2p </p><p>In order to get one, you must configure either Tor or I2P to create a new hidden service / server tunnel. If you do not yet have one, you can still go on, and make it right later in RetroShare's Options-&gt;Network-&gt;Hidden Service configuration panel.</p></body></html> + + + true + + + + + + + PGP key length + + + + + + + + + + + + + + + 255 + 255 + 255 + + + + + + + 255 + 255 + 178 + + + + + + + + + 255 + 255 + 255 + + + + + + + 255 + 255 + 178 + + + + + + + + + 255 + 255 + 178 + + + + + + + 255 + 255 + 178 + + + + + + + + true + + + QFrame::Box + + + <html><head/><body><p align="justify">Before proceeding, move your mouse around to help Retroshare collect as much randomness as possible. Filling the progressbar to 20% is needed, 100% is advised.</p></body></html> + + + true + + + + + + + 24 + + + + + + + + + + 16777215 + 32 + + + + Your profile is associated with a PGP key pair + + + Create a new profile + + + + :/icons/svg/profile.svg:/icons/svg/profile.svg + + + + 32 + 32 + + + + false + + + + + + + + 0 + 0 + + + + Import new profile + + + + + + + + 0 + 0 + + + + Export selected profile + + + + + + + + 16777215 + 32 + + + + Your profile is associated with a PGP key pair + + + Advanced options + + + + :/icons/settings/general.svg:/icons/settings/general.svg + + + + 24 + 24 + + + + false + + + + + + + Create a hidden node + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + 16777215 + 26 + + + + + 50 + false + + + + + + + Create new profile + + + + + + + + + + + + + + 255 + 255 + 255 + + + + + + + 255 + 255 + 178 + + + + + + + + + 255 + 255 + 255 + + + + + + + 255 + 255 + 178 + + + + + + + + + 255 + 255 + 178 + + + + + + + 255 + 255 + 178 + + + + + + + + true + + + QFrame::Box + + + You can create a new profile with this form. +Alternatively you can use an existing profile. Just uncheck "Create a new profile" + + + true + + + + @@ -741,7 +738,6 @@ anonymous, you can use a fake email. importIdentity_PB exportIdentity_PB adv_checkbox - hidden_checkbox genPGPuser name_input nickname_input @@ -751,10 +747,10 @@ anonymous, you can use a fake email. node_input hiddenaddr_input hiddenport_spinBox - genButton + diff --git a/retroshare-gui/src/gui/icons.qrc b/retroshare-gui/src/gui/icons.qrc index df040eb60..d34e11b0c 100644 --- a/retroshare-gui/src/gui/icons.qrc +++ b/retroshare-gui/src/gui/icons.qrc @@ -1,5 +1,6 @@ + icons/svg/profile.svg icons/svg/download.svg icons/svg/folders.svg icons/svg/folders1.svg diff --git a/retroshare-gui/src/gui/icons/svg/profile.svg b/retroshare-gui/src/gui/icons/svg/profile.svg new file mode 100644 index 000000000..c915ebd49 --- /dev/null +++ b/retroshare-gui/src/gui/icons/svg/profile.svg @@ -0,0 +1,77 @@ + + + +image/svg+xml \ No newline at end of file diff --git a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss index 29f7d1036..c229c2052 100644 --- a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss +++ b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss @@ -639,3 +639,22 @@ IdEditDialog QLabel#info_label background-color: qlineargradient(x1:0, y1:0, x2:0, y2:1, stop:0 #FFFFD7, stop:1 #FFFFB2); } +GenCertDialog QPushButton#genButton { + border-image: url(:/images/btn_blue.png) 4; + border-width: 4; + padding: 0px 6px; + font-size: 16px; + font: bold; + color: white; +} + +GenCertDialog QPushButton#genButton:hover { + border-image: url(:/images/btn_blue_hover.png) 4; +} + +GenCertDialog QPushButton#genButton:disabled { + border-image: url(:/images/btn_27.png) 4; + font-size: 16px; + font: bold; + color: black; +} From ad1ba7a77dddf94cc89a0505b9f2d509f1d26372 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 6 Sep 2016 23:05:46 +0200 Subject: [PATCH 060/174] fixed loading/saving of remote directories --- .../src/file_sharing/dir_hierarchy.cc | 39 ++++++++++++++----- .../src/file_sharing/directory_storage.cc | 5 +++ libretroshare/src/file_sharing/filelist_io.h | 2 + libretroshare/src/file_sharing/p3filelists.cc | 4 ++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index b6713c9d6..d0d40d7bf 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -570,6 +570,8 @@ bool InternalFileHierarchyStorage::save(const std::string& fname) uint32_t file_section_offset = 0 ; uint32_t file_section_size = 0; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,(uint32_t)fe.parent_index)) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ROW ,(uint32_t)fe.row )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,fe.file_name )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,fe.file_size )) throw std::runtime_error("Write error") ; @@ -588,6 +590,8 @@ bool InternalFileHierarchyStorage::save(const std::string& fname) uint32_t dir_section_offset = 0 ; uint32_t dir_section_size = 0; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,(uint32_t)de.parent_index )) throw std::runtime_error("Write error") ; + if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_ROW ,(uint32_t)de.row )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,(uint32_t)i )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,de.dir_name )) throw std::runtime_error("Write error") ; if(!FileListIO::writeField(dir_section_data,dir_section_size,dir_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,de.dir_parent_path )) throw std::runtime_error("Write error") ; @@ -633,7 +637,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) try { - if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_offset) ) + if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_size) ) throw std::runtime_error("Read error") ; // Read some header @@ -667,17 +671,26 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint64_t file_size ; RsFileHash file_hash ; uint32_t file_modtime ; + uint32_t row ; + uint32_t parent_index ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw std::runtime_error("Write error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw std::runtime_error("Write error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw std::runtime_error("Read error") ; if(node_index >= mNodes.size()) mNodes.resize(node_index+1,NULL) ; - mNodes[node_index] = new FileEntry(file_name,file_size,file_modtime,file_hash); + FileEntry *fe = new FileEntry(file_name,file_size,file_modtime,file_hash); + + fe->parent_index = parent_index ; + fe->row = row ; + + mNodes[node_index] = fe ; } else if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size)) { @@ -686,8 +699,12 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) std::string dir_parent_path ; uint32_t dir_modtime ; uint32_t dir_update_time ; - uint64_t dir_most_recent_time ; + uint32_t dir_most_recent_time ; + uint32_t row ; + uint32_t parent_index ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; @@ -705,6 +722,9 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) de->dir_update_time = dir_update_time ; de->dir_most_recent_time = dir_most_recent_time ; + de->parent_index = parent_index ; + de->row = row ; + uint32_t n_subdirs = 0 ; uint32_t n_subfiles = 0 ; @@ -725,6 +745,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,fi)) throw std::runtime_error("Read error") ; de->subfiles.push_back(fi) ; } + mNodes[node_index] = de ; } else throw std::runtime_error("Unknown node section.") ; @@ -736,7 +757,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) } catch(std::exception& e) { - std::cerr << "Error while writing: " << e.what() << std::endl; + std::cerr << "Error while reading: " << e.what() << std::endl; if(buffer != NULL) free(buffer) ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index ba10db1a2..94e5f74ed 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -545,6 +545,11 @@ RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::st : DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname) { load(fname) ; + + std::cerr << "***************************************" << std::endl; + std::cerr << "Loaded following directory for peer " << pid << std::endl; + mFileHierarchy->print(); + std::cerr << "***************************************" << std::endl; } void RemoteDirectoryStorage::checkSave() diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 408ba26c8..db4df001a 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -28,6 +28,8 @@ static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x0d ; static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x0e ; static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x0f ; +static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x10 ; +static const uint8_t FILE_LIST_IO_TAG_ROW = 0x11 ; static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9e1b323e2..600e9edec 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -199,6 +199,9 @@ int p3FileDatabase::tick() } mLastRemoteDirSweepTS = now; + +#warning hack to make loaded directories show up in the GUI, because the GUI isn't ready at the time they are actually loaded up. + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } return 0; @@ -412,6 +415,7 @@ void p3FileDatabase::cleanup() mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } From eaa8ad883a65ecc4e41da856583d7c95c433a293 Mon Sep 17 00:00:00 2001 From: MrAlice Date: Wed, 7 Sep 2016 22:31:12 +0200 Subject: [PATCH 061/174] half implemented the hash-based communication for dirs between friends --- .../src/file_sharing/dir_hierarchy.cc | 4 +- .../src/file_sharing/dir_hierarchy.h | 16 ++++- .../src/file_sharing/directory_storage.h | 5 +- libretroshare/src/file_sharing/p3filelists.cc | 66 ++++++++++++++----- .../src/file_sharing/rsfilelistitems.cc | 16 ++--- .../src/file_sharing/rsfilelistitems.h | 16 ++--- 6 files changed, 87 insertions(+), 36 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d0d40d7bf..f7e8fe61a 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -393,9 +393,9 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(Direct bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list& results) { - std::map::const_iterator it = mHashes.find(hash); + std::map::const_iterator it = mFileHashes.find(hash); - if( it != mHashes.end() ) + if( it != mFileHashes.end() ) { results.clear(); results.push_back(it->second) ; diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index f0b109c7d..3115543d5 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -117,5 +117,19 @@ private: bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); - std::map mHashes ; // used for fast search access. We should try something faster than std::map. hash_map?? + // Map of the hash of all files and all directories. The file hashes are the sha1sum of the file data. + // is used for fast search access for FT. + // Note: We should try something faster than std::map. hash_map?? + + std::map mFileHashes ; + + // The directory hashes are the sha1sum of the + // full public path to the directory. + // The later is used by synchronisation items in order + // to avoid sending explicit EntryIndex values. + // This is kept separate from mFileHashes because the two are used + // in very different ways. + // + std::map mDirHashes ; }; + diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 302c24be5..7d98730ac 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -94,13 +94,16 @@ class DirectoryStorage const RsPeerId& peerId() const { return mPeerId ; } int parentRow(EntryIndex e) const ; - bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; + bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); + bool getHashFromIndex(const EntryIndex& index,RsFileHash& hash) const { NOT_IMPLEMENTED() ; return false; } + bool getIndexFromHash(const RsFileHash& hash,EntryIndex& index) const { NOT_IMPLEMENTED() ; return false; } + void print(); void cleanup(); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 600e9edec..d53f3c578 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -200,7 +200,7 @@ int p3FileDatabase::tick() mLastRemoteDirSweepTS = now; -#warning hack to make loaded directories show up in the GUI, because the GUI isn't ready at the time they are actually loaded up. +#warning hack to make loaded directories show up in the GUI, because the GUI isn_t ready at the time they are actually loaded up. RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } @@ -864,6 +864,9 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // - local node sends the last known modf time to friends, // - friends respond with either a full directory content, or an acknowledge that the time is right // +// Directories are designated by their hash, instead of their index. This allows to hide the non shared directories +// behind a layer of abstraction, at the cost of a logarithmic search, which is acceptable as far as dir sync-ing between +// friends is concerned (We obviously could not do that for GUI display, which has a small and constant cost). void p3FileDatabase::tickRecv() { @@ -898,12 +901,20 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) { RS_STACK_MUTEX(mFLSMtx) ; - P3FILELISTS_DEBUG() << "Received directory sync request. index=" << item->entry_index << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; + P3FILELISTS_DEBUG() << "Received directory sync request. hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; - uint32_t entry_type = mLocalSharedDirs->getEntryType(item->entry_index) ; + EntryIndex entry_index = DirectoryStorage::NO_INDEX; + + if(!mLocalSharedDirs->getIndexFromHash(item->entry_hash,entry_index)) + { + std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + return; + } + + uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ; ritem->PeerId(item->PeerId()) ; ritem->request_id = item->request_id; - ritem->entry_index = item->entry_index ; + ritem->entry_hash = item->entry_hash ; std::list node_groups; FileStorageFlags node_flags; @@ -913,15 +924,15 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } - else if(item->entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(item->entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) + else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)item->entry_index << ", or permission denied." << std::endl; + std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else { time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getDirUpdateTS(item->entry_index,local_recurs_max_time,local_update_time); + mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time); if(item->last_known_recurs_modf_TS < local_recurs_max_time) { @@ -931,7 +942,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) ritem->last_known_recurs_modf_TS = local_recurs_max_time; // We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer. - mLocalSharedDirs->serialiseDirEntry(item->entry_index,ritem->directory_content_data,item->PeerId()) ; + mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ; } else { @@ -950,13 +961,21 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { - P3FILELISTS_DEBUG() << "Handling sync response for directory with index " << item->entry_index << std::endl; + P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl; + + EntryIndex entry_index = DirectoryStorage::NO_INDEX; // remove the original request from pending list { RS_STACK_MUTEX(mFLSMtx) ; + if(entry_index == DirectoryStorage::NO_INDEX) + { + std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + return; + } + std::map::iterator it = mPendingSyncRequests.find(item->request_id) ; if(it == mPendingSyncRequests.end()) @@ -984,12 +1003,19 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(mRemoteDirectories[fi] == NULL) mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); + + if(!mRemoteDirectories[fi]->getIndexFromHash(item->entry_hash,entry_index)) + { + std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; + return ; + } + std::cerr << " entry index is " << entry_index ; } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) { - P3FILELISTS_DEBUG() << " removing directory with index " << item->entry_index << " because it does not exist." << std::endl; - mRemoteDirectories[fi]->removeDirectory(item->entry_index); + P3FILELISTS_DEBUG() << " removing directory with index " << entry_index << " because it does not exist." << std::endl; + mRemoteDirectories[fi]->removeDirectory(entry_index); mRemoteDirectories[fi]->print(); } @@ -997,13 +1023,13 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setDirUpdateTS(item->entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; - if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(item->entry_index,item->directory_content_data)) + if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; @@ -1051,7 +1077,13 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; - item->entry_index = e ; + + if(!rds->getHashFromIndex(e,item->entry_hash) ) + { + std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; + return; + } + item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; item->request_id = sync_req_id ; item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; @@ -1080,14 +1112,16 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) { + static uint64_t random_bias = RSRandom::random_u64(); uint64_t r = e ; - // This is kind of arbitrary. The important thing is that the same ID needs to be generated for a given (peer_id,e) pair. + // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,e) pair, in a way + // that cannot be brute-forced or reverse-engineered, which explains the random bias. for(uint32_t i=0;ientry_index); + ok &= item->entry_hash.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -206,7 +206,7 @@ RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncRespons uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. uint64_t request_id; // use to determine if changes that have occured since last hash - ok &= getRawUInt32(data, *size, &offset, &item->entry_index); + ok &= item->entry_hash.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -274,7 +274,7 @@ uint32_t RsFileListsSyncRequestItem::serial_size()const uint32_t s = 8; //header size - s += 4; // entry index + s += RsFileHash::serial_size(); // entry hash s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -287,7 +287,7 @@ uint32_t RsFileListsSyncResponseItem::serial_size()const uint32_t s = 8; //header size - s += 4; // entry index + s += RsFileHash::serial_size(); // entry hash s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -308,7 +308,7 @@ std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t inde printRsItemBase(out, "RsFileListsSyncReqItem", indent); uint16_t int_Indent = indent + 2; - printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; @@ -323,7 +323,7 @@ std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t ind printRsItemBase(out, "RsFileListsSyncDirItem", indent); uint16_t int_Indent = indent + 2; - printIndent(out , int_Indent); out << "Entry index: " << entry_index << std::endl; + printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 1ff00d57a..5644d4648 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -84,10 +84,10 @@ public: virtual bool serialise(void *data,uint32_t& size) const; virtual uint32_t serial_size() const ; - uint32_t entry_index ; // index of the directory to sync - uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + RsFileHash entry_hash ; // hash of the directory to sync + uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash }; class RsFileListsSyncResponseItem : public RsFileListsItem @@ -102,10 +102,10 @@ public: virtual bool serialise(void *data,uint32_t& size) const; virtual uint32_t serial_size() const ; - uint32_t entry_index ; // advises whether to use sync hash - uint32_t flags; // is it a partial/final item (used for large items only) - uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. - uint64_t request_id; // use to determine if changes that have occured since last hash + RsFileHash entry_hash ; // hash of the directory to sync + uint32_t flags; // is it a partial/final item (used for large items only) + uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. + uint64_t request_id; // use to determine if changes that have occured since last hash RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser. }; From fafe684cc40917f72b5305f805b50f2ea9f06d3e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 8 Sep 2016 21:34:53 +0200 Subject: [PATCH 062/174] finished implementing hash-based transactions. To be tested. --- .../src/file_sharing/dir_hierarchy.cc | 104 ++++++++++++++---- .../src/file_sharing/dir_hierarchy.h | 2 +- .../src/file_sharing/directory_storage.cc | 46 +++++--- 3 files changed, 110 insertions(+), 42 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index f7e8fe61a..94fa8eaff 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -227,7 +227,7 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex return true; } -bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array) +bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_hash,const std::vector& subfiles_hash) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) { @@ -241,47 +241,103 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI d.dir_update_time = time(NULL); d.dir_name = dir_name; - d.subfiles = subfiles_array ; - d.subdirs = subdirs_array ; + d.subdirs.clear(); + d.subfiles.clear(); // check that all subdirs already exist. If not, create. - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subdirs_array[i]+1,NULL); + DirectoryStorage::EntryIndex dir_index = 0; + std::map::const_iterator it = mDirHashes.find(subdirs_hash[i]) ; - FileStorageNode *& node(mNodes[subdirs_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_DIR) + if(it == mDirHashes.end() || it->second >= mNodes.size()) { - delete node ; - node = NULL ; - } + // find an epty slot + int found = -1 ; - if(node == NULL) + for(uint32_t j=0;jsecond; + + if(mNodes[dir_index] != NULL && mNodes[dir_index]->type() != FileStorageNode::TYPE_DIR) + { + delete mNodes[dir_index] ; + mNodes[dir_index] = NULL ; + } + } + FileStorageNode *& node(mNodes[dir_index]); + if(!node) node = new DirEntry(""); + d.subdirs.push_back(dir_index) ; + ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; node->row = i ; node->parent_index = indx ; } - for(uint32_t i=0;i= mNodes.size() ) - mNodes.resize(subfiles_array[i]+1,NULL); + DirectoryStorage::EntryIndex file_index = 0; + std::map::const_iterator it = mFileHashes.find(subfiles_hash[i]) ; - FileStorageNode *& node(mNodes[subfiles_array[i]]); - - if(node != NULL && node->type() != FileStorageNode::TYPE_FILE) + if(it == mFileHashes.end()) { - delete node ; - node = NULL ; + // find an epty slot + int found = -1; + + for(uint32_t j=0;jsecond ; - if(node == NULL) - node = new FileEntry("",0,0); + if(mNodes[file_index] != NULL && mNodes[file_index]->type() != FileStorageNode::TYPE_FILE) + { + delete mNodes[file_index] ; + mNodes[file_index] = NULL ; + } - node->row = subdirs_array.size()+i ; + file_index = it->second; + } + FileStorageNode *& node(mNodes[file_index]); + if(!node) + node = new FileEntry("",0,0,subfiles_hash[i]); + + d.subfiles.push_back(file_index) ; + + node->row = subdirs_hash.size()+i ; node->parent_index = indx ; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 3115543d5..13a3f5c3e 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -73,7 +73,7 @@ public: bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); - bool updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_array,const std::vector& subfiles_array); + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_hash); bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 94e5f74ed..bdcd24c84 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -466,7 +466,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary } // compute list of allowed subdirs - std::vector allowed_subdirs ; + std::vector allowed_subdirs ; FileStorageFlags node_flags ; std::list node_groups ; @@ -474,7 +474,16 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary for(uint32_t i=0;isubdirs.size();++i) if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) - allowed_subdirs.push_back(dir->subdirs[i]) ; + { + RsFileHash hash ; + + if(!getHashFromIndex(dir->subdirs[i],hash)) + { + std::cerr << "(EE) cannot get hash from index for subdir " << dir->subdirs[i] << " at position " << i << " in subdirs list. Weird." << std::endl; + continue ; + } + allowed_subdirs.push_back(hash) ; + } unsigned char *section_data = NULL; uint32_t section_size = 0; @@ -498,7 +507,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // serialise subdirs entry indexes for(uint32_t i=0;isubfiles[i] )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ; if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ; @@ -594,14 +602,14 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co // serialise subdirs entry indexes - std::vector subdirs_array ; - uint32_t subdir_index ; + std::vector subdirs_hashes ; + RsFileHash subdir_hash ; for(uint32_t i=0;iupdateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_array,subfiles_array)) + // First create the entries for each subdir and each subfile, if needed. + if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_hash)) { std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl; return false ; } - // then update the subfiles - for(uint32_t i=0;iupdateFile(subfiles_array[i],subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) - std::cerr << "(EE) Cannot update file with index " << subfiles_array[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; + if(!mFileHierarchy->updateFile(file_index,subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) + std::cerr << "(EE) Cannot update file with index " << file_index <<" and hash " << subfiles_hash[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; } mChanged = true ; From 0ea695beb3dd1b1f71828ff0e8fcdcb38ed60f07 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 8 Sep 2016 23:43:14 +0200 Subject: [PATCH 063/174] fixed a few bugs in hash-based sync. Not there yet. --- .../src/file_sharing/dir_hierarchy.cc | 32 +++++++++++++++++ .../src/file_sharing/dir_hierarchy.h | 7 ++++ .../src/file_sharing/directory_storage.cc | 18 ++++++++-- .../src/file_sharing/directory_storage.h | 4 +-- libretroshare/src/file_sharing/p3filelists.cc | 34 ++++++++----------- retroshare-gui/src/gui/SharedFilesDialog.cpp | 20 +++++++++++ 6 files changed, 91 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 94fa8eaff..8cf824110 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -1,3 +1,5 @@ +#include "util/rsdir.h" + #include "dir_hierarchy.h" #include "filelist_io.h" @@ -28,6 +30,36 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0) de->dir_modtime=0; mNodes.push_back(de) ; +#warning not very elegant. We should remove the leading / + mDirHashes[computeDirHash("/")] = 0 ; +} + +RsFileHash InternalFileHierarchyStorage::computeDirHash(const std::string& dir_path) +{ + return RsDirUtil::sha1sum((unsigned char*)dir_path.c_str(),dir_path.length()) ; +} +bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const +{ + if(!checkIndex(index,FileStorageNode::TYPE_DIR)) + return false ; + + DirEntry& d = *static_cast(mNodes[index]) ; + + hash = computeDirHash( d.dir_parent_path + "/" + d.dir_name ) ; + + std::cerr << "Computing dir hash from index " << index << ". Dir=\"" << d.dir_parent_path + "/" + d.dir_name << "\" hash=" << hash << std::endl; + + return true; +} +bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const +{ + std::map::const_iterator it = mDirHashes.find(hash) ; + + if(it == mDirHashes.end()) + return false; + + index = it->second; + return true; } int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 13a3f5c3e..5d7fb033a 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -81,6 +81,13 @@ public: time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index); + // hash stuff + + static RsFileHash computeDirHash(const std::string& dir_path); + + bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ; + bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; + // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index bdcd24c84..c8a0b0cc3 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -193,6 +193,8 @@ bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) { + RS_STACK_MUTEX(mDirStorageMtx) ; + d.children.clear() ; time_t now = time(NULL) ; @@ -266,6 +268,18 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return true; } + +bool DirectoryStorage::getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->getDirHashFromIndex(index,hash) ; +} +bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->getIndexFromDirHash(hash,index) ; +} + /******************************************************************************************************************/ /* Local Directory Storage */ /******************************************************************************************************************/ @@ -477,7 +491,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary { RsFileHash hash ; - if(!getHashFromIndex(dir->subdirs[i],hash)) + if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash)) { std::cerr << "(EE) cannot get hash from index for subdir " << dir->subdirs[i] << " at position " << i << " in subdirs list. Weird." << std::endl; continue ; @@ -664,7 +678,7 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co { DirectoryStorage::EntryIndex file_index ; - if(!getIndexFromHash(subfiles_hash[i],file_index)) + if(!getIndexFromDirHash(subfiles_hash[i],file_index)) { std::cerr << "(EE) Cannot optain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; continue; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 7d98730ac..fc9aadf7f 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -101,8 +101,8 @@ class DirectoryStorage bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); - bool getHashFromIndex(const EntryIndex& index,RsFileHash& hash) const { NOT_IMPLEMENTED() ; return false; } - bool getIndexFromHash(const RsFileHash& hash,EntryIndex& index) const { NOT_IMPLEMENTED() ; return false; } + bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; + bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; void print(); void cleanup(); diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index d53f3c578..60312124e 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -12,7 +12,7 @@ #include "rsserver/p3face.h" -#define P3FILELISTS_DEBUG() std::cerr << "p3FileLists: " +#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << ": p3FileLists: " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; @@ -187,7 +187,7 @@ int p3FileDatabase::tick() { if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) { - std::cerr << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; + P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; #ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[i]->print(); #endif @@ -210,20 +210,20 @@ int p3FileDatabase::tick() void p3FileDatabase::startThreads() { RS_STACK_MUTEX(mFLSMtx) ; - std::cerr << "Starting directory watcher thread..." ; + P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ; mLocalDirWatcher->start(); - std::cerr << "Done." << std::endl; + P3FILELISTS_DEBUG() << "Done." << std::endl; } void p3FileDatabase::stopThreads() { RS_STACK_MUTEX(mFLSMtx) ; - std::cerr << "Stopping hash cache thread..." ; std::cerr.flush() ; + P3FILELISTS_DEBUG() << "Stopping hash cache thread..." ; std::cerr.flush() ; mHashCache->fullstop(); - std::cerr << "Done." << std::endl; + P3FILELISTS_DEBUG() << "Done." << std::endl; - std::cerr << "Stopping directory watcher thread..." ; std::cerr.flush() ; + P3FILELISTS_DEBUG() << "Stopping directory watcher thread..." ; std::cerr.flush() ; mLocalDirWatcher->fullstop(); - std::cerr << "Done." << std::endl; + P3FILELISTS_DEBUG() << "Done." << std::endl; } void p3FileDatabase::tickWatchers() @@ -367,7 +367,7 @@ void p3FileDatabase::cleanup() { RS_STACK_MUTEX(mFLSMtx) ; - std::cerr << "p3FileDatabase::cleanup()" << std::endl; + P3FILELISTS_DEBUG() << "p3FileDatabase::cleanup()" << std::endl; // look through the list of friend directories. Remove those who are not our friends anymore. // @@ -429,7 +429,7 @@ void p3FileDatabase::cleanup() for(std::map::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();) if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now) { - std::cerr << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; + P3FILELISTS_DEBUG() << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; std::map::iterator tmp(it); ++tmp; @@ -438,7 +438,7 @@ void p3FileDatabase::cleanup() } else { - std::cerr << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; + P3FILELISTS_DEBUG() << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; ++it ; } } @@ -905,7 +905,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) EntryIndex entry_index = DirectoryStorage::NO_INDEX; - if(!mLocalSharedDirs->getIndexFromHash(item->entry_hash,entry_index)) + if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index)) { std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; return; @@ -970,12 +970,6 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { RS_STACK_MUTEX(mFLSMtx) ; - if(entry_index == DirectoryStorage::NO_INDEX) - { - std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; - return; - } - std::map::iterator it = mPendingSyncRequests.find(item->request_id) ; if(it == mPendingSyncRequests.end()) @@ -1004,7 +998,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(mRemoteDirectories[fi] == NULL) mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); - if(!mRemoteDirectories[fi]->getIndexFromHash(item->entry_hash,entry_index)) + if(!mRemoteDirectories[fi]->getIndexFromDirHash(item->entry_hash,entry_index)) { std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; @@ -1078,7 +1072,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; - if(!rds->getHashFromIndex(e,item->entry_hash) ) + if(!rds->getDirHashFromIndex(e,item->entry_hash) ) { std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; return; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 209878f68..a0864839a 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -85,6 +85,8 @@ // #define DONT_USE_SEARCH_IN_TREE_VIEW 1 +//#define DEBUG_SHARED_FILES_DIALOG 1 + const QString Image_AddNewAssotiationForFile = ":/images/kcmsystem24.png"; class SFDSortFilterProxyModel : public QSortFilterProxyModel @@ -842,7 +844,9 @@ void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexe if(ui.dirTreeView->model() == NULL) return ; +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Saving expanded items. " << std::endl; +#endif for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); @@ -855,7 +859,9 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand if(ui.dirTreeView->model() == NULL) return ; +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Restoring expanded items. " << std::endl; +#endif for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); @@ -866,29 +872,39 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp) { std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; +#endif if(ui.dirTreeView->isExpanded(index)) { +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Index " << local_path << " is expanded." << std::endl; +#endif if(index.isValid()) exp.insert(local_path) ; for(int row=0;rowmodel()->rowCount(index);++row) recursSaveExpandedItems(index.child(row,0),local_path,exp) ; } +#ifdef DEBUG_SHARED_FILES_DIALOG else std::cerr << "Index is not expanded." << std::endl; +#endif } void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set& exp) { std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; +#endif if(exp.find(local_path) != exp.end()) { +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "re expanding index " << local_path << std::endl; +#endif ui.dirTreeView->setExpanded(index,true) ; for(int row=0;rowmodel()->rowCount(index);++row) @@ -904,7 +920,9 @@ void SharedFilesDialog::postModDirectories(bool local) } std::set expanded_indexes; saveExpandedPaths(expanded_indexes) ; +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; +#endif /* Notify both models, only one is visible */ tree_model->postMods(); @@ -916,7 +934,9 @@ void SharedFilesDialog::postModDirectories(bool local) if (ui.filterPatternLineEdit->text().isEmpty() == false) FilterItems(); +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "****** updated directories! ******" << std::endl; +#endif QCoreApplication::flush(); } From 384d001f3bd9a5ba22ad16a0d8c6879216cf96bb Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 10 Sep 2016 13:57:05 +0200 Subject: [PATCH 064/174] progress in hash-based sync. Not there yet. --- .../src/file_sharing/dir_hierarchy.cc | 70 ++++++++++++++++--- .../src/file_sharing/dir_hierarchy.h | 4 +- .../src/file_sharing/directory_storage.cc | 65 ++++++++++++++--- .../src/file_sharing/directory_storage.h | 4 ++ libretroshare/src/file_sharing/filelist_io.h | 42 ++++++----- libretroshare/src/file_sharing/p3filelists.cc | 4 +- 6 files changed, 147 insertions(+), 42 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 8cf824110..f62000343 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -28,26 +28,20 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0) de->row=0; de->parent_index=0; de->dir_modtime=0; + de->dir_hash=RsFileHash() ; // null hash is root by convention. mNodes.push_back(de) ; + #warning not very elegant. We should remove the leading / - mDirHashes[computeDirHash("/")] = 0 ; + mDirHashes[de->dir_hash] = 0 ; } -RsFileHash InternalFileHierarchyStorage::computeDirHash(const std::string& dir_path) -{ - return RsDirUtil::sha1sum((unsigned char*)dir_path.c_str(),dir_path.length()) ; -} bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) return false ; - DirEntry& d = *static_cast(mNodes[index]) ; - - hash = computeDirHash( d.dir_parent_path + "/" + d.dir_name ) ; - - std::cerr << "Computing dir hash from index " << index << ". Dir=\"" << d.dir_parent_path + "/" + d.dir_name << "\" hash=" << hash << std::endl; + hash = static_cast(mNodes[index])->dir_hash ; return true; } @@ -119,6 +113,10 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage de->row = mNodes.size(); de->parent_index = indx; de->dir_modtime = it->second; + de->dir_parent_path = d.dir_parent_path + "/" + d.dir_name ; + de->dir_hash = createDirHash(de->dir_name,de->dir_parent_path) ; + + mDirHashes[de->dir_hash] = mNodes.size() ; d.subdirs.push_back(mNodes.size()) ; mNodes.push_back(de) ; @@ -126,6 +124,24 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage return true; } + +RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string& dir_name,const std::string& dir_parent_path) +{ + // What we need here: a unique identifier + // - that cannot be bruteforced to find the real directory name and path + // - that is used by friends to refer to a specific directory. + + // Option 1: compute H(some_secret_salt + dir_name + dir_parent_path) + // and keep the same salt so that we can re-create the hash + // + // Option 2: compute H(virtual_path). That only works at the level of LocalDirectoryStorage + // + // Option 3: just compute something random, but then we need to store it so as to not + // confuse friends when restarting. + + return RsFileHash::random(); +} + bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! { // check that it's a directory @@ -268,6 +284,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } DirEntry& d(*static_cast(mNodes[indx])) ; + std::cerr << "Updating dir entry: name=\"" << dir_name << "\", most_recent_time=" << most_recent_time << ", modtime=" << dir_modtime << std::endl; + d.dir_most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; d.dir_update_time = time(NULL); @@ -279,6 +297,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI // check that all subdirs already exist. If not, create. for(uint32_t i=0;i::const_iterator it = mDirHashes.find(subdirs_hash[i]) ; @@ -303,6 +323,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI dir_index = found; mDirHashes[subdirs_hash[i]] = dir_index ; + + std::cerr << " created, at new index " << dir_index << std::endl; } else { @@ -313,6 +335,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI delete mNodes[dir_index] ; mNodes[dir_index] = NULL ; } + std::cerr << " already exists, index=" << dir_index << "." << std::endl; } FileStorageNode *& node(mNodes[dir_index]); if(!node) @@ -329,6 +352,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI DirectoryStorage::EntryIndex file_index = 0; std::map::const_iterator it = mFileHashes.find(subfiles_hash[i]) ; + std::cerr << " subfile hash " << i << ": " << subfiles_hash[i] ; + if(it == mFileHashes.end()) { // find an epty slot @@ -350,6 +375,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI file_index = found; mFileHashes[subfiles_hash[i]] = file_index ; + + std::cerr << " created, at new index " << file_index << std::endl; } else { @@ -362,8 +389,11 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } file_index = it->second; + + std::cerr << " already exists, index=" << file_index << "." << std::endl; } FileStorageNode *& node(mNodes[file_index]); + if(!node) node = new FileEntry("",0,0,subfiles_hash[i]); @@ -579,6 +609,14 @@ void InternalFileHierarchyStorage::print() const std::cerr << "Total nodes: " << mNodes.size() << " (" << nfiles << " files, " << ndirs << " dirs, " << nempty << " empty slots)" << std::endl; recursPrint(0,DirectoryStorage::EntryIndex(0)); + + std::cerr << "Known dir hashes: " << std::endl; + for(std::map::const_iterator it(mDirHashes.begin());it!=mDirHashes.end();++it) + std::cerr << " " << it->first << " at index " << it->second << std::endl; + + std::cerr << "Known file hashes: " << std::endl; + for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) + std::cerr << " " << it->first << " at index " << it->second << std::endl; } void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::EntryIndex node) const { @@ -591,7 +629,7 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry } DirEntry& d(*static_cast(mNodes[node])); - std::cerr << indent << "dir:" << d.dir_name << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.dir_most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; + std::cerr << indent << "dir hash=" << d.dir_hash << ". name:" << d.dir_name << ", parent_path:" << d.dir_parent_path << ", modf time: " << d.dir_modtime << ", recurs_last_modf_time: " << d.dir_most_recent_time << ", parent: " << d.parent_index << ", row: " << d.row << ", subdirs: " ; for(uint32_t i=0;i(mNodes[dir])) ; + RsFileHash hash = d.dir_hash ; + for(uint32_t i=0;irow = row ; mNodes[node_index] = fe ; + mFileHashes[fe->file_hash] = node_index ; } else if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY,node_section_data,node_section_size)) { uint32_t node_index ; std::string dir_name ; std::string dir_parent_path ; + RsFileHash dir_hash ; uint32_t dir_modtime ; uint32_t dir_update_time ; uint32_t dir_most_recent_time ; @@ -795,6 +840,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ,dir_hash )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw std::runtime_error("Read error") ; if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw std::runtime_error("Read error") ; @@ -806,6 +852,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) DirEntry *de = new DirEntry(dir_name) ; de->dir_name = dir_name ; de->dir_parent_path = dir_parent_path ; + de->dir_hash = dir_hash ; de->dir_modtime = dir_modtime ; de->dir_update_time = dir_update_time ; de->dir_most_recent_time = dir_most_recent_time ; @@ -834,6 +881,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) de->subfiles.push_back(fi) ; } mNodes[node_index] = de ; + mDirHashes[de->dir_hash] = node_index ; } else throw std::runtime_error("Unknown node section.") ; diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 5d7fb033a..d84e9e8d4 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -49,6 +49,7 @@ public: // local stuff std::string dir_name ; std::string dir_parent_path ; + RsFileHash dir_hash ; std::vector subdirs ; std::vector subfiles ; @@ -83,8 +84,6 @@ public: // hash stuff - static RsFileHash computeDirHash(const std::string& dir_path); - bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ; bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; @@ -118,6 +117,7 @@ public: private: void recursPrint(int depth,DirectoryStorage::EntryIndex node) const; static bool nodeAccessError(const std::string& s); + static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ; // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning // phase. That allows to keep file information when moving them around. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index c8a0b0cc3..f27bfb16b 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -467,6 +467,58 @@ bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& i return true; } +std::string LocalDirectoryStorage::locked_getVirtualDirName(EntryIndex indx) const +{ + if(indx == 0) + return std::string() ; + + const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); + + if(dir->parent_index != 0) + return dir->dir_name ; + + std::map::const_iterator it = mLocalDirs.find(dir->dir_name) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl; + return std::string() ; + } + + return it->second.virtualname ; +} +std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const +{ + if(indx == 0) + return std::string() ; + + std::string res ; + const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); + + while(dir->parent_index != 0) + { + dir = mFileHierarchy->getDirEntry(dir->parent_index) ; + res += dir->dir_name + "/"+ res ; + } + + std::map::const_iterator it = mLocalDirs.find(dir->dir_name) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl; + return std::string() ; + } + return it->second.virtualname + "/" + res; +} +RsFileHash LocalDirectoryStorage::locked_getDirHashFromIndex(EntryIndex indx) const +{ + // hash the full virtual path + + std::string virtual_path = locked_getVirtualPath(indx) ; + + return RsDirUtil::sha1sum((unsigned char*)virtual_path.c_str(),virtual_path.length()) ; +} + bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id) { RS_STACK_MUTEX(mDirStorageMtx) ; @@ -489,13 +541,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary for(uint32_t i=0;isubdirs.size();++i) if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - RsFileHash hash ; - - if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash)) - { - std::cerr << "(EE) cannot get hash from index for subdir " << dir->subdirs[i] << " at position " << i << " in subdirs list. Weird." << std::endl; - continue ; - } + RsFileHash hash = locked_getDirHashFromIndex(dir->subdirs[i]) ; allowed_subdirs.push_back(hash) ; } @@ -508,10 +554,11 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // - the index entry for each subdir (the updte TS are exchanged at a higher level) // - the file info for each subfile // + std::string virtual_dir_name = locked_getVirtualDirName(indx) ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir->dir_name )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,virtual_dir_name )) return false ; if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ; // serialise number of subdirs and number of subfiles diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index fc9aadf7f..7a69eb7f6 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -205,6 +205,10 @@ public: bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ; private: + RsFileHash locked_getDirHashFromIndex(EntryIndex indx) const ; + std::string locked_getVirtualPath(EntryIndex indx) const ; + std::string locked_getVirtualDirName(EntryIndex indx) const ; + bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index db4df001a..06eee527c 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -11,25 +11,31 @@ static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ; -static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ; +static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x01 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; -static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; -static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; -static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; -static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; -static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; -static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; -static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x09 ; -static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x0a ; -static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x0b ; -static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x0c ; -static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x0d ; -static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x0e ; -static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x0f ; -static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x10 ; -static const uint8_t FILE_LIST_IO_TAG_ROW = 0x11 ; +static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x10 ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x11 ; +static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x12 ; +static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x13 ; + +static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x20 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x21 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x22 ; + +static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x30 ; +static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x31 ; +static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x32 ; + +static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x40 ; +static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x41 ; + +static const uint8_t FILE_LIST_IO_TAG_DIR_HASH = 0x50 ; +static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x51 ; + +static const uint8_t FILE_LIST_IO_TAG_ROW = 0x60 ; +static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x61 ; +static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x62 ; static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 60312124e..c0c603ecc 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -154,9 +154,9 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; -#ifdef DEBUG_FILE_HIERARCHY +//#ifdef DEBUG_FILE_HIERARCHY mLocalSharedDirs->print(); -#endif +//#endif last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI From 88f85383df2e93283b543daf5d0debefabd99b65 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 10 Sep 2016 17:25:29 +0200 Subject: [PATCH 065/174] fixed a few bugs in hash-based syncing --- .../src/file_sharing/dir_hierarchy.cc | 11 +++++++++++ .../src/file_sharing/dir_hierarchy.h | 1 + .../src/file_sharing/directory_storage.cc | 19 ++++++++----------- .../src/file_sharing/directory_storage.h | 1 - libretroshare/src/file_sharing/p3filelists.cc | 17 ++++++++++++----- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index f62000343..5765dd394 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -55,6 +55,16 @@ bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,Di index = it->second; return true; } +bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const +{ + std::map::const_iterator it = mFileHashes.find(hash) ; + + if(it == mFileHashes.end()) + return false; + + index = it->second; + return true; +} int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) { @@ -344,6 +354,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI d.subdirs.push_back(dir_index) ; ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; + ((DirEntry*&)node)->dir_hash = subdirs_hash[i]; node->row = i ; node->parent_index = indx ; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index d84e9e8d4..7d7eb35af 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -86,6 +86,7 @@ public: bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ; bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; + bool getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ; // file/dir access and modification bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index f27bfb16b..fae385d24 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -510,14 +510,6 @@ std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const } return it->second.virtualname + "/" + res; } -RsFileHash LocalDirectoryStorage::locked_getDirHashFromIndex(EntryIndex indx) const -{ - // hash the full virtual path - - std::string virtual_path = locked_getVirtualPath(indx) ; - - return RsDirUtil::sha1sum((unsigned char*)virtual_path.c_str(),virtual_path.length()) ; -} bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id) { @@ -541,7 +533,12 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary for(uint32_t i=0;isubdirs.size();++i) if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - RsFileHash hash = locked_getDirHashFromIndex(dir->subdirs[i]) ; + RsFileHash hash ; + if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash)) + { + std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ; + return false; + } allowed_subdirs.push_back(hash) ; } @@ -725,9 +722,9 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co { DirectoryStorage::EntryIndex file_index ; - if(!getIndexFromDirHash(subfiles_hash[i],file_index)) + if(!mFileHierarchy->getIndexFromFileHash(subfiles_hash[i],file_index)) { - std::cerr << "(EE) Cannot optain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; + std::cerr << "(EE) Cannot obtain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; continue; } std::cerr << " updating file entry " << subfiles_hash[i] << std::endl; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 7a69eb7f6..78fcb46ba 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -205,7 +205,6 @@ public: bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ; private: - RsFileHash locked_getDirHashFromIndex(EntryIndex indx) const ; std::string locked_getVirtualPath(EntryIndex indx) const ; std::string locked_getVirtualDirName(EntryIndex indx) const ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c0c603ecc..07f3cd30b 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1003,7 +1003,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; } - std::cerr << " entry index is " << entry_index ; + std::cerr << " entry index is " << entry_index << " " ; } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) @@ -1029,9 +1029,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; std::cerr << " new content after update: " << std::endl; -#ifdef DEBUG_FILE_HIERARCHY +//#ifdef DEBUG_FILE_HIERARCHY mRemoteDirectories[fi]->print(); -#endif +//#endif } } @@ -1097,7 +1097,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // Dont recurs into sub-directories, since we dont know yet were to go. - return ; + //return ; } for(DirectoryStorage::DirIterator it(rds,e);it;++it) @@ -1113,8 +1113,15 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId // that cannot be brute-forced or reverse-engineered, which explains the random bias. for(uint32_t i=0;i Date: Sat, 10 Sep 2016 20:47:51 +0200 Subject: [PATCH 066/174] fixed last bugs in hash-based dir list exchange. It works! --- .../src/file_sharing/dir_hierarchy.cc | 192 ++++++++++-------- .../src/file_sharing/dir_hierarchy.h | 10 +- .../src/file_sharing/directory_storage.cc | 45 ++-- libretroshare/src/file_sharing/p3filelists.cc | 4 +- 4 files changed, 133 insertions(+), 118 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 5765dd394..0ad931077 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -285,7 +285,26 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex return true; } -bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,time_t most_recent_time,time_t dir_modtime,const std::vector& subdirs_hash,const std::vector& subfiles_hash) +DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex() +{ + int found = -1; + for(uint32_t j=0;j& subdirs_hash,const std::vector& subfiles_array) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) { @@ -301,118 +320,131 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI d.dir_update_time = time(NULL); d.dir_name = dir_name; + std::map existing_subdirs ; + + for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_hash] = d.subdirs[i] ; + d.subdirs.clear(); - d.subfiles.clear(); // check that all subdirs already exist. If not, create. for(uint32_t i=0;i::iterator it = existing_subdirs.find(subdirs_hash[i]) ; DirectoryStorage::EntryIndex dir_index = 0; - std::map::const_iterator it = mDirHashes.find(subdirs_hash[i]) ; - if(it == mDirHashes.end() || it->second >= mNodes.size()) + if(it != existing_subdirs.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_DIR) { - // find an epty slot - int found = -1 ; + dir_index = it->second ; - for(uint32_t j=0;jdir_parent_path = d.dir_parent_path + "/" + dir_name ; + de->dir_hash = subdirs_hash[i]; mDirHashes[subdirs_hash[i]] = dir_index ; std::cerr << " created, at new index " << dir_index << std::endl; } - else - { - dir_index = it->second; - - if(mNodes[dir_index] != NULL && mNodes[dir_index]->type() != FileStorageNode::TYPE_DIR) - { - delete mNodes[dir_index] ; - mNodes[dir_index] = NULL ; - } - std::cerr << " already exists, index=" << dir_index << "." << std::endl; - } - FileStorageNode *& node(mNodes[dir_index]); - if(!node) - node = new DirEntry(""); d.subdirs.push_back(dir_index) ; - - ((DirEntry*&)node)->dir_parent_path = d.dir_parent_path + "/" + dir_name ; - ((DirEntry*&)node)->dir_hash = subdirs_hash[i]; - node->row = i ; - node->parent_index = indx ; + mDirHashes[subdirs_hash[i]] = dir_index ; } - for(uint32_t i=0;i::const_iterator it = existing_subdirs.begin();it!=existing_subdirs.end();++it) { - DirectoryStorage::EntryIndex file_index = 0; - std::map::const_iterator it = mFileHashes.find(subfiles_hash[i]) ; + std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; - std::cerr << " subfile hash " << i << ": " << subfiles_hash[i] ; - - if(it == mFileHashes.end()) + if(!checkIndex(it->second,FileStorageNode::TYPE_DIR)) { - // find an epty slot - int found = -1; - - for(uint32_t j=0;jsecond << " because it is not a file. Inconsistency error!" << std::endl; + continue ; } - else + recursRemoveDirectory(it->second) ; + } + + // now update subfiles. This is more stricky because we need to not suppress hash duplicates + + std::map existing_subfiles ; + + for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_name] = d.subfiles[i] ; + + d.subfiles.clear(); + + for(uint32_t i=0;i::iterator it = existing_subfiles.find(subfiles_array[i].file_name) ; + const FileEntry& f(subfiles_array[i]) ; + DirectoryStorage::EntryIndex file_index ; + + std::cerr << " subfile name = " << subfiles_array[i].file_name << ": " ; + + if(it != existing_subfiles.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_FILE) { file_index = it->second ; - if(mNodes[file_index] != NULL && mNodes[file_index]->type() != FileStorageNode::TYPE_FILE) - { - delete mNodes[file_index] ; - mNodes[file_index] = NULL ; - } + std::cerr << " already exists, at index " << file_index << std::endl; - file_index = it->second; + if(!updateFile(file_index,f.file_hash,f.file_name,f.file_size,f.file_modtime)) + std::cerr << "(EE) Cannot update file with index " << it->second <<" and hash " << f.file_hash << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; - std::cerr << " already exists, index=" << file_index << "." << std::endl; + existing_subfiles.erase(it) ; } - FileStorageNode *& node(mNodes[file_index]); + else + { + file_index = allocateNewIndex() ; - if(!node) - node = new FileEntry("",0,0,subfiles_hash[i]); + mNodes[file_index] = new FileEntry(f.file_name,f.file_size,f.file_modtime,f.file_hash) ; + mFileHashes[f.file_hash] = file_index ; + + std::cerr << " created, at new index " << file_index << std::endl; + } d.subfiles.push_back(file_index) ; - - node->row = subdirs_hash.size()+i ; - node->parent_index = indx ; } + // remove subfiles that do not exist anymore + + for(std::map::const_iterator it = existing_subfiles.begin();it!=existing_subfiles.end();++it) + { + std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; + + if(!checkIndex(it->second,FileStorageNode::TYPE_FILE)) + { + std::cerr << "(EE) Cannot delete node of index " << it->second << " because it is not a file. Inconsistency error!" << std::endl; + continue ; + } + delete mNodes[it->second] ; + mNodes[it->second] = NULL ; + } + + // now update row and parent index for all subnodes + + uint32_t n=0; + for(uint32_t i=0;iparent_index = indx ; + mNodes[d.subdirs[i]]->row = n++ ; + } + for(uint32_t i=0;iparent_index = indx ; + mNodes[d.subfiles[i]]->row = n++ ; + } + return true; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 7d7eb35af..33c5a562e 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -25,6 +25,7 @@ public: class FileEntry: public FileStorageNode { public: + FileEntry() : file_size(0), file_modtime(0) {} FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {} FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {} @@ -74,7 +75,7 @@ public: bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); - bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_hash); + bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_array); bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); @@ -120,14 +121,19 @@ private: static bool nodeAccessError(const std::string& s); static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ; + // Allocates a new entry in mNodes, possible re-using an empty slot and returns its index. + + DirectoryStorage::EntryIndex allocateNewIndex(); + // Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning // phase. That allows to keep file information when moving them around. bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir); - // Map of the hash of all files and all directories. The file hashes are the sha1sum of the file data. + // Map of the hash of all files. The file hashes are the sha1sum of the file data. // is used for fast search access for FT. // Note: We should try something faster than std::map. hash_map?? + // Unlike directories, multiple files may have the same hash. So this cannot be used for anything else than FT. std::map mFileHashes ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index fae385d24..933660986 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -671,11 +671,8 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co } // deserialise directory subfiles, with info for each of them - std::vector subfiles_array ; - std::vector subfiles_name ; - std::vector subfiles_size ; - std::vector subfiles_hash ; - std::vector subfiles_modtime ; + + std::vector subfiles_array ; for(uint32_t i=0;iupdateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_hash)) + if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_array)) { std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl; return false ; } - // Then update the subfiles - for(uint32_t i=0;igetIndexFromFileHash(subfiles_hash[i],file_index)) - { - std::cerr << "(EE) Cannot obtain file entry index for hash " << subfiles_hash[i] << ". This is very unexpected." << std::endl; - continue; - } - std::cerr << " updating file entry " << subfiles_hash[i] << std::endl; - - if(!mFileHierarchy->updateFile(file_index,subfiles_hash[i],subfiles_name[i],subfiles_size[i],subfiles_modtime[i])) - std::cerr << "(EE) Cannot update file with index " << file_index <<" and hash " << subfiles_hash[i] << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; - } mChanged = true ; return true ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 07f3cd30b..c7b659a85 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1106,6 +1106,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) { +#warning needs to be improved. It's quite likely that random_bias and then e can be bruteforced from the result of this function static uint64_t random_bias = RSRandom::random_u64(); uint64_t r = e ; @@ -1117,11 +1118,8 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId r ^= (0x011933ff92892a94 * e + peer_id.toByteArray()[i] * 0x1001fff92ee640f9) ; r <<= 8 ; r ^= 0xf392843890321808; - - std::cerr << std::hex << "r=" << r << std::endl; } - std::cerr << "making request ID: peer_id=" << peer_id << ", e=" << e << ", random_bias=" << std::hex<< random_bias << " returning " << (r^random_bias) << std::dec << std::endl; return r ^ random_bias; } From c1e538ff131b472e8a4f86aa71488f892f3c83ff Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 11 Sep 2016 17:52:12 +0200 Subject: [PATCH 067/174] added interactive update of dir entries when browsing files from an online friend --- libretroshare/src/file_sharing/p3filelists.cc | 146 ++++++++++++------ libretroshare/src/file_sharing/p3filelists.h | 13 +- libretroshare/src/ft/ftserver.cc | 7 +- libretroshare/src/ft/ftserver.h | 1 + libretroshare/src/retroshare/rsfiles.h | 2 + retroshare-gui/src/gui/RemoteDirModel.cpp | 5 + retroshare-gui/src/gui/RemoteDirModel.h | 9 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 9 +- retroshare-gui/src/gui/SharedFilesDialog.h | 1 + 9 files changed, 136 insertions(+), 57 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c7b659a85..9a2663991 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -192,7 +192,7 @@ int p3FileDatabase::tick() mRemoteDirectories[i]->print(); #endif - locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root()) ; + locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root(),0) ; } mRemoteDirectories[i]->checkSave() ; @@ -549,6 +549,30 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi return true; } +void p3FileDatabase::requestDirUpdate(void *ref) +{ + uint32_t fi; + DirectoryStorage::EntryIndex e ; + + convertPointerToEntryIndex(ref,e,fi); + + if(fi == 0) + return ; // not updating current directory (should we?) + + time_t recurs_max_modf_TS_remote_time,local_update_TS; + + std::cerr << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; + + if(!mRemoteDirectories[fi-1]->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) + { + std::cerr << " (EE) Cannot get max known recurs modf time!" << std::endl; + return ; + } + + if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e,recurs_max_modf_TS_remote_time)) + P3FILELISTS_DEBUG() << " Succeed." << std::endl; +} + // This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files. int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const @@ -1035,20 +1059,24 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) } } -void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e) +void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e,int depth) { time_t now = time(NULL) ; + std::string indent(2*depth,' ') ; + // if not up to date, request update, and return (content is not certified, so no need to recurs yet). // if up to date, return, because TS is about the last modif TS below, so no need to recurs either. // get the info for this entry + P3FILELISTS_DEBUG() << "currently at entry index " << e << std::endl; + time_t recurs_max_modf_TS_remote_time,local_update_TS; if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << "(EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; + std::cerr << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; } @@ -1056,73 +1084,89 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if(now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) // we need to compare local times only. We cannot compare local (now) with remote time. { - // check if a request already exists and is not too old either: no need to re-ask. + if(generateAndSendSyncRequest(rds,e,recurs_max_modf_TS_remote_time)) + P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),e) ; + // Dont recurs into sub-directories, since we dont know yet were to go. - std::map::iterator it = mPendingSyncRequests.find(sync_req_id) ; - - if(it != mPendingSyncRequests.end()) - { - P3FILELISTS_DEBUG() << "Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; - return ; - } - - P3FILELISTS_DEBUG() << "Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - - RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; - - if(!rds->getDirHashFromIndex(e,item->entry_hash) ) - { - std::cerr << "(EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; - return; - } - - item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; - item->request_id = sync_req_id ; - item->last_known_recurs_modf_TS = recurs_max_modf_TS_remote_time ; - item->PeerId(rds->peerId()) ; - - DirSyncRequestData data ; - - data.request_TS = now ; - data.peer_id = item->PeerId(); - data.flags = item->flags; - - std::cerr << "Pushing req in pending list with peer id " << data.peer_id << std::endl; - - mPendingSyncRequests[sync_req_id] = data ; - - sendItem(item) ; // at end! Because item is destroyed by the process. - - // Dont recurs into sub-directories, since we dont know yet were to go. - - //return ; + //return ; } for(DirectoryStorage::DirIterator it(rds,e);it;++it) - locked_recursSweepRemoteDirectory(rds,*it); + locked_recursSweepRemoteDirectory(rds,*it,depth+1); } -p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) +p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId& peer_id,const RsFileHash& hash) { -#warning needs to be improved. It's quite likely that random_bias and then e can be bruteforced from the result of this function static uint64_t random_bias = RSRandom::random_u64(); - uint64_t r = e ; + uint64_t r = 0 ; - // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,e) pair, in a way - // that cannot be brute-forced or reverse-engineered, which explains the random bias. + // This is kind of arbitrary. The important thing is that the same ID needs to be generated every time for a given (peer_id,entry index) pair, in a way + // that cannot be brute-forced or reverse-engineered, which explains the random bias and the usage of the hash, that is itself random. for(uint32_t i=0;igetDirHashFromIndex(e,entry_hash) ) + { + std::cerr << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; + return false; + } + + // check if a request already exists and is not too old either: no need to re-ask. + + DirSyncRequestId sync_req_id = makeDirSyncReqId(rds->peerId(),entry_hash) ; + + std::map::iterator it = mPendingSyncRequests.find(sync_req_id) ; + + if(it != mPendingSyncRequests.end()) + { + P3FILELISTS_DEBUG() << " Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; + return false ; + } + + RsFileListsSyncRequestItem *item = new RsFileListsSyncRequestItem ; + + item->entry_hash = entry_hash ; + item->flags = RsFileListsItem::FLAGS_SYNC_REQUEST ; + item->request_id = sync_req_id ; + item->last_known_recurs_modf_TS = max_known_recurs_modf_time ; + item->PeerId(rds->peerId()) ; + + DirSyncRequestData data ; + + data.request_TS = now ; + data.peer_id = item->PeerId(); + data.flags = item->flags; + + std::cerr << " Pushing req in pending list with peer id " << data.peer_id << std::endl; + + mPendingSyncRequests[sync_req_id] = data ; + + sendItem(item) ; // at end! Because item is destroyed by the process. + + return true; +} + + diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index ec5f1b517..8861ed26d 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -92,6 +92,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. + void requestDirUpdate(void *ref) ; // triggers an update. Used when browsing. int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; uint32_t getType(void *) const ; @@ -142,7 +143,15 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub typedef uint64_t DirSyncRequestId ; - static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id,DirectoryStorage::EntryIndex e) ; + static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id, const RsFileHash &hash) ; + + /*! + * \brief generateAndSendSyncRequest + * \param rds Remote directory storage for the request + * \param e Entry index to update + * \return true if the request is correctly sent. + */ + bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e,time_t max_known_recurs_modf_time); // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. @@ -185,7 +194,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub time_t mLastRemoteDirSweepTS ; // TS for friend list update std::map mPendingSyncRequests ; // pending requests, waiting for an answer - void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds,DirectoryStorage::EntryIndex e); + void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds, DirectoryStorage::EntryIndex e, int depth); // We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name. // diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 1268ea831..4b4d4c1e3 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -345,7 +345,12 @@ bool ftServer::FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& return mFtController->getFileDownloadChunksDetails(hash,info); } - /* Directory Handling */ +void ftServer::requestDirUpdate(void *ref) +{ + mFileDatabase->requestDirUpdate(ref) ; +} + + /* Directory Handling */ void ftServer::setDownloadDirectory(std::string path) { mFtController->setDownloadDirectory(path); diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 4e5ae3a61..da1deb009 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -199,6 +199,7 @@ public: /*** * Directory Handling ***/ + virtual void requestDirUpdate(void *ref) ; // triggers the update of the given reference. Used when browsing. virtual void setDownloadDirectory(std::string path); virtual void setPartialsDirectory(std::string path); virtual std::string getDownloadDirectory(); diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index cf1bfc157..d49324933 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -205,6 +205,8 @@ class RsFiles /*** * Directory Control ***/ + virtual void requestDirUpdate(void *ref) =0 ; // triggers the update of the given reference. Used when browsing. + virtual void setDownloadDirectory(std::string path) = 0; virtual void setPartialsDirectory(std::string path) = 0; virtual std::string getDownloadDirectory() = 0; diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index eb5800867..2153f255d 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -80,6 +80,11 @@ void RetroshareDirModel::treeStyle() peerIcon = QIcon(":/images/user/identity16.png"); } +void TreeStyle_RDM::updateRef(const QModelIndex& indx) const +{ + rsFiles->requestDirUpdate(indx.internalPointer()) ; +} + bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const { diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index 062a49a43..54f60ffad 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -78,6 +78,8 @@ class RetroshareDirModel : public QAbstractItemModel bool requestDirDetails(void *ref, bool remote,DirDetails& d) const; virtual void update() {} + virtual void updateRef(const QModelIndex&) const =0; + public: virtual QMimeData * mimeData ( const QModelIndexList & indexes ) const; virtual QStringList mimeTypes () const; @@ -167,7 +169,9 @@ class TreeStyle_RDM: public RetroshareDirModel virtual ~TreeStyle_RDM() ; protected: - /* These are all overloaded Virtual Functions */ + virtual void updateRef(const QModelIndex&) const ; + + /* These are all overloaded Virtual Functions */ virtual int rowCount(const QModelIndex &parent = QModelIndex()) const; virtual int columnCount(const QModelIndex &parent = QModelIndex()) const; @@ -203,7 +207,8 @@ class FlatStyle_RDM: public RetroshareDirModel void updateRefs() ; protected: - virtual void postMods(); + virtual void updateRef(const QModelIndex&) const {} + virtual void postMods(); virtual int rowCount(const QModelIndex &parent = QModelIndex()) const; virtual int columnCount(const QModelIndex &parent = QModelIndex()) const; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index a0864839a..aaaa79284 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -228,6 +228,7 @@ RemoteSharedFilesDialog::RemoteSharedFilesDialog(QWidget *parent) ui.checkButton->hide() ; connect(ui.downloadButton, SIGNAL(clicked()), this, SLOT(downloadRemoteSelected())); + connect(ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), this, SLOT( expanded(const QModelIndex & ) ) ); // load settings processSettings(true); @@ -366,7 +367,6 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) } // connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), this, SLOT( collapsed(const QModelIndex & ) ) ); -// connect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), this, SLOT( expanded(const QModelIndex & ) ) ); ui.dirTreeView->setModel(proxyModel); ui.dirTreeView->update(); @@ -517,6 +517,13 @@ QModelIndexList SharedFilesDialog::getSelected() return proxyList ; } +void RemoteSharedFilesDialog::expanded(const QModelIndex& indx) +{ + std::cerr << "Expanding at " << indx.row() << " and " << indx.column() << " ref=" << indx.internalPointer() << ", pointer at 1: " << proxyModel->mapToSource(indx).internalPointer() << std::endl; + + model->updateRef(proxyModel->mapToSource(indx)) ; +} + void RemoteSharedFilesDialog::downloadRemoteSelected() { /* call back to the model (which does all the interfacing? */ diff --git a/retroshare-gui/src/gui/SharedFilesDialog.h b/retroshare-gui/src/gui/SharedFilesDialog.h index 9b7387224..594df6fab 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.h +++ b/retroshare-gui/src/gui/SharedFilesDialog.h @@ -196,6 +196,7 @@ class RemoteSharedFilesDialog : public SharedFilesDialog private slots: void downloadRemoteSelected(); + void expanded(const QModelIndex& indx); }; #endif From af69790db6c4a7474806947e9c019df6b64f322b Mon Sep 17 00:00:00 2001 From: cyril soler Date: Mon, 12 Sep 2016 10:15:13 +0200 Subject: [PATCH 068/174] added control file for debian jessie --- build_scripts/Debian+Ubuntu/control.jessie | 44 +++++++++++++++++++ .../Debian+Ubuntu/makeSourcePackage.sh | 15 ++++--- build_scripts/Debian+Ubuntu/ppa_upload.sh | 2 +- 3 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 build_scripts/Debian+Ubuntu/control.jessie diff --git a/build_scripts/Debian+Ubuntu/control.jessie b/build_scripts/Debian+Ubuntu/control.jessie new file mode 100644 index 000000000..9b557e3b8 --- /dev/null +++ b/build_scripts/Debian+Ubuntu/control.jessie @@ -0,0 +1,44 @@ +Source: retroshare06 +Section: devel +Priority: standard +Maintainer: Cyril Soler +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.6, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev, qtmultimedia5-dev, qttools5-dev, libqt5x11extras5-dev, qt5-default +Standards-Version: 3.9.6 +Homepage: http://retroshare.sourceforge.net + +Package: retroshare06-voip-plugin +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, retroshare06, libspeex1, libspeexdsp1, libqt5multimedia5 +Description: RetroShare VOIP plugin + This package provides a plugin for RetroShare, a secured Friend-to-Friend communication + plateform. The plugin adds voice-over-IP functionality to the private chat window. Both + friends chatting together need the plugin installed to be able to talk together. + +Package: retroshare06-feedreader-plugin +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, retroshare06 +Description: RetroShare FeedReader plugin + This package provides a plugin for RetroShare, a secured Friend-to-Friend communication + plateform. The plugin adds a RSS feed reader tab to retroshare. + +Package: retroshare06-nogui +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, gnome-keyring +Conflicts: retroshare06 +Description: Secure communication with friends + This is the command-line client for RetroShare network. This client + can be contacted and talked-to using SSL. Clients exist for portable + devices running e.g. Android. + +Package: retroshare06 +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, gnome-keyring +Conflicts: retroshare06-nogui +Description: Secure communication with friends + RetroShare is a Open Source cross-platform, private and secure decentralised + commmunication platform. It lets you to securely chat and share files with your + friends and family, using a web-of-trust to authenticate peers and OpenSSL to + encrypt all communication. RetroShare provides filesharing, chat, messages, + forums and channels. + + diff --git a/build_scripts/Debian+Ubuntu/makeSourcePackage.sh b/build_scripts/Debian+Ubuntu/makeSourcePackage.sh index 71b4dd070..bd2cd2ba5 100755 --- a/build_scripts/Debian+Ubuntu/makeSourcePackage.sh +++ b/build_scripts/Debian+Ubuntu/makeSourcePackage.sh @@ -20,6 +20,12 @@ dist="" # This is the key for "Cyril Soler " gpgkey="0932399B" +date=`git log --pretty=format:"%ai" | head -1 | cut -d\ -f1 | sed -e s/-//g` +time=`git log --pretty=format:"%aD" | head -1 | cut -d\ -f5 | sed -e s/://g` +hhsh=`git log --pretty=format:"%H" | head -1 | cut -c1-8` + +rev=${date}.${hhsh} + while [ ${#} -gt 0 ]; do case ${1} in "-rev") shift @@ -54,12 +60,6 @@ echo Attempting to get revision number... ccount=`git rev-list --count --all` ccount=`expr $ccount + 8613 - 8267` -date=`git log --pretty=format:"%ai" | head -1 | cut -d\ -f1 | sed -e s/-//g` -time=`git log --pretty=format:"%aD" | head -1 | cut -d\ -f5 | sed -e s/://g` -hhsh=`git log --pretty=format:"%H" | head -1 | cut -c1-8` - -rev=${date}.${hhsh} - echo " "Using PGP key id : ${gpgkey} echo " "Using distributions: ${dist} echo " "Commit count : ${ccount} @@ -137,7 +137,8 @@ for i in ${dist}; do cp ../control.xenial debian/control elif test "${i}" = "stretch" ; then cp ../control.${i} debian/control - cp ../rules.${i} debian/rules + elif test "${i}" = "jessie" ; then + cp ../control.${i} debian/control else cp ../debian/control debian/control fi diff --git a/build_scripts/Debian+Ubuntu/ppa_upload.sh b/build_scripts/Debian+Ubuntu/ppa_upload.sh index 1e5d0aeb3..f328c5172 100644 --- a/build_scripts/Debian+Ubuntu/ppa_upload.sh +++ b/build_scripts/Debian+Ubuntu/ppa_upload.sh @@ -1,4 +1,4 @@ #!/bin/sh -for i in `ls retroshare06_0.6.0-1.*.changes` ; do +for i in `ls retroshare06_0.6.1-1.*.changes` ; do dput ppa:retroshare/unstable $i done From 274f924ca01674ef9a8c3220cd91025cd4295ba1 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 12 Sep 2016 20:55:20 +0200 Subject: [PATCH 069/174] fixed getFileInfo and transfer from new database --- .../src/file_sharing/dir_hierarchy.cc | 4 +++ .../src/file_sharing/directory_storage.cc | 31 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 0ad931077..d58d91170 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -260,6 +260,7 @@ bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; + mFileHashes[hash] = file_index ; old_hash = hash ; @@ -282,6 +283,9 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex fe.file_modtime = modf_time; fe.file_name = fname; + if(!hash.isNull()) + mFileHashes[hash] = file_index ; + return true; } diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 933660986..779a2c04d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -185,9 +185,36 @@ void DirectoryStorage::print() bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) { - RS_STACK_MUTEX(mDirStorageMtx) ; + DirDetails d; + extractData(i,d) ; + + info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file. + info.parent_groups = d.parent_groups; + info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_* + info.path = d.path + "/" + d.name; + info.fname = d.name; + info.hash = d.hash; + info.size = d.count; + + // all this stuff below is not useful in this case. + + info.mId = 0; /* (GUI) Model Id -> unique number */ + info.ext.clear(); + info.avail = 0; /* how much we have */ + info.rank = 0; + info.age = 0; + info.queue_position =0; + info.searchId = 0; /* 0 if none */ + + /* Transfer Stuff */ + info.transfered = 0; + info.tfRate = 0; /* in kbytes */ + info.downloadStatus = FT_STATE_COMPLETE ; + std::list peers; + + info.priority = SPEED_NORMAL; + info.lastTS = 0; - NOT_IMPLEMENTED(); return true; } From 9f66c0050b63bca5e22119cf6b090ea624379186 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Mon, 12 Sep 2016 23:37:19 +0200 Subject: [PATCH 070/174] moved rsexpr to file_sharing directory --- .../src/file_sharing/dir_hierarchy.cc | 44 +++++++++ .../src/file_sharing/dir_hierarchy.h | 5 ++ .../src/file_sharing/directory_storage.cc | 90 ++++++++++--------- .../src/file_sharing/directory_storage.h | 4 +- .../src/{dbase => file_sharing}/rsexpr.cc | 0 libretroshare/src/libretroshare.pro | 13 +-- libretroshare/src/retroshare/rstypes.h | 2 +- 7 files changed, 102 insertions(+), 56 deletions(-) rename libretroshare/src/{dbase => file_sharing}/rsexpr.cc (100%) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d58d91170..778d1cffb 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -2,6 +2,7 @@ #include "dir_hierarchy.h" #include "filelist_io.h" +//#include "rsexpr.h" #define DEBUG_DIRECTORY_STORAGE 1 @@ -570,6 +571,49 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list &results) const +{ +#ifdef TODO + for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) + { + const std::string &str1 = static_cast(mNodes[fit->second])->file_name; + + { + /*Evaluate the boolean expression and add it to the results if its true*/ + bool ret = exp->eval(fit->second); + if (ret == true){ + results.push_back(fit->second); + } + } +#endif +} + +int InternalFileHierarchyStorage::searchTerms(const std::list& terms, std::list &results) const +{ + // most entries are likely to be files, so we could do a linear search over the entries tab. + // instead we go through the table of hashes. + + for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) + if(mNodes[it->second] != NULL) + { + const std::string &str1 = static_cast(mNodes[it->second])->file_name; + + for(std::list::const_iterator iter(terms.begin()); iter != terms.end(); ++iter) + { + /* always ignore case */ + const std::string &str2 = (*iter); +#ifdef TODO + if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), CompareCharIC() )) + { + results.push_back(fit->second); + break; + } +#endif + } + } + return 0 ; +} + bool InternalFileHierarchyStorage::check(std::string& error_string) const // checks consistency of storage. { // recurs go through all entries, check that all diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 33c5a562e..f2e81844f 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -110,7 +110,12 @@ public: uint32_t getType(DirectoryStorage::EntryIndex indx) const; DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index); DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index); + + // search. SearchHash is logarithmic. The other two are linear. + bool searchHash(const RsFileHash& hash,std::list& results); + int searchBoolExp(Expression * exp, std::list &results) const ; + int searchTerms(const std::list& terms, std::list &results) const ; bool check(std::string& error_string) const ;// checks consistency of storage. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 779a2c04d..4130481df 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -160,62 +160,29 @@ void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_ void DirectoryStorage::load(const std::string& local_file_name) { - std::cerr << "DirectoryStorage::load()" << std::endl; - - // first load the header, than all fields. - RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy->load(local_file_name); } void DirectoryStorage::save(const std::string& local_file_name) { - std::cerr << "DirectoryStorage::Save()" << std::endl; - RS_STACK_MUTEX(mDirStorageMtx) ; mFileHierarchy->save(local_file_name); - - // first write the header, than all fields. } void DirectoryStorage::print() { RS_STACK_MUTEX(mDirStorageMtx) ; - std::cerr << "LocalDirectoryStorage:" << std::endl; mFileHierarchy->print(); } -bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +int DirectoryStorage::searchTerms(const std::list& terms, std::list &results) const { - DirDetails d; - extractData(i,d) ; - - info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file. - info.parent_groups = d.parent_groups; - info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_* - info.path = d.path + "/" + d.name; - info.fname = d.name; - info.hash = d.hash; - info.size = d.count; - - // all this stuff below is not useful in this case. - - info.mId = 0; /* (GUI) Model Id -> unique number */ - info.ext.clear(); - info.avail = 0; /* how much we have */ - info.rank = 0; - info.age = 0; - info.queue_position =0; - info.searchId = 0; /* 0 if none */ - - /* Transfer Stuff */ - info.transfered = 0; - info.tfRate = 0; /* in kbytes */ - info.downloadStatus = FT_STATE_COMPLETE ; - std::list peers; - - info.priority = SPEED_NORMAL; - info.lastTS = 0; - - return true; + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->searchTerms(terms,results); +} +int DirectoryStorage::searchBoolExp(Expression * exp, std::list &results) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + return mFileHierarchy->searchBoolExp(exp,results); } bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) @@ -444,6 +411,47 @@ bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) return getFileSharingPermissions(indx,d.flags,d.parent_groups) ; } +bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) +{ + DirDetails d; + extractData(i,d) ; + + if(d.type != DIR_TYPE_FILE) + { + std::cerr << "(EE) LocalDirectoryStorage: asked for file info for index " << i << " which is not a file." << std::endl; + return false; + } + + info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file. + info.parent_groups = d.parent_groups; + info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_* + info.path = d.path + "/" + d.name; + info.fname = d.name; + info.hash = d.hash; + info.size = d.count; + + // all this stuff below is not useful in this case. + + info.mId = 0; /* (GUI) Model Id -> unique number */ + info.ext.clear(); + info.avail = 0; /* how much we have */ + info.rank = 0; + info.age = 0; + info.queue_position =0; + info.searchId = 0; /* 0 if none */ + + /* Transfer Stuff */ + info.transfered = 0; + info.tfRate = 0; /* in kbytes */ + info.downloadStatus = FT_STATE_COMPLETE ; + std::list peers; + + info.priority = SPEED_NORMAL; + info.lastTS = 0; + + return true; +} + bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list& parent_groups) { RS_STACK_MUTEX(mDirStorageMtx) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 78fcb46ba..7c88ffe21 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -24,9 +24,9 @@ class DirectoryStorage void save() const ; - virtual int searchTerms(const std::list& terms, std::list &results) const { NOT_IMPLEMENTED() ; return 0;} + virtual int searchTerms(const std::list& terms, std::list &results) const ; + virtual int searchBoolExp(Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; - virtual int searchBoolExp(Expression * exp, std::list &results) const { NOT_IMPLEMENTED() ; return 0; } bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; bool setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; diff --git a/libretroshare/src/dbase/rsexpr.cc b/libretroshare/src/file_sharing/rsexpr.cc similarity index 100% rename from libretroshare/src/dbase/rsexpr.cc rename to libretroshare/src/file_sharing/rsexpr.cc diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 56d36469c..f839ab2d5 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -55,6 +55,7 @@ file_lists { file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ file_sharing/dir_hierarchy.cc \ + file_sharing/rsexpr.cc \ file_sharing/rsfilelistitems.cc } @@ -366,11 +367,6 @@ INCLUDEPATH *= $${OPENPGPSDK_DIR} PRE_TARGETDEPS *= $${OPENPGPSDK_DIR}/lib/libops.a LIBS *= $${OPENPGPSDK_DIR}/lib/libops.a -lbz2 -HEADERS += dbase/cachestrapper.h \ - dbase/fimonitor.h \ - dbase/findex.h \ - dbase/fistore.h - HEADERS += ft/ftchunkmap.h \ ft/ftcontroller.h \ ft/ftdata.h \ @@ -532,13 +528,6 @@ HEADERS += util/folderiterator.h \ util/rsscopetimer.h \ util/stacktrace.h -SOURCES += dbase/cachestrapper.cc \ - dbase/fimonitor.cc \ - dbase/findex.cc \ - dbase/fistore.cc \ - dbase/rsexpr.cc - - SOURCES += ft/ftchunkmap.cc \ ft/ftcontroller.cc \ ft/ftdatamultiplex.cc \ diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 8ca770a76..0a8537bb7 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -236,7 +236,7 @@ public: RsPeerId id; std::string name; RsFileHash hash; - std::string path; + std::string path; // full path of the parent directory, when it is a file; full path of the dir otherwise. uint64_t count; uint32_t age; FileStorageFlags flags; From e9418bb5c65aae018e965ad06764c3aecb29528f Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 00:01:26 +0200 Subject: [PATCH 071/174] added file headers with proper licence terms --- .../src/file_sharing/dir_hierarchy.cc | 24 +++++++++++++++++ .../src/file_sharing/dir_hierarchy.h | 26 ++++++++++++++++++ .../src/file_sharing/directory_storage.cc | 24 +++++++++++++++++ .../src/file_sharing/directory_storage.h | 25 +++++++++++++++++ .../src/file_sharing/directory_updater.cc | 24 +++++++++++++++++ .../src/file_sharing/directory_updater.h | 26 ++++++++++++++++++ .../src/file_sharing/file_sharing_defaults.h | 26 ++++++++++++++++++ libretroshare/src/file_sharing/filelist_io.cc | 24 +++++++++++++++++ libretroshare/src/file_sharing/filelist_io.h | 26 ++++++++++++++++++ libretroshare/src/file_sharing/hash_cache.cc | 24 +++++++++++++++++ libretroshare/src/file_sharing/hash_cache.h | 26 ++++++++++++++++++ libretroshare/src/file_sharing/p3filelists.cc | 24 +++++++++++++++++ libretroshare/src/file_sharing/p3filelists.h | 27 +++++++++++++++++++ .../src/file_sharing/rsfilelistitems.cc | 24 +++++++++++++++++ .../src/file_sharing/rsfilelistitems.h | 11 ++++---- 15 files changed, 355 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 778d1cffb..aa1e351ae 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -1,3 +1,27 @@ +/* + * RetroShare Internal directory storage class. + * + * file_sharing/dir_hierarchy.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "util/rsdir.h" #include "dir_hierarchy.h" diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index f2e81844f..f01f2c056 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ Internal directory hierarchy class. + * + * file_sharing/dir_hierarchy.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once #include diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 4130481df..2adc54498 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File list storage system. + * + * file_sharing/directory_storage.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include #include "serialiser/rstlvbinary.h" #include "retroshare/rspeers.h" diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 7c88ffe21..cfec2a018 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -1,3 +1,28 @@ +/* + * RetroShare C++ Directory Storage system. + * + * file_sharing/directory_storage.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + #pragma once #include diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index adc3e8fff..23c1fe7c6 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -1,3 +1,27 @@ +/* + * RetroShare Directory watching system. + * + * file_sharing/directory_updater.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "util/folderiterator.h" #include "rsserver/p3face.h" diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 3fb52c81e..25f94d28a 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ Directory parsing code. + * + * file_sharing/directory_updater.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + // This class crawls the given directry hierarchy and updates it. It does so by calling the // shared file list source. This source may be of two types: // - local: directories are crawled n disk and files are hashed / requested from a cache diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 5bc404c5f..4c47584e9 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ File sharing default variables + * + * file_sharing/file_sharing_defaults.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 47d8899e3..f4cd1f6be 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File lists IO methods. + * + * file_sharing/filelist_io.h + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "retroshare/rsids.h" #include "pqi/authssl.h" #include "util/rsdir.h" diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h index 06eee527c..f02d07ed7 100644 --- a/libretroshare/src/file_sharing/filelist_io.h +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ File lists IO methods. + * + * file_sharing/filelist_io.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once #include diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index c2ed3ed4e..f92964ac4 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,3 +1,27 @@ +/* + * RetroShare Hash cache + * + * file_sharing/hash_cache.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "util/rsdir.h" #include "util/rsprint.h" #include "rsserver/p3face.h" diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 6413ce68c..467c18d6a 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -1,3 +1,29 @@ +/* + * RetroShare C++ Hash cache. + * + * file_sharing/hash_cache.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + #pragma once #include diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9a2663991..db73972e2 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File lists service. + * + * file_sharing/p3filelists.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "serialiser/rsserviceids.h" #include "file_sharing/p3filelists.h" diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 8861ed26d..b15a74488 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -1,3 +1,30 @@ +/* + * RetroShare C++ File lists service. + * + * file_sharing/p3filelists.h + * + * Copyright 2016 by Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ + + +// // This class is responsible for // - maintaining a list of shared file hierarchies for each known friends // - talking to the GUI diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index 841e1d1a7..b3fd1db59 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -1,3 +1,27 @@ +/* + * RetroShare File lists service items + * + * file_sharing/rsfilelistsitems.cc + * + * Copyright 2016 Mr.Alice + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com". + * + */ #include "serialiser/rsbaseserial.h" #include "file_sharing/rsfilelistitems.h" diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index 5644d4648..deaa06e60 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -1,11 +1,9 @@ -#pragma once - /* - * libretroshare/src/serialiser: rsnxssitems.h + * RetroShare File lists service items. * - * RetroShare Serialiser. + * file_sharing/rsfilelistitems.h * - * Copyright 2012 Christopher Evi-Parker, Robert Fernie. + * Copyright 2016 Mr.Alice * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -21,9 +19,10 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * USA. * - * Please report all bugs and problems to "retroshare@lunamutt.com". + * Please report all bugs and problems to "retroshare.project@gmail.com". * */ +#pragma once #include #include From a2e34f9cc62e12b8627720927284517ef05f00aa Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 12:05:22 +0200 Subject: [PATCH 072/174] moved regular expression classes into a separate namespace --- libresapi/src/api/FileSearchHandler.cpp | 4 +- .../src/file_sharing/dir_hierarchy.cc | 2 +- .../src/file_sharing/dir_hierarchy.h | 2 +- .../src/file_sharing/directory_storage.cc | 2 +- .../src/file_sharing/directory_storage.h | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 2 +- libretroshare/src/file_sharing/p3filelists.h | 2 +- libretroshare/src/file_sharing/rsexpr.cc | 331 ++++++++------- libretroshare/src/ft/ftserver.cc | 4 +- libretroshare/src/ft/ftserver.h | 4 +- libretroshare/src/libretroshare.pro | 1 + libretroshare/src/retroshare/rsexpr.h | 382 +++++++++--------- libretroshare/src/retroshare/rsfiles.h | 6 +- libretroshare/src/retroshare/rsturtle.h | 4 +- libretroshare/src/turtle/p3turtle.cc | 4 +- libretroshare/src/turtle/p3turtle.h | 2 +- libretroshare/src/turtle/rsturtleitem.h | 2 +- .../src/gui/FileTransfer/SearchDialog.cpp | 8 +- .../src/gui/FileTransfer/SearchDialog.h | 5 +- .../gui/advsearch/advancedsearchdialog.cpp | 6 +- .../src/gui/advsearch/advancedsearchdialog.h | 4 +- .../src/gui/advsearch/expressionwidget.cpp | 30 +- .../src/gui/advsearch/expressionwidget.h | 4 +- .../src/gui/advsearch/guiexprelement.cpp | 36 +- .../src/gui/advsearch/guiexprelement.h | 16 +- 25 files changed, 439 insertions(+), 426 deletions(-) diff --git a/libresapi/src/api/FileSearchHandler.cpp b/libresapi/src/api/FileSearchHandler.cpp index 386db94c9..3833b4c9d 100644 --- a/libresapi/src/api/FileSearchHandler.cpp +++ b/libresapi/src/api/FileSearchHandler.cpp @@ -173,8 +173,8 @@ void FileSearchHandler::handleCreateSearch(Request &req, Response &resp) return; } - NameExpression exprs(ContainsAllStrings,words,true) ; - LinearizedExpression lin_exp ; + RsRegularExpression::NameExpression exprs(RsRegularExpression::ContainsAllStrings,words,true) ; + RsRegularExpression::LinearizedExpression lin_exp ; exprs.linearize(lin_exp) ; uint32_t search_id = RSRandom::random_u32(); diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index aa1e351ae..1735beb79 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -595,7 +595,7 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list &results) const +int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const { #ifdef TODO for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index f01f2c056..fb83e5d9d 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -140,7 +140,7 @@ public: // search. SearchHash is logarithmic. The other two are linear. bool searchHash(const RsFileHash& hash,std::list& results); - int searchBoolExp(Expression * exp, std::list &results) const ; + int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; int searchTerms(const std::list& terms, std::list &results) const ; bool check(std::string& error_string) const ;// checks consistency of storage. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 2adc54498..2a7cd8f91 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -203,7 +203,7 @@ int DirectoryStorage::searchTerms(const std::list& terms, std::list RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->searchTerms(terms,results); } -int DirectoryStorage::searchBoolExp(Expression * exp, std::list &results) const +int DirectoryStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->searchBoolExp(exp,results); diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index cfec2a018..be420f618 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -50,7 +50,7 @@ class DirectoryStorage void save() const ; virtual int searchTerms(const std::list& terms, std::list &results) const ; - virtual int searchBoolExp(Expression * exp, std::list &results) const ; + virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index db73972e2..ea77de3a6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -814,7 +814,7 @@ int p3FileDatabase::SearchKeywords(const std::list& keywords, std:: return filterResults(firesults,results,flags,client_peer_id) ; } -int p3FileDatabase::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const +int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const { RS_STACK_MUTEX(mFLSMtx) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index b15a74488..f832dfc53 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -103,7 +103,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; - virtual int SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; + virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; // ftSearch virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const; diff --git a/libretroshare/src/file_sharing/rsexpr.cc b/libretroshare/src/file_sharing/rsexpr.cc index c9e1ff798..6b4d7ce20 100644 --- a/libretroshare/src/file_sharing/rsexpr.cc +++ b/libretroshare/src/file_sharing/rsexpr.cc @@ -26,6 +26,7 @@ #include "dbase/findex.h" #include "retroshare/rsexpr.h" +#include "retroshare/rstypes.h" #include #include @@ -34,33 +35,35 @@ eval functions of relational expressions. ******************************************************************************************/ +namespace RsRegularExpression +{ template<> void RelExpression::linearize(LinearizedExpression& e) const { - e._ints.push_back(Op) ; - e._ints.push_back(LowerValue) ; - e._ints.push_back(HigherValue) ; + e._ints.push_back(Op) ; + e._ints.push_back(LowerValue) ; + e._ints.push_back(HigherValue) ; } -bool DateExpression::eval(FileEntry *file) +bool DateExpression::eval(const ExpFileEntry& file) { - return evalRel(file->modtime); + return evalRel(file.file_modtime()); } -bool SizeExpressionMB::eval(FileEntry *file) +bool SizeExpressionMB::eval(const ExpFileEntry& file) { - return evalRel((int)(file->size/(uint64_t)(1024*1024))); + return evalRel((int)(file.file_size()/(uint64_t)(1024*1024))); } -bool SizeExpression::eval(FileEntry *file) +bool SizeExpression::eval(const ExpFileEntry& file) { - return evalRel(file->size); + return evalRel(file.file_size()); } -bool PopExpression::eval(FileEntry *file) +bool PopExpression::eval(const ExpFileEntry& file) { - return evalRel(file->pop); + return evalRel(file.file_popularity()); } /****************************************************************************************** @@ -68,98 +71,93 @@ Code for evaluating string expressions ******************************************************************************************/ -bool NameExpression::eval(FileEntry *file) +bool NameExpression::eval(const ExpFileEntry& file) { - return evalStr(file->name); + return evalStr(file.file_name()); } -bool PathExpression::eval(FileEntry *file){ - std::string path; - /*Construct the path of this file*/ - DirEntry * curr = file->parent; - while ( curr != NULL ){ - path = curr->name+"/"+ path; - curr = curr->parent; - } - return evalStr(path); +bool PathExpression::eval(const ExpFileEntry& file) +{ + return evalStr(file.file_parent_path()); } -bool ExtExpression::eval(FileEntry *file){ - std::string ext; - /*Get the part of the string after the last instance of . in the filename */ - size_t index = file->name.find_last_of('.'); - if (index != std::string::npos) { - ext = file->name.substr(index+1); - if (ext != "" ){ - return evalStr(ext); - } - } - return false; +bool ExtExpression::eval(const ExpFileEntry& file) +{ + std::string ext; + /*Get the part of the string after the last instance of . in the filename */ + size_t index = file.file_name().find_last_of('.'); + if (index != std::string::npos) { + ext = file.file_name().substr(index+1); + if (ext != "" ){ + return evalStr(ext); + } + } + return false; } -bool HashExpression::eval(FileEntry *file){ - return evalStr(file->hash.toStdString()); +bool HashExpression::eval(const ExpFileEntry& file){ + return evalStr(file.file_hash().toStdString()); } /*Check whether two strings are 'equal' to each other*/ static bool StrEquals(const std::string & str1, const std::string & str2, - bool IgnoreCase ){ - if ( str1.size() != str2.size() ){ - return false; - } else if (IgnoreCase) { - std::equal( str1.begin(), str1.end(), - str2.begin(), CompareCharIC() ); - } - return std::equal( str1.begin(), str1.end(), - str2.begin()); + bool IgnoreCase ){ + if ( str1.size() != str2.size() ){ + return false; + } else if (IgnoreCase) { + std::equal( str1.begin(), str1.end(), + str2.begin(), CompareCharIC() ); + } + return std::equal( str1.begin(), str1.end(), + str2.begin()); } /*Check whether one string contains the other*/ static bool StrContains( const std::string & str1, const std::string & str2, - bool IgnoreCase){ + bool IgnoreCase){ - std::string::const_iterator iter ; - if (IgnoreCase) { - iter = std::search( str1.begin(), str1.end(), - str2.begin(), str2.end(), CompareCharIC() ); - } else { - iter = std::search( str1.begin(), str1.end(), - str2.begin(), str2.end()); - } - - return ( iter != str1.end() ); + std::string::const_iterator iter ; + if (IgnoreCase) { + iter = std::search( str1.begin(), str1.end(), + str2.begin(), str2.end(), CompareCharIC() ); + } else { + iter = std::search( str1.begin(), str1.end(), + str2.begin(), str2.end()); + } + + return ( iter != str1.end() ); } bool StringExpression :: evalStr ( const std::string &str ){ - std::list::iterator iter; - switch (Op) { - case ContainsAllStrings: - for ( iter = terms.begin(); iter != terms.end(); ++iter ) { - if ( StrContains (str, *iter, IgnoreCase) == false ){ - return false; - } - } - return true; - break; - case ContainsAnyStrings: - for ( iter = terms.begin(); iter != terms.end(); ++iter ) { - if ( StrContains (str,*iter, IgnoreCase) == true ) { - return true; - } - } - break; - case EqualsString: - for ( iter = terms.begin(); iter != terms.end(); ++iter ) { - if ( StrEquals (str,*iter, IgnoreCase) == true ) { - return true; - } - } - break; - default: - return false; - } - return false; + std::list::iterator iter; + switch (Op) { + case ContainsAllStrings: + for ( iter = terms.begin(); iter != terms.end(); ++iter ) { + if ( StrContains (str, *iter, IgnoreCase) == false ){ + return false; + } + } + return true; + break; + case ContainsAnyStrings: + for ( iter = terms.begin(); iter != terms.end(); ++iter ) { + if ( StrContains (str,*iter, IgnoreCase) == true ) { + return true; + } + } + break; + case EqualsString: + for ( iter = terms.begin(); iter != terms.end(); ++iter ) { + if ( StrEquals (str,*iter, IgnoreCase) == true ) { + return true; + } + } + break; + default: + return false; + } + return false; } /************************************************************************* @@ -168,122 +166,123 @@ bool StringExpression :: evalStr ( const std::string &str ){ void CompoundExpression::linearize(LinearizedExpression& e) const { - e._tokens.push_back(LinearizedExpression::EXPR_COMP) ; - e._ints.push_back(Op) ; + e._tokens.push_back(LinearizedExpression::EXPR_COMP) ; + e._ints.push_back(Op) ; - Lexp->linearize(e) ; - Rexp->linearize(e) ; + Lexp->linearize(e) ; + Rexp->linearize(e) ; } void StringExpression::linearize(LinearizedExpression& e) const { - e._ints.push_back(Op) ; - e._ints.push_back(IgnoreCase) ; - e._ints.push_back(terms.size()) ; + e._ints.push_back(Op) ; + e._ints.push_back(IgnoreCase) ; + e._ints.push_back(terms.size()) ; - for(std::list::const_iterator it(terms.begin());it!=terms.end();++it) - e._strings.push_back(*it) ; + for(std::list::const_iterator it(terms.begin());it!=terms.end();++it) + e._strings.push_back(*it) ; } Expression *LinearizedExpression::toExpr(const LinearizedExpression& e) { - int i=0,j=0,k=0 ; - return toExpr(e,i,j,k) ; + int i=0,j=0,k=0 ; + return toExpr(e,i,j,k) ; } void LinearizedExpression::readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list& strings,bool& b,StringOperator& op) { - op = static_cast(e._ints[n_ints++]) ; - b = e._ints[n_ints++] ; - int n = e._ints[n_ints++] ; + op = static_cast(e._ints[n_ints++]) ; + b = e._ints[n_ints++] ; + int n = e._ints[n_ints++] ; - strings.clear() ; - for(int i=0;i(e._tokens[n_tok++]) ; + LinearizedExpression::token tok = static_cast(e._tokens[n_tok++]) ; - switch(tok) - { - case EXPR_DATE: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + switch(tok) + { + case EXPR_DATE: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new DateExpression(op,lv,hv) ; - } + return new DateExpression(op,lv,hv) ; + } - case EXPR_POP: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + case EXPR_POP: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new PopExpression(op,lv,hv) ; - } - case EXPR_SIZE: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + return new PopExpression(op,lv,hv) ; + } + case EXPR_SIZE: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new SizeExpression(op,lv,hv) ; - } - case EXPR_HASH: { - std::list strings ; - StringOperator op ; - bool b ; + return new SizeExpression(op,lv,hv) ; + } + case EXPR_HASH: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new HashExpression(op,strings) ; - } - case EXPR_NAME: { - std::list strings ; - StringOperator op ; - bool b ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; + return new HashExpression(op,strings) ; + } + case EXPR_NAME: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new NameExpression(op,strings,b) ; - } - case EXPR_PATH: { - std::list strings ; - StringOperator op ; - bool b ; + return new NameExpression(op,strings,b) ; + } + case EXPR_PATH: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new ExtExpression(op,strings,b) ; - } - case EXPR_EXT: { - std::list strings ; - StringOperator op ; - bool b ; + return new ExtExpression(op,strings,b) ; + } + case EXPR_EXT: { + std::list strings ; + StringOperator op ; + bool b ; - readStringExpr(e,n_ints,n_strings,strings,b,op) ; + readStringExpr(e,n_ints,n_strings,strings,b,op) ; - return new ExtExpression(op,strings,b) ; - } - case EXPR_COMP: { - LogicalOperator op = static_cast(e._ints[n_ints++]) ; + return new ExtExpression(op,strings,b) ; + } + case EXPR_COMP: { + LogicalOperator op = static_cast(e._ints[n_ints++]) ; - Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ; - Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ; + Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ; + Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ; - return new CompoundExpression(op,e1,e2) ; - } - case EXPR_SIZE_MB: { - RelOperator op = static_cast(e._ints[n_ints++]) ; - int lv = e._ints[n_ints++] ; - int hv = e._ints[n_ints++] ; + return new CompoundExpression(op,e1,e2) ; + } + case EXPR_SIZE_MB: { + RelOperator op = static_cast(e._ints[n_ints++]) ; + int lv = e._ints[n_ints++] ; + int hv = e._ints[n_ints++] ; - return new SizeExpressionMB(op,lv,hv) ; - } - default: - std::cerr << "No expression match the current value " << tok << std::endl ; - return NULL ; - } + return new SizeExpressionMB(op,lv,hv) ; + } + default: + std::cerr << "No expression match the current value " << tok << std::endl ; + return NULL ; + } } +} diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 4b4d4c1e3..1a558f9d6 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -569,11 +569,11 @@ int ftServer::SearchKeywords(std::list keywords, std::listSearchKeywords(keywords, results,flags,peer_id); } -int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags) +int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags) { return mFileDatabase->SearchBoolExp(exp, results,flags,RsPeerId()); } -int ftServer::SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) +int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) { return mFileDatabase->SearchBoolExp(exp,results,flags,peer_id) ; } diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index da1deb009..1173d8311 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -184,8 +184,8 @@ public: virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags); - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags); + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id); /*** * Utility Functions diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index f839ab2d5..e30e3637e 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -59,6 +59,7 @@ file_lists { file_sharing/rsfilelistitems.cc } + dsdv { DEFINES *= SERVICES_DSDV HEADERS += services/p3dsdv.h \ diff --git a/libretroshare/src/retroshare/rsexpr.h b/libretroshare/src/retroshare/rsexpr.h index 87d216387..fc339e87c 100644 --- a/libretroshare/src/retroshare/rsexpr.h +++ b/libretroshare/src/retroshare/rsexpr.h @@ -1,6 +1,3 @@ -#ifndef RS_EXPRESSIONS_H -#define RS_EXPRESSIONS_H - /* * rs-core/src/rsiface: rsexpr.h * @@ -26,38 +23,41 @@ * */ +#pragma once #include #include #include +#include "retroshare/rstypes.h" /****************************************************************************************** Enumerations defining the Operators usable in the Boolean search expressions ******************************************************************************************/ - +namespace RsRegularExpression +{ enum LogicalOperator{ - AndOp=0, /* exp AND exp */ - OrOp=1, /* exp OR exp */ - XorOp=2 /* exp XOR exp */ + AndOp=0, /* exp AND exp */ + OrOp=1, /* exp OR exp */ + XorOp=2 /* exp XOR exp */ }; -/*Operators for String Queries*/ +/* Operators for String Queries */ enum StringOperator{ - ContainsAnyStrings = 0, /* e.g. name contains any of 'conference' 'meeting' 'presentation' */ - ContainsAllStrings = 1, /* same as above except that it contains ALL of the strings */ - EqualsString = 2 /* exactly equal*/ + ContainsAnyStrings = 0, /* e.g. name contains any of 'conference' 'meeting' 'presentation' */ + ContainsAllStrings = 1, /* same as above except that it contains ALL of the strings */ + EqualsString = 2 /* exactly equal*/ }; -/*Relational operators ( >, <, >=, <=, == and InRange )*/ +/* Comparison operators ( >, <, >=, <=, == and InRange ) */ enum RelOperator{ - Equals = 0, - GreaterEquals = 1, - Greater = 2, - SmallerEquals = 3, - Smaller = 4, - InRange = 5 /* lower limit <= value <= upper limit*/ + Equals = 0, + GreaterEquals = 1, + Greater = 2, + SmallerEquals = 3, + Smaller = 4, + InRange = 5 /* lower limit <= value <= upper limit*/ }; /******************************************************************************************** @@ -69,26 +69,26 @@ class Expression ; class LinearizedExpression { - public: - std::vector _tokens ; - std::vector _ints ; - std::vector _strings ; +public: + std::vector _tokens ; + std::vector _ints ; + std::vector _strings ; - typedef enum { EXPR_DATE= 0, - EXPR_POP = 1, - EXPR_SIZE= 2, - EXPR_HASH= 3, - EXPR_NAME= 4, - EXPR_PATH= 5, - EXPR_EXT = 6, - EXPR_COMP= 7, - EXPR_SIZE_MB=8 } token ; + typedef enum { EXPR_DATE = 0, + EXPR_POP = 1, + EXPR_SIZE = 2, + EXPR_HASH = 3, + EXPR_NAME = 4, + EXPR_PATH = 5, + EXPR_EXT = 6, + EXPR_COMP = 7, + EXPR_SIZE_MB = 8 } token ; - static Expression *toExpr(const LinearizedExpression& e) ; + static Expression *toExpr(const LinearizedExpression& e) ; - private: - static Expression *toExpr(const LinearizedExpression& e,int&,int&,int&) ; - static void readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list& strings,bool& b,StringOperator& op) ; +private: + static Expression *toExpr(const LinearizedExpression& e,int&,int&,int&) ; + static void readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list& strings,bool& b,StringOperator& op) ; }; @@ -96,110 +96,122 @@ class LinearizedExpression Boolean Search Expression classes: - Expression: The base class of all expression typest - CompoundExpression: The expression which uses a logical operator to combine - the results of two expressions - StringExpression: An expression which uses some sort of string comparison. - RelExpression: A Relational Expression where > < >= <= == make sense. - e.g. size date etc + Expression: The base class of all expression typest + CompoundExpression: The expression which uses a logical operator to combine + the results of two expressions + StringExpression: An expression which uses some sort of string comparison. + RelExpression: A Relational Expression where > < >= <= == make sense. + e.g. size date etc ******************************************************************************************/ -class FileEntry; +/*! + * \brief The ExpFileEntry class + * This is the base class the regular expressions can operate on. Derive it to fit your own needs! + */ +class ExpFileEntry +{ +public: + virtual const std::string& file_name() const =0; + virtual uint64_t file_size() const =0; + virtual time_t file_modtime() const =0; + virtual time_t file_popularity() const =0; + virtual const std::string file_parent_path() const =0; + virtual const RsFileHash& file_hash() const =0; +}; class Expression { - public: - virtual bool eval (FileEntry *file) = 0; - virtual ~Expression() {}; +public: + virtual bool eval (const ExpFileEntry& file) = 0; + virtual ~Expression() {}; - virtual void linearize(LinearizedExpression& e) const = 0 ; + virtual void linearize(LinearizedExpression& e) const = 0 ; }; - class CompoundExpression : public Expression { - public: - CompoundExpression( enum LogicalOperator op, Expression * exp1, Expression *exp2) - : Lexp(exp1), Rexp(exp2), Op(op){ } +public: + CompoundExpression( enum LogicalOperator op, Expression * exp1, Expression *exp2) + : Lexp(exp1), Rexp(exp2), Op(op){ } - bool eval (FileEntry *file) { - if (Lexp == NULL or Rexp == NULL) { - return false; - } - switch (Op){ - case AndOp: - return Lexp->eval(file) && Rexp->eval(file); - case OrOp: - return Lexp->eval(file) || Rexp->eval(file); - case XorOp: - return Lexp->eval(file) ^ Rexp->eval(file); - default: - return false; - } - } - virtual ~CompoundExpression(){ - delete Lexp; - delete Rexp; - } + bool eval (const ExpFileEntry& file) { + if (Lexp == NULL or Rexp == NULL) { + return false; + } + switch (Op){ + case AndOp: + return Lexp->eval(file) && Rexp->eval(file); + case OrOp: + return Lexp->eval(file) || Rexp->eval(file); + case XorOp: + return Lexp->eval(file) ^ Rexp->eval(file); + default: + return false; + } + } + virtual ~CompoundExpression(){ + delete Lexp; + delete Rexp; + } - virtual void linearize(LinearizedExpression& e) const ; - private: - Expression *Lexp; - Expression *Rexp; - enum LogicalOperator Op; + virtual void linearize(LinearizedExpression& e) const ; +private: + Expression *Lexp; + Expression *Rexp; + enum LogicalOperator Op; }; class StringExpression: public Expression { - public: - StringExpression(enum StringOperator op, std::list &t, bool ic): Op(op),terms(t), IgnoreCase(ic){} +public: + StringExpression(enum StringOperator op, std::list &t, bool ic): Op(op),terms(t), IgnoreCase(ic){} - virtual void linearize(LinearizedExpression& e) const ; - protected: - bool evalStr(const std::string &str); + virtual void linearize(LinearizedExpression& e) const ; +protected: + bool evalStr(const std::string &str); - enum StringOperator Op; - std::list terms; - bool IgnoreCase; + enum StringOperator Op; + std::list terms; + bool IgnoreCase; }; template class RelExpression: public Expression { - public: - RelExpression(enum RelOperator op, T lv, T hv): Op(op), LowerValue(lv), HigherValue(hv) {} +public: + RelExpression(enum RelOperator op, T lv, T hv): Op(op), LowerValue(lv), HigherValue(hv) {} - virtual void linearize(LinearizedExpression& e) const ; - protected: - bool evalRel(T val); + virtual void linearize(LinearizedExpression& e) const ; +protected: + bool evalRel(T val); - enum RelOperator Op; - T LowerValue; - T HigherValue; + enum RelOperator Op; + T LowerValue; + T HigherValue; }; template<> void RelExpression::linearize(LinearizedExpression& e) const ; template bool RelExpression::evalRel(T val) { - switch (Op) { - case Equals: - return LowerValue == val; - case GreaterEquals: - return LowerValue >= val; - case Greater: - return LowerValue > val; - case SmallerEquals: - return LowerValue <= val; - case Smaller: - return LowerValue < val; - case InRange: - return (LowerValue <= val) && (val <= HigherValue); - default: - return false; - } + switch (Op) { + case Equals: + return LowerValue == val; + case GreaterEquals: + return LowerValue >= val; + case Greater: + return LowerValue > val; + case SmallerEquals: + return LowerValue <= val; + case Smaller: + return LowerValue < val; + case InRange: + return (LowerValue <= val) && (val <= HigherValue); + default: + return false; + } } @@ -212,12 +224,12 @@ Binary Predicate for Case Insensitive search *Factor locales in the comparison */ struct CompareCharIC : - public std::binary_function< char , char , bool> { + public std::binary_function< char , char , bool> { - bool operator () ( char ch1 , char ch2 ) const { - return tolower( static_cast < unsigned char > (ch1) ) - == tolower( static_cast < unsigned char > (ch2) ); - } + bool operator () ( char ch1 , char ch2 ) const { + return tolower( static_cast < unsigned char > (ch1) ) + == tolower( static_cast < unsigned char > (ch2) ); + } }; @@ -228,55 +240,55 @@ Some implementations of StringExpressions. class NameExpression: public StringExpression { - public: - NameExpression(enum StringOperator op, std::list &t, bool ic): - StringExpression(op,t,ic) {} - bool eval(FileEntry *file); +public: + NameExpression(enum StringOperator op, std::list &t, bool ic): + StringExpression(op,t,ic) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_NAME) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_NAME) ; + StringExpression::linearize(e) ; + } }; class PathExpression: public StringExpression { public: - PathExpression(enum StringOperator op, std::list &t, bool ic): - StringExpression(op,t,ic) {} - bool eval(FileEntry *file); + PathExpression(enum StringOperator op, std::list &t, bool ic): + StringExpression(op,t,ic) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_PATH) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_PATH) ; + StringExpression::linearize(e) ; + } }; class ExtExpression: public StringExpression { public: - ExtExpression(enum StringOperator op, std::list &t, bool ic): - StringExpression(op,t,ic) {} - bool eval(FileEntry *file); + ExtExpression(enum StringOperator op, std::list &t, bool ic): + StringExpression(op,t,ic) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_EXT) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_EXT) ; + StringExpression::linearize(e) ; + } }; class HashExpression: public StringExpression { public: - HashExpression(enum StringOperator op, std::list &t): - StringExpression(op,t, true) {} - bool eval(FileEntry *file); + HashExpression(enum StringOperator op, std::list &t): + StringExpression(op,t, true) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_HASH) ; - StringExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_HASH) ; + StringExpression::linearize(e) ; + } }; /****************************************************************************************** @@ -286,62 +298,62 @@ Some implementations of Relational Expressions. class DateExpression: public RelExpression { - public: - DateExpression(enum RelOperator op, int v): RelExpression(op,v,v){} - DateExpression(enum RelOperator op, int lv, int hv): - RelExpression(op,lv,hv) {} - bool eval(FileEntry *file); +public: + DateExpression(enum RelOperator op, int v): RelExpression(op,v,v){} + DateExpression(enum RelOperator op, int lv, int hv): + RelExpression(op,lv,hv) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_DATE) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_DATE) ; + RelExpression::linearize(e) ; + } }; class SizeExpression: public RelExpression { - public: - SizeExpression(enum RelOperator op, int v): RelExpression(op,v,v){} - SizeExpression(enum RelOperator op, int lv, int hv): - RelExpression(op,lv,hv) {} - bool eval(FileEntry *file); +public: + SizeExpression(enum RelOperator op, int v): RelExpression(op,v,v){} + SizeExpression(enum RelOperator op, int lv, int hv): + RelExpression(op,lv,hv) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_SIZE) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_SIZE) ; + RelExpression::linearize(e) ; + } }; class SizeExpressionMB: public RelExpression { - public: - SizeExpressionMB(enum RelOperator op, int v): RelExpression(op,v,v){} - SizeExpressionMB(enum RelOperator op, int lv, int hv): - RelExpression(op,lv,hv) {} - bool eval(FileEntry *file); +public: + SizeExpressionMB(enum RelOperator op, int v): RelExpression(op,v,v){} + SizeExpressionMB(enum RelOperator op, int lv, int hv): + RelExpression(op,lv,hv) {} + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_SIZE_MB) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_SIZE_MB) ; + RelExpression::linearize(e) ; + } }; class PopExpression: public RelExpression { - public: - PopExpression(enum RelOperator op, int v): RelExpression(op,v,v){} - PopExpression(enum RelOperator op, int lv, int hv): RelExpression(op,lv,hv) {} - PopExpression(const LinearizedExpression& e) ; - bool eval(FileEntry *file); +public: + PopExpression(enum RelOperator op, int v): RelExpression(op,v,v){} + PopExpression(enum RelOperator op, int lv, int hv): RelExpression(op,lv,hv) {} + PopExpression(const LinearizedExpression& e) ; + bool eval(const ExpFileEntry& file); - virtual void linearize(LinearizedExpression& e) const - { - e._tokens.push_back(LinearizedExpression::EXPR_POP) ; - RelExpression::linearize(e) ; - } + virtual void linearize(LinearizedExpression& e) const + { + e._tokens.push_back(LinearizedExpression::EXPR_POP) ; + RelExpression::linearize(e) ; + } }; +} -#endif /* RS_EXPRESSIONS_H */ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index d49324933..2236a6a8b 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -36,7 +36,7 @@ class RsFiles; extern RsFiles *rsFiles; -class Expression; +namespace RsRegularExpression { class Expression; } class CacheStrapper ; class CacheTransfer; @@ -190,8 +190,8 @@ class RsFiles virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) = 0; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags) = 0; - virtual int SearchBoolExp(Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags) = 0; + virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; /*** * Utility Functions. diff --git a/libretroshare/src/retroshare/rsturtle.h b/libretroshare/src/retroshare/rsturtle.h index 6fb1b045c..59b787c8e 100644 --- a/libretroshare/src/retroshare/rsturtle.h +++ b/libretroshare/src/retroshare/rsturtle.h @@ -35,7 +35,7 @@ #include "retroshare/rstypes.h" -class LinearizedExpression ; +namespace RsRegularExpression { class LinearizedExpression ; } class RsTurtleClientService ; class RsTurtle; @@ -102,7 +102,7 @@ class RsTurtle // as they come back. // virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ; - virtual TurtleRequestId turtleSearch(const LinearizedExpression& expr) = 0 ; + virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ; // Initiates tunnel handling for the given file hash. tunnels. Launches // an exception if an error occurs during the initialization process. The diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index b3bcb1630..25d1972a5 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -1763,7 +1763,7 @@ void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list initialResults; // to do: split search string into words. - Expression *exp = LinearizedExpression::toExpr(expr) ; + RsRegularExpression::Expression *exp = RsRegularExpression::LinearizedExpression::toExpr(expr) ; if(exp == NULL) return ; @@ -1820,7 +1820,7 @@ TurtleRequestId p3turtle::turtleSearch(const std::string& string_to_match) return id ; } -TurtleRequestId p3turtle::turtleSearch(const LinearizedExpression& expr) +TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpression& expr) { // generate a new search id. diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index e5faca3bc..bfda7663e 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -239,7 +239,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config // remove the specific file search packets from the turtle router. // virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ; - virtual TurtleSearchRequestId turtleSearch(const LinearizedExpression& expr) ; + virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ; // Initiates tunnel handling for the given file hash. tunnels. Launches // an exception if an error occurs during the initialization process. The diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 114ee97ef..11e1969e7 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -102,7 +102,7 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem RsTurtleRegExpSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {} RsTurtleRegExpSearchRequestItem(void *data,uint32_t size) ; - LinearizedExpression expr ; // Reg Exp in linearised mode + RsRegularExpression::LinearizedExpression expr ; // Reg Exp in linearised mode virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; } virtual void performLocalSearch(std::list&) const ; diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp index 4c6d9481e..fa80c7ec9 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp @@ -774,7 +774,7 @@ void SearchDialog::initSearchResult(const QString& txt, qulonglong searchId, int ui.searchSummaryWidget->setCurrentItem(item2); } -void SearchDialog::advancedSearch(Expression* expression) +void SearchDialog::advancedSearch(RsRegularExpression::Expression* expression) { advSearchDialog->hide(); @@ -782,7 +782,7 @@ void SearchDialog::advancedSearch(Expression* expression) std::list results; // send a turtle search request - LinearizedExpression e ; + RsRegularExpression::LinearizedExpression e ; expression->linearize(e) ; TurtleRequestId req_id = rsTurtle->turtleSearch(e) ; @@ -843,8 +843,8 @@ void SearchDialog::searchKeywords(const QString& keywords) if (n < 1) return; - NameExpression exprs(ContainsAllStrings,words,true) ; - LinearizedExpression lin_exp ; + RsRegularExpression::NameExpression exprs(RsRegularExpression::ContainsAllStrings,words,true) ; + RsRegularExpression::LinearizedExpression lin_exp ; exprs.linearize(lin_exp) ; TurtleRequestId req_id ; diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.h b/retroshare-gui/src/gui/FileTransfer/SearchDialog.h index c0682ecb0..bfc5892cc 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.h +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.h @@ -27,9 +27,10 @@ #include "mainpage.h" class AdvancedSearchDialog; -class Expression; class RSTreeWidgetItemCompareRole; +namespace RsRegularExpression { class Expression; } + #define FRIEND_SEARCH 1 #define ANONYMOUS_SEARCH 2 class SearchDialog : public MainPage @@ -99,7 +100,7 @@ private slots: void showAdvSearchDialog(bool=true); /** perform the advanced search */ - void advancedSearch(Expression*); + void advancedSearch(RsRegularExpression::Expression*); void selectSearchResults(int index = -1); void hideOrShowSearchResult(QTreeWidgetItem* resultItem, QString currentSearchId = QString(), int fileTypeIndex = -1); diff --git a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp index 3d60260be..d4b629b4b 100644 --- a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp +++ b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp @@ -119,9 +119,9 @@ void AdvancedSearchDialog::prepareSearch() } -Expression * AdvancedSearchDialog::getRsExpr() +RsRegularExpression::Expression * AdvancedSearchDialog::getRsExpr() { - Expression * wholeExpression; + RsRegularExpression::Expression * wholeExpression; // process the special case: first expression wholeExpression = expressions->at(0)->getRsExpression(); @@ -131,7 +131,7 @@ Expression * AdvancedSearchDialog::getRsExpr() for (int i = 1; i < expressions->size(); ++i) { // extract the expression information and compound it with the // first expression - wholeExpression = new CompoundExpression(expressions->at(i)->getOperator(), + wholeExpression = new RsRegularExpression::CompoundExpression(expressions->at(i)->getOperator(), wholeExpression, expressions->at(i)->getRsExpression()); } diff --git a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h index 296829057..02626283a 100644 --- a/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h +++ b/retroshare-gui/src/gui/advsearch/advancedsearchdialog.h @@ -36,10 +36,10 @@ class AdvancedSearchDialog : public QDialog, public Ui::AdvancedSearchDialog public: AdvancedSearchDialog(QWidget * parent = 0 ); - Expression * getRsExpr(); + RsRegularExpression::Expression * getRsExpr(); QString getSearchAsString(); signals: - void search(Expression*); + void search(RsRegularExpression::Expression*); private slots: void deleteExpression(ExpressionWidget*); diff --git a/retroshare-gui/src/gui/advsearch/expressionwidget.cpp b/retroshare-gui/src/gui/advsearch/expressionwidget.cpp index 013bec182..d5e83c3fc 100644 --- a/retroshare-gui/src/gui/advsearch/expressionwidget.cpp +++ b/retroshare-gui/src/gui/advsearch/expressionwidget.cpp @@ -130,7 +130,7 @@ void ExpressionWidget::deleteExpression() emit signalDelete(this); } -LogicalOperator ExpressionWidget::getOperator() +RsRegularExpression::LogicalOperator ExpressionWidget::getOperator() { return exprOpElem->getLogicalOperator(); } @@ -145,9 +145,9 @@ static int checkedConversion(uint64_t s) return (int)s ; } -Expression* ExpressionWidget::getRsExpression() +RsRegularExpression::Expression* ExpressionWidget::getRsExpression() { - Expression * expr = NULL; + RsRegularExpression::Expression * expr = NULL; std::list wordList; uint64_t lowVal = 0; @@ -174,54 +174,54 @@ Expression* ExpressionWidget::getRsExpression() switch (searchType) { case NameSearch: - expr = new NameExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::NameExpression(exprCondElem->getStringOperator(), wordList, exprParamElem->ignoreCase()); break; case PathSearch: - expr = new PathExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::PathExpression(exprCondElem->getStringOperator(), wordList, exprParamElem->ignoreCase()); break; case ExtSearch: - expr = new ExtExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::ExtExpression(exprCondElem->getStringOperator(), wordList, exprParamElem->ignoreCase()); break; case HashSearch: - expr = new HashExpression(exprCondElem->getStringOperator(), + expr = new RsRegularExpression::HashExpression(exprCondElem->getStringOperator(), wordList); break; case DateSearch: if (inRangedConfig) { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); } else { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); } break; case PopSearch: if (inRangedConfig) { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal)); } else { - expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); + expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue())); } break; case SizeSearch: if (inRangedConfig) { if(lowVal >= (uint64_t)(1024*1024*1024) || highVal >= (uint64_t)(1024*1024*1024)) - expr = new SizeExpressionMB(exprCondElem->getRelOperator(), (int)(lowVal / (1024*1024)), (int)(highVal / (1024*1024))); + expr = new RsRegularExpression::SizeExpressionMB(exprCondElem->getRelOperator(), (int)(lowVal / (1024*1024)), (int)(highVal / (1024*1024))); else - expr = new SizeExpression(exprCondElem->getRelOperator(), lowVal, highVal); + expr = new RsRegularExpression::SizeExpression(exprCondElem->getRelOperator(), lowVal, highVal); } else { uint64_t s = exprParamElem->getIntValue() ; if(s >= (uint64_t)(1024*1024*1024)) - expr = new SizeExpressionMB(exprCondElem->getRelOperator(), (int)(s/(1024*1024))) ; + expr = new RsRegularExpression::SizeExpressionMB(exprCondElem->getRelOperator(), (int)(s/(1024*1024))) ; else - expr = new SizeExpression(exprCondElem->getRelOperator(), (int)s) ; + expr = new RsRegularExpression::SizeExpression(exprCondElem->getRelOperator(), (int)s) ; } break; }; diff --git a/retroshare-gui/src/gui/advsearch/expressionwidget.h b/retroshare-gui/src/gui/advsearch/expressionwidget.h index 21f964449..2ebc6b851 100644 --- a/retroshare-gui/src/gui/advsearch/expressionwidget.h +++ b/retroshare-gui/src/gui/advsearch/expressionwidget.h @@ -45,11 +45,11 @@ public: /** delivers the expression represented by this widget the operator to join this expression with any previous expressions is provided by the getOperator method */ - Expression* getRsExpression(); + RsRegularExpression::Expression* getRsExpression(); /** supplies the operator to be used when joining this expression to the whole query */ - LogicalOperator getOperator(); + RsRegularExpression::LogicalOperator getOperator(); QString toString(); diff --git a/retroshare-gui/src/gui/advsearch/guiexprelement.cpp b/retroshare-gui/src/gui/advsearch/guiexprelement.cpp index f3ed6debe..7cf7e0732 100644 --- a/retroshare-gui/src/gui/advsearch/guiexprelement.cpp +++ b/retroshare-gui/src/gui/advsearch/guiexprelement.cpp @@ -62,9 +62,9 @@ QStringList GuiExprElement::relOptionsList; QMap GuiExprElement::TermsIndexMap; -QMap GuiExprElement::logicalOpIndexMap; -QMap GuiExprElement::strConditionIndexMap; -QMap GuiExprElement::relConditionIndexMap; +QMap GuiExprElement::logicalOpIndexMap; +QMap GuiExprElement::strConditionIndexMap; +QMap GuiExprElement::relConditionIndexMap; QMap GuiExprElement::logicalOpStrMap; QMap GuiExprElement::termsStrMap; @@ -141,9 +141,9 @@ void GuiExprElement::initialiseOptionsLists() GuiExprElement::relOptionsList.append(RANGE); // now the maps - GuiExprElement::logicalOpIndexMap[GuiExprElement::AND_INDEX] = AndOp; - GuiExprElement::logicalOpIndexMap[GuiExprElement::OR_INDEX] = OrOp; - GuiExprElement::logicalOpIndexMap[GuiExprElement::XOR_INDEX] = XorOp; + GuiExprElement::logicalOpIndexMap[GuiExprElement::AND_INDEX] = RsRegularExpression::AndOp; + GuiExprElement::logicalOpIndexMap[GuiExprElement::OR_INDEX] = RsRegularExpression::OrOp; + GuiExprElement::logicalOpIndexMap[GuiExprElement::XOR_INDEX] = RsRegularExpression::XorOp; GuiExprElement::TermsIndexMap[GuiExprElement::NAME_INDEX] = NameSearch; GuiExprElement::TermsIndexMap[GuiExprElement::PATH_INDEX] = PathSearch; @@ -153,9 +153,9 @@ void GuiExprElement::initialiseOptionsLists() GuiExprElement::TermsIndexMap[GuiExprElement::SIZE_INDEX] = SizeSearch; GuiExprElement::TermsIndexMap[GuiExprElement::POP_INDEX] = PopSearch; - GuiExprElement::strConditionIndexMap[GuiExprElement::CONTAINS_INDEX] = ContainsAnyStrings; - GuiExprElement::strConditionIndexMap[GuiExprElement::CONTALL_INDEX] = ContainsAllStrings; - GuiExprElement::strConditionIndexMap[GuiExprElement::IS_INDEX] = EqualsString; + GuiExprElement::strConditionIndexMap[GuiExprElement::CONTAINS_INDEX] = RsRegularExpression::ContainsAnyStrings; + GuiExprElement::strConditionIndexMap[GuiExprElement::CONTALL_INDEX] = RsRegularExpression::ContainsAllStrings; + GuiExprElement::strConditionIndexMap[GuiExprElement::IS_INDEX] = RsRegularExpression::EqualsString; /* W A R N I N G !!!! the cb elements correspond to their inverse rel op counterparts in rsexpr.h due to the nature of @@ -166,12 +166,12 @@ void GuiExprElement::initialiseOptionsLists() files where the condition is greater than the file size i.e. files whose size is less than or equal to the condition Therefore we invert the mapping of rel conditions here to match the behaviour of the impl. */ - GuiExprElement::relConditionIndexMap[GuiExprElement::LT_INDEX] = GreaterEquals; - GuiExprElement::relConditionIndexMap[GuiExprElement::LTE_INDEX] = Greater; - GuiExprElement::relConditionIndexMap[GuiExprElement::EQUAL_INDEX] = Equals; - GuiExprElement::relConditionIndexMap[GuiExprElement::GTE_INDEX] = Smaller; - GuiExprElement::relConditionIndexMap[GuiExprElement::GT_INDEX] = SmallerEquals; - GuiExprElement::relConditionIndexMap[GuiExprElement::RANGE_INDEX] = InRange; + GuiExprElement::relConditionIndexMap[GuiExprElement::LT_INDEX] = RsRegularExpression::GreaterEquals; + GuiExprElement::relConditionIndexMap[GuiExprElement::LTE_INDEX] = RsRegularExpression::Greater; + GuiExprElement::relConditionIndexMap[GuiExprElement::EQUAL_INDEX] = RsRegularExpression::Equals; + GuiExprElement::relConditionIndexMap[GuiExprElement::GTE_INDEX] = RsRegularExpression::Smaller; + GuiExprElement::relConditionIndexMap[GuiExprElement::GT_INDEX] = RsRegularExpression::SmallerEquals; + GuiExprElement::relConditionIndexMap[GuiExprElement::RANGE_INDEX] = RsRegularExpression::InRange; // the string to index map GuiExprElement::termsStrMap[GuiExprElement::NAME_INDEX] = NAME; @@ -260,7 +260,7 @@ QString ExprOpElement::toString() } -LogicalOperator ExprOpElement::getLogicalOperator() +RsRegularExpression::LogicalOperator ExprOpElement::getLogicalOperator() { return GuiExprElement::logicalOpIndexMap[cb->currentIndex()]; } @@ -313,12 +313,12 @@ QString ExprConditionElement::toString() return GuiExprElement::relConditionStrMap[cb->currentIndex()]; } -RelOperator ExprConditionElement::getRelOperator() +RsRegularExpression::RelOperator ExprConditionElement::getRelOperator() { return GuiExprElement::relConditionIndexMap[cb->currentIndex()]; } -StringOperator ExprConditionElement::getStringOperator() +RsRegularExpression::StringOperator ExprConditionElement::getStringOperator() { return GuiExprElement::strConditionIndexMap[cb->currentIndex()]; } diff --git a/retroshare-gui/src/gui/advsearch/guiexprelement.h b/retroshare-gui/src/gui/advsearch/guiexprelement.h index 075eddf41..0b78f7975 100644 --- a/retroshare-gui/src/gui/advsearch/guiexprelement.h +++ b/retroshare-gui/src/gui/advsearch/guiexprelement.h @@ -114,9 +114,9 @@ protected: static QStringList relOptionsList; // provides a mapping of condition operators to RSExpr reloperators - static QMap logicalOpIndexMap; - static QMap strConditionIndexMap; - static QMap relConditionIndexMap; + static QMap logicalOpIndexMap; + static QMap strConditionIndexMap; + static QMap relConditionIndexMap; // provides a mapping of indexes to translatable strings static QMap logicalOpStrMap; @@ -134,7 +134,7 @@ class ExprOpElement : public GuiExprElement public: ExprOpElement(QWidget * parent = 0); - LogicalOperator getLogicalOperator(); + RsRegularExpression::LogicalOperator getLogicalOperator(); QString toString(); private: QComboBox * cb; @@ -148,8 +148,8 @@ class ExprTermsElement : public GuiExprElement public: ExprTermsElement(QWidget * parent = 0); int getTermsIndex(); - RelOperator getRelOperator(); - StringOperator getStringOperator(); + RsRegularExpression::RelOperator getRelOperator(); + RsRegularExpression::StringOperator getStringOperator(); void set(int i) {cb->setCurrentIndex(i);} QString toString(); @@ -167,8 +167,8 @@ class ExprConditionElement : public GuiExprElement public: ExprConditionElement(ExprSearchType, QWidget * parent = 0); - RelOperator getRelOperator(); - StringOperator getStringOperator(); + RsRegularExpression::RelOperator getRelOperator(); + RsRegularExpression::StringOperator getStringOperator(); void adjustForSearchType(ExprSearchType); void set(int i) {cb->setCurrentIndex(i);} QString toString(); From 6ac9b27d7b743fd2f00d2e3b61ad4500f1d4c47e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 12:14:16 +0200 Subject: [PATCH 073/174] moved rsexpr.cc into util/ --- libretroshare/src/libretroshare.pro | 2 +- libretroshare/src/{file_sharing => util}/rsexpr.cc | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename libretroshare/src/{file_sharing => util}/rsexpr.cc (100%) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index e30e3637e..260626db0 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -55,7 +55,6 @@ file_lists { file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc \ file_sharing/dir_hierarchy.cc \ - file_sharing/rsexpr.cc \ file_sharing/rsfilelistitems.cc } @@ -653,6 +652,7 @@ SOURCES += turtle/p3turtle.cc \ SOURCES += util/folderiterator.cc \ util/rsdebug.cc \ + util/rsexpr.cc \ util/rscompress.cc \ util/smallobject.cc \ util/rsdir.cc \ diff --git a/libretroshare/src/file_sharing/rsexpr.cc b/libretroshare/src/util/rsexpr.cc similarity index 100% rename from libretroshare/src/file_sharing/rsexpr.cc rename to libretroshare/src/util/rsexpr.cc From 02d2fb42b7b6093084354afc5cbbe10fa59780a8 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 13:53:51 +0200 Subject: [PATCH 074/174] fixed search and filter for regular expressions and terms --- .../src/file_sharing/dir_hierarchy.cc | 35 ++++++++++----- libretroshare/src/file_sharing/p3filelists.cc | 45 ++++++++++++++----- libretroshare/src/retroshare/rsexpr.h | 4 +- 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 1735beb79..8b03060d1 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -23,6 +23,7 @@ * */ #include "util/rsdir.h" +#include "retroshare/rsexpr.h" #include "dir_hierarchy.h" #include "filelist_io.h" @@ -595,21 +596,33 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list &results) const { -#ifdef TODO for(std::map::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it) - { - const std::string &str1 = static_cast(mNodes[fit->second])->file_name; + if(mNodes[it->second] != NULL && exp->eval( + DirectoryStorageExprFileEntry(*static_cast(mNodes[it->second]), + *static_cast(mNodes[mNodes[it->second]->parent_index]) + ))) + results.push_back(it->second); - { - /*Evaluate the boolean expression and add it to the results if its true*/ - bool ret = exp->eval(fit->second); - if (ret == true){ - results.push_back(fit->second); - } - } -#endif + return 0; } int InternalFileHierarchyStorage::searchTerms(const std::list& terms, std::list &results) const diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ea77de3a6..3eaf21c87 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -807,19 +807,37 @@ bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags, int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) { - RS_STACK_MUTEX(mFLSMtx) ; - std::list firesults; - mLocalSharedDirs->searchTerms(keywords,firesults) ; + { + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalSharedDirs->searchTerms(keywords,firesults) ; + + for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,0,p); + *it = (intptr_t)p ; + } + } return filterResults(firesults,results,flags,client_peer_id) ; } int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const { - RS_STACK_MUTEX(mFLSMtx) ; - std::list firesults; - mLocalSharedDirs->searchBoolExp(exp,firesults) ; + { + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalSharedDirs->searchBoolExp(exp,firesults) ; + + for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,0,p); + *it = (intptr_t)p ; + } + } return filterResults(firesults,results,flags,client_peer_id) ; } @@ -854,18 +872,21 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li { results.clear(); -#ifdef P3FILELISTS_DEBUG - if((flags & ~RS_FILE_HINTS_PERMISSION_MASK) > 0) - std::cerr << "(EE) ***** FileIndexMonitor:: Flags ERROR in filterResults!!" << std::endl; -#endif + flags &= RS_FILE_HINTS_PERMISSION_MASK; // remove other flags. + /* translate/filter results */ for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) { DirDetails cdetails ; - RequestDirDetails ((void*)(intptr_t)*rit,cdetails,FileSearchFlags(0u)); + + if(!RequestDirDetails ((void*)(intptr_t)*rit,cdetails,RS_FILE_HINTS_LOCAL)) + { + std::cerr << "(EE) Cannot get dir details for entry " << (void*)(intptr_t)*rit << std::endl; + continue ; + } #ifdef P3FILELISTS_DEBUG - std::cerr << "Filtering candidate " << (*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; + std::cerr << "Filtering candidate " << (void*)(intptr_t)(*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; #endif if(!peer_id.isNull()) diff --git a/libretroshare/src/retroshare/rsexpr.h b/libretroshare/src/retroshare/rsexpr.h index fc339e87c..4cf9e7983 100644 --- a/libretroshare/src/retroshare/rsexpr.h +++ b/libretroshare/src/retroshare/rsexpr.h @@ -115,8 +115,8 @@ public: virtual const std::string& file_name() const =0; virtual uint64_t file_size() const =0; virtual time_t file_modtime() const =0; - virtual time_t file_popularity() const =0; - virtual const std::string file_parent_path() const =0; + virtual uint32_t file_popularity() const =0; + virtual std::string file_parent_path() const =0; virtual const RsFileHash& file_hash() const =0; }; From 53c65fff9fb7cbd85f2dca68c952dada1b4585a4 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 18:54:00 +0200 Subject: [PATCH 075/174] removed cache transfers from the GUI --- libretroshare/src/ft/ftserver.cc | 13 ------- libretroshare/src/ft/ftserver.h | 7 ---- libretroshare/src/libretroshare.pro | 2 -- libretroshare/src/plugins/pluginmanager.cc | 12 ++----- libretroshare/src/retroshare/rsfiles.h | 4 --- libretroshare/src/rsserver/rsinit.cc | 5 --- .../src/gui/FileTransfer/TransfersDialog.cpp | 18 ---------- .../src/gui/FileTransfer/TransfersDialog.h | 3 -- .../src/gui/settings/TransferPage.cpp | 10 ------ .../src/gui/settings/TransferPage.h | 1 - .../src/gui/settings/TransferPage.ui | 36 +++++-------------- 11 files changed, 11 insertions(+), 100 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 1a558f9d6..06c538a70 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -44,11 +44,6 @@ const int ftserverzone = 29539; #include "pqi/p3notify.h" #include "rsserver/p3face.h" - -// Includes CacheStrapper / FiMonitor / FiStore for us. - -#include "ft/ftdbase.h" - #include "pqi/pqi.h" #include "pqi/p3linkmgr.h" @@ -299,14 +294,6 @@ bool ftServer::FileClearCompleted() { return mFtController->FileClearCompleted(); } -void ftServer::setMinPrioritizedTransfers(uint32_t s) -{ - mFtController->setMinPrioritizedTransfers(s) ; -} -uint32_t ftServer::getMinPrioritizedTransfers() -{ - return mFtController->getMinPrioritizedTransfers() ; -} void ftServer::setQueueSize(uint32_t s) { mFtController->setQueueSize(s) ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 1173d8311..e61a72e49 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -47,7 +47,6 @@ #include "turtle/turtleclientservice.h" #include "services/p3service.h" #include "retroshare/rsfiles.h" -//#include "dbase/cachestrapper.h" #include "pqi/pqi.h" #include "pqi/p3cfgmgr.h" @@ -55,10 +54,6 @@ class p3ConnectMgr; class p3FileDatabase; -class CacheStrapper; -class CacheTransfer; - -class ftCacheStrapper; class ftFiStore; class ftFiMonitor; @@ -145,8 +140,6 @@ public: /*** * Control of Downloads Priority. ***/ - virtual uint32_t getMinPrioritizedTransfers() ; - virtual void setMinPrioritizedTransfers(uint32_t s) ; virtual uint32_t getQueueSize() ; virtual void setQueueSize(uint32_t s) ; virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv); diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 260626db0..971578c0a 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -371,7 +371,6 @@ HEADERS += ft/ftchunkmap.h \ ft/ftcontroller.h \ ft/ftdata.h \ ft/ftdatamultiplex.h \ - ft/ftdbase.h \ ft/ftextralist.h \ ft/ftfilecreator.h \ ft/ftfileprovider.h \ @@ -531,7 +530,6 @@ HEADERS += util/folderiterator.h \ SOURCES += ft/ftchunkmap.cc \ ft/ftcontroller.cc \ ft/ftdatamultiplex.cc \ - ft/ftdbase.cc \ ft/ftextralist.cc \ ft/ftfilecreator.cc \ ft/ftfileprovider.cc \ diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index af1bfbb65..688cc0bb0 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -410,7 +410,8 @@ void RsPluginManager::slowTickPlugins(time_t seconds) void RsPluginManager::registerCacheServices() { -#warning this code should go + // this is removed since the old cache syste is gone, but we need to make it register new GXS group services instead. +#ifdef REMOVED std::cerr << " Registering cache services." << std::endl; for(uint32_t i=0;i<_plugins.size();++i) @@ -419,6 +420,7 @@ void RsPluginManager::registerCacheServices() //rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ; std::cerr << " adding new cache pair for plugin " << _plugins[i].plugin->getPluginName() << ", with RS_ID " << _plugins[i].plugin->rs_service_id() << std::endl ; } +#endif } void RsPluginManager::registerClientServices(p3ServiceServer *pqih) @@ -547,14 +549,6 @@ bool RsPluginManager::saveList(bool& cleanup, std::list& list) return true; } -// RsCacheService::RsCacheService(uint16_t service_type,uint32_t tick_delay, RsPluginHandler* pgHandler) -// : CacheSource(service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getLocalCacheDir()), -// CacheStore (service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getFileServer()->getCacheTransfer(), pgHandler->getRemoteCacheDir()), -// p3Config(), -// _tick_delay_in_seconds(tick_delay) -// { -// } - RsPQIService::RsPQIService(uint16_t /*service_type*/, uint32_t /*tick_delay_in_seconds*/, RsPluginHandler* /*pgHandler*/) : p3Service(),p3Config() { diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 2236a6a8b..247270007 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -37,8 +37,6 @@ class RsFiles; extern RsFiles *rsFiles; namespace RsRegularExpression { class Expression; } -class CacheStrapper ; -class CacheTransfer; /* These are used mainly by ftController at the moment */ const uint32_t RS_FILE_CTRL_PAUSE = 0x00000100; @@ -147,8 +145,6 @@ class RsFiles /*** * Control of Downloads Priority. ***/ - virtual uint32_t getMinPrioritizedTransfers() = 0 ; - virtual void setMinPrioritizedTransfers(uint32_t s) = 0 ; virtual uint32_t getQueueSize() = 0 ; virtual void setQueueSize(uint32_t s) = 0 ; virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove mv) = 0; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 066616633..82c5168f9 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1675,11 +1675,6 @@ int RsServer::StartupRetroShare() /**************************************************************************/ std::cerr << "(2) Load configuration files" << std::endl; - /* NOTE: CacheStrapper's load causes Cache Files to be - * loaded into all the CacheStores/Sources. This happens - * after all the other configurations have happened. - */ - mConfigMgr->loadConfiguration(); /**************************************************************************/ diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index c19762027..195959e16 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -349,9 +349,6 @@ TransfersDialog::TransfersDialog(QWidget *parent) // ui.tunnelInfoWidget->setFocusPolicy(Qt::NoFocus); /** Setup the actions for the context menu */ - toggleShowCacheTransfersAct = new QAction(tr( "Show file list transfers" ), this ); - toggleShowCacheTransfersAct->setCheckable(true) ; - connect(toggleShowCacheTransfersAct,SIGNAL(triggered()),this,SLOT(toggleShowCacheTransfers())) ; // Actions. Only need to be defined once. pauseAct = new QAction(QIcon(IMAGE_PAUSE), tr("Pause"), this); @@ -519,12 +516,6 @@ UserNotify *TransfersDialog::getUserNotify(QObject *parent) return new TransferUserNotify(parent); } -void TransfersDialog::toggleShowCacheTransfers() -{ - _show_cache_transfers = !_show_cache_transfers ; - insertTransfers() ; -} - void TransfersDialog::processSettings(bool bLoad) { m_bProcessSettings = true; @@ -537,9 +528,6 @@ void TransfersDialog::processSettings(bool bLoad) if (bLoad) { // load settings - // state of checks - _show_cache_transfers = Settings->value("showCacheTransfers", false).toBool(); - // state of the lists DLHeader->restoreState(Settings->value("downloadList").toByteArray()); ULHeader->restoreState(Settings->value("uploadList").toByteArray()); @@ -565,9 +553,6 @@ void TransfersDialog::processSettings(bool bLoad) } else { // save settings - // state of checks - Settings->setValue("showCacheTransfers", _show_cache_transfers); - // state of the lists Settings->setValue("downloadList", DLHeader->saveState()); Settings->setValue("uploadList", ULHeader->saveState()); @@ -801,9 +786,6 @@ void TransfersDialog::downloadListCustomPopupMenu( QPoint /*point*/ ) contextMnu.addSeparator() ;//----------------------------------------------- - contextMnu.addAction( toggleShowCacheTransfersAct ) ; - toggleShowCacheTransfersAct->setChecked(_show_cache_transfers) ; - collCreateAct->setEnabled(true) ; collModifAct->setEnabled(single && add_CollActions) ; collViewAct->setEnabled(single && add_CollActions) ; diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h index 0e6e9ad4b..8477d0749 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h @@ -199,7 +199,6 @@ private: QAction *chunkProgressiveAct; QAction *chunkStreamingAct; QAction *detailsFileAct; - QAction *toggleShowCacheTransfersAct; QAction *renameFileAct; QAction *specifyDestinationDirectoryAct; QAction *expandAllAct; @@ -249,7 +248,6 @@ private: /** Qt Designer generated object */ Ui::TransfersDialog ui; - bool _show_cache_transfers ; public slots: // these two functions add entries to the transfers dialog, and return the row id of the entry modified/added // @@ -259,7 +257,6 @@ public slots: int addUploadItem(const QString& symbol, const QString& name, const QString& coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, const QString& sources,const QString& source_id, const QString& status, qlonglong completed, qlonglong remaining); void showFileDetails() ; - void toggleShowCacheTransfers() ; double getProgress(int row, QStandardItemModel *model); double getSpeed(int row, QStandardItemModel *model); diff --git a/retroshare-gui/src/gui/settings/TransferPage.cpp b/retroshare-gui/src/gui/settings/TransferPage.cpp index 273f5be5b..bccdcea3c 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.cpp +++ b/retroshare-gui/src/gui/settings/TransferPage.cpp @@ -36,7 +36,6 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags) ui.setupUi(this); ui._queueSize_SB->setValue(rsFiles->getQueueSize()) ; - ui._minPrioritized_SB->setValue(rsFiles->getMinPrioritizedTransfers()) ; switch(rsFiles->defaultChunkStrategy()) { @@ -48,7 +47,6 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags) ui._diskSpaceLimit_SB->setValue(rsFiles->freeDiskSpaceLimit()) ; QObject::connect(ui._queueSize_SB,SIGNAL(valueChanged(int)),this,SLOT(updateQueueSize(int))) ; - QObject::connect(ui._minPrioritized_SB,SIGNAL(valueChanged(int)),this,SLOT(updateMinPrioritized(int))) ; QObject::connect(ui._defaultStrategy_CB,SIGNAL(activated(int)),this,SLOT(updateDefaultStrategy(int))) ; QObject::connect(ui._diskSpaceLimit_SB,SIGNAL(valueChanged(int)),this,SLOT(updateDiskSizeLimit(int))) ; QObject::connect(ui._max_tr_up_per_sec_SB, SIGNAL( valueChanged( int ) ), this, SLOT( updateMaxTRUpRate(int) ) ); @@ -80,15 +78,7 @@ void TransferPage::updateDiskSizeLimit(int s) rsFiles->setFreeDiskSpaceLimit(s) ; } -void TransferPage::updateMinPrioritized(int s) -{ - rsFiles->setMinPrioritizedTransfers(s) ; -} void TransferPage::updateQueueSize(int s) { - if(ui._minPrioritized_SB->value() > s) - { - ui._minPrioritized_SB->setValue(s) ; - } rsFiles->setQueueSize(s) ; } diff --git a/retroshare-gui/src/gui/settings/TransferPage.h b/retroshare-gui/src/gui/settings/TransferPage.h index 3a28f41ac..373d9a17f 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.h +++ b/retroshare-gui/src/gui/settings/TransferPage.h @@ -46,7 +46,6 @@ class TransferPage: public ConfigPage public slots: void updateQueueSize(int) ; - void updateMinPrioritized(int) ; void updateDefaultStrategy(int) ; void updateDiskSizeLimit(int) ; void updateMaxTRUpRate(int); diff --git a/retroshare-gui/src/gui/settings/TransferPage.ui b/retroshare-gui/src/gui/settings/TransferPage.ui index 14023f0ab..ceeae7f47 100644 --- a/retroshare-gui/src/gui/settings/TransferPage.ui +++ b/retroshare-gui/src/gui/settings/TransferPage.ui @@ -6,8 +6,8 @@ 0 0 - 604 - 340 + 700 + 356 @@ -28,13 +28,6 @@ - - - - Slots reserved for non-cache transfers: - - - @@ -76,19 +69,6 @@ - - - - <html><head/><body><p>You can use this to force RetroShare to download your files rather <br/>than cache files for as many slots as requested. Setting that number <br/>to be equal to the queue size above will always prioritize your files<br/>over cache. <br/><br/>It is however recommended to leave at least a few slots for cache files. For now, cache files are only used to transfer friend file lists.</p></body></html> - - - 1 - - - 3 - - - @@ -168,12 +148,12 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> <html><head><meta name="qrichtext" content="1" /><style type="text/css"> p, li { white-space: pre-wrap; } -</style></head><body style=" font-family:'Ubuntu'; font-size:11pt; font-weight:400; font-style:normal;"> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-family:'Sans'; font-size:8pt; font-weight:600;">RetroShare</span><span style=" font-family:'Sans'; font-size:8pt;"> is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.</span></p> -<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-family:'Sans'; font-size:8pt;"><br /></p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-family:'Sans'; font-size:8pt;">You can separately setup share flags for each shared directory in the shared files dialog to be:</span></p> -<ul style="margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;"><li style=" font-family:'Sans'; font-size:8pt;" style=" margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Browsable by friends</span>: files are seen by your friends.</li> -<li style=" font-family:'Sans'; font-size:8pt;" style=" margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Anonymously shared</span>: files are anonymously reachable through distant F2F tunnels.</li></ul></body></html> +</style></head><body style=" font-family:'Sans'; font-size:9pt; font-weight:400; font-style:normal;"> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:8pt; font-weight:600;">RetroShare</span><span style=" font-size:8pt;"> is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.</span></p> +<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-size:8pt;"><br /></p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:8pt;">You can separately setup share flags for each shared directory in the shared files dialog to be:</span></p> +<ul style="margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;"><li style=" font-size:8pt;" style=" margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Browsable by friends</span>: files are seen by your friends.</li> +<li style=" font-size:8pt;" style=" margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">Anonymously shared</span>: files are anonymously reachable through distant F2F tunnels.</li></ul></body></html> From 93818f6088db01ee36fd22ca2f7477634c4a44e9 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Tue, 13 Sep 2016 21:23:27 +0200 Subject: [PATCH 076/174] simplified a bit the code in Flat_Model, and tried to fix the crash when sorting items --- libretroshare/src/ft/ftcontroller.h | 1 - libretroshare/src/plugins/pluginmanager.cc | 1 - libretroshare/src/rsserver/rsinit.cc | 1 - libretroshare/src/util/rsexpr.cc | 1 - retroshare-gui/src/gui/RemoteDirModel.cpp | 104 +++++++++++-------- retroshare-gui/src/gui/RemoteDirModel.h | 5 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 23 ++-- 7 files changed, 77 insertions(+), 59 deletions(-) diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index 5ee84623b..516b07050 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -48,7 +48,6 @@ class ftDataMultiplex; class p3turtle ; class p3ServiceControl; -#include "dbase/cachestrapper.h" #include "util/rsthreads.h" #include "pqi/pqiservicemonitor.h" #include "pqi/p3cfgmgr.h" diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 688cc0bb0..98335ea5a 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 82c5168f9..6726c1656 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -805,7 +805,6 @@ bool RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port) #include //#include -#include "dbase/cachestrapper.h" #include "ft/ftserver.h" #include "ft/ftcontroller.h" diff --git a/libretroshare/src/util/rsexpr.cc b/libretroshare/src/util/rsexpr.cc index 6b4d7ce20..7e35a1873 100644 --- a/libretroshare/src/util/rsexpr.cc +++ b/libretroshare/src/util/rsexpr.cc @@ -24,7 +24,6 @@ */ -#include "dbase/findex.h" #include "retroshare/rsexpr.h" #include "retroshare/rstypes.h" #include diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 2153f255d..1afdc94ef 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -194,6 +194,7 @@ int FlatStyle_RDM::rowCount(const QModelIndex &parent) const std::cerr << "RetroshareDirModel::rowCount(): " << parent.internalPointer(); std::cerr << ": "; #endif + RS_STACK_MUTEX(_ref_mutex) ; return _ref_entries.size() ; } @@ -501,7 +502,12 @@ QVariant FlatStyle_RDM::sortRole(const QModelIndex& index,const DirDetails& deta case 1: return (qulonglong) details.count; case 2: return details.age; case 3: return QString::fromUtf8(rsPeers->getPeerName(details.id).c_str()); - case 4: return _ref_entries[index.row()].second ; + + case 4: { + RS_STACK_MUTEX(_ref_mutex) ; + + return computeDirectoryPath(details); + } } } return QVariant(); @@ -765,9 +771,11 @@ QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent if(row < 0) return QModelIndex() ; - if(row < (int) _ref_entries.size()) + RS_STACK_MUTEX(_ref_mutex) ; + + if(row < (int) _ref_entries.size()) { - void *ref = _ref_entries[row].first ; + void *ref = _ref_entries[row]; #ifdef RDM_DEBUG std::cerr << "Creating index 2 row=" << row << ", column=" << column << ", ref=" << (void*)ref << std::endl; @@ -903,7 +911,7 @@ void RetroshareDirModel::postMods() // changePersistentIndexList(piList, empty); /* Clear caches */ - mCache.clear(); + //mCache.clear(); #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::postMods()" << std::endl; @@ -1316,17 +1324,18 @@ TreeStyle_RDM::~TreeStyle_RDM() } void FlatStyle_RDM::postMods() { - if(visible()) + if(visible()) { - _ref_entries.clear() ; - _ref_stack.clear() ; + emit layoutAboutToBeChanged(); - _ref_stack.push_back(NULL) ; // init the stack with the topmost parent directory - - std::cerr << "FlatStyle_RDM::postMods(): cleared ref entries" << std::endl; - _needs_update = false ; - updateRefs() ; - } + { + RS_STACK_MUTEX(_ref_mutex) ; + _ref_stack.clear() ; + _ref_stack.push_back(NULL) ; // init the stack with the topmost parent directory + _needs_update = false ; + } + QTimer::singleShot(100,this,SLOT(updateRefs())) ; + } else _needs_update = true ; } @@ -1339,47 +1348,56 @@ void FlatStyle_RDM::updateRefs() return ; } - RetroshareDirModel::preMods() ; + RetroshareDirModel::preMods() ; - static const uint32_t MAX_REFS_PER_SECOND = 2000 ; + + static const uint32_t MAX_REFS_PER_SECOND = 2000 ; uint32_t nb_treated_refs = 0 ; - while(!_ref_stack.empty()) - { - void *ref = _ref_stack.back() ; + { + RS_STACK_MUTEX(_ref_mutex) ; + + _ref_entries.clear() ; + + std::cerr << "FlatStyle_RDM::postMods(): cleared ref entries" << std::endl; + + while(!_ref_stack.empty()) + { + void *ref = _ref_stack.back() ; #ifdef RDM_DEBUG - std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl; + std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl; #endif - _ref_stack.pop_back() ; + _ref_stack.pop_back() ; - DirDetails details ; + DirDetails details ; - if (requestDirDetails(ref, RemoteMode,details)) - { - if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons. - _ref_entries.push_back(std::pair(ref,computeDirectoryPath(details))); + if (requestDirDetails(ref, RemoteMode,details)) + { + if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons. + _ref_entries.push_back(ref) ; #ifdef RDM_DEBUG - std::cerr << "FlatStyle_RDM::postMods(): adding ref " << ref << std::endl; + std::cerr << "FlatStyle_RDM::postMods(): adding ref " << ref << std::endl; #endif - for(uint32_t i=0;i MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to - { // the user and setup a timer to finish the job later. - _needs_update = true ; + for(uint32_t i=0;i MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to + { // the user and setup a timer to finish the job later. + _needs_update = true ; - if(visible()) - QTimer::singleShot(2000,this,SLOT(updateRefs())) ; - else - std::cerr << "Not visible: suspending update"<< std::endl; - break ; - } - } - std::cerr << "reference tab contains " << _ref_entries.size() << " files" << std::endl; + if(visible()) + QTimer::singleShot(2000,this,SLOT(updateRefs())) ; + else + std::cerr << "Not visible: suspending update"<< std::endl; + break ; + } + } + std::cerr << "reference tab contains " << _ref_entries.size() << " files" << std::endl; - if(_ref_stack.empty()) - _needs_update = false ; + if(_ref_stack.empty()) + _needs_update = false ; + } - RetroshareDirModel::postMods() ; + RetroshareDirModel::postMods() ; } diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index 54f60ffad..72a3fad7c 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -194,7 +194,7 @@ class FlatStyle_RDM: public RetroshareDirModel public: FlatStyle_RDM(bool mode) - : RetroshareDirModel(mode) + : RetroshareDirModel(mode), _ref_mutex("Flat file list") { _needs_update = true ; } @@ -223,7 +223,8 @@ class FlatStyle_RDM: public RetroshareDirModel QString computeDirectoryPath(const DirDetails& details) const ; - std::vector > _ref_entries ;// used to store the refs to display + mutable RsMutex _ref_mutex ; + std::vector _ref_entries ;// used to store the refs to display std::vector _ref_stack ; // used to store the refs to update bool _needs_update ; }; diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index aaaa79284..80cbfa856 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -144,14 +144,17 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD tree_proxyModel->setSortRole(RetroshareDirModel::SortRole); tree_proxyModel->sort(COLUMN_NAME); - flat_proxyModel = new SFDSortFilterProxyModel(flat_model, this); - flat_proxyModel->setDynamicSortFilter(true); - flat_proxyModel->setSourceModel(flat_model); - flat_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive); - flat_proxyModel->setSortRole(RetroshareDirModel::SortRole); - flat_proxyModel->sort(COLUMN_NAME); + flat_proxyModel = new SFDSortFilterProxyModel(flat_model, this); + flat_proxyModel->setSourceModel(flat_model); + flat_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive); + flat_proxyModel->setSortRole(RetroshareDirModel::SortRole); + flat_proxyModel->sort(COLUMN_NAME); - connect(ui.filterClearButton, SIGNAL(clicked()), this, SLOT(clearFilter())); + // Mr.Alice: I removed this because it causes a crash for some obscur reason. Apparently when the model is changed, the proxy model cannot + // deal with the change by itself. Should I call something specific? I've no idea. Removing this does not seem to cause any harm either. + //flat_proxyModel->setDynamicSortFilter(true); + + connect(ui.filterClearButton, SIGNAL(clicked()), this, SLOT(clearFilter())); connect(ui.filterStartButton, SIGNAL(clicked()), this, SLOT(startFilter())); connect(ui.filterPatternLineEdit, SIGNAL(returnPressed()), this, SLOT(startFilter())); connect(ui.filterPatternLineEdit, SIGNAL(textChanged(const QString &)), this, SLOT(filterRegExpChanged())); @@ -346,13 +349,13 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) if(viewTypeIndex==VIEW_TYPE_TREE) { model = tree_model ; - proxyModel = tree_proxyModel ; + proxyModel = tree_proxyModel ; } else { model = flat_model ; - proxyModel = flat_proxyModel ; - } + proxyModel = flat_proxyModel ; + } showProperColumns() ; From afbb641b58b63405000307d3987feea755a88c1a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 20:41:22 +0200 Subject: [PATCH 077/174] fixed forceDirectoryCheck and inDirectoryCheck methods --- .../src/file_sharing/directory_updater.cc | 5 ++ .../src/file_sharing/directory_updater.h | 3 +- .../src/file_sharing/file_sharing_defaults.h | 6 +- libretroshare/src/file_sharing/p3filelists.cc | 58 ++++++++++--------- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 23c1fe7c6..06b9af252 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -164,6 +164,11 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p } } +bool LocalDirectoryUpdater::inDirectoryCheck() const +{ + return mHashCache->isRunning(); +} + void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) { if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index 25f94d28a..a4a3f4242 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -38,7 +38,8 @@ public: LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ; virtual ~LocalDirectoryUpdater() {} - virtual void forceUpdate(); + void forceUpdate(); + bool inDirectoryCheck() const ; void setFileWatchPeriod(uint32_t seconds) { mDelayBetweenDirectoryUpdates = seconds ; } uint32_t fileWatchPeriod() const { return mDelayBetweenDirectoryUpdates ; } diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 4c47584e9..43cf5406f 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -26,9 +26,9 @@ #pragma once -static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 100 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 10 ; // 10 seconds for testing. Should be much more!! -static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 10 ; // 10 seconds for testing. Should be much more!! +static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes +static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 300 ; // 5 minutes static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 3eaf21c87..db52d40e3 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -36,7 +36,8 @@ #include "rsserver/p3face.h" -#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << ": p3FileLists: " +#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : " +#define P3FILELISTS_ERROR() std::cerr << "***ERROR***" << " : FILE_LISTS : " << __FUNCTION__ << " : " static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; @@ -564,7 +565,7 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi if(fi+1 >= (1<= (1<< NB_ENTRY_INDEX_BITS)) { - std::cerr << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; + P3FILELISTS_ERROR() << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; return false ; } @@ -585,11 +586,11 @@ void p3FileDatabase::requestDirUpdate(void *ref) time_t recurs_max_modf_TS_remote_time,local_update_TS; - std::cerr << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; + P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; if(!mRemoteDirectories[fi-1]->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << " (EE) Cannot get max known recurs modf time!" << std::endl; + P3FILELISTS_ERROR() << " (EE) Cannot get max known recurs modf time!" << std::endl; return ; } @@ -660,8 +661,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.count = d.children.size(); #ifdef DEBUG_FILE_HIERARCHY - std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; - std::cerr << d << std::endl; + P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; #endif return true ; @@ -675,7 +675,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // check consistency if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) { - std::cerr << "remote request on local index or local request on remote index. This should not happen." << std::endl; + P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl; return false ; } DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); @@ -710,8 +710,8 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.id = storage->peerId(); #ifdef DEBUG_FILE_HIERARCHY - std::cerr << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; - std::cerr << d << std::endl; + P3FILELISTS_DEBUG() << "ExtractData: ref=" << ref << ", flags=" << flags << " : returning this: " << std::endl; + P3FILELISTS_DEBUG() << d << std::endl; #endif return true; @@ -751,11 +751,12 @@ uint32_t p3FileDatabase::getType(void *ref) const void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed { NOT_IMPLEMENTED(); + mLocalDirWatcher->forceUpdate(); } bool p3FileDatabase::inDirectoryCheck() { NOT_IMPLEMENTED(); - return 0; + return mLocalDirWatcher->inDirectoryCheck(); } void p3FileDatabase::setWatchPeriod(uint32_t seconds) { @@ -882,11 +883,11 @@ int p3FileDatabase::filterResults(const std::list& firesults,std::li if(!RequestDirDetails ((void*)(intptr_t)*rit,cdetails,RS_FILE_HINTS_LOCAL)) { - std::cerr << "(EE) Cannot get dir details for entry " << (void*)(intptr_t)*rit << std::endl; + P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << (void*)(intptr_t)*rit << std::endl; continue ; } #ifdef P3FILELISTS_DEBUG - std::cerr << "Filtering candidate " << (void*)(intptr_t)(*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; + P3FILELISTS_DEBUG() << "Filtering candidate " << (void*)(intptr_t)(*rit) << ", flags=" << cdetails.flags << ", peer=" << peer_id ; #endif if(!peer_id.isNull()) @@ -950,7 +951,7 @@ void p3FileDatabase::tickRecv() case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: handleDirSyncResponse( dynamic_cast(item) ) ; break ; default: - std::cerr << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; + P3FILELISTS_ERROR() << "(EE) unhandled packet subtype " << item->PacketSubType() << " in " << __PRETTY_FUNCTION__ << std::endl; } delete item ; @@ -976,7 +977,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index)) { - std::cerr << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; + P3FILELISTS_ERROR() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; return; } @@ -995,7 +996,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) } else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) { - std::cerr << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl; + P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl; ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else @@ -1043,7 +1044,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(it == mPendingSyncRequests.end()) { - std::cerr << " request " << std::hex << item->request_id << std::dec << " cannot be found. ERROR!" << std::endl; + P3FILELISTS_ERROR() << " request " << std::hex << item->request_id << std::dec << " cannot be found. ERROR!" << std::endl; return ; } mPendingSyncRequests.erase(it) ; @@ -1069,38 +1070,39 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) if(!mRemoteDirectories[fi]->getIndexFromDirHash(item->entry_hash,entry_index)) { - std::cerr << std::endl << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; + std::cerr << std::endl; + P3FILELISTS_ERROR() << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; } - std::cerr << " entry index is " << entry_index << " " ; + P3FILELISTS_DEBUG() << " entry index is " << entry_index << " " ; } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) { - P3FILELISTS_DEBUG() << " removing directory with index " << entry_index << " because it does not exist." << std::endl; + std::cerr << " removing directory with index " << entry_index << " because it does not exist." << std::endl; mRemoteDirectories[fi]->removeDirectory(entry_index); mRemoteDirectories[fi]->print(); } else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) { - P3FILELISTS_DEBUG() << " Directory is up to date. Setting local TS." << std::endl; + std::cerr << " Directory is up to date. Setting local TS." << std::endl; mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL)); } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { - P3FILELISTS_DEBUG() << " Item contains directory data. Deserialising/Updating." << std::endl; + std::cerr << " Item contains directory data. Deserialising/Updating." << std::endl; if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed else - std::cerr << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; + P3FILELISTS_ERROR() << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; - std::cerr << " new content after update: " << std::endl; -//#ifdef DEBUG_FILE_HIERARCHY +#ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << " new content after update: " << std::endl; mRemoteDirectories[fi]->print(); -//#endif +#endif } } @@ -1121,7 +1123,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) { - std::cerr << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; + P3FILELISTS_ERROR() << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; } @@ -1172,7 +1174,7 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons if(!rds->getDirHashFromIndex(e,entry_hash) ) { - std::cerr << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; + P3FILELISTS_ERROR() << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; return false; } @@ -1202,7 +1204,7 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons data.peer_id = item->PeerId(); data.flags = item->flags; - std::cerr << " Pushing req in pending list with peer id " << data.peer_id << std::endl; + P3FILELISTS_DEBUG() << " Pushing req in pending list with peer id " << data.peer_id << std::endl; mPendingSyncRequests[sync_req_id] = data ; From e985a2c9271555f13ec0399d255332934045ee9d Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 20:43:52 +0200 Subject: [PATCH 078/174] removed call to setDynamicSortFilter in SharedFilesDialog, as a workaround for the crash when sorting --- retroshare-gui/src/gui/SharedFilesDialog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 80cbfa856..8ebb22d39 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -138,7 +138,6 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD flat_model = _flat_model ; tree_proxyModel = new SFDSortFilterProxyModel(tree_model, this); - tree_proxyModel->setDynamicSortFilter(true); tree_proxyModel->setSourceModel(tree_model); tree_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive); tree_proxyModel->setSortRole(RetroshareDirModel::SortRole); @@ -152,6 +151,7 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD // Mr.Alice: I removed this because it causes a crash for some obscur reason. Apparently when the model is changed, the proxy model cannot // deal with the change by itself. Should I call something specific? I've no idea. Removing this does not seem to cause any harm either. + //tree_proxyModel->setDynamicSortFilter(true); //flat_proxyModel->setDynamicSortFilter(true); connect(ui.filterClearButton, SIGNAL(clicked()), this, SLOT(clearFilter())); From ddc89a653502c50431ee1801a93c8ec3a698d158 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 21:52:16 +0200 Subject: [PATCH 079/174] fixed core dump in printout of empty slots. Improved dir sync-ing strategy --- .../src/file_sharing/dir_hierarchy.cc | 11 ++++--- libretroshare/src/file_sharing/p3filelists.cc | 30 ++++++++++++------- retroshare-gui/src/gui/SharedFilesDialog.cpp | 13 +++++--- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 8b03060d1..45ac0e8bb 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -466,6 +466,8 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI uint32_t n=0; for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_update_time = 0 ; // force the update of the subdir. + mNodes[d.subdirs[i]]->parent_index = indx ; mNodes[d.subdirs[i]]->row = n++ ; } @@ -767,10 +769,11 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry recursPrint(depth+1,d.subdirs[i]) ; for(uint32_t i=0;i(mNodes[d.subfiles[i]])); - std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; - } + if(mNodes[d.subfiles[i]] != NULL) + { + FileEntry& f(*static_cast(mNodes[d.subfiles[i]])); + std::cerr << indent << " hash:" << f.file_hash << " ts:" << (uint64_t)f.file_modtime << " " << f.file_size << " " << f.file_name << ", parent: " << f.parent_index << ", row: " << f.row << std::endl; + } } bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index db52d40e3..0a51b72dd 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -930,9 +930,23 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // - the max time is computed upward until the root of the hierarchy // - because the hash is performed late, the last modf time upward is updated only when the hash is obtained. // -// Remote dirs store the last modif time of the files/dir in the friend's time -// - local node sends the last known modf time to friends, -// - friends respond with either a full directory content, or an acknowledge that the time is right +// Remote dirs store +// 1 - recursive modif time of the files/dir in the friend's time +// 2 - update TS in the local time +// +// The update algorithm is the following: +// +// [Client side] +// - every 30 sec, send a sync packet for the root of the hierarchy containing the remote known max recurs TS. +// - crawl through all directories and if the update TS is 0, ask for sync too. +// - when a sync respnse is received: +// - if response is ACK, set the update time to now. Nothing changed. +// - if response is an update +// * locally update the subfiles +// * for all subdirs, set the local update time to 0 (so as to force a sync) +// +// [Server side] +// - when sync req is received, compare to local recurs TS, and send ACK or full update accordingly // // Directories are designated by their hash, instead of their index. This allows to hide the non shared directories // behind a layer of abstraction, at the cost of a logarithmic search, which is acceptable as far as dir sync-ing between @@ -1004,7 +1018,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) time_t local_recurs_max_time,local_update_time; mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time); - if(item->last_known_recurs_modf_TS < local_recurs_max_time) + if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust. { P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl; @@ -1129,16 +1143,10 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // compare TS - if(now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) // we need to compare local times only. We cannot compare local (now) with remote time. - { + if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. if(generateAndSendSyncRequest(rds,e,recurs_max_modf_TS_remote_time)) P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; - // Dont recurs into sub-directories, since we dont know yet were to go. - - //return ; - } - for(DirectoryStorage::DirIterator it(rds,e);it;++it) locked_recursSweepRemoteDirectory(rds,*it,depth+1); } diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 8ebb22d39..f48394cca 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -163,10 +163,10 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD QHeaderView * header = ui.dirTreeView->header () ; header->resizeSection ( COLUMN_NAME, 490 ); - header->resizeSection ( COLUMN_SIZE, 70 ); - header->resizeSection ( COLUMN_AGE, 100 ); - header->resizeSection ( COLUMN_FRIEND, 100 ); - header->resizeSection ( COLUMN_DIR, 100 ); + header->resizeSection ( COLUMN_SIZE, 70 ); + header->resizeSection ( COLUMN_AGE, 100 ); + header->resizeSection ( COLUMN_FRIEND,100); + header->resizeSection ( COLUMN_DIR, 100 ); header->setStretchLastSection(false); @@ -869,6 +869,10 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand if(ui.dirTreeView->model() == NULL) return ; + // we need to disable this, because the signal will trigger unnecessary update at the friend. + + ui.dirTreeView->blockSignals(true) ; + #ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Restoring expanded items. " << std::endl; #endif @@ -877,6 +881,7 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); recursRestoreExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); } + ui.dirTreeView->blockSignals(false) ; } void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp) From f58e9c1a731262df18a216868ad46a3bd7ee6893 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 14 Sep 2016 22:41:02 +0200 Subject: [PATCH 080/174] improved debug output of directory storage, and fixed bug causing loading error (previously crashing files should be removed) --- .../src/file_sharing/dir_hierarchy.cc | 73 ++++++++++++------- libretroshare/src/file_sharing/filelist_io.cc | 6 +- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 45ac0e8bb..fe5f67d5c 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -22,14 +22,16 @@ * Please report all bugs and problems to "retroshare.project@gmail.com". * */ +#include #include "util/rsdir.h" +#include "util/rsprint.h" #include "retroshare/rsexpr.h" #include "dir_hierarchy.h" #include "filelist_io.h" //#include "rsexpr.h" -#define DEBUG_DIRECTORY_STORAGE 1 +//#define DEBUG_DIRECTORY_STORAGE 1 /******************************************************************************************************************/ /* Internal File Hierarchy Storage */ @@ -891,6 +893,22 @@ bool InternalFileHierarchyStorage::save(const std::string& fname) } } +class read_error +{ +public: + read_error(unsigned char *sec,uint32_t size,uint32_t offset,uint8_t expected_tag) + { + std::ostringstream s ; + s << "At offset " << offset << "/" << size << ": expected section tag " << std::hex << (int)expected_tag << std::dec << " but got " << RsUtil::BinToHex(&sec[offset],std::min((int)size-(int)offset, 15)) << "..." << std::endl; + err_string = s.str(); + } + read_error(const std::string& s) : err_string(s) {} + + const std::string& what() const { return err_string ; } +private: + std::string err_string ; +}; + bool InternalFileHierarchyStorage::load(const std::string& fname) { unsigned char *buffer = NULL ; @@ -900,16 +918,16 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) try { if(!FileListIO::loadEncryptedDataFromFile(fname,buffer,buffer_size) ) - throw std::runtime_error("Read error") ; + throw read_error("Cannot decrypt") ; // Read some header uint32_t version, n_nodes ; - if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,version)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION,version)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION) ; if(version != (uint32_t) FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001) throw std::runtime_error("Wrong version number") ; - if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_nodes)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_nodes)) throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; // Write all file/dir entries @@ -925,6 +943,9 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) unsigned char *node_section_data = NULL ; uint32_t node_section_size = 0 ; uint32_t node_section_offset = 0 ; +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "reading node " << i << ", offset " << buffer_offset << " : " << RsUtil::BinToHex(&buffer[buffer_offset],std::min((int)buffer_size - (int)buffer_offset,100)) << "..." << std::endl; +#endif if(FileListIO::readField(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY,node_section_data,node_section_size)) { @@ -936,13 +957,13 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint32_t row ; uint32_t parent_index ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index)) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file_name )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file_size )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file_hash )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,file_modtime)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ) ; if(node_index >= mNodes.size()) mNodes.resize(node_index+1,NULL) ; @@ -967,15 +988,15 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint32_t row ; uint32_t parent_index ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ,dir_hash )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw std::runtime_error("Read error") ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir_most_recent_time )) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ,parent_index )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_PARENT_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ,row )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ROW ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,node_index )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,dir_name )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_NAME ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ,dir_hash )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_DIR_HASH ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,dir_parent_path )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_MODIF_TS ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,dir_update_time )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_UPDATE_TS ) ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,dir_most_recent_time )) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS) ; if(node_index >= mNodes.size()) mNodes.resize(node_index+1,NULL) ; @@ -994,35 +1015,35 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) uint32_t n_subdirs = 0 ; uint32_t n_subfiles = 0 ; - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; for(uint32_t j=0;jsubdirs.push_back(di) ; } - if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw std::runtime_error("Read error") ; + if(!FileListIO::readField(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles)) throw read_error(node_section_data,node_section_size,node_section_offset,FILE_LIST_IO_TAG_RAW_NUMBER) ; for(uint32_t j=0;jsubfiles.push_back(fi) ; } mNodes[node_index] = de ; mDirHashes[de->dir_hash] = node_index ; } else - throw std::runtime_error("Unknown node section.") ; + throw read_error(buffer,buffer_size,buffer_offset,FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY) ; free(node_section_data) ; } free(buffer) ; return true ; } - catch(std::exception& e) + catch(read_error& e) { std::cerr << "Error while reading: " << e.what() << std::endl; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index f4cd1f6be..90297ab0e 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -88,8 +88,10 @@ bool FileListIO::write125Size(unsigned char *data,uint32_t data_size,uint32_t& o if(offset+2 > data_size) return false; - data[offset++] = (uint8_t)((S >> 8) + 192) ; - data[offset++] = (uint8_t)((S & 255) - 192) ; + data[offset+1] = (S - 192) & 0xff ; + data[offset ] = ((S - 192 - data[offset+1]) >> 8) + 192; + + offset += 2 ; return true; } From ca30b0da4476f57aaa62d1bcffa42792767ce20a Mon Sep 17 00:00:00 2001 From: MrAlice Date: Thu, 15 Sep 2016 10:41:40 +0200 Subject: [PATCH 081/174] fixed terrible cost of calling requestDirDetails before createIndex --- .../src/file_sharing/dir_hierarchy.cc | 21 +++++++++ .../src/file_sharing/dir_hierarchy.h | 1 + .../src/file_sharing/directory_storage.cc | 6 +++ .../src/file_sharing/directory_storage.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 45 +++++++++++++++++++ libretroshare/src/file_sharing/p3filelists.h | 1 + libretroshare/src/ft/ftserver.cc | 4 ++ libretroshare/src/ft/ftserver.h | 1 + libretroshare/src/retroshare/rsfiles.h | 3 +- retroshare-gui/src/gui/RemoteDirModel.cpp | 42 ++++------------- 10 files changed, 90 insertions(+), 35 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index fe5f67d5c..cbe6eeab6 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -94,6 +94,27 @@ bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,D return true; } +bool InternalFileHierarchyStorage::getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const +{ + if(!checkIndex(e,FileStorageNode::TYPE_DIR)) + return false ; + + const DirEntry& d = *static_cast(mNodes[e]) ; + + if((uint32_t)row < d.subdirs.size()) + { + c = d.subdirs[row] ; + return true ; + } + + if((uint32_t)row < d.subdirs.size() + d.subfiles.size()) + { + c = d.subfiles[row - (int)d.subdirs.size()] ; + return true ; + } + return false; +} + int InternalFileHierarchyStorage::parentRow(DirectoryStorage::EntryIndex e) { if(!checkIndex(e,FileStorageNode::TYPE_DIR | FileStorageNode::TYPE_FILE) || e==0) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index fb83e5d9d..905fe0552 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -95,6 +95,7 @@ public: int parentRow(DirectoryStorage::EntryIndex e); bool isIndexValid(DirectoryStorage::EntryIndex e) const; bool stampDirectory(const DirectoryStorage::EntryIndex& indx); + bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const; bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); bool removeDirectory(DirectoryStorage::EntryIndex indx) ; bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 2a7cd8f91..faa3ceaee 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -96,6 +96,12 @@ int DirectoryStorage::parentRow(EntryIndex e) const return mFileHierarchy->parentRow(e) ; } +bool DirectoryStorage::getChildIndex(EntryIndex e,int row,EntryIndex& c) const +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + + return mFileHierarchy->getChildIndex(e,row,c) ; +} uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx) { diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index be420f618..dd1202726 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -118,6 +118,7 @@ class DirectoryStorage EntryIndex root() const ; // returns the index of the root directory entry. const RsPeerId& peerId() const { return mPeerId ; } int parentRow(EntryIndex e) const ; + bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0a51b72dd..5a7621b17 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -598,6 +598,51 @@ void p3FileDatabase::requestDirUpdate(void *ref) P3FILELISTS_DEBUG() << " Succeed." << std::endl; } +bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const +{ + RS_STACK_MUTEX(mFLSMtx) ; + + result = NULL ; + + if (ref == NULL) + if(flags & RS_FILE_HINTS_LOCAL) + { + if(row != 0) + return false ; + + convertEntryIndexToPointer(0,0,result); + + return true ; + } + else for(uint32_t i=0;iroot(),i+1,result); + + return true; + } + + uint32_t fi; + DirectoryStorage::EntryIndex e ; + + convertPointerToEntryIndex(ref,e,fi); + + // check consistency + if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) + { + P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl; + return false ; + } + DirectoryStorage *storage = (fi==0)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]); + + // Case where the index is the top of a single person. Can be us, or a friend. + + EntryIndex c = 0; + bool res = storage->getChildIndex(e,row,c); + + convertEntryIndexToPointer(c,fi,result) ; + + return res; +} // This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files. int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index f832dfc53..f72628877 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -116,6 +116,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; int RequestDirDetails(const std::string& path, DirDetails &details) const ; + bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const; // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 06c538a70..6d6547884 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -535,6 +535,10 @@ int ftServer::RequestDirDetails(const RsPeerId& uid, const std::string& path, Di return mFileDatabase->RequestDirDetails(uid, path, details); } +bool ftServer::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) +{ + return mFileDatabase->findChildPointer(ref,row,result,flags) ; +} int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) { return mFileDatabase->RequestDirDetails(ref,details,flags) ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index e61a72e49..cc4004263 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -173,6 +173,7 @@ public: ***/ virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details); virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags); + virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) ; virtual uint32_t getType(void *ref,FileSearchFlags flags) ; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags); diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 247270007..7f6de8be9 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -182,7 +182,8 @@ class RsFiles */ virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) = 0; virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) = 0; - virtual uint32_t getType(void *ref,FileSearchFlags flags) = 0; + virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) =0; + virtual uint32_t getType(void *ref,FileSearchFlags flags) = 0; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) = 0; virtual int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0; diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 1afdc94ef..a2f3fd09b 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -710,6 +710,9 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent std::cerr << ": row:" << row << " col:" << column << " "; #endif + // This function is used extensively. There's no way we can use requestDirDetails() in it, which would + // cause far too much overhead. So we use a dedicated function that only grabs the required information. + if(row < 0) return QModelIndex() ; @@ -722,44 +725,15 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent } ********/ - DirDetails details ; + void *result ; - if (! requestDirDetails(ref, RemoteMode,details)) - { -#ifdef RDM_DEBUG - std::cerr << "lookup failed -> invalid"; - std::cerr << std::endl; -#endif + if(rsFiles->findChildPointer(ref, row, result, ((RemoteMode) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL))) + return createIndex(row, column, result) ; + else return QModelIndex(); - } - - - /* now iterate through the details to - * get the reference number - */ - - if (row >= (int) details.children.size()) - { -#ifdef RDM_DEBUG - std::cerr << "wrong number of children -> invalid"; - std::cerr << std::endl; -#endif - return QModelIndex(); - } - -#ifdef RDM_DEBUG - std::cerr << "success index(" << row << "," << column << "," << details->childrenVector[row].ref << ")"; - std::cerr << std::endl; -#endif - - /* we can just grab the reference now */ - -#ifdef RDM_DEBUG - std::cerr << "Creating index 1 row=" << row << ", column=" << column << ", ref=" << (void*)details.children[row].ref << std::endl; -#endif - return createIndex(row, column, details.children[row].ref); } + QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const { Q_UNUSED(parent); From d3b46221ff22d3964c0130786b4ee39b31491b71 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 15 Sep 2016 21:45:00 +0200 Subject: [PATCH 082/174] merged modifications from origin --- .../src/file_sharing/dir_hierarchy.cc | 23 +++----- .../src/file_sharing/dir_hierarchy.h | 8 ++- .../src/file_sharing/directory_storage.cc | 57 ++++++++++--------- .../src/file_sharing/directory_storage.h | 9 ++- .../src/file_sharing/directory_updater.cc | 4 ++ libretroshare/src/file_sharing/p3filelists.cc | 31 +++++----- libretroshare/src/file_sharing/p3filelists.h | 2 +- libretroshare/src/util/folderiterator.cc | 19 +++++++ libretroshare/src/util/folderiterator.h | 7 +++ 9 files changed, 96 insertions(+), 64 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index cbe6eeab6..ad1004715 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -129,14 +129,6 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e) { return e < mNodes.size() && mNodes[e] != NULL ; } -bool InternalFileHierarchyStorage::stampDirectory(const DirectoryStorage::EntryIndex& indx) -{ - if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) - return false; - - static_cast(mNodes[indx])->dir_modtime = time(NULL) ; - return true; -} bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) { @@ -504,33 +496,32 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI return true; } -bool InternalFileHierarchyStorage::getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) +bool InternalFileHierarchyStorage::getTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* m) const { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + std::cerr << "[directory storage] (EE) cannot get TS for index " << index << ". Not a valid index or not a directory." << std::endl; return false; } DirEntry& d(*static_cast(mNodes[index])) ; - recurs_max_modf_TS = d.dir_most_recent_time ; - local_update_TS = d.dir_update_time ; + TS = d.*m ; return true; } -bool InternalFileHierarchyStorage::setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS) + +bool InternalFileHierarchyStorage::setTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* m) { if(!checkIndex(index,FileStorageNode::TYPE_DIR)) { - std::cerr << "[directory storage] (EE) cannot update TS for index " << index << ". Not a valid index or not a directory." << std::endl; + std::cerr << "[directory storage] (EE) cannot get TS for index " << index << ". Not a valid index or not a directory." << std::endl; return false; } DirEntry& d(*static_cast(mNodes[index])) ; - d.dir_most_recent_time = recurs_max_modf_TS ; - d.dir_update_time = local_update_TS ; + d.*m = TS; return true; } diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 905fe0552..9320f3102 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -94,7 +94,6 @@ public: int parentRow(DirectoryStorage::EntryIndex e); bool isIndexValid(DirectoryStorage::EntryIndex e) const; - bool stampDirectory(const DirectoryStorage::EntryIndex& indx); bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const; bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); bool removeDirectory(DirectoryStorage::EntryIndex indx) ; @@ -103,8 +102,11 @@ public: bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash); bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time); bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector &subdirs_hash, const std::vector &subfiles_array); - bool getDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); - bool setDirUpdateTS(const DirectoryStorage::EntryIndex& index,time_t& recurs_max_modf_TS,time_t& local_update_TS); + + // TS get/set functions. Take one of the class members as argument. + + bool getTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) const; + bool setTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) ; // Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index faa3ceaee..d5292498a 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -115,16 +115,14 @@ uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx) return DIR_TYPE_UNKNOWN; } } -bool DirectoryStorage::getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) -{ - RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->getDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; -} -bool DirectoryStorage::setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) -{ - RS_STACK_MUTEX(mDirStorageMtx) ; - return mFileHierarchy->setDirUpdateTS(index,recurs_max_modf_TS,local_update_TS) ; -} + +bool DirectoryStorage::getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); } +bool DirectoryStorage::getDirectoryRecursModTime(EntryIndex index,time_t& rec_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); } +bool DirectoryStorage::getDirectoryLocalModTime (EntryIndex index,time_t& loc_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); } + +bool DirectoryStorage::setDirectoryUpdateTime (EntryIndex index,time_t update_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); } +bool DirectoryStorage::setDirectoryRecursModTime(EntryIndex index,time_t rec_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); } +bool DirectoryStorage::setDirectoryLocalModTime (EntryIndex index,time_t loc_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); } bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs) { @@ -364,26 +362,33 @@ static bool sameLists(const std::list& l1,const std::list::iterator it = mLocalDirs.find(info.filename) ; - - if(it == mLocalDirs.end()) { - std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; - return ; + RS_STACK_MUTEX(mDirStorageMtx) ; + + std::map::iterator it = mLocalDirs.find(info.filename) ; + + if(it == mLocalDirs.end()) + { + std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl; + return ; + } + + // we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just + // want to make sure that if they are different, flags get updated. + + if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname) + { + it->second = info; + + std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; + changed = true ; + } } - // we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just - // want to make sure that if they are different, flags get updated. - - if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname) - { - it->second = info; - mFileHierarchy->stampDirectory(0) ; - - std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; - } + if(changed) + setDirectoryLocalModTime(0,time(NULL)) ; } bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index dd1202726..abd91b6dd 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -53,8 +53,13 @@ class DirectoryStorage virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; - bool getDirUpdateTS(EntryIndex index,time_t& recurs_max_modf_TS,time_t& local_update_TS) ; - bool setDirUpdateTS(EntryIndex index,time_t recurs_max_modf_TS,time_t local_update_TS) ; + bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; + bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; + bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; + + bool setDirectoryRecursModTime(EntryIndex index,time_t recurs_max_modf_TS) ; + bool setDirectoryLocalModTime (EntryIndex index,time_t modtime_TS) ; + bool setDirectoryUpdateTime (EntryIndex index,time_t update_TS) ; uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 06b9af252..1a9bad468 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -134,6 +134,10 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; } } + // update folder modificatoin time, which is the only way to detect e.g. removed or renamed files. + + mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ; + // update file and dir lists for current directory. mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 5a7621b17..88d7ea8bd 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -584,17 +584,9 @@ void p3FileDatabase::requestDirUpdate(void *ref) if(fi == 0) return ; // not updating current directory (should we?) - time_t recurs_max_modf_TS_remote_time,local_update_TS; - P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; - if(!mRemoteDirectories[fi-1]->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) - { - P3FILELISTS_ERROR() << " (EE) Cannot get max known recurs modf time!" << std::endl; - return ; - } - - if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e,recurs_max_modf_TS_remote_time)) + if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) P3FILELISTS_DEBUG() << " Succeed." << std::endl; } @@ -1060,8 +1052,8 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) } else { - time_t local_recurs_max_time,local_update_time; - mLocalSharedDirs->getDirUpdateTS(entry_index,local_recurs_max_time,local_update_time); + time_t local_recurs_max_time ; + mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ; if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust. { @@ -1147,7 +1139,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) { std::cerr << " Directory is up to date. Setting local TS." << std::endl; - mRemoteDirectories[fi]->setDirUpdateTS(entry_index,item->last_known_recurs_modf_TS,time(NULL)); + mRemoteDirectories[fi]->setDirectoryUpdateTime(entry_index,time(NULL)) ; } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { @@ -1178,9 +1170,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r P3FILELISTS_DEBUG() << "currently at entry index " << e << std::endl; - time_t recurs_max_modf_TS_remote_time,local_update_TS; + time_t local_update_TS; - if(!rds->getDirUpdateTS(e,recurs_max_modf_TS_remote_time,local_update_TS)) + if(!rds->getDirectoryUpdateTime(e,local_update_TS)) { P3FILELISTS_ERROR() << " (EE) lockec_recursSweepRemoteDirectory(): cannot get update TS for directory with index " << e << ". This is a consistency bug." << std::endl; return; @@ -1189,7 +1181,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // compare TS if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. - if(generateAndSendSyncRequest(rds,e,recurs_max_modf_TS_remote_time)) + if(generateAndSendSyncRequest(rds,e)) P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; for(DirectoryStorage::DirIterator it(rds,e);it;++it) @@ -1220,11 +1212,18 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId return r ^ random_bias; } -bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e,time_t max_known_recurs_modf_time) +bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e) { RsFileHash entry_hash ; time_t now = time(NULL) ; + time_t max_known_recurs_modf_time ; + + if(!rds->getDirectoryRecursModTime(e,max_known_recurs_modf_time)) + { + P3FILELISTS_ERROR() << " (EE) cannot find recurs mod time for entry index " << e << ". This is very unexpected." << std::endl; + return false; + } if(!rds->getDirHashFromIndex(e,entry_hash) ) { P3FILELISTS_ERROR() << " (EE) cannot find hash for entry index " << e << ". This is very unexpected." << std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index f72628877..5198a0a4e 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -179,7 +179,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub * \param e Entry index to update * \return true if the request is correctly sent. */ - bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e,time_t max_known_recurs_modf_time); + bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e); // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 105c7776c..9e8ba66d3 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -13,6 +13,23 @@ namespace librs { namespace util { FolderIterator::FolderIterator(const std::string& folderName) : mFolderName(folderName) { + // Grab the last modification time for the directory + + struct stat64 buf ; + +#ifdef WINDOWS_SYS + std::wstring wfullname; + librs::util::ConvertUtf8ToUtf16(folderName, wfullname); + if ( 0 == _wstati64(wfullname.c_str(), &buf)) +#else + if ( 0 == stat64(folderName.c_str(), &buf)) +#endif + { + mFolderModTime = buf.st_mtime ; + } + + // Now open directory content and read the first entry + #ifdef WINDOWS_SYS std::wstring utf16Name; if(! ConvertUtf8ToUtf16(folderName, utf16Name)) { @@ -123,6 +140,8 @@ bool FolderIterator::d_name(std::string& dest) return true; } +time_t FolderIterator::dir_modtime() const { return mFolderModTime ; } + const std::string& FolderIterator::file_fullpath() { return mFullPath ; } const std::string& FolderIterator::file_name() { return mFileName ; } uint64_t FolderIterator::file_size() { return mFileSize ; } diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index dcf8ed3f3..1dcc07bc3 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -30,10 +30,16 @@ public: TYPE_DIR = 0x02 }; + // info about current parent directory + time_t dir_modtime() const ; + + // info about directory content + bool isValid() const { return validity; } bool readdir(); void next(); +#warning this one should go, as it reports the same information than file_name() bool d_name(std::string& dest); bool closedir(); @@ -58,6 +64,7 @@ private: bool mStatInfoOk ; time_t mFileModTime ; + time_t mFolderModTime ; uint64_t mFileSize ; uint8_t mType ; std::string mFileName ; From 2343c910554f6780cf7c166e6fbfc31ebb875b1e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 15 Sep 2016 21:40:53 +0200 Subject: [PATCH 083/174] added display of hash progress --- libretroshare/src/file_sharing/hash_cache.cc | 37 ++++++++++++++++++-- libretroshare/src/file_sharing/hash_cache.h | 5 +++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index f92964ac4..47edf9b5a 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -41,12 +41,33 @@ HashStorage::HashStorage(const std::string& save_file_name) mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; mRunning = false ; mLastSaveTime = 0 ; + mTotalSizeToHash = 0; + mTotalFilesToHash = 0; { RS_STACK_MUTEX(mHashMtx) ; locked_load() ; } } +static std::string friendlyUnit(uint64_t val) +{ + const std::string units[5] = {"B","KB","MB","GB","TB"}; + char buf[50] ; + + double fact = 1.0 ; + + for(unsigned int i=0; i<5; ++i) + if(double(val)/fact < 1024.0) + { + sprintf(buf,"%2.2f",double(val)/fact) ; + return std::string(buf) + " " + units[i]; + } + else + fact *= 1024.0f ; + + sprintf(buf,"%2.2f",double(val)/fact*1024.0f) ; + return std::string(buf) + " TB"; +} void HashStorage::data_tick() { @@ -93,6 +114,8 @@ void HashStorage::data_tick() std::cerr << "Stopping hashing thread." << std::endl; shutdown(); mRunning = false ; + mTotalSizeToHash = 0; + mTotalFilesToHash = 0; std::cerr << "done." << std::endl; } @@ -110,6 +133,7 @@ void HashStorage::data_tick() { RS_STACK_MUTEX(mHashMtx) ; + job = mFilesToHash.begin()->second ; mFilesToHash.erase(mFilesToHash.begin()) ; } @@ -117,9 +141,9 @@ void HashStorage::data_tick() std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); std::string tmpout; - //rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", cnt+1, n_files, friendlyUnit(size).c_str(), int(size/double(total_size)*100.0), fe.name.c_str()) ; + rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, job.full_path) ; + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; @@ -139,6 +163,8 @@ void HashStorage::data_tick() info.hash = hash; mChanged = true ; + ++mHashCounter ; + mTotalHashedSize += size ; } } // call the client @@ -181,16 +207,23 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t FileHashJob job ; job.client = c ; + job.size = size ; job.client_param = client_param ; job.full_path = full_path ; job.ts = mod_time ; mFilesToHash[full_path] = job; + mTotalSizeToHash += size ; + ++mTotalFilesToHash; + if(!mRunning) { mRunning = true ; std::cerr << "Starting hashing thread." << std::endl; + mHashCounter = 0; + mTotalHashedSize = 0; + start() ; } diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 467c18d6a..9ab4199a6 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -97,6 +97,7 @@ private: struct FileHashJob { std::string full_path; + uint64_t size ; HashStorageClient *client; uint32_t client_param ; time_t ts; @@ -110,7 +111,11 @@ private: RsMutex mHashMtx ; bool mRunning; + uint32_t mHashCounter; uint32_t mInactivitySleepTime ; + uint64_t mTotalSizeToHash ; + uint64_t mTotalHashedSize ; + uint64_t mTotalFilesToHash ; time_t mLastSaveTime ; }; From 3f9acb5ff81830b1ebae484a7d1b5af1402fb3db Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 15 Sep 2016 23:51:46 +0200 Subject: [PATCH 084/174] added item splitting and checksum checking in p3filelists --- .../src/file_sharing/file_sharing_defaults.h | 2 + libretroshare/src/file_sharing/p3filelists.cc | 120 +++++++++++++++++- libretroshare/src/file_sharing/p3filelists.h | 6 + .../src/file_sharing/rsfilelistitems.cc | 4 + .../src/file_sharing/rsfilelistitems.h | 3 + 5 files changed, 130 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 43cf5406f..486c684ee 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -38,3 +38,5 @@ static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-co static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this + +static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 88d7ea8bd..2d57fb76b 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -606,12 +606,13 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe return true ; } - else for(uint32_t i=0;iroot(),i+1,result); - + convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; } + else + return false; uint32_t fi; DirectoryStorage::EntryIndex e ; @@ -1077,11 +1078,120 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) // sends the response. - sendItem(ritem); + splitAndSendItem(ritem) ; } -void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *item) +void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) { + ritem->checksum = RsDirUtil::sha1sum((uint8_t*)ritem->directory_content_data.bin_data,ritem->directory_content_data.bin_len); + + while(ritem->directory_content_data.bin_len > MAX_DIR_SYNC_RESPONSE_DATA_SIZE) + { + P3FILELISTS_DEBUG() << "Chopping off partial chunk of size " << MAX_DIR_SYNC_RESPONSE_DATA_SIZE << " from item data of size " << ritem->directory_content_data.bin_len << std::endl; + + RsFileListsSyncResponseItem *subitem = new RsFileListsSyncResponseItem() ; + + subitem->entry_hash = ritem->entry_hash; + subitem->flags = ritem->flags | RsFileListsItem::FLAGS_SYNC_PARTIAL; + subitem->last_known_recurs_modf_TS = ritem->last_known_recurs_modf_TS; + subitem->request_id = ritem->request_id; + subitem->checksum = ritem->checksum ; + + // copy a subpart of the data + subitem->directory_content_data.tlvtype = ritem->directory_content_data.tlvtype ; + subitem->directory_content_data.setBinData(ritem->directory_content_data.bin_data, MAX_DIR_SYNC_RESPONSE_DATA_SIZE) ; + + // update ritem to chop off the data that was sent. + memmove(ritem->directory_content_data.bin_data, &((unsigned char*)ritem->directory_content_data.bin_data)[MAX_DIR_SYNC_RESPONSE_DATA_SIZE], ritem->directory_content_data.bin_len - MAX_DIR_SYNC_RESPONSE_DATA_SIZE) ; + ritem->directory_content_data.bin_len -= MAX_DIR_SYNC_RESPONSE_DATA_SIZE ; + + // send + subitem->PeerId(ritem->PeerId()) ; + + sendItem(subitem) ; + + // fix up last chunk + if(ritem->directory_content_data.bin_len <= MAX_DIR_SYNC_RESPONSE_DATA_SIZE) + ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL_END ; + } + + sendItem(ritem) ; +} + +// This function should not take memory ownership of ritem, so it makes copies. + +RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncResponseItem *ritem) +{ +#warning make sure about how robust that is to disconnections, etc. + if(!(ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL )) + return ritem ; + + // item is a partial item. Look first for a starting entry + + P3FILELISTS_DEBUG() << "Item from peer " << ritem->PeerId() << " is partial. Size = " << ritem->directory_content_data.bin_len << std::endl; + + RS_STACK_MUTEX(mFLSMtx) ; + + bool is_ending = (ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL_END); + std::map::iterator it = mPartialResponseItems.find(ritem->request_id) ; + + if(it == mPartialResponseItems.end()) + { + if(is_ending) + { + P3FILELISTS_ERROR() << "Impossible situation: partial item ended right away. Dropping..." << std::endl; + return NULL; + } + P3FILELISTS_DEBUG() << "Creating new item buffer" << std::endl; + + mPartialResponseItems[ritem->request_id] = new RsFileListsSyncResponseItem(*ritem) ; + return NULL ; + } + else if(it->second->checksum != ritem->checksum) + { + P3FILELISTS_ERROR() << "Impossible situation: partial items with different checksums. Dropping..." << std::endl; + mPartialResponseItems.erase(it); + return NULL; + } + + // collapse the item at the end of the existing partial item. Dont delete ritem as it will be by the caller. + + it->second->directory_content_data.bin_data = realloc(it->second->directory_content_data.bin_data,it->second->directory_content_data.bin_len + ritem->directory_content_data.bin_len) ; + memcpy(it->second->directory_content_data.bin_data,ritem->directory_content_data.bin_data,ritem->directory_content_data.bin_len); + + // if finished, return the item + + if(is_ending) + { + P3FILELISTS_DEBUG() << "Item is complete. Returning it" << std::endl; + + RsFileListsSyncResponseItem *ret = it->second ; + mPartialResponseItems.erase(it) ; + + ret->flags &= ~RsFileListsItem::FLAGS_SYNC_PARTIAL_END ; + ret->flags &= ~RsFileListsItem::FLAGS_SYNC_PARTIAL ; + + return ret ; + } + else + return NULL ; +} + +void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) +{ + RsFileListsSyncResponseItem *item = recvAndRebuildItem(sitem) ; + + if(!item) + return ; + + // check the hash. If anything goes wrong (in the chunking for instance) the hash will not match + + if(RsDirUtil::sha1sum((uint8_t*)item->directory_content_data.bin_data,item->directory_content_data.bin_len) != item->checksum) + { + P3FILELISTS_ERROR() << "Checksum error in response item " << std::hex << item->request_id << std::dec << " . This is unexpected, and might be due to connection problems." << std::endl; + return ; + } + P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl; EntryIndex entry_index = DirectoryStorage::NO_INDEX; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 5198a0a4e..52dd00967 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -173,6 +173,11 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id, const RsFileHash &hash) ; + // utility functions to send items with some maximum size. + + void splitAndSendItem(RsFileListsSyncResponseItem *ritem); + RsFileListsSyncResponseItem *recvAndRebuildItem(RsFileListsSyncResponseItem *ritem); + /*! * \brief generateAndSendSyncRequest * \param rds Remote directory storage for the request @@ -221,6 +226,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub time_t mLastRemoteDirSweepTS ; // TS for friend list update std::map mPendingSyncRequests ; // pending requests, waiting for an answer + std::map mPartialResponseItems; void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds, DirectoryStorage::EntryIndex e, int depth); diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index b3fd1db59..bd07bb5f6 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -152,6 +152,7 @@ bool RsFileListsSyncResponseItem::serialise(void *data, uint32_t& size) const /* RsFileListsSyncMsgItem */ ok &= entry_hash.serialise(data, size, offset); + ok &= checksum.serialise(data, size, offset); ok &= setRawUInt32(data, size, &offset, flags ); ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS); ok &= setRawUInt64(data, size, &offset, request_id); @@ -231,6 +232,7 @@ RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncRespons uint64_t request_id; // use to determine if changes that have occured since last hash ok &= item->entry_hash.deserialise(data, *size, offset); + ok &= item->checksum.deserialise(data, *size, offset); ok &= getRawUInt32(data, *size, &offset, &item->flags); ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS); ok &= getRawUInt64(data, *size, &offset, &item->request_id); @@ -312,6 +314,7 @@ uint32_t RsFileListsSyncResponseItem::serial_size()const uint32_t s = 8; //header size s += RsFileHash::serial_size(); // entry hash + s += RsFileHash::serial_size(); // checksum s += 4; // flags s += 4; // last_known_recurs_modf_TS s += 8; // request_id @@ -348,6 +351,7 @@ std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t ind uint16_t int_Indent = indent + 2; printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl; + printIndent(out , int_Indent); out << "Checksum : " << checksum << std::endl; printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl; printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl; printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.h b/libretroshare/src/file_sharing/rsfilelistitems.h index deaa06e60..8c1d466e1 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.h +++ b/libretroshare/src/file_sharing/rsfilelistitems.h @@ -66,6 +66,8 @@ public: static const uint32_t FLAGS_SYNC_DIR_CONTENT = 0x0004 ; static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0008 ; static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0010 ; + static const uint32_t FLAGS_SYNC_PARTIAL = 0x0020 ; + static const uint32_t FLAGS_SYNC_PARTIAL_END = 0x0040 ; }; /*! @@ -102,6 +104,7 @@ public: virtual uint32_t serial_size() const ; RsFileHash entry_hash ; // hash of the directory to sync + RsFileHash checksum ; // checksum of the bindary data, for checking uint32_t flags; // is it a partial/final item (used for large items only) uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. uint64_t request_id; // use to determine if changes that have occured since last hash From 04c908e0466579d1912b599c671da6b7d669379f Mon Sep 17 00:00:00 2001 From: MrAlice Date: Fri, 16 Sep 2016 09:35:03 +0200 Subject: [PATCH 085/174] fixed two bugs in dirlist chunking system causing checksum errors --- libretroshare/src/file_sharing/p3filelists.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 2d57fb76b..3b743e3d6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1089,10 +1089,12 @@ void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) { P3FILELISTS_DEBUG() << "Chopping off partial chunk of size " << MAX_DIR_SYNC_RESPONSE_DATA_SIZE << " from item data of size " << ritem->directory_content_data.bin_len << std::endl; + ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL ; + RsFileListsSyncResponseItem *subitem = new RsFileListsSyncResponseItem() ; subitem->entry_hash = ritem->entry_hash; - subitem->flags = ritem->flags | RsFileListsItem::FLAGS_SYNC_PARTIAL; + subitem->flags = ritem->flags ; subitem->last_known_recurs_modf_TS = ritem->last_known_recurs_modf_TS; subitem->request_id = ritem->request_id; subitem->checksum = ritem->checksum ; @@ -1156,8 +1158,14 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR // collapse the item at the end of the existing partial item. Dont delete ritem as it will be by the caller. - it->second->directory_content_data.bin_data = realloc(it->second->directory_content_data.bin_data,it->second->directory_content_data.bin_len + ritem->directory_content_data.bin_len) ; - memcpy(it->second->directory_content_data.bin_data,ritem->directory_content_data.bin_data,ritem->directory_content_data.bin_len); + uint32_t old_len = it->second->directory_content_data.bin_len ; + uint32_t added = ritem->directory_content_data.bin_len; + + it->second->directory_content_data.bin_data = realloc(it->second->directory_content_data.bin_data,old_len + added) ; + memcpy(&((unsigned char*)it->second->directory_content_data.bin_data)[old_len],ritem->directory_content_data.bin_data,added); + it->second->directory_content_data.bin_len = old_len + added ; + + P3FILELISTS_DEBUG() << "Added new chunk of length " << added << ". New size is " << old_len + added << std::endl; // if finished, return the item From de104d3e344fe397ded0a4be3b3f089111a66c3e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 18:34:39 +0200 Subject: [PATCH 086/174] fixed up GUI parameters in Directories. Removed HashCache duration handles since this is now automatic --- .../src/file_sharing/directory_storage.cc | 20 ++++++- .../src/file_sharing/directory_storage.h | 20 +++++-- .../src/file_sharing/directory_updater.cc | 39 ++++++++++++- .../src/file_sharing/directory_updater.h | 10 +++- .../src/file_sharing/file_sharing_defaults.h | 16 ++--- libretroshare/src/file_sharing/hash_cache.cc | 58 +++++++++---------- libretroshare/src/file_sharing/hash_cache.h | 9 ++- libretroshare/src/file_sharing/p3filelists.cc | 51 ++++++++-------- libretroshare/src/file_sharing/p3filelists.h | 7 +-- libretroshare/src/ft/ftserver.cc | 32 ++-------- libretroshare/src/ft/ftserver.h | 7 +-- libretroshare/src/retroshare/rsfiles.h | 12 ++-- libretroshare/src/rsserver/rsinit.cc | 4 +- .../src/gui/settings/DirectoriesPage.cpp | 51 ++-------------- .../src/gui/settings/DirectoriesPage.h | 3 - .../src/gui/settings/DirectoriesPage.ui | 50 +--------------- 16 files changed, 169 insertions(+), 220 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index d5292498a..efb88bd08 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -337,6 +337,8 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list::const_iterator it(processed_list.begin());it!=processed_list.end();++it) mLocalDirs[it->filename] = *it; + + mTSChanged = true ; } void LocalDirectoryStorage::getSharedDirectoryList(std::list& lst) { @@ -388,7 +390,10 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) } if(changed) + { setDirectoryLocalModTime(0,time(NULL)) ; + mTSChanged = true ; + } } bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath) @@ -411,13 +416,24 @@ bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std:: return true; } +void LocalDirectoryStorage::notifyTSChanged() +{ + RS_STACK_MUTEX(mDirStorageMtx) ; + mTSChanged = true ; +} void LocalDirectoryStorage::updateTimeStamps() { RS_STACK_MUTEX(mDirStorageMtx) ; - time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ; + if(mTSChanged) + { + std::cerr << "Updating recursive TS for local shared dirs..." << std::endl; - std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; + time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ; + mTSChanged = false ; + + std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; + } } std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index abd91b6dd..b76d73ebd 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -201,7 +201,17 @@ public: void updateShareFlags(const SharedDirInfo& info) ; bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ; + /*! + * \brief updateTimeStamps + * Checks recursive TS and update the if needed. + */ void updateTimeStamps(); + + /*! + * \brief notifyTSChanged + * Use this to force an update of the recursive TS, when calling updateTimeStamps(); + */ + void notifyTSChanged(); /*! * \brief getFileInfo Converts an index info a full file info structure. * \param i index in the directory structure @@ -239,11 +249,13 @@ private: std::string locked_getVirtualPath(EntryIndex indx) const ; std::string locked_getVirtualDirName(EntryIndex indx) const ; - bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); - std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; + bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list& parent_groups); + std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const; - std::map mLocalDirs ; // map is better for search. it->first=it->second.filename - std::string mFileName; + std::map mLocalDirs ; // map is better for search. it->first=it->second.filename + std::string mFileName; + + bool mTSChanged ; }; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 1a9bad468..16ca35be8 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -42,16 +42,35 @@ LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStora mLastTSUpdateTime = 0; mDelayBetweenDirectoryUpdates = DELAY_BETWEEN_DIRECTORY_UPDATES; + mIsEnabled = false ; +} + +bool LocalDirectoryUpdater::isEnabled() const +{ + return mIsEnabled ; +} +void LocalDirectoryUpdater::setEnabled(bool b) +{ + if(mIsEnabled == b) + return ; + + if(b) + start() ; + else + shutdown(); + + mIsEnabled = b ; } void LocalDirectoryUpdater::data_tick() { time_t now = time(NULL) ; - if(now > DELAY_BETWEEN_DIRECTORY_UPDATES + mLastSweepTime) + if(now > mDelayBetweenDirectoryUpdates + mLastSweepTime) { sweepSharedDirectories() ; mLastSweepTime = now; + mSharedDirectories->notifyTSChanged() ; } if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime) @@ -177,10 +196,24 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri { if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; + + mSharedDirectories->notifyTSChanged() ; +} + +bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param) +{ + return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ; +} + +void LocalDirectoryUpdater::setFileWatchPeriod(int seconds) +{ + mDelayBetweenDirectoryUpdates = seconds ; +} +uint32_t LocalDirectoryUpdater::fileWatchPeriod() const +{ + return mDelayBetweenDirectoryUpdates ; } - - diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index a4a3f4242..d0f920cb0 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -41,13 +41,18 @@ public: void forceUpdate(); bool inDirectoryCheck() const ; - void setFileWatchPeriod(uint32_t seconds) { mDelayBetweenDirectoryUpdates = seconds ; } - uint32_t fileWatchPeriod() const { return mDelayBetweenDirectoryUpdates ; } + void setFileWatchPeriod(int seconds) ; + uint32_t fileWatchPeriod() const ; + + void setEnabled(bool b) ; + bool isEnabled() const ; protected: virtual void data_tick() ; virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size); + virtual bool hash_confirm(uint32_t client_param) ; + void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx); void sweepSharedDirectories(); @@ -59,5 +64,6 @@ private: time_t mLastTSUpdateTime; uint32_t mDelayBetweenDirectoryUpdates; + bool mIsEnabled ; }; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 486c684ee..94e2981f7 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -28,15 +28,17 @@ static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes -static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 300 ; // 5 minutes +static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 sec. Buy we only update for real if something has changed. -static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time -static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files +static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time +static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files +static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch -static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. -static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. +static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. +static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. -static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. -static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this +static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs. +static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission +static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 47edf9b5a..4748e7125 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -43,6 +43,7 @@ HashStorage::HashStorage(const std::string& save_file_name) mLastSaveTime = 0 ; mTotalSizeToHash = 0; mTotalFilesToHash = 0; + mMaxStorageDurationDays = DEFAULT_HASH_STORAGE_DURATION_DAYS ; { RS_STACK_MUTEX(mHashMtx) ; @@ -138,40 +139,42 @@ void HashStorage::data_tick() mFilesToHash.erase(mFilesToHash.begin()) ; } - std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); - - std::string tmpout; - rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; - - if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) - std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; - else - std::cerr << "done."<< std::endl; - - // store the result - + if(job.client->hash_confirm(job.client_param)) { - RS_STACK_MUTEX(mHashMtx) ; - HashStorageInfo& info(mFiles[job.full_path]); + std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); - info.filename = job.full_path ; - info.size = size ; - info.modf_stamp = job.ts ; - info.time_stamp = time(NULL); - info.hash = hash; + std::string tmpout; + rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; - mChanged = true ; - ++mHashCounter ; - mTotalHashedSize += size ; + RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; + + if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) + std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; + else + std::cerr << "done."<< std::endl; + + // store the result + + { + RS_STACK_MUTEX(mHashMtx) ; + HashStorageInfo& info(mFiles[job.full_path]); + + info.filename = job.full_path ; + info.size = size ; + info.modf_stamp = job.ts ; + info.time_stamp = time(NULL); + info.hash = hash; + + mChanged = true ; + ++mHashCounter ; + mTotalHashedSize += size ; + } } } // call the client if(!hash.isNull()) job.client->hash_callback(job.client_param, job.full_path, hash, size); - } bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param) @@ -234,14 +237,11 @@ void HashStorage::clean() { RS_STACK_MUTEX(mHashMtx) ; -#ifdef HASHSTORAGE_DEBUG - std::cerr << "Cleaning HashStorage..." << std::endl ; -#endif time_t now = time(NULL) ; time_t duration = mMaxStorageDurationDays * 24 * 3600 ; // seconds #ifdef HASHSTORAGE_DEBUG - std::cerr << "cleaning hash cache." << std::endl ; + std::cerr << "Cleaning hash cache." << std::endl ; #endif for(std::map::iterator it(mFiles.begin());it!=mFiles.end();) diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 9ab4199a6..0f92f09c1 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -36,7 +36,14 @@ public: HashStorageClient() {} virtual ~HashStorageClient() {} + // the result of the hashing info is sent to this method + virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)=0; + + // this method is used to check that the client param is still valid just before hashing. This avoids hashing files + // that are still in queue while removed from shared lists. + + virtual bool hash_confirm(uint32_t client_param)=0 ; }; class HashStorage: public RsTickingThread @@ -90,7 +97,7 @@ private: // Local configuration and storage uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries - std::map mFiles ; // stored as (full_path, hash_info) + std::map mFiles ; // stored as (full_path, hash_info) std::string mFilePath ; bool mChanged ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 3b743e3d6..c67f0b322 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -312,6 +312,15 @@ cleanup = true; rskv->tlvkvs.pairs.push_back(kv); } + { + RsTlvKeyValue kv; + + kv.key = WATCH_FILE_ENABLED_SS; + kv.value = watchEnabled()?"YES":"NO" ; + + rskv->tlvkvs.pairs.push_back(kv); + } + /* Add KeyValue to saveList */ sList.push_back(rskv); @@ -354,6 +363,10 @@ bool p3FileDatabase::loadList(std::list& load) if(sscanf(kit->value.c_str(),"%d",&t) == 1) setWatchPeriod(t); } + else if(kit->key == WATCH_FILE_ENABLED_SS) + { + setWatchEnabled(kit->value == "YES") ; + } delete *it ; continue ; } @@ -788,14 +801,23 @@ uint32_t p3FileDatabase::getType(void *ref) const void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed { - NOT_IMPLEMENTED(); mLocalDirWatcher->forceUpdate(); } bool p3FileDatabase::inDirectoryCheck() { - NOT_IMPLEMENTED(); return mLocalDirWatcher->inDirectoryCheck(); } +void p3FileDatabase::setWatchEnabled(bool b) +{ + RS_STACK_MUTEX(mFLSMtx) ; + mLocalDirWatcher->setEnabled(b) ; + IndicateConfigChanged(); +} +bool p3FileDatabase::watchEnabled() +{ + RS_STACK_MUTEX(mFLSMtx) ; + return mLocalDirWatcher->isEnabled() ; +} void p3FileDatabase::setWatchPeriod(uint32_t seconds) { RS_STACK_MUTEX(mFLSMtx) ; @@ -808,31 +830,6 @@ uint32_t p3FileDatabase::watchPeriod() RS_STACK_MUTEX(mFLSMtx) ; return mLocalDirWatcher->fileWatchPeriod(); } -void p3FileDatabase::setRememberHashCacheDuration(uint32_t days) -{ - RS_STACK_MUTEX(mFLSMtx) ; - mHashCache->setRememberHashFilesDuration(days) ; -} -uint32_t p3FileDatabase::rememberHashCacheDuration() -{ - RS_STACK_MUTEX(mFLSMtx) ; - return mHashCache->rememberHashFilesDuration() ; -} -void p3FileDatabase::clearHashCache() -{ - RS_STACK_MUTEX(mFLSMtx) ; - mHashCache->clear() ; -} -bool p3FileDatabase::rememberHashCache() -{ - NOT_IMPLEMENTED(); - return false; -} -void p3FileDatabase::setRememberHashCache(bool) -{ - NOT_IMPLEMENTED(); -} - bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const { RS_STACK_MUTEX(mFLSMtx) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 52dd00967..6b7052677 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -135,11 +135,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void setWatchPeriod(uint32_t seconds); uint32_t watchPeriod() ; - void setRememberHashCacheDuration(uint32_t days) ; - uint32_t rememberHashCacheDuration() ; - void clearHashCache() ; - bool rememberHashCache() ; - void setRememberHashCache(bool) ; + void setWatchEnabled(bool b) ; + bool watchEnabled() ; // interfact for directory parsing diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 6d6547884..832530ade 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -675,35 +675,11 @@ bool ftServer::removeSharedDirectory(std::string dir) return true; } -void ftServer::setWatchPeriod(int minutes) -{ - mFileDatabase->setWatchPeriod(minutes*60) ; -} -int ftServer::watchPeriod() const -{ - return mFileDatabase->watchPeriod()/60 ; -} +bool ftServer::watchEnabled() { return mFileDatabase->watchEnabled() ; } +int ftServer::watchPeriod() const { return mFileDatabase->watchPeriod()/60 ; } -void ftServer::setRememberHashFiles(bool b) -{ - mFileDatabase->setRememberHashCache(b) ; -} -bool ftServer::rememberHashFiles() const -{ - return mFileDatabase->rememberHashCache() ; -} -void ftServer::setRememberHashFilesDuration(uint32_t days) -{ - mFileDatabase->setRememberHashCacheDuration(days) ; -} -uint32_t ftServer::rememberHashFilesDuration() const -{ - return mFileDatabase->rememberHashCacheDuration() ; -} -void ftServer::clearHashCache() -{ - mFileDatabase->clearHashCache() ; -} +void ftServer::setWatchEnabled(bool b) { mFileDatabase->setWatchEnabled(b) ; } +void ftServer::setWatchPeriod(int minutes) { mFileDatabase->setWatchPeriod(minutes*60) ; } bool ftServer::getShareDownloadDirectory() { diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index cc4004263..c5d1d23eb 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -208,13 +208,10 @@ public: virtual bool getShareDownloadDirectory(); virtual bool shareDownloadDirectory(bool share); - virtual void setRememberHashFilesDuration(uint32_t days) ; - virtual uint32_t rememberHashFilesDuration() const ; - virtual bool rememberHashFiles() const ; - virtual void setRememberHashFiles(bool) ; - virtual void clearHashCache() ; virtual void setWatchPeriod(int minutes) ; virtual int watchPeriod() const ; + virtual void setWatchEnabled(bool b) ; + virtual bool watchEnabled() ; /***************************************************************/ /*************** Data Transfer Interface ***********************/ diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 7f6de8be9..cd12396fb 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -213,13 +213,11 @@ class RsFiles virtual bool addSharedDirectory(const SharedDirInfo& dir) = 0; virtual bool updateShareFlags(const SharedDirInfo& dir) = 0; // updates the flags. The directory should already exist ! virtual bool removeSharedDirectory(std::string dir) = 0; - virtual void setRememberHashFilesDuration(uint32_t days) = 0 ; - virtual uint32_t rememberHashFilesDuration() const = 0 ; - virtual void clearHashCache() = 0 ; - virtual bool rememberHashFiles() const =0; - virtual void setRememberHashFiles(bool) =0; - virtual void setWatchPeriod(int minutes) =0; - virtual int watchPeriod() const =0; + + virtual void setWatchPeriod(int minutes) =0; + virtual void setWatchEnabled(bool b) =0; + virtual int watchPeriod() const =0; + virtual bool watchEnabled() =0; virtual bool getShareDownloadDirectory() = 0; virtual bool shareDownloadDirectory(bool share) = 0; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 6726c1656..c34ae7b94 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1239,8 +1239,8 @@ int RsServer::StartupRetroShare() //pqih = new pqipersongrpDummy(none, flags); /****** New Ft Server **** !!! */ - ftServer *ftserver = new ftServer(mPeerMgr, serviceCtrl); - ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory()); + ftServer *ftserver = new ftServer(mPeerMgr, serviceCtrl); + ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory()); ftserver->SetupFtServer() ; diff --git a/retroshare-gui/src/gui/settings/DirectoriesPage.cpp b/retroshare-gui/src/gui/settings/DirectoriesPage.cpp index 1bb387a26..b9d2ecc5e 100755 --- a/retroshare-gui/src/gui/settings/DirectoriesPage.cpp +++ b/retroshare-gui/src/gui/settings/DirectoriesPage.cpp @@ -36,18 +36,9 @@ DirectoriesPage::DirectoriesPage(QWidget * parent, Qt::WindowFlags flags) connect(ui.incomingButton, SIGNAL(clicked( bool ) ), this , SLOT( setIncomingDirectory() ) ); connect(ui.partialButton, SIGNAL(clicked( bool ) ), this , SLOT( setPartialsDirectory() ) ); connect(ui.editShareButton, SIGNAL(clicked()), this, SLOT(editDirectories())); - connect(ui.cleanHashCachePB, SIGNAL(clicked()), this, SLOT(clearHashCache())); - connect(ui.rememberHashesCB, SIGNAL(clicked(bool)), this, SLOT(clickedRememberHashes(bool))); - connect(ui.rememberHashesCB, SIGNAL(clicked(bool)), this, SLOT(toggleRememberHashes())); connect(ui.autoCheckDirectories_CB, SIGNAL(clicked(bool)), this, SLOT(toggleAutoCheckDirectories(bool))); } -void DirectoriesPage::clearHashCache() -{ - if(QMessageBox::question(this, tr("Cache cleaning confirmation"), tr("This will forget any former hash of non shared files. Do you confirm ?"), QMessageBox::Yes | QMessageBox::No, QMessageBox::No) == QMessageBox::Yes) - rsFiles->clearHashCache() ; -} - void DirectoriesPage::toggleAutoCheckDirectories(bool b) { ui.autoCheckDirectoriesDelay_SB->setEnabled(b); @@ -58,28 +49,9 @@ void DirectoriesPage::editDirectories() ShareManager::showYourself() ; } -void DirectoriesPage::clickedRememberHashes(bool b) -{ - if (!b) { - if (QMessageBox::question(this,tr("Cache cleaning confirmation"), tr("This will forget any former hash of non shared files. Do you confirm ?"), QMessageBox::Yes | QMessageBox::No, QMessageBox::No) == QMessageBox::No) { - ui.rememberHashesCB->setChecked(true); - } - } -} - -void DirectoriesPage::toggleRememberHashes() -{ - bool b = ui.rememberHashesCB->isChecked(); - ui.rememberHashesSB->setEnabled(b); - ui.cleanHashCachePB->setEnabled(b); -} - /** Saves the changes on this page */ bool DirectoriesPage::save(QString &/*errmsg*/) { - rsFiles->setRememberHashFilesDuration(ui.rememberHashesSB->value()); - rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value()); - std::string dir = ui.incomingDir->text().toUtf8().constData(); if (!dir.empty()) { @@ -92,18 +64,8 @@ bool DirectoriesPage::save(QString &/*errmsg*/) rsFiles->setPartialsDirectory(dir); } - if (ui.rememberHashesCB->isChecked()) { - rsFiles->setRememberHashFiles(true); - } else { - rsFiles->setRememberHashFiles(false); - rsFiles->clearHashCache() ; - } - - if (ui.autoCheckDirectories_CB->isChecked()) { - rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value()); - } else { - rsFiles->setWatchPeriod(-ui.autoCheckDirectoriesDelay_SB->value()); - } + rsFiles->setWatchEnabled(ui.autoCheckDirectories_CB->isChecked()) ; + rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value()); rsFiles->shareDownloadDirectory(ui.shareDownloadDirectoryCB->isChecked()); @@ -115,14 +77,9 @@ void DirectoriesPage::load() { ui.shareDownloadDirectoryCB->setChecked(rsFiles->getShareDownloadDirectory()); - ui.rememberHashesSB->setValue(rsFiles->rememberHashFilesDuration()); - ui.rememberHashesCB->setChecked(rsFiles->rememberHashFiles()); - toggleRememberHashes(); - int u = rsFiles->watchPeriod() ; - ui.autoCheckDirectoriesDelay_SB->setValue(abs(u)) ; - ui.autoCheckDirectories_CB->setChecked(u>0) ; - ui.autoCheckDirectoriesDelay_SB->setEnabled(u>0) ; + ui.autoCheckDirectoriesDelay_SB->setValue(u) ; + ui.autoCheckDirectories_CB->setChecked(rsFiles->watchEnabled()) ; ; ui.incomingDir->setText(QString::fromUtf8(rsFiles->getDownloadDirectory().c_str())); ui.partialsDir->setText(QString::fromUtf8(rsFiles->getPartialsDirectory().c_str())); diff --git a/retroshare-gui/src/gui/settings/DirectoriesPage.h b/retroshare-gui/src/gui/settings/DirectoriesPage.h index 97eedad6b..ffeb1bf34 100755 --- a/retroshare-gui/src/gui/settings/DirectoriesPage.h +++ b/retroshare-gui/src/gui/settings/DirectoriesPage.h @@ -45,9 +45,6 @@ private slots: void editDirectories() ; void setIncomingDirectory(); void setPartialsDirectory(); - void clearHashCache(); - void clickedRememberHashes(bool); - void toggleRememberHashes(); void toggleAutoCheckDirectories(bool); private: diff --git a/retroshare-gui/src/gui/settings/DirectoriesPage.ui b/retroshare-gui/src/gui/settings/DirectoriesPage.ui index b9f13e9b6..9413ac2f9 100755 --- a/retroshare-gui/src/gui/settings/DirectoriesPage.ui +++ b/retroshare-gui/src/gui/settings/DirectoriesPage.ui @@ -6,7 +6,7 @@ 0 0 - 485 + 895 549 @@ -144,50 +144,7 @@ - - - - - Remember file hashes even if not shared. -This might be useful if you're sharing an -external HD, to avoid re-hashing files when -you plug it in. - - - Remember hashed files for - - - true - - - - - - - days - - - 1 - - - 365 - - - 10 - - - - - - - Forget any hashed file that is not anymore shared. - - - Clean Hash Cache - - - - + @@ -240,9 +197,6 @@ you plug it in. shareDownloadDirectoryCB editShareButton - rememberHashesCB - rememberHashesSB - cleanHashCachePB autoCheckDirectories_CB autoCheckDirectoriesDelay_SB incomingDir From e4568a896f31611c5d4fabaa3a151f1ed7161951 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 21:16:25 +0200 Subject: [PATCH 087/174] removed some unused code --- libretroshare/src/file_sharing/hash_cache.cc | 5 +++-- libretroshare/src/file_sharing/p3filelists.cc | 20 +++++-------------- libretroshare/src/file_sharing/p3filelists.h | 14 +++++-------- 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 4748e7125..becbf48f5 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -314,7 +314,7 @@ void HashStorage::locked_save() return ; } - std::cerr << mFiles.size() << " Entries saved." << std::endl; + std::cerr << mFiles.size() << " entries saved in hash cache." << std::endl; free(data) ; } @@ -356,9 +356,10 @@ bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_si // now write the whole string into a single section in the file if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ; - +#ifdef HASHSTORAGE_DEBUG std::cerr << "Writing hash storage section " << RsUtil::BinToHex(section_data,section_offset) << std::endl; std::cerr << "Info.filename = " << info.filename << std::endl; +#endif free(section_data) ; return true; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c67f0b322..489164c8f 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -773,11 +773,11 @@ int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& pat NOT_IMPLEMENTED(); return 0; } -int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const -{ - NOT_IMPLEMENTED(); - return 0; -} +//int p3FileDatabase::RequestDirDetails(const std::string& path, DirDetails &details) const +//{ +// NOT_IMPLEMENTED(); +// return 0; +//} uint32_t p3FileDatabase::getType(void *ref) const { RS_STACK_MUTEX(mFLSMtx) ; @@ -830,16 +830,6 @@ uint32_t p3FileDatabase::watchPeriod() RS_STACK_MUTEX(mFLSMtx) ; return mLocalDirWatcher->fileWatchPeriod(); } -bool p3FileDatabase::findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const -{ - RS_STACK_MUTEX(mFLSMtx) ; - - std::list firesults; - mLocalSharedDirs->searchHash(hash,firesults) ; - - NOT_IMPLEMENTED(); - return false; -} int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) { diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 6b7052677..ec4e583ba 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -98,15 +98,10 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub // virtual int tick() ; - // access to own/remote shared files - // - virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; - - virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; - virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - // ftSearch virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const; + virtual int SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) ; + virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; // Interface for browsing dir hierarchy // @@ -114,8 +109,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void stopThreads() ; void startThreads() ; - int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; - int RequestDirDetails(const std::string& path, DirDetails &details) const ; bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const; // void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally. @@ -124,6 +117,9 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; uint32_t getType(void *) const ; + // proxy method used by the web UI. + int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; + // set/update shared directories void setSharedDirectories(const std::list& dirs); From ff4c43819ea4b7f15deadc5cfd2b57cb957de52e Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 21:41:18 +0200 Subject: [PATCH 088/174] fixed reloading of ShareManager due to postModDirectories and made it reload after addign new share --- libretroshare/src/file_sharing/p3filelists.cc | 1 - retroshare-gui/src/gui/MainWindow.cpp | 5 ++++- retroshare-gui/src/gui/ShareManager.cpp | 8 ++++++-- retroshare-gui/src/gui/common/GroupFlagsWidget.cpp | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 489164c8f..9608ef8a6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1111,7 +1111,6 @@ void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncResponseItem *ritem) { -#warning make sure about how robust that is to disconnections, etc. if(!(ritem->flags & RsFileListsItem::FLAGS_SYNC_PARTIAL )) return ritem ; diff --git a/retroshare-gui/src/gui/MainWindow.cpp b/retroshare-gui/src/gui/MainWindow.cpp index 4f8cfb37e..b6f8913df 100644 --- a/retroshare-gui/src/gui/MainWindow.cpp +++ b/retroshare-gui/src/gui/MainWindow.cpp @@ -756,7 +756,10 @@ void MainWindow::updateFriends() void MainWindow::postModDirectories(bool update_local) { RSettingsWin::postModDirectories(update_local); - ShareManager::postModDirectories(update_local); + + // Why would we need that?? The effect is to reset the flags while we're changing them, so it's really not + // a good idea. + //ShareManager::postModDirectories(update_local); QCoreApplication::flush(); } diff --git a/retroshare-gui/src/gui/ShareManager.cpp b/retroshare-gui/src/gui/ShareManager.cpp index 3e9477fdd..61aadefb4 100644 --- a/retroshare-gui/src/gui/ShareManager.cpp +++ b/retroshare-gui/src/gui/ShareManager.cpp @@ -148,7 +148,7 @@ void ShareManager::load() GroupFlagsWidget *widget = new GroupFlagsWidget(NULL,(*it).shareflags); - listWidget->setRowHeight(row, 32); + listWidget->setRowHeight(row, 32 * QFontMetricsF(font()).height()/14.0); listWidget->setCellWidget(row, COLUMN_SHARE_FLAGS, widget); listWidget->setItem(row, COLUMN_GROUPS, new QTableWidgetItem()) ; @@ -157,7 +157,7 @@ void ShareManager::load() //connect(widget,SIGNAL(flagsChanged(FileStorageFlags)),this,SLOT(updateFlags())) ; } - listWidget->setColumnWidth(COLUMN_SHARE_FLAGS,132) ; + listWidget->setColumnWidth(COLUMN_SHARE_FLAGS,132 * QFontMetricsF(font()).height()/14.0) ; //ui.incomingDir->setText(QString::fromStdString(rsFiles->getDownloadDirectory())); @@ -282,6 +282,7 @@ void ShareManager::editShareDirectory() ShareDialog sharedlg (it->filename, this); sharedlg.setWindowTitle(tr("Edit Shared Folder")); sharedlg.exec(); + load(); break; } } @@ -318,6 +319,7 @@ void ShareManager::showShareDialog() { ShareDialog sharedlg ("", this); sharedlg.exec(); + load(); } void ShareManager::shareddirListCurrentCellChanged(int currentRow, int currentColumn, int previousRow, int previousColumn) @@ -387,4 +389,6 @@ void ShareManager::dropEvent(QDropEvent *event) event->setDropAction(Qt::CopyAction); event->accept(); + + load(); } diff --git a/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp b/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp index be72f7eb1..a86703fc7 100644 --- a/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupFlagsWidget.cpp @@ -35,8 +35,8 @@ GroupFlagsWidget::GroupFlagsWidget(QWidget *parent,FileStorageFlags flags) { _layout = new QHBoxLayout(this) ; - setMinimumSize(128,32) ; - setMaximumSize(128,32) ; + setMinimumSize(128 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0) ; + setMaximumSize(128 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0) ; setSizePolicy(QSizePolicy::Fixed,QSizePolicy::Fixed); _icons[INDEX_GROUP_BROWSABLE] = new QIcon(FLAGS_GROUP_BROWSABLE_ICON) ; @@ -58,7 +58,7 @@ GroupFlagsWidget::GroupFlagsWidget(QWidget *parent,FileStorageFlags flags) _buttons[i] = new QPushButton(this) ; _buttons[i]->setCheckable(true) ; _buttons[i]->setChecked(flags & _flags[i]) ; - _buttons[i]->setIconSize(QSize(32,32)); + _buttons[i]->setIconSize(QSize(32 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0)); update_button_state(_buttons[i]->isChecked(),i) ; _layout->addWidget(_buttons[i]) ; } From 9cf950472cbbb5452f9eca63ad7e0f37725a9535 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 22:05:27 +0200 Subject: [PATCH 089/174] removed most of debug info in file list sharing code --- .../src/file_sharing/directory_storage.cc | 20 ++++-- .../src/file_sharing/directory_updater.cc | 21 ++++-- libretroshare/src/file_sharing/hash_cache.cc | 4 +- libretroshare/src/file_sharing/p3filelists.cc | 69 +++++++++++++++++-- libretroshare/src/file_sharing/p3filelists.h | 3 - 5 files changed, 99 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index efb88bd08..45cb3ccca 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -32,6 +32,8 @@ #include "dir_hierarchy.h" #include "filelist_io.h" +//#define DEBUG_REMOTE_DIRECTORY_STORAGE 1 + /******************************************************************************************************************/ /* Iterators */ /******************************************************************************************************************/ @@ -384,7 +386,9 @@ void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info) { it->second = info; +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl; +#endif changed = true ; } } @@ -427,12 +431,16 @@ void LocalDirectoryStorage::updateTimeStamps() if(mTSChanged) { +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "Updating recursive TS for local shared dirs..." << std::endl; +#endif time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ; mTSChanged = false ; +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; +#endif } } @@ -444,7 +452,7 @@ std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const if (cit == mLocalDirs.end()) { - std::cerr << "FileIndexMonitor::locked_findRealRoot() Invalid RootDir: " << virtual_rootdir << std::endl; + std::cerr << "(EE) locked_findRealRootFromVirtualFilename() Invalid RootDir: " << virtual_rootdir << std::endl; return std::string(); } return cit->second.filename; @@ -681,7 +689,9 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary free(file_section_data) ; } +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl; +#endif bindata.bin_data = section_data ; bindata.bin_len = section_offset ; @@ -698,10 +708,10 @@ RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::st { load(fname) ; - std::cerr << "***************************************" << std::endl; - std::cerr << "Loaded following directory for peer " << pid << std::endl; + std::cerr << "Loaded remote directory for peer " << pid << std::endl; +#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE mFileHierarchy->print(); - std::cerr << "***************************************" << std::endl; +#endif } void RemoteDirectoryStorage::checkSave() @@ -721,7 +731,9 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co uint32_t section_size = bindata.bin_len ; uint32_t section_offset=0 ; +#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl; +#endif std::string dir_name ; uint32_t most_recent_time ,dir_modtime ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 16ca35be8..c2e20aeca 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -29,7 +29,7 @@ #include "directory_updater.h" #include "file_sharing_defaults.h" -#define DEBUG_LOCAL_DIR_UPDATER 1 +//#define DEBUG_LOCAL_DIR_UPDATER 1 //=============================================================================================================// // Local Directory Updater // @@ -89,8 +89,9 @@ void LocalDirectoryUpdater::forceUpdate() void LocalDirectoryUpdater::sweepSharedDirectories() { RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - std::cerr << "LocalDirectoryUpdater::sweep()" << std::endl; +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl; +#endif // recursive update algorithm works that way: // - the external loop starts on the shared directory list and goes through sub-directories @@ -114,7 +115,9 @@ void LocalDirectoryUpdater::sweepSharedDirectories() for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it) { - std::cerr << " recursing into " << stored_dir_it.name() << std::endl; +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl; +#endif recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it) ; // here we need to use the list that was stored, instead of the shared dir list, because the two // are not necessarily in the same order. @@ -124,7 +127,9 @@ void LocalDirectoryUpdater::sweepSharedDirectories() void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx) { - std::cerr << " parsing directory " << cumulated_path << ", index=" << indx << std::endl; +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl; +#endif // make sure list of subdirs is the same // make sure list of subfiles is the same @@ -143,11 +148,15 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p { case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; subfiles[dirIt.file_name()].size = dirIt.file_size(); +#ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; +#endif break; case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime(); +#ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; +#endif break; default: std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; @@ -182,7 +191,9 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p for(std::map::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) { +#ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " recursing into " << stored_dir_it.name() << std::endl; +#endif recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ; } } diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index becbf48f5..3f96abec1 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -30,7 +30,7 @@ #include "filelist_io.h" #include "file_sharing_defaults.h" -#define HASHSTORAGE_DEBUG 1 +//#define HASHSTORAGE_DEBUG 1 static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000; static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000; @@ -100,7 +100,9 @@ void HashStorage::data_tick() // sleep off mutex! if(empty) { +#ifdef HASHSTORAGE_DEBUG std::cerr << "nothing to hash. Sleeping for " << st << " us" << std::endl; +#endif usleep(st); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 9608ef8a6..0fd6bc5db 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -39,6 +39,8 @@ #define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : " #define P3FILELISTS_ERROR() std::cerr << "***ERROR***" << " : FILE_LISTS : " << __FUNCTION__ << " : " +//#define DEBUG_P3FILELISTS 1 + static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ; static const uint32_t P3FILELISTS_UPDATE_FLAG_LOCAL_DIRS_CHANGED = 0x0002 ; @@ -179,9 +181,9 @@ int p3FileDatabase::tick() { RS_STACK_MUTEX(mFLSMtx) ; -//#ifdef DEBUG_FILE_HIERARCHY +#ifdef DEBUG_FILE_HIERARCHY mLocalSharedDirs->print(); -//#endif +#endif last_print_time = now ; //#warning this should be removed, but it's necessary atm for updating the GUI @@ -200,7 +202,7 @@ int p3FileDatabase::tick() mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; } -#warning we need to make sure that one req per directory will not cause to keep re-asking the top level dirs. + if(mLastRemoteDirSweepTS + 5 < now) { RS_STACK_MUTEX(mFLSMtx) ; @@ -235,20 +237,29 @@ int p3FileDatabase::tick() void p3FileDatabase::startThreads() { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ; +#endif mLocalDirWatcher->start(); +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; +#endif } void p3FileDatabase::stopThreads() { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Stopping hash cache thread..." ; std::cerr.flush() ; +#endif mHashCache->fullstop(); +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; - P3FILELISTS_DEBUG() << "Stopping directory watcher thread..." ; std::cerr.flush() ; +#endif mLocalDirWatcher->fullstop(); +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; +#endif } void p3FileDatabase::tickWatchers() @@ -404,8 +415,9 @@ void p3FileDatabase::cleanup() { { RS_STACK_MUTEX(mFLSMtx) ; - +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "p3FileDatabase::cleanup()" << std::endl; +#endif // look through the list of friend directories. Remove those who are not our friends anymore. // @@ -422,7 +434,10 @@ void p3FileDatabase::cleanup() for(uint32_t i=0;ipeerId()) == friend_set.end()) { + +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; +#endif delete mRemoteDirectories[i]; mRemoteDirectories[i] = NULL ; @@ -446,7 +461,9 @@ void p3FileDatabase::cleanup() if(mRemoteDirectories.size() > friend_index && mRemoteDirectories[friend_index] != NULL) continue ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; +#endif if(mRemoteDirectories.size() <= friend_index) mRemoteDirectories.resize(friend_index+1,NULL) ; @@ -467,7 +484,9 @@ void p3FileDatabase::cleanup() for(std::map::iterator it = mPendingSyncRequests.begin();it!=mPendingSyncRequests.end();) if(online_peers.find(it->second.peer_id) == online_peers.end() || it->second.request_TS + DELAY_BEFORE_DROP_REQUEST < now) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " removing pending request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << ", because peer is offline or request is too old." << std::endl; +#endif std::map::iterator tmp(it); ++tmp; @@ -476,7 +495,9 @@ void p3FileDatabase::cleanup() } else { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " keeping request " << std::hex << it->first << std::dec << " for peer " << it->second.peer_id << std::endl; +#endif ++it ; } } @@ -597,10 +618,16 @@ void p3FileDatabase::requestDirUpdate(void *ref) if(fi == 0) return ; // not updating current directory (should we?) +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; +#endif if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) + { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Succeed." << std::endl; +#endif + } } bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const @@ -1010,7 +1037,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) { RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Received directory sync request. hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; +#endif EntryIndex entry_index = DirectoryStorage::NO_INDEX; @@ -1030,7 +1059,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) if(entry_type != DIR_TYPE_DIR) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl; +#endif ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ; } else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE))) @@ -1045,7 +1076,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust. { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl; +#endif ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT; ritem->last_known_recurs_modf_TS = local_recurs_max_time; @@ -1055,7 +1088,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) } else { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl; +#endif ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ; ritem->last_known_recurs_modf_TS = local_recurs_max_time ; @@ -1074,7 +1109,9 @@ void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem) while(ritem->directory_content_data.bin_len > MAX_DIR_SYNC_RESPONSE_DATA_SIZE) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Chopping off partial chunk of size " << MAX_DIR_SYNC_RESPONSE_DATA_SIZE << " from item data of size " << ritem->directory_content_data.bin_len << std::endl; +#endif ritem->flags |= RsFileListsItem::FLAGS_SYNC_PARTIAL ; @@ -1116,7 +1153,9 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR // item is a partial item. Look first for a starting entry +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Item from peer " << ritem->PeerId() << " is partial. Size = " << ritem->directory_content_data.bin_len << std::endl; +#endif RS_STACK_MUTEX(mFLSMtx) ; @@ -1130,7 +1169,9 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR P3FILELISTS_ERROR() << "Impossible situation: partial item ended right away. Dropping..." << std::endl; return NULL; } +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Creating new item buffer" << std::endl; +#endif mPartialResponseItems[ritem->request_id] = new RsFileListsSyncResponseItem(*ritem) ; return NULL ; @@ -1151,13 +1192,17 @@ RsFileListsSyncResponseItem *p3FileDatabase::recvAndRebuildItem(RsFileListsSyncR memcpy(&((unsigned char*)it->second->directory_content_data.bin_data)[old_len],ritem->directory_content_data.bin_data,added); it->second->directory_content_data.bin_len = old_len + added ; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Added new chunk of length " << added << ". New size is " << old_len + added << std::endl; +#endif // if finished, return the item if(is_ending) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Item is complete. Returning it" << std::endl; +#endif RsFileListsSyncResponseItem *ret = it->second ; mPartialResponseItems.erase(it) ; @@ -1186,7 +1231,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) return ; } +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Handling sync response for directory with hash " << item->entry_hash << std::endl; +#endif EntryIndex entry_index = DirectoryStorage::NO_INDEX; @@ -1229,7 +1276,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) P3FILELISTS_ERROR() << " (EE) cannot find index from hash " << item->entry_hash << ". Dropping the response." << std::endl; return ; } +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " entry index is " << entry_index << " " ; +#endif } if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) @@ -1272,7 +1321,9 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // get the info for this entry +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "currently at entry index " << e << std::endl; +#endif time_t local_update_TS; @@ -1286,7 +1337,11 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. if(generateAndSendSyncRequest(rds,e)) + { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; +#endif + } for(DirectoryStorage::DirIterator it(rds,e);it;++it) locked_recursSweepRemoteDirectory(rds,*it,depth+1); @@ -1342,7 +1397,9 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons if(it != mPendingSyncRequests.end()) { +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Not asking for sync of directory " << e << " to friend " << rds->peerId() << " because a recent pending request still exists." << std::endl; +#endif return false ; } @@ -1360,7 +1417,9 @@ bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,cons data.peer_id = item->PeerId(); data.flags = item->flags; +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Pushing req in pending list with peer id " << data.peer_id << std::endl; +#endif mPendingSyncRequests[sync_req_id] = data ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index ec4e583ba..bd35a8015 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -139,9 +139,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed bool inDirectoryCheck(); - // debug - void full_print(); - protected: int filterResults(const std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const; From 0579fb03faa7ff9a094d910351eb9b5a235db24a Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 22:14:25 +0200 Subject: [PATCH 090/174] removed unused tickWatchers() --- libretroshare/src/file_sharing/p3filelists.cc | 8 -------- libretroshare/src/file_sharing/p3filelists.h | 1 - 2 files changed, 9 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0fd6bc5db..0e734e56a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -154,10 +154,6 @@ RsServiceInfo p3FileDatabase::getServiceInfo() } int p3FileDatabase::tick() { - // tick the watchers, possibly create new ones if additional friends do connect. - // - tickWatchers(); - // tick the input/output list of update items and process them // tickRecv() ; @@ -262,10 +258,6 @@ void p3FileDatabase::stopThreads() #endif } -void p3FileDatabase::tickWatchers() -{ -} - bool p3FileDatabase::saveList(bool &cleanup, std::list& sList) { cleanup = true; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index bd35a8015..08afd0e88 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -153,7 +153,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub void cleanup(); void tickRecv(); void tickSend(); - void tickWatchers(); private: p3ServiceControl *mServCtrl ; From 7488ad457d024a25bb7071d653c5c0861d879f12 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sun, 18 Sep 2016 22:32:30 +0200 Subject: [PATCH 091/174] added display of files being hashed with green color and blue icon. This could be improved I suppose. --- retroshare-gui/src/gui/RemoteDirModel.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index a2f3fd09b..bf644713e 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -299,7 +299,10 @@ QVariant RetroshareDirModel::decorationRole(const DirDetails& details,int coln) else if (details.type == DIR_TYPE_FILE) /* File */ { // extensions predefined - return FilesDefs::getIconFromFilename(QString::fromUtf8(details.name.c_str())); + if(details.hash.isNull()) + return QIcon(":/images/reset.png") ; // file is being hashed + else + return FilesDefs::getIconFromFilename(QString::fromUtf8(details.name.c_str())); } else return QVariant(); @@ -541,7 +544,9 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const if (role == Qt::TextColorRole) { - if(details.min_age > ageIndicator) + if(details.type == DIR_TYPE_FILE && details.hash.isNull()) + return QVariant(QColor(Qt::green)) ; + else if(details.min_age > ageIndicator) return QVariant(QColor(Qt::gray)) ; else if(RemoteMode) { From 0f5cf56cc6caa3a42e63c4296c2be0b2814bc9ba Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 21 Sep 2016 22:27:53 +0200 Subject: [PATCH 092/174] removed dbase directory --- libretroshare/src/dbase/cachestrapper.cc | 1323 ----------------- libretroshare/src/dbase/cachestrapper.h | 475 ------ libretroshare/src/dbase/cachetest.h | 63 - libretroshare/src/dbase/fimonitor.cc | 1706 ---------------------- libretroshare/src/dbase/fimonitor.h | 220 --- libretroshare/src/dbase/findex.cc | 1516 ------------------- libretroshare/src/dbase/findex.h | 278 ---- libretroshare/src/dbase/fistore.cc | 459 ------ libretroshare/src/dbase/fistore.h | 102 -- 9 files changed, 6142 deletions(-) delete mode 100644 libretroshare/src/dbase/cachestrapper.cc delete mode 100644 libretroshare/src/dbase/cachestrapper.h delete mode 100644 libretroshare/src/dbase/cachetest.h delete mode 100644 libretroshare/src/dbase/fimonitor.cc delete mode 100644 libretroshare/src/dbase/fimonitor.h delete mode 100644 libretroshare/src/dbase/findex.cc delete mode 100644 libretroshare/src/dbase/findex.h delete mode 100644 libretroshare/src/dbase/fistore.cc delete mode 100644 libretroshare/src/dbase/fistore.h diff --git a/libretroshare/src/dbase/cachestrapper.cc b/libretroshare/src/dbase/cachestrapper.cc deleted file mode 100644 index 4ecdedf64..000000000 --- a/libretroshare/src/dbase/cachestrapper.cc +++ /dev/null @@ -1,1323 +0,0 @@ -/* - * RetroShare FileCache Module: cachestrapper.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "dbase/cachestrapper.h" -#include "serialiser/rsserviceids.h" -#include "serialiser/rsconfigitems.h" -#include "pqi/p3servicecontrol.h" -#include "pqi/p3peermgr.h" -#include "util/rsdir.h" - -#include -#include - -/**** -* #define CS_DEBUG 1 -***/ - -bool operator<(const CacheId &a, const CacheId &b) -{ - if (a.type == b.type) - return (a.subid < b.subid); - return (a.type < b.type); -} - -bool operator<(const CachePair &a, const CachePair &b) -{ - return (a.id < b.id); -} - - -std::ostream &operator<<(std::ostream &out, const RsCacheData &d) -{ - out << "[ p: " << d.pid << " id: <" << d.cid.type << "," << d.cid.subid; - out << "> #" << d.hash << " size: " << d.size; - out << " \"" << d.name << "\"@\"" << d.path; - out << "\" ]"; - return out; -} - -/********************************* Cache Store / Source ************************** - * This is a generic interface which interacts with the FileTransfer Unit - * to collect Data to be Cached. - * - ********************************* Cache Store / Source *************************/ - -CacheSource::CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir) - :cacheType(t), multiCache(m), mStrapper(cs), cacheDir(cachedir), cMutex("CacheSource") - { - return; - } - - /* Mutex Stuff -> to be done... */ -void CacheSource::lockData() const -{ -#ifdef CS_DEBUG - std::cerr << "CacheSource::lockData()" << std::endl; -#endif - cMutex.lock(); -} - -void CacheSource::unlockData() const -{ -#ifdef CS_DEBUG - std::cerr << "CacheSource::unlockData()" << std::endl; -#endif - cMutex.unlock(); -} - - /* to be overloaded for inherited Classes */ -bool CacheSource::loadLocalCache(const RsCacheData &data) -{ - return refreshCache(data); -} - - /* control Caches available */ -bool CacheSource::refreshCache(const RsCacheData &data,const std::set& destination_peers) -{ - bool ret = false; - { - RsStackMutex mtx(cMutex); /* LOCK MUTEX */ - - if (data.cid.type == getCacheType()) - { - int subid = 0; - if (isMultiCache()) - { - subid = data.cid.subid; - } - - /* Backup the old Caches */ - CacheSet::const_iterator it; - if (caches.end() != (it = caches.find(subid))) - { - mOldCaches[it->second.hash] = it->second; - } - - /* store new cache */ - caches[subid] = data; - ret = true; - } - } - // Strip down destination peers to eliminate peers that are not allowed to receive cache items. - - if (mStrapper) /* allow testing without full feedback */ - { - std::set allowed_dest_peers ; - - for(std::set::const_iterator it(destination_peers.begin());it!=destination_peers.end();++it) - if(isPeerAcceptedAsCacheReceiver(*it)) - allowed_dest_peers.insert(*it) ; - - mStrapper->refreshCache(data,allowed_dest_peers); - } - - return ret; -} -bool CacheSource::refreshCache(const RsCacheData &data) -{ - bool ret = false; - { - RsStackMutex mtx(cMutex); /* LOCK MUTEX */ - - if (data.cid.type == getCacheType()) - { - int subid = 0; - if (isMultiCache()) - { - subid = data.cid.subid; - } - - /* Backup the old Caches */ - CacheSet::const_iterator it; - if (caches.end() != (it = caches.find(subid))) - { - mOldCaches[it->second.hash] = it->second; - } - - /* store new cache */ - caches[subid] = data; - ret = true; - } - } - // Strip down destination peers to eliminate peers that are not allowed to receive cache items. - - std::list ids; - rsPeers->getOnlineList(ids); - - if (mStrapper) /* allow testing without full feedback */ - { - std::set allowed_dest_peers ; - - for(std::list::const_iterator it(ids.begin());it!=ids.end();++it) - if(isPeerAcceptedAsCacheReceiver(*it)) - allowed_dest_peers.insert(*it) ; - - mStrapper->refreshCache(data,allowed_dest_peers); - } - - return ret; - -} - -// bool CacheSource::refreshCache(const RsCacheData &data) -// { -// bool ret = false; -// { -// RsStackMutex mtx(cMutex); /* LOCK MUTEX */ -// -// if (data.cid.type == getCacheType()) -// { -// int subid = 0; -// if (isMultiCache()) -// { -// subid = data.cid.subid; -// } -// -// /* Backup the old Caches */ -// CacheSet::const_iterator it; -// if (caches.end() != (it = caches.find(subid))) -// { -// mOldCaches[it->second.hash] = it->second; -// } -// -// /* store new cache */ -// caches[subid] = data; -// ret = true; -// } -// } -// -// if (mStrapper) /* allow testing without full feedback */ -// mStrapper->refreshCache(data); -// -// return ret; -// } -bool CacheSource::clearCache(CacheId id) -{ - lockData(); /* LOCK MUTEX */ - - bool ret = false; - if (id.type == getCacheType()) - { - CacheSet::iterator it; - if (caches.end() != (it = caches.find(id.subid))) - { - /* Backup the old Caches */ - mOldCaches[it->second.hash] = it->second; - caches.erase(it); - ret = true; - } - } - - unlockData(); /* UNLOCK MUTEX */ - return ret; -} - -//bool CacheSource::cachesAvailable(const RsPeerId& pid, std::map &ids) -//{ -// if(!isPeerAcceptedAsCacheReceiver(pid)) -// return false ; -// -// lockData(); /* LOCK MUTEX */ -// -// /* can overwrite for more control! */ -// CacheSet::iterator it; -// for(it = caches.begin(); it != caches.end(); ++it) -// { -// ids[(it->second).cid] = it->second; -// } -// bool ret = (caches.size() > 0); -// -// unlockData(); /* UNLOCK MUTEX */ -// return ret; -// -//} - - -bool CacheSource::findCache(const RsFileHash &hash, RsCacheData &data) const -{ - lockData(); /* LOCK MUTEX */ - - bool found = false; - CacheSet::const_iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - if (hash == (it->second).hash) - { - data = it->second; - found = true; - break; - } - } - - if (!found) - { - std::map::const_iterator oit; - oit = mOldCaches.find(hash); - if (oit != mOldCaches.end()) - { - data = oit->second; - found = true; - } - } - - unlockData(); /* UNLOCK MUTEX */ - - return found; -} - - -void CacheSource::listCaches(std::ostream &out) -{ - lockData(); /* LOCK MUTEX */ - - /* can overwrite for more control! */ - CacheSet::iterator it; - out << "CacheSource::listCaches() [" << getCacheType(); - out << "] Total: " << caches.size() << std::endl; - int i; - for(i = 0, it = caches.begin(); it != caches.end(); ++it, ++i) - { - out << "\tC[" << i << "] : " << it->second << std::endl; - } - - unlockData(); /* UNLOCK MUTEX */ - return; -} - - -CacheStore::CacheStore(uint16_t t, bool m, - CacheStrapper *cs, CacheTransfer *cft, std::string cachedir) - :cacheType(t), multiCache(m), mStrapper(cs), - cacheTransfer(cft), cacheDir(cachedir), cMutex("CacheStore") - { - /* not much */ - return; - } - - /* Mutex Stuff -> to be done... */ -void CacheStore::lockData() const -{ -#ifdef CS_DEBUG -// std::cerr << "CacheStore::lockData()" << std::endl; -#endif - cMutex.lock(); -} - -void CacheStore::unlockData() const -{ -#ifdef CS_DEBUG -// std::cerr << "CacheStore::unlockData()" << std::endl; -#endif - cMutex.unlock(); -} - -void CacheStore::listCaches(std::ostream &out) -{ - lockData(); /* LOCK MUTEX */ - - /* can overwrite for more control! */ - std::map::iterator pit; - out << "CacheStore::listCaches() [" << getCacheType(); - out << "] Total People: " << caches.size(); - out << std::endl; - for(pit = caches.begin(); pit != caches.end(); ++pit) - { - CacheSet::iterator it; - out << "\tTotal for [" << pit->first << "] : " << (pit->second).size(); - out << std::endl; - for(it = (pit->second).begin(); it != (pit->second).end(); ++it) - { - out << "\t\t" << it->second; - out << std::endl; - } - } - - unlockData(); /* UNLOCK MUTEX */ - return; -} - - - /* look for stored data. using pic/cid in RsCacheData - */ -bool CacheStore::getStoredCache(RsCacheData &data) -{ - lockData(); /* LOCK MUTEX */ - - bool ok = locked_getStoredCache(data); - - unlockData(); /* UNLOCK MUTEX */ - return ok; -} - - -bool CacheStore::locked_getStoredCache(RsCacheData &data) -{ - if (data.cid.type != getCacheType()) - { - return false; - } - - std::map::iterator pit; - if (caches.end() == (pit = caches.find(data.pid))) - { - return false; - } - - CacheSet::iterator cit; - if (isMultiCache()) - { - /* look for subid */ - if ((pit->second).end() == - (cit = (pit->second).find(data.cid.subid))) - { - return false; - } - } - else - { - if ((pit->second).end() == - (cit = (pit->second).find(0))) - { - return false; - } - } - - /* we found it! (cit) */ - data = cit->second; - - return true; -} - - - -bool CacheStore::getAllStoredCaches(std::list &data) -{ - lockData(); /* LOCK MUTEX */ - - std::map::iterator pit; - for(pit = caches.begin(); pit != caches.end(); ++pit) - { - CacheSet::iterator cit; - /* look for subid */ - for(cit = (pit->second).begin(); - cit != (pit->second).end(); ++cit) - { - data.push_back(cit->second); - } - } - - unlockData(); /* UNLOCK MUTEX */ - - return true; -} - - - /* input from CacheStrapper. - * check if we want to download it... - * determine the new name/path - * then request it. - */ -void CacheStore::availableCache(const RsCacheData &data) -{ -#ifdef CS_DEBUG - std::cerr << "CacheStore::availableCache() :" << data << std::endl; -#endif - - /* basic checks */ - lockData(); /* LOCK MUTEX */ - - bool rightCache = (data.cid.type == getCacheType()); - - unlockData(); /* UNLOCK MUTEX */ - - if (!rightCache) - { - return; /* bad id */ - } - - /* These Functions lock the Mutex themselves - */ - - if (!fetchCache(data)) - { - return; /* ignore it */ - } - - RsCacheData rData = data; - - /* get new name */ - if (!nameCache(rData)) - { - return; /* error naming */ - } - - /* request it */ - if(isPeerAcceptedAsCacheProvider(rData.pid)) // Check for service permission - cacheTransfer -> RequestCache(rData, this); - - /* will get callback when it is complete */ - return; -} - - - - /* called when the download is completed ... updates internal data */ -void CacheStore::downloadedCache(const RsCacheData &data) -{ -#ifdef CS_DEBUG - std::cerr << "CacheStore::downloadedCache() :" << data << std::endl; -#endif - - /* updates data */ - if (!loadCache(data)) - { - return; - } -} - /* called when the download is completed ... updates internal data */ -void CacheStore::failedCache(const RsCacheData &data) -{ - (void) data; -#ifdef CS_DEBUG - std::cerr << "CacheStore::failedCache() :" << data << std::endl; -#endif - - return; -} - - - /* virtual function overloaded by cache implementor */ -bool CacheStore::fetchCache(const RsCacheData &data) -{ - /* should we fetch it? */ -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() tofetch?:" << data << std::endl; -#endif - - RsCacheData incache = data; - - lockData(); /* LOCK MUTEX */ - - bool haveCache = ((locked_getStoredCache(incache)) && (data.hash == incache.hash)); - - unlockData(); /* UNLOCK MUTEX */ - - - if (haveCache) - { -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() Already have it: false" << std::endl; -#endif - return false; - } - -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() Missing this cache: true" << std::endl; -#endif - return true; -} - - -int CacheStore::nameCache(RsCacheData &data) -{ - /* name it... */ - lockData(); /* LOCK MUTEX */ - -#ifdef CS_DEBUG - std::cerr << "CacheStore::nameCache() for:" << data << std::endl; -#endif - - data.name = data.hash.toStdString(); - data.path = getCacheDir(); - -#ifdef CS_DEBUG - std::cerr << "CacheStore::nameCache() done:" << data << std::endl; -#endif - - unlockData(); /* UNLOCK MUTEX */ - - return 1; -} - - -int CacheStore::loadCache(const RsCacheData &data) -{ - /* attempt to load -> dummy function */ -#ifdef CS_DEBUG - std::cerr << "CacheStore::loadCache() Dummy Load for:" << data << std::endl; -#endif - - lockData(); /* LOCK MUTEX */ - - locked_storeCacheEntry(data); - - unlockData(); /* UNLOCK MUTEX */ - - return 1; -} - -/* This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - -void CacheStore::locked_storeCacheEntry(const RsCacheData &data) -{ - /* store what we loaded - overwriting if necessary */ - std::map::iterator pit; - if (caches.end() == (pit = caches.find(data.pid))) - { - /* add in a new CacheSet */ - CacheSet emptySet; - caches[data.pid] = emptySet; - - pit = caches.find(data.pid); - } - - if (isMultiCache()) - { - (pit->second)[data.cid.subid] = data; - } - else - { - (pit->second)[0] = data; - } - - /* tell the strapper we've loaded one */ - if (mStrapper) - { - mStrapper->refreshCacheStore(data); - } - return; -} - - -/********************************* CacheStrapper ********************************* - * This is the bit which handles queries - * - ********************************* CacheStrapper ********************************/ - -CacheStrapper::CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId) - :p3Config(), mServiceCtrl(sc), mFtServiceId(ftServiceId), - csMtx("CacheStrapper") -{ - return; -} - - -void CacheStrapper::addCachePair(CachePair set) -{ - caches[set.id.type] = set; -} - - - /**************** from pqimonclient ********************/ - -void CacheStrapper::statusChange(const std::list &plist) -{ - std::list::const_iterator it; - for(it = plist.begin(); it != plist.end(); ++it) - { - if(it->actions & RS_SERVICE_PEER_CONNECTED) - { - /* grab all the cache ids and add */ - - std::map hashs; - std::map::iterator cit; - - handleCacheQuery(it->id, hashs); - - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - for(cit = hashs.begin(); cit != hashs.end(); ++cit) - { - mCacheUpdates.push_back(std::make_pair(it->id, cit->second)); - } - } - } -} - - /**************** from pqimonclient ********************/ - -void CacheStrapper::refreshCache(const RsCacheData &data,const std::set& destination_peers) -{ - /* we've received an update - * send to all online peers + self intersected with online peers. - */ -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::refreshCache() : " << data << std::endl; -#endif - const RsPeerId& ownid = mServiceCtrl->getOwnId() ; - std::set ids; - - // Need to use ftServiceID as packets are passed through there. - mServiceCtrl->getPeersConnected(mFtServiceId, ids); - ids.insert(ownid) ; - - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - for(std::set::const_iterator it = ids.begin(); it != ids.end(); ++it) - if(destination_peers.find(*it) != destination_peers.end()) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::refreshCache() Send To: " << *it << std::endl; -#endif - mCacheUpdates.push_back(std::make_pair(*it, data)); - } - - IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ -} - -// void CacheStrapper::refreshCache(const RsCacheData &data) -// { -// /* we've received an update -// * send to all online peers + self -// */ -// #ifdef CS_DEBUG -// std::cerr << "CacheStrapper::refreshCache() : " << data << std::endl; -// #endif -// -// std::string ownid = mServiceCtrl->getOwnId() ; -// std::set ids; -// mServiceCtrl->getOnlineList(ids); -// ids.push_back(ownid) ; -// -// { -// RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ -// for(std::set::const_iterator it = ids.begin(); it != ids.end(); ++it) -// if(*it == ownid || isPeerPartipating(*it)) -// mCacheUpdates.push_back(std::make_pair(*it, data)); -// } -// IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ -// } - -void CacheStrapper::refreshCacheStore(const RsCacheData & /* data */ ) -{ - /* indicate to save data */ - IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ - -} - -bool CacheStrapper::getCacheUpdates(std::list > &updates) -{ - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - updates = mCacheUpdates; - mCacheUpdates.clear(); - - return true; -} - - - - /* pass to correct CacheSet */ -void CacheStrapper::recvCacheResponse(RsCacheData &data, time_t /* ts */) -{ - /* find cache store */ - std::map::iterator it2; - if (caches.end() == (it2 = caches.find(data.cid.type))) - { - /* error - don't have this type of cache */ - return; - } - - /* notify the CacheStore */ - (it2 -> second).store -> availableCache(data); - -} - -void CacheStrapper::handleCacheQuery(const RsPeerId& id, std::map &hashs) -{ - /* basic version just iterates through .... - * more complex could decide who gets what! - * - * or that can be handled on a cache by cache basis. - */ - - std::map::iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - (it->second).source -> cachesAvailable(id, hashs); - } - return; -} - -void CacheStrapper::listCaches(std::ostream &out) -{ - /* can overwrite for more control! */ - std::map::iterator it; - out << "CacheStrapper::listCaches() [" << mServiceCtrl->getOwnId(); - out << "] " << " Total Caches: " << caches.size(); - out << std::endl; - for(it = caches.begin(); it != caches.end(); ++it) - { - out << "CacheType: " << it->first; - out << std::endl; - - (it->second).source->listCaches(out); - (it->second).store->listCaches(out); - out << std::endl; - } - return; -} - -void CacheStrapper::listPeerStatus(std::ostream & /* out */) -{ -#if 0 - std::map::iterator it; - out << "CacheStrapper::listPeerStatus() [" << ownId; - out << "] Total Peers: " << status.size() << " Total Caches: " << caches.size(); - out << std::endl; - for(it = status.begin(); it != status.end(); ++it) - { - out << "Peer: " << it->first; - out << " Query: " << (it->second).query; - out << " Answer: " << (it->second).answer; - out << std::endl; - } - return; -#endif -} - - -bool CacheStrapper::findCache(const RsFileHash& hash, RsCacheData &data) const -{ - /* can overwrite for more control! */ - std::map::const_iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - if ((it->second).source->findCache(hash, data)) - { - return true; - } - } - return false; -} - -bool CacheStrapper::CacheExist(RsCacheData& data){ - - std::string filename = data.path + "/" + data.name; - FILE* file = NULL; - file = RsDirUtil::rs_fopen(filename.c_str(), "r"); - - if(file == NULL) - return false; - - fclose(file); - return true; -} - -/***************************************************************************/ -/****************************** CONFIGURATION HANDLING *********************/ -/***************************************************************************/ - -/**** OVERLOADED FROM p3Config ****/ - -RsSerialiser *CacheStrapper::setupSerialiser() -{ - RsSerialiser *rss = new RsSerialiser(); - - /* add in the types we need! */ - rss->addSerialType(new RsCacheConfigSerialiser()); - - return rss; -} - - -bool CacheStrapper::saveList(bool &cleanup, std::list& saveData) -{ - - /* it can delete them! */ - cleanup = true; - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::saveList()" << std::endl; -#endif - - /* iterate through the Caches (local first) */ - - std::list::iterator cit; - std::list ownCaches; - std::list remoteCaches; - const RsPeerId& ownId = mServiceCtrl->getOwnId(); - - std::map::iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - std::map::iterator tit; - std::map ownTmp; - (it->second).source -> cachesAvailable(ownId, ownTmp); - (it->second).store -> getAllStoredCaches(remoteCaches); - - for(tit = ownTmp.begin(); tit != ownTmp.end(); ++tit) - { - if(CacheExist(tit->second)) - ownCaches.push_back(tit->second); - } - } - - for(cit = ownCaches.begin(); cit != ownCaches.end(); ++cit) - { - RsCacheConfig *rscc = new RsCacheConfig(); - - // Fixup lazy behaviour in clients... - // This ensures correct loading later. - // (used to be: rscc->pid = cit->pid;) - rscc->pid = ownId; - - //rscc->pname = cit->pname; - rscc->cachetypeid = cit->cid.type; - rscc->cachesubid = cit->cid.subid; - rscc->path = cit->path; - rscc->name = cit->name; - rscc->hash = cit->hash; - rscc->size = cit->size; - rscc->recvd = cit->recvd; - - saveData.push_back(rscc); - } - - for(cit = remoteCaches.begin(); cit != remoteCaches.end(); ++cit) - { - if (cit->pid == ownId) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() discarding Own Remote Cache"; - std::cerr << std::endl; -#endif - continue; /* skip own caches -> will get transferred anyway */ - } - - RsCacheConfig *rscc = new RsCacheConfig(); - - rscc->pid = cit->pid; - //rscc->pname = cit->pname; - rscc->cachetypeid = cit->cid.type; - rscc->cachesubid = cit->cid.subid; - rscc->path = cit->path; - rscc->name = cit->name; - rscc->hash = cit->hash; - rscc->size = cit->size; - rscc->recvd = cit->recvd; - - saveData.push_back(rscc); - } - - /* list completed! */ - return true; -} - - -bool CacheStrapper::loadList(std::list& load) -{ - std::list::iterator it; - RsCacheConfig *rscc; - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Item Count: " << load.size(); - std::cerr << std::endl; -#endif - std::list ownCaches; - std::list remoteCaches; - const RsPeerId& ownId = mServiceCtrl->getOwnId(); - - //peerConnectState ownState; - //mPeerMgr->getOwnNetStatus(ownState); - //std::string ownName = ownState.name+" ("+ownState.location+")"; - - std::map > saveFiles; - std::map >::iterator sit; - - for(it = load.begin(); it != load.end(); ++it) - { - /* switch on type */ - if (NULL != (rscc = dynamic_cast(*it))) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Item: "; - std::cerr << std::endl; - rscc->print(std::cerr, 10); - std::cerr << std::endl; -#endif - RsCacheData cd; - - cd.pid = RsPeerId(rscc->pid) ; - -#if 0 - if(cd.pid == ownId) - { - cd.pname = ownName; - } - else - { - peerConnectState pca; - mPeerMgr->getFriendNetStatus(rscc->pid, pca); - cd.pname = pca.name+" ("+pca.location+")"; - } -#endif - - cd.cid.type = rscc->cachetypeid; - cd.cid.subid = rscc->cachesubid; - cd.path = RsDirUtil::convertPathToUnix(rscc->path); - cd.name = rscc->name; - cd.hash = rscc->hash; - cd.size = rscc->size; - cd.recvd = rscc->recvd; - - /* store files that we want to keep */ - (saveFiles[cd.path]).insert(cd.name); - - std::map::iterator it2; - if (caches.end() == (it2 = caches.find(cd.cid.type))) - { - /* error - don't have this type of cache */ -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Can't Find Cache discarding"; - std::cerr << std::endl; -#endif - } - else - { - /* check that the file exists first.... */ - std::string filename = cd.path; - filename += "/"; - filename += cd.name; -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Checking File: " << filename; - std::cerr << std::endl; -#endif - - uint64_t tmp_size ; - bool fileExists = RsDirUtil::checkFile(filename,tmp_size); - if (fileExists) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Exists ... continuing"; - std::cerr << std::endl; -#endif - if (cd.pid == ownId) - { - - /* load local */ - (it2 -> second).source -> loadLocalCache(cd); -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() loaded Local"; - std::cerr << std::endl; -#endif - } - else - { - /* load remote */ - (it2 -> second).store -> loadCache(cd); -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() loaded Remote"; - std::cerr << std::endl; -#endif - } - } - else - { - std::cerr << "CacheStrapper::loadList() Doesn't Exist dropping: " << filename; - std::cerr << std::endl; - } - - } - - /* cleanup */ - delete (*it); - - } - else - { - /* cleanup */ - delete (*it); - } - } - load.clear() ; - - /* assemble a list of dirs to clean (union of cache dirs) */ - std::list cacheDirs; - std::list::iterator dit; -#ifdef CS_DEBUG - std::set::iterator fit; -#endif - std::map::iterator cit; - for(cit = caches.begin(); cit != caches.end(); ++cit) - { - std::string lcdir = (cit->second).source->getCacheDir(); - std::string rcdir = (cit->second).store->getCacheDir(); - - if (cacheDirs.end() == std::find(cacheDirs.begin(), cacheDirs.end(), lcdir)) - { - cacheDirs.push_back(lcdir); - } - - if (cacheDirs.end() == std::find(cacheDirs.begin(), cacheDirs.end(), rcdir)) - { - cacheDirs.push_back(rcdir); - } - } - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Files To Save:" << std::endl; -#endif - -#ifdef CS_DEBUG - for(sit = saveFiles.begin(); sit != saveFiles.end(); ++sit) - { - std::cerr << "CacheStrapper::loadList() Files To Save in dir: <" << sit->first << ">" << std::endl; - for(fit = (sit->second).begin(); fit != (sit->second).end(); ++fit) - { - std::cerr << "\tFile: " << *fit << std::endl; - } - } -#endif - - std::set emptySet; - for(dit = cacheDirs.begin(); dit != cacheDirs.end(); ++dit) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Cleaning cache dir: <" << *dit << ">" << std::endl; -#endif - sit = saveFiles.find(RsDirUtil::convertPathToUnix(*dit)); - - if (sit != saveFiles.end()) - { -#ifdef CS_DEBUG - for(fit = (sit->second).begin(); fit != (sit->second).end(); ++fit) - { - std::cerr << "CacheStrapper::loadList() Keeping File: " << *fit << std::endl; - } -#endif - RsDirUtil::cleanupDirectoryFaster(*dit, sit->second); - } - else - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() No Files to save here!" << std::endl; -#endif - RsDirUtil::cleanupDirectoryFaster(*dit, emptySet); - } - } - - return true; - -} - - -/********************************* CacheStrapper ********************************* - * This is the bit which handles queries - * - ********************************* CacheStrapper ********************************/ - - -/* request from CacheStore */ -bool CacheTransfer::RequestCache(RsCacheData &data, CacheStore *cbStore) -{ - /* check for a previous request -> and cancel - * - * - if duplicate pid, cid -> cancel old transfer - * - if duplicate hash -> Fail Transfer - */ - - std::map::iterator dit; - std::map::iterator sit; - - for(dit = cbData.begin(); dit != cbData.end(); ++dit) - { - if (((dit->second).pid == data.pid) && - ((dit->second).cid.type == data.cid.type) && - ((dit->second).cid.subid == data.cid.subid)) - { - sit = cbStores.find(dit->second.hash); - - /* if identical to previous request, then we don't want to cancel - * a partially transferred cache file - * - * We wouldn't expect to have to request it again, however the feedback loop - * from ftController is not completed (it should callback and tell us if it cancels - * the cache file. XXX TO FIX. - */ - if ((data.hash == dit->second.hash) && - (data.path == dit->second.path) && - (data.size == dit->second.size)) - { - std::cerr << "Re-request duplicate cache... let it continue"; - std::cerr << std::endl; - /* request data */ - RequestCacheFile(data.pid, data.path, data.hash, data.size); - - return true; - } - - /* cancel old transfer */ - CancelCacheFile(dit->second.pid, dit->second.path, - dit->second.hash, dit->second.size); - - sit = cbStores.find(dit->second.hash); - cbData.erase(dit); - cbStores.erase(sit); - - break; - } - } - - /* find in store.... */ - sit = cbStores.find(data.hash); - if (sit != cbStores.end()) - { - /* Duplicate Current Request */ - cbStore -> failedCache(data); - return false; - } - - - /* store request */ - cbData[data.hash] = data; - cbStores[data.hash] = cbStore; - - /* request data */ - RequestCacheFile(data.pid, data.path, data.hash, data.size); - - /* wait for answer */ - return true; -} - - -/* to be overloaded */ -bool CacheTransfer::RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size) -{ - (void) id; - (void) path; - (void) size; -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::RequestCacheFile() : from:" << id << " #"; - std::cerr << hash << " size: " << size; - std::cerr << " savepath: " << path << std::endl; - std::cerr << "CacheTransfer::RequestCacheFile() Dummy... saying completed"; - std::cerr << std::endl; -#endif - - /* just tell them we've completed! */ - CompletedCache(hash); - return true; -} - -/* to be overloaded */ -bool CacheTransfer::CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash &hash, uint64_t size) -{ - (void) id; - (void) path; - (void) hash; - (void) size; -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CancelCacheFile() : from:" << id << " #"; - std::cerr << hash << " size: " << size; - std::cerr << " savepath: " << path << std::endl; - std::cerr << "CacheTransfer::CancelCacheFile() Dummy fn"; - std::cerr << std::endl; -#endif - - return true; -} - - -/* internal completion -> does cb */ -bool CacheTransfer::CompletedCache(const RsFileHash& hash) -{ - std::map::iterator dit; - std::map::iterator sit; - -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache(" << hash << ")"; - std::cerr << std::endl; -#endif - - /* find in store.... */ - sit = cbStores.find(hash); - dit = cbData.find(hash); - - if ((sit == cbStores.end()) || (dit == cbData.end())) - { -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache() Failed to find it"; - std::cerr << std::endl; -#endif - - return false; - } - -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache() callback to store"; - std::cerr << std::endl; -#endif - /* callback */ - (sit -> second) -> downloadedCache(dit->second); - - /* clean up store */ - cbStores.erase(sit); - cbData.erase(dit); - - return true; -} - -/* internal completion -> does cb */ -bool CacheTransfer::FailedCache(const RsFileHash& hash) -{ - std::map::iterator dit; - std::map::iterator sit; - - /* find in store.... */ - sit = cbStores.find(hash); - dit = cbData.find(hash); - - if ((sit == cbStores.end()) || (dit == cbData.end())) - { - return false; - } - - /* callback */ - (sit -> second) -> failedCache(dit->second); - - /* clean up store */ - cbStores.erase(sit); - cbData.erase(dit); - - return true; -} - - -bool CacheTransfer::FindCacheFile(const RsFileHash &hash, std::string &path, uint64_t &size) -{ - RsCacheData data; - if (strapper->findCache(hash, data)) - { - path = data.path + "/" + data.name; - size = data.size; - return true; - } - - return false; -} - - - diff --git a/libretroshare/src/dbase/cachestrapper.h b/libretroshare/src/dbase/cachestrapper.h deleted file mode 100644 index 3be595fc4..000000000 --- a/libretroshare/src/dbase/cachestrapper.h +++ /dev/null @@ -1,475 +0,0 @@ -/* - * RetroShare FileCache Module: cachestrapper.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_CACHE_STRAPPER_H -#define MRK_CACHE_STRAPPER_H - -#include "pqi/p3cfgmgr.h" -#include "pqi/pqiservicemonitor.h" -#include "util/rsthreads.h" - -#include -#include -#include -#include -#include - -/******************* CacheStrapper and Related Classes ******************* - * A generic Cache Update system. - * - * CacheStrapper: maintains a set of CacheSources, and CacheStores, - * queries and updates as new information arrives. - * - * CacheTransfer: Interface for FileTransfer Class to support. - * - * CacheSource: Base Class for cache data provider. eg. FileIndexMonitor. - * CacheStore: Base Class for data cache. eg. FileCache/Store. - * - * Still TODO: - * (1) Design and Implement the Upload side of CacheTransfer/CacheStrapper. - * (2) CacheStrapper:: Save / Load Cache lists.... - * (3) Clean up lists, maps on shutdown etc. - * (4) Consider Mutexes for multithreaded operations. - * (5) Test the MultiSource/Store capabilities. - * - ******************* CacheStrapper and Related Classes *******************/ - - -class CacheTransfer; /* Interface for File Transfer */ -class CacheSource; /* Interface for local File Index/Monitor */ -class CacheStore; /* Interface for the actual Cache */ -class CacheStrapper; /* Controlling Class */ - -/******************************** CacheId ********************************/ - -/*! - * Use this to identify the type of cache source, strapper, - */ -class CacheId -{ - public: - CacheId() :type(0), subid(0) { return; } - CacheId(uint16_t a, uint16_t b) :type(a), subid(b) { return; } - uint16_t type; /// cache types, this should be set to services type - uint16_t subid; /// should be initialised when using multicache feature of cachestrapper -}; - - -bool operator<(const CacheId &a, const CacheId &b); - -/*! - * Use for identifying physical files that have been chosen as cache data - * note: this does not actual store the data but serves to locate on network (via hash attribute, - * and on file via path) - */ -class RsCacheData -{ - public: - - RsPeerId pid; /// peer id - /// REMOVED as a WASTE to look it up everywhere! std::string pname; /// peer name (can be used by cachestore) - CacheId cid; /// cache id - std::string path; /// file system path where physical cache data is located - std::string name; - RsFileHash hash; - uint64_t size; - time_t recvd; /// received timestamp -}; - - -std::ostream &operator<<(std::ostream &out, const RsCacheData &d); - -/***************************** CacheTransfer *****************************/ - -/** - * Interface for FileTransfer Class to support cache - */ -class CacheTransfer -{ - public: - - CacheTransfer(CacheStrapper *cs) :strapper(cs) { return; } - virtual ~CacheTransfer() {} - - /*! - * upload side of things .... searches through CacheStrapper. - */ - bool FindCacheFile(const RsFileHash& hash, std::string &path, uint64_t &size); - - /*! - * At the download side RequestCache() => overloaded RequestCacheFile() - * the class should then call CompletedCache() or FailedCache() - */ - bool RequestCache(RsCacheData &data, CacheStore *cbStore); /* request from CacheStore */ - - protected: - - /*! - * to be overloaded - */ - virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - - bool CompletedCache(const RsFileHash &hash); /* internal completion -> does cb */ - bool FailedCache(const RsFileHash &hash); /* internal completion -> does cb */ - - private: - - CacheStrapper *strapper; - - std::map cbData; - std::map cbStores; -}; - - - -/************************ CacheSource/CacheStore *************************/ - -typedef std::map CacheSet; - -/*! - * Implements features needed for a service to act as a cachesource and allow pushing a of cache data from service to strapper - * Service is able to use this class for refresh its cache (push cache data) - * and interface to load and check cache availablility among peers (source of cache data) - * Architecturally Cachestrapper maintains the cachesource (which is passed as a pointer handle) while the cachesource-inheriting - * service can update cachesource as to new cache sources (cache data) created. Equivalently it enquiries through cache source for - * new cache data from peers - * @see p3Distrib - */ -class CacheSource -{ - public: - - CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir); - virtual ~CacheSource() {} - - /*! - * called to determine available cache for peer - - * default acceptable (returns all) - */ - virtual bool cachesAvailable(const RsPeerId& pid, std::map &ids)=0; - - /*! - * function called at startup to load from - * configuration file.... - * to be overloaded by inherited class - */ - virtual bool loadLocalCache(const RsCacheData &data); - - /* control Caches available */ - bool refreshCache(const RsCacheData &data,const std::set& destination_peers); - bool refreshCache(const RsCacheData &data); - bool clearCache(CacheId id); - - /* controls if peer is an accepted receiver for cache items. Default is yes. To be overloaded. */ - virtual bool isPeerAcceptedAsCacheReceiver(const RsPeerId& /*peer_id*/) { return true ; } - - /* get private data */ - std::string getCacheDir() { return cacheDir; } - bool isMultiCache() { return multiCache; } - uint16_t getCacheType() { return cacheType; } - - /* display */ - void listCaches(std::ostream &out); - - /* search */ - bool findCache(const RsFileHash& hash, RsCacheData &data) const; - - protected: - - uint16_t cacheType; /// for checking of cache type (usually of child class of source) - bool multiCache; /// whether multisource is in use or not. - CacheStrapper *mStrapper; - - /*** MUTEX LOCKING */ - void lockData() const; - void unlockData() const; - - CacheSet caches; /// all local cache data stored here - std::map mOldCaches; /// replaced/cleared caches are pushed here (in case requested) - - private: - - std::string cacheDir; - mutable RsMutex cMutex; - -}; - - -/*! - * Base Class for data cache. eg. FileCache/Store. - * This is best used to deal with external caches from other peers - * @see p3Distrib. pqiMonitor - */ -class CacheStore -{ - public: - - /*! - * - * @param t set to particular rs_service id. see rsserviceids.h - * @param m whether this is multicache service (true) or not(false) - * @param cs cache strapper instance responsible for maintaining the cache service - * @param cft cache transfer instance responsible for rquestiing and tranfering caches - * @param cachedir directory used to store cache related info for cachestore client - * @return - */ - CacheStore(uint16_t t, bool m, CacheStrapper *cs, CacheTransfer *cft, std::string cachedir); - virtual ~CacheStore() {} - - /* current stored data */ - - /*! - * - * @param data returns cache data for pid/cid set in data itself - * @return false is unsuccessful and vice versa - */ - bool getStoredCache(RsCacheData &data); /* use pid/cid in data */ - - /*! - * - * @param data all cache store by cachestore is store here - * @return false not returned, only true at the moment - */ - bool getAllStoredCaches(std::list &data); /* use pid/cid in data */ - - /*! - * input from CacheStrapper -> store can then download new data - */ - void availableCache(const RsCacheData &data); - - /*! - * should be called when the download is completed ... cache data is loaded - */ - void downloadedCache(const RsCacheData &data); - - /*! - * called if the download fails, TODO: nothing done yet - */ - void failedCache(const RsCacheData &data); - - /* virtual functions overloaded by cache implementor */ - - /* controls if peer is an accepted provider for cache items. Default is yes. To be overloaded. */ - virtual bool isPeerAcceptedAsCacheProvider(const RsPeerId& /*peer_id*/) { return true ; } - - /*! - * @param data cache data is stored here - * @return false is failed (cache does not exist), otherwise true - */ - virtual bool fetchCache(const RsCacheData &data); /* a question? */ - virtual int nameCache(RsCacheData &data); /* fill in the name/path */ - virtual int loadCache(const RsCacheData &data); /* actual load, once data available */ - - /* get private data */ - - std::string getCacheDir() { return cacheDir; } - bool isMultiCache() { return multiCache; } - uint16_t getCacheType() { return cacheType; } - - /*! - * display, e.g. can pass std::out, cerr, ofstream, etc - */ - void listCaches(std::ostream &out); - - protected: - - /*! - * ** MUTEX LOCKING - */ - void lockData() const; - - /*! - * ** MUTEX LOCKING - */ - void unlockData() const; - - /*! This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - void locked_storeCacheEntry(const RsCacheData &data); - - /*! This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - bool locked_getStoredCache(RsCacheData &data); - - private: - - uint16_t cacheType; /* for checking */ - bool multiCache; /* do we care about subid's */ - - CacheStrapper *mStrapper; - CacheTransfer *cacheTransfer; - - std::string cacheDir; - - mutable RsMutex cMutex; - std::map caches; - -}; - - - -/***************************** CacheStrapper *****************************/ - - -/*! - * a convenient to pass cache handles to cachestrapper to maintain the cache service - * Make Sure you get the Ids right! see rsservicesids.h. - * When creating a cache service this data structure is - * source, usually the child class of store and source also serves as both handles - * @see CacheStrapper - */ -class CachePair -{ - public: - - /*! - * Default constructor, all variables set to NULL - */ - CachePair() - :source(NULL), store(NULL), id(0, 0) { return; } - - /*! - * - * @param a cache source for service - * @param b cache store for service - * @param c the cache service id, c.type should be set to service id service-child class of store and source - */ - CachePair(CacheSource *a, CacheStore *b, CacheId c) - :source(a), store(b), id(c) { return; } - - CacheSource *source; - CacheStore *store; - CacheId id; /// should be set id type to service types of store and source service-child class, and subid for multicache use -}; - - -bool operator<(const CachePair &a, const CachePair &b); - -/*! - * CacheStrapper: maintains a set of CacheSources, and CacheStores, - * queries and updates as new information arrives. - */ - -class p3ServiceControl; - -class CacheStrapper: public pqiServiceMonitor, public p3Config -{ - public: - - /*! - * @param cm handle used by strapper for getting peer connection information (online peers, sslids...) - * @return - */ - CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId); -virtual ~CacheStrapper() { return; } - - /************* from pqiMonitor *******************/ -virtual void statusChange(const std::list &plist); - /************* from pqiMonitor *******************/ - - /* Feedback from CacheSources */ - -/*! - * send data to peers online and self - * @param data - * - */ -void refreshCache(const RsCacheData &data); -void refreshCache(const RsCacheData &data,const std::set& destination_peers); // specify a particular list of destination peers (self not added!) - -/*! - * forces config savelist - * @param data - * @see saveList() - */ -void refreshCacheStore(const RsCacheData &data); - -/*! - * list of Caches to send out - */ -bool getCacheUpdates(std::list > &updates); - -/*! - * add to strapper's cachepair set so a related service's store and source can be maintained - * @param pair the source and store handle for a service - */ -void addCachePair(CachePair pair); - - /*** I/O (2) ***/ -void recvCacheResponse(RsCacheData &data, time_t ts); -void handleCacheQuery(const RsPeerId& id, std::map &data); - - -/*! - * search through CacheSources. - * @return false if cachedate mapping to hash not found - */ -bool findCache(const RsFileHash &hash, RsCacheData &data) const; - - /* display */ -void listCaches(std::ostream &out); - -/*! - * does not do anything - * @param out - * @deprecated - */ -void listPeerStatus(std::ostream &out); - -/** - * Checks if the cache physically exist at path given - * @param data - * @return whether it exists or not - */ -bool CacheExist(RsCacheData& data); - - /* Config */ - protected: - - /* Key Functions to be overloaded for Full Configuration */ -virtual RsSerialiser *setupSerialiser(); -virtual bool saveList(bool &cleanup, std::list&); -virtual bool loadList(std::list& load); - - private: - - /* these are static - so shouldn't need mutex */ - p3ServiceControl *mServiceCtrl; - uint32_t mFtServiceId; - - std::map caches; - - RsMutex csMtx; /* protect below */ - - std::list > mCacheUpdates; -}; - - -#endif diff --git a/libretroshare/src/dbase/cachetest.h b/libretroshare/src/dbase/cachetest.h deleted file mode 100644 index 59d55b5be..000000000 --- a/libretroshare/src/dbase/cachetest.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * RetroShare FileCache Module: cachetest.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_TEST_CACHE_H -#define MRK_TEST_CACHE_H - -#include "cachestrapper.h" - -#define TESTID 0xffff -#define TESTID2 0xffee - -class CacheTestSource: public CacheSource -{ - public: - CacheTestSource(CacheStrapper *cs, std::string dir) - :CacheSource(cs, TESTID, false, dir) { return; } -}; - -class CacheTestStore: public CacheStore -{ - public: - CacheTestStore(CacheTransfer *cft, std::string dir) - :CacheStore(TESTID, false, cft, dir) { return; } -}; - - -class CacheTestMultiSource: public CacheSource -{ - public: - CacheTestMultiSource(CacheStrapper *cs, std::string dir) - :CacheSource(cs, TESTID2, true, dir) { return; } -}; - -class CacheTestMultiStore: public CacheStore -{ - public: - CacheTestMultiStore(CacheTransfer *cft, std::string dir) - :CacheStore(TESTID2, true, cft, dir) { return; } -}; - - -#endif - diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc deleted file mode 100644 index 3269469f7..000000000 --- a/libretroshare/src/dbase/fimonitor.cc +++ /dev/null @@ -1,1706 +0,0 @@ -/* - * RetroShare FileCache Module: fimonitor.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifdef WINDOWS_SYS -#include "util/rsstring.h" -#include "util/rswin.h" -#endif - -#include "rsserver/p3face.h" -#include "dbase/fimonitor.h" -#include "util/rsdir.h" -#include "util/rsmemory.h" -#include "pqi/authssl.h" -#include "serialiser/rsserviceids.h" -#include "retroshare/rsiface.h" -#include "pqi/p3notify.h" -#include "retroshare/rspeers.h" -#include "retroshare/rstypes.h" -#include "util/folderiterator.h" -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// *********** -//#define FIM_DEBUG 1 -// ***********/ - -FileIndexMonitor::FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid,const std::string& config_dir) - :CacheSource(RS_SERVICE_TYPE_FILE_INDEX, true, cs, cachedir), fiMutex("FileIndexMonitor"), fi(pid), - pendingDirs(false), pendingForceCacheWrite(false), - mForceCheck(false), mInCheck(false), hashCache(config_dir+"/" + "file_cache"),useHashCache(true) - -{ - updatePeriod = 15 * 60; // 15 minutes - reference_time = 0 ; -} - -bool FileIndexMonitor::autoCheckEnabled() const -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return updatePeriod > 0 ; -} - -bool FileIndexMonitor::rememberHashFiles() -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return useHashCache ; -} -void FileIndexMonitor::setRememberHashFiles(bool b) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ -#ifdef FIM_DEBUG - std::cerr << "Setting useHashCache to " << b << std::endl; -#endif - useHashCache = b ; -} -void FileIndexMonitor::setRememberHashFilesDuration(uint32_t days) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - -#ifdef FIM_DEBUG - std::cerr << "Setting HashCache duration to " << days << std::endl; -#endif - hashCache.setRememberHashFilesDuration(days) ; -} - -uint32_t FileIndexMonitor::rememberHashFilesDuration() const -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - return hashCache.rememberHashFilesDuration() ; -} - -// Remove any memory of formerly hashed files that are not shared anymore -void FileIndexMonitor::clearHashFiles() -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - hashCache.clear() ; - hashCache.save() ; -} - -HashCache::HashCache(const std::string& path) - : _path(path) -{ - _max_cache_duration_days = 10 ; // 10 days is the default value. - _files.clear() ; - _changed = false ; - - // check for unencrypted - - std::istream *f = NULL ; - uint64_t file_size ; - - if(RsDirUtil::checkFile( _path+".bin",file_size,false ) ) - { - std::cerr << "Encrypted hash cache file present. Reading it." << std::endl; - - // read the binary stream into memory. - // - void *buffer = rs_malloc(file_size) ; - - if(buffer == NULL) - return ; - - FILE *F = fopen( (_path+".bin").c_str(),"rb") ; - if (!F) - { - std::cerr << "Cannot open file for reading encrypted file cache, filename " << (_path+".bin") << std::endl; - free(buffer); - return; - } - if(fread(buffer,1,file_size,F) != file_size) - { - std::cerr << "Cannot read from file " + _path+".bin" << ": something's wrong." << std::endl; - free(buffer) ; - fclose(F) ; - return ; - } - fclose(F) ; - - void *decrypted_data =NULL; - int decrypted_data_size =0; - - if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) - { - std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; - free(buffer) ; - return ; - } - free(buffer) ; - - std::string s((char *)decrypted_data,decrypted_data_size) ; - f = new std::istringstream(s) ; - - free(decrypted_data) ; - } - else - { - std::cerr << "Encrypted file cache not present. Trying unencrypted..." << std::endl; - - f = new std::ifstream( (_path+".lst").c_str()) ; - - if(!f->good()) - { - delete f ; - std::cerr << "Unencrypted file cache not present either." << std::endl; - return ; - } - } - - std::streamsize max_line_size = 2000 ; // should be enough. Anyway, if we - // miss one entry, we just lose some - // cache itemsn but this is not too - // much of a problem. - char *buff = new char[max_line_size] ; - -#ifdef FIM_DEBUG - std::cerr << "Loading HashCache from file " << path << std::endl ; - int n=0 ; -#endif - - while(!f->eof()) - { - HashCacheInfo info ; - - f->getline(buff,max_line_size,'\n') ; - std::string name(buff) ; - - f->getline(buff,max_line_size,'\n') ; //if(sscanf(buff,"%llu",&info.size) != 1) break ; - - info.size = 0 ; - sscanf(buff, RsDirUtil::scanf_string_for_uint(sizeof(info.size)), &info.size); - - f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.time_stamp)),&info.time_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } - f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.modf_stamp)),&info.modf_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } - f->getline(buff,max_line_size,'\n') ; info.hash = RsFileHash(std::string(buff)) ; - -#ifdef FIM_DEBUG - std::cerr << " (" << name << ", " << info.size << ", " << info.time_stamp << ", " << info.modf_stamp << ", " << info.hash << std::endl ; - ++n ; -#endif - _files[name] = info ; - } - - delete[] buff ; - delete f ; -#ifdef FIM_DEBUG - std::cerr << n << " entries loaded." << std::endl ; -#endif -} - -void HashCache::save() -{ - if(!_changed) - { -#ifdef FIM_DEBUG - std::cerr << "Hash cache not changed. Not saving." << std::endl ; -#endif - return; - } - -#ifdef FIM_DEBUG - std::cerr << "Saving Hash Cache to file " << _path << "..." << std::endl ; -#endif - - std::ostringstream f ; - for(std::map::const_iterator it(_files.begin()); - it != _files.end(); - ++it) - { - f << it->first << std::endl ; - f << it->second.size << std::endl; - f << it->second.time_stamp << std::endl; - f << it->second.modf_stamp << std::endl; - f << it->second.hash << std::endl; - } - - void *encryptedData = NULL ; - int encDataLen = 0 ; - - if(!AuthSSL::getAuthSSL()->encrypt( - encryptedData, - encDataLen, - f.str().c_str(), - f.str().length(), - AuthSSL::getAuthSSL()->OwnId())) - { - std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; - return; - } - - FILE *F = fopen( (_path+".bin.tmp").c_str(),"wb" ) ; - - if(!F) - { - std::cerr << "Cannot open encrypted file cache for writing: " << _path+".bin.tmp" << std::endl; - goto save_free; - } - if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) - { - std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; - fclose(F) ; - goto save_free; - } - - fclose(F) ; - - RsDirUtil::renameFile(_path+".bin.tmp",_path+".bin") ; -#ifdef FIM_DEBUG - std::cerr << "done." << std::endl ; -#endif - - _changed = false; - -save_free: - free(encryptedData); -} - -bool HashCache::find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) -{ -#ifdef FIM_DEBUG - std::cerr << "HashCache: looking for " << full_path << std::endl ; -#endif - time_t now = time(NULL) ; - std::map::iterator it(_files.find(full_path)) ; - - if(it != _files.end() && (uint64_t)time_stamp == it->second.modf_stamp && size == it->second.size) - { - hash = it->second.hash ; - it->second.time_stamp = now ; -#ifdef FIM_DEBUG - std::cerr << "Found in cache." << std::endl ; -#endif - return true ; - } - else - { -#ifdef FIM_DEBUG - std::cerr << "not found in cache." << std::endl ; -#endif - return false ; - } -} -void HashCache::insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) -{ - HashCacheInfo info ; - info.size = size ; - info.modf_stamp = time_stamp ; - info.time_stamp = time(NULL) ; - info.hash = hash ; - - _files[full_path] = info ; - _changed = true ; - -#ifdef FIM_DEBUG - std::cerr << "Entry inserted in cache: " << full_path << ", " << size << ", " << time_stamp << std::endl ; -#endif -} -void HashCache::clean() -{ -#ifdef FIM_DEBUG - std::cerr << "Cleaning HashCache..." << std::endl ; -#endif - time_t now = time(NULL) ; - time_t duration = _max_cache_duration_days * 24 * 3600 ; // seconds - -#ifdef FIM_DEBUG - std::cerr << "cleaning hash cache." << std::endl ; -#endif - - for(std::map::iterator it(_files.begin());it!=_files.end();) - if(it->second.time_stamp + duration < (uint64_t)now) - { -#ifdef FIM_DEBUG - std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ; -#endif - std::map::iterator tmp(it) ; - ++tmp ; - _files.erase(it) ; - it=tmp ; - _changed = true ; - } - else - ++it ; - -#ifdef FIM_DEBUG - std::cerr << "Done." << std::endl; -#endif -} - -FileIndexMonitor::~FileIndexMonitor() -{ - /* Data cleanup - TODO */ -} - -int FileIndexMonitor::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) -{ - results.clear(); - std::list firesults; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - fi.searchTerms(keywords, firesults); - } - - return filterResults(firesults,results,flags,peer_id) ; -} - -int FileIndexMonitor::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const -{ - results.clear(); - std::list firesults; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - fi.searchBoolExp(exp, firesults); - } - - return filterResults(firesults,results,flags,peer_id) ; -} - -int FileIndexMonitor::filterResults(std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const -{ -#ifdef DEBUG - if((flags & ~RS_FILE_HINTS_PERMISSION_MASK) > 0) - std::cerr << "(EE) ***** FileIndexMonitor:: Flags ERROR in filterResults!!" << std::endl; -#endif - /* translate/filter results */ - - for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) - { - DirDetails cdetails ; - RequestDirDetails (*rit,cdetails,FileSearchFlags(0u)); -#ifdef FIM_DEBUG - std::cerr << "Filtering candidate " << (*rit)->name << ", flags=" << cdetails.flags << ", peer=" << peer_id ; -#endif - - if(!peer_id.isNull()) - { - FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ; - - if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags )) - { - cdetails.id.clear() ; - results.push_back(cdetails); -#ifdef FIM_DEBUG - std::cerr << ": kept" << std::endl ; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << ": discarded" << std::endl ; -#endif - } - else - results.push_back(cdetails); - } - - return !results.empty() ; -} - -bool FileIndexMonitor::findLocalFile(const RsFileHash& hash,FileSearchFlags hint_flags, const RsPeerId& peer_id,std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const -{ - std::list results; - bool ok = false; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - -#ifdef FIM_DEBUG -// std::cerr << "FileIndexMonitor::findLocalFile() Hash: " << hash << std::endl; -#endif - /* search through the fileIndex */ - fi.searchHash(hash, results); - - if ( !results.empty() ) - { - /* find the full path for the first entry */ - FileEntry *fe = results.front(); - DirEntry *de = fe->parent; /* all files must have a valid parent! */ - - locked_findShareFlagsAndParentGroups(fe,storage_flags,parent_groups) ; - - // turn share flags into hint flags - - FileSearchFlags shflh = peer_id.isNull()?(RS_FILE_HINTS_BROWSABLE|RS_FILE_HINTS_NETWORK_WIDE):rsPeers->computePeerPermissionFlags(peer_id,storage_flags,parent_groups) ; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile: Filtering candidate " << fe->name << ", flags=" << storage_flags << ", hint_flags=" << hint_flags << ", peer_id = " << peer_id << std::endl ; -#endif - - if(peer_id.isNull() || (shflh & hint_flags)) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile() Found Name: " << fe->name << std::endl; -#endif - std::string shpath = RsDirUtil::removeRootDir(de->path); - std::string basedir = RsDirUtil::getRootDir(de->path); - std::string realroot = locked_findRealRoot(basedir); - - /* construct full name */ - if (realroot.length() > 0) - { - fullpath = realroot + "/"; - if (shpath != "") - { - fullpath += shpath + "/"; - } - fullpath += fe->name; - - size = fe->size; - ok = true; - } -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile() Found Path: " << fullpath << std::endl; - std::cerr << "FileIndexMonitor::findLocalFile() Found Size: " << size << std::endl; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << "FileIndexMonitor::findLocalFile() discarded" << std::endl ; -#endif - } - } /* UNLOCKED DIRS */ - - return ok; -} - -bool FileIndexMonitor::convertSharedFilePath(std::string path, std::string &fullpath) -{ - bool ok = false; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::convertSharedFilePath() path: " << path << std::endl; -#endif - - std::string shpath = RsDirUtil::removeRootDir(path); - std::string basedir = RsDirUtil::getRootDir(path); - std::string realroot = locked_findRealRoot(basedir); - - /* construct full name */ - if (realroot.length() > 0) - { - fullpath = realroot + "/"; - fullpath += shpath; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::convertSharedFilePath() Found Path: " << fullpath << std::endl; -#endif - ok = true; - } - - } /* UNLOCKED DIRS */ - - return ok; -} - - -bool FileIndexMonitor::loadLocalCache(const RsCacheData &data) /* called with stored data */ -{ - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadLocalCache(): subid = " << data.cid.subid << ", filename=" << data.name << ", peer id = " << data.pid << std::endl; -#endif - - if(!strcmp(data.name.c_str()+data.name.size()-5,".rsfc"))// this trick allows to load the complete file. Not the one being shared. - { // other files are discarded and re-created in case permissions have changed. - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - /* More error checking needed here! */ - - std::string name = data.name ; - - if ( fi.loadIndex(data.path + '/' + name, RsFileHash(), data.size) ) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadCache() Success!"; - std::cerr << std::endl; -#endif - fi.root->row = 0; - fi.root->name = data.pid.toStdString(); // XXX Hack here - TODO - - std::string fname_browsable = data.path + '/' + name ; - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(fname_browsable, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(fname_browsable.c_str(), &buf)) -#endif - { - reference_time = buf.st_mtime ; -#ifdef FIM_DEBUG - std::cerr << "Read new reference time of created file " << fname_browsable << ", to " << reference_time << std::endl; -#endif - } - else - { - std::cerr << "(EE) Error. Cannot get the proper modification time for file " << fname_browsable << " errno=" << errno << std::endl; - reference_time = 0 ; - } -#ifdef FIM_DEBUG - std::cerr << "Current reference time is now : " << reference_time << std::endl; -#endif - } - else - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadCache() Failed!"; - std::cerr << std::endl; -#endif - reference_time = 0 ; - } - - fi.updateMaxModTime() ; - - // The index is re-saved. - // - we might have new friends - // - the cache system removes old cache items so we need to re-create it. - // - locked_saveFileIndexes(false) ; - } -#ifdef FIM_DEBUG - else - std::cerr << "FileIndexMonitor:: not loading cache item " << data.name << std::endl; -#endif - - return false; -} - -bool FileIndexMonitor::updateCache(const RsCacheData &data,const std::set& destination_peers) /* we call this one */ -{ - return refreshCache(data,destination_peers); -} - - -int FileIndexMonitor::getPeriod() const -{ -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setPeriod() getting watch period" << std::endl; -#endif - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return updatePeriod ; -} - -void FileIndexMonitor::setPeriod(int period) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - updatePeriod = period; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setPeriod() Setting watch period to " << updatePeriod << std::endl; -#endif -} - -void FileIndexMonitor::data_tick() -{ - if(autoCheckEnabled()) - updateCycle(); - - int i=0 ; - for(;;++i) - { - if(!isRunning()) - return; - - usleep(1*1000*1000); // 1 sec - - /* check dirs if they've changed */ - if (internal_setSharedDirectories()) - break; - - { - RsStackMutex mtx(fiMutex) ; - - if(i >= abs(updatePeriod)) - break ; - } - } - - if(i < abs(updatePeriod) || autoCheckEnabled()) - updateCycle(); -} - -void FileIndexMonitor::updateCycle() -{ - time_t startstamp = time(NULL); - -#ifdef FIM_DEBUG - std::cerr << "Checking directory for new/modified files. Reference time is " << reference_time << std::endl; -#endif - /* iterate through all out-of-date directories */ - bool moretodo = true; - bool fiMods = false; - - { - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - mInCheck = true; - } - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_EXAMINING_FILES, "") ; - - std::vector to_hash ; - - bool cache_is_new ; - { - RsStackMutex mtx(fiMutex) ; - cache_is_new = useHashCache && hashCache.empty() ; - } - struct stat64 buf; - - while(isRunning() && moretodo) - { - /* sleep a bit for each loop */ -// csoler: I'm disabling this since it causes a very long update cycle when the number -// of directories to go through is very large. -// -// usleep(100*1000); /* 100 msec */ - - /* check if directories have been updated */ - - if (internal_setSharedDirectories()) /* reset start time */ - startstamp = time(NULL); - - /* Handle a Single out-of-date directory */ - - time_t stamp = time(NULL); - - /* lock dirs from now on */ - RsStackMutex mtx(fiMutex) ; - - DirEntry *olddir = fi.findOldDirectory(startstamp); - - if (!olddir) - { - /* finished */ - moretodo = false; - continue; - } - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " Checking: " << olddir->path << std::endl; -#endif - - FileEntry fe; - - /* determine the full root path */ - std::string dirpath = olddir->path; - std::string rootdir = RsDirUtil::getRootDir(olddir->path); - std::string remdir = RsDirUtil::removeRootDir(olddir->path); - std::string realroot = locked_findRealRoot(rootdir); - std::string realpath = realroot; - - if (remdir != "") - realpath += "/" + remdir; - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " RealPath: " << realpath << std::endl; -#endif - - /* check for the dir existance */ - librs::util::FolderIterator dirIt(realpath); - if (!dirIt.isValid()) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " Missing Dir: " << realpath << std::endl; - std::cerr << " Root Dir: " << rootdir << std::endl; - std::cerr << " remdir: " << remdir << std::endl; -#endif - if(directoryMap.end() != directoryMap.find(rootdir) && remdir=="") - { -#ifdef FIM_DEBUG - std::cerr << " This is a root directory. Keeping it empty." << std::endl; -#endif - } - else - { - if (!fi.removeOldDirectory(olddir->parent->path, olddir->name, stamp))/* bad directory - delete */ - { - /* bad... drop out of updateCycle() - hopefully the initial cleanup - * will deal with it next time! - otherwise we're in a continual loop - */ - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << "ERROR Failed to Remove: " << olddir->path << std::endl; - } - continue; - } - } - - /* update this dir - as its valid */ - fe.name = olddir->name; - fi.updateDirEntry(olddir->parent->path, fe, stamp); - - /* update the directories and files here */ - std::map::iterator dit; - std::map::iterator fit; - - /* flag existing subdirs as old */ - for(dit = olddir->subdirs.begin(); dit != olddir->subdirs.end(); ++dit) - { - fe.name = (dit->second)->name; - /* set the age as out-of-date so that it gets checked */ - fi.updateDirEntry(olddir->path, fe, 0); - } - - /* now iterate through the directory... - * directories - flags as old, - * files checked to see if they have changed. (rehashed) - */ - - to_hash.push_back(DirContentToHash()) ; - to_hash.back().realpath = realpath ; - to_hash.back().dirpath = dirpath ; - - for(;dirIt.isValid() && isRunning();dirIt.next()) - { - /* check entry type */ - std::string fname; - dirIt.d_name(fname); - std::string fullname = realpath + "/" + fname; -#ifdef FIM_DEBUG - std::cerr << "calling stats on " << fullname <path, fe, 0); - } - else if (S_ISREG(buf.st_mode)) - { - /* is file */ - bool toadd = false; -#ifdef FIM_DEBUG - std::cerr << "Is File: " << fullname << std::endl; -#endif - - fe.name = fname; - fe.size = buf.st_size; - fe.modtime = buf.st_mtime; - - /* check if it exists already */ - fit = olddir->files.find(fname); - if (fit == olddir->files.end()) - { - /* needs to be added */ -#ifdef FIM_DEBUG - std::cerr << "File Missing from List:" << fname << std::endl; -#endif - toadd = true; - } - else - { - /* check size / modtime are the same */ - // if reference_time was not inited, we revert to the old method: we test the saved mod time for the file - // versus the measured mod time. If reference is inited (this is what should always happen) we compare the measured - // mod time with the reference time. - // - if ((fe.size != (fit->second)->size) || (reference_time==0 && fe.modtime != (fit->second)->modtime) || fe.modtime > reference_time) //(fit->second)->modtime)) - { -#ifdef FIM_DEBUG - std::cerr << "File ModTime/Size changed:" << fname << std::endl; - std::cerr << "fe.modtime = " << fe.modtime << std::endl; - std::cerr << "fit.mdtime = " << fit->second->modtime << std::endl; -#endif - toadd = true; - } - else - fe.hash = (fit->second)->hash; /* keep old info */ - } - if (toadd) - { - /* push onto Hash List */ -#ifdef FIM_DEBUG - std::cerr << "Adding to Update List: "; - std::cerr << olddir->path; - std::cerr << fname << std::endl; -#endif - to_hash.back().fentries.push_back(fe); - fiMods = true ; - } - else /* update with new time */ - { -#ifdef FIM_DEBUG - std::cerr << "File Hasn't Changed:" << fname << std::endl; -#endif - fi.updateFileEntry(olddir->path, fe, stamp); - - if(cache_is_new) - hashCache.insert(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash) ; - } - } - } -#ifdef FIM_DEBUG - else - std::cout << "stat error " << errno << std::endl ; -#endif - } - - if(to_hash.back().fentries.empty()) - to_hash.pop_back() ; - - /* now we unlock the lock, and iterate through the - * next files - hashing them, before adding into the system. - */ - /* for safety - blank out data we cannot use (TODO) */ - olddir = NULL; - - /* close directory */ - dirIt.closedir(); - } - - // Now, hash all files at once. - // - if(isRunning() && !to_hash.empty()) - hashFiles(to_hash) ; - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; - - int cleanedCount = 0; - - { /* LOCKED DIRS */ - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - /* finished update cycle - cleanup extra dirs/files that - * have not had their timestamps updated. - */ - - cleanedCount = fi.cleanOldEntries(startstamp) ; - -#ifdef FIM_DEBUG - /* print out the new directory structure */ -// fi.printFileIndex(std::cerr); -#endif - /* now if we have changed things -> restore file/hash it/and - * tell the CacheSource - */ - - if (pendingForceCacheWrite) - { - pendingForceCacheWrite = false; - fiMods = true; - } - - if (fiMods) - { - reference_time = locked_saveFileIndexes(true) ; -#ifdef FIM_DEBUG - std::cerr << "Index saved. New reference time is " << reference_time << std::endl; -#endif - } - - fi.updateHashIndex() ; // update hash map that is used to accelerate search. - fi.updateMaxModTime() ; // Update modification times for proper display. - - mInCheck = false; - - if(useHashCache) - { - hashCache.clean() ; - hashCache.save() ; - } - } - - if (cleanedCount > 0) { - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - } -} - -static std::string friendlyUnit(uint64_t val) -{ - const std::string units[5] = {"B","KB","MB","GB","TB"}; - char buf[50] ; - - double fact = 1.0 ; - - for(unsigned int i=0; i<5; ++i) - if(double(val)/fact < 1024.0) - { - sprintf(buf,"%2.2f",double(val)/fact) ; - return std::string(buf) + " " + units[i]; - } - else - fact *= 1024.0f ; - - sprintf(buf,"%2.2f",double(val)/fact*1024.0f) ; - return std::string(buf) + " TB"; -} - - -void FileIndexMonitor::hashFiles(const std::vector& to_hash) -{ - // Size interval at which we save the file lists - static const uint64_t MAX_SIZE_WITHOUT_SAVING = 10737418240ull ; // 10 GB - - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - time_t stamp = time(NULL); - - // compute total size of files to hash - uint64_t total_size = 0 ; - uint32_t n_files = 0 ; - - for(uint32_t i=0;inotifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; - - std::string real_path = RsDirUtil::makePath(to_hash[i].realpath, fe.name); - - // 1st look into the hash cache if this file already exists. - // - if(useHashCache && hashCache.find(real_path,fe.size,fe.modtime,fe.hash)) - fi.updateFileEntry(to_hash[i].dirpath,fe,stamp); - else if(RsDirUtil::getFileHash(real_path, fe.hash,fe.size, this)) // not found, then hash it. - { - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - /* update fileIndex with new time */ - /* update with new time */ - - // Check again that the hashed file hasn't been modified since the beginning of the hashing process. - // If so, drop it. - // - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(real_path, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(real_path.c_str(), &buf)) -#endif - { - if(buf.st_mtime != fe.modtime) - std::cerr << "File " << real_path << " has been modified while being hashed. It will be dropped to avoid data inconsistency" << std::endl; - else - { - fi.updateFileEntry(to_hash[i].dirpath,fe,stamp); - - hashed_size += to_hash[i].fentries[j].size ; - - // Update the hash cache - // - if(useHashCache) - hashCache.insert(real_path,fe.size,fe.modtime,fe.hash) ; - } - } - } - else - std::cerr << "Failed to Hash File " << fe.name << std::endl; - - size += to_hash[i].fentries[j].size ; - - // don't hit the disk too hard! - usleep(10*1000); // 10 msec - - // Save the hashing result every 60 seconds, so has to save what is already hashed. -#ifdef FIM_DEBUG - std::cerr << "size - last_save_size = " << hashed_size - last_save_size << ", max=" << MAX_SIZE_WITHOUT_SAVING << std::endl ; -#endif - - if(hashed_size > last_save_size + MAX_SIZE_WITHOUT_SAVING) - { - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_SAVE_FILE_INDEX, "") ; - - //Waiting 1 sec before pass to other hash ??is it really needed?? - usleep(1*1000*1000); // 1 sec - - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - fi.updateHashIndex() ; - FileIndexMonitor::locked_saveFileIndexes(true) ; - last_save_size = hashed_size ; - - if(useHashCache) - hashCache.save() ; - } - - // check if thread is running - running = isRunning(); - } - - fi.updateHashIndex() ; - - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - - -time_t FileIndexMonitor::locked_saveFileIndexes(bool update_cache) -{ - /* store to the cacheDirectory */ - - std::string path = getCacheDir(); - - // Multiple files are saved: for every kind of peers, the set of browsable files will be different. A specific file is - // prepared for all situations, and shared by all peers having the same situation. - // - // A complete file collection is also saved, and serves as memory for the FileIndexMonitor system. - // -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() FileIndex modified ... updating" << std::endl; -#endif - // Make for each peer the list of forbidden shared directories. Make a separate cache file for each different set. - // To figure out which sets are different, we index them by the set of forbidden indexes from the directory list. - // This is probably a bit costly, but we can't suppose that the number of shared directories is bounded. - // - std::list all_friend_ids ; - rsPeers->getFriendList(all_friend_ids); - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle(): got list of all friends." << std::endl ; - for(std::list::const_iterator it(all_friend_ids.begin());it!=all_friend_ids.end();++it) - std::cerr << " " << *it << std::endl; -#endif - - std::map, std::set > peers_per_directory_combination ; - - for(std::list::const_iterator it(all_friend_ids.begin());it!=all_friend_ids.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "About to save, with the following restrictions:" << std::endl ; - std::cerr << "Peer : " << *it << std::endl; -#endif - - std::set forbidden_dirs ; - for(std::map::const_iterator dit(directoryMap.begin());dit!=directoryMap.end();++dit) - { -#ifdef FIM_DEBUG - std::cerr << " dir=" << dit->first << ", " ; - std::cerr << "parent groups: " ; - for(std::list::const_iterator mit(dit->second.parent_groups.begin());mit!=dit->second.parent_groups.end();++mit) - std::cerr << (*mit) << ", " ; - std::cerr << std::endl;; -#endif - - FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(*it,dit->second.shareflags,dit->second.parent_groups) ; - - if(!(permission_flags & RS_FILE_HINTS_BROWSABLE)) - { -#ifdef FIM_DEBUG - std::cerr << "forbidden" << std::endl; -#endif - forbidden_dirs.insert(dit->first) ; - } -#ifdef FIM_DEBUG - else - std::cerr << "autorized" << std::endl; -#endif - } - - peers_per_directory_combination[forbidden_dirs].insert(*it) ; - } - RsPeerId ownId = rsPeers->getOwnId() ; - peers_per_directory_combination[std::set()].insert(ownId) ; // add full configuration to self, i.e. no forbidden directories. - - int n=0 ; - time_t now = time(NULL) ; - time_t mod_time = 0 ; - - for(std::map, std::set >::const_iterator it(peers_per_directory_combination.begin()); - it!=peers_per_directory_combination.end();++it,++n) - { - std::string tmpname_browsable; - - if(it->first.empty()) - rs_sprintf(tmpname_browsable, "fc-own-%ld.rsfc",now,n); - else - rs_sprintf(tmpname_browsable, "fc-own-%ld.%04d",now,n); - - std::string fname_browsable = path + "/" + tmpname_browsable; - -#ifdef FIM_DEBUG - std::cerr << "Sending file list: " << std::endl; - std::cerr << " filename : " << tmpname_browsable << std::endl; - std::cerr << " to peers : " << std::endl; - for(std::set::const_iterator itt(it->second.begin());itt!= it->second.end();++itt) - std::cerr << " " << *itt << std::endl; - std::cerr << " forbidden : " << std::endl; - for(std::set::const_iterator itt(it->first.begin());itt!= it->first.end();++itt) - std::cerr << " " << *itt << std::endl; -#endif - - RsFileHash hash ; - uint64_t size ; - -#ifdef FIM_DEBUG - std::cerr << "writing file " << fname_browsable << std::endl; -#endif - fi.saveIndex(fname_browsable, hash, size,it->first); // save only browsable files - - if(size > 0) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() saved with hash:" << hash << std::endl; -#endif - - /* should clean up the previous cache.... */ - - /* flag as new info */ - RsCacheData data; - data.pid = fi.root->id; - data.cid.type = getCacheType(); - data.cid.subid = n; - data.path = path; - data.name = tmpname_browsable; - data.hash = hash; - data.size = size; - data.recvd = time(NULL); - - for(std::set::const_iterator ff(it->second.begin());ff!=it->second.end();++ff) - _cache_items_per_peer[*ff] = data ; - - data.cid.subid = n; - - if(update_cache) - updateCache(data,it->second); - } - - if(it->first.empty()) - { - // Computes the reference time. - // - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(fname_browsable, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(fname_browsable.c_str(), &buf)) -#endif - { - mod_time = buf.st_mtime ; - } - else - { - std::cerr << "(EE) Error. Cannot get the proper modification time for file " << fname_browsable << " errno=" << errno << std::endl; - mod_time = 0 ; - } - } - } - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() called updateCache()"; - std::cerr << std::endl; -#endif - return mod_time ; -} - -bool FileIndexMonitor::cachesAvailable(const RsPeerId &pid,std::map &ids) -{ - lockData() ; -#ifdef FIM_DEBUG - std::cerr << "In cachesAvailable..." << std::endl; -#endif - - // Go through the list of saved cache items for that particular peer. - // - ids.clear() ; - std::map::const_iterator it(_cache_items_per_peer.find(pid)) ; - RsPeerId ownId = rsPeers->getOwnId(); - - if(it != _cache_items_per_peer.end()) - { - ids[it->second.cid] = it->second ; - - if(pid != ownId) - ids[it->second.cid].cid.subid = 0 ; // Force subid to be 0, so that it's - // not going to be mixed up at the client with other files received if the - // subid changes for that peer. - // -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor: caches available for peer " << pid << ": " << it->second.name << std::endl ; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << "No cache item for peer " << pid << std::endl; -#endif - - unlockData() ; - - return true ; -} - -void FileIndexMonitor::updateShareFlags(const SharedDirInfo& dir) -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - bool fimods = false ; -#ifdef FIM_DEBUG - std::cerr << "*** FileIndexMonitor: Updating flags for " << dir.filename << " to " << dir.shareflags << std::endl ; -#endif - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - for(std::list::iterator it(pendingDirList.begin());it!=pendingDirList.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "** testing pending dir " << (*it).filename << std::endl ; -#endif - if((*it).filename == dir.filename) - { -#ifdef FIM_DEBUG - std::cerr << "** Updating to " << (*it).shareflags << "!!" << std::endl ; -#endif - (*it).shareflags = dir.shareflags ; - (*it).parent_groups = dir.parent_groups ; - break ; - } - } - - for(std::map::iterator it(directoryMap.begin());it!=directoryMap.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "** testing " << (*it).second.filename << std::endl ; -#endif - if((*it).second.filename == dir.filename) - { -#ifdef FIM_DEBUG - std::cerr << "** Updating from " << it->second.shareflags << "!!" << std::endl ; -#endif - (*it).second.shareflags = dir.shareflags ; - (*it).second.parent_groups = dir.parent_groups ; - fimods = true ; - break ; - } - } - } - if(fimods) - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - locked_saveFileIndexes(true) ; - } - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - /* interface */ -void FileIndexMonitor::setSharedDirectories(const std::list& dirs) -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - std::list checkeddirs; - - std::list::const_iterator it; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setSharedDirectories() :\n"; -#endif - - for(it = dirs.begin(); it != dirs.end(); ++it) - { - -#ifdef FIM_DEBUG - std::cerr << "\t" << (*it).filename; - std::cerr << std::endl; -#endif - - checkeddirs.push_back(*it); - -#ifdef REMOVED_CODE - // this code has been removed because it prevents unmounted shared directories to stay in the list of shared files. It's better - // to keep them showing empty than removing them completely. - // - /* check if dir exists before adding in */ -// std::string path = (*it).filename; -// if (!RsDirUtil::checkDirectory(path)) -// { -//#ifdef FIM_DEBUG -// std::cerr << "FileIndexMonitor::setSharedDirectories()"; -// std::cerr << " Ignoring NonExistant SharedDir: " << path << std::endl; -//#endif -// } -// else -// { -// checkeddirs.push_back(*it); -// } -#endif - } - - { - RsStackMutex stack(fiMutex) ;/* LOCKED DIRS */ - - pendingDirs = true; - pendingDirList = checkeddirs; - } - - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - - /* interface */ -void FileIndexMonitor::getSharedDirectories(std::list &dirs) -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - /* must provide pendingDirs, as other parts depend on instanteous response */ - if (pendingDirs) - dirs = pendingDirList; - else - { - /* get actual list (not pending stuff) */ - std::map::const_iterator it; - - for(it = directoryMap.begin(); it != directoryMap.end(); ++it) - dirs.push_back(it->second) ; - } -} - - - /* interface */ -void FileIndexMonitor::forceDirectoryCheck() -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - if (!mInCheck) - mForceCheck = true; -} - - -void FileIndexMonitor::forceDirListsRebuildAndSend() -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - locked_saveFileIndexes(true) ; -} - - /* interface */ -bool FileIndexMonitor::inDirectoryCheck() -{ - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - return mInCheck; -} - - -bool FileIndexMonitor::internal_setSharedDirectories() -{ - bool changed = false; - - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - if (!pendingDirs) - { - if (mForceCheck) - { - mForceCheck = false; - return true; - } - - return false; - } - - mForceCheck = false; - pendingDirs = false; - pendingForceCacheWrite = true; - - /* clear old directories */ - directoryMap.clear(); - - /* iterate through the directories */ - std::list::iterator it; - std::map::const_iterator cit; - for(it = pendingDirList.begin(); it != pendingDirList.end(); ++it) - { - /* get the head directory */ - std::string root_dir = (*it).filename; - std::string top_dir = it->virtualname; - if (top_dir.empty()) { - top_dir = RsDirUtil::getTopDir(root_dir); - } - - /* if unique -> add, else add modifier */ - bool unique = false; - for(int i = 0; !unique; ++i) - { - std::string tst_dir = top_dir; - if (i > 0) - { - rs_sprintf_append(tst_dir, "-%d", i); - } - if (directoryMap.end()== (cit=directoryMap.find(tst_dir))) - { - unique = true; - /* store calculated name */ - it->virtualname = tst_dir; - /* add it! */ - directoryMap[tst_dir.c_str()] = *it; -#ifdef FIM_DEBUG - std::cerr << "Added [" << tst_dir << "] => " << root_dir << std::endl; -#endif - } - } - } - - pendingDirList.clear(); - - /* now we've decided on the 'root' dirs set them to the - * fileIndex - */ - std::list topdirs; - for(cit = directoryMap.begin(); cit != directoryMap.end(); ++cit) - { - topdirs.push_back(cit->first); - } - - if (fi.setRootDirectories(topdirs, 0) > 0) - { - changed = true; - } - - locked_saveFileIndexes(true) ; - } - - if (changed) - { - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - } - - return true; -} - -/* lookup directory function */ -std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const -{ - /**** MUST ALREADY BE LOCKED ****/ - std::string realroot = ""; - - std::map::const_iterator cit; - if (directoryMap.end()== (cit=directoryMap.find(rootdir))) - { - std::cerr << "FileIndexMonitor::locked_findRealRoot() Invalid RootDir: "; - std::cerr << rootdir << std::endl; - } - else - { - realroot = cit->second.filename; - } - - return realroot; -} - -int FileIndexMonitor::RequestDirDetails(const std::string& path, DirDetails& details) const -{ - /* lock it up */ - RsStackMutex mutex(fiMutex) ; - return fi.extractData(path,details) ; -} - -uint32_t FileIndexMonitor::getType(void *ref) const -{ - RsStackMutex mutex(fiMutex) ; - - return fi.getType(ref) ; -} -int FileIndexMonitor::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const -{ - /* remove unused parameter warnings */ - (void) flags; - - RsStackMutex mutex(fiMutex) ; - -#ifdef FIM_DEBUG2 - std::cerr << "FileIndexMonitor::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl; -#endif - - /* root case */ - -#ifdef FIM_DEBUG2 - fi.root->checkParentPointers(); -#endif - - // If ref is NULL, we build a root node - - if (ref == NULL) - { -#ifdef FI_DEBUG2 - std::cerr << "FileIndex::RequestDirDetails() ref=NULL (root)" << std::endl; -#endif - /* local only */ - DirStub stub; - stub.type = DIR_TYPE_PERSON; - stub.name = fi.root->name; - stub.ref = fi.root; - details.children.push_back(stub); - details.count = 1; - - details.parent = NULL; - details.prow = -1; - details.ref = NULL; - details.type = DIR_TYPE_ROOT; - details.name = "root"; - details.hash.clear() ; - details.path = "root"; - details.age = 0; - details.flags.clear() ; - details.min_age = 0 ; - - return true ; - } - - bool b = FileIndex::extractData(ref,details) ; - - if(!b) - return false ; - - // look for the top level and setup flags accordingly - // The top level directory is the first dir in parents for which - // dir->parent->parent == NULL - - if(ref != NULL) - { - FileEntry *file = (FileEntry *) ref; - locked_findShareFlagsAndParentGroups(file,details.flags,details.parent_groups) ; - } - return true ; -} - -void FileIndexMonitor::locked_findShareFlagsAndParentGroups(FileEntry *file,FileStorageFlags& flags,std::list& parent_groups) const -{ - flags.clear() ; - static const FileStorageFlags PERMISSION_MASK = DIR_FLAGS_BROWSABLE_OTHERS | DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_GROUPS | DIR_FLAGS_NETWORK_WIDE_GROUPS ; - - DirEntry *dir = dynamic_cast(file) ; - if(dir == NULL) - dir = dynamic_cast(file->parent) ; - - if(dir != NULL && dir->parent != NULL) - while(dir->parent->parent != NULL) - dir = dir->parent ; - - if(dir != NULL && dir->parent != NULL) - { -#ifdef FIM_DEBUG2 - std::cerr << "FileIndexMonitor::RequestDirDetails: top parent name=" << dir->name << std::endl ; -#endif - std::map::const_iterator it = directoryMap.find(dir->name) ; - - if(it == directoryMap.end()) - std::cerr << "*********** ERROR *********** In " << __PRETTY_FUNCTION__ << std::endl ; - else - { - flags = it->second.shareflags ; - flags &= PERMISSION_MASK ; - flags &= ~DIR_FLAGS_NETWORK_WIDE_GROUPS ; // Disabling this flag for now, because it has inconsistent effects. - parent_groups = it->second.parent_groups ; - } -#ifdef FIM_DEBUG2 - std::cerr << "flags = " << flags << std::endl ; -#endif - } -} - - diff --git a/libretroshare/src/dbase/fimonitor.h b/libretroshare/src/dbase/fimonitor.h deleted file mode 100644 index 2cdddb4af..000000000 --- a/libretroshare/src/dbase/fimonitor.h +++ /dev/null @@ -1,220 +0,0 @@ -/* - * RetroShare FileCache Module: fimonitor.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef FILE_INDEX_MONITOR_H -#define FILE_INDEX_MONITOR_H - -#include "dbase/cachestrapper.h" -#include "dbase/findex.h" -#include "util/rsthreads.h" -#include "retroshare/rsfiles.h" - -/****************************************************************************************** - * The Local Monitoring Class: FileIndexMonitor. - * - * This periodically scans the directory tree, and updates any modified directories/files. - * - *****************************************************************************************/ - -/****************************************************************************************** - STILL TODO: - - (1) Implement Hash function. - -bool FileIndexMonitor::hashFile(std::string path, FileEntry &fi); - - (2) Add Shared directory controls to Monitor. - -int FileIndexMonitor::addSharedDirectory(std::path); -int FileIndexMonitor::removeSharedDirectory(std::path); -std::string FileIndexMonitor::findRealRoot(std::string base); - - These must be split into / and the mapping saved. - eg: addSharedDirectory("c:/home/stuff/dir1") --> "c:/home/stuff" <-> "dir1" - This code has been written already, and can just be moved over. - - FOR LATER: - (2) Port File/Directory lookup code to windoze. (or compile dirent.c under windoze) - (3) Add Load/Store interface to FileIndexMonitor. (later) - (4) Integrate with real Thread/Mutex code (last thing to do) - -******************************************************************************************/ - - -class DirContentToHash -{ - public: - std::vector fentries ; - - std::string realpath ; - std::string dirpath ; -}; - -class HashCache -{ - public: - HashCache(const std::string& save_file_name) ; - - void save() ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - void clean() ; - - typedef struct - { - uint64_t size ; - uint64_t time_stamp ; - uint64_t modf_stamp ; - RsFileHash hash ; - } HashCacheInfo ; - - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } - private: - uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries - std::map _files ; - std::string _path ; - bool _changed ; -}; - -/****************************************************************************************** - * FileIndexMonitor - *****************************************************************************************/ - -class FileIndexMonitor: public CacheSource, public RsTickingThread -{ - public: - FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid, const std::string& config_dir); - virtual ~FileIndexMonitor(); - - /* external interface for filetransfer */ - bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; - - int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) ; - int SearchBoolExp(Expression *exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - - int filterResults(std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - - - /* external interface for local access to files */ - bool convertSharedFilePath(std::string path, std::string &fullpath); - - - /* Interacting with CacheSource */ - /* overloaded from CacheSource */ - virtual bool loadLocalCache(const RsCacheData &data); /* called with stored data */ - bool updateCache(const RsCacheData &data,const std::set& dest_peers); /* we call when we have a new cache for others */ - - - /* the FileIndexMonitor inner workings */ - //virtual void run(std::string& currentJob); /* overloaded from RsThread */ - //void updateCycle(std::string& currentJob); - virtual void data_tick(); /* overloaded from RsThread */ - void updateCycle(); - - // Interface for browsing dir hirarchy - int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ; - uint32_t getType(void*) const ; - int RequestDirDetails(const std::string& path, DirDetails &details) const ; - - // set/update shared directories - virtual void setSharedDirectories(const std::list& dirs); - void getSharedDirectories(std::list& dirs); - void updateShareFlags(const SharedDirInfo& info) ; - - void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed - void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files. - bool inDirectoryCheck(); - - /* util fns */ - - // from CacheSource - virtual bool cachesAvailable(const RsPeerId& pid, std::map &ids) ; - - protected: - // Sets/gets the duration period within which already hashed files are remembered. - // - void setRememberHashFilesDuration(uint32_t days) ; - uint32_t rememberHashFilesDuration() const ; - void setRememberHashFiles(bool) ; - bool rememberHashFiles() ; - // Remove any memory of formerly hashed files that are not shared anymore - void clearHashFiles() ; - void setPeriod(int insecs); - int getPeriod() const; - - bool autoCheckEnabled() const ; - - private: - /* the mutex should be locked before calling these 3. */ - - // Saves file indexs and update the cache. Returns the name of the main - // file index, which becomes the new reference file for mod times. - // - time_t locked_saveFileIndexes(bool update_cache) ; - - // Finds the share flags associated with this file entry. - void locked_findShareFlagsAndParentGroups(FileEntry *fe, FileStorageFlags& shareflags, std::list &parent_groups) const ; - - std::string locked_findRealRoot(std::string base) const; - - void hashFiles(const std::vector& to_hash) ; - bool hashFile(std::string path, FileEntry &fi); /* To Implement */ - - /* data */ - - mutable RsMutex fiMutex; - - FileIndex fi; - - int updatePeriod; - std::map directoryMap; /* used by findRealRoot */ - - /* flags to kick - if we were busy or sleeping */ - bool pendingDirs; - bool pendingForceCacheWrite; - - /* flags to force Check, to tell if we're in check */ - bool mForceCheck; - bool mInCheck; - - std::list pendingDirList; - bool internal_setSharedDirectories(); - - HashCache hashCache ; - bool useHashCache ; - - std::map _cache_items_per_peer ; // stored the cache items to be sent to each peer. - - // This file is the location of the current index file. When checking for new files, we compare the modification time - // of this file to the mod time of the files on the disk. This allows to now account for time-shift in the computer. - // - time_t reference_time ; -}; - - -#endif - - diff --git a/libretroshare/src/dbase/findex.cc b/libretroshare/src/dbase/findex.cc deleted file mode 100644 index 17b5dba0b..000000000 --- a/libretroshare/src/dbase/findex.cc +++ /dev/null @@ -1,1516 +0,0 @@ -/* - * RetroShare FileCache Module: findex.cc - * - * Copyright 2004-2007 by Robert Fernie, Kefei Zhou. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include -#include -#include "dbase/findex.h" -#include "retroshare/rsexpr.h" -#include "util/rsdir.h" -#include "util/rsstring.h" -#include "util/rscompress.h" - -#include -#include -#include -#include -#ifdef __MACH__ -#include -#else -#include -#endif -#include -#include -#include -#include -#include - -#include -#include - -// This char is used to separate fields in the file list cache. It is supposed to be -// sufficiently safe on all systems. -// -static const char FILE_CACHE_SEPARATOR_CHAR = '|' ; - -/**** - * #define FI_DEBUG 1 - * #define FI_DEBUG_ALL 1 - ****/ - -static RsMutex FIndexPtrMtx("FIndexPtrMtx") ; -#ifdef __MACH__ -std::unordered_set FileIndex::_pointers ; -#else -std::tr1::unordered_set FileIndex::_pointers ; -#endif -void FileIndex::registerEntry(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - _pointers.insert(p) ; -} -void FileIndex::unregisterEntry(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - _pointers.erase(p) ; -} -bool FileIndex::isValid(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - return _pointers.find(p) != _pointers.end() ; -} - -DirEntry::~DirEntry() -{ - /* cleanup */ - std::map::iterator dit; - std::map::iterator fit; - - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { - FileIndex::unregisterEntry((void*)dit->second) ; - delete (dit->second); - } - subdirs.clear(); - - for(fit = files.begin(); fit != files.end(); ++fit) - { - FileIndex::unregisterEntry((void*)fit->second) ; - delete (fit->second); - } - files.clear(); -} - - -int DirEntry::checkParentPointers() -{ -#ifdef FI_DEBUG - updateChildRows(); - std::map::iterator dit; - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { - /* debug check */ - (dit->second)->checkParentPointers(); - } -#endif - return 1; -} - - - -int DirEntry::updateChildRows() -{ - /* iterate through children and set row (parent should be good) */ - std::map::iterator dit; - std::map::iterator fit; - int i = 0; - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { -#ifdef FI_DEBUG - /* debug check */ - if ((dit->second)->parent != this) - { - std::cerr << "DirEntry::updateChildRows()"; - std::cerr << "****WARNING subdir Parent pointer invalid!"; - std::cerr << std::endl; - (dit->second)->parent = this; - } -#endif - (dit->second)->row = i++; - } - - for(fit = files.begin(); fit != files.end(); ++fit) - { -#ifdef FI_DEBUG - /* debug check */ - if ((fit->second)->parent != this) - { - std::cerr << "DirEntry::updateChildRows()"; - std::cerr << "****WARNING file Parent pointer invalid!"; - std::cerr << std::endl; - (fit->second)->parent = this; - } -#endif - (fit->second)->row = i++; - } - return 1; -} - - -int DirEntry::removeDir(const std::string& name) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() != (it = subdirs.find(name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeDir() Cleaning up dir: " << name; - std::cerr << std::endl; -#endif - ndir = (it->second); - - subdirs.erase(it); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - /* update row counters */ - updateChildRows(); - return 1; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeDir() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - -int DirEntry::removeFile(const std::string& name) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - FileEntry *nfile = NULL; - - if (files.end() != (it = files.find(name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeFile() removing File: " << name; - std::cerr << std::endl; -#endif - nfile = (it->second); - - files.erase(it); - FileIndex::unregisterEntry((void*)nfile) ; - delete nfile; - /* update row counters */ - updateChildRows(); - return 1; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeFile() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - - - -int DirEntry::removeOldDir(const std::string& name, time_t old) -{ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() != (it = subdirs.find(name))) - { - ndir = (it->second); - if (ndir->updtime < old) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeOldDir() Removing Old dir: " << name; - std::cerr << std::endl; -#endif - subdirs.erase(it); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - - /* update row counters */ - updateChildRows(); - return 1; - } -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeOldDir() Keeping UptoDate dir: " << name; - std::cerr << std::endl; -#endif - return 0; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeDir() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - - - -int DirEntry::removeOldEntries(time_t old, bool recursive) -{ - /* remove old dirs from our lists -> then do children and files */ - - /* get all dirs with old time */ - std::list removeList; - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - if ((it->second)->updtime < old) - { - removeList.push_back(it->second); - } - } - - int count = removeList.size(); - - /* now remove the old entries */ - std::list::iterator rit; - for(rit = removeList.begin(); rit != removeList.end(); ++rit) - { - removeDir((*rit)->name); - } - - if (recursive) - { - /* now handle children */ - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - count += (it->second)->removeOldEntries(old, recursive); - } - } - - /* now handle files similarly */ - std::list removeFileList; - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - if ((fit->second)->updtime < old) - { - removeFileList.push_back(fit->second); - } - } - - count += removeFileList.size(); - - /* now remove the old entries */ - std::list::iterator rfit; - for(rfit = removeFileList.begin(); rfit != removeFileList.end(); ++rfit) - { - removeFile((*rfit)->name); - } - - return count; -} - - -DirEntry *DirEntry::findOldDirectory(time_t old) -{ - /* check if one of our directories is old ... - */ - - /* get all dirs with old time */ - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - if ((it->second)->updtime < old) - { - return (it->second); - } - } - - /* - * else check chlidren. - */ - - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - DirEntry *olddir = (it->second)->findOldDirectory(old); - if (olddir) - { - return olddir; - } - } - - return NULL; -} - - -DirEntry *DirEntry::findDirectory(const std::string& fpath) -{ - std::string nextdir = RsDirUtil::getRootDir(fpath); - std::map::iterator it; - if (subdirs.end() == (it = subdirs.find(nextdir))) - { -#ifdef FI_DEBUG - std::cerr << "DirEntry::findDirectory() Missing subdir:"; - std::cerr << "\"" << nextdir << "\""; - std::cerr << std::endl; -#endif - return NULL; - } - - std::string rempath = RsDirUtil::removeRootDir(fpath); - if (rempath == "") - { - return it->second; - } - - - // Adding more lenient directory look up. - // returns lower directory to fpath is a FILE. - DirEntry *subdir = (it->second)->findDirectory(rempath); - if (subdir) - return subdir; - else - return it->second; -} - - -int DirEntry::updateDirectories(const std::string& fpath, int new_pop, int new_modtime) -{ - int ret = 1; - if (path != "") /* if not there -> continue down tree */ - { - std::string nextdir = RsDirUtil::getRootDir(fpath); - std::map::iterator it; - if (subdirs.end() == (it = subdirs.find(nextdir))) - { - return 0; - } - - std::string rempath = RsDirUtil::removeRootDir(fpath); - ret = (it->second)->updateDirectories(rempath, new_pop, new_modtime); - } - - if (ret) /* if full path is okay -> update and return ok */ - { - /* this is assumes that pop always increases! */ - if (new_pop > pop) - { - pop = new_pop; - } - if (new_modtime > modtime) - { - modtime = new_modtime; - } - } - return ret; -} - -DirEntry *DirEntry::updateDir(const FileEntry& fe, time_t utime) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() == (it = subdirs.find(fe.name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateDir() Adding Entry"; - std::cerr << std::endl; -#endif - ndir = new DirEntry(); - FileIndex::registerEntry((void*)ndir) ; - ndir -> parent = this; - ndir -> path = path + "/" + fe.name; - ndir -> name = fe.name; - ndir -> pop = 0; - ndir -> modtime = 0; - ndir -> updtime = utime; - - subdirs[fe.name] = ndir; - - /* update row counters */ - updateChildRows(); - return ndir; - } - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateDir() Updating Entry"; - std::cerr << std::endl; -#endif - - /* update utime */ - ndir = (it->second); - ndir->updtime = utime; - - return ndir; -} - - -FileEntry *DirEntry::updateFile(const FileEntry& fe, time_t utime) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - FileEntry *nfile = NULL; - if (files.end() == (it = files.find(fe.name))) - { - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateFile() Adding Entry"; - std::cerr << std::endl; -#endif - - nfile = new FileEntry(); - FileIndex::registerEntry((void*)nfile) ; - nfile -> parent = this; - nfile -> name = fe.name; - nfile -> hash = fe.hash; - nfile -> size = fe.size; - nfile -> pop = 0; - nfile -> modtime = fe.modtime; - nfile -> updtime = utime; - - files[fe.name] = nfile; - - /* update row counters */ - updateChildRows(); - return nfile; - } - - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateFile() Updating Entry"; - std::cerr << std::endl; -#endif - - - /* update utime */ - nfile = (it->second); - nfile -> parent = this; - nfile -> name = fe.name; - nfile -> hash = fe.hash; - nfile -> size = fe.size; - nfile -> modtime = fe.modtime; - nfile -> updtime = utime; - //nfile -> pop = 0; // not handled here. - - return nfile; -} - - -int FileEntry::print(std::string &out) -{ - /* print this dir, then subdirs, then files */ - - rs_sprintf_append(out, "file %03d [%ld/%ld] : ", row, updtime, modtime); - - if (parent) - out += parent->path; - else - out += "[MISSING PARENT]"; - - rs_sprintf_append(out, " %s [ s: %lld ] ==> [ %s ]\n", name.c_str(), size, hash.toStdString().c_str()); - - return 1; -} - - -int DirEntry::print(std::string &out) -{ - /* print this dir, then subdirs, then files */ - rs_sprintf_append(out, "dir %03d [%ld] : %s\n", row, updtime, path.c_str()); - - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - (it->second)->print(out); - } - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - (fit->second)->print(out); - } - return 1; -} - -FileIndex::FileIndex(const RsPeerId& pid) -{ - root = new PersonEntry(pid); - registerEntry(root) ; - _file_hashes.clear() ; -} - -FileIndex::~FileIndex() -{ - FileIndex::unregisterEntry((void*)root) ; - delete root; -} - -int FileIndex::setRootDirectories(const std::list &inlist, time_t updtime) -{ - /* set update time to zero */ - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - (it->second)->updtime = 0; - } - - std::list::const_iterator ait; - FileEntry fe; - time_t utime = 1; - for(ait = inlist.begin(); ait != inlist.end(); ++ait) - { - fe.name = (*ait); - - /* see if it exists */ - root->updateDir(fe, utime); - } - - /* remove all dirs with zero time (non recursive) */ - int cleanedCount = root->removeOldEntries(utime, false); - - /* now flag remaining directories with correct update time */ - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - (it->second)->updtime = updtime; - } - - // update file hash index. - - updateHashIndex() ; - - return cleanedCount; -} - -void FileIndex::updateHashIndex() -{ - _file_hashes.clear() ; - recursUpdateHashIndex(root) ; -} - -void FileIndex::recursUpdateHashIndex(DirEntry *dir) -{ - for(std::map::iterator it(dir->subdirs.begin());it!=dir->subdirs.end();++it) - recursUpdateHashIndex(it->second) ; - - for(std::map::iterator it(dir->files.begin());it!=dir->files.end();++it) - _file_hashes[it->second->hash] = it->second ; -} - -void FileIndex::updateMaxModTime() -{ - RecursUpdateMaxModTime(root) ; -} -void FileIndex::RecursUpdateMaxModTime(DirEntry *dir) -{ - time_t max_mod_t = 0 ; - - for(std::map::iterator it(dir->subdirs.begin());it!=dir->subdirs.end();++it) - { - RecursUpdateMaxModTime(it->second) ; - max_mod_t = std::max(max_mod_t, it->second->most_recent_time) ; - } - for(std::map::iterator it(dir->files.begin());it!=dir->files.end();++it) - max_mod_t = std::max(max_mod_t, it->second->modtime) ; - - dir->most_recent_time = max_mod_t ; -} - -int FileIndex::getRootDirectories(std::list &outlist) -{ - /* set update time to zero */ - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - outlist.push_back(it->first); - } - return 1; -} - -/* update (index building) */ -DirEntry *FileIndex::updateDirEntry(const std::string& fpath, const FileEntry& fe, time_t utime) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::updateDirEntry() Path: \""; - std::cerr << fpath << "\"" << " + \"" << fe.name << "\""; - std::cerr << std::endl; -#endif - DirEntry *parent = NULL; - if (fpath == "") - { - parent = root; - } - else - { - parent = root->findDirectory(fpath); - } - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::updateDirEntry() NULL parent"; - std::cerr << std::endl; -#endif - return NULL; - } - return parent -> updateDir(fe, utime); -} - - -FileEntry *FileIndex::updateFileEntry(const std::string& fpath, const FileEntry& fe, time_t utime) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::updateFileEntry() Path: \""; - std::cerr << fpath << "\"" << " + \"" << fe.name << "\""; - std::cerr << std::endl; -#endif - DirEntry *parent = root->findDirectory(fpath); - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::updateFileEntry() NULL parent"; - std::cerr << std::endl; -#endif - return NULL; - } - - return parent -> updateFile(fe, utime); -} - - -DirEntry *FileIndex::findOldDirectory(time_t old) /* finds directories older than old */ -{ - DirEntry *olddir = root->findOldDirectory(old); -#ifdef FI_DEBUG - - std::cerr << "FileIndex::findOldDirectory(" << old << ") -> "; - if (olddir) - { - std::cerr << olddir->path; - } - else - { - std::cerr << "NONE"; - } - std::cerr << std::endl; - -#endif - return olddir; -} - -int FileIndex::removeOldDirectory(const std::string& fpath, const std::string& name, time_t old) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::removeOldDirectory() Path: \""; - std::cerr << fpath << "\"" << " + \"" << name << "\""; - std::cerr << std::endl; -#endif - - /* because of this find - we cannot get a child of - * root (which is what we want!) - */ - - DirEntry *parent = root->findDirectory(fpath); - /* for root directory case ... no subdir. */ - if (fpath == "") - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::removeOldDirectory() removing a root dir"; - std::cerr << std::endl; -#endif - parent = root; - } - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::removeOldDirectory() NULL parent"; - std::cerr << std::endl; -#endif - return 0; - } - return parent -> removeOldDir(name, old); -} - - -int FileIndex::cleanOldEntries(time_t old) /* removes entries older than old */ -{ - int count = 0; - - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - count += (it->second)->removeOldEntries(old, true); - } - return count; -} - - - -int FileIndex::printFileIndex(std::ostream &out) -{ - std::string sout ; - printFileIndex(sout) ; - out << sout << std::endl; - - return 1 ; -} -int FileIndex::printFileIndex(std::string &out) -{ - out += "FileIndex::printFileIndex()\n"; - root->print(out); - return 1; -} - -int FileIndex::loadIndex(const std::string& filename, const RsFileHash& expectedHash, uint64_t size) -{ - FILE *file = RsDirUtil::rs_fopen(filename.c_str(),"rb") ; - - if (!file) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::loadIndex error opening file: " << filename; - std::cerr << std::endl; -#endif - return 0; - } - - std::string s ; - - { - /* load file into memory, close file */ - RsTemporaryMemory compressed_data(size) ; - - if(!compressed_data) - { - std::cerr << "FileIndex::loadIndex(): can't allocate memory for " << size << " bytes." << std::endl; - fclose(file); - return 0; - } - uint64_t bytesread = 0 ; - if(size != (bytesread = fread(compressed_data,1,size,file))) - { - std::cerr << "FileIndex::loadIndex(): can't read " << size << " bytes from file " << filename << ". Only " << bytesread << " actually read." << std::endl; - fclose(file); - return 0; - } - fclose(file); - - RsFileHash tmpout = RsDirUtil::sha1sum((unsigned char *)(compressed_data),size); - - // /* calculate hash */ - // unsigned char sha_buf[SHA_DIGEST_LENGTH]; - // SHA_CTX *sha_ctx = new SHA_CTX; - // SHA1_Init(sha_ctx); - // SHA1_Update(sha_ctx, s.c_str(), s.length()); - // SHA1_Final(&sha_buf[0], sha_ctx); - // delete sha_ctx; - // - // std::string tmpout; - // for(int i = 0; i < SHA_DIGEST_LENGTH; ++i) - // { - // rs_sprintf_append(tmpout, "%02x", (unsigned int) (sha_buf[i])); - // } - - if (!expectedHash.isNull() && expectedHash != tmpout) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::loadIndex expected hash does not match" << std::endl; - std::cerr << "Expected hash: " << expectedHash << std::endl; - std::cerr << "Hash found: " << tmpout << std::endl; -#endif - return 0; - } - // now uncompress the string - // - - uint8_t *uncompressed_data = NULL ; - unsigned int uncompressed_data_size = 0 ; - - if(!RsCompress::uncompress_memory_chunk(compressed_data,size,uncompressed_data,uncompressed_data_size)) - { - std::cerr << "FileIndex::loadIndex() Decompression failed! Fileindex can't be read." << std::endl; - free(uncompressed_data); - return 0 ; - } - s = std::string((char *)uncompressed_data,uncompressed_data_size) ; - - std::cerr << " file = " << filename << std::endl; - std::cerr << " uncompressed size = " << uncompressed_data_size << std::endl; - std::cerr << " compressed size = " << size << std::endl; - std::cerr << " hash = " << tmpout << std::endl; - - free(uncompressed_data) ; - } - -#define FIND_NEXT(s,start,end,c) end = s.find(c, start); if (end == std::string::npos) end = s.length(); - - DirEntry *ndir = NULL; - FileEntry *nfile = NULL; - std::list dirlist; - std::string word; - char ch; - - std::string::size_type pos = 0; - while (pos < s.length()) - { - ch = s[pos]; - ++pos; - if (ch == '-') - { - FIND_NEXT(s, pos, pos, '\n'); - ++pos; - - switch(dirlist.size()) - { - /* parse error: out of directory */ - case 0: - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " Ran out of dirs"; - std::cerr << std::endl; -#endif - goto error; - } - /* finished parse, last dir is root */ - case 1: - { - RsPeerId pid = root -> id; - FileIndex::unregisterEntry((void*)root) ; - delete root; /* to clean up old entries */ - root = new PersonEntry(pid); - registerEntry((void*)root) ; - - /* shallow copy of all except id */ - ndir = dirlist.back(); - dirlist.pop_back(); /* empty list */ - (*root) = (*ndir); - - /* now cleanup (can't call standard delete) */ - ndir->subdirs.clear(); - ndir->files.clear(); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - ndir = NULL; - - /* must reset parent pointers now */ - std::map::iterator it; - for(it = root->subdirs.begin(); - it != root->subdirs.end(); ++it) - { - (it->second)->parent = root; - } - - break; - } - /* pop stack */ - default: dirlist.pop_back(); ndir = dirlist.back(); - } - continue; - } - - // Ignore comments - else if (ch == '#') - { - FIND_NEXT(s, pos, pos, '\n'); - ++pos; - } - - else { - std::vector tokens; - /* parse line */ - std::string::size_type lineend; - FIND_NEXT(s, pos, lineend, '\n'); - std::string line = s.substr(pos, lineend - pos); - pos = lineend + 1; - - std::string::size_type start = 0; - while (start < line.length()) - { - std::string::size_type end; - FIND_NEXT(line, start, end, FILE_CACHE_SEPARATOR_CHAR); - tokens.push_back(line.substr(start, end - start)); - start = end + 1; - } - - /* create new file and add it to last directory*/ - if (ch == 'f') - { - if (tokens.size() != 6) - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " File token count wrong: " << tokens.size(); - std::cerr << std::endl; - for(unsigned int i = 0; i < tokens.size(); ++i) - { - std::cerr << "\tToken[" << i << "]:" << tokens[i]; - std::cerr << std::endl; - } - -#endif - goto error; - } - nfile = new FileEntry(); - registerEntry((void*)nfile) ; - nfile->name = tokens[0]; - nfile->hash = RsFileHash(tokens[1]); - nfile->size = atoll(tokens[2].c_str()); - nfile->modtime = atoi(tokens[3].c_str()); - nfile->pop = atoi(tokens[4].c_str()); - nfile->updtime = atoi(tokens[5].c_str()); - nfile->parent = ndir; - if (ndir) { - nfile->row = ndir->subdirs.size() + ndir->files.size(); - ndir->files[nfile->name] = nfile; - } - - } - /* create new dir and add to stack */ - else if (ch == 'd') - { - if (tokens.size() != 6) - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " Dir token count wrong: " << tokens.size(); - std::cerr << std::endl; -#endif - goto error; - } - ndir = new DirEntry(); - registerEntry((void*)ndir) ; - ndir->name = tokens[0]; - ndir->path = tokens[1]; - ndir->size = atoi(tokens[2].c_str()); - ndir->modtime = atoi(tokens[3].c_str()); - ndir->pop = atoi(tokens[4].c_str()); - ndir->updtime = atoi(tokens[5].c_str()); - if (!dirlist.empty()) - { - ndir->parent = (dirlist.back()); - ndir->row = dirlist.back()->subdirs.size(); - dirlist.back()->subdirs[ndir->name] = ndir; - } - dirlist.push_back(ndir); - } - } - } - - updateHashIndex() ; - - return 1; - - /* parse error encountered */ -error: -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << std::endl; -#endif - return 0; -} - - -int FileIndex::saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size,const std::set& forbidden_dirs) -{ - std::string filenametmp = filename + ".tmp" ; - std::string s; - - size = 0 ; - - /* print version and header */ - s += "# FileIndex version 0.1\n"; - s += "# Dir: d name, path, parent, size, modtime, pop, updtime;\n"; - s += "# File: f name, hash, size, modtime, pop, updtime;\n"; - s += "#\n"; - - /* begin recusion */ - root->writeDirInfo(s) ; - - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { -#ifdef FI_DEBUG - std::cout << "writting root directory: name=" << it->second->name << ", path=" << it->second->path << std::endl ; -#endif - if(forbidden_dirs.find(it->second->name) != forbidden_dirs.end()) - { -#ifdef FI_DEBUG - std::cerr << " will be suppressed." << std::endl ; -#endif - } - else - { -#ifdef FI_DEBUG - std::cerr << " will be saved." << std::endl ; -#endif - (it->second)->saveEntry(s); - } - } - - root->writeFileInfo(s) ; // this should never do anything - - /* signal to pop directory from stack in loadIndex() */ - s += "-\n"; - - // now compress the data. -#ifdef FI_DEBUG - std::cerr << "FileIndex::saveIndex(): compressign data." << std::endl; -#endif - - uint8_t *compressed_data = NULL ; - uint32_t compressed_data_size = 0 ; - - if(!RsCompress::compress_memory_chunk((unsigned char *)s.c_str(),s.length(),compressed_data,compressed_data_size)) - { - std::cerr << "(EE) ERROR in file list compression ! file list can't be saved" << std::endl; - free(compressed_data); - return false ; - } - - fileHash = RsDirUtil::sha1sum((unsigned char *)compressed_data,compressed_data_size); - -#ifdef FI_DEBUG - std::cerr << " file = " << filename << std::endl; - std::cerr << " old size = " << s.length() << std::endl; - std::cerr << " new size = " << compressed_data_size << std::endl; - std::cerr << " hash = " << fileHash << std::endl; -#endif - -// /* calculate sha1 hash */ -// SHA_CTX *sha_ctx = new SHA_CTX; -// SHA1_Init(sha_ctx); -// SHA1_Update(sha_ctx, s.c_str(), s.length()); -// SHA1_Final(&sha_buf[0], sha_ctx); -// delete sha_ctx; -// -// for(int i = 0; i < SHA_DIGEST_LENGTH; ++i) -// { -// rs_sprintf_append(fileHash, "%02x", (unsigned int) (sha_buf[i])); -// } - - /* finally, save to file */ - - FILE *file = RsDirUtil::rs_fopen(filenametmp.c_str(), "wb"); - if (file == NULL) - { - std::cerr << "FileIndex::saveIndex error opening file for writting: " << filename << ". Giving up." << std::endl; - return 0; - } - uint32_t outwritten ; - - if(compressed_data_size != (outwritten=fwrite(compressed_data,1,compressed_data_size,file))) - { - std::cerr << "FileIndex::saveIndex error. File not entirely written. Only " << outwritten << " bytes wrote out of " << compressed_data_size << " check for disk full, or disk quotas." << std::endl; - fclose(file); - return 0; - } - - fclose(file); - free(compressed_data) ; - - // Use a temp file name so that the file is never half saved. - // - if(!RsDirUtil::renameFile(filenametmp,filename)) - return false ; - - /* get the size out */ - struct stat64 buf; - - if(-1 == stat64(filename.c_str(), &buf)) - { - std::cerr << "Can't determine size of file " << filename << ": errno = " << errno << std::endl ; - return false ; - } - - size=buf.st_size; - - return true; -} - - -std::string FixName(const std::string& _in) -{ - /* replace any , with _ */ - std::string in(_in) ; - for(unsigned int i = 0; i < in.length(); ++i) - { - if (in[i] == FILE_CACHE_SEPARATOR_CHAR) - { - in[i] = '_'; - } - } - return in; -} - -void DirEntry::writeDirInfo(std::string& s) -{ - /* print node info */ - rs_sprintf_append(s, "d%s%c%s%c%lld%c%ld%c%d%c%ld%c\n", - FixName(name).c_str(), FILE_CACHE_SEPARATOR_CHAR, - FixName(path).c_str(), FILE_CACHE_SEPARATOR_CHAR, - size, FILE_CACHE_SEPARATOR_CHAR, - modtime, FILE_CACHE_SEPARATOR_CHAR, - pop, FILE_CACHE_SEPARATOR_CHAR, - updtime, FILE_CACHE_SEPARATOR_CHAR); -} - -void DirEntry::writeFileInfo(std::string& s) -{ - /* print file info */ - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - rs_sprintf_append(s, "f%s%c%s%c%lld%c%ld%c%d%c%ld%c\n", - FixName((fit->second)->name).c_str(), FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->hash.toStdString().c_str(), FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->size, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->modtime, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->pop, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->updtime, FILE_CACHE_SEPARATOR_CHAR); - } -} - -/* recusive function for traversing the dir tree in preorder */ -int DirEntry::saveEntry(std::string &s) -{ - writeDirInfo(s) ; - - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - (it->second)->saveEntry(s); - } - - writeFileInfo(s) ; - - /* signal to pop directory from stack in loadIndex() */ - s += "-\n"; - return 1; -} - - -int FileIndex::searchHash(const RsFileHash& hash, std::list &results) const -{ -#ifdef FI_DEBUG - std::cerr << "FileIndex::searchHash(" << hash << ")"; - std::cerr << std::endl; -#endif - - std::map::const_iterator it = _file_hashes.find(hash) ; - - if(it!=_file_hashes.end() && isValid((void*)it->second)) - results.push_back(it->second) ; - -#ifdef OLD_CODE_PLZ_REMOVE - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - /* add subdirs to stack */ - std::map::iterator it; - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - std::map::iterator fit; - /* search in current dir */ - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - if (hash == (fit->second)->hash) - { - results.push_back(fit->second); -#ifdef FI_DEBUG - std::cerr << "FileIndex::searchHash(" << hash << ")"; - std::cerr << " found: " << fit->second->name; - std::cerr << std::endl; -#endif - } - } - } -#endif - - return 0; -} - - -int FileIndex::searchTerms(const std::list& terms, std::list &results) const -{ - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - /* iterators */ - std::map::iterator it; - std::map::iterator fit; - std::list::const_iterator iter; - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - for (iter = terms.begin(); iter != terms.end(); ++iter) { - std::string::const_iterator it2; - const std::string &str1 = ndir->name; - const std::string &str2 = *iter; - it2 = std::search(str1.begin(), str1.end(), str2.begin(), str2.end(), CompareCharIC()); - if (it2 != str1.end()) { - /* first search to see if its parent is in the results list */ - bool addDir = true; - for (std::list::iterator rit(results.begin()); rit != results.end() && addDir; ++rit) { - DirEntry *de = dynamic_cast(*rit); - if (de && (de == root)) - continue; - if (de && (de == ndir->parent)) - addDir = false; - } - if (addDir) { - results.push_back((FileEntry *) ndir); - break; - } - } - } - - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - /* cycle through terms */ - for(iter = terms.begin(); iter != terms.end(); ++iter) - { - /* always ignore case */ - std::string::const_iterator it2 ; - const std::string &str1 = fit->second->name; - const std::string &str2 = (*iter); - - it2 = std::search( str1.begin(), str1.end(), - str2.begin(), str2.end(), CompareCharIC() ); - if (it2 != str1.end()) - { - results.push_back(fit->second); - break; - } - /* original case specific term search ****** - if (fit->second->name.find(*iter) != std::string::npos) - { - results.push_back(fit->second); - break; - } - ************/ - } - } - } //while - - return 0; -} - -int FileIndex::searchBoolExp(Expression * exp, std::list &results) const -{ - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - /* iterators */ - std::map::iterator it; - std::map::iterator fit; - std::list::const_iterator iter; - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - /*Evaluate the boolean expression and add it to the results if its true*/ - bool ret = exp->eval(fit->second); - if (ret == true){ - results.push_back(fit->second); - } - } - } //while - - return 0; -} - -uint32_t FileIndex::getType(void *ref) -{ - if(ref == NULL) - return DIR_TYPE_ROOT ; - - if(!isValid(ref)) - return DIR_TYPE_ROOT ; - - return static_cast(ref)->type() ; -} - -bool FileIndex::extractData(const std::string& fpath,DirDetails& details) const -{ - void *ref = findRef(fpath) ; - - if(ref == NULL) - return false ; - - return extractData(ref,details) ; -} - -void *FileIndex::findRef(const std::string& fpath) const -{ - DirEntry *parent = root->findDirectory(fpath); - - std::cerr << "findRef() Called on " << fpath << std::endl; - - if (!parent) - { -//#ifdef FI_DEBUG - std::cerr << "FileIndex::updateFileEntry() NULL parent"; - std::cerr << std::endl; -//#endif - return NULL; - } - std::cerr << "Found parent directory: " << std::endl; - std::cerr << " parent.name = " << parent->name << std::endl; - std::cerr << " parent.path = " << parent->path << std::endl; - - if(parent->path == fpath) // directory! - { - std::cerr << " fpath is a directory. Returning parent!" << std::endl; - return parent ; - } - else - { - std::cerr << " fpath is a file. Looking into parent directory..." << std::endl; - /* search in current dir */ - for(std::map::iterator fit = parent->files.begin(); fit != parent->files.end(); ++fit) - { - std::cerr << " trying " << parent->path + "/" + fit->second->name << std::endl; - if(parent->path + "/" + fit->second->name == fpath) - { - std::cerr << " found !" << std::endl; - return fit->second ; - } - } - - std::cerr << " (EE) not found !" << std::endl; - return NULL ; - } -} - -bool FileIndex::extractData(void *ref,DirDetails& details) -{ - if(!isValid(ref)) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() asked for an invalid pointer " << (void*)ref << std::endl; -#endif - return false ; - } - - FileEntry *file = static_cast(ref); - DirEntry *dir = (file->hash.isNull())?static_cast(file):NULL ; // This is a hack to avoid doing a dynamic_cast - - details.children.clear() ; - time_t now = time(NULL) ; - - if (dir!=NULL) /* has children --- fill */ - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() ref=dir" << std::endl; -#endif - /* extract all the entries */ - for(std::map::const_iterator dit(dir->subdirs.begin()); dit != dir->subdirs.end(); ++dit) - { - DirEntry *dirEntry = dit->second; - - DirStub stub; - stub.type = DIR_TYPE_DIR; - stub.name = dirEntry -> name; - stub.ref = dirEntry; - - details.children.push_back(stub); - } - - for(std::map::const_iterator fit(dir->files.begin()); fit != dir->files.end(); ++fit) - { - FileEntry *fileEntry = fit->second; - - DirStub stub; - stub.type = DIR_TYPE_FILE; - stub.name = fileEntry -> name; - stub.ref = fileEntry; - - details.children.push_back(stub); - } - - if(dir->parent == NULL) - details.type = DIR_TYPE_PERSON ; - else - details.type = DIR_TYPE_DIR; - details.hash.clear() ; - details.count = dir->subdirs.size() + dir->files.size(); - details.min_age = now - dir->most_recent_time ; - } - else - { -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; -#endif - details.type = DIR_TYPE_FILE; - details.count = file->size; - details.min_age = now - file->modtime ; - } - -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() name: " << file->name << std::endl; -#endif - details.ref = file; - details.hash = file->hash; - details.age = now - file->modtime; - details.flags.clear() ; - - /* find parent pointer, and row */ - details.parent = file->parent ; - - details.prow = (file->parent==NULL)?0:file->parent->row ; - - if(details.type == DIR_TYPE_DIR) - { - details.name = file->name; - details.path = dir->path; - } - else - { - details.name = file->name; - details.path = (file->parent==NULL)?"":file->parent->path; - } - - /* find peer id */ - FileEntry *f ; - for(f=file;f->parent!=NULL;f=f->parent) ; - - details.id = static_cast(f)->id; // The topmost parent is necessarily a personEntrY, so we can avoid a dynamic_cast. - -#ifdef FI_DEBUG - assert(details.parent != details.ref) ; - std::cout << "details: ref=" << (void*)ref << ", prow=" << details.prow << ", parent=" << (void*)details.parent << ", children=" ; - for(std::list::iterator it(details.children.begin());it!=details.children.end();++it) - std::cout << " " << (void*)it->ref ; - std::cout << std::endl ; -#endif - - return true; -} - diff --git a/libretroshare/src/dbase/findex.h b/libretroshare/src/dbase/findex.h deleted file mode 100644 index 87ad257d9..000000000 --- a/libretroshare/src/dbase/findex.h +++ /dev/null @@ -1,278 +0,0 @@ -/* - * RetroShare FileCache Module: findex.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef FILE_INDEX_H -#define FILE_INDEX_H - -#include -#include -#include -#if __MACH__ -#include -#else -#include -#endif -#include -#include -#include -#include "retroshare/rstypes.h" - -/****************************************************************************************** - * The Key Data Types for the File Index: - - FileEntry : Information about a single file. - DirEntry : Information about a directory and its children - PersonEntry : Information the root of a FileIndex. - - FileIndex : A Collection of root directories, with a set of functions to manipulate them. - In terms of retroshare, There will be a single 'Local' FileIndex used to store - the shared files, and a set of 'Remote' FileIndices which are used to store - the available files of all the peers. - -******************************************************************************************/ -/****************************************************************************************** - STILL TODO: - - (1) Load/Store a FileIndex to file... - int FileIndex::loadIndex(FILE *input); - int FileIndex::saveIndex(FILE *input); - - This can be done in a recursive manner, or handled completely within FileIndex. - - (2) Search Functions for Partial File Names and Hashes. - - int FileIndex::searchHash(std::string hash, std::list &results); - int FileIndex::searchTerms(std::list terms, std::list &results); - - This is probably best done in a recursive manner. - - The search could also be extended to handle complex Boolean searches such as : - match (size > 100K) && (name contains 'Blue') .... if anyone is interested. - But this can get quite complicated, and can be left to a later date. - -******************************************************************************************/ - - -/****************************************************************************************** - * FileEntry - *****************************************************************************************/ - -#include - -class DirEntry; - -class FileEntry: public RsMemoryManagement::SmallObject -{ - public: - FileEntry() - : size(0), modtime(0), pop(0), updtime(0), parent(NULL), row(0) - { return; } - - virtual ~FileEntry() { return; } - virtual uint32_t type() const { return DIR_TYPE_FILE ; } - -virtual int print(std::string &out); - - /* Data */ - std::string name; - RsFileHash hash; - uint64_t size; /* file size */ - time_t modtime; /* modification time - most recent mod time for a sub entry for dirs */ - int pop; /* popularity rating */ - - time_t updtime; /* last updated */ - - /* References for easy manipulation */ - DirEntry *parent; - int row; - std::list parent_groups ; -}; - -/****************************************************************************************** - * DirEntry - *****************************************************************************************/ - -class DirEntry: public FileEntry -{ - public: - - DirEntry() : most_recent_time(0) {} - /* cleanup */ -virtual ~DirEntry(); - - /* update local entries */ -DirEntry * updateDir(const FileEntry& fe, time_t updtime); -FileEntry * updateFile(const FileEntry& fe, time_t updtime); - - - virtual uint32_t type() const { return DIR_TYPE_DIR ; } -int checkParentPointers(); -int updateChildRows(); - - /* remove local entries */ -int removeFile(const std::string& name); -int removeDir(const std::string& name); -int removeOldDir(const std::string& name, time_t old); /* checks ts first */ - - /* recursive cleanup */ -int removeOldEntries(time_t old, bool recursive); - - /* recursive searches */ -DirEntry * findOldDirectory(time_t old); -DirEntry * findDirectory(const std::string& path); - - /* recursive update directory mod/pop values */ -int updateDirectories(const std::string& path, int pop, int modtime); - - /* output */ -int print(std::string &out); - -int saveEntry(std::string &out); -void writeDirInfo(std::string&); -void writeFileInfo(std::string&); - - /* Data */ - std::string path; /* full path (includes name) */ - std::map subdirs; - std::map files; - - time_t most_recent_time; /* last updated */ - - /* Inherited members from FileEntry: - int size - count for dirs - std::string name; - directory name - std::string hash; - not used - int size; - not used - int modtime; - most recent modication time of any child file (recursive) - int pop; - most popular child file (recursive) - int updtime; - last updated - */ - -}; - -/****************************************************************************************** - * PersonEntry - *****************************************************************************************/ - -class PersonEntry: public DirEntry -{ - public: - /* cleanup */ - PersonEntry(const RsPeerId& pid) : id(pid) { return; } -virtual ~PersonEntry() { return; } - -DirEntry &operator=(DirEntry &src) -{ - DirEntry *pdest = this; - (*pdest) = src; - return *this; -} - virtual uint32_t type() const { return DIR_TYPE_PERSON ; } - - /* Data */ - RsPeerId id; - - /* Inherited members from FileEntry: - int size - count for dirs - std::string name; - directory name - std::string hash; - not used - int size; - not used - int modtime; - most recent modication time of any child file (recursive) - int pop; - most popular child file (recursive) - int updtime; - last updated - */ - -}; - -/****************************************************************************************** - * FileIndex - *****************************************************************************************/ - -class Expression; - -class FileIndex -{ - public: - FileIndex(const RsPeerId& pid); - ~FileIndex(); - - /* control root entries */ - int setRootDirectories(const std::list &inlist, time_t utime); - int getRootDirectories(std::list &outlist); - - /* update (index building) */ - DirEntry * updateDirEntry(const std::string& path, const FileEntry& fe, time_t utime); - FileEntry * updateFileEntry(const std::string& path, const FileEntry& fe, time_t utime); - - DirEntry * findOldDirectory(time_t old); /* finds directories older than old */ - int removeOldDirectory(const std::string& fpath, const std::string& name, time_t old); - - int cleanOldEntries(time_t old); /* removes entries older than old */ - - /* debug */ - int printFileIndex(std::string &out); - int printFileIndex(std::ostream &out); - - /* load/save to file */ - int loadIndex(const std::string& filename, const RsFileHash &expectedHash, uint64_t size); - int saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size, const std::set& forbidden_roots); - - /* search through this index */ - int searchTerms(const std::list& terms, std::list &results) const; - int searchHash(const RsFileHash& hash, std::list &results) const; - int searchBoolExp(Expression * exp, std::list &results) const; - - /// Recursively compute the maximum modification time of children. - /// Used to easily retrieve mose recent files. - // - void updateMaxModTime() ; - void RecursUpdateMaxModTime(DirEntry *) ; - - PersonEntry *root; - -#ifdef __MACH__ - static std::unordered_set _pointers ; -#else - static std::tr1::unordered_set _pointers ; -#endif - static void registerEntry(void*p) ; - static void unregisterEntry(void*p) ; - static bool isValid(void*p) ; - - /// Fills up details from the data contained in ref. - // - static bool extractData(void *ref,DirDetails& details) ; - static uint32_t getType(void *ref) ; - - void *findRef(const std::string& path) const ; - bool extractData(const std::string& path,DirDetails& details) const ; - - void updateHashIndex() ; - void recursUpdateHashIndex(DirEntry *) ; - - std::map _file_hashes ; -}; - - -#endif - diff --git a/libretroshare/src/dbase/fistore.cc b/libretroshare/src/dbase/fistore.cc deleted file mode 100644 index 00d70a214..000000000 --- a/libretroshare/src/dbase/fistore.cc +++ /dev/null @@ -1,459 +0,0 @@ -/* - * RetroShare FileCache Module: fistore.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include - -#include "rsserver/p3face.h" -#include "dbase/fistore.h" -#include "retroshare/rsexpr.h" -#include "retroshare/rsfiles.h" -#include "serialiser/rsserviceids.h" -#include "pqi/p3peermgr.h" -#include "pqi/p3notify.h" - -FileIndexStore::FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, - p3PeerMgr *cnmgr, RsPeerId ownid, std::string cachedir) - :CacheStore(RS_SERVICE_TYPE_FILE_INDEX, false, cs, cft, cachedir), - localId(ownid), localindex(NULL), mPeerMgr(cnmgr) -{ - return; -} - -FileIndexStore::~FileIndexStore() -{ - /* clean up the Index */ - return; -} - -/*** - * #define FIS_DEBUG2 1 - * #define FIS_DEBUG 1 - **/ - - /* actual load, once data available */ -int FileIndexStore::loadCache(const RsCacheData &data) -{ - -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() hash: " << data.hash << std::endl; - std::cerr << "FileIndexStore::loadCache() path: " << data.path << std::endl; - std::cerr << "FileIndexStore::loadCache() name: " << data.name << std::endl; - std::cerr << "FileIndexStore::loadCache() size: " << data.size << std::endl; -#endif - - /* do Callback */ - AboutToModify(); - - /* lock it up */ - lockData(); - - FileIndex *fiold = NULL; - bool local = (data.pid == localId); - - std::map::iterator it; - /* remove old cache */ - if (local) - { - fiold = localindex; - localindex = NULL; - } - else if (indices.end() != (it = indices.find(data.pid))) - { - fiold = it->second; - indices.erase(it); - //delete fi; - } - - if(mPeerMgr->isFriend(data.pid)) - { - // We discard file lists from non friends. This is the place to remove file lists of deleted friends - // from the cache. Doing this, the file list still shows in a session where we deleted a friend, but will be removed - // at next restart. - // - - /* load Cache */ - FileIndex *finew = new FileIndex(data.pid); - - if (finew->loadIndex(data.path + '/' + data.name, data.hash, data.size)) - { -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() Succeeded!" << std::endl; -#endif - /* This is not the place to set the peername. - * It is a hack, which makes it programmatically impossible - * to get the file data out.... - * - * peername should not be used in dbase. - */ - finew->root->name = data.pid.toStdString(); - - if (local) - { - localindex = finew; - } - else - { - indices[data.pid] = finew; - } - delete fiold; - - /* store in tale */ - locked_storeCacheEntry(data); - } - else - { -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() Failed!" << std::endl; -#endif - /* reinstall the old one! */ - delete finew; - if (fiold) - { - if (local) - { - localindex = fiold; - } - else - { - indices[data.pid] = fiold; - } - } - } - } -#ifdef FIS_DEBUG - else - std::cerr << "Discarding file list from deleted peer " << data.pid << std::endl ; -#endif - - /* need to correct indices(row) for the roots of the FileIndex */ - int i = 0; - for(it = indices.begin(); it != indices.end(); ++it) - { - (it->second)->root->row = i++; - it->second->FileIndex::updateMaxModTime() ; - } - if (localindex) - { - localindex->root->row = 0; - localindex->updateMaxModTime() ; - } - - unlockData(); - - ModCompleted(); - bool ret = false; - return ret; -} - - - /* Search Interface - For Directory Access */ -int FileIndexStore::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails& details) const -{ - lockData(); - - std::map::const_iterator it = indices.find(uid); - bool found = true; - - if (it != indices.end()) - found = it->second->extractData(path,details) ; - - unlockData(); - - return found ; - -#ifdef OLD_STUFF_TO_REMOVE - /* lock it up */ - lockData(); - - std::map::const_iterator it; - it = indices.find(uid); - bool found = true; - - if (it != indices.end()) - { - //DirEntry *fdir = (it->second).lookupDirectory(path); - /* translate it - */ - bool b = FileIndex::extractData((it->second)->root,details) ; - - found = found && b ; - } - else - found = false; - - unlockData(); - return found; -#endif -} - -int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const -{ - /* remove unused parameter warnings */ - (void) flags; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl; -#endif - - std::map::const_iterator pit; - - lockData(); - -// checked by FileIndex::extractData -// if(ref != NULL && !FileIndex::isValid(ref)) -// { -// unlockData() ; -// return false ; -// } - - /* so cast *ref to a DirEntry */ - /* root case */ - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() CHKS" << std::endl; - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - (pit->second)->root->checkParentPointers(); - } -#endif - - if (ref == NULL) - { -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() ref=NULL (root)" << std::endl; -#endif - - /* get remote root entries */ - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - /* - */ - FileIndex *fileIndex = pit->second; - - DirStub stub; - stub.type = DIR_TYPE_PERSON; - stub.name = fileIndex->root->name; - stub.ref = fileIndex->root; - - details.children.push_back(stub); - } - details.parent = NULL; - details.prow = -1; - details.ref = NULL; - details.type = DIR_TYPE_ROOT; - details.name = ""; - details.hash.clear() ; - details.path = ""; - details.count = indices.size(); - details.age = 0; - details.flags.clear() ; - details.min_age = 0; - - unlockData(); - return true ; - } - - bool b = FileIndex::extractData(ref,details) ; - - unlockData(); - return b; -} -uint32_t FileIndexStore::getType(void *ref) const -{ - lockData() ; - uint32_t b = FileIndex::getType(ref) ; - unlockData(); - - return b; -} - -int FileIndexStore::SearchHash(const RsFileHash& hash, std::list &results) const -{ - lockData(); - results.clear() ; - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - - time_t now = time(NULL); - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash()" << std::endl; -#endif - for(pit = indices.begin(); pit != indices.end(); ++pit) - { -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash() Searching: Peer "; - std::cerr << pit->first << std::endl; -#endif - firesults.clear(); - - (pit->second)->searchHash(hash, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - FileDetail fd; - fd.id = pit->first; - fd.name = (*rit)->name; - fd.hash = (*rit)->hash; - fd.path = ""; /* TODO */ - fd.size = (*rit)->size; - fd.age = now - (*rit)->modtime; - fd.rank = (*rit)->pop; - - results.push_back(fd); - } - - } - - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash() Found " << results.size(); - std::cerr << " Results from " << indices.size() << " Peers" << std::endl; -#endif - - unlockData(); - return results.size(); -} - - -int FileIndexStore::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) const -{ - lockData(); - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - - results.clear() ; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchKeywords()" << std::endl; -#endif - if(flags & RS_FILE_HINTS_REMOTE) - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - firesults.clear(); - - (pit->second)->searchTerms(keywords, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - - if(!FileIndex::extractData(*rit, dd)) - continue ; - - results.push_back(dd); - } - } - - if(flags & RS_FILE_HINTS_LOCAL) - if (localindex) - { - firesults.clear(); - - localindex->searchTerms(keywords, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - - if(!FileIndex::extractData(*rit, dd)) - continue ; - - dd.id.clear() ; - results.push_back(dd); - } - - } - - unlockData(); - return results.size(); -} - - -int FileIndexStore::searchBoolExp(Expression * exp, std::list &results) const -{ - lockData(); - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::searchBoolExp()" << std::endl; -#endif - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - firesults.clear(); - - (pit->second)->searchBoolExp(exp, firesults); - - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - FileIndex::extractData(*rit, dd); - results.push_back(dd); - } - - } - - /* finally search local files */ - if (localindex) - { - firesults.clear(); - - localindex->searchBoolExp(exp, firesults); - - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - FileIndex::extractData(*rit, dd); - dd.id.clear() ; - results.push_back(dd); - } - - } - - - unlockData(); - return results.size(); -} - -int FileIndexStore::AboutToModify() -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - - return 1; -} - - -int FileIndexStore::ModCompleted() -{ - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - - return 1; -} - - diff --git a/libretroshare/src/dbase/fistore.h b/libretroshare/src/dbase/fistore.h deleted file mode 100644 index e175adec8..000000000 --- a/libretroshare/src/dbase/fistore.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * RetroShare FileCache Module: fistore.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_FILE_INDEX_STORE_H -#define MRK_FILE_INDEX_STORE_H - - -/********** - * Stores the FileCaches of the Peers - * must implement 'loadCache' to - * - * This class is also accessed by the GUI.... - * and the FileTransfer class. - * - */ - -class p3PeerMgr ; - -#include "dbase/findex.h" -#include "dbase/cachestrapper.h" -#include "retroshare/rsiface.h" - -class FileStoreResult -{ - public: - std::string id; - std::string path; - std::string hash; - std::string name; -}; - -class NotifyCallback -{ - public: - NotifyCallback() { return; } -virtual ~NotifyCallback() { return; } -virtual void AboutToModify() { return; } -virtual void ModCompleted() { return; } -}; - - -class Expression; - -class FileIndexStore: public CacheStore -{ - public: - - FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, p3PeerMgr *pmgr, RsPeerId ownid, std::string cachedir); -virtual ~FileIndexStore(); - - /* virtual functions overloaded by cache implementor */ -virtual int loadCache(const RsCacheData &data); /* actual load, once data available */ - - /* Search Interface - For FileTransfer Lookup */ - int SearchHash(const RsFileHash &hash, std::list &results) const; - - /* Search Interface - For Search Interface */ - int SearchKeywords(std::list terms, std::list &results,FileSearchFlags flags) const; - - /* Search Interface - for Adv Search Interface */ - int searchBoolExp(Expression * exp, std::list &results) const; - - - /* Search Interface - For Directory Access */ - int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) const; - int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const; - uint32_t getType(void *ref) const ; - - private: - int AboutToModify(); - int ModCompleted(); - - std::map indices; - - RsPeerId localId; - FileIndex *localindex; - - p3PeerMgr *mPeerMgr ; -}; - - -#endif From 1be599008afe275baa9f1cdefedfd92badec1e71 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Wed, 21 Sep 2016 22:31:39 +0200 Subject: [PATCH 093/174] disabled dbase from unittests.pro, which should be cnverted into proper tests for file_sharing. --- tests/unittests/unittests.pro | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/unittests.pro b/tests/unittests/unittests.pro index 2409b8845..d0485ca92 100644 --- a/tests/unittests/unittests.pro +++ b/tests/unittests/unittests.pro @@ -353,9 +353,9 @@ SOURCES += libretroshare/gxs/data_service/rsdataservice_test.cc \ ################################ dbase ##################################### -SOURCES += libretroshare/dbase/fisavetest.cc \ - libretroshare/dbase/fitest2.cc \ - libretroshare/dbase/searchtest.cc \ +#SOURCES += libretroshare/dbase/fisavetest.cc \ +# libretroshare/dbase/fitest2.cc \ +# libretroshare/dbase/searchtest.cc \ # libretroshare/dbase/ficachetest.cc \ # libretroshare/dbase/fimontest.cc \ From 7f99bc2b70c9abca4345a95fe8e720acd5cd8c41 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 22 Sep 2016 10:26:05 +0200 Subject: [PATCH 094/174] fixed compilation after removal of dbase/ --- libretroshare/src/plugins/rscacheservice.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libretroshare/src/plugins/rscacheservice.h b/libretroshare/src/plugins/rscacheservice.h index 6e928da61..82f43e0de 100644 --- a/libretroshare/src/plugins/rscacheservice.h +++ b/libretroshare/src/plugins/rscacheservice.h @@ -1,6 +1,5 @@ #pragma once -#include #include "plugins/pluginmanager.h" // This code needs to be re-written to work with GXS. For now it is obsolete. @@ -9,7 +8,7 @@ // The following class abstracts the construction of a cache service. The user only has to // supply RS with a type ID. If the ID is already in use, RS will complain. // -class RsCacheService: public CacheSource, public CacheStore, public p3Config +class RsCacheService: /* public CacheSource, public CacheStore, */ public p3Config { public: RsCacheService(uint16_t type,uint32_t tick_delay_in_seconds, RsPluginHandler* pgHandler) ; From e6e95829bcc4446b882c3263d5769d39467721df Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 22 Sep 2016 13:44:53 +0200 Subject: [PATCH 095/174] fixed bug in estimation of next FT rate that caused downloads to stuck for a while --- libretroshare/src/ft/fttransfermodule.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index f5d5dfe84..7943a394b 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -739,7 +739,7 @@ bool ftTransferModule::locked_tickPeerTransfer(peerInfo &info) #endif /* update rate */ - if(info.lastTransfers > 0 || ageReq > 2) + if( (info.lastTransfers > 0 && ageReq > 0) || ageReq > 2) { info.actualRate = info.actualRate * 0.75 + 0.25 * info.lastTransfers / (float)ageReq; info.lastTransfers = 0; From 0cc4ebd89c34c66442185eefc918bf0d09ffb2fc Mon Sep 17 00:00:00 2001 From: mr-alice Date: Thu, 22 Sep 2016 21:47:58 +0200 Subject: [PATCH 096/174] added comments in the file lists code. Removed unused functions. Added a few missing mutexes. --- .../src/file_sharing/dir_hierarchy.cc | 33 +++++++++++++- .../src/file_sharing/directory_storage.cc | 15 ------- .../src/file_sharing/directory_storage.h | 43 +++++++++++++------ libretroshare/src/file_sharing/hash_cache.h | 19 ++++++-- libretroshare/src/file_sharing/p3filelists.cc | 37 ++++++++++++---- libretroshare/src/file_sharing/p3filelists.h | 4 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 2 + 7 files changed, 108 insertions(+), 45 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index ad1004715..61785753f 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -142,14 +142,18 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage for(uint32_t i=0;i(mNodes[d.subdirs[i]])->dir_name) == subdirs.end()) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] Removing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; +#endif if( !removeDirectory(d.subdirs[i])) i++ ; } else { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] Keeping existing subdirectory " << static_cast(mNodes[d.subdirs[i]])->dir_name << " with index " << d.subdirs[i] << std::endl; +#endif should_create.erase(static_cast(mNodes[d.subdirs[i]])->dir_name) ; ++i; @@ -157,7 +161,9 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage for(std::map::const_iterator it(should_create.begin());it!=should_create.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] adding new subdirectory " << it->first << " at index " << mNodes.size() << std::endl; +#endif DirEntry *de = new DirEntry(it->first) ; @@ -258,7 +264,9 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En if(it == subfiles.end()) // file does not exist anymore => delete { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] removing non existing file " << f.file_name << " at index " << d.subfiles[i] << std::endl; +#endif delete mNodes[d.subfiles[i]] ; mNodes[d.subfiles[i]] = NULL ; @@ -281,7 +289,9 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En for(std::map::const_iterator it(new_files.begin());it!=new_files.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] adding new file " << it->first << " at index " << mNodes.size() << std::endl; +#endif d.subfiles.push_back(mNodes.size()) ; mNodes.push_back(new FileEntry(it->first,it->second.size,it->second.modtime)); @@ -297,8 +307,9 @@ bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl; return false; } - +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl; +#endif RsFileHash& old_hash (static_cast(mNodes[file_index])->file_hash) ; mFileHashes[hash] = file_index ; @@ -317,7 +328,9 @@ bool InternalFileHierarchyStorage::updateFile(const DirectoryStorage::EntryIndex FileEntry& fe(*static_cast(mNodes[file_index])) ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "[directory storage] updating file entry at index " << file_index << ", name=" << fe.file_name << " size=" << fe.file_size << ", hash=" << fe.file_hash << std::endl; +#endif fe.file_hash = hash; fe.file_size = size; @@ -358,7 +371,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI } DirEntry& d(*static_cast(mNodes[indx])) ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "Updating dir entry: name=\"" << dir_name << "\", most_recent_time=" << most_recent_time << ", modtime=" << dir_modtime << std::endl; +#endif d.dir_most_recent_time = most_recent_time; d.dir_modtime = dir_modtime; @@ -375,7 +390,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI // check that all subdirs already exist. If not, create. for(uint32_t i=0;i::iterator it = existing_subdirs.find(subdirs_hash[i]) ; DirectoryStorage::EntryIndex dir_index = 0; @@ -384,7 +401,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI { dir_index = it->second ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " already exists, at index " << dir_index << std::endl; +#endif existing_subdirs.erase(it) ; } @@ -401,7 +420,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI mDirHashes[subdirs_hash[i]] = dir_index ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " created, at new index " << dir_index << std::endl; +#endif } d.subdirs.push_back(dir_index) ; @@ -411,7 +432,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI for(std::map::const_iterator it = existing_subdirs.begin();it!=existing_subdirs.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; +#endif if(!checkIndex(it->second,FileStorageNode::TYPE_DIR)) { @@ -436,13 +459,17 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI const FileEntry& f(subfiles_array[i]) ; DirectoryStorage::EntryIndex file_index ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " subfile name = " << subfiles_array[i].file_name << ": " ; +#endif if(it != existing_subfiles.end() && mNodes[it->second] != NULL && mNodes[it->second]->type() == FileStorageNode::TYPE_FILE) { file_index = it->second ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " already exists, at index " << file_index << std::endl; +#endif if(!updateFile(file_index,f.file_hash,f.file_name,f.file_size,f.file_modtime)) std::cerr << "(EE) Cannot update file with index " << it->second <<" and hash " << f.file_hash << ". This is very weird. Entry should have just been created and therefore should exist. Skipping." << std::endl; @@ -456,7 +483,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI mNodes[file_index] = new FileEntry(f.file_name,f.file_size,f.file_modtime,f.file_hash) ; mFileHashes[f.file_hash] = file_index ; +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " created, at new index " << file_index << std::endl; +#endif } d.subfiles.push_back(file_index) ; @@ -465,7 +494,9 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI for(std::map::const_iterator it = existing_subfiles.begin();it!=existing_subfiles.end();++it) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << " removing existing subfile that is not in the dirctory anymore: name=" << it->first << " index=" << it->second << std::endl; +#endif if(!checkIndex(it->second,FileStorageNode::TYPE_FILE)) { diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 45cb3ccca..9a8b76c9e 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -173,21 +173,6 @@ int DirectoryStorage::searchHash(const RsFileHash& hash, std::list & return mFileHierarchy->searchHash(hash,results); } -// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; -// static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; - -void DirectoryStorage::loadNextTag(const unsigned char *data,uint32_t& offset,uint8_t& entry_tag,uint32_t& entry_size) -{ - entry_tag = data[offset++] ; -} -void DirectoryStorage::saveNextTag(unsigned char *data, uint32_t& offset, uint8_t entry_tag, uint32_t entry_size) -{ -} - void DirectoryStorage::load(const std::string& local_file_name) { RS_STACK_MUTEX(mDirStorageMtx) ; diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index b76d73ebd..2d8119445 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -49,23 +49,28 @@ class DirectoryStorage void save() const ; + // These functions are to be used by file transfer and file search. + virtual int searchTerms(const std::list& terms, std::list &results) const ; virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; virtual int searchHash(const RsFileHash& hash, std::list &results) const ; - bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; - bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; - bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; + // gets/sets the various time stamps: + // + bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; // last modification time, computed recursively over all subfiles and directories + bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; // last modification time for that index only + bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; // last time the entry was updated. This is only used on the RemoteDirectoryStorage side. bool setDirectoryRecursModTime(EntryIndex index,time_t recurs_max_modf_TS) ; bool setDirectoryLocalModTime (EntryIndex index,time_t modtime_TS) ; bool setDirectoryUpdateTime (EntryIndex index,time_t update_TS) ; - uint32_t getEntryType(const EntryIndex& indx) ; // returns DIR_TYPE_*, not the internal directory storage stuff. + uint32_t getEntryType(const EntryIndex& indx) ; // WARNING: returns DIR_TYPE_*, not the internal directory storage stuff. virtual bool extractData(const EntryIndex& indx,DirDetails& d); // This class allows to abstractly browse the stored directory hierarchy in a depth-first manner. - // It gives access to sub-files and sub-directories below. + // It gives access to sub-files and sub-directories below. When using it, the client should make sure + // that the DirectoryStorage is properly locked, since the iterator cannot lock it. // class DirIterator { @@ -120,20 +125,28 @@ class DirectoryStorage time_t modtime; }; - EntryIndex root() const ; // returns the index of the root directory entry. - const RsPeerId& peerId() const { return mPeerId ; } - int parentRow(EntryIndex e) const ; - bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; + EntryIndex root() const ; // returns the index of the root directory entry. This is generally 0. + const RsPeerId& peerId() const { return mPeerId ; } // peer ID of who owns that file list. + int parentRow(EntryIndex e) const ; // position of the current node, in the array of children at its parent node. Used by GUI for display. + bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; // returns the index of the children node at position "row" in the children nodes. Used by GUI for display. + // Sets the subdirectory/subfiles list of entry indx the supplied one, possible adding and removing directories (resp.files). New directories are set empty with + // just a name and need to be updated later on. New files are returned in a list so that they can be sent to hash cache. + // bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; + // Updates relevant information for the file at the given index. + bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ; bool updateHash(const EntryIndex& index,const RsFileHash& hash); - bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; - bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; + // Returns the hash of the directory at the given index and reverse. This hash is set as random the first time it is used (when updating directories). It will be + // used by the sync system to designate the directory without referring to index (index could be used to figure out the existance of hidden directories) + + bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; // constant cost + bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; // log cost. void print(); void cleanup(); @@ -144,9 +157,6 @@ class DirectoryStorage private: - void loadNextTag(const unsigned char *data, uint32_t& offset, uint8_t& entry_tag, uint32_t& entry_size) ; - void saveNextTag(unsigned char *data,uint32_t& offset,uint8_t entry_tag,uint32_t entry_size) ; - // debug void locked_check(); @@ -195,6 +205,11 @@ public: LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {} virtual ~LocalDirectoryStorage() {} + /*! + * \brief [gs]etSharedDirectoryList + * Gets/sets the list of shared directories. Each directory is supplied with a virtual name (the name the friends will see), and sharing flags/groups. + * \param lst + */ void setSharedDirectoryList(const std::list& lst) ; void getSharedDirectoryList(std::list& lst) ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 0f92f09c1..1afa756ca 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -30,6 +30,11 @@ #include "util/rsthreads.h" #include "retroshare/rsfiles.h" +/*! + * \brief The HashStorageClient class + * Used by clients of the hash cache for receiving hash results when done. This is asynchrone of course since hashing + * might be quite costly. + */ class HashStorageClient { public: @@ -59,7 +64,7 @@ public: * \param mod_time Actual file modification time * \param known_hash Returned hash for the file. * \param c Hash cache client to which the hash should be sent once calculated - * \param client_param Param to be passed to the client callback + * \param client_param Param to be passed to the client callback. Useful if the client needs a file ID. * * \return true if the supplied hash info is up to date. */ @@ -75,9 +80,9 @@ public: } ; // interaction with GUI, called from p3FileLists - void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } + void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } // duration for which the hash is kept even if the file is not shared anymore uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; } - void clear() { mFiles.clear(); mChanged=true; } + void clear() { mFiles.clear(); mChanged=true; } // drop all known hashes. Not something to do, except if you want to rehash the entire database bool empty() const { return mFiles.empty() ; } // Functions called by the thread @@ -86,8 +91,14 @@ public: friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ; private: + /*! + * \brief clean + * This function is responsible for removing old hashes, etc + */ void clean() ; + // loading/saving the entire hash database to a file + void locked_save() ; void locked_load() ; @@ -98,7 +109,7 @@ private: uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries std::map mFiles ; // stored as (full_path, hash_info) - std::string mFilePath ; + std::string mFilePath ; // file where the hash database is stored bool mChanged ; struct FileHashJob diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 0e734e56a..409dd4637 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -223,8 +223,10 @@ int p3FileDatabase::tick() mLastRemoteDirSweepTS = now; -#warning hack to make loaded directories show up in the GUI, because the GUI isn_t ready at the time they are actually loaded up. - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); + // This is a hash to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up, + // so the first notify is ignored, and no other notify will happen next. + + RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); } return 0; @@ -614,7 +616,9 @@ void p3FileDatabase::requestDirUpdate(void *ref) P3FILELISTS_DEBUG() << "Trying to force sync of entry ndex " << e << " to friend " << mRemoteDirectories[fi-1]->peerId() << std::endl; #endif - if(generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) + RS_STACK_MUTEX(mFLSMtx) ; + + if(locked_generateAndSendSyncRequest(mRemoteDirectories[fi-1],e)) { #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Succeed." << std::endl; @@ -638,7 +642,7 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe return true ; } - else if(row < mRemoteDirectories.size()) + else if((uint32_t)row < mRemoteDirectories.size()) { convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; @@ -752,7 +756,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // Case where the index is the top of a single person. Can be us, or a friend. - bool res = storage->extractData(e,d); + if(!storage->extractData(e,d)) + { + P3FILELISTS_ERROR() << "(EE) request on index " << e << ", for directory ID=" << storage->peerId() << " failed. This should not happen." << std::endl; + return false ; + } // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // a pointer into an int. @@ -824,6 +832,7 @@ void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the di } bool p3FileDatabase::inDirectoryCheck() { + RS_STACK_MUTEX(mFLSMtx) ; return mLocalDirWatcher->inDirectoryCheck(); } void p3FileDatabase::setWatchEnabled(bool b) @@ -975,8 +984,9 @@ bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& // - because the hash is performed late, the last modf time upward is updated only when the hash is obtained. // // Remote dirs store -// 1 - recursive modif time of the files/dir in the friend's time -// 2 - update TS in the local time +// 1 - recursive max modif time of the files/dir in the friend's time +// 2 - modif time of the files/dir in the friend's time +// 3 - update TS in the local time // // The update algorithm is the following: // @@ -1275,20 +1285,29 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) if(item->flags & RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED) { + RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS std::cerr << " removing directory with index " << entry_index << " because it does not exist." << std::endl; +#endif mRemoteDirectories[fi]->removeDirectory(entry_index); mRemoteDirectories[fi]->print(); } else if(item->flags & RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE) { + RS_STACK_MUTEX(mFLSMtx) ; +#ifdef DEBUG_P3FILELISTS std::cerr << " Directory is up to date. Setting local TS." << std::endl; +#endif mRemoteDirectories[fi]->setDirectoryUpdateTime(entry_index,time(NULL)) ; } else if(item->flags & RsFileListsItem::FLAGS_SYNC_DIR_CONTENT) { +#ifdef DEBUG_P3FILELISTS std::cerr << " Item contains directory data. Deserialising/Updating." << std::endl; +#endif + RS_STACK_MUTEX(mFLSMtx) ; if(mRemoteDirectories[fi]->deserialiseUpdateDirEntry(entry_index,item->directory_content_data)) RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); // notify the GUI if the hierarchy has changed @@ -1328,7 +1347,7 @@ void p3FileDatabase::locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *r // compare TS if((e == 0 && now > local_update_TS + DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ) || local_update_TS == 0) // we need to compare local times only. We cannot compare local (now) with remote time. - if(generateAndSendSyncRequest(rds,e)) + if(locked_generateAndSendSyncRequest(rds,e)) { #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " Asking for sync of directory " << e << " to peer " << rds->peerId() << " because it's " << (now - local_update_TS) << " secs old since last check." << std::endl; @@ -1363,7 +1382,7 @@ p3FileDatabase::DirSyncRequestId p3FileDatabase::makeDirSyncReqId(const RsPeerId return r ^ random_bias; } -bool p3FileDatabase::generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e) +bool p3FileDatabase::locked_generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e) { RsFileHash entry_hash ; time_t now = time(NULL) ; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 08afd0e88..67834bfa7 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -117,7 +117,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ; uint32_t getType(void *) const ; - // proxy method used by the web UI. + // proxy method used by the web UI. Dont't delete! int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const; // set/update shared directories @@ -173,7 +173,7 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub * \param e Entry index to update * \return true if the request is correctly sent. */ - bool generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e); + bool locked_generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e); // File sync request queues. The fast one is used for online browsing when friends are connected. // The slow one is used for background update of file lists. diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index f48394cca..036dd96e2 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -522,7 +522,9 @@ QModelIndexList SharedFilesDialog::getSelected() void RemoteSharedFilesDialog::expanded(const QModelIndex& indx) { +#ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Expanding at " << indx.row() << " and " << indx.column() << " ref=" << indx.internalPointer() << ", pointer at 1: " << proxyModel->mapToSource(indx).internalPointer() << std::endl; +#endif model->updateRef(proxyModel->mapToSource(indx)) ; } From 7bddd4bcd49e9dfe606224e1a6cb7c106d723992 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Fri, 23 Sep 2016 13:18:55 +0200 Subject: [PATCH 097/174] removed call to exit causing the software to stop when local network interfaces are not ready --- libretroshare/src/pqi/pqinetwork.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/pqi/pqinetwork.cc b/libretroshare/src/pqi/pqinetwork.cc index 4937aef64..db442f380 100644 --- a/libretroshare/src/pqi/pqinetwork.cc +++ b/libretroshare/src/pqi/pqinetwork.cc @@ -279,12 +279,6 @@ int inet_aton(const char *name, struct in_addr *addr) #include #endif // WINDOWS_SYS -void getLocalAddressesFailed() -{ - std::cerr << "FATAL ERROR: getLocalAddresses failed!" << std::endl; - exit(1); -} - bool getLocalAddresses(std::list & addrs) { addrs.clear(); @@ -304,7 +298,11 @@ bool getLocalAddresses(std::list & addrs) NULL, adapter_addresses, &bf_size); - if (error != ERROR_SUCCESS) getLocalAddressesFailed(); + if (error != ERROR_SUCCESS) + { + std::cerr << "FATAL ERROR: getLocalAddresses failed!" << std::endl; + return false ; + } IP_ADAPTER_ADDRESSES* adapter(NULL); for(adapter = adapter_addresses; NULL != adapter; adapter = adapter->Next) @@ -321,7 +319,11 @@ bool getLocalAddresses(std::list & addrs) free(adapter_addresses); #else // WINDOWS_SYS struct ifaddrs *ifsaddrs, *ifa; - if(getifaddrs(&ifsaddrs) != 0) getLocalAddressesFailed(); + if(getifaddrs(&ifsaddrs) != 0) + { + std::cerr << "FATAL ERROR: getLocalAddresses failed!" << std::endl; + return false ; + } for ( ifa = ifsaddrs; ifa; ifa = ifa->ifa_next ) if ( ifa->ifa_addr && (ifa->ifa_flags & IFF_UP) ) { From 9f76b7601a1d9d478bc762bd88facaa632fdd693 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 23 Sep 2016 19:01:14 +0200 Subject: [PATCH 098/174] forced closedir in order to avoid leak of file descriptor in FolderIterator --- libretroshare/src/util/folderiterator.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 9e8ba66d3..35a661882 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -150,9 +150,6 @@ uint8_t FolderIterator::file_type() { return mType ; } bool FolderIterator::closedir() { - if(!validity) - return false; - validity = false; #ifdef WINDOWS_SYS From ca281e4bb871ba981742e573ce41bf2edb51ad5e Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 18:32:36 +0200 Subject: [PATCH 099/174] Fixed error with 32 bit GCC Both uint32_t and size_t are resolved to unsigned int, it makes 2 lines conflicted. This happens in filelist_io.cc, lines: 31-32, 36-37, 41-42 --- libretroshare/src/file_sharing/filelist_io.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 90297ab0e..33a1bc5a1 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -29,17 +29,17 @@ #include "filelist_io.h" template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; } -template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const size_t & val) { return setRawUInt64(buff,size,&offset,val) ; } +template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint64_t & val) { return setRawUInt64(buff,size,&offset,val) ; } template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; } template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const Sha1CheckSum & val) { return val.serialise(buff,size,offset) ; } template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint32_t & val) { return getRawUInt32(const_cast(buff),size,&offset,&val) ; } -template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,size_t & val) { return getRawUInt64(const_cast(buff),size,&offset,&val) ; } +template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint64_t & val) { return getRawUInt64(const_cast(buff),size,&offset,&val) ; } template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,std::string & val) { return getRawString(const_cast(buff),size,&offset,val) ; } template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,Sha1CheckSum & val) { return val.deserialise(const_cast(buff),size,offset) ; } template<> uint32_t FileListIO::serial_size(const uint32_t & ) { return 4 ; } -template<> uint32_t FileListIO::serial_size(const size_t & ) { return 8 ; } +template<> uint32_t FileListIO::serial_size(const uint64_t & ) { return 8 ; } template<> uint32_t FileListIO::serial_size(const std::string & val) { return getRawStringSize(val) ; } template<> uint32_t FileListIO::serial_size(const Sha1CheckSum & ) { return Sha1CheckSum::serial_size(); } From 529ab64ae383209c3ca56626821e1d9f9aa1926d Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 18:43:48 +0200 Subject: [PATCH 100/174] Fixed missing time.h includes --- libretroshare/src/file_sharing/dir_hierarchy.cc | 1 + libretroshare/src/file_sharing/directory_storage.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 61785753f..023206d29 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -23,6 +23,7 @@ * */ #include +#include #include "util/rsdir.h" #include "util/rsprint.h" #include "retroshare/rsexpr.h" diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 9a8b76c9e..0f0c4e2cb 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -23,6 +23,7 @@ * */ #include +#include #include "serialiser/rstlvbinary.h" #include "retroshare/rspeers.h" #include "util/rsdir.h" From 70c8e14d164b69fb181069a913dd300e32845436 Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 19:00:07 +0200 Subject: [PATCH 101/174] Fix folderiterator.cc:23: error: cannot convert '_stat64*' to '_stati64*' for argument '2' to 'int _wstati64(const wchar_t*, _stati64*)' --- libretroshare/src/util/folderiterator.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 35a661882..bf9bfd68b 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -4,6 +4,10 @@ #include #include +#ifdef WINDOWS_SYS +#include "util/rswin.h" +#endif + #include "folderiterator.h" #include "rsstring.h" From 71a5bffed887f2bf3422c7fc4002a30882ab9745 Mon Sep 17 00:00:00 2001 From: mr-alice Date: Fri, 23 Sep 2016 20:07:57 +0200 Subject: [PATCH 102/174] added small hack against re-hashing due to stat64 depending on local time on windows --- libretroshare/src/file_sharing/hash_cache.cc | 22 +++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 3f96abec1..8311aa77a 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -191,9 +191,29 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t time_t now = time(NULL) ; std::map::iterator it = mFiles.find(full_path) ; - if(it != mFiles.end() && (uint64_t)mod_time == it->second.modf_stamp && size == it->second.size) + // On windows we compare the time up to +/- 3600 seconds. This avoids re-hashing files in case of daylight saving change. + // + // See: + // https://support.microsoft.com/en-us/kb/190315 + // + if(it != mFiles.end() +#ifdef WINDOWS_SYS + && ( (uint64_t)mod_time == it->second.modf_stamp || (uint64_t)mod_time+3600 == it->second.modf_stamp ||(uint64_t)mod_time == it->second.modf_stamp+3600) +#else + && (uint64_t)mod_time == it->second.modf_stamp +#endif + && size == it->second.size) { it->second.time_stamp = now ; + +#ifdef WINDOWS_SYS + if(it->second.time_stamp != (uint64_t)mod_time) + { + std::cerr << "(WW) detected a 1 hour shift in file modification time. This normally happens to many files at once, when daylight saving time shifts (file=\"" << full_path << "\")." << std::endl; + it->second.time_stamp = (uint64_t)mod_time; + } +#endif + known_hash = it->second.hash; #ifdef HASHSTORAGE_DEBUG std::cerr << "Found in cache." << std::endl ; From 8500b5e9f562ee4ebce0e396d7156004b3214c2d Mon Sep 17 00:00:00 2001 From: hunbernd Date: Fri, 23 Sep 2016 21:56:41 +0200 Subject: [PATCH 103/174] Fixed inconsistency in integer sizes that caused segmentation fault while hashing files also broken message on the GUI --- libretroshare/src/file_sharing/hash_cache.cc | 2 +- libretroshare/src/file_sharing/hash_cache.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 3f96abec1..2ae365885 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -146,7 +146,7 @@ void HashStorage::data_tick() std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush(); std::string tmpout; - rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", mHashCounter+1, mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; + rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ; RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 1afa756ca..3860aeabc 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -129,7 +129,7 @@ private: RsMutex mHashMtx ; bool mRunning; - uint32_t mHashCounter; + uint64_t mHashCounter; uint32_t mInactivitySleepTime ; uint64_t mTotalSizeToHash ; uint64_t mTotalHashedSize ; From 5de73df86fa0a4f15bd1742749ede1ed51c6345e Mon Sep 17 00:00:00 2001 From: Silenoz Byronsk Date: Sat, 24 Sep 2016 08:48:35 +0200 Subject: [PATCH 104/174] Update retroshare_de.ts --- retroshare-gui/src/lang/retroshare_de.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/retroshare-gui/src/lang/retroshare_de.ts b/retroshare-gui/src/lang/retroshare_de.ts index 4470cd4dd..0fd9b3ae3 100644 --- a/retroshare-gui/src/lang/retroshare_de.ts +++ b/retroshare-gui/src/lang/retroshare_de.ts @@ -402,7 +402,7 @@ p, li { white-space: pre-wrap; } Description: - Beschreibung + Beschreibung: @@ -514,7 +514,7 @@ p, li { white-space: pre-wrap; } Choose the language used in RetroShare - Wähle die Sprache, die RetroShare verwendet + Wähle die Sprache, die RetroShare verwenden soll @@ -771,7 +771,7 @@ Aber denke daran, dass alle Daten hier VERLOREN gehen werden, wenn wir die Proto Click to change your avatar - Klick zum Ändern deines Avatars + Klicke zum Ändern deines Avatars @@ -1417,7 +1417,7 @@ Aber denke daran, dass alle Daten hier VERLOREN gehen werden, wenn wir die Proto Select lobbies at left to show details. Double click lobbies to enter and chat. Keine Lobby ausgewählt. -Wähle links eine Lobby aus, um Details anzuzeigen. +Wähle links eine Lobby aus, um die Details anzusehen. Doppelklicke auf Lobbys um sie zu betreten und zu chatten. @@ -1630,7 +1630,7 @@ Doppelklicke auf Lobbys um sie zu betreten und zu chatten. Send as plain text by default - + Sende einen standart Klartext From e03bd6cbacca61e71deb58cb50411a956edea2aa Mon Sep 17 00:00:00 2001 From: mr-alice Date: Sat, 24 Sep 2016 13:56:02 +0200 Subject: [PATCH 105/174] fixed bug causing call to closedir on invalid handle --- libretroshare/src/util/folderiterator.cc | 9 +++++++-- libretroshare/src/util/folderiterator.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index bf9bfd68b..415aeb085 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -44,10 +44,10 @@ FolderIterator::FolderIterator(const std::string& folderName) utf16Name += L"/*.*"; handle = FindFirstFileW(utf16Name.c_str(), &fileInfo); - validity = handle != INVALID_HANDLE_VALUE; + is_open = validity = handle != INVALID_HANDLE_VALUE; #else handle = opendir(folderName.c_str()); - validity = handle != NULL; + is_open = validity = handle != NULL; next(); #endif } @@ -156,6 +156,11 @@ bool FolderIterator::closedir() { validity = false; + if(!is_open) + return true ; + + is_open = false ; + #ifdef WINDOWS_SYS return FindClose(handle) != 0; #else diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 1dcc07bc3..21ebacd06 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -50,6 +50,7 @@ public: time_t file_modtime() ; private: + bool is_open; bool validity; #ifdef WINDOWS_SYS From 52ee76ed24a237c27e6a7a8a26d66daf1c00bd57 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 24 Sep 2016 14:14:32 +0200 Subject: [PATCH 106/174] removed annoying debug message in ID system --- libretroshare/src/services/p3idservice.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index bc79d98a1..6914ebaa5 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -2288,7 +2288,9 @@ void p3IdService::requestIdsFromNet() } else { +#ifdef DEBUG_IDS std::cerr << "(EE) no online peers among supply list in ID request for groupId " << cit->first << ". Keeping it until peers show up."<< std::endl; +#endif ++cit ; } } From cb4dc962298f5a7a5fea3cae032d59ce0ac30b68 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 24 Sep 2016 17:12:38 +0200 Subject: [PATCH 107/174] removed debug info in p3filelists --- libretroshare/src/file_sharing/p3filelists.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 409dd4637..366df028c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -210,8 +210,8 @@ int p3FileDatabase::tick() { if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) { - P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; #ifdef DEBUG_FILE_HIERARCHY + P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; mRemoteDirectories[i]->print(); #endif From 19b1c4f5d9350b0bdea85de8f7d5417710a1a41c Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 25 Sep 2016 11:04:42 +0200 Subject: [PATCH 108/174] fixed potential crash due to not checking if mRemoteDirectory[i] is empty in p3FileLists --- libretroshare/src/file_sharing/p3filelists.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 366df028c..2b9d4a660 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -426,22 +426,22 @@ void p3FileDatabase::cleanup() } for(uint32_t i=0;ipeerId()) == friend_set.end()) + if(mRemoteDirectories[i] != NULL && friend_set.find(mRemoteDirectories[i]->peerId()) == friend_set.end()) { #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " removing file list of non friend " << mRemoteDirectories[i]->peerId() << std::endl; #endif - delete mRemoteDirectories[i]; - mRemoteDirectories[i] = NULL ; - mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; friend_set.erase(mRemoteDirectories[i]->peerId()); mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); mFriendIndexTab[i].clear(); + + delete mRemoteDirectories[i]; + mRemoteDirectories[i] = NULL ; } // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. @@ -1040,7 +1040,7 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) RS_STACK_MUTEX(mFLSMtx) ; #ifdef DEBUG_P3FILELISTS - P3FILELISTS_DEBUG() << "Received directory sync request. hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; + P3FILELISTS_DEBUG() << "Received directory sync request from peer " << item->PeerId() << ". hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl; #endif EntryIndex entry_index = DirectoryStorage::NO_INDEX; @@ -1314,7 +1314,7 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) else P3FILELISTS_ERROR() << "(EE) Cannot deserialise dir entry. ERROR. "<< std::endl; -#ifdef DEBUG_FILE_HIERARCHY +#ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << " new content after update: " << std::endl; mRemoteDirectories[fi]->print(); #endif From ddc870c6b15ca57b83f6ddebd314bb91910c97b2 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 25 Sep 2016 11:05:01 +0200 Subject: [PATCH 109/174] added some more debug info in DirectoryStorage class --- .../src/file_sharing/directory_storage.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 0f0c4e2cb..914ff5679 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -597,6 +597,9 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx); +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE + std::cerr << "Serialising Dir entry " << std::hex << indx << " for client id " << client_id << std::endl; +#endif if(dir == NULL) { std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl; @@ -620,7 +623,14 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary return false; } allowed_subdirs.push_back(hash) ; +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE + std::cerr << " pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << std::endl; +#endif } +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE + else + std::cerr << " not pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << ": permission denied for this peer." << std::endl; +#endif unsigned char *section_data = NULL; uint32_t section_size = 0; @@ -673,6 +683,10 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) return false ; free(file_section_data) ; + +#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE + std::cerr << " pushing subfile " << file->hash << ", array position=" << i << " indx=" << dir->subfiles[i] << std::endl; +#endif } #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE From bb4b57776887e5c92d88699b036d99200d67ca2c Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 25 Sep 2016 18:44:28 +0200 Subject: [PATCH 110/174] prevent storage for hash of files that cannot be hashed --- libretroshare/src/file_sharing/hash_cache.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index e42c961fd..2f631abb6 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -150,14 +150,12 @@ void HashStorage::data_tick() RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; - if(!RsDirUtil::getFileHash(job.full_path, hash,size, this)) - std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; - else + if(RsDirUtil::getFileHash(job.full_path, hash,size, this)) + { + // store the result + std::cerr << "done."<< std::endl; - // store the result - - { RS_STACK_MUTEX(mHashMtx) ; HashStorageInfo& info(mFiles[job.full_path]); @@ -168,9 +166,12 @@ void HashStorage::data_tick() info.hash = hash; mChanged = true ; - ++mHashCounter ; mTotalHashedSize += size ; } + else + std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl; + + ++mHashCounter ; } } // call the client From 068955d419edc892a5afb0be91a28b3011fb9c3b Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 25 Sep 2016 18:51:33 +0200 Subject: [PATCH 111/174] fixed up a few debug messages in p3filelists --- libretroshare/src/file_sharing/dir_hierarchy.cc | 2 +- libretroshare/src/file_sharing/filelist_io.cc | 12 ++++++------ libretroshare/src/file_sharing/p3filelists.cc | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 023206d29..28606513d 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -982,7 +982,7 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) mNodes.clear(); mNodes.resize(n_nodes,NULL) ; - for(uint32_t i=0;iencrypt( encryptedData, encDataLen, data,total_size, AuthSSL::getAuthSSL()->OwnId())) { - std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; + std::cerr << "Cannot encrypt encrypted file. Something's wrong." << std::endl; return false; } @@ -159,14 +159,14 @@ bool FileListIO::saveEncryptedDataToFile(const std::string& fname,const unsigned if(!F) { - std::cerr << "Cannot open encrypted file cache for writing: " << fname+".tmp" << std::endl; + std::cerr << "Cannot open encrypted file for writing: " << fname+".tmp" << std::endl; free(encryptedData); return false; } if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) { - std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; + std::cerr << "Could not write entire encrypted file. Out of disc space??" << std::endl; fclose(F) ; free(encryptedData); @@ -190,7 +190,7 @@ bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned cha if(!RsDirUtil::checkFile( fname,file_size,false ) ) { - std::cerr << "Encrypted hash cache file not present." << std::endl; + std::cerr << "Encrypted file " << fname << " not available." << std::endl; return false; } @@ -204,7 +204,7 @@ bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned cha FILE *F = fopen( fname.c_str(),"rb") ; if (!F) { - std::cerr << "Cannot open file for reading encrypted file cache, filename " << fname << std::endl; + std::cerr << "Cannot open encrypted file, filename " << fname << std::endl; return false; } if(fread(buffer,1,file_size,F) != file_size) @@ -221,7 +221,7 @@ bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned cha if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) { - std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; + std::cerr << "Cannot decrypt encrypted file. Something's wrong." << std::endl; return false; } diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 2b9d4a660..d3785c29d 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1262,7 +1262,9 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) RS_STACK_MUTEX(mFLSMtx) ; fi = locked_getFriendIndex(item->PeerId()); - std::cerr << " friend index is " << fi ; +#ifdef DEBUG_P3FILELISTS + P3FILELISTS_DEBUG() << " friend index is " << fi ; +#endif // make sure we have a remote directory for that friend. From 93e70ca30b074359696947ee0f0bec11d17f89e8 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 25 Sep 2016 21:01:03 +0200 Subject: [PATCH 112/174] fixed search on remote files --- libretroshare/src/file_sharing/p3filelists.cc | 120 +++++++++++++++--- 1 file changed, 100 insertions(+), 20 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index d3785c29d..329928d26 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -861,39 +861,119 @@ uint32_t p3FileDatabase::watchPeriod() int p3FileDatabase::SearchKeywords(const std::list& keywords, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) { - std::list firesults; + if(flags & RS_FILE_HINTS_LOCAL) { - RS_STACK_MUTEX(mFLSMtx) ; - - mLocalSharedDirs->searchTerms(keywords,firesults) ; - - for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + std::list firesults; { - void *p=NULL; - convertEntryIndexToPointer(*it,0,p); - *it = (intptr_t)p ; + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalSharedDirs->searchTerms(keywords,firesults) ; + + for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,0,p); + *it = (intptr_t)p ; + } } + + filterResults(firesults,results,flags,client_peer_id) ; } - return filterResults(firesults,results,flags,client_peer_id) ; + if(flags & RS_FILE_HINTS_REMOTE) + { + std::list firesults; + + { + RS_STACK_MUTEX(mFLSMtx) ; + + for(uint32_t i=0;i local_results; + mRemoteDirectories[i]->searchTerms(keywords,local_results) ; + + for(std::list::iterator it(local_results.begin());it!=local_results.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,i+1,p); + firesults.push_back((intptr_t)p) ; + } + } + } + + for(std::list::const_iterator rit ( firesults.begin()); rit != firesults.end(); ++rit) + { + DirDetails dd; + + if(!RequestDirDetails ((void*)(intptr_t)*rit,dd,RS_FILE_HINTS_REMOTE)) + continue ; + + results.push_back(dd); + } + + } + + return !results.empty() ; } + int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& client_peer_id) const { - std::list firesults; + if(flags & RS_FILE_HINTS_LOCAL) { - RS_STACK_MUTEX(mFLSMtx) ; - - mLocalSharedDirs->searchBoolExp(exp,firesults) ; - - for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + std::list firesults; { - void *p=NULL; - convertEntryIndexToPointer(*it,0,p); - *it = (intptr_t)p ; + RS_STACK_MUTEX(mFLSMtx) ; + + mLocalSharedDirs->searchBoolExp(exp,firesults) ; + + for(std::list::iterator it(firesults.begin());it!=firesults.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,0,p); + *it = (intptr_t)p ; + } + } + + filterResults(firesults,results,flags,client_peer_id) ; + } + + if(flags & RS_FILE_HINTS_REMOTE) + { + std::list firesults; + + { + RS_STACK_MUTEX(mFLSMtx) ; + + for(uint32_t i=0;i local_results; + mRemoteDirectories[i]->searchBoolExp(exp,local_results) ; + + for(std::list::iterator it(local_results.begin());it!=local_results.end();++it) + { + void *p=NULL; + convertEntryIndexToPointer(*it,i+1,p); + firesults.push_back((intptr_t)p) ; + } + + } + } + + for(std::list::const_iterator rit ( firesults.begin()); rit != firesults.end(); ++rit) + { + DirDetails dd; + + if(!RequestDirDetails ((void*)(intptr_t)*rit,dd,RS_FILE_HINTS_REMOTE)) + continue ; + + results.push_back(dd); } } - return filterResults(firesults,results,flags,client_peer_id) ; + return !results.empty() ; + } bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const { From be6370ef132165d3acbe4dcb7d078a735b0bb4bf Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 25 Sep 2016 21:34:18 +0200 Subject: [PATCH 113/174] added missing code for searchTerms in new p3filelists service --- libretroshare/src/file_sharing/dir_hierarchy.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 28606513d..d5cd7b4f6 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -23,6 +23,7 @@ * */ #include +#include #include #include "util/rsdir.h" #include "util/rsprint.h" @@ -687,13 +688,12 @@ int InternalFileHierarchyStorage::searchTerms(const std::list& term { /* always ignore case */ const std::string &str2 = (*iter); -#ifdef TODO - if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), CompareCharIC() )) + + if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), RsRegularExpression::CompareCharIC() )) { - results.push_back(fit->second); + results.push_back(it->second); break; } -#endif } } return 0 ; From 1125dfe6d1e07899e5f6c1f71bd5590e0d469a2d Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 25 Sep 2016 23:42:20 +0200 Subject: [PATCH 114/174] fixed folder iterator to properly handle broken symbolic links --- .../src/file_sharing/dir_hierarchy.h | 2 +- libretroshare/src/util/folderiterator.cc | 102 +++++++++++------- 2 files changed, 62 insertions(+), 42 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index 9320f3102..7220aacc8 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -144,7 +144,7 @@ public: bool searchHash(const RsFileHash& hash,std::list& results); int searchBoolExp(RsRegularExpression::Expression * exp, std::list &results) const ; - int searchTerms(const std::list& terms, std::list &results) const ; + int searchTerms(const std::list& terms, std::list &results) const ; // does a logical OR between items of the list of terms bool check(std::string& error_string) const ;// checks consistency of storage. diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 415aeb085..62d451e21 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -11,6 +11,8 @@ #include "folderiterator.h" #include "rsstring.h" +#define DEBUG_FOLDER_ITERATOR 1 + namespace librs { namespace util { @@ -59,57 +61,75 @@ FolderIterator::~FolderIterator() void FolderIterator::next() { - do { - if(!readdir()) - { - validity = false ; - break ; - } - + while(readdir()) + { d_name(mFileName); - } while(mFileName == "." || mFileName == "..") ; - mFullPath = mFolderName + "/" + mFileName ; + if(mFileName == "." || mFileName == "..") + continue ; - struct stat64 buf ; + mFullPath = mFolderName + "/" + mFileName ; + + struct stat64 buf ; + +#ifdef DEBUG_FOLDER_ITERATOR + std::cerr << "FolderIterator: next. Looking into file " << mFileName ; +#endif #ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) + std::wstring wfullname; + librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname); + if ( 0 == _wstati64(wfullname.c_str(), &buf)) #else - if ( 0 == stat64(mFullPath.c_str(), &buf)) + if ( 0 == stat64(mFullPath.c_str(), &buf)) #endif - { - mFileModTime = buf.st_mtime ; - mStatInfoOk = true; + { + mFileModTime = buf.st_mtime ; + mStatInfoOk = true; - if (S_ISDIR(buf.st_mode)) - { - mType = TYPE_DIR ; - mFileSize = 0 ; - mFileModTime = buf.st_mtime; - } - else if (S_ISREG(buf.st_mode)) - { - mType = TYPE_FILE ; - mFileSize = buf.st_size; - mFileModTime = buf.st_mtime; - } - else - { - mType = TYPE_UNKNOWN ; - mFileSize = 0 ; - mFileModTime = 0; + if (S_ISDIR(buf.st_mode)) + { +#ifdef DEBUG_FOLDER_ITERATOR + std::cerr << ": is a directory" << std::endl; +#endif + + mType = TYPE_DIR ; + mFileSize = 0 ; + mFileModTime = buf.st_mtime; + + return ; + } + + if (S_ISREG(buf.st_mode)) + { +#ifdef DEBUG_FOLDER_ITERATOR + std::cerr << ": is a file" << std::endl; +#endif + + mType = TYPE_FILE ; + mFileSize = buf.st_size; + mFileModTime = buf.st_mtime; + + return ; + } } + +#ifdef DEBUG_FOLDER_ITERATOR + std::cerr << ": is unknown skipping" << std::endl; +#endif + + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; } - else - { - mType = TYPE_UNKNOWN ; - mFileSize = 0 ; - mFileModTime = 0; - validity = false ; - } +#ifdef DEBUG_FOLDER_ITERATOR + std::cerr << "End of directory." << std::endl; +#endif + + mType = TYPE_UNKNOWN ; + mFileSize = 0 ; + mFileModTime = 0; + validity = false ; } bool FolderIterator::readdir() From 3beaa974b08d84a01c2a688ae1cd3440d2392a32 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Mon, 26 Sep 2016 10:03:51 +0200 Subject: [PATCH 115/174] fixed potential crash when an entry in mRemoteDirectories is NULL --- libretroshare/src/file_sharing/p3filelists.cc | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 329928d26..ff0021a6e 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -207,19 +207,20 @@ int p3FileDatabase::tick() mServCtrl->getPeersConnected(getServiceInfo().mServiceType, online_peers) ; for(uint32_t i=0;ipeerId()) != online_peers.end()) + if(mRemoteDirectories[i] != NULL) { + if(online_peers.find(mRemoteDirectories[i]->peerId()) != online_peers.end()) + { #ifdef DEBUG_FILE_HIERARCHY - P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; - mRemoteDirectories[i]->print(); + P3FILELISTS_DEBUG() << "Launching recurs sweep of friend directory " << mRemoteDirectories[i]->peerId() << ". Content currently is:" << std::endl; + mRemoteDirectories[i]->print(); #endif - locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root(),0) ; - } + locked_recursSweepRemoteDirectory(mRemoteDirectories[i],mRemoteDirectories[i]->root(),0) ; + } - mRemoteDirectories[i]->checkSave() ; - } + mRemoteDirectories[i]->checkSave() ; + } mLastRemoteDirSweepTS = now; From b95fbe34bcb2ee6f357b2f537c4f7b4cb70efaa7 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Mon, 26 Sep 2016 10:49:59 +0200 Subject: [PATCH 116/174] added a few more missing checks for null entries in mRemoteDirectories --- libretroshare/src/file_sharing/p3filelists.cc | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index ff0021a6e..1c7dbaa36 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -722,16 +722,17 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags d.children.push_back(stub); } else for(uint32_t i=0;iroot(),i+1,p); + if(mRemoteDirectories[i] != NULL) + { + void *p; + convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i+1,p); - DirStub stub; - stub.type = DIR_TYPE_PERSON; - stub.name = mRemoteDirectories[i]->peerId().toStdString(); - stub.ref = p; - d.children.push_back(stub); - } + DirStub stub; + stub.type = DIR_TYPE_PERSON; + stub.name = mRemoteDirectories[i]->peerId().toStdString(); + stub.ref = p; + d.children.push_back(stub); + } d.count = d.children.size(); @@ -757,7 +758,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags // Case where the index is the top of a single person. Can be us, or a friend. - if(!storage->extractData(e,d)) + if(storage==NULL || !storage->extractData(e,d)) { P3FILELISTS_ERROR() << "(EE) request on index " << e << ", for directory ID=" << storage->peerId() << " failed. This should not happen." << std::endl; return false ; @@ -823,8 +824,10 @@ uint32_t p3FileDatabase::getType(void *ref) const if(fi == 0) return mLocalSharedDirs->getEntryType(e) ; - else + else if(mRemoteDirectories[fi-1]!=NULL) return mRemoteDirectories[fi-1]->getEntryType(e) ; + else + return DIR_TYPE_ROOT ;// some failure case. Should not happen } void p3FileDatabase::forceDirectoryCheck() // Force re-sweep the directories and see what's changed From 15f69b223689e02e0a005d609c54c4e5ba9f1235 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Mon, 26 Sep 2016 11:11:22 +0200 Subject: [PATCH 117/174] fixed crash in debug output in p3filelists --- libretroshare/src/file_sharing/p3filelists.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 1c7dbaa36..c0f29bf7c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -760,7 +760,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags if(storage==NULL || !storage->extractData(e,d)) { - P3FILELISTS_ERROR() << "(EE) request on index " << e << ", for directory ID=" << storage->peerId() << " failed. This should not happen." << std::endl; + P3FILELISTS_ERROR() << "(EE) request on index " << e << ", for directory ID=" << ((storage==NULL)?("[NULL]"):(storage->peerId().toStdString())) << " failed. This should not happen." << std::endl; return false ; } From fc1a350869a19a0f8d30836f7f2292c623ef4543 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 26 Sep 2016 21:12:13 +0200 Subject: [PATCH 118/174] added automated import of old hash cache in order to avoid re-hashing everything when switching to the new system --- libretroshare/src/file_sharing/hash_cache.cc | 120 ++++++++++++++++++- libretroshare/src/file_sharing/hash_cache.h | 3 +- 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 2f631abb6..bfe9529ad 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -47,7 +47,9 @@ HashStorage::HashStorage(const std::string& save_file_name) { RS_STACK_MUTEX(mHashMtx) ; - locked_load() ; + + if(!locked_load()) + try_load_import_old_hash_cache(); } } static std::string friendlyUnit(uint64_t val) @@ -287,7 +289,7 @@ void HashStorage::clean() #endif } -void HashStorage::locked_load() +bool HashStorage::locked_load() { unsigned char *data = NULL ; uint32_t data_size=0; @@ -295,7 +297,7 @@ void HashStorage::locked_load() if(!FileListIO::loadEncryptedDataFromFile(mFilePath,data,data_size)) { std::cerr << "(EE) Cannot read hash cache." << std::endl; - return ; + return false; } uint32_t offset = 0 ; HashStorageInfo info ; @@ -315,6 +317,7 @@ void HashStorage::locked_load() #ifdef HASHSTORAGE_DEBUG std::cerr << n << " entries loaded." << std::endl ; #endif + return true ; } void HashStorage::locked_save() @@ -392,3 +395,114 @@ std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& inf { return o << info.hash << " " << info.size << " " << info.filename ; } + +/********************************************************************************************************************************/ +/* 09/26/2016 */ +/* This method should be removed in a few months. It only helps importing the old hash cache in order to */ +/* spare the re-hashing of everything. If successful, it also renames the old hash cache into a new name so that */ +/* the file is not found at the next attempt. */ +/********************************************************************************************************************************/ +// +#include "rsserver/rsaccounts.h" +#include + +bool HashStorage::try_load_import_old_hash_cache() +{ + // compute file name + + std::string base_dir = rsAccounts->PathAccountDirectory(); + std::string old_cache_filename = base_dir + "/" + "file_cache.bin" ; + + // check for unencrypted + + std::istream *f = NULL ; + uint64_t file_size ; + + if(RsDirUtil::checkFile( old_cache_filename,file_size,false ) ) + { + std::cerr << "Encrypted hash cache file present. Reading it." << std::endl; + + // read the binary stream into memory. + // + RsTemporaryMemory buffer(file_size) ; + + if(buffer == NULL) + return false; + + FILE *F = fopen( old_cache_filename.c_str(),"rb") ; + if (!F) + { + std::cerr << "Cannot open file for reading encrypted file cache, filename " << old_cache_filename << std::endl; + return false; + } + if(fread(buffer,1,file_size,F) != file_size) + { + std::cerr << "Cannot read from file " << old_cache_filename << ": something's wrong." << std::endl; + fclose(F) ; + return false; + } + fclose(F) ; + + void *decrypted_data =NULL; + int decrypted_data_size =0; + + if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) + { + std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; + return false; + } + std::string s((char *)decrypted_data,decrypted_data_size) ; + f = new std::istringstream(s) ; + + free(decrypted_data) ; + } + else + return false; + + std::streamsize max_line_size = 2000 ; // should be enough. Anyway, if we + // miss one entry, we just lose some + // cache itemsn but this is not too + // much of a problem. + char *buff = new char[max_line_size] ; + + std::cerr << "Importing hashCache from file " << old_cache_filename << std::endl ; + int n=0 ; + + std::map tmp_files ; // stored as (full_path, hash_info) + + while(!f->eof()) + { + HashStorageInfo info ; + + f->getline(buff,max_line_size,'\n') ; + info.filename = std::string(buff) ; + + f->getline(buff,max_line_size,'\n') ; //if(sscanf(buff,"%llu",&info.size) != 1) break ; + + info.size = 0 ; + sscanf(buff, RsDirUtil::scanf_string_for_uint(sizeof(info.size)), &info.size); + + f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.time_stamp)),&info.time_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } + f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.modf_stamp)),&info.modf_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } + f->getline(buff,max_line_size,'\n') ; info.hash = RsFileHash(std::string(buff)) ; + + std::cerr << " (" << info.filename << ", " << info.size << ", " << info.time_stamp << ", " << info.modf_stamp << ", " << info.hash << std::endl ; + ++n ; + + tmp_files[info.filename] = info ; + } + + delete[] buff ; + delete f ; + + std::cerr << "(II) Successfully imported hash cache from file " << old_cache_filename << " for a total of " << n << " entries." << std::endl; + std::cerr << "(II) This file is now obsolete, and will be renamed " << old_cache_filename + ".bak" << " in case you needed it. But you can safely remove it." << std::endl; + + RsDirUtil::renameFile(old_cache_filename,old_cache_filename+".bak") ; + + mFiles = tmp_files ; + locked_save() ; // this is called explicitly here because the ticking thread is not active. + + return true; +} +/********************************************************************************************************************************/ diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 3860aeabc..512056596 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -100,7 +100,8 @@ private: // loading/saving the entire hash database to a file void locked_save() ; - void locked_load() ; + bool locked_load() ; + bool try_load_import_old_hash_cache(); bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const; bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const; From a2ff5d27c540d3b88a4e44bcd99c61cb4ad2aaff Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 26 Sep 2016 22:06:30 +0200 Subject: [PATCH 119/174] simplified p3FileList code a little bit: removed mFriendIndexTab and made locked_getFriendIndex() create the remote directories on demand --- libretroshare/src/file_sharing/p3filelists.cc | 97 ++++++++----------- libretroshare/src/file_sharing/p3filelists.h | 1 - 2 files changed, 41 insertions(+), 57 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c0f29bf7c..46f7dfde9 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -67,7 +67,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) throw std::runtime_error("Cannot create base directory to store/access file sharing files.") ; // loads existing indexes for friends. Some might be already present here. - // + // mRemoteDirectories.clear() ; // we should load them! mOwnId = mpeers->getOwnId() ; @@ -76,7 +76,7 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers) mLocalDirWatcher = new LocalDirectoryUpdater(mHashCache,mLocalSharedDirs) ; - mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; + mUpdateFlags = P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED ; mLastRemoteDirSweepTS = 0 ; mLastCleanupTime = 0 ; @@ -154,16 +154,16 @@ RsServiceInfo p3FileDatabase::getServiceInfo() } int p3FileDatabase::tick() { - // tick the input/output list of update items and process them - // - tickRecv() ; - tickSend() ; + // tick the input/output list of update items and process them + // + tickRecv() ; + tickSend() ; time_t now = time(NULL) ; // cleanup - // - remove/delete shared file lists for people who are not friend anymore - // - + // - remove/delete shared file lists for people who are not friend anymore + // - if(mLastCleanupTime + 5 < now) { @@ -176,7 +176,7 @@ int p3FileDatabase::tick() if(last_print_time + 20 < now) { RS_STACK_MUTEX(mFLSMtx) ; - + #ifdef DEBUG_FILE_HIERARCHY mLocalSharedDirs->print(); #endif @@ -439,10 +439,18 @@ void p3FileDatabase::cleanup() friend_set.erase(mRemoteDirectories[i]->peerId()); mFriendIndexMap.erase(mRemoteDirectories[i]->peerId()); - mFriendIndexTab[i].clear(); delete mRemoteDirectories[i]; mRemoteDirectories[i] = NULL ; + + // now, in order to avoid empty seats, just move the last one here, and update indexes + + mRemoteDirectories[i] = mRemoteDirectories.back(); + mRemoteDirectories.pop_back(); + + mFriendIndexMap[mRemoteDirectories[i]->peerId()] = i; + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ; } // look through the remaining list of friends, which are the ones for which no remoteDirectoryStorage class has been allocated. @@ -452,21 +460,6 @@ void p3FileDatabase::cleanup() // Check if a remote directory exists for that friend, possibly creating the index. uint32_t friend_index = locked_getFriendIndex(*it) ; - - if(mRemoteDirectories.size() > friend_index && mRemoteDirectories[friend_index] != NULL) - continue ; - -#ifdef DEBUG_P3FILELISTS - P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; -#endif - - if(mRemoteDirectories.size() <= friend_index) - mRemoteDirectories.resize(friend_index+1,NULL) ; - - mRemoteDirectories[friend_index] = new RemoteDirectoryStorage(*it,makeRemoteFileName(*it)); - - mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ; - mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; } // cancel existing requests for which the peer is offline @@ -511,8 +504,8 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid) { // allocate a new index for that friend, and tell that we should save. uint32_t found = 0 ; - for(uint32_t i=0;i= (1 << NB_FRIEND_INDEX_BITS) ) + if(mRemoteDirectories.size() >= (1 << NB_FRIEND_INDEX_BITS) ) { std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than " << (1<second) - mRemoteDirectories.resize(it->second,NULL) ; + { + mRemoteDirectories.resize(it->second+1,NULL) ; + mRemoteDirectories[it->second] = new RemoteDirectoryStorage(pid,makeRemoteFileName(pid)); + + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ; + mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED ; + +#ifdef DEBUG_P3FILELISTS + P3FILELISTS_DEBUG() << " adding missing remote dir entry for friend " << *it << ", with index " << friend_index << std::endl; +#endif + } return it->second; } } -const RsPeerId& p3FileDatabase::locked_getFriendFromIndex(uint32_t indx) const -{ - static const RsPeerId null_id ; - - if(indx >= mFriendIndexTab.size()) - return null_id ; - - if(mFriendIndexTab[indx].isNull()) - { - std::cerr << "(EE) null friend id requested from index " << indx << ": this is a bug, most likely" << std::endl; - return null_id ; - } - - return mFriendIndexTab[indx]; -} bool p3FileDatabase::convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) { // trust me, I can do this ;-) @@ -1352,12 +1343,6 @@ void p3FileDatabase::handleDirSyncResponse(RsFileListsSyncResponseItem *sitem) // make sure we have a remote directory for that friend. - if(mRemoteDirectories.size() <= fi) - mRemoteDirectories.resize(fi+1,NULL) ; - - if(mRemoteDirectories[fi] == NULL) - mRemoteDirectories[fi] = new RemoteDirectoryStorage(item->PeerId(),makeRemoteFileName(item->PeerId())); - if(!mRemoteDirectories[fi]->getIndexFromDirHash(item->entry_hash,entry_index)) { std::cerr << std::endl; diff --git a/libretroshare/src/file_sharing/p3filelists.h b/libretroshare/src/file_sharing/p3filelists.h index 67834bfa7..3b155a109 100644 --- a/libretroshare/src/file_sharing/p3filelists.h +++ b/libretroshare/src/file_sharing/p3filelists.h @@ -196,7 +196,6 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p); static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ; uint32_t locked_getFriendIndex(const RsPeerId& pid); - const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const; void handleDirSyncRequest (RsFileListsSyncRequestItem *) ; void handleDirSyncResponse (RsFileListsSyncResponseItem *) ; From 9d72b064e1ffb894f54c1bb01312ba03ad9e634e Mon Sep 17 00:00:00 2001 From: sehraf Date: Tue, 27 Sep 2016 09:50:59 +0200 Subject: [PATCH 120/174] add thread names & remove some warnings --- libretroshare/src/file_sharing/dir_hierarchy.cc | 4 ++-- libretroshare/src/file_sharing/directory_storage.cc | 4 ++++ libretroshare/src/file_sharing/directory_updater.cc | 4 ++-- libretroshare/src/file_sharing/hash_cache.cc | 4 +++- libretroshare/src/file_sharing/p3filelists.cc | 11 +++++------ libretroshare/src/file_sharing/rsfilelistitems.cc | 2 ++ 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index d5cd7b4f6..3c360d973 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -184,7 +184,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage return true; } -RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string& dir_name,const std::string& dir_parent_path) +RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string &/*dir_name*/, const std::string &/*dir_parent_path*/) { // What we need here: a unique identifier // - that cannot be bruteforced to find the real directory name and path @@ -226,7 +226,7 @@ bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex parent_dir.subdirs[i] = parent_dir.subdirs.back() ; parent_dir.subdirs.pop_back(); - bool res = recursRemoveDirectory(indx) ; + recursRemoveDirectory(indx) ; #ifdef DEBUG_DIRECTORY_STORAGE print(); std::string err ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 914ff5679..6d5bab024 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -426,6 +426,10 @@ void LocalDirectoryStorage::updateTimeStamps() #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl; +#else + // remove unused variable warning + // variable is only used for debugging + (void)last_modf_time; #endif } } diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index c2e20aeca..0a656417a 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -55,7 +55,7 @@ void LocalDirectoryUpdater::setEnabled(bool b) return ; if(b) - start() ; + start("fs dir updater") ; else shutdown(); @@ -203,7 +203,7 @@ bool LocalDirectoryUpdater::inDirectoryCheck() const return mHashCache->isRunning(); } -void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size) +void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string &/*name*/, const RsFileHash &hash, uint64_t /*size*/) { if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash)) std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index bfe9529ad..abca37add 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -252,7 +252,7 @@ bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mHashCounter = 0; mTotalHashedSize = 0; - start() ; + start("fs hash cache") ; } return false; @@ -301,7 +301,9 @@ bool HashStorage::locked_load() } uint32_t offset = 0 ; HashStorageInfo info ; +#ifdef HASHSTORAGE_DEBUG uint32_t n=0; +#endif while(offset < data_size) if(readHashStorageInfo(data,data_size,offset,info)) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 46f7dfde9..268b2e6f4 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -239,7 +239,7 @@ void p3FileDatabase::startThreads() #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ; #endif - mLocalDirWatcher->start(); + mLocalDirWatcher->start("fs dir watcher"); #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; #endif @@ -458,8 +458,7 @@ void p3FileDatabase::cleanup() for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) { // Check if a remote directory exists for that friend, possibly creating the index. - - uint32_t friend_index = locked_getFriendIndex(*it) ; + locked_getFriendIndex(*it) ; } // cancel existing requests for which the peer is offline @@ -639,8 +638,8 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; } - else - return false; + else + return false; uint32_t fi; DirectoryStorage::EntryIndex e ; @@ -788,7 +787,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags return true; } -int p3FileDatabase::RequestDirDetails(const RsPeerId& uid,const std::string& path, DirDetails &details) const +int p3FileDatabase::RequestDirDetails(const RsPeerId &/*uid*/, const std::string &/*path*/, DirDetails &/*details*/) const { NOT_IMPLEMENTED(); return 0; diff --git a/libretroshare/src/file_sharing/rsfilelistitems.cc b/libretroshare/src/file_sharing/rsfilelistitems.cc index bd07bb5f6..800f2f3d8 100644 --- a/libretroshare/src/file_sharing/rsfilelistitems.cc +++ b/libretroshare/src/file_sharing/rsfilelistitems.cc @@ -226,10 +226,12 @@ RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncRespons RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem(); + /* uint32_t entry_index ; // index of the directory to sync uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc. uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below. uint64_t request_id; // use to determine if changes that have occured since last hash + */ ok &= item->entry_hash.deserialise(data, *size, offset); ok &= item->checksum.deserialise(data, *size, offset); From 2916c8b7c4e0a65ea710d793a98b24a4e8058f67 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Tue, 27 Sep 2016 10:13:51 +0200 Subject: [PATCH 121/174] removed debug output in folderiterator --- libretroshare/src/util/folderiterator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 62d451e21..b50ef99aa 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -11,7 +11,7 @@ #include "folderiterator.h" #include "rsstring.h" -#define DEBUG_FOLDER_ITERATOR 1 +//#define DEBUG_FOLDER_ITERATOR 1 namespace librs { namespace util { From 37a4e7750c2143a569dd2bab0bd8d22717ee4c7c Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 27 Sep 2016 20:13:02 +0200 Subject: [PATCH 122/174] removed debug info in directory_storage.cc --- libretroshare/src/file_sharing/directory_storage.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 6d5bab024..612fda1d6 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -746,9 +746,11 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) return false ; +#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE std::cerr << " dir name : \"" << dir_name << "\"" << std::endl; std::cerr << " most recent time : " << most_recent_time << std::endl; std::cerr << " modification time : " << dir_modtime << std::endl; +#endif // serialise number of subdirs and number of subfiles @@ -757,8 +759,10 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs )) return false ; if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles )) return false ; +#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE std::cerr << " number of subdirs : " << n_subdirs << std::endl; std::cerr << " number of files : " << n_subfiles << std::endl; +#endif // serialise subdirs entry indexes @@ -803,8 +807,9 @@ bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,co } RS_STACK_MUTEX(mDirStorageMtx) ; - +#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE std::cerr << " updating dir entry..." << std::endl; +#endif // First create the entries for each subdir and each subfile, if needed. if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_array)) From c26970feba1c84e7ff32813b6ad1cfc58276ee3b Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 27 Sep 2016 20:20:12 +0200 Subject: [PATCH 123/174] fixed open folder from File Sharing dialog --- retroshare-gui/src/gui/RemoteDirModel.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index bf644713e..be9e28580 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -1133,10 +1133,10 @@ void RetroshareDirModel::openSelected(const QModelIndexList &qmil) { if ((*it).type & DIR_TYPE_PERSON) continue; - std::string path, name; - rsFiles->ConvertSharedFilePath((*it).path, path); + //std::string path, name; + //rsFiles->ConvertSharedFilePath((*it).path, path); - QDir dir(QString::fromUtf8(path.c_str())); + QDir dir(QString::fromUtf8((*it).path.c_str())); QString dest; if ((*it).type & DIR_TYPE_FILE) { dest = dir.absoluteFilePath(QString::fromUtf8(it->name.c_str())); From 5547d71ac74204584a92a4bc8727d09cb6884dde Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 27 Sep 2016 21:09:13 +0200 Subject: [PATCH 124/174] fixed advanced search not closing --- retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp index fa80c7ec9..fbb78200c 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp @@ -738,8 +738,8 @@ void SearchDialog::showAdvSearchDialog(bool show) if (advSearchDialog == 0 && show) { advSearchDialog = new AdvancedSearchDialog(); - connect(advSearchDialog, SIGNAL(search(Expression*)), - this, SLOT(advancedSearch(Expression*))); + connect(advSearchDialog, SIGNAL(search(RsRegularExpression::Expression*)), + this, SLOT(advancedSearch(RsRegularExpression::Expression*))); } if (show) { advSearchDialog->show(); From bf0777fd9e62852ecda34bc31b2f10b8de1c2cb8 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 27 Sep 2016 23:13:59 +0200 Subject: [PATCH 125/174] fixed update of DirHash list and FileHash list when files and dirs are removed, while keeping the cost low --- .../src/file_sharing/dir_hierarchy.cc | 88 +++++++++++++------ .../src/file_sharing/dir_hierarchy.h | 6 +- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 3c360d973..3b0e49bed 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -61,8 +61,6 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0) de->dir_hash=RsFileHash() ; // null hash is root by convention. mNodes.push_back(de) ; - -#warning not very elegant. We should remove the leading / mDirHashes[de->dir_hash] = 0 ; } @@ -75,24 +73,43 @@ bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::E return true; } -bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const +bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) { - std::map::const_iterator it = mDirHashes.find(hash) ; + std::map::iterator it = mDirHashes.find(hash) ; if(it == mDirHashes.end()) return false; index = it->second; + + // make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of dir hashes: when we need them, we check them. + if(!checkIndex(index, FileStorageNode::TYPE_DIR) || static_cast(mNodes[index])->dir_hash != hash) + { + std::cerr << "(II) removing non existing hash from dir hash list: " << hash << std::endl; + + mDirHashes.erase(it) ; + return false ; + } return true; } -bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const +bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) { - std::map::const_iterator it = mFileHashes.find(hash) ; + std::map::iterator it = mFileHashes.find(hash) ; if(it == mFileHashes.end()) return false; index = it->second; + + // make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of file hashes: when we need them, we check them. + if(!checkIndex(it->second, FileStorageNode::TYPE_FILE) || static_cast(mNodes[index])->file_hash != hash) + { + std::cerr << "(II) removing non existing hash from file hash list: " << hash << std::endl; + + mFileHashes.erase(it) ; + return false ; + } + return true; } @@ -226,7 +243,7 @@ bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex parent_dir.subdirs[i] = parent_dir.subdirs.back() ; parent_dir.subdirs.pop_back(); - recursRemoveDirectory(indx) ; + recursRemoveDirectory(indx) ; #ifdef DEBUG_DIRECTORY_STORAGE print(); std::string err ; @@ -633,12 +650,12 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::getSubDirIndex(Direct bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,std::list& results) { - std::map::const_iterator it = mFileHashes.find(hash); + DirectoryStorage::EntryIndex indx ; - if( it != mFileHashes.end() ) + if(getIndexFromFileHash(hash,indx)) { results.clear(); - results.push_back(it->second) ; + results.push_back(indx) ; return true ; } else @@ -699,55 +716,68 @@ int InternalFileHierarchyStorage::searchTerms(const std::list& term return 0 ; } -bool InternalFileHierarchyStorage::check(std::string& error_string) const // checks consistency of storage. +bool InternalFileHierarchyStorage::check(std::string& error_string) // checks consistency of storage. { // recurs go through all entries, check that all std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. hits[0] = 1 ; // because 0 is never the child of anyone + std::map tmp_hashes ; for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) { // stamp the kids - const DirEntry& de = *static_cast(mNodes[i]) ; + DirEntry& de = *static_cast(mNodes[i]) ; - for(uint32_t j=0;j= mNodes.size()) { - error_string = "Node child out of tab!" ; - return false ; + error_string += " - Node child out of tab!" ; + de.subdirs[j] = de.subdirs.back() ; + de.subdirs.pop_back(); } - if(hits[de.subdirs[j]] != 0) + else if(hits[de.subdirs[j]] != 0) { - error_string = "Double hit on a single node" ; - return false; + error_string += " - Double hit on a single node" ; + de.subdirs[j] = de.subdirs.back() ; + de.subdirs.pop_back(); + } + else + { + hits[de.subdirs[j]] = 1; + ++j ; } - hits[de.subdirs[j]] = 1; } - for(uint32_t j=0;j= mNodes.size()) { - error_string = "Node child out of tab!" ; - return false ; + error_string += " - Node child out of tab!" ; + de.subfiles[j] = de.subfiles.back() ; + de.subfiles.pop_back(); } - if(hits[de.subfiles[j]] != 0) + else if(hits[de.subfiles[j]] != 0) { - error_string = "Double hit on a single node" ; - return false; + error_string += " - Double hit on a single node" ; + de.subfiles[j] = de.subfiles.back() ; + de.subfiles.pop_back(); + } + else + { + hits[de.subfiles[j]] = 1; + ++j ; } - hits[de.subfiles[j]] = 1; } - } for(uint32_t i=0;i &results) const ; int searchTerms(const std::list& terms, std::list &results) const ; // does a logical OR between items of the list of terms - bool check(std::string& error_string) const ;// checks consistency of storage. + bool check(std::string& error_string) ;// checks consistency of storage. void print() const; From 9672fe33d0d237d00e445ec9ec780e2497c6f636 Mon Sep 17 00:00:00 2001 From: Phenom Date: Wed, 28 Sep 2016 11:35:44 +0200 Subject: [PATCH 126/174] Fix InternalFileHierarchyStorage::check, error_string will never be shown as always return true. --- libretroshare/src/file_sharing/dir_hierarchy.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 3b0e49bed..0a65ee371 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -720,9 +720,9 @@ bool InternalFileHierarchyStorage::check(std::string& error_string) // checks co { // recurs go through all entries, check that all + error_string = ""; std::vector hits(mNodes.size(),0) ; // count hits of children. Should be 1 for all in the end. Otherwise there's an error. hits[0] = 1 ; // because 0 is never the child of anyone - std::map tmp_hashes ; for(uint32_t i=0;itype() == FileStorageNode::TYPE_DIR) @@ -734,13 +734,13 @@ bool InternalFileHierarchyStorage::check(std::string& error_string) // checks co { if(de.subdirs[j] >= mNodes.size()) { - error_string += " - Node child out of tab!" ; + error_string += " - Node child dir out of tab!" ; de.subdirs[j] = de.subdirs.back() ; de.subdirs.pop_back(); } else if(hits[de.subdirs[j]] != 0) { - error_string += " - Double hit on a single node" ; + error_string += " - Double hit on a single node dir." ; de.subdirs[j] = de.subdirs.back() ; de.subdirs.pop_back(); } @@ -754,13 +754,13 @@ bool InternalFileHierarchyStorage::check(std::string& error_string) // checks co { if(de.subfiles[j] >= mNodes.size()) { - error_string += " - Node child out of tab!" ; + error_string += " - Node child file out of tab!" ; de.subfiles[j] = de.subfiles.back() ; de.subfiles.pop_back(); } else if(hits[de.subfiles[j]] != 0) { - error_string += " - Double hit on a single node" ; + error_string += " - Double hit on a single node file." ; de.subfiles[j] = de.subfiles.back() ; de.subfiles.pop_back(); } @@ -780,7 +780,7 @@ bool InternalFileHierarchyStorage::check(std::string& error_string) // checks co mNodes[i] = NULL ; } - return true; + return error_string.empty();; } void InternalFileHierarchyStorage::print() const From 6e4ba76bbb29b633debe9cc04de961b909022830 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 28 Sep 2016 21:55:03 +0200 Subject: [PATCH 127/174] improved display of shared directories for friends with no update info --- .../src/file_sharing/directory_storage.cc | 1 + retroshare-gui/src/gui/RemoteDirModel.cpp | 34 +++++++++++++------ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 612fda1d6..0f0f065d4 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -242,6 +242,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.hash.clear() ; d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size(); d.min_age = now - dir_entry->dir_most_recent_time ; + d.age = now - dir_entry->dir_modtime ; d.name = dir_entry->dir_name; d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; d.parent = (void*)(intptr_t)dir_entry->parent_index ; diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index be9e28580..ca550ea88 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -63,6 +63,11 @@ Qt::DropActions RetroshareDirModel::supportedDragActions() const } #endif +static bool isNewerThanEpoque(uint32_t age) +{ + return age < time(NULL) - 1000 ; // this should be conservative enough +} + void FlatStyle_RDM::update() { if(_needs_update) @@ -241,7 +246,7 @@ QString RetroshareDirModel::getAgeIndicatorString(const DirDetails &details) con QString ret(""); QString nind = tr("NEW"); // QString oind = tr("OLD"); - uint32_t age = details.age; + uint32_t age = details.min_age; switch (ageIndicator) { case IND_LAST_DAY: @@ -271,7 +276,7 @@ QVariant RetroshareDirModel::decorationRole(const DirDetails& details,int coln) return QVariant() ; if (details.type == DIR_TYPE_PERSON) - { + { if(details.min_age > ageIndicator) return QIcon(":/images/folder_grey.png"); else if (ageIndicator == IND_LAST_DAY ) @@ -321,14 +326,16 @@ QVariant TreeStyle_RDM::displayRole(const DirDetails& details,int coln) const if (details.type == DIR_TYPE_PERSON) /* Person */ { - switch(coln) + switch(coln) { case 0: return (RemoteMode)?(QString::fromUtf8(rsPeers->getPeerName(details.id).c_str())):tr("My files"); case 1: return QString() ; - case 2: - return misc::userFriendlyDuration(details.min_age); + case 2: if(!isNewerThanEpoque(details.min_age)) + return QString(); + else + return misc::userFriendlyDuration(details.min_age); default: return QString() ; } @@ -342,7 +349,7 @@ QVariant TreeStyle_RDM::displayRole(const DirDetails& details,int coln) const case 1: return misc::friendlyUnit(details.count); case 2: - return misc::userFriendlyDuration(details.age); + return misc::userFriendlyDuration(details.min_age); case 3: return getFlagsString(details.flags); // case 4: @@ -417,7 +424,7 @@ QVariant FlatStyle_RDM::displayRole(const DirDetails& details,int coln) const { case 0: return QString::fromUtf8(details.name.c_str()); case 1: return misc::friendlyUnit(details.count); - case 2: return misc::userFriendlyDuration(details.age); + case 2: return misc::userFriendlyDuration(details.min_age); case 3: return QString::fromUtf8(rsPeers->getPeerName(details.id).c_str()); case 4: return computeDirectoryPath(details); default: @@ -457,7 +464,7 @@ QVariant TreeStyle_RDM::sortRole(const QModelIndex& /*index*/,const DirDetails& case 1: return (qulonglong) details.count; case 2: - return details.age; + return details.min_age; case 3: return getFlagsString(details.flags); case 4: @@ -503,7 +510,7 @@ QVariant FlatStyle_RDM::sortRole(const QModelIndex& index,const DirDetails& deta { case 0: return QString::fromUtf8(details.name.c_str()); case 1: return (qulonglong) details.count; - case 2: return details.age; + case 2: return details.min_age; case 3: return QString::fromUtf8(rsPeers->getPeerName(details.id).c_str()); case 4: { @@ -570,7 +577,11 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const if(role == Qt::DecorationRole) return decorationRole(details,coln) ; - /***************** + if(role == Qt::ToolTipRole) + if(!isNewerThanEpoque(details.min_age)) + return tr("This node hasn't sent any directory information yet.") ; + + /***************** Qt::EditRole Qt::ToolTipRole Qt::StatusTipRole @@ -842,7 +853,8 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const switch(details.type) { - case DIR_TYPE_PERSON: return Qt::ItemIsEnabled; + // we grey out a person that has never been updated. It's easy to spot these, since the min age of the directory is approx equal to time(NULL), which exceeds 40 years. + case DIR_TYPE_PERSON:return isNewerThanEpoque(details.min_age)? (Qt::ItemIsEnabled):(Qt::NoItemFlags) ; case DIR_TYPE_DIR: return Qt::ItemIsSelectable | Qt::ItemIsEnabled; case DIR_TYPE_FILE: return Qt::ItemIsSelectable | Qt::ItemIsDragEnabled | Qt::ItemIsEnabled; } From c73a898a742175aa62631b4cfafe07fd31df17f3 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 28 Sep 2016 22:26:22 +0200 Subject: [PATCH 128/174] updated ubuntu changelog --- build_scripts/Debian+Ubuntu/changelog | 101 ++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/build_scripts/Debian+Ubuntu/changelog b/build_scripts/Debian+Ubuntu/changelog index 6230d6c4b..fa6a37703 100644 --- a/build_scripts/Debian+Ubuntu/changelog +++ b/build_scripts/Debian+Ubuntu/changelog @@ -1,5 +1,106 @@ retroshare06 (0.6.1-1.XXXXXX~YYYYYY) YYYYYY; urgency=low + Warning: new file list sharing system. + * Old list of shares is not preserved and should be re-defined. + * The hash cache however is imported from the old format, meaning that Retroshare should not re-hash new files that were already shared before. + * New system offers interactive browsing of shared dirs from online friends and background update of directory lists + * The old cachestrapper system is removed since it is now obsolete. + + 6e4ba76 csoler Wed, 28 Sep 2016 21:55:03 +0200 improved display of shared directories for friends with no update info + 1d977dd csoler Wed, 28 Sep 2016 16:39:48 +0200 Merge pull request #518 from PhenomRetroShare/Fix_InternalFileHierarchyStorage__check + 9672fe3 Phenom Wed, 28 Sep 2016 11:35:44 +0200 Fix InternalFileHierarchyStorage::check, error_string will never be shown as always return true. + bf0777f csoler Tue, 27 Sep 2016 23:13:59 +0200 fixed update of DirHash list and FileHash list when files and dirs are removed, while keeping the cost low + 5547d71 csoler Tue, 27 Sep 2016 21:09:13 +0200 fixed advanced search not closing + c26970f csoler Tue, 27 Sep 2016 20:20:12 +0200 fixed open folder from File Sharing dialog + 37a4e77 csoler Tue, 27 Sep 2016 20:13:02 +0200 removed debug info in directory_storage.cc + 2916c8b csoler Tue, 27 Sep 2016 10:13:51 +0200 removed debug output in folderiterator + 0bcb6c3 csoler Tue, 27 Sep 2016 09:59:51 +0200 Merge pull request #517 from sehraf/pr-filelist-fixes + 9d72b06 sehraf Tue, 27 Sep 2016 09:50:59 +0200 add thread names & remove some warnings + a2ff5d2 csoler Mon, 26 Sep 2016 22:06:30 +0200 simplified p3FileList code a little bit: removed mFriendIndexTab and made locked_getFriendIndex() create the remote directories on demand + fc1a350 csoler Mon, 26 Sep 2016 21:12:13 +0200 added automated import of old hash cache in order to avoid re-hashing everything when switching to the new system + 15f69b2 csoler Mon, 26 Sep 2016 11:11:22 +0200 fixed crash in debug output in p3filelists + b95fbe3 csoler Mon, 26 Sep 2016 10:49:59 +0200 added a few more missing checks for null entries in mRemoteDirectories + 3beaa97 csoler Mon, 26 Sep 2016 10:03:51 +0200 fixed potential crash when an entry in mRemoteDirectories is NULL + 1125dfe csoler Sun, 25 Sep 2016 23:42:20 +0200 fixed folder iterator to properly handle broken symbolic links + be6370e csoler Sun, 25 Sep 2016 21:34:18 +0200 added missing code for searchTerms in new p3filelists service + 93e70ca csoler Sun, 25 Sep 2016 21:01:03 +0200 fixed search on remote files + 068955d csoler Sun, 25 Sep 2016 18:51:33 +0200 fixed up a few debug messages in p3filelists + bb4b577 csoler Sun, 25 Sep 2016 18:44:28 +0200 prevent storage for hash of files that cannot be hashed + ddc870c csoler Sun, 25 Sep 2016 11:05:01 +0200 added some more debug info in DirectoryStorage class + 19b1c4f csoler Sun, 25 Sep 2016 11:04:42 +0200 fixed potential crash due to not checking if mRemoteDirectory[i] is empty in p3FileLists + cb4dc96 csoler Sat, 24 Sep 2016 17:12:38 +0200 removed debug info in p3filelists + ee54c02 csoler Sat, 24 Sep 2016 14:47:45 +0200 merged branch v0.6-FileLists locally into master + 52ee76e csoler Sat, 24 Sep 2016 14:14:32 +0200 removed annoying debug message in ID system + 34f1ebb csoler Sat, 24 Sep 2016 14:13:38 +0200 Merge pull request #507 from mr-alice/v0.6-FileLists + e03bd6c mr-alice Sat, 24 Sep 2016 13:56:02 +0200 fixed bug causing call to closedir on invalid handle + 96b333d Mr.Alice Sat, 24 Sep 2016 09:04:25 +0200 Merge pull request #1 from hunbernd/v0.6-FileLists + 8500b5e hunbernd Fri, 23 Sep 2016 21:56:41 +0200 Fixed inconsistency in integer sizes that caused segmentation fault while hashing files also broken message on the GUI + 71a5bff mr-alice Fri, 23 Sep 2016 20:07:57 +0200 added small hack against re-hashing due to stat64 depending on local time on windows + 70c8e14 hunbernd Fri, 23 Sep 2016 19:00:07 +0200 Fix folderiterator.cc:23: error: cannot convert '_stat64*' to '_stati64*' for argument '2' to 'int _wstati64(const wchar_t*, _stati64*)' + 529ab64 hunbernd Fri, 23 Sep 2016 18:43:48 +0200 Fixed missing time.h includes + ca281e4 hunbernd Fri, 23 Sep 2016 18:32:36 +0200 Fixed error with 32 bit GCC Both uint32_t and size_t are resolved to unsigned int, it makes 2 lines conflicted. This happens in filelist_io.cc, lines: 31-32, 36-37, 41-42 + 9ce4b13 mr-alice Fri, 23 Sep 2016 19:03:54 +0200 merged upstream/master + 9f76b76 mr-alice Fri, 23 Sep 2016 19:01:14 +0200 forced closedir in order to avoid leak of file descriptor in FolderIterator + 7bddd4b csoler Fri, 23 Sep 2016 13:18:55 +0200 removed call to exit causing the software to stop when local network interfaces are not ready + 0cc4ebd mr-alice Thu, 22 Sep 2016 21:47:58 +0200 added comments in the file lists code. Removed unused functions. Added a few missing mutexes. + e6e9582 csoler Thu, 22 Sep 2016 13:44:53 +0200 fixed bug in estimation of next FT rate that caused downloads to stuck for a while + 7f99bc2 mr-alice Thu, 22 Sep 2016 10:26:05 +0200 fixed compilation after removal of dbase/ + 1be5990 mr-alice Wed, 21 Sep 2016 22:31:39 +0200 disabled dbase from unittests.pro, which should be cnverted into proper tests for file_sharing. + 0f5cf56 mr-alice Wed, 21 Sep 2016 22:27:53 +0200 removed dbase directory + 0757fb3 mr-alice Tue, 20 Sep 2016 20:51:50 +0200 merged upstream/master + 7488ad4 mr-alice Sun, 18 Sep 2016 22:32:30 +0200 added display of files being hashed with green color and blue icon. This could be improved I suppose. + 0579fb0 mr-alice Sun, 18 Sep 2016 22:14:25 +0200 removed unused tickWatchers() + 9cf9504 mr-alice Sun, 18 Sep 2016 22:05:27 +0200 removed most of debug info in file list sharing code + ff4c438 mr-alice Sun, 18 Sep 2016 21:41:18 +0200 fixed reloading of ShareManager due to postModDirectories and made it reload after addign new share + e4568a8 mr-alice Sun, 18 Sep 2016 21:16:25 +0200 removed some unused code + de104d3 mr-alice Sun, 18 Sep 2016 18:34:39 +0200 fixed up GUI parameters in Directories. Removed HashCache duration handles since this is now automatic + 04c908e MrAlice Fri, 16 Sep 2016 09:35:03 +0200 fixed two bugs in dirlist chunking system causing checksum errors + 3f9acb5 mr-alice Thu, 15 Sep 2016 23:51:46 +0200 added item splitting and checksum checking in p3filelists + 2343c91 mr-alice Thu, 15 Sep 2016 21:40:53 +0200 added display of hash progress + d3b4622 mr-alice Thu, 15 Sep 2016 21:45:00 +0200 merged modifications from origin + ca30b0d MrAlice Thu, 15 Sep 2016 10:41:40 +0200 fixed terrible cost of calling requestDirDetails before createIndex + f58e9c1 mr-alice Wed, 14 Sep 2016 22:41:02 +0200 improved debug output of directory storage, and fixed bug causing loading error (previously crashing files should be removed) + ddc89a6 mr-alice Wed, 14 Sep 2016 21:52:16 +0200 fixed core dump in printout of empty slots. Improved dir sync-ing strategy + e985a2c mr-alice Wed, 14 Sep 2016 20:43:52 +0200 removed call to setDynamicSortFilter in SharedFilesDialog, as a workaround for the crash when sorting + afbb641 mr-alice Wed, 14 Sep 2016 20:41:22 +0200 fixed forceDirectoryCheck and inDirectoryCheck methods + 93818f6 mr-alice Tue, 13 Sep 2016 21:23:27 +0200 simplified a bit the code in Flat_Model, and tried to fix the crash when sorting items + 53c65ff mr-alice Tue, 13 Sep 2016 18:54:00 +0200 removed cache transfers from the GUI + 02d2fb4 mr-alice Tue, 13 Sep 2016 13:53:51 +0200 fixed search and filter for regular expressions and terms + 6ac9b27 mr-alice Tue, 13 Sep 2016 12:14:16 +0200 moved rsexpr.cc into util/ + a2e34f9 mr-alice Tue, 13 Sep 2016 12:05:22 +0200 moved regular expression classes into a separate namespace + e9418bb mr-alice Tue, 13 Sep 2016 00:01:26 +0200 added file headers with proper licence terms + 9f66c00 mr-alice Mon, 12 Sep 2016 23:37:19 +0200 moved rsexpr to file_sharing directory + 274f924 mr-alice Mon, 12 Sep 2016 20:55:20 +0200 fixed getFileInfo and transfer from new database + af69790 csoler Mon, 12 Sep 2016 10:15:13 +0200 added control file for debian jessie + c1e538f mr-alice Sun, 11 Sep 2016 17:52:12 +0200 added interactive update of dir entries when browsing files from an online friend + 3718e7e mr-alice Sat, 10 Sep 2016 20:47:51 +0200 fixed last bugs in hash-based dir list exchange. It works! + 88f8538 mr-alice Sat, 10 Sep 2016 17:25:29 +0200 fixed a few bugs in hash-based syncing + 384d001 mr-alice Sat, 10 Sep 2016 13:57:05 +0200 progress in hash-based sync. Not there yet. + 0ea695b mr-alice Thu, 8 Sep 2016 23:43:14 +0200 fixed a few bugs in hash-based sync. Not there yet. + fafe684 mr-alice Thu, 8 Sep 2016 21:34:53 +0200 finished implementing hash-based transactions. To be tested. + e3d12c4 csoler Thu, 8 Sep 2016 17:08:34 +0200 Merge pull request #491 from G10h4ck/warnings_cleanup + eaa8ad8 MrAlice Wed, 7 Sep 2016 22:31:12 +0200 half implemented the hash-based communication for dirs between friends + ea380b2 csoler Wed, 7 Sep 2016 21:29:48 +0200 Merge pull request #487 from AsamK/fix_wayland + 798ffd8 csoler Wed, 7 Sep 2016 21:28:28 +0200 Merge pull request #488 from sehraf/pr-stats-api + ad1ba7a mr-alice Tue, 6 Sep 2016 23:05:46 +0200 fixed loading/saving of remote directories + e9540d7 defnax Tue, 6 Sep 2016 17:45:42 +0200 Improved look and feel layout of Profile Generation Dialog + 679c53b mr-alice Mon, 5 Sep 2016 21:22:11 +0200 added load/save of directory hierarchies. Fixed bug in loading of hash cache (caused y non initialisation of dest string in GetRawString()) + 5f5ce4a thunder2 Tue, 17 May 2016 19:02:14 +0200 Fixed version_detail_bash_script with shadow build + ea6431e thunder2 Mon, 5 Sep 2016 14:03:39 +0200 Fixed Windows shadow build with Qt 5.6 or higher + 5b221c5 mr-alice Sat, 3 Sep 2016 18:46:03 +0200 moved encrypted data output/input to FileListIO + 371171f mr-alice Sat, 3 Sep 2016 13:29:23 +0200 moved InternalFileHierarchyStorage to separate file. Fixed loading/saving infrastructure (not working yet) + 153783a mr-alice Fri, 2 Sep 2016 22:08:27 +0200 fixed bug in saving hash cache + 7b566f2 mr-alice Fri, 2 Sep 2016 21:49:43 +0200 added load/save for hash cache in the proper place + 52ef7ad mr-alice Thu, 1 Sep 2016 21:41:12 +0200 fixed saving of config when diretory list is changed + 6fc02e0 mr-alice Thu, 1 Sep 2016 21:04:48 +0200 added load/save of configuration and shared dirs list + f191fe6 mr-alice Thu, 1 Sep 2016 20:02:47 +0200 fixed compilation with new RsNodeGroupId + ade6017 mr-alice Thu, 1 Sep 2016 19:52:31 +0200 merged with upstream/master + 1f3ea6f mr-alice Wed, 31 Aug 2016 22:47:05 +0200 added selection of subdirs to be seen for friends depending on permissions. Enabled dynamic update at friends + + -- Cyril Soler Wed, 28 Sep 2016 22:00:00 +0100 + +retroshare06 (0.6.1-1.20160831.53e26983~trusty) trusty; urgency=low + Release of version 0.6.1. * See detailed logs: https://github.com/RetroShare/RetroShare/blob/master/build_scripts/Debian%2BUbuntu/changelog * Release blog post: https://retroshareteam.wordpress.com From c2761659e9afa502d80d24792e06fe25525dd85f Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 28 Sep 2016 22:28:43 +0200 Subject: [PATCH 129/174] updated clean.sh script to 0.6.1 --- build_scripts/Debian+Ubuntu/clean.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_scripts/Debian+Ubuntu/clean.sh b/build_scripts/Debian+Ubuntu/clean.sh index 6e68064f9..c1dbb219b 100755 --- a/build_scripts/Debian+Ubuntu/clean.sh +++ b/build_scripts/Debian+Ubuntu/clean.sh @@ -1,11 +1,11 @@ #!/bin/sh rm -f ./libssh-0.5.4.tar.gz.* -rm -f ./retroshare06_0.6.0-1.*_source.build -rm -f ./retroshare06_0.6.0-1.*_source.changes -rm -f ./retroshare06_0.6.0-1.*.tar.gz -rm -f ./retroshare06_0.6.0-1.*.diff.gz -rm -f ./retroshare06_0.6.0-1.*.dsc +rm -f ./retroshare06_0.6.1-1.*_source.build +rm -f ./retroshare06_0.6.1-1.*_source.changes +rm -f ./retroshare06_0.6.1-1.*.tar.gz +rm -f ./retroshare06_0.6.1-1.*.diff.gz +rm -f ./retroshare06_0.6.1-1.*.dsc rm -f *.upload rm -f *~ From e185bcdab4ff6aeff085b621436f1fad7bcd727a Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 28 Sep 2016 23:02:24 +0200 Subject: [PATCH 130/174] fixed removal of directory when removing a friend --- libretroshare/src/file_sharing/p3filelists.cc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 268b2e6f4..e53bc8e9a 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -239,7 +239,7 @@ void p3FileDatabase::startThreads() #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Starting directory watcher thread..." ; #endif - mLocalDirWatcher->start("fs dir watcher"); + mLocalDirWatcher->start("fs dir watcher"); #ifdef DEBUG_P3FILELISTS P3FILELISTS_DEBUG() << "Done." << std::endl; #endif @@ -445,10 +445,14 @@ void p3FileDatabase::cleanup() // now, in order to avoid empty seats, just move the last one here, and update indexes - mRemoteDirectories[i] = mRemoteDirectories.back(); - mRemoteDirectories.pop_back(); + while(i < mRemoteDirectories.size() && mRemoteDirectories[i] == NULL) + { + mRemoteDirectories[i] = mRemoteDirectories.back(); + mRemoteDirectories.pop_back(); + } - mFriendIndexMap[mRemoteDirectories[i]->peerId()] = i; + if(i < mRemoteDirectories.size() && mRemoteDirectories[i] != NULL) // this test is needed in the case we have deleted the last element + mFriendIndexMap[mRemoteDirectories[i]->peerId()] = i; mUpdateFlags |= P3FILELISTS_UPDATE_FLAG_REMOTE_DIRS_CHANGED ; } @@ -458,7 +462,7 @@ void p3FileDatabase::cleanup() for(std::set::const_iterator it(friend_set.begin());it!=friend_set.end();++it) { // Check if a remote directory exists for that friend, possibly creating the index. - locked_getFriendIndex(*it) ; + locked_getFriendIndex(*it) ; } // cancel existing requests for which the peer is offline @@ -638,8 +642,8 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; } - else - return false; + else + return false; uint32_t fi; DirectoryStorage::EntryIndex e ; From 0ea909df104091b8691607f2339ff195ca57ba00 Mon Sep 17 00:00:00 2001 From: Phenom Date: Thu, 29 Sep 2016 12:45:01 +0200 Subject: [PATCH 131/174] Fix Windows Compilation after 6e4ba76b --- retroshare-gui/src/gui/RemoteDirModel.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index ca550ea88..bfe7e2dbe 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -38,6 +38,7 @@ #include #include +#include /***** * #define RDM_DEBUG From b62b66a24cefb13c682f9669932b32e1e83cae5c Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 29 Sep 2016 20:55:16 +0200 Subject: [PATCH 132/174] replace std::vector for std::set to represent _hashes_to_remove, so as to avoid duplicates (suggested by Jo) --- libretroshare/src/turtle/p3turtle.cc | 29 ++++++++++++---------------- libretroshare/src/turtle/p3turtle.h | 2 +- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 25d1972a5..2044b52ac 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -430,21 +430,21 @@ void p3turtle::autoWash() { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - for(unsigned int i=0;i<_hashes_to_remove.size();++i) + for(std::set::const_iterator hit(_hashes_to_remove.begin());hit!=_hashes_to_remove.end();++hit) { - std::map::iterator it(_incoming_file_hashes.find(_hashes_to_remove[i])) ; + std::map::iterator it(_incoming_file_hashes.find(*hit)) ; if(it == _incoming_file_hashes.end()) { #ifdef P3TURTLE_DEBUG - std::cerr << "p3turtle: asked to stop monitoring file hash " << _hashes_to_remove[i] << ", but this hash is actually not handled by the turtle router." << std::endl ; + std::cerr << "p3turtle: asked to stop monitoring file hash " << *hit << ", but this hash is actually not handled by the turtle router." << std::endl ; #endif continue ; } // copy the list of tunnels to remove. #ifdef P3TURTLE_DEBUG - std::cerr << "p3turtle: stopping monitoring for file hash " << _hashes_to_remove[i] << ", and closing " << it->second.tunnels.size() << " tunnels (" ; + std::cerr << "p3turtle: stopping monitoring for file hash " << *hit << ", and closing " << it->second.tunnels.size() << " tunnels (" ; #endif std::vector tunnels_to_remove ; @@ -463,11 +463,8 @@ void p3turtle::autoWash() _incoming_file_hashes.erase(it) ; } - if(!_hashes_to_remove.empty()) - { - IndicateConfigChanged() ; // initiates saving of handled hashes. - _hashes_to_remove.clear() ; - } + + _hashes_to_remove.clear() ; } // look for tunnels and stored temporary info that have not been used for a while. @@ -657,7 +654,7 @@ void p3turtle::stopMonitoringTunnels(const RsFileHash& hash) std::cerr << "p3turtle: Marking hash " << hash << " to be removed during autowash." << std::endl ; #endif // We don't do the deletion in this process, because it can cause a race with tunnel management. - _hashes_to_remove.push_back(hash) ; + _hashes_to_remove.insert(hash) ; } // -----------------------------------------------------------------------------------// @@ -1855,15 +1852,13 @@ void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *clie // First, check if the hash is tagged for removal (there's a delay) - for(uint32_t i=0;i<_hashes_to_remove.size();++i) - if(_hashes_to_remove[i] == hash) - { - _hashes_to_remove[i] = _hashes_to_remove.back() ; - _hashes_to_remove.pop_back() ; + if(_hashes_to_remove.find(hash) != _hashes_to_remove.end()) + { + _hashes_to_remove.erase(hash) ; #ifdef P3TURTLE_DEBUG - std::cerr << "p3turtle: File hash " << hash << " Was scheduled for removal. Canceling the removal." << std::endl ; + std::cerr << "p3turtle: File hash " << hash << " Was scheduled for removal. Canceling the removal." << std::endl ; #endif - } + } // Then, check if the hash is already there // diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index bfda7663e..ceb137e7f 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -409,7 +409,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config std::map _virtual_peers ; /// Hashes marked to be deleted. - std::vector _hashes_to_remove ; + std::set _hashes_to_remove ; /// List of client services that have regitered. std::list _registered_services ; From 84341f26b38d3848f29e81eaf5830400bf29c31e Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 29 Sep 2016 22:30:26 +0200 Subject: [PATCH 133/174] changed _outgoing_file_hashes into a new map that connects services to tunnel ID instead of file hash, hence avoiding to remove a hash on server side when a tunnel is closed (bug found by Jolavillette) --- libretroshare/src/turtle/p3turtle.cc | 19 ++++++++++--------- libretroshare/src/turtle/p3turtle.h | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 2044b52ac..8208c08b8 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -624,8 +624,9 @@ void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector::iterator itHash = _outgoing_file_hashes.find(it->second.hash); - if(itHash != _outgoing_file_hashes.end()) + std::map::iterator itHash = _outgoing_tunnel_client_services.find(tid); + + if(itHash != _outgoing_tunnel_client_services.end()) { TurtleVirtualPeerId vpid = it->second.vpid ; TurtleFileHash hash = it->second.hash ; @@ -634,7 +635,7 @@ void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector >(itHash->second,hash_vpid)) ; - _outgoing_file_hashes.erase(itHash) ; + _outgoing_tunnel_client_services.erase(itHash) ; // Also remove the associated virtual peer // @@ -1197,11 +1198,11 @@ bool p3turtle::getTunnelServiceInfo(TurtleTunnelId tunnel_id,RsPeerId& vpid,RsFi } else if(tunnel.local_dst == _own_id) { - std::map::const_iterator it = _outgoing_file_hashes.find(hash) ; + std::map::const_iterator it = _outgoing_tunnel_client_services.find(tunnel_id) ; - if(it == _outgoing_file_hashes.end()) + if(it == _outgoing_tunnel_client_services.end()) { - std::cerr << "p3turtle::handleRecvGenericTunnelItem(): hash " << hash << " for server side tunnel endpoint " << std::hex << tunnel_id << std::dec << " has been removed (probably a late response)! Dropping the item. " << std::endl; + std::cerr << "p3turtle::handleRecvGenericTunnelItem(): hash " << tunnel.hash << " for server side tunnel endpoint " << std::hex << tunnel_id << std::dec << " has been removed (probably a late response)! Dropping the item. " << std::endl; return false; } @@ -1475,7 +1476,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item) // Store some info string about the tunnel. // - _outgoing_file_hashes[item->file_hash] = service ; + _outgoing_tunnel_client_services[t_id] = service ; // Notify the client service that there's a new virtual peer id available as a client. // @@ -2096,8 +2097,8 @@ void p3turtle::dumpState() //std::cerr << ", last_req=" << (void*)it->second.last_request << ", time_stamp = " << it->second.time_stamp << "(" << now-it->second.time_stamp << " secs ago)" << std::endl ; } std::cerr << " Active outgoing file hashes: " << _outgoing_file_hashes.size() << std::endl ; - for(std::map::const_iterator it(_outgoing_file_hashes.begin());it!=_outgoing_file_hashes.end();++it) - std::cerr << " hash=0x" << it->first << std::endl ; + for(std::map::const_iterator it(_outgoing_file_hashes.begin());it!=_outgoing_file_hashes.end();++it) + std::cerr << " TID=0x" << it->first << std::endl ; std::cerr << " Local tunnels:" << std::endl ; for(std::map::const_iterator it(_local_tunnels.begin());it!=_local_tunnels.end();++it) diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index ceb137e7f..d9d51456b 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -400,7 +400,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config std::map _incoming_file_hashes ; /// stores file info for each file we provide. - std::map _outgoing_file_hashes ; + std::map _outgoing_tunnel_client_services ; /// local tunnels, stored by ids (Either transiting or ending). std::map _local_tunnels ; From a75115585b33c53b4fcd9fae268b9c92551f77b7 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 1 Oct 2016 15:46:32 +0200 Subject: [PATCH 134/174] do not send un-hashed files to friends, and allow to re-send the directory content when all files are hashed --- libretroshare/src/file_sharing/dir_hierarchy.cc | 15 ++++++++++++++- .../src/file_sharing/directory_storage.cc | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 0a65ee371..a697fa120 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -583,13 +583,26 @@ time_t InternalFileHierarchyStorage::recursUpdateLastModfTime(const DirectorySto DirEntry& d(*static_cast(mNodes[dir_index])) ; time_t largest_modf_time = d.dir_modtime ; + bool unfinished_files_present = false ; for(uint32_t i=0;i(mNodes[d.subfiles[i]])->file_modtime) ; + { + FileEntry *f = static_cast(mNodes[d.subfiles[i]]) ; + + if(!f->file_hash.isNull()) + largest_modf_time = std::max(largest_modf_time, f->file_modtime) ; // only account for hashed files, since we never send unhashed files to friends. + else + unfinished_files_present = true ; + } for(uint32_t i=0;i 0) + largest_modf_time-- ; + d.dir_most_recent_time = largest_modf_time ; return largest_modf_time ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 0f0f065d4..02c406c5d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -637,6 +637,17 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary std::cerr << " not pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << ": permission denied for this peer." << std::endl; #endif + // now count the files that do not have a null hash (meaning the hash has indeed been computed) + + uint32_t allowed_subfiles = 0 ; + + for(uint32_t i=0;isubfiles.size();++i) + { + const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ; + if(file != NULL && !file->file_hash.isNull()) + allowed_subfiles++ ; + } + unsigned char *section_data = NULL; uint32_t section_size = 0; uint32_t section_offset = 0; @@ -655,7 +666,7 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary // serialise number of subdirs and number of subfiles if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) return false ; - if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subfiles )) return false ; // serialise subdirs entry indexes @@ -672,9 +683,9 @@ bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinary const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ; - if(file == NULL) + if(file == NULL || file->file_hash.isNull()) { - std::cerr << "(EE) cannot reach file entry " << dir->subfiles[i] << " to get/send file info." << std::endl; + std::cerr << "(II) skipping unhashed or Null file entry " << dir->subfiles[i] << " to get/send file info." << std::endl; continue ; } From 9ae05112be6bc2a7a60a2be1bdb2170d80504613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bryon=20Gloden=2C=20CISSP=C2=AE?= Date: Sat, 1 Oct 2016 12:01:09 -0400 Subject: [PATCH 135/174] uninitialized variable: size [libretroshare/src/file_sharing/hash_cache.cc:182]: (error) Uninitialized variable: size Found by https://github.com/bryongloden/cppcheck --- libretroshare/src/file_sharing/hash_cache.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index abca37add..538aec5d8 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -76,7 +76,7 @@ void HashStorage::data_tick() { FileHashJob job; RsFileHash hash; - uint64_t size ; + uint64_t size = 0; { bool empty ; From c968490b438f00bc52639cf70e91e739f99ad12a Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 1 Oct 2016 22:14:50 +0200 Subject: [PATCH 136/174] switched to floating point time for bandwidth estimation in pqistreamer. Helps a lot RTTs since rounding to int prveviously caused packets to be delayed up to 1 sec (improvement by Jollavilette) --- libretroshare/src/pqi/pqistreamer.cc | 49 +++++++++++++++++++--------- libretroshare/src/pqi/pqistreamer.h | 4 +-- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 9d68a50b2..6cc051ee2 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -76,6 +76,19 @@ static uint8_t PACKET_SLICING_PROBE_BYTES[8] = { 0x02, 0xaa, 0xbb, 0xcc, 0x00, #include "util/rsprint.h" #endif +static double getCurrentTS() +{ +#ifndef WINDOWS_SYS + struct timeval cts_tmp; + gettimeofday(&cts_tmp, NULL); + double cts = (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0; +#else + struct _timeb timebuf; + _ftime( &timebuf); + double cts = (timebuf.time) + ((double) timebuf.millitm) / 1000.0; +#endif + return cts; +} pqistreamer::pqistreamer(RsSerialiser *rss, const RsPeerId& id, BinInterface *bio_in, int bio_flags_in) :PQInterface(id), mStreamerMtx("pqistreamer"), @@ -97,7 +110,9 @@ pqistreamer::pqistreamer(RsSerialiser *rss, const RsPeerId& id, BinInterface *bi mAcceptsPacketSlicing = false ; // by default. Will be turned into true when everyone's ready. mLastSentPacketSlicingProbe = 0 ; - mAvgLastUpdate = mCurrReadTS = mCurrSentTS = time(NULL); + mAvgLastUpdate = time(NULL); + mCurrSentTS = mCurrReadTS = getCurrentTS(); + mIncomingSize = 0 ; mStatisticsTimeStamp = 0 ; @@ -1099,7 +1114,7 @@ float pqistreamer::outTimeSlice_locked() // very simple..... int pqistreamer::outAllowedBytes_locked() { - int t = time(NULL); // get current timestep. + double t = getCurrentTS() ; // Grabs today's time in sec, with ms accuracy. Allows a much more accurate allocation of bw /* allow a lot if not bandwidthLimited */ if (!mBio->bandwidthLimited()) @@ -1109,17 +1124,18 @@ int pqistreamer::outAllowedBytes_locked() return PQISTREAM_ABS_MAX; } - int dt = t - mCurrSentTS; - // limiter -> for when currSentTs -> 0. - if (dt > 5) - dt = 5; + double dt = t - mCurrSentTS; + + // limiter -> for when currSentTs -> 0. + if (dt > 5) + dt = 5; + + double maxout = getMaxRate(false) * 1024.0; + + mCurrSent -= int(dt * maxout); - int maxout = (int) (getMaxRate(false) * 1000.0); - mCurrSent -= dt * maxout; if (mCurrSent < 0) - { mCurrSent = 0; - } mCurrSentTS = t; @@ -1137,7 +1153,7 @@ int pqistreamer::outAllowedBytes_locked() int pqistreamer::inAllowedBytes_locked() { - int t = time(NULL); // get current timestep. + double t = getCurrentTS(); // in secs, with a ms accuracy /* allow a lot if not bandwidthLimited */ if (!mBio->bandwidthLimited()) @@ -1147,17 +1163,18 @@ int pqistreamer::inAllowedBytes_locked() return PQISTREAM_ABS_MAX; } - int dt = t - mCurrReadTS; + double dt = t - mCurrReadTS; + // limiter -> for when currReadTs -> 0. if (dt > 5) dt = 5; - int maxin = (int) (getMaxRate(true) * 1000.0); - mCurrRead -= dt * maxin; + double maxin = getMaxRate(true) * 1024.0; + + mCurrRead -= int(dt * maxin); + if (mCurrRead < 0) - { mCurrRead = 0; - } mCurrReadTS = t; diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index 5cdbab9d7..b4351670c 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -159,8 +159,8 @@ class pqistreamer: public PQInterface int mCurrRead; int mCurrSent; - time_t mCurrReadTS; // TS from which these are measured. - time_t mCurrSentTS; + double mCurrReadTS; // TS from which these are measured. + double mCurrSentTS; time_t mAvgLastUpdate; // TS from which these are measured. uint32_t mAvgReadCount; From ed8d78660fec7156e8f27d76e64dfb38000d6fc5 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 3 Oct 2016 20:49:49 +0200 Subject: [PATCH 137/174] added missing mChanged=false that caused remote directories to save far too often --- libretroshare/src/file_sharing/directory_storage.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 02c406c5d..7780717ca 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -738,6 +738,7 @@ void RemoteDirectoryStorage::checkSave() { save(mFileName); mLastSavedTime = now ; + mChanged = false ; } } From ebfc82cc1e29f1e8c22c582b81492c04be92174d Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 3 Oct 2016 21:44:34 +0200 Subject: [PATCH 138/174] save/restore selection in shared dir list tree view --- libretroshare/src/file_sharing/p3filelists.cc | 2 +- retroshare-gui/src/gui/SharedFilesDialog.cpp | 45 ++++++++++++------- retroshare-gui/src/gui/SharedFilesDialog.h | 8 ++-- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index e53bc8e9a..49ae0aa0c 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -224,7 +224,7 @@ int p3FileDatabase::tick() mLastRemoteDirSweepTS = now; - // This is a hash to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up, + // This is a hack to make loaded directories show up in the GUI, because the GUI generally isn't ready at the time they are actually loaded up, // so the first notify is ignored, and no other notify will happen next. RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); diff --git a/retroshare-gui/src/gui/SharedFilesDialog.cpp b/retroshare-gui/src/gui/SharedFilesDialog.cpp index 036dd96e2..db2129b62 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/SharedFilesDialog.cpp @@ -256,13 +256,14 @@ void SharedFilesDialog::showEvent(QShowEvent *) { if(model!=NULL) { - std::set expanded_indexes ; - saveExpandedPaths(expanded_indexes); + std::set expanded_indexes,selected_indexes ; + + saveExpandedPathsAndSelection(expanded_indexes,selected_indexes); model->setVisible(true) ; model->update() ; - restoreExpandedPaths(expanded_indexes); + restoreExpandedPathsAndSelection(expanded_indexes,selected_indexes); } } RemoteSharedFilesDialog::~RemoteSharedFilesDialog() @@ -359,8 +360,9 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) showProperColumns() ; - std::set expanded_indexes ; - saveExpandedPaths(expanded_indexes); + std::set expanded_indexes,selected_indexes ; + + saveExpandedPathsAndSelection(expanded_indexes,selected_indexes); if(isVisible()) { @@ -374,7 +376,7 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex) ui.dirTreeView->setModel(proxyModel); ui.dirTreeView->update(); - restoreExpandedPaths(expanded_indexes); + restoreExpandedPathsAndSelection(expanded_indexes,selected_indexes); QHeaderView * header = ui.dirTreeView->header () ; QHeaderView_setSectionResizeModeColumn(header, COLUMN_NAME, QHeaderView::Interactive); @@ -851,7 +853,7 @@ void SharedFilesDialog::preModDirectories(bool local) flat_model->preMods(); } -void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexes) +void SharedFilesDialog::saveExpandedPathsAndSelection(std::set& expanded_indexes, std::set& selected_indexes) { if(ui.dirTreeView->model() == NULL) return ; @@ -862,11 +864,12 @@ void SharedFilesDialog::saveExpandedPaths(std::set& expanded_indexe for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); - recursSaveExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); + + recursSaveExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes,selected_indexes); } } -void SharedFilesDialog::restoreExpandedPaths(const std::set& expanded_indexes) +void SharedFilesDialog::restoreExpandedPathsAndSelection(const std::set& expanded_indexes, const std::set& selected_indexes) { if(ui.dirTreeView->model() == NULL) return ; @@ -881,18 +884,23 @@ void SharedFilesDialog::restoreExpandedPaths(const std::set& expand for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row) { std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString(); - recursRestoreExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes); + recursRestoreExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes,selected_indexes); } + QItemSelection selection ; + ui.dirTreeView->blockSignals(false) ; } -void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp) +void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set& exp,std::set& sel) { std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); #ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; #endif + if(ui.dirTreeView->selectionModel()->selection().contains(index)) + sel.insert(local_path) ; + if(ui.dirTreeView->isExpanded(index)) { #ifdef DEBUG_SHARED_FILES_DIALOG @@ -902,7 +910,7 @@ void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const s exp.insert(local_path) ; for(int row=0;rowmodel()->rowCount(index);++row) - recursSaveExpandedItems(index.child(row,0),local_path,exp) ; + recursSaveExpandedItems(index.child(row,0),local_path,exp,sel) ; } #ifdef DEBUG_SHARED_FILES_DIALOG else @@ -910,12 +918,14 @@ void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const s #endif } -void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set& exp) +void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set& exp, const std::set &sel) { std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString(); #ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl; #endif + if(sel.find(local_path) != sel.end()) + ui.dirTreeView->selectionModel()->select(index, QItemSelectionModel::Select | QItemSelectionModel::Rows); if(exp.find(local_path) != exp.end()) { @@ -925,7 +935,7 @@ void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, con ui.dirTreeView->setExpanded(index,true) ; for(int row=0;rowmodel()->rowCount(index);++row) - recursRestoreExpandedItems(index.child(row,0),local_path,exp) ; + recursRestoreExpandedItems(index.child(row,0),local_path,exp,sel) ; } } @@ -935,8 +945,9 @@ void SharedFilesDialog::postModDirectories(bool local) if (isRemote() == local) { return; } - std::set expanded_indexes; - saveExpandedPaths(expanded_indexes) ; + std::set expanded_indexes,selected_indexes; + + saveExpandedPathsAndSelection(expanded_indexes,selected_indexes) ; #ifdef DEBUG_SHARED_FILES_DIALOG std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl; #endif @@ -946,7 +957,7 @@ void SharedFilesDialog::postModDirectories(bool local) flat_model->postMods(); ui.dirTreeView->update() ; - restoreExpandedPaths(expanded_indexes) ; + restoreExpandedPathsAndSelection(expanded_indexes,selected_indexes) ; if (ui.filterPatternLineEdit->text().isEmpty() == false) FilterItems(); diff --git a/retroshare-gui/src/gui/SharedFilesDialog.h b/retroshare-gui/src/gui/SharedFilesDialog.h index 594df6fab..3d199fe98 100644 --- a/retroshare-gui/src/gui/SharedFilesDialog.h +++ b/retroshare-gui/src/gui/SharedFilesDialog.h @@ -94,10 +94,10 @@ protected: Ui::SharedFilesDialog ui; virtual void processSettings(bool bLoad) = 0; - void recursRestoreExpandedItems(const QModelIndex& index,const std::string& path,const std::set& exp); - void recursSaveExpandedItems(const QModelIndex& index, const std::string &path, std::set &exp); - void saveExpandedPaths(std::set& paths) ; - void restoreExpandedPaths(const std::set& paths) ; + void recursRestoreExpandedItems(const QModelIndex& index,const std::string& path,const std::set& exp,const std::set& sel); + void recursSaveExpandedItems(const QModelIndex& index, const std::string &path, std::set &exp, std::set& sel); + void saveExpandedPathsAndSelection(std::set& paths, std::set& selected_indexes) ; + void restoreExpandedPathsAndSelection(const std::set& paths, const std::set& selected_indexes) ; protected: //now context menu are created again every time theu are called ( in some From a35985e3fc7f3da0e0b09ef2b852c6fb6db58a08 Mon Sep 17 00:00:00 2001 From: thunder2 Date: Tue, 6 Sep 2016 21:31:41 +0200 Subject: [PATCH 139/174] Windows build environment: - Added build script - Added build-installer script - Added pack script - Added gitlog script - Use shadow build --- build_scripts/Windows/.gitignore | 1 + .../Windows/build/build-installer.bat | 56 +++ build_scripts/Windows/build/build.bat | 81 ++++ build_scripts/Windows/build/clean.bat | 21 + build_scripts/Windows/build/env.bat | 31 ++ build_scripts/Windows/build/git-log.bat | 106 +++++ build_scripts/Windows/build/pack.bat | 210 ++++++++++ build_scripts/Windows/build_libs/.gitignore | 6 - .../Windows/build_libs/1-prepare.bat | 43 -- .../Windows/build_libs/2-install-msys.bat | 70 ---- .../Windows/build_libs/3-build-libs.bat | 29 -- build_scripts/Windows/build_libs/Makefile | 371 ++++++++++++------ build_scripts/Windows/build_libs/_env.bat | 9 - .../Windows/build_libs/build-libs.bat | 46 +++ .../Windows/build_libs/clean-all.bat | 30 -- build_scripts/Windows/build_libs/clean.bat | 35 +- build_scripts/Windows/build_libs/env.bat | 14 + .../Windows/build_libs/update-msys.bat | 16 - build_scripts/Windows/env.bat | 19 + build_scripts/Windows/env/env-msys.bat | 16 + build_scripts/Windows/env/env.bat | 29 ++ .../Windows/env/tools/prepare-msys.bat | 83 ++++ .../Windows/env/tools/prepare-tools.bat | 139 +++++++ .../Windows/env/tools/root/update-msys.bat | 15 + .../Windows/{ => installer}/HeaderImage.bmp | Bin .../{ => installer}/HeaderImageEmpty.bmp | Bin .../Windows/{ => installer}/lang/ca_ES.nsh | 0 .../Windows/{ => installer}/lang/de.nsh | 0 .../Windows/{ => installer}/lang/en.nsh | 0 .../Windows/{ => installer}/lang/es.nsh | 0 .../Windows/{ => installer}/lang/fr.nsh | 0 .../Windows/{ => installer}/lang/pl.nsh | 0 .../Windows/{ => installer}/lang/ru.nsh | 0 .../Windows/{ => installer}/lang/tr.nsh | 0 .../Windows/{ => installer}/lang/ts/ca_ES.ts | 0 .../lang/ts/convert_from_ts.bat | 0 .../lang/ts/convert_from_ts.xsl | 0 .../{ => installer}/lang/ts/convert_to_ts.bat | 0 .../Windows/{ => installer}/lang/ts/de.ts | 0 .../Windows/{ => installer}/lang/ts/en.ts | 0 .../Windows/{ => installer}/lang/ts/es.ts | 0 .../Windows/{ => installer}/lang/ts/fr.ts | 0 .../Windows/{ => installer}/lang/ts/pl.ts | 0 .../Windows/{ => installer}/lang/ts/ru.ts | 0 .../Windows/{ => installer}/lang/ts/tr.ts | 0 .../{ => installer}/lang/ts/xsltproc.exe | Bin .../Windows/{ => installer}/lang/ts/zh_CN.ts | 0 .../Windows/{ => installer}/lang/zh_CN.nsh | 0 .../{ => installer}/retroshare-Qt4.nsi | 21 +- .../{ => installer}/retroshare-Qt5.nsi | 23 +- build_scripts/Windows/make_installer.bat | 46 --- build_scripts/Windows/tools/depends.bat | 37 ++ build_scripts/Windows/tools/find-in-path.bat | 26 ++ .../Windows/tools/get-gcc-version.bat | 42 ++ build_scripts/Windows/tools/get-git-ref.bat | 39 ++ .../Windows/tools/get-qt-version.bat | 34 ++ .../get-rs-version.bat} | 22 +- build_scripts/Windows/tools/msys-path.bat | 20 + build_scripts/Windows/tools/remove-dir.bat | 15 + .../{build_libs/Tools => tools}/winhttpjs.bat | 0 60 files changed, 1357 insertions(+), 444 deletions(-) create mode 100644 build_scripts/Windows/.gitignore create mode 100644 build_scripts/Windows/build/build-installer.bat create mode 100644 build_scripts/Windows/build/build.bat create mode 100644 build_scripts/Windows/build/clean.bat create mode 100644 build_scripts/Windows/build/env.bat create mode 100644 build_scripts/Windows/build/git-log.bat create mode 100644 build_scripts/Windows/build/pack.bat delete mode 100644 build_scripts/Windows/build_libs/.gitignore delete mode 100644 build_scripts/Windows/build_libs/1-prepare.bat delete mode 100644 build_scripts/Windows/build_libs/2-install-msys.bat delete mode 100644 build_scripts/Windows/build_libs/3-build-libs.bat delete mode 100644 build_scripts/Windows/build_libs/_env.bat create mode 100644 build_scripts/Windows/build_libs/build-libs.bat delete mode 100644 build_scripts/Windows/build_libs/clean-all.bat create mode 100644 build_scripts/Windows/build_libs/env.bat delete mode 100644 build_scripts/Windows/build_libs/update-msys.bat create mode 100644 build_scripts/Windows/env.bat create mode 100644 build_scripts/Windows/env/env-msys.bat create mode 100644 build_scripts/Windows/env/env.bat create mode 100644 build_scripts/Windows/env/tools/prepare-msys.bat create mode 100644 build_scripts/Windows/env/tools/prepare-tools.bat create mode 100644 build_scripts/Windows/env/tools/root/update-msys.bat rename build_scripts/Windows/{ => installer}/HeaderImage.bmp (100%) rename build_scripts/Windows/{ => installer}/HeaderImageEmpty.bmp (100%) rename build_scripts/Windows/{ => installer}/lang/ca_ES.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/de.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/en.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/es.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/fr.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/pl.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/ru.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/tr.nsh (100%) rename build_scripts/Windows/{ => installer}/lang/ts/ca_ES.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/convert_from_ts.bat (100%) rename build_scripts/Windows/{ => installer}/lang/ts/convert_from_ts.xsl (100%) rename build_scripts/Windows/{ => installer}/lang/ts/convert_to_ts.bat (100%) rename build_scripts/Windows/{ => installer}/lang/ts/de.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/en.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/es.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/fr.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/pl.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/ru.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/tr.ts (100%) rename build_scripts/Windows/{ => installer}/lang/ts/xsltproc.exe (100%) rename build_scripts/Windows/{ => installer}/lang/ts/zh_CN.ts (100%) rename build_scripts/Windows/{ => installer}/lang/zh_CN.nsh (100%) rename build_scripts/Windows/{ => installer}/retroshare-Qt4.nsi (94%) rename build_scripts/Windows/{ => installer}/retroshare-Qt5.nsi (95%) delete mode 100644 build_scripts/Windows/make_installer.bat create mode 100644 build_scripts/Windows/tools/depends.bat create mode 100644 build_scripts/Windows/tools/find-in-path.bat create mode 100644 build_scripts/Windows/tools/get-gcc-version.bat create mode 100644 build_scripts/Windows/tools/get-git-ref.bat create mode 100644 build_scripts/Windows/tools/get-qt-version.bat rename build_scripts/Windows/{GetRsVersion.bat => tools/get-rs-version.bat} (54%) create mode 100644 build_scripts/Windows/tools/msys-path.bat create mode 100644 build_scripts/Windows/tools/remove-dir.bat rename build_scripts/Windows/{build_libs/Tools => tools}/winhttpjs.bat (100%) diff --git a/build_scripts/Windows/.gitignore b/build_scripts/Windows/.gitignore new file mode 100644 index 000000000..a7b500b79 --- /dev/null +++ b/build_scripts/Windows/.gitignore @@ -0,0 +1 @@ +/build/env-mod.bat \ No newline at end of file diff --git a/build_scripts/Windows/build/build-installer.bat b/build_scripts/Windows/build/build-installer.bat new file mode 100644 index 000000000..9d3aa837e --- /dev/null +++ b/build_scripts/Windows/build/build-installer.bat @@ -0,0 +1,56 @@ +@echo off + +setlocal + +:: Initialize environment +call "%~dp0..\env.bat" +if errorlevel 1 goto error_env +call "%EnvPath%\env.bat" +if errorlevel 1 goto error_env + +:: Get gcc versions +call "%ToolsPath%\get-gcc-version.bat" GCCVersion +if "%GCCVersion%"=="" echo Cannot get gcc version.& exit /B 1 + +:: Check external libraries +if not exist "%RootPath%\libs" echo Please build external libraries first.& exit /B 1 + +:: Check gcc version of external libraries +if not exist "%RootPath%\libs\gcc-version" echo Cannot get gcc version of external libraries.& exit /B 1 +set /P LibsGCCVersion=<"%RootPath%\libs\gcc-version" +if "%LibsGCCVersion%" NEQ "%GCCVersion%" echo Please use correct version of external libraries. (gcc %GCCVersion% ^<^> libs %LibsGCCVersion%).& exit /B 1 + +:: Initialize environment +call "%~dp0env.bat" +if errorlevel 1 goto error_env + +:: Build defines for script +set NSIS_PARAM= + +set NSIS_PARAM=%NSIS_PARAM% /DRELEASEDIR="%RsBuildPath%" +set NSIS_PARAM=%NSIS_PARAM% /DQTDIR="%QtPath%\.." +set NSIS_PARAM=%NSIS_PARAM% /DMINGWDIR="%MinGWPath%\.." +set NSIS_PARAM=%NSIS_PARAM% /DOUTDIR="%RsPackPath%" +set NSIS_PARAM=%NSIS_PARAM% /DINSTALLERADD="%RsArchiveAdd%" + +:: Scan version from source +set RsRevision= +set RsBuildAdd= +call "%ToolsPath%\get-rs-version.bat" RS_REVISION_STRING RsRevision +if "%RsRevision%"=="" echo Revision not found.& exit /B 1 +call "%ToolsPath%\get-rs-version.bat" RS_BUILD_NUMBER_ADD RsBuildAdd +if errorlevel 1 exit /B 1 + +set NSIS_PARAM=%NSIS_PARAM% /DREVISION=%RsRevision% /DBUILDADD=%RsBuildAdd% + +set QtMainVersion=%QtVersion:~0,1% + +:: Create installer +"%EnvMakeNSISExe%" %NSIS_PARAM% "%SourcePath%\build_scripts\Windows\installer\retroshare-Qt%QtMainVersion%.nsi" + +exit /B %ERRORLEVEL% + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/build/build.bat b/build_scripts/Windows/build/build.bat new file mode 100644 index 000000000..4893a51d7 --- /dev/null +++ b/build_scripts/Windows/build/build.bat @@ -0,0 +1,81 @@ +@echo off + +setlocal + +:: Initialize environment +call "%~dp0..\env.bat" +if errorlevel 1 goto error_env +call "%EnvPath%\env.bat" +if errorlevel 1 goto error_env + +:: Get gcc versions +call "%ToolsPath%\get-gcc-version.bat" GCCVersion +if "%GCCVersion%"=="" echo Cannot get gcc version.& exit /B 1 + +:: Check external libraries +if not exist "%RootPath%\libs" echo Please build external libraries first.& exit /B 1 + +:: Check gcc version of external libraries +if not exist "%RootPath%\libs\gcc-version" echo Cannot get gcc version of external libraries.& exit /B 1 +set /P LibsGCCVersion=<"%RootPath%\libs\gcc-version" +if "%LibsGCCVersion%" NEQ "%GCCVersion%" echo Please use correct version of external libraries. (gcc %GCCVersion% ^<^> libs %LibsGCCVersion%).& exit /B 1 + +:: Initialize environment +call "%~dp0env.bat" +if errorlevel 1 goto error_env + +:: Check git executable +set GitPath= +call "%ToolsPath%\find-in-path.bat" GitPath git.exe +if "%GitPath%" NEQ "" goto found_git +choice /M "Git not found in PATH. Version information cannot be calculated. Do you want to proceed?" +if %errorlevel%==2 exit /B 1 +:found_git + +echo. +echo === Version +echo. + +title Build - %SourceName%-%RsBuildConfig% [Version] + +pushd "%SourcePath%\retroshare-gui\src\gui\images" +:: Touch resource file +copy /b retroshare_win.rc +,, +popd + +if not exist "%RsBuildPath%" mkdir "%RsBuildPath%" +pushd "%RsBuildPath%" + +echo. +echo === qmake +echo. + +title Build - %SourceName%-%RsBuildConfig% [qmake] + +qmake "%SourcePath%\RetroShare.pro" -r "CONFIG+=%RsBuildConfig% version_detail_bash_script" +if errorlevel 1 goto error + +echo. +echo === make +echo. + +title Build - %SourceName%-%RsBuildConfig% [make] + +if exist "%EnvJomExe%" ( + "%EnvJomExe%" +) else ( + mingw32-make +) + +:error +popd + +title %COMSPEC% + +if errorlevel 1 echo.& echo Build failed& echo. +exit /B %ERRORLEVEL% + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/build/clean.bat b/build_scripts/Windows/build/clean.bat new file mode 100644 index 000000000..0797e7681 --- /dev/null +++ b/build_scripts/Windows/build/clean.bat @@ -0,0 +1,21 @@ +@echo off + +setlocal + +:: Initialize environment +call "%~dp0..\env.bat" +if errorlevel 1 goto error_env +call "%EnvPath%\env.bat" +if errorlevel 1 goto error_env +call "%~dp0env.bat" +if errorlevel 1 goto error_env + +if not exist "%RsBuildPath%" exit /B 0 +call "%ToolsPath%\remove-dir.bat" "%RsBuildPath%" + +exit /B %ERRORLEVEL% + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/build/env.bat b/build_scripts/Windows/build/env.bat new file mode 100644 index 000000000..a99905161 --- /dev/null +++ b/build_scripts/Windows/build/env.bat @@ -0,0 +1,31 @@ +set CurPath=%~dp0 +set BuildPath=%EnvRootPath%\builds +set DeployPath=%EnvRootPath%\deploy + +if not exist "%BuildPath%" mkdir "%BuildPath%" +if not exist "%DeployPath%" mkdir "%DeployPath%" + +:: Check Qt environment +set QtPath= +call "%ToolsPath%\find-in-path.bat" QtPath qmake.exe +if "%QtPath%"=="" echo Please run command in the Qt Command Prompt.& exit /B 1 + +:: Check MinGW environment +set MinGWPath= +call "%ToolsPath%\find-in-path.bat" MinGWPath gcc.exe +if "%MinGWPath%"=="" echo Please run command in the Qt Command Prompt.& exit /B 1 + +:: Get Qt version +call "%ToolsPath%\get-qt-version.bat" QtVersion +if "%QtVersion%"=="" echo Cannot get Qt version.& exit /B 1 + +set RsBuildConfig=release +set RsBuildPath=%BuildPath%\Qt-%QtVersion%-%RsBuildConfig% +set RsDeployPath=%DeployPath%\Qt-%QtVersion%-%RsBuildConfig% +set RsPackPath=%DeployPath% +set RsArchiveAdd= + +if exist "%~dp0env-mod.bat" call "%~dp0env-mod.bat" +if errorlevel 1 exit /B %ERRORLEVEL% + +exit /B 0 \ No newline at end of file diff --git a/build_scripts/Windows/build/git-log.bat b/build_scripts/Windows/build/git-log.bat new file mode 100644 index 000000000..49cfaef1e --- /dev/null +++ b/build_scripts/Windows/build/git-log.bat @@ -0,0 +1,106 @@ +@echo off + +setlocal + +:: Initialize environment +call "%~dp0..\env.bat" +if errorlevel 1 goto error_env +call "%EnvPath%\env.bat" +if errorlevel 1 goto error_env +call "%~dp0env.bat" +if errorlevel 1 goto error_env + +:: Check git executable +set GitPath= +call "%ToolsPath%\find-in-path.bat" GitPath git.exe +if "%GitPath%"=="" echo Git executable not found in PATH.& exit /B 1 + +:: Get compiled revision +set GetRsVersion=%SourcePath%\build_scripts\Windows\tools\get-rs-version.bat +if not exist "%GetRsVersion%" ( + echo File not found + echo %GetRsVersion% + exit /B 1 +) + +call "%GetRsVersion%" RS_REVISION_STRING RsRevision +if "%RsRevision%"=="" echo Revision not found.& exit /B 1 + +:: Get compiled version +call "%GetRsVersion%" RS_REVISION_STRING RsRevision +if "%RsRevision%"=="" echo Revision not found.& exit /B 1 + +call "%GetRsVersion%" RS_MAJOR_VERSION RsMajorVersion +if "%RsMajorVersion%"=="" echo Major version not found.& exit /B 1 + +call "%GetRsVersion%" RS_MINOR_VERSION RsMinorVersion +if "%RsMinorVersion%"=="" echo Minor version not found.& exit /B 1 + +call "%GetRsVersion%" RS_BUILD_NUMBER RsBuildNumber +if "%RsBuildNumber%"=="" echo Build number not found.& exit /B 1 + +call "%GetRsVersion%" RS_BUILD_NUMBER_ADD RsBuildNumberAdd + +set RsVersion=%RsMajorVersion%.%RsMinorVersion%.%RsBuildNumber%%RsBuildNumberAdd% + +:: Check WMIC is available +wmic.exe alias /? >nul 2>&1 || echo WMIC is not available.&& exit /B 1 + +:: Use WMIC to retrieve date in format YYYYMMDD +set RsDate= +for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /format:list') do set RsDate=%%I +set RsDate=%RsDate:~0,4%%RsDate:~4,2%%RsDate:~6,2% + +:: Get last revision +set RsLastRefFile=%BuildPath%\Qt-%QtVersion%-%RsBuildConfig%-LastRef.txt +set RsLastRef= +if exist "%RsLastRefFile%" set /P RsLastRef=<"%RsLastRefFile%" + +if not "%RsLastRef%"=="" echo Last Revision was %RsLastRef% +set /P RsLastRefInput=Last Revision: +if "%RsLastRefInput%" NEQ "" set RsLastRef=%RsLastRefInput% + +:: Get current revision +pushd "%SourcePath%" +call "%ToolsPath%\get-git-ref.bat" RsRef +popd + +if errorlevel 1 exit /B 1 +if "%RsRef%"=="" echo Cannot get git revision.& exit /B 1 + +echo. +echo Creating log from %RsLastRef% +echo to %RsRef% +choice /M "Do you want to proceed?" +if %errorlevel%==2 exit /B 1 + +if "%RsBuildConfig%" NEQ "release" ( + set RsGitLog=%DeployPath%\RetroShare-%RsVersion%-Windows-Portable-%RsDate%-%RsRevision%-Qt-%QtVersion%%RsArchiveAdd%-%RsBuildConfig%.txt +) else ( + set RsGitLog=%DeployPath%\RetroShare-%RsVersion%-Windows-Portable-%RsDate%-%RsRevision%-Qt-%QtVersion%%RsArchiveAdd%.txt +) + +title %SourceName%-%RsBuildConfig% [git log] + +pushd "%SourcePath%" +if "%RsLastRef%"=="" ( + git log %RsRef% >"%RsGitLog%" +) else ( + if "%RsLastRef%"=="%RsRef%" ( + git log %RsRef% --max-count=1 >"%RsGitLog%" + ) else ( + git log %RsLastRef%..%RsRef% >"%RsGitLog%" + ) +) +popd + +title %COMSPEC% + +echo %RsRef%>"%RsLastRefFile%" + +exit /B %ERRORLEVEL% + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/build/pack.bat b/build_scripts/Windows/build/pack.bat new file mode 100644 index 000000000..a8e7fdaff --- /dev/null +++ b/build_scripts/Windows/build/pack.bat @@ -0,0 +1,210 @@ +@echo off + +setlocal + +set Quite=^>nul + +:: Initialize environment +call "%~dp0..\env.bat" +if errorlevel 1 goto error_env +call "%EnvPath%\env.bat" +if errorlevel 1 goto error_env + +:: Get gcc versions +call "%ToolsPath%\get-gcc-version.bat" GCCVersion +if "%GCCVersion%"=="" echo Cannot get gcc version.& exit /B 1 + +:: Check external libraries +if not exist "%RootPath%\libs" echo Please build external libraries first.& exit /B 1 + +:: Check gcc version of external libraries +if not exist "%RootPath%\libs\gcc-version" echo Cannot get gcc version of external libraries.& exit /B 1 +set /P LibsGCCVersion=<"%RootPath%\libs\gcc-version" +if "%LibsGCCVersion%" NEQ "%GCCVersion%" echo Please use correct version of external libraries. (gcc %GCCVersion% ^<^> libs %LibsGCCVersion%).& exit /B 1 + +:: Initialize environment +call "%~dp0env.bat" +if errorlevel 1 goto error_env + +:: Remove deploy path +if exist "%RsDeployPath%" rmdir /S /Q "%RsDeployPath%" + +:: Check compilation +if not exist "%RsBuildPath%\Makefile" echo Project is not compiled.& goto error + +:: Get compiled revision +set GetRsVersion=%SourcePath%\build_scripts\Windows\tools\get-rs-version.bat +if not exist "%GetRsVersion%" ( + echo File not found + echo %GetRsVersion% + goto error +) + +call "%GetRsVersion%" RS_REVISION_STRING RsRevision +if "%RsRevision%"=="" echo Revision not found.& goto error + +:: Get compiled version +call "%GetRsVersion%" RS_MAJOR_VERSION RsMajorVersion +if "%RsMajorVersion%"=="" echo Major version not found.& goto error + +call "%GetRsVersion%" RS_MINOR_VERSION RsMinorVersion +if "%RsMinorVersion%"=="" echo Minor version not found.& goto error + +call "%GetRsVersion%" RS_BUILD_NUMBER RsBuildNumber +if "%RsBuildNumber%"=="" echo Build number not found.& goto error + +call "%GetRsVersion%" RS_BUILD_NUMBER_ADD RsBuildNumberAdd + +set RsVersion=%RsMajorVersion%.%RsMinorVersion%.%RsBuildNumber%%RsBuildNumberAdd% + +:: Check WMIC is available +wmic.exe alias /? >nul 2>&1 || echo WMIC is not available.&& goto error + +:: Use WMIC to retrieve date in format YYYYMMDD +set RsDate= +for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /format:list') do set RsDate=%%I +set RsDate=%RsDate:~0,4%%RsDate:~4,2%%RsDate:~6,2% + +set QtMainVersion=%QtVersion:~0,1% + +rem Qt 4 = QtSvg4.dll +rem Qt 5 = Qt5Svg.dll +set QtMainVersion1= +set QtMainVersion2= +if "%QtMainVersion%"=="4" set QtMainVersion2=4 +if "%QtMainVersion%"=="5" set QtMainVersion1=5 + +if "%RsBuildConfig%" NEQ "release" ( + set Archive=%RsPackPath%\RetroShare-%RsVersion%-Windows-Portable-%RsDate%-%RsRevision%-Qt-%QtVersion%%RsArchiveAdd%-%RsBuildConfig%.7z +) else ( + set Archive=%RsPackPath%\RetroShare-%RsVersion%-Windows-Portable-%RsDate%-%RsRevision%-Qt-%QtVersion%%RsArchiveAdd%.7z +) + +if exist "%Archive%" del /Q "%Archive%" + +:: Create deploy path +mkdir "%RsDeployPath%" + +title Pack - %SourceName%-%RsBuildConfig% [copy files] + +set ExtensionsFile=%SourcePath%\libretroshare\src\rsserver\rsinit.cc +set Extensions= +for /f %%e in ('type "%ExtensionsFile%" ^| "%EnvSedExe%" -n "s/^.*\/\(extensions[^/]*\)\/.*$/\1/p" ^| "%EnvSedExe%" -n "1,1p"') do set Extensions=%%e +if "%Extensions%"=="" echo Folder for extensions not found in %ExtensionsFile%& goto error + +:: Copy files +mkdir "%RsDeployPath%\Data\%Extensions%" +mkdir "%RsDeployPath%\imageformats" +mkdir "%RsDeployPath%\qss" +mkdir "%RsDeployPath%\stylesheets" +mkdir "%RsDeployPath%\sounds" +mkdir "%RsDeployPath%\translations" + +copy nul "%RsDeployPath%\portable" %Quite% + +echo copy binaries +copy "%RsBuildPath%\retroshare-gui\src\%RsBuildConfig%\RetroShare*.exe" "%RsDeployPath%" %Quite% +copy "%RsBuildPath%\retroshare-nogui\src\%RsBuildConfig%\retroshare*-nogui.exe" "%RsDeployPath%" %Quite% + +echo copy extensions +for /D %%D in ("%RsBuildPath%\plugins\*") do ( + call :copy_extension "%%D" "%RsDeployPath%\Data\%Extensions%" + call :copy_dependencies "%RsDeployPath%\Data\%Extensions%\%%~nxD.dll" "%RsDeployPath%" +) + +echo copy external binaries +copy "%RootPath%\libs\bin\*.dll" "%RsDeployPath%" %Quite% + +echo copy dependencies +call :copy_dependencies "%RsDeployPath%\RetroShare06.exe" "%RsDeployPath%" + +echo copy Qt DLL's +copy "%QtPath%\Qt%QtMainVersion1%Svg%QtMainVersion2%.dll" "%RsDeployPath%" %Quite% + +if "%QtMainVersion%"=="5" ( + mkdir "%RsDeployPath%\platforms" + copy "%QtPath%\..\plugins\platforms\qwindows.dll" "%RsDeployPath%\platforms" %Quite% + mkdir "%RsDeployPath%\audio" + copy "%QtPath%\..\plugins\audio\qtaudio_windows.dll" "%RsDeployPath%\audio" %Quite% +) + +copy "%QtPath%\..\plugins\imageformats\*.dll" "%RsDeployPath%\imageformats" %Quite% +del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite% + +echo copy qss +xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite% + +echo copy stylesheets +xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite% +rmdir /S /Q "%RsDeployPath%\stylesheets\compact" %Quite% +rmdir /S /Q "%RsDeployPath%\stylesheets\standard" %Quite% +rmdir /S /Q "%RsDeployPath%\stylesheets\__MACOSX__Bubble" %Quite% + +echo copy sounds +xcopy /S "%SourcePath%\retroshare-gui\src\sounds" "%RsDeployPath%\sounds" %Quite% + +echo copy translation +copy "%SourcePath%\retroshare-gui\src\translations\qt_tr.qm" "%RsDeployPath%\translations" %Quite% +copy "%QtPath%\..\translations\qt_*.qm" "%RsDeployPath%\translations" %Quite% +if "%QtMainVersion%"=="5" ( + copy "%QtPath%\..\translations\qtbase_*.qm" "%RsDeployPath%\translations" %Quite% + copy "%QtPath%\..\translations\qtscript_*.qm" "%RsDeployPath%\translations" %Quite% + copy "%QtPath%\..\translations\qtquick1_*.qm" "%RsDeployPath%\translations" %Quite% + copy "%QtPath%\..\translations\qtmultimedia_*.qm" "%RsDeployPath%\translations" %Quite% + copy "%QtPath%\..\translations\qtxmlpatterns_*.qm" "%RsDeployPath%\translations" %Quite% +) + +echo copy bdboot.txt +copy "%SourcePath%\libbitdht\src\bitdht\bdboot.txt" "%RsDeployPath%" %Quite% + +echo copy changelog.txt +copy "%SourcePath%\retroshare-gui\src\changelog.txt" "%RsDeployPath%" %Quite% + +if exist "%SourcePath%\libresapi\src\webui" ( + echo copy webui + mkdir "%RsDeployPath%\webui" + xcopy /S "%SourcePath%\libresapi\src\webui" "%RsDeployPath%\webui" %Quite% +) + +rem pack files +title Pack - %SourceName%-%RsBuildConfig% [pack files] + +"%SevenZipExe%" a -mx=9 -t7z "%Archive%" "%RsDeployPath%\*" + +title %COMSPEC% + +call :cleanup + +endlocal +exit /B 0 + +:error +call :Cleanup +endlocal +exit /B 1 + +:cleanup +goto :EOF + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 + +:copy_extension +if exist "%~1\%RsBuildConfig%\%~n1.dll" ( + copy "%~1\%RsBuildConfig%\%~n1.dll" %2 %Quite% +) +goto :EOF + +:copy_dependencies +for /F "usebackq" %%A in (`%ToolsPath%\depends.bat list %1`) do ( + if exist "%QtPath%\%%A" ( + copy "%QtPath%\%%A" %2 %Quite% + ) else ( + if exist "%MinGWPath%\%%A" ( + copy "%MinGWPath%\%%A" %2 %Quite% + ) + ) +) +goto :EOF diff --git a/build_scripts/Windows/build_libs/.gitignore b/build_scripts/Windows/build_libs/.gitignore deleted file mode 100644 index 9f502e129..000000000 --- a/build_scripts/Windows/build_libs/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -/download -/msys -/libs -/tools/7z.dll -/tools/7z.exe -/tools/curl.exe \ No newline at end of file diff --git a/build_scripts/Windows/build_libs/1-prepare.bat b/build_scripts/Windows/build_libs/1-prepare.bat deleted file mode 100644 index 4c93d54ef..000000000 --- a/build_scripts/Windows/build_libs/1-prepare.bat +++ /dev/null @@ -1,43 +0,0 @@ -@setlocal - -@echo off - -:: Initialize environment -call "%~dp0_env.bat" - -set SevenZipUrl=http://7-zip.org/a/7z1602.msi -set SevenZipInstall=7z1602.msi -set CurlUrl=https://bintray.com/artifact/download/vszakats/generic/curl-7.50.1-win32-mingw.7z -set CurlInstall=curl-7.50.1-win32-mingw.7z - -if not exist "%DownloadPath%" mkdir "%DownloadPath%" - -call :remove_dir "%TempPath%" - -echo Download installation files -if not exist "%DownloadPath%\%SevenZipInstall%" call "%ToolsPath%\winhttpjs.bat" %SevenZipUrl% -saveTo "%DownloadPath%\%SevenZipInstall%" -if not exist "%DownloadPath%\%SevenZipInstall%" echo Cannot download 7z& goto :exit - -if not exist "%DownloadPath%\%CurlInstall%" call "%ToolsPath%\winhttpjs.bat" %CurlUrl% -saveTo "%DownloadPath%\%CurlInstall%" -if not exist "%DownloadPath%\%CurlInstall%" echo Cannot download Curl& goto :exit - -echo Unpack 7z -msiexec /a "%DownloadPath%\%SevenZipInstall%" /qb TARGETDIR="%TempPath%" -copy "%TempPath%\Files\7-Zip\7z.dll" "%ToolsPath%" -copy "%TempPath%\Files\7-Zip\7z.exe" "%ToolsPath%" -call :remove_dir "%TempPath%" - -echo Unpack Curl -"%SevenZipExe%" x -o"%TempPath%" "%DownloadPath%\%CurlInstall%" -copy "%TempPath%\curl-7.50.1-win32-mingw\bin\curl.exe" "%ToolsPath%" -call :remove_dir "%TempPath%" - -:exit -endlocal -exit /B 0 - -:remove_dir -if not exist %1 goto :EOF -del /s /f /q %1 >nul -rmdir /s /q %1 -goto :EOF diff --git a/build_scripts/Windows/build_libs/2-install-msys.bat b/build_scripts/Windows/build_libs/2-install-msys.bat deleted file mode 100644 index 7e17a2dc0..000000000 --- a/build_scripts/Windows/build_libs/2-install-msys.bat +++ /dev/null @@ -1,70 +0,0 @@ -@setlocal - -@echo off - -:: Initialize environment -call "%~dp0_env.bat" - -set MSYSInstall=mingw-get-0.6.2-mingw32-beta-20131004-1-bin.zip -set CMakeInstall=cmake-3.1.0-win32-x86.zip -set CMakeUnpackPath=%MSYSPath%\msys\1.0 - -if not exist "%DownloadPath%" mkdir "%DownloadPath%" - -echo Check existing installation -if not exist "%MSYSPath%\bin\mingw-get.exe" goto proceed -choice /M "Found existing MSYS version. Do you want to proceed?" -if %ERRORLEVEL%==2 goto exit - -:proceed -echo Remove previous MSYS version -call :remove_dir "%MSYSPath%" - -echo Download installation files -if not exist "%DownloadPath%\%MSYSInstall%" "%CurlExe%" -L -k http://sourceforge.net/projects/mingw/files/Installer/mingw-get/mingw-get-0.6.2-beta-20131004-1/%MSYSInstall%/download -o "%DownloadPath%\%MSYSInstall%" -if not exist "%DownloadPath%%\MSYSInstall%" echo Cannot download MSYS& goto :exit - -if not exist "%DownloadPath%\%CMakeInstall%" "%CurlExe%" -L -k http://www.cmake.org/files/v3.1/cmake-3.1.0-win32-x86.zip -o "%DownloadPath%\%CMakeInstall%" -if not exist "%DownloadPath%\%CMakeInstall%" echo Cannot download CMake& goto :exit - -echo Unpack MSYS -"%SevenZipExe%" x -o"%MSYSPath%" "%DownloadPath%\%MSYSInstall%" - -echo Install MSYS -if not exist "%MSYSPath%\var\lib\mingw-get\data\profile.xml" copy "%MSYSPath%\var\lib\mingw-get\data\defaults.xml" "%MSYSPath%\var\lib\mingw-get\data\profile.xml" -pushd "%MSYSPath%\bin" -mingw-get.exe install mingw32-mingw-get -mingw-get.exe install msys-coreutils -mingw-get.exe install msys-base -mingw-get.exe install msys-autoconf -mingw-get.exe install msys-automake -mingw-get.exe install msys-autogen -mingw-get.exe install msys-mktemp -mingw-get.exe install msys-wget -popd - -echo Unpack CMake -"%SevenZipExe%" x -o"%CMakeUnpackPath%" "%DownloadPath%\%CMakeInstall%" - -echo Install CMake -set CMakeVersion= -for /D %%F in (%CMakeUnpackPath%\cmake*) do set CMakeVersion=%%~nxF -if "%CMakeVersion%"=="" echo CMake version not found.& goto :exit -echo Found CMake version %CMakeVersion% - -set FoundProfile= -for /f "tokens=3" %%F in ('find /c /i "%CMakeVersion%" "%MSYSPath%\msys\1.0\etc\profile"') do set FoundProfile=%%F - -if "%FoundProfile%"=="0" ( - echo export PATH="${PATH}:/%CMakeVersion%/bin">>"%MSYSPath%\msys\1.0\etc\profile" -) - -:exit -endlocal -exit /B 0 - -:remove_dir -if not exist %1 goto :EOF -del /s /f /q %1 >nul -rmdir /s /q %1 -goto :EOF diff --git a/build_scripts/Windows/build_libs/3-build-libs.bat b/build_scripts/Windows/build_libs/3-build-libs.bat deleted file mode 100644 index 3b7a3c92c..000000000 --- a/build_scripts/Windows/build_libs/3-build-libs.bat +++ /dev/null @@ -1,29 +0,0 @@ -@setlocal - -@echo off - -:: Initialize environment -call "%~dp0_env.bat" - -set MSYSSH=%MSYSPath%\msys\1.0\bin\sh.exe -set MSYSCurPath=/%CurPath:~0,1%/%CurPath:~3% -set MSYSCurPath=%MSYSCurPath:\=/% - -if not exist "%MSYSSH%" echo Please install MSYS first.&& exit /B 1 - -set GCCPath= -call :FIND_IN_PATH g++.exe GCCPath -if "%GCCPath%"=="" echo Please run %~nx0 in the Qt Command Prompt or add the path to MinGW bin folder to PATH variable.&& exit /B 1 - -"%MSYSSH%" --login -i -c "cd "%MSYSCurPath%" && make -f makefile %*" - -exit /B %ERRORLEVEL% - -:FIND_IN_PATH -SET PathTemp="%Path:;=";"%" -FOR %%P IN (%PathTemp%) DO ( - IF EXIST "%%~P.\%~1" ( - set %2=%%~P - goto :EOF - ) -) diff --git a/build_scripts/Windows/build_libs/Makefile b/build_scripts/Windows/build_libs/Makefile index c53ada54f..5f0a2351f 100755 --- a/build_scripts/Windows/build_libs/Makefile +++ b/build_scripts/Windows/build_libs/Makefile @@ -13,214 +13,331 @@ SQLCIPHER_VERSION=2.2.1 LIBMICROHTTPD_VERSION=0.9.46 FFMPEG_VERSION=3.1.2 +MAKEFILE_PATH=$(dir $(MAKEFILE_LIST)) +LIBS_PATH?=$(MAKEFILE_PATH)../../../../libs + +DOWNLOAD_PATH?=download +COPY_ANSWER?= + all: dirs zlib bzip2 miniupnpc openssl speex speexdsp opencv libxml2 libxslt curl sqlcipher libmicrohttpd ffmpeg copylibs download: \ - download/zlib-$(ZLIB_VERSION).tar.gz \ - download/bzip2-$(BZIP2_VERSION).tar.gz \ - download/miniupnpc-$(MINIUPNPC_VERSION).tar.gz \ - download/openssl-$(OPENSSL_VERSION).tar.gz \ - download/speex-$(SPEEX_VERSION).tar.gz \ - download/speexdsp-$(SPEEXDSP_VERSION).tar.gz \ - download/opencv-$(OPENCV_VERSION).tar.gz \ - download/libxml2-$(LIBXML2_VERSION).tar.gz \ - download/libxslt-$(LIBXSLT_VERSION).tar.gz \ - download/curl-$(CURL_VERSION).tar.gz \ - download/tcl$(TCL_VERSION)-src.tar.gz \ - download/sqlcipher-$(SQLCIPHER_VERSION).tar.gz \ - download/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz \ - download/ffmpeg-$(FFMPEG_VERSION).tar.gz + $(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/openssl-$(OPENSSL_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/speex-$(SPEEX_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/speexdsp-$(SPEEXDSP_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/libxslt-$(LIBXSLT_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/curl-$(CURL_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/tcl$(TCL_VERSION)-src.tar.gz \ + $(DOWNLOAD_PATH)/sqlcipher-$(SQLCIPHER_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz \ + $(DOWNLOAD_PATH)/ffmpeg-$(FFMPEG_VERSION).tar.gz + +clean: + rm -r -f libs dirs: - mkdir -p download - mkdir -p libs/include - mkdir -p libs/lib - mkdir -p libs/bin + mkdir -p $(DOWNLOAD_PATH) + mkdir -p libs + gcc --version | head --lines 1 | tr ' ' '\n' | tail -1 >libs/gcc-version -download/zlib-$(ZLIB_VERSION).tar.gz: - wget --no-check-certificate http://sourceforge.net/projects/libpng/files/zlib/$(ZLIB_VERSION)/zlib-$(ZLIB_VERSION).tar.gz/download -O download/zlib-$(ZLIB_VERSION).tar.gz +$(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz: + wget --no-check-certificate http://sourceforge.net/projects/libpng/files/zlib/$(ZLIB_VERSION)/zlib-$(ZLIB_VERSION).tar.gz/download -O $(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz -zlib: download/zlib-$(ZLIB_VERSION).tar.gz - tar xvf download/zlib-$(ZLIB_VERSION).tar.gz +zlib: libs/zlib-$(ZLIB_VERSION) + +libs/zlib-$(ZLIB_VERSION): $(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz + # prepare + rm -r -f libs/zlib-* + tar xvf $(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz + # build cd zlib-$(ZLIB_VERSION) && ./configure #cd zlib-$(ZLIB_VERSION) && make install prefix="`pwd`/../libs" cd zlib-$(ZLIB_VERSION) && make - cp zlib-$(ZLIB_VERSION)/zlib.h libs/include/ - cp zlib-$(ZLIB_VERSION)/zconf.h libs/include/ - cp zlib-$(ZLIB_VERSION)/libz.a libs/lib/ + # copy files + mkdir -p libs/zlib-$(ZLIB_VERSION).tmp/include + cp zlib-$(ZLIB_VERSION)/zlib.h libs/zlib-$(ZLIB_VERSION).tmp/include/ + cp zlib-$(ZLIB_VERSION)/zconf.h libs/zlib-$(ZLIB_VERSION).tmp/include/ + mkdir -p libs/zlib-$(ZLIB_VERSION).tmp/lib + cp zlib-$(ZLIB_VERSION)/libz.a libs/zlib-$(ZLIB_VERSION).tmp/lib/ rm -r -f zlib-$(ZLIB_VERSION) - touch zlib + mv libs/zlib-$(ZLIB_VERSION).tmp libs/zlib-$(ZLIB_VERSION) -download/bzip2-$(BZIP2_VERSION).tar.gz: - wget http://bzip.org/$(BZIP2_VERSION)/bzip2-$(BZIP2_VERSION).tar.gz -O download/bzip2-$(BZIP2_VERSION).tar.gz +bzip2: libs/bzip2-$(BZIP2_VERSION) -bzip2: download/bzip2-$(BZIP2_VERSION).tar.gz - tar xvf download/bzip2-$(BZIP2_VERSION).tar.gz +$(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz: + wget http://bzip.org/$(BZIP2_VERSION)/bzip2-$(BZIP2_VERSION).tar.gz -O $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz + +libs/bzip2-$(BZIP2_VERSION): $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz + # prepare + rm -r -f libs/bzip2-* + tar xvf $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz + # build #cd bzip2-$(BZIP2_VERSION) && make install PREFIX="`pwd`/../libs" cd bzip2-$(BZIP2_VERSION) && make - cp bzip2-$(BZIP2_VERSION)/bzlib.h libs/include/ - cp bzip2-$(BZIP2_VERSION)/libbz2.a libs/lib/ + # copy files + mkdir -p libs/bzip2-$(BZIP2_VERSION).tmp/include + cp bzip2-$(BZIP2_VERSION)/bzlib.h libs/bzip2-$(BZIP2_VERSION).tmp/include/ + mkdir -p libs/bzip2-$(BZIP2_VERSION).tmp/lib + cp bzip2-$(BZIP2_VERSION)/libbz2.a libs/bzip2-$(BZIP2_VERSION).tmp/lib/ rm -r -f bzip2-$(BZIP2_VERSION) - touch bzip2 + mv libs/bzip2-$(BZIP2_VERSION).tmp libs/bzip2-$(BZIP2_VERSION) -download/miniupnpc-$(MINIUPNPC_VERSION).tar.gz: - wget http://miniupnp.free.fr/files/download.php?file=miniupnpc-$(MINIUPNPC_VERSION).tar.gz -O download/miniupnpc-$(MINIUPNPC_VERSION).tar.gz +miniupnpc: libs/miniupnpc-$(MINIUPNPC_VERSION) -miniupnpc: download/miniupnpc-$(MINIUPNPC_VERSION).tar.gz - tar xvf download/miniupnpc-$(MINIUPNPC_VERSION).tar.gz +$(DOWNLOAD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tar.gz: + wget http://miniupnp.free.fr/files/download.php?file=miniupnpc-$(MINIUPNPC_VERSION).tar.gz -O $(DOWNLOAD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tar.gz + +libs/miniupnpc-$(MINIUPNPC_VERSION): $(DOWNLOAD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tar.gz + # prepare + rm -r -f libs/miniupnpc-* + tar xvf $(DOWNLOAD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tar.gz + # build cd miniupnpc-$(MINIUPNPC_VERSION) && CC=gcc && export CC && make -f Makefile.mingw init libminiupnpc.a miniupnpc.dll - mkdir -p libs/include/miniupnpc && cp miniupnpc-$(MINIUPNPC_VERSION)/*.h libs/include/miniupnpc/ - cp miniupnpc-$(MINIUPNPC_VERSION)/miniupnpc.lib libs/lib/ - cp miniupnpc-$(MINIUPNPC_VERSION)/miniupnpc.dll libs/bin/ + # copy files + mkdir -p libs/miniupnpc-$(MINIUPNPC_VERSION).tmp/include/miniupnpc + cp miniupnpc-$(MINIUPNPC_VERSION)/*.h libs/miniupnpc-$(MINIUPNPC_VERSION).tmp/include/miniupnpc/ + mkdir -p libs/miniupnpc-$(MINIUPNPC_VERSION).tmp/lib + cp miniupnpc-$(MINIUPNPC_VERSION)/miniupnpc.lib libs/miniupnpc-$(MINIUPNPC_VERSION).tmp/lib/ + mkdir -p libs/miniupnpc-$(MINIUPNPC_VERSION).tmp/bin + cp miniupnpc-$(MINIUPNPC_VERSION)/miniupnpc.dll libs/miniupnpc-$(MINIUPNPC_VERSION).tmp/bin/ rm -r -f miniupnpc-$(MINIUPNPC_VERSION) - touch miniupnpc + mv libs/miniupnpc-$(MINIUPNPC_VERSION).tmp libs/miniupnpc-$(MINIUPNPC_VERSION) -download/openssl-$(OPENSSL_VERSION).tar.gz: - wget --no-check-certificate https://www.openssl.org/source/openssl-$(OPENSSL_VERSION).tar.gz -O download/openssl-$(OPENSSL_VERSION).tar.gz +openssl: libs/openssl-$(OPENSSL_VERSION) -openssl: download/openssl-$(OPENSSL_VERSION).tar.gz - tar xvf download/openssl-$(OPENSSL_VERSION).tar.gz +$(DOWNLOAD_PATH)/openssl-$(OPENSSL_VERSION).tar.gz: + wget --no-check-certificate https://www.openssl.org/source/openssl-$(OPENSSL_VERSION).tar.gz -O $(DOWNLOAD_PATH)/openssl-$(OPENSSL_VERSION).tar.gz + +libs/openssl-$(OPENSSL_VERSION): $(DOWNLOAD_PATH)/openssl-$(OPENSSL_VERSION).tar.gz + # prepare + rm -r -f libs/openssl-* + tar xvf $(DOWNLOAD_PATH)/openssl-$(OPENSSL_VERSION).tar.gz + # build #cd openssl-$(OPENSSL_VERSION) && ./config --prefix="`pwd`/../libs" #cd openssl-$(OPENSSL_VERSION) && make install cd openssl-$(OPENSSL_VERSION) && ./config shared cd openssl-$(OPENSSL_VERSION) && make - mkdir -p libs/include/openssl && cp openssl-$(OPENSSL_VERSION)/include/openssl/*.h libs/include/openssl/ - cp openssl-$(OPENSSL_VERSION)/libeay32.dll libs/bin/ - cp openssl-$(OPENSSL_VERSION)/ssleay32.dll libs/bin/ - cp openssl-$(OPENSSL_VERSION)/libcrypto.dll.a libs/lib/ - cp openssl-$(OPENSSL_VERSION)/libssl.dll.a libs/lib/ + # copy files + mkdir -p libs/openssl-$(OPENSSL_VERSION).tmp/include/openssl + cp openssl-$(OPENSSL_VERSION)/include/openssl/*.h libs/openssl-$(OPENSSL_VERSION).tmp/include/openssl/ + mkdir -p libs/openssl-$(OPENSSL_VERSION).tmp/bin + cp openssl-$(OPENSSL_VERSION)/libeay32.dll libs/openssl-$(OPENSSL_VERSION).tmp/bin/ + cp openssl-$(OPENSSL_VERSION)/ssleay32.dll libs/openssl-$(OPENSSL_VERSION).tmp/bin/ + mkdir -p libs/openssl-$(OPENSSL_VERSION).tmp/lib + cp openssl-$(OPENSSL_VERSION)/libcrypto.dll.a libs/openssl-$(OPENSSL_VERSION).tmp/lib/ + cp openssl-$(OPENSSL_VERSION)/libssl.dll.a libs/openssl-$(OPENSSL_VERSION).tmp/lib/ rm -r -f openssl-$(OPENSSL_VERSION) - touch openssl + mv libs/openssl-$(OPENSSL_VERSION).tmp libs/openssl-$(OPENSSL_VERSION) -download/speex-$(SPEEX_VERSION).tar.gz: - wget http://downloads.xiph.org/releases/speex/speex-$(SPEEX_VERSION).tar.gz -O download/speex-$(SPEEX_VERSION).tar.gz +speex: libs/speex-$(SPEEX_VERSION) -speex: download/speex-$(SPEEX_VERSION).tar.gz - tar xvf download/speex-$(SPEEX_VERSION).tar.gz +$(DOWNLOAD_PATH)/speex-$(SPEEX_VERSION).tar.gz: + wget http://downloads.xiph.org/releases/speex/speex-$(SPEEX_VERSION).tar.gz -O $(DOWNLOAD_PATH)/speex-$(SPEEX_VERSION).tar.gz + +libs/speex-$(SPEEX_VERSION): $(DOWNLOAD_PATH)/speex-$(SPEEX_VERSION).tar.gz + # prepare + rm -r -f libs/speex-* + tar xvf $(DOWNLOAD_PATH)/speex-$(SPEEX_VERSION).tar.gz + # build cd speex-$(SPEEX_VERSION) && ./configure #cd speex-$(SPEEX_VERSION) && make install exec_prefix="`pwd`/../libs" cd speex-$(SPEEX_VERSION) && make - mkdir -p libs/include/speex && cp speex-$(SPEEX_VERSION)/include/speex/*.h libs/include/speex/ - cp speex-$(SPEEX_VERSION)/libspeex/.libs/libspeex.a libs/lib + # copy files + mkdir -p libs/speex-$(SPEEX_VERSION).tmp/include/speex + cp speex-$(SPEEX_VERSION)/include/speex/*.h libs/speex-$(SPEEX_VERSION).tmp/include/speex/ + mkdir -p libs/speex-$(SPEEX_VERSION).tmp/lib + cp speex-$(SPEEX_VERSION)/libspeex/.libs/libspeex.a libs/speex-$(SPEEX_VERSION).tmp/lib rm -r -f speex-$(SPEEX_VERSION) - touch speex + mv libs/speex-$(SPEEX_VERSION).tmp libs/speex-$(SPEEX_VERSION) -download/speexdsp-$(SPEEXDSP_VERSION).tar.gz: - wget http://downloads.xiph.org/releases/speex/speexdsp-$(SPEEXDSP_VERSION).tar.gz -O download/speexdsp-$(SPEEXDSP_VERSION).tar.gz +speexdsp: libs/speexdsp-$(SPEEXDSP_VERSION) -speexdsp: download/speexdsp-$(SPEEXDSP_VERSION).tar.gz - tar xvf download/speexdsp-$(SPEEXDSP_VERSION).tar.gz +$(DOWNLOAD_PATH)/speexdsp-$(SPEEXDSP_VERSION).tar.gz: + wget http://downloads.xiph.org/releases/speex/speexdsp-$(SPEEXDSP_VERSION).tar.gz -O $(DOWNLOAD_PATH)/speexdsp-$(SPEEXDSP_VERSION).tar.gz + +libs/speexdsp-$(SPEEXDSP_VERSION): $(DOWNLOAD_PATH)/speexdsp-$(SPEEXDSP_VERSION).tar.gz + # prepare + rm -r -f libs/speexdsp-* + tar xvf $(DOWNLOAD_PATH)/speexdsp-$(SPEEXDSP_VERSION).tar.gz + # build cd speexdsp-$(SPEEXDSP_VERSION) && ./configure cd speexdsp-$(SPEEXDSP_VERSION) && make - mkdir -p libs/include/speex && cp speexdsp-$(SPEEXDSP_VERSION)/include/speex/*.h libs/include/speex/ - cp speexdsp-$(SPEEXDSP_VERSION)/libspeexdsp/.libs/libspeexdsp.a libs/lib + # copy files + mkdir -p libs/speexdsp-$(SPEEXDSP_VERSION).tmp/include/speex + cp speexdsp-$(SPEEXDSP_VERSION)/include/speex/*.h libs/speexdsp-$(SPEEXDSP_VERSION).tmp/include/speex/ + mkdir -p libs/speexdsp-$(SPEEXDSP_VERSION).tmp/lib + cp speexdsp-$(SPEEXDSP_VERSION)/libspeexdsp/.libs/libspeexdsp.a libs/speexdsp-$(SPEEXDSP_VERSION).tmp/lib rm -r -f speexdsp-$(SPEEXDSP_VERSION) - touch speexdsp + mv libs/speexdsp-$(SPEEXDSP_VERSION).tmp libs/speexdsp-$(SPEEXDSP_VERSION) -download/opencv-$(OPENCV_VERSION).tar.gz: - wget --no-check-certificate https://github.com/Itseez/opencv/archive/$(OPENCV_VERSION).tar.gz -O download/opencv-$(OPENCV_VERSION).tar.gz +opencv: libs/opencv-$(OPENCV_VERSION) -opencv: download/opencv-$(OPENCV_VERSION).tar.gz - tar xvf download/opencv-$(OPENCV_VERSION).tar.gz +$(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz: + wget --no-check-certificate https://github.com/Itseez/opencv/archive/$(OPENCV_VERSION).tar.gz -O $(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz + +libs/opencv-$(OPENCV_VERSION): $(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz + # prepare + rm -r -f libs/opencv-* + tar xvf $(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz + # build mkdir -p opencv-$(OPENCV_VERSION)/build #cd opencv-$(OPENCV_VERSION)/build && cmake .. -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="`pwd`/../../libs" cd opencv-$(OPENCV_VERSION)/build && cmake .. -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="`pwd`/install" cd opencv-$(OPENCV_VERSION)/build && make install - cp -r opencv-$(OPENCV_VERSION)/build/install/include/* libs/include/ - mkdir -p libs/lib/opencv && cp -r opencv-$(OPENCV_VERSION)/build/install/x86/mingw/staticlib/* libs/lib/opencv/ + # copy files + mkdir -p libs/opencv-$(OPENCV_VERSION).tmp/include + cp -r opencv-$(OPENCV_VERSION)/build/install/include/* libs/opencv-$(OPENCV_VERSION).tmp/include/ + mkdir -p libs/opencv-$(OPENCV_VERSION).tmp/lib/opencv + cp -r opencv-$(OPENCV_VERSION)/build/install/x86/mingw/staticlib/* libs/opencv-$(OPENCV_VERSION).tmp/lib/opencv/ rm -r -f opencv-$(OPENCV_VERSION) - touch opencv + mv libs/opencv-$(OPENCV_VERSION).tmp libs/opencv-$(OPENCV_VERSION) -download/libxml2-$(LIBXML2_VERSION).tar.gz: - wget ftp://xmlsoft.org/libxml2/libxml2-$(LIBXML2_VERSION).tar.gz -O download/libxml2-$(LIBXML2_VERSION).tar.gz +libxml2: libs/libxml2-$(LIBXML2_VERSION) -libxml2: download/libxml2-$(LIBXML2_VERSION).tar.gz - tar xvf download/libxml2-$(LIBXML2_VERSION).tar.gz +$(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz: + wget ftp://xmlsoft.org/libxml2/libxml2-$(LIBXML2_VERSION).tar.gz -O $(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz + +libs/libxml2-$(LIBXML2_VERSION): $(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz + # prepare + rm -r -f libs/libxml2-* + tar xvf $(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz + # build cd libxml2-$(LIBXML2_VERSION) && ./configure --without-iconv -enable-shared=no #cd libxml2-$(LIBXML2_VERSION) && make install exec_prefix="`pwd`/../libs" cd libxml2-$(LIBXML2_VERSION) && make - mkdir -p libs/include/libxml && cp libxml2-$(LIBXML2_VERSION)/include/libxml/*.h libs/include/libxml/ - cp libxml2-$(LIBXML2_VERSION)/.libs/libxml2.a libs/lib/ - touch libxml2 + # copy files + mkdir -p libs/libxml2-$(LIBXML2_VERSION).tmp/include/libxml + cp libxml2-$(LIBXML2_VERSION)/include/libxml/*.h libs/libxml2-$(LIBXML2_VERSION).tmp/include/libxml/ + mkdir -p libs/libxml2-$(LIBXML2_VERSION).tmp/lib + cp libxml2-$(LIBXML2_VERSION)/.libs/libxml2.a libs/libxml2-$(LIBXML2_VERSION).tmp/lib/ + #rm -r -f libxml2-$(LIBXML2_VERSION) # see libxslt + mv libs/libxml2-$(LIBXML2_VERSION).tmp libs/libxml2-$(LIBXML2_VERSION) -download/libxslt-$(LIBXSLT_VERSION).tar.gz: - wget ftp://xmlsoft.org/libxml2/libxslt-$(LIBXSLT_VERSION).tar.gz -O download/libxslt-$(LIBXSLT_VERSION).tar.gz +libxslt: libs/libxslt-$(LIBXSLT_VERSION) -libxslt: download/libxml2-$(LIBXML2_VERSION).tar.gz download/libxslt-$(LIBXSLT_VERSION).tar.gz - tar xvf download/libxml2-$(LIBXML2_VERSION).tar.gz - tar xvf download/libxslt-$(LIBXSLT_VERSION).tar.gz - tar xvf libxslt-$(LIBXSLT_VERSION)-fix.tar.gz +$(DOWNLOAD_PATH)/libxslt-$(LIBXSLT_VERSION).tar.gz: + wget ftp://xmlsoft.org/libxml2/libxslt-$(LIBXSLT_VERSION).tar.gz -O $(DOWNLOAD_PATH)/libxslt-$(LIBXSLT_VERSION).tar.gz + +libs/libxslt-$(LIBXSLT_VERSION): $(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz $(DOWNLOAD_PATH)/libxslt-$(LIBXSLT_VERSION).tar.gz + # prepare + rm -r -f libs/libxslt-* + tar xvf $(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz + tar xvf $(DOWNLOAD_PATH)/libxslt-$(LIBXSLT_VERSION).tar.gz + tar xvf $(MAKEFILE_PATH)libxslt-$(LIBXSLT_VERSION)-fix.tar.gz + # build cd libxslt-$(LIBXSLT_VERSION) && ./configure --with-libxml-src=../libxml2-$(LIBXML2_VERSION) -enable-shared=no CFLAGS=-DLIBXML_STATIC cd libxslt-$(LIBXSLT_VERSION) && make - mkdir -p libs/include/libxslt && cp libxslt-$(LIBXSLT_VERSION)/libxslt/*.h libs/include/libxslt/ - cp libxslt-$(LIBXSLT_VERSION)/libxslt/.libs/libxslt.a libs/lib/ - cp libxslt-$(LIBXSLT_VERSION)/libexslt/.libs/libexslt.a libs/lib/ + # copy files + mkdir -p libs/libxslt-$(LIBXSLT_VERSION).tmp/include/libxslt + cp libxslt-$(LIBXSLT_VERSION)/libxslt/*.h libs/libxslt-$(LIBXSLT_VERSION).tmp/include/libxslt/ + mkdir -p libs/libxslt-$(LIBXSLT_VERSION).tmp/lib + cp libxslt-$(LIBXSLT_VERSION)/libxslt/.libs/libxslt.a libs/libxslt-$(LIBXSLT_VERSION).tmp/lib/ + cp libxslt-$(LIBXSLT_VERSION)/libexslt/.libs/libexslt.a libs/libxslt-$(LIBXSLT_VERSION).tmp/lib/ rm -r -f libxml2-$(LIBXML2_VERSION) rm -r -f libxslt-$(LIBXSLT_VERSION) - touch libxslt + mv libs/libxslt-$(LIBXSLT_VERSION).tmp libs/libxslt-$(LIBXSLT_VERSION) -download/curl-$(CURL_VERSION).tar.gz: - wget --no-check-certificate http://curl.haxx.se/download/curl-$(CURL_VERSION).tar.gz -O download/curl-$(CURL_VERSION).tar.gz +curl: libs/curl-$(CURL_VERSION) -curl: download/curl-$(CURL_VERSION).tar.gz - tar xvf download/curl-$(CURL_VERSION).tar.gz - cd curl-$(CURL_VERSION) && ./configure --disable-shared --with-ssl="`pwd`/../libs" +$(DOWNLOAD_PATH)/curl-$(CURL_VERSION).tar.gz: + wget --no-check-certificate http://curl.haxx.se/$(DOWNLOAD_PATH)/curl-$(CURL_VERSION).tar.gz -O $(DOWNLOAD_PATH)/curl-$(CURL_VERSION).tar.gz + +libs/curl-$(CURL_VERSION): $(DOWNLOAD_PATH)/curl-$(CURL_VERSION).tar.gz + # prepare + rm -r -f libs/curl-* + tar xvf $(DOWNLOAD_PATH)/curl-$(CURL_VERSION).tar.gz + # build + cd curl-$(CURL_VERSION) && ./configure --disable-shared --with-ssl="`pwd`/../libs/openssl-$(OPENSSL_VERSION)" #cd curl-$(CURL_VERSION) && make install exec_prefix="`pwd`/../libs" cd curl-$(CURL_VERSION) && make - mkdir -p libs/include/curl && cp curl-$(CURL_VERSION)/include/curl/*.h libs/include/curl/ - cp curl-$(CURL_VERSION)/lib/.libs/libcurl.a libs/lib/ + # copy files + mkdir -p libs/curl-$(CURL_VERSION).tmp/include/curl + cp curl-$(CURL_VERSION)/include/curl/*.h libs/curl-$(CURL_VERSION).tmp/include/curl/ + mkdir -p libs/curl-$(CURL_VERSION).tmp/lib + cp curl-$(CURL_VERSION)/lib/.libs/libcurl.a libs/curl-$(CURL_VERSION).tmp/lib/ rm -r -f curl-$(CURL_VERSION) - touch curl + mv libs/curl-$(CURL_VERSION).tmp libs/curl-$(CURL_VERSION) -download/tcl$(TCL_VERSION)-src.tar.gz: - wget http://prdownloads.sourceforge.net/tcl/tcl$(TCL_VERSION)-src.tar.gz -O download/tcl$(TCL_VERSION)-src.tar.gz +sqlcipher: libs/sqlcipher-$(SQLCIPHER_VERSION) -download/sqlcipher-$(SQLCIPHER_VERSION).tar.gz: - wget --no-check-certificate https://github.com/sqlcipher/sqlcipher/archive/v$(SQLCIPHER_VERSION).tar.gz -O download/sqlcipher-$(SQLCIPHER_VERSION).tar.gz +$(DOWNLOAD_PATH)/tcl$(TCL_VERSION)-src.tar.gz: + wget http://prdownloads.sourceforge.net/tcl/tcl$(TCL_VERSION)-src.tar.gz -O $(DOWNLOAD_PATH)/tcl$(TCL_VERSION)-src.tar.gz -sqlcipher: download/tcl$(TCL_VERSION)-src.tar.gz download/sqlcipher-$(SQLCIPHER_VERSION).tar.gz +$(DOWNLOAD_PATH)/sqlcipher-$(SQLCIPHER_VERSION).tar.gz: + wget --no-check-certificate https://github.com/sqlcipher/sqlcipher/archive/v$(SQLCIPHER_VERSION).tar.gz -O $(DOWNLOAD_PATH)/sqlcipher-$(SQLCIPHER_VERSION).tar.gz + +libs/sqlcipher-$(SQLCIPHER_VERSION): $(DOWNLOAD_PATH)/tcl$(TCL_VERSION)-src.tar.gz $(DOWNLOAD_PATH)/sqlcipher-$(SQLCIPHER_VERSION).tar.gz + # prepare + rm -r -f libs/sqlcipher-* # tcl - tar xvf download/tcl$(TCL_VERSION)-src.tar.gz + tar xvf $(DOWNLOAD_PATH)/tcl$(TCL_VERSION)-src.tar.gz mkdir -p tcl$(TCL_VERSION)/build cd tcl$(TCL_VERSION)/build && ../win/configure cd tcl$(TCL_VERSION)/build && make #sqlcipher - tar xvf download/sqlcipher-$(SQLCIPHER_VERSION).tar.gz + tar xvf $(DOWNLOAD_PATH)/sqlcipher-$(SQLCIPHER_VERSION).tar.gz cd sqlcipher-$(SQLCIPHER_VERSION) && ln -s ../tcl$(TCL_VERSION)/build/tclsh86.exe tclsh mkdir -p tcl$(TCL_VERSION)/lib ln -s `pwd`/tcl$(TCL_VERSION)/library `pwd`/tcl$(TCL_VERSION)/lib/tcl8.6 - cd sqlcipher-$(SQLCIPHER_VERSION) && PATH=.:$$PATH:`pwd`/../tcl$(TCL_VERSION)/build && LIBS="-L`pwd`/../libs/lib -lgdi32 $$LIBS" && export LIBS && ./configure --disable-shared --enable-static --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC -I`pwd`/../libs/include -I`pwd`/../tcl$(TCL_VERSION)/generic" LDFLAGS="-L`pwd`/../libs/lib -lcrypto -lgdi32" --with-tcl="`pwd`/../tcl$(TCL_VERSION)/build" && make install prefix="`pwd`/install" - cp -r sqlcipher-$(SQLCIPHER_VERSION)/install/include/* libs/include/ - cp sqlcipher-$(SQLCIPHER_VERSION)/install/lib/libsqlcipher.a libs/lib/ - cp sqlcipher-$(SQLCIPHER_VERSION)/install/bin/sqlcipher.exe libs/bin/ + # build + cd sqlcipher-$(SQLCIPHER_VERSION) && PATH=.:$$PATH:`pwd`/../tcl$(TCL_VERSION)/build && LIBS="-L`pwd`/../libs/openssl-$(OPENSSL_VERSION)/lib -lgdi32 $$LIBS" && export LIBS && ./configure --disable-shared --enable-static --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC -I`pwd`/../libs/openssl-$(OPENSSL_VERSION)/include -I`pwd`/../tcl$(TCL_VERSION)/generic" LDFLAGS="-L`pwd`/../libs/openssl-$(OPENSSL_VERSION)/lib -lcrypto -lgdi32" --with-tcl="`pwd`/../tcl$(TCL_VERSION)/build" && make install prefix="`pwd`/install" + # copy files + mkdir -p libs/sqlcipher-$(SQLCIPHER_VERSION).tmp/include + cp -r sqlcipher-$(SQLCIPHER_VERSION)/install/include/* libs/sqlcipher-$(SQLCIPHER_VERSION).tmp/include/ + mkdir -p libs/sqlcipher-$(SQLCIPHER_VERSION).tmp/lib + cp sqlcipher-$(SQLCIPHER_VERSION)/install/lib/libsqlcipher.a libs/sqlcipher-$(SQLCIPHER_VERSION).tmp/lib/ + mkdir -p libs/sqlcipher-$(SQLCIPHER_VERSION).tmp/bin + cp sqlcipher-$(SQLCIPHER_VERSION)/install/bin/sqlcipher.exe libs/sqlcipher-$(SQLCIPHER_VERSION).tmp/bin/ rm -r -f sqlcipher-$(SQLCIPHER_VERSION) rm -r -f tcl$(TCL_VERSION) - touch sqlcipher + mv libs/sqlcipher-$(SQLCIPHER_VERSION).tmp libs/sqlcipher-$(SQLCIPHER_VERSION) -download/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz: - wget --no-check-certificate http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz -O download/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz +libmicrohttpd: libs/libmicrohttpd-$(LIBMICROHTTPD_VERSION) -libmicrohttpd: download/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz - tar xvf download/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz - cd libmicrohttpd-$(LIBMICROHTTPD_VERSION) && ./configure --disable-shared --enable-static --prefix="`pwd`/../libs" +$(DOWNLOAD_PATH)/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz: + wget --no-check-certificate http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz -O $(DOWNLOAD_PATH)/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz + +libs/libmicrohttpd-$(LIBMICROHTTPD_VERSION): $(DOWNLOAD_PATH)/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz + # prepare + rm -r -f libs/libmicrohttpd-* + tar xvf $(DOWNLOAD_PATH)/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tar.gz + # build + cd libmicrohttpd-$(LIBMICROHTTPD_VERSION) && ./configure --disable-shared --enable-static --prefix="`pwd`/../libs/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tmp" cd libmicrohttpd-$(LIBMICROHTTPD_VERSION) && make install + # copy files rm -r -f libmicrohttpd-$(LIBMICROHTTPD_VERSION) - touch libmicrohttpd + mv libs/libmicrohttpd-$(LIBMICROHTTPD_VERSION).tmp libs/libmicrohttpd-$(LIBMICROHTTPD_VERSION) -download/ffmpeg-$(FFMPEG_VERSION).tar.gz: - wget --no-check-certificate https://ffmpeg.org/releases/ffmpeg-$(FFMPEG_VERSION).tar.gz -O download/ffmpeg-$(FFMPEG_VERSION).tar.gz +ffmpeg: libs/ffmpeg-$(FFMPEG_VERSION) -ffmpeg: download/ffmpeg-$(FFMPEG_VERSION).tar.gz - tar xvf download/ffmpeg-$(FFMPEG_VERSION).tar.gz - cd ffmpeg-$(FFMPEG_VERSION) && ./configure --disable-shared --enable-static --disable-programs --disable-ffmpeg --disable-ffplay --disable-ffprobe --disable-ffserver --disable-doc --disable-htmlpages --disable-manpages --disable-podpages --disable-txtpages --disable-yasm --disable-everything --enable-encoder=mpeg4 --enable-decoder=mpeg4 --prefix="`pwd`/../libs" +$(DOWNLOAD_PATH)/ffmpeg-$(FFMPEG_VERSION).tar.gz: + wget --no-check-certificate https://ffmpeg.org/releases/ffmpeg-$(FFMPEG_VERSION).tar.gz -O $(DOWNLOAD_PATH)/ffmpeg-$(FFMPEG_VERSION).tar.gz + +libs/ffmpeg-$(FFMPEG_VERSION): $(DOWNLOAD_PATH)/ffmpeg-$(FFMPEG_VERSION).tar.gz + # prepare + rm -r -f libs/ffmpeg-* + tar xvf $(DOWNLOAD_PATH)/ffmpeg-$(FFMPEG_VERSION).tar.gz + # build + cd ffmpeg-$(FFMPEG_VERSION) && ./configure --disable-shared --enable-static --disable-programs --disable-ffmpeg --disable-ffplay --disable-ffprobe --disable-ffserver --disable-doc --disable-htmlpages --disable-manpages --disable-podpages --disable-txtpages --disable-yasm --disable-everything --enable-encoder=mpeg4 --enable-decoder=mpeg4 --prefix="`pwd`/../libs/ffmpeg-$(FFMPEG_VERSION).tmp" cd ffmpeg-$(FFMPEG_VERSION) && make install + # copy files rm -r -f ffmpeg-$(FFMPEG_VERSION) - touch ffmpeg + mv libs/ffmpeg-$(FFMPEG_VERSION).tmp libs/ffmpeg-$(FFMPEG_VERSION) - -copylibs: - read -p "Do you want to copy libs to retroshare? (yes|no)" answer; \ - if [ "$$answer" = "yes" ] ; then \ - cp -r libs ../../../../ ; \ +copylibs: + if [ "$(COPY_ANSWER)" = "" ] ; then \ + read -p "Do you want to copy libs to retroshare? (y|n)" answer; \ + else \ + answer=$(COPY_ANSWER) ; \ + fi ; \ + if [ "$$answer" = "y" ] ; then \ + rm -r -f $(LIBS_PATH) ; \ + mkdir -p $(LIBS_PATH) ; \ + cp ./libs/gcc-version $(LIBS_PATH) ; \ + find ./libs -mindepth 1 -maxdepth 1 -type d -not -name "*.tmp" -print -exec cp -r {}/. $(LIBS_PATH) \; ; \ fi - diff --git a/build_scripts/Windows/build_libs/_env.bat b/build_scripts/Windows/build_libs/_env.bat deleted file mode 100644 index 8ca4f15ec..000000000 --- a/build_scripts/Windows/build_libs/_env.bat +++ /dev/null @@ -1,9 +0,0 @@ -set CurPath=%~dp0 -set DownloadPath=%CurPath%download -set ToolsPath=%CurPath%tools -set TempPath=%CurPath%tmp -set MSYSPath=%CurPath%msys -set LibsPath=%CurPath%libs - -set CurlExe=%ToolsPath%\curl.exe -set SevenZipExe=%ToolsPath%\7z.exe diff --git a/build_scripts/Windows/build_libs/build-libs.bat b/build_scripts/Windows/build_libs/build-libs.bat new file mode 100644 index 000000000..197f0f97f --- /dev/null +++ b/build_scripts/Windows/build_libs/build-libs.bat @@ -0,0 +1,46 @@ +:: Usage: +:: call build-libs.bat [auto-copy] [make tasks] + +@echo off + +setlocal + +:: Parameter +set MakeParam="DOWNLOAD_PATH=../download" +if "%~1"=="auto-copy" set MakeParam=%MakeParam% "COPY_ANSWER=y"& shift /1 + +set MakeTask= +:param_loop +if "%~1" NEQ "" ( + set MakeTask=%MakeTask% %1 + shift /1 + goto param_loop +) + +:: Initialize environment +call "%~dp0..\env.bat" +if errorlevel 1 goto error_env +call "%EnvPath%\env-msys.bat" +if errorlevel 1 goto error_env + +:: Check MSYS environment +set MSYSSH=%EnvMSYSPath%\msys\1.0\bin\sh.exe +if not exist "%MSYSSH%" echo Please install MSYS first.& exit /B 1 + +:: Initialize environment +call "%~dp0env.bat" +if errorlevel 1 goto error_env + +call "%ToolsPath%\msys-path.bat" "%CurPath%" MSYSCurPath +call "%ToolsPath%\msys-path.bat" "%BuildLibsPath%" MSYSBuildLibsPath + +if not exist "%BuildLibsPath%" mkdir "%BuildLibsPath%" + +"%MSYSSH%" --login -i -c "cd "%MSYSBuildLibsPath%" && make -f %MSYSCurPath%/makefile %MakeParam% %MakeTask%" + +exit /B %ERRORLEVEL% + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/build_libs/clean-all.bat b/build_scripts/Windows/build_libs/clean-all.bat deleted file mode 100644 index 740bf9d3e..000000000 --- a/build_scripts/Windows/build_libs/clean-all.bat +++ /dev/null @@ -1,30 +0,0 @@ -@setlocal - -@echo off - -:: Initialize environment -call "%~dp0_env.bat" - -::call :remove_dir "%DownloadPath%" -call :remove_dir "%MSYSPath%" -call :remove_dir "%TempPath%" - -call :remove_file "%ToolsPath%\7z.exe" -call :remove_file "%ToolsPath%\7z.dll" -call :remove_file "%ToolsPath%\curl.exe" - -call "%~dp0clean.bat" - -endlocal -exit /B 0 - -:remove_dir -if not exist %1 goto :EOF -del /s /f /q %1 >nul -rmdir /s /q %1 -goto :EOF - -:remove_file -if not exist %1 goto :EOF -del /q %1 >nul -goto :EOF diff --git a/build_scripts/Windows/build_libs/clean.bat b/build_scripts/Windows/build_libs/clean.bat index 171130fc2..a8c35a512 100644 --- a/build_scripts/Windows/build_libs/clean.bat +++ b/build_scripts/Windows/build_libs/clean.bat @@ -1,36 +1,3 @@ -@setlocal - @echo off -:: Initialize environment -call "%~dp0_env.bat" - -call :remove_dir "%LibsPath%" - -call :remove_file "%CurPath%bzip2" -call :remove_file "%CurPath%curl" -call :remove_file "%CurPath%ffmpeg" -call :remove_file "%CurPath%libmicrohttpd" -call :remove_file "%CurPath%libxml2" -call :remove_file "%CurPath%libxslt" -call :remove_file "%CurPath%miniupnpc" -call :remove_file "%CurPath%opencv" -call :remove_file "%CurPath%openssl" -call :remove_file "%CurPath%speex" -call :remove_file "%CurPath%speexdsp" -call :remove_file "%CurPath%sqlcipher" -call :remove_file "%CurPath%zlib" - -endlocal -exit /B 0 - -:remove_dir -if not exist %1 goto :EOF -del /s /f /q %1 >nul -rmdir /s /q %1 -goto :EOF - -:remove_file -if not exist %1 goto :EOF -del /q %1 >nul -goto :EOF +call "%~dp0build-libs.bat" clean diff --git a/build_scripts/Windows/build_libs/env.bat b/build_scripts/Windows/build_libs/env.bat new file mode 100644 index 000000000..affde05f9 --- /dev/null +++ b/build_scripts/Windows/build_libs/env.bat @@ -0,0 +1,14 @@ +set CurPath=%~dp0 + +:: Check MinGW environment +set MinGWPath= +call "%ToolsPath%\find-in-path.bat" MinGWPath gcc.exe +if "%MinGWPath%"=="" echo Please run command in the Qt Command Prompt or add the path to MinGW bin folder to PATH variable.& exit /B 1 + +:: Get gcc versions +call "%ToolsPath%\get-gcc-version.bat" GCCVersion +if "%GCCVersion%"=="" echo Cannot get gcc version.& exit /B 1 + +set BuildLibsPath=%EnvRootPath%\build-libs\gcc-%GCCVersion% + +exit /B 0 diff --git a/build_scripts/Windows/build_libs/update-msys.bat b/build_scripts/Windows/build_libs/update-msys.bat deleted file mode 100644 index 6ecc30eff..000000000 --- a/build_scripts/Windows/build_libs/update-msys.bat +++ /dev/null @@ -1,16 +0,0 @@ -@setlocal - -@echo off - -:: Initialize environment -call "%~dp0_env.bat" - -if not exist "%MSYSPath%\bin\mingw-get.exe" exit /B 0 - -echo Update MSYS -pushd "%MSYSPath%\bin" -mingw-get.exe update -mingw-get.exe upgrade -popd - -exit /B %ERRORLEVEL% diff --git a/build_scripts/Windows/env.bat b/build_scripts/Windows/env.bat new file mode 100644 index 000000000..863369162 --- /dev/null +++ b/build_scripts/Windows/env.bat @@ -0,0 +1,19 @@ +call :make_path SourcePath "%~dp0..\.." +call :make_path RootPath "%SourcePath%\.." +call :source_name SourceName "%SourcePath%" +set ToolsPath=%~dp0tools +set EnvPath=%~dp0env + +exit /B 0 + +:make_path +setlocal +set Var=%~1 +pushd %2 +set CD=%cd% +popd +endlocal & set %Var%=%CD% +goto :EOF + +:source_name +set %~1=%~nx2 diff --git a/build_scripts/Windows/env/env-msys.bat b/build_scripts/Windows/env/env-msys.bat new file mode 100644 index 000000000..9b1d98620 --- /dev/null +++ b/build_scripts/Windows/env/env-msys.bat @@ -0,0 +1,16 @@ +:: Usage: +:: call find-in-path.bat [reinstall|clean] + +:: Initialize environment +call "%~dp0env.bat" +if errorlevel 1 goto error_env + +set EnvMSYSPath=%EnvRootPath%\msys + +call "%~dp0tools\prepare-msys.bat" %1 +exit /B %ERRORLEVEL% + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/env/env.bat b/build_scripts/Windows/env/env.bat new file mode 100644 index 000000000..d9052b5f3 --- /dev/null +++ b/build_scripts/Windows/env/env.bat @@ -0,0 +1,29 @@ +:: Initialize environment +call "%~dp0..\env.bat" +if errorlevel 1 goto error_env + +set EnvRootPath=%RootPath%\%SourceName%-env +set EnvToolsPath=%EnvRootPath%\tools +set EnvTempPath=%EnvRootPath%\tmp +set EnvDownloadPath=%EnvRootPath%\download + +set EnvCurlExe=%EnvToolsPath%\curl.exe +set EnvSevenZipExe=%EnvToolsPath%\7z.exe +set EnvJomExe=%EnvToolsPath%\jom.exe +set EnvSedExe=%EnvToolsPath%\sed.exe +set EnvCutExe=%EnvToolsPath%\cut.exe +set EnvDependsExe=%EnvToolsPath%\depends.exe +set EnvMakeNSISExe=%EnvToolsPath%\NSIS\makensis.exe + +:: Create folders +if not exist "%EnvRootPath%" mkdir "%EnvRootPath%" +if not exist "%EnvToolsPath%" mkdir "%EnvToolsPath%" +if not exist "%EnvDownloadPath%" mkdir "%EnvDownloadPath%" + +call "%~dp0tools\prepare-tools.bat" +exit /B %ERRORLEVEL% + +:error_env +echo Failed to initialize environment. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/env/tools/prepare-msys.bat b/build_scripts/Windows/env/tools/prepare-msys.bat new file mode 100644 index 000000000..4799715ca --- /dev/null +++ b/build_scripts/Windows/env/tools/prepare-msys.bat @@ -0,0 +1,83 @@ +:: Usage: +:: call prepare-msys.bat [reinstall|clean] + +setlocal enabledelayedexpansion + +if "%EnvMSYSPath%"=="" exit /B 1 +if not exist "%EnvRootPath%"=="" exit /B 1 + +copy "%~dp0root\update-msys.bat" "%EnvRootPath%" >nul + +if "%~1"=="clean" ( + echo Clean MSYS + call "%ToolsPath%\remove-dir.bat" "%EnvMSYSPath%" + goto exit +) + +if exist "%EnvMSYSPath%\bin\mingw-get.exe" ( + if "%~1"=="reinstall" ( + choice /M "Found existing MSYS version. Do you want to proceed?" + if !ERRORLEVEL!==2 goto exit + ) else ( + goto exit + ) +) + +set MSYSInstall=mingw-get-0.6.2-mingw32-beta-20131004-1-bin.zip +set CMakeInstall=cmake-3.1.0-win32-x86.zip +set CMakeUnpackPath=%EnvMSYSPath%\msys\1.0 + +echo Remove previous MSYS version +call "%ToolsPath%\remove-dir.bat" "%EnvMSYSPath%" + +echo Download installation files +if not exist "%EnvDownloadPath%\%MSYSInstall%" "%EnvCurlExe%" -L -k http://sourceforge.net/projects/mingw/files/Installer/mingw-get/mingw-get-0.6.2-beta-20131004-1/%MSYSInstall%/download -o "%EnvDownloadPath%\%MSYSInstall%" +if not exist "%EnvDownloadPath%%\MSYSInstall%" echo Cannot download MSYS& goto error + +if not exist "%EnvDownloadPath%\%CMakeInstall%" "%EnvCurlExe%" -L -k http://www.cmake.org/files/v3.1/cmake-3.1.0-win32-x86.zip -o "%EnvDownloadPath%\%CMakeInstall%" +if not exist "%EnvDownloadPath%\%CMakeInstall%" echo Cannot download CMake& goto error + +echo Unpack MSYS +"%EnvSevenZipExe%" x -o"%EnvMSYSPath%" "%EnvDownloadPath%\%MSYSInstall%" + +echo Install MSYS +if not exist "%EnvMSYSPath%\var\lib\mingw-get\data\profile.xml" copy "%EnvMSYSPath%\var\lib\mingw-get\data\defaults.xml" "%EnvMSYSPath%\var\lib\mingw-get\data\profile.xml" +pushd "%EnvMSYSPath%\bin" +mingw-get.exe install mingw32-mingw-get +mingw-get.exe install msys-coreutils +mingw-get.exe install msys-base +mingw-get.exe install msys-autoconf +mingw-get.exe install msys-automake +mingw-get.exe install msys-autogen +mingw-get.exe install msys-mktemp +mingw-get.exe install msys-wget +popd + +echo Unpack CMake +"%EnvSevenZipExe%" x -o"%CMakeUnpackPath%" "%EnvDownloadPath%\%CMakeInstall%" + +echo Install CMake +set CMakeVersion= +for /D %%F in (%CMakeUnpackPath%\cmake*) do set CMakeVersion=%%~nxF +if "%CMakeVersion%"=="" echo CMake version not found.& goto :exit +echo Found CMake version %CMakeVersion% + +set FoundProfile= +for /f "tokens=3" %%F in ('find /c /i "%CMakeVersion%" "%EnvMSYSPath%\msys\1.0\etc\profile"') do set FoundProfile=%%F + +if "%FoundProfile%"=="0" ( + echo export PATH="${PATH}:/%CMakeVersion%/bin">>"%EnvMSYSPath%\msys\1.0\etc\profile" +) + +:exit +endlocal +exit /B 0 + +:error +endlocal +exit /B 1 + +:error_vars +echo Failed to initialize variables. +endlocal +exit /B 1 diff --git a/build_scripts/Windows/env/tools/prepare-tools.bat b/build_scripts/Windows/env/tools/prepare-tools.bat new file mode 100644 index 000000000..f453f35da --- /dev/null +++ b/build_scripts/Windows/env/tools/prepare-tools.bat @@ -0,0 +1,139 @@ +setlocal + +if "%EnvRootPath%"=="" exit /B 1 + +set SevenZipUrl=http://7-zip.org/a/7z1602.msi +set SevenZipInstall=7z1602.msi +set CurlUrl=https://bintray.com/artifact/download/vszakats/generic/curl-7.50.1-win32-mingw.7z +set CurlInstall=curl-7.50.1-win32-mingw.7z +set JomUrl=http://download.qt.io/official_releases/jom/jom.zip +set JomInstall=jom.zip +set DependsUrl=http://www.dependencywalker.com/depends22_x86.zip +set DependsInstall=depends22_x86.zip +set UnixToolsUrl=http://unxutils.sourceforge.net/UnxUpdates.zip +set UnixToolsInstall=UnxUpdates.zip +set NSISUrl=http://prdownloads.sourceforge.net/nsis/nsis-3.0-setup.exe?download +set NSISInstall=nsis-3.0-setup.exe +set NSISInstallPath=%EnvToolsPath%\NSIS + +if not exist "%EnvToolsPath%\7z.exe" ( + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" + mkdir "%EnvTempPath%" + + echo Download 7z installation + + if not exist "%EnvDownloadPath%\%SevenZipInstall%" call "%ToolsPath%\winhttpjs.bat" %SevenZipUrl% -saveTo "%EnvDownloadPath%\%SevenZipInstall%" + if not exist "%EnvDownloadPath%\%SevenZipInstall%" echo Cannot download 7z installation& goto error + + echo Unpack 7z + msiexec /a "%EnvDownloadPath%\%SevenZipInstall%" /qb TARGETDIR="%EnvTempPath%" + copy "%EnvTempPath%\Files\7-Zip\7z.dll" "%EnvToolsPath%" + copy "%EnvTempPath%\Files\7-Zip\7z.exe" "%EnvToolsPath%" + + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" +) + +if not exist "%EnvToolsPath%\curl.exe" ( + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" + mkdir "%EnvTempPath%" + + echo Download Curl installation + + if not exist "%EnvDownloadPath%\%CurlInstall%" call "%ToolsPath%\winhttpjs.bat" %CurlUrl% -saveTo "%EnvDownloadPath%\%CurlInstall%" + if not exist "%EnvDownloadPath%\%CurlInstall%" echo Cannot download Curl installation& goto error + + echo Unpack Curl + "%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%CurlInstall%" + copy "%EnvTempPath%\curl-7.50.1-win32-mingw\bin\curl.exe" "%EnvToolsPath%" + + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" +) + +if not exist "%EnvToolsPath%\jom.exe" ( + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" + mkdir "%EnvTempPath%" + + echo Download jom installation + + if not exist "%EnvDownloadPath%\%JomInstall%" call "%ToolsPath%\winhttpjs.bat" %JomUrl% -saveTo "%EnvDownloadPath%\%JomInstall%" + if not exist "%EnvDownloadPath%\%JomInstall%" echo Cannot download jom installation& goto error + + echo Unpack jom + "%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%JomInstall%" + copy "%EnvTempPath%\jom.exe" "%EnvToolsPath%" + + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" +) + +if not exist "%EnvToolsPath%\depends.exe" ( + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" + mkdir "%EnvTempPath%" + + echo Download Dependency Walker installation + + if not exist "%EnvDownloadPath%\%DependsInstall%" call "%ToolsPath%\winhttpjs.bat" %DependsUrl% -saveTo "%EnvDownloadPath%\%DependsInstall%" + if not exist "%EnvDownloadPath%\%DependsInstall%" echo Cannot download Dependendy Walker installation& goto error + + echo Unpack Dependency Walker + "%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%DependsInstall%" + copy "%EnvTempPath%\*" "%EnvToolsPath%" + + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" +) + +if not exist "%EnvToolsPath%\cut.exe" ( + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" + mkdir "%EnvTempPath%" + + echo Download Unix Tools installation + + if not exist "%EnvDownloadPath%\%UnixToolsInstall%" call "%ToolsPath%\winhttpjs.bat" %UnixToolsUrl% -saveTo "%EnvDownloadPath%\%UnixToolsInstall%" + if not exist "%EnvDownloadPath%\%UnixToolsInstall%" echo Cannot download unix Tools installation& goto error + + echo Unpack Unix Tools + "%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%UnixToolsInstall%" + copy "%EnvTempPath%\cut.exe" "%EnvToolsPath%" + + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" +) + +if not exist "%EnvToolsPath%\sed.exe" ( + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" + mkdir "%EnvTempPath%" + + echo Download Unix Tools installation + + if not exist "%EnvDownloadPath%\%UnixToolsInstall%" call "%ToolsPath%\winhttpjs.bat" %UnixToolsUrl% -saveTo "%EnvDownloadPath%\%UnixToolsInstall%" + if not exist "%EnvDownloadPath%\%UnixToolsInstall%" echo Cannot download Unix Tools installation& goto error + + echo Unpack Unix Tools + "%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%UnixToolsInstall%" + copy "%EnvTempPath%\sed.exe" "%EnvToolsPath%" + + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" +) + +if not exist "%EnvToolsPath%\NSIS\nsis.exe" ( + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" + mkdir "%EnvTempPath%" + + echo Download NSIS installation + + if not exist "%EnvDownloadPath%\%NSISInstall%" "%EnvCurlExe%" -L -k %NSISUrl% -o "%EnvDownloadPath%\%NSISInstall%" + if not exist "%EnvDownloadPath%\%NSISInstall%" echo Cannot download NSIS installation& goto error + + echo Unpack NSIS + "%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%NSISInstall%" + if not exist "%NSISInstallPath%" mkdir "%NSISInstallPath%" + xcopy /s "%EnvTempPath%" "%NSISInstallPath%" + + call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%" +) + +:exit +endlocal +exit /B 0 + +:error +endlocal +exit /B 1 diff --git a/build_scripts/Windows/env/tools/root/update-msys.bat b/build_scripts/Windows/env/tools/root/update-msys.bat new file mode 100644 index 000000000..b0a109d01 --- /dev/null +++ b/build_scripts/Windows/env/tools/root/update-msys.bat @@ -0,0 +1,15 @@ +@echo off + +setlocal + +set MSYSPath=%~dp0msys + +if not exist "%MSYSPath%\bin\mingw-get.exe" echo MSYS is not installed& exit /B 0 + +echo Update MSYS +pushd "%MSYSPath%\bin" +mingw-get.exe update +mingw-get.exe upgrade +popd + +exit /B %ERRORLEVEL% diff --git a/build_scripts/Windows/HeaderImage.bmp b/build_scripts/Windows/installer/HeaderImage.bmp similarity index 100% rename from build_scripts/Windows/HeaderImage.bmp rename to build_scripts/Windows/installer/HeaderImage.bmp diff --git a/build_scripts/Windows/HeaderImageEmpty.bmp b/build_scripts/Windows/installer/HeaderImageEmpty.bmp similarity index 100% rename from build_scripts/Windows/HeaderImageEmpty.bmp rename to build_scripts/Windows/installer/HeaderImageEmpty.bmp diff --git a/build_scripts/Windows/lang/ca_ES.nsh b/build_scripts/Windows/installer/lang/ca_ES.nsh similarity index 100% rename from build_scripts/Windows/lang/ca_ES.nsh rename to build_scripts/Windows/installer/lang/ca_ES.nsh diff --git a/build_scripts/Windows/lang/de.nsh b/build_scripts/Windows/installer/lang/de.nsh similarity index 100% rename from build_scripts/Windows/lang/de.nsh rename to build_scripts/Windows/installer/lang/de.nsh diff --git a/build_scripts/Windows/lang/en.nsh b/build_scripts/Windows/installer/lang/en.nsh similarity index 100% rename from build_scripts/Windows/lang/en.nsh rename to build_scripts/Windows/installer/lang/en.nsh diff --git a/build_scripts/Windows/lang/es.nsh b/build_scripts/Windows/installer/lang/es.nsh similarity index 100% rename from build_scripts/Windows/lang/es.nsh rename to build_scripts/Windows/installer/lang/es.nsh diff --git a/build_scripts/Windows/lang/fr.nsh b/build_scripts/Windows/installer/lang/fr.nsh similarity index 100% rename from build_scripts/Windows/lang/fr.nsh rename to build_scripts/Windows/installer/lang/fr.nsh diff --git a/build_scripts/Windows/lang/pl.nsh b/build_scripts/Windows/installer/lang/pl.nsh similarity index 100% rename from build_scripts/Windows/lang/pl.nsh rename to build_scripts/Windows/installer/lang/pl.nsh diff --git a/build_scripts/Windows/lang/ru.nsh b/build_scripts/Windows/installer/lang/ru.nsh similarity index 100% rename from build_scripts/Windows/lang/ru.nsh rename to build_scripts/Windows/installer/lang/ru.nsh diff --git a/build_scripts/Windows/lang/tr.nsh b/build_scripts/Windows/installer/lang/tr.nsh similarity index 100% rename from build_scripts/Windows/lang/tr.nsh rename to build_scripts/Windows/installer/lang/tr.nsh diff --git a/build_scripts/Windows/lang/ts/ca_ES.ts b/build_scripts/Windows/installer/lang/ts/ca_ES.ts similarity index 100% rename from build_scripts/Windows/lang/ts/ca_ES.ts rename to build_scripts/Windows/installer/lang/ts/ca_ES.ts diff --git a/build_scripts/Windows/lang/ts/convert_from_ts.bat b/build_scripts/Windows/installer/lang/ts/convert_from_ts.bat similarity index 100% rename from build_scripts/Windows/lang/ts/convert_from_ts.bat rename to build_scripts/Windows/installer/lang/ts/convert_from_ts.bat diff --git a/build_scripts/Windows/lang/ts/convert_from_ts.xsl b/build_scripts/Windows/installer/lang/ts/convert_from_ts.xsl similarity index 100% rename from build_scripts/Windows/lang/ts/convert_from_ts.xsl rename to build_scripts/Windows/installer/lang/ts/convert_from_ts.xsl diff --git a/build_scripts/Windows/lang/ts/convert_to_ts.bat b/build_scripts/Windows/installer/lang/ts/convert_to_ts.bat similarity index 100% rename from build_scripts/Windows/lang/ts/convert_to_ts.bat rename to build_scripts/Windows/installer/lang/ts/convert_to_ts.bat diff --git a/build_scripts/Windows/lang/ts/de.ts b/build_scripts/Windows/installer/lang/ts/de.ts similarity index 100% rename from build_scripts/Windows/lang/ts/de.ts rename to build_scripts/Windows/installer/lang/ts/de.ts diff --git a/build_scripts/Windows/lang/ts/en.ts b/build_scripts/Windows/installer/lang/ts/en.ts similarity index 100% rename from build_scripts/Windows/lang/ts/en.ts rename to build_scripts/Windows/installer/lang/ts/en.ts diff --git a/build_scripts/Windows/lang/ts/es.ts b/build_scripts/Windows/installer/lang/ts/es.ts similarity index 100% rename from build_scripts/Windows/lang/ts/es.ts rename to build_scripts/Windows/installer/lang/ts/es.ts diff --git a/build_scripts/Windows/lang/ts/fr.ts b/build_scripts/Windows/installer/lang/ts/fr.ts similarity index 100% rename from build_scripts/Windows/lang/ts/fr.ts rename to build_scripts/Windows/installer/lang/ts/fr.ts diff --git a/build_scripts/Windows/lang/ts/pl.ts b/build_scripts/Windows/installer/lang/ts/pl.ts similarity index 100% rename from build_scripts/Windows/lang/ts/pl.ts rename to build_scripts/Windows/installer/lang/ts/pl.ts diff --git a/build_scripts/Windows/lang/ts/ru.ts b/build_scripts/Windows/installer/lang/ts/ru.ts similarity index 100% rename from build_scripts/Windows/lang/ts/ru.ts rename to build_scripts/Windows/installer/lang/ts/ru.ts diff --git a/build_scripts/Windows/lang/ts/tr.ts b/build_scripts/Windows/installer/lang/ts/tr.ts similarity index 100% rename from build_scripts/Windows/lang/ts/tr.ts rename to build_scripts/Windows/installer/lang/ts/tr.ts diff --git a/build_scripts/Windows/lang/ts/xsltproc.exe b/build_scripts/Windows/installer/lang/ts/xsltproc.exe similarity index 100% rename from build_scripts/Windows/lang/ts/xsltproc.exe rename to build_scripts/Windows/installer/lang/ts/xsltproc.exe diff --git a/build_scripts/Windows/lang/ts/zh_CN.ts b/build_scripts/Windows/installer/lang/ts/zh_CN.ts similarity index 100% rename from build_scripts/Windows/lang/ts/zh_CN.ts rename to build_scripts/Windows/installer/lang/ts/zh_CN.ts diff --git a/build_scripts/Windows/lang/zh_CN.nsh b/build_scripts/Windows/installer/lang/zh_CN.nsh similarity index 100% rename from build_scripts/Windows/lang/zh_CN.nsh rename to build_scripts/Windows/installer/lang/zh_CN.nsh diff --git a/build_scripts/Windows/retroshare-Qt4.nsi b/build_scripts/Windows/installer/retroshare-Qt4.nsi similarity index 94% rename from build_scripts/Windows/retroshare-Qt4.nsi rename to build_scripts/Windows/installer/retroshare-Qt4.nsi index 139113a02..517c0745a 100644 --- a/build_scripts/Windows/retroshare-Qt4.nsi +++ b/build_scripts/Windows/installer/retroshare-Qt4.nsi @@ -3,7 +3,6 @@ # Needed defines ;!define BUILDADD "" -;!define SOURCEDIR "" ;!define RELEASEDIR "" ;!define QTDIR "" ;!define MINGWDIR "" @@ -16,10 +15,6 @@ !error "BUILDADD is not defined" !endif -!ifndef SOURCEDIR -!error "SOURCEDIR is not defined" -!endif - !ifndef RELEASEDIR !error "RELEASEDIR is not defined" !endif @@ -38,12 +33,22 @@ !define OUTDIR_ "" !endif +!ifndef INSTALLERADD +!define INSTALLERADD "" +!endif + +# Source directory +!define SOURCEDIR "..\..\.." + # Get version from executable !GetDllVersion "${RELEASEDIR}\retroshare-gui\src\release\RetroShare06.exe" VERSION_ - !define VERSION ${VERSION_1}.${VERSION_2}.${VERSION_3}${BUILDADD} ;!define REVISION ${VERSION_4} +# Get version of Qt +!GetDllVersion "${QTDIR}\bin\QtCore4.dll" QTVERSION_ +!define QTVERSION ${QTVERSION_1}.${QTVERSION_2}.${QTVERSION_3} + # Check version !ifndef REVISION !error "REVISION is not defined" @@ -71,7 +76,7 @@ # Main Install settings Name "${APPNAMEANDVERSION}" InstallDirRegKey HKLM "Software\${APPNAME}" "" -OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-setup.exe" +OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-Qt${QTVERSION}${INSTALLERADD}-setup.exe" BrandingText "${APPNAMEANDVERSION}" RequestExecutionlevel highest # Use compression @@ -92,7 +97,7 @@ Var StyleSheetDir # Interface Settings !define MUI_ABORTWARNING !define MUI_HEADERIMAGE -!define MUI_HEADERIMAGE_BITMAP "${SOURCEDIR}\build_scripts\Windows\HeaderImage.bmp" +!define MUI_HEADERIMAGE_BITMAP "${SOURCEDIR}\build_scripts\Windows\installer\HeaderImage.bmp" ;!define MUI_WELCOMEFINISHPAGE_BITMAP "...bmp" # MUI defines diff --git a/build_scripts/Windows/retroshare-Qt5.nsi b/build_scripts/Windows/installer/retroshare-Qt5.nsi similarity index 95% rename from build_scripts/Windows/retroshare-Qt5.nsi rename to build_scripts/Windows/installer/retroshare-Qt5.nsi index 918ce53b1..916710a5e 100644 --- a/build_scripts/Windows/retroshare-Qt5.nsi +++ b/build_scripts/Windows/installer/retroshare-Qt5.nsi @@ -3,7 +3,6 @@ # Needed defines ;!define BUILDADD "" -;!define SOURCEDIR "" ;!define RELEASEDIR "" ;!define QTDIR "" ;!define MINGWDIR "" @@ -16,10 +15,6 @@ !error "BUILDADD is not defined" !endif -!ifndef SOURCEDIR -!error "SOURCEDIR is not defined" -!endif - !ifndef RELEASEDIR !error "RELEASEDIR is not defined" !endif @@ -38,12 +33,22 @@ !define OUTDIR_ "" !endif +!ifndef INSTALLERADD +!define INSTALLERADD "" +!endif + +# Source directory +!define SOURCEDIR "..\..\.." + # Get version from executable !GetDllVersion "${RELEASEDIR}\retroshare-gui\src\release\RetroShare06.exe" VERSION_ - !define VERSION ${VERSION_1}.${VERSION_2}.${VERSION_3}${BUILDADD} ;!define REVISION ${VERSION_4} +# Get version of Qt +!GetDllVersion "${QTDIR}\bin\Qt5Core.dll" QTVERSION_ +!define QTVERSION ${QTVERSION_1}.${QTVERSION_2}.${QTVERSION_3} + # Check version !ifndef REVISION !error "REVISION is not defined" @@ -71,7 +76,7 @@ # Main Install settings Name "${APPNAMEANDVERSION}" InstallDirRegKey HKLM "Software\${APPNAME}" "" -OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-setup.exe" +OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-Qt${QTVERSION}${INSTALLERADD}-setup.exe" BrandingText "${APPNAMEANDVERSION}" RequestExecutionlevel highest # Use compression @@ -92,7 +97,7 @@ Var StyleSheetDir # Interface Settings !define MUI_ABORTWARNING !define MUI_HEADERIMAGE -!define MUI_HEADERIMAGE_BITMAP "${SOURCEDIR}\build_scripts\Windows\HeaderImage.bmp" +!define MUI_HEADERIMAGE_BITMAP "${SOURCEDIR}\build_scripts\Windows\installer\HeaderImage.bmp" ;!define MUI_WELCOMEFINISHPAGE_BITMAP "...bmp" # MUI defines @@ -216,9 +221,7 @@ Section $(Section_Main) Section_Main File /r "${QTDIR}\plugins\imageformats\qgif.dll" File /r "${QTDIR}\plugins\imageformats\qicns.dll" File /r "${QTDIR}\plugins\imageformats\qico.dll" - File /r "${QTDIR}\plugins\imageformats\qjp2.dll" File /r "${QTDIR}\plugins\imageformats\qjpeg.dll" - File /r "${QTDIR}\plugins\imageformats\qmng.dll" File /r "${QTDIR}\plugins\imageformats\qsvg.dll" File /r "${QTDIR}\plugins\imageformats\qtga.dll" File /r "${QTDIR}\plugins\imageformats\qtiff.dll" diff --git a/build_scripts/Windows/make_installer.bat b/build_scripts/Windows/make_installer.bat deleted file mode 100644 index 556680da8..000000000 --- a/build_scripts/Windows/make_installer.bat +++ /dev/null @@ -1,46 +0,0 @@ -@echo off - -setlocal - -:: Modify variable when makensis.exe doesn't exist in PATH -set NSIS_EXE=makensis.exe - -:: Needed environment variables -set SourceDir=%~dp0..\.. -::set ReleaseDir= -::set QtDir= -::set MinGWDir= - -:: Optional environment variables -::set OutDir= - -:: Build defines for script -set NSIS_PARAM= - -if "%SourceDir%" NEQ "" set NSIS_PARAM=%NSIS_PARAM% /DSOURCEDIR="%SourceDir%" -if "%ReleaseDir%" NEQ "" set NSIS_PARAM=%NSIS_PARAM% /DRELEASEDIR="%ReleaseDir%" -if "%QtDir%" NEQ "" set NSIS_PARAM=%NSIS_PARAM% /DQTDIR="%QtDir%" -if "%MinGWDir%" NEQ "" set NSIS_PARAM=%NSIS_PARAM% /DMINGWDIR="%MinGWDir%" -if "%OutDir%" NEQ "" set NSIS_PARAM=%NSIS_PARAM% /DOUTDIR="%OutDir%" - -:: Scan version from source -set Revision= -set BuildAdd= -call "%~dp0GetRsVersion.bat" RS_REVISION_STRING Revision -if errorlevel 1 goto exit -call "%~dp0GetRsVersion.bat" RS_BUILD_NUMBER_ADD BuildAdd -if errorlevel 1 goto exit - -if "%Revision%"=="" ( - echo. - echo Version not found - goto exit -) - -set NSIS_PARAM=%NSIS_PARAM% /DREVISION=%Revision% /DBUILDADD=%BuildAdd% - -:: Create installer -"%NSIS_EXE%" %NSIS_PARAM% "%~dp0retroshare.nsi" - -:exit -endlocal diff --git a/build_scripts/Windows/tools/depends.bat b/build_scripts/Windows/tools/depends.bat new file mode 100644 index 000000000..3f1bb3d08 --- /dev/null +++ b/build_scripts/Windows/tools/depends.bat @@ -0,0 +1,37 @@ +:: Usage: +:: call depends.bat [list^|missing] file + +setlocal + +if "%2"=="" ( + echo Usage: %~nx0 [list^|missing] File + goto :exit +) + +if not exist "%EnvDependsExe%" echo depends.exe not found in %EnvToolsPath%.& goto exit +if not exist "%EnvCutExe%" echo cut.exe not found in %EnvToolsPath%.& goto exit + +start /wait "" "%EnvDependsExe%" /c /oc:"%~dp0depends.tmp" %2 +if "%1"=="missing" ( + "%EnvCutExe%" --delimiter=, --fields=1,2 "%~dp0depends.tmp" >"%~dp0depends1.tmp" + for /F "tokens=1,2 delims=," %%A in (%~sdp0depends1.tmp) do ( + if "%%A"=="?" ( + echo %%~B + ) + ) +) + +if "%1"=="list" ( + "%EnvCutExe%" --delimiter=, --fields=2 "%~dp0depends.tmp" >"%~dp0depends1.tmp" + for /F "tokens=1 delims=," %%A in (%~sdp0depends1.tmp) do ( + if "%%A" NEQ "Module" ( + echo %%~A + ) + ) +) + +if exist "%~dp0depends.tmp" del /Q "%~dp0depends.tmp" +if exist "%~dp0depends1.tmp" del /Q "%~dp0depends1.tmp" + +:exit +endlocal diff --git a/build_scripts/Windows/tools/find-in-path.bat b/build_scripts/Windows/tools/find-in-path.bat new file mode 100644 index 000000000..0ee7407bf --- /dev/null +++ b/build_scripts/Windows/tools/find-in-path.bat @@ -0,0 +1,26 @@ +:: Usage: +:: call find-in-path.bat variable file + +setlocal + +set Var=%~1 +set File=%~2 + +if "%File%"=="" ( + echo. + echo Parameter error. + exit /B 1 +) + +set FoundPath= + +SET PathTemp="%Path:;=";"%" +FOR %%P IN (%PathTemp%) DO ( + IF EXIST "%%~P.\%File%" ( + set FoundPath=%%~P + goto :found + ) +) + +:found +endlocal & set %Var%=%FoundPath% diff --git a/build_scripts/Windows/tools/get-gcc-version.bat b/build_scripts/Windows/tools/get-gcc-version.bat new file mode 100644 index 000000000..c9998376b --- /dev/null +++ b/build_scripts/Windows/tools/get-gcc-version.bat @@ -0,0 +1,42 @@ +:: Usage: +:: call get-gcc-version.bat variable + +setlocal + +set Var=%~1 +if "%Var%"=="" ( + echo. + echo Parameter error. + exit /B 1 +) + +set GCCVersion= + +call "%~dp0find-in-path.bat" GCCPath gcc.exe +if "%GCCPath%"=="" ( + echo. + echo Cannot find gcc.exe in PATH. + goto exit +) + +gcc --version >"%~dp0gccversion.tmp" +for /F "tokens=1*" %%A in (%~sdp0gccversion.tmp) do ( + if "%%A"=="gcc" ( + call :find_version %%B + goto exit + ) +) + +:exit +if exist "%~dp0gccversion.tmp" del /Q "%~dp0gccversion.tmp" + +endlocal & set %Var%=%GCCVersion% +goto :EOF + +:find_version +:loop +if "%2" NEQ "" ( + shift + goto loop +) +set GCCVersion=%1 diff --git a/build_scripts/Windows/tools/get-git-ref.bat b/build_scripts/Windows/tools/get-git-ref.bat new file mode 100644 index 000000000..f8f6efdeb --- /dev/null +++ b/build_scripts/Windows/tools/get-git-ref.bat @@ -0,0 +1,39 @@ +REM Usage: +REM call get-git-ref.bat Variable [Branch] + +setlocal + +set Variable=%~1 +if "%Variable%"=="" ( + echo. + echo Parameter error + exit /B 1 +) + +set Ref= + +:: Check git executable +set GitPath= +call "%~dp0find-in-path.bat" GitPath git.exe +if "%GitPath%"=="" ( + echo. + echo Git executable not found in PATH. + goto exit +) + +set GitParameter= +set Branch=%~2 +if "%Branch%"=="" ( + set Branch=HEAD + set GitParameter=--head +) + +for /F "tokens=1*" %%A in ('git show-ref %GitParameter% %Branch%') do ( + if "%%B"=="%Branch%" ( + set Ref=%%A + ) +) + +:exit +endlocal & set %Variable%=%Ref% +exit /B 0 diff --git a/build_scripts/Windows/tools/get-qt-version.bat b/build_scripts/Windows/tools/get-qt-version.bat new file mode 100644 index 000000000..c49939970 --- /dev/null +++ b/build_scripts/Windows/tools/get-qt-version.bat @@ -0,0 +1,34 @@ +:: Usage: +:: call get-qt-version.bat variable + +setlocal + +set Var=%~1 +if "%Var%"=="" ( + echo. + echo Parameter error. + exit /B 1 +) + +set QtVersion= + +call "%~dp0find-in-path.bat" QMakePath qmake.exe +if "%QMakePath%"=="" ( + echo. + echo Cannot find qmake.exe in PATH. + goto exit +) + +qmake.exe -version >"%~dp0qtversion.tmp" +for /F "tokens=1,2,3,4" %%A in (%~sdp0qtversion.tmp) do ( + if "%%A"=="Using" ( + set QtVersion=%%D + goto exit + ) +) + +:exit +if exist "%~dp0qtversion.tmp" del /Q "%~dp0qtversion.tmp" + +endlocal & set %Var%=%QtVersion% +exit /B 0 \ No newline at end of file diff --git a/build_scripts/Windows/GetRsVersion.bat b/build_scripts/Windows/tools/get-rs-version.bat similarity index 54% rename from build_scripts/Windows/GetRsVersion.bat rename to build_scripts/Windows/tools/get-rs-version.bat index 153b5969d..c99ded405 100644 --- a/build_scripts/Windows/GetRsVersion.bat +++ b/build_scripts/Windows/tools/get-rs-version.bat @@ -1,33 +1,23 @@ -@:: Usage: -@:: call GetRsVersion.bat Define Variable +:: Usage: +:: call get-rs-version.bat Define Variable -@setlocal -@echo off +setlocal set Define=%~1 -if "%Define%"=="" ( - echo. - echo Parameter error - endlocal - exit /B1 -) - set Variable=%~2 if "%Variable%"=="" ( echo. - echo Parameter error - endlocal - exit /B1 + echo Parameter error. + exit /B 1 ) set Result= -set VersionFile="%~dp0..\..\libretroshare\src\retroshare\rsversion.h" +set VersionFile="%~dp0..\..\..\libretroshare\src\retroshare\rsversion.h" if not exist "%VersionFile%" ( echo. echo Version file doesn't exist. echo %VersionFile% - endlocal exit /B1 ) diff --git a/build_scripts/Windows/tools/msys-path.bat b/build_scripts/Windows/tools/msys-path.bat new file mode 100644 index 000000000..3b0c5fd69 --- /dev/null +++ b/build_scripts/Windows/tools/msys-path.bat @@ -0,0 +1,20 @@ +:: Usage: +:: call msys-path.bat path variable + +setlocal + +set WinPath=%~1 +set MSYSVar=%~2 + +if "%MSYSVar%"=="" ( + echo. + echo Parameter error. + exit /B 1 +) + +set MSYSPath=/%WinPath:~0,1%/%WinPath:~3% +set MSYSPath=%MSYSPath:\=/% + +endlocal & set %MSYSVar%=%MSYSPath% + +exit /B 0 diff --git a/build_scripts/Windows/tools/remove-dir.bat b/build_scripts/Windows/tools/remove-dir.bat new file mode 100644 index 000000000..2bcb14c51 --- /dev/null +++ b/build_scripts/Windows/tools/remove-dir.bat @@ -0,0 +1,15 @@ +:: Usage: +:: call remove-dir.bat path + +if "%~1"=="" ( + echo. + echo Parameter error. + exit /B 1 +) + +if exist %1 ( + del /s /f /q %1 >nul + rmdir /s /q %1 +) + +exit /B 0 diff --git a/build_scripts/Windows/build_libs/Tools/winhttpjs.bat b/build_scripts/Windows/tools/winhttpjs.bat similarity index 100% rename from build_scripts/Windows/build_libs/Tools/winhttpjs.bat rename to build_scripts/Windows/tools/winhttpjs.bat From 133676fdee02bf95264e63916265c5e87eb2b381 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 4 Oct 2016 21:24:53 +0200 Subject: [PATCH 140/174] simplified the code to remove old locations and fixed bug that in some situation would let a PGP key with no locations --- libretroshare/src/pqi/p3peermgr.cc | 127 ++++++----------------------- 1 file changed, 27 insertions(+), 100 deletions(-) diff --git a/libretroshare/src/pqi/p3peermgr.cc b/libretroshare/src/pqi/p3peermgr.cc index f37f80b55..e5f17d518 100644 --- a/libretroshare/src/pqi/p3peermgr.cc +++ b/libretroshare/src/pqi/p3peermgr.cc @@ -286,7 +286,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht) void p3PeerMgrIMPL::tick() { - static const time_t INTERVAL_BETWEEN_LOCATION_CLEANING = 300 ; // Remove unused locations and clean IPs every 10 minutes. + static const time_t INTERVAL_BETWEEN_LOCATION_CLEANING = 100 ; // Remove unused locations and clean IPs every 10 minutes. static time_t last_friends_check = time(NULL) ; // first cleaning after 1 hour. @@ -2841,121 +2841,48 @@ bool p3PeerMgrIMPL::removeBannedIps() bool p3PeerMgrIMPL::removeUnusedLocations() { std::list toRemove; - - std::map hasRecentLocation; std::map mostRecentTime; - std::map mostRecentLocation; - - // init maps - { - std::list pgpList; - - if(!rsPeers->getGPGAcceptedList(pgpList)) - return false; - - std::list::iterator it; - for(it = pgpList.begin(); it != pgpList.end(); ++it) - { - hasRecentLocation[*it] = false; - mostRecentTime[*it] = (time_t)0; - } - } const time_t now = time(NULL); - RsPgpId pgpID; - + + std::list pgpList; + + if(!rsPeers->getGPGAcceptedList(pgpList)) + return false; { RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ + // First put a sensible number in all PGP ids + + for(std::list::const_iterator it = pgpList.begin(); it != pgpList.end(); ++it) + mostRecentTime[*it] = (time_t)0; + #ifdef PEER_DEBUG std::cerr << "p3PeerMgr::removeUnusedLocations()" << std::endl; #endif + // Then compute the most recently used location for all PGP ids - std::map::iterator it; - for(it = mFriendList.begin(); it != mFriendList.end(); ++it) - { - pgpID = it->second.gpg_id; + for( std::map::iterator it = mFriendList.begin(); it != mFriendList.end(); ++it) + { + time_t& bst(mostRecentTime[it->second.gpg_id]) ; + bst = std::max(bst,it->second.lastcontact) ; + } - // store some references to speed up accessing - RsPeerId &idRef = mostRecentLocation[pgpID]; - bool &recentRef = hasRecentLocation[pgpID]; + // And remove all locations that are too old and also older than the most recent location. Doing this we're sure to always keep at least one location per PGP id. - if (now > it->second.lastcontact + RS_PEER_OFFLINE_DELETE) - { - // location is too old - if(recentRef) - { - // there is already one location that won't get removed - // -> we can safely remove this one - toRemove.push_back(it->first); + for( std::map::iterator it = mFriendList.begin(); it != mFriendList.end(); ++it) + { + if (now > it->second.lastcontact + RS_PEER_OFFLINE_DELETE && it->second.lastcontact < mostRecentTime[it->second.gpg_id]) + toRemove.push_back(it->first); #ifdef PEER_DEBUG - std::cerr << "p3PeerMgr::removeUnusedLocations() removing Old SSL Id: " << it->first << std::endl; + std::cerr << "Location " << it->first << " PGP id " << it->second.gpg_id << " last contact " << it->second.lastcontact << " remove: " << (now > it->second.lastcontact + RS_PEER_OFFLINE_DELETE) << " most recent: " << mostRecentTime[it->second.gpg_id] + << ". Final result remove: " << (it->second.lastcontact < mostRecentTime[it->second.gpg_id] && now > it->second.lastcontact + RS_PEER_OFFLINE_DELETE )<< std::endl; #endif - } - else - { - // we need to take care that the most recent location it not removed - time_t &timeRef = mostRecentTime[pgpID]; - - if(timeRef > it->second.lastcontact) - { - // this (it) location is longer offline compared to mostRecentLocation - // -> we can remove this one - toRemove.push_back(it->first); -#ifdef PEER_DEBUG - std::cerr << "p3PeerMgr::removeUnusedLocations() removing Old SSL Id: " << it->first << std::endl; -#endif - } - else - { - // this (it) location is more recent compared to mostRecentLocation - // -> we can remove mostRecentLocation - if(!idRef.isNull()) - { - toRemove.push_back(idRef); -#ifdef PEER_DEBUG - std::cerr << "p3PeerMgr::removeUnusedLocations() removing Old SSL Id: " << it->first << std::endl; -#endif - } - // update maps - idRef = it->first; - timeRef = it->second.lastcontact; - } - } - } - else - { - // found a location that won't get removed - recentRef = true; - - // we can remove mostRecentLocation if it is set - if(!idRef.isNull()) - { - toRemove.push_back(idRef); -#ifdef PEER_DEBUG - std::cerr << "p3PeerMgr::removeUnusedLocations() removing Old SSL Id: " << it->first << std::endl; -#endif - } - } - -// if (isDummyFriend(it->first)) -// { -// toRemove.push_back(it->first); -// -//#ifdef PEER_DEBUG -// std::cerr << "p3PeerMgr::removeUnusedLocations() removing Dummy Id: " << it->first << std::endl; -//#endif -// -// } - - } + } } - std::list::iterator it; - for(it = toRemove.begin(); it != toRemove.end(); ++it) - { - removeFriend(*it, false); - } + for( std::list::iterator it = toRemove.begin(); it != toRemove.end(); ++it) + removeFriend(*it, false); return true; } From b08a62afc9a8e65dc67b8070a5fdd6e8d3c8eb31 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 4 Oct 2016 21:25:48 +0200 Subject: [PATCH 141/174] restored delay between location cleaning to 5 mins --- libretroshare/src/pqi/p3peermgr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/pqi/p3peermgr.cc b/libretroshare/src/pqi/p3peermgr.cc index e5f17d518..f85edb511 100644 --- a/libretroshare/src/pqi/p3peermgr.cc +++ b/libretroshare/src/pqi/p3peermgr.cc @@ -286,7 +286,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht) void p3PeerMgrIMPL::tick() { - static const time_t INTERVAL_BETWEEN_LOCATION_CLEANING = 100 ; // Remove unused locations and clean IPs every 10 minutes. + static const time_t INTERVAL_BETWEEN_LOCATION_CLEANING = 300 ; // Remove unused locations and clean IPs every 10 minutes. static time_t last_friends_check = time(NULL) ; // first cleaning after 1 hour. From e42dae8f78a3dc6c071518106513eef6af4bc7b6 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 4 Oct 2016 21:34:23 +0200 Subject: [PATCH 142/174] continue sending packet slicing probes in case the peer has restarted --- libretroshare/src/pqi/pqistreamer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 6cc051ee2..a96c93148 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -555,7 +555,7 @@ int pqistreamer::handleoutgoing_locked() // if so, we enable it for the session. This should be removed (because it's unnecessary) when all users have switched to the new version. time_t now = time(NULL) ; - if((!mAcceptsPacketSlicing) && now > mLastSentPacketSlicingProbe + PQISTREAM_PACKET_SLICING_PROBE_DELAY) + if(now > mLastSentPacketSlicingProbe + PQISTREAM_PACKET_SLICING_PROBE_DELAY) { #ifdef DEBUG_PACKET_SLICING std::cerr << "(II) Inserting packet slicing probe in traffic" << std::endl; From 20cd123e55e7d64638a7c7288db3e6c326d25336 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 4 Oct 2016 21:41:38 +0200 Subject: [PATCH 143/174] removed annoying ERROR output that is more a debug info than a real error, in p3filelists --- libretroshare/src/file_sharing/p3filelists.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index 49ae0aa0c..f8f87c701 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1125,7 +1125,9 @@ void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item) if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index)) { - P3FILELISTS_ERROR() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; +#ifdef DEBUG_P3FILELISTS + P3FILELISTS_DEBUG() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl; +#endif return; } From d367491274310bb3c7fd87d117645bcbb1eec540 Mon Sep 17 00:00:00 2001 From: thunder2 Date: Tue, 4 Oct 2016 21:45:59 +0200 Subject: [PATCH 144/174] Fixed env.bat for Windows build --- build_scripts/Windows/build/env.bat | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_scripts/Windows/build/env.bat b/build_scripts/Windows/build/env.bat index a99905161..c4758f0d4 100644 --- a/build_scripts/Windows/build/env.bat +++ b/build_scripts/Windows/build/env.bat @@ -25,7 +25,9 @@ set RsDeployPath=%DeployPath%\Qt-%QtVersion%-%RsBuildConfig% set RsPackPath=%DeployPath% set RsArchiveAdd= -if exist "%~dp0env-mod.bat" call "%~dp0env-mod.bat" +if not exist "%~dp0env-mod.bat" goto no_mod +call "%~dp0env-mod.bat" if errorlevel 1 exit /B %ERRORLEVEL% +:no_mod exit /B 0 \ No newline at end of file From 9177c8277acfd90d3109f6ee6fab9cbd6a10941f Mon Sep 17 00:00:00 2001 From: cave beat Date: Tue, 4 Oct 2016 23:14:50 +0200 Subject: [PATCH 145/174] updated MSys2 Install Guide to .md --- WindowsMSys2_InstallGuide.md | 87 ++++++++++++++++++++++++++++++++++ WindowsMSys2_InstallGuide.txt | 89 ----------------------------------- 2 files changed, 87 insertions(+), 89 deletions(-) create mode 100644 WindowsMSys2_InstallGuide.md delete mode 100644 WindowsMSys2_InstallGuide.txt diff --git a/WindowsMSys2_InstallGuide.md b/WindowsMSys2_InstallGuide.md new file mode 100644 index 000000000..c6782744d --- /dev/null +++ b/WindowsMSys2_InstallGuide.md @@ -0,0 +1,87 @@ +##Compilation on Windows + +###Qt Installation + +Install Qt via: [Qt Download](http://www.qt.io/download/) + +Use default options. +Add to the PATH environment variable + + ;C:\Qt\5.5\mingw492_32\bin;C:\Qt\Tools\mingw492_32\bin;C:\Qt\Tools\mingw492_32\opt\bin + +Depends on wich version of Qt you use. +Change build-all-mingw32make.bat with these values too if you don't use MSys2. + +###MSYS2 INSTALLATION + +Choose your MSYS2 installer here: [MSYS2](http://msys2.github.io/) + +Follow install procedure. +Don't forget to sync & Update pacman. + + pacman --needed -Sy bash pacman pacman-mirrors msys2-runtime + +Restart console + + pacman -Su + +Install all default programms + + pacman -S base-devel git mercurial cvs wget p7zip gcc perl ruby python2 + +Choose only w64-i686 if you want only compilation in 32b architecture. + + pacman -S mingw-w64-i686-toolchain mingw-w64-x86_64-toolchain + +###Install other binutils: + pacman -S mingw-w64-i686-miniupnpc mingw-w64-x86_64-miniupnpc + pacman -S mingw-w64-i686-sqlite3 mingw-w64-x86_64-sqlite3 + pacman -S mingw-w64-i686-speex mingw-w64-x86_64-speex + pacman -S mingw-w64-i686-opencv mingw-w64-x86_64-opencv + pacman -S mingw-w64-i686-ffmpeg mingw-w64-x86_64-ffmpeg + pacman -S mingw-w64-i686-libmicrohttpd mingw-w64-x86_64-libmicrohttpd + pacman -S mingw-w64-i686-libxslt mingw-w64-x86_64-libxslt + +Add MSYS2 to PATH environment variable depends your windows + + ;C:\msys64\mingw32\bin + ;C:\msys32\mingw32\bin + + +###Git Installation + +Install Git Gui or other client: [Git Scm](https://git-scm.com/download/win) + +Create a new directory named: + + C:\Development\GIT + +Right-click on it and choose: *Git Bash Here* + +Paste this text on git console: +git clone https://github.com/RetroShare/RetroShare.git + + +###Last Settings + + +In QtCreator Option Git add its path: *C:\Program Files\Git\bin* +and select "Pull" with "Rebase" + + +Open an MSys2 32|64 shell +Move to build_scripts: + + cd /c/Development/GIT/RetroShare/msys2_build_libs/ + +###Compile missing library + make + +You can now compile RS into Qt Creator + +For using, and debugging Plugins, you can use [Symlinker](http://amd989.github.io/Symlinker/) to link +the files in + + \build-RetroShare-Desktop_Qt_X_Y_Z_MinGW_32bit-Debug\plugins\PluginName\debug\*.dll +to +*%appdata%\RetroShare\extensions6* diff --git a/WindowsMSys2_InstallGuide.txt b/WindowsMSys2_InstallGuide.txt deleted file mode 100644 index ded6a1420..000000000 --- a/WindowsMSys2_InstallGuide.txt +++ /dev/null @@ -1,89 +0,0 @@ -############################# -###--- QT INSTALLATION ---### -############################# - -###Install Qt via: -http://www.qt.io/download/ - -###Use default options. -###Add to the PATH environment variable. -;C:\Qt\5.5\mingw492_32\bin;C:\Qt\Tools\mingw492_32\bin;C:\Qt\Tools\mingw492_32\opt\bin - -###Depends on wich version of Qt you use. - -###Change build-all-mingw32make.bat with these values too if you don't use MSys2. - - -############################### -###---MSYS2 INSTALLATION ---### -############################### - -###Choose your MSYS2 installer here: -http://msys2.github.io/ - -###Follow install procedure. -###Don't forget to sync & Update pacman. -pacman --needed -Sy bash pacman pacman-mirrors msys2-runtime - -###Restart console -pacman -Su - -###Install all default programms -pacman -S base-devel git mercurial cvs wget p7zip gcc perl ruby python2 - -###Choose only w64-i686 if you want only compilation in 32b architecture. -pacman -S mingw-w64-i686-toolchain mingw-w64-x86_64-toolchain - -###Install other binutils: -pacman -S mingw-w64-i686-miniupnpc mingw-w64-x86_64-miniupnpc -pacman -S mingw-w64-i686-sqlite3 mingw-w64-x86_64-sqlite3 -pacman -S mingw-w64-i686-speex mingw-w64-x86_64-speex -pacman -S mingw-w64-i686-opencv mingw-w64-x86_64-opencv -pacman -S mingw-w64-i686-ffmpeg mingw-w64-x86_64-ffmpeg -pacman -S mingw-w64-i686-libmicrohttpd mingw-w64-x86_64-libmicrohttpd -pacman -S mingw-w64-i686-libxslt mingw-w64-x86_64-libxslt - -### Add MSYS2 to PATH environment variable depends your windows -;C:\msys64\mingw32\bin -;C:\msys32\mingw32\bin - - -############################## -###--- GIT INSTALLATION ---### -############################## - -###Install Git Gui or other client: -https://git-scm.com/download/win - -###Create a new directory named: -C:\Development\GIT - -###Right-click on it and choose: -Git Bash Here - -###Paste this text on git console: -git clone https://github.com/RetroShare/RetroShare.git - - -########################### -###--- LAST SETTINGS ---### -########################### - -###In QtCreator Option Git add its path: -C:\Program Files\Git\bin -### and select "Pull" with "Rebase" - - -###Open an MSys2 32|64 shell -###Move to build_scripts: -cd /c/Development/GIT/RetroShare/msys2_build_libs/ - -###Compil missing library -make - -###You can now compile RS into Qt Creator - -###For using, and debugging Plugins, you can use SymLinker (http://amd989.github.io/Symlinker/) to link -### the files in \build-RetroShare-Desktop_Qt_X_Y_Z_MinGW_32bit-Debug\plugins\PluginName\debug\*.dll -### to %appdata%\RetroShare\extensions6 - From 891e61e16b5c8757d83c7ffd2d0708670abadabb Mon Sep 17 00:00:00 2001 From: cave beat Date: Tue, 4 Oct 2016 23:27:40 +0200 Subject: [PATCH 146/174] MacOS to .md, + README update --- MacOS_X_InstallGuide.md | 71 ++++++++++++++++++++++++++++++++++++++++ MacOS_X_InstallGuide.txt | 66 ------------------------------------- README.md | 4 +-- 3 files changed, 73 insertions(+), 68 deletions(-) create mode 100644 MacOS_X_InstallGuide.md delete mode 100644 MacOS_X_InstallGuide.txt diff --git a/MacOS_X_InstallGuide.md b/MacOS_X_InstallGuide.md new file mode 100644 index 000000000..ab20893e3 --- /dev/null +++ b/MacOS_X_InstallGuide.md @@ -0,0 +1,71 @@ +##Compilation on MacOS + +### Qt Installation + +Install Qt via: [Qt Download](http://www.qt.io/download/) + +Use default options. +Add to the PATH environment variable with this temporary solution. + + export PATH=/users/$USER/Qt/5.5/clang_64/bin:$PATH + +Depends on wich version of Qt you use. + +###MacPort Installation + +Install MacPort and XCode following this guide: [MacPort and XCode](http://guide.macports.org/#installing.xcode) + +Start XCode to get it updated and to able C compiler to create executables. + +###Install libraries + + $sudo port -v selfupdate + $sudo port install openssl + $sudo port install miniupnpc + +For VOIP Plugin: + + $sudo port install speex-devel + $sudo port install opencv + +Get Your OSX SDK if missing: [MacOSX-SDKs](https://github.com/phracker/MacOSX-SDKs) + +###Last Settings + +In QtCreator Option Git add its path: + + C:\Program Files\Git\bin + +and select "Pull" with "Rebase" + +###Compil missing libraries +####SQLCipher + + cd + git clone https://github.com/sqlcipher/sqlcipher.git + cd sqlcipher + ./configure --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC" LDFLAGS="-lcrypto" + make + sudo make install + +NOTE, might be necessary to *chmod 000 /usr/local/ssl* temporarily during *./configure* if +homebrew uses newer, non-stock ssl dependencies found there. configure might get confused. + +####libMicroHTTPD +The one with port don't have good support. + + cd + wget http://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz + tar zxvf libmicrohttpd-0.9.46.tar.gz + cd libmicrohttpd-0.9.46 + bash ./configure + sudo make install + +You can now compile RS into Qt Creator or with terminal + + cd + git clone https://github.com/RetroShare/RetroShare.git retroshare + cd retroshare + qmake; make + +You can find compiled application on *./retroshare/retroshare-gui/src/RetroShare06.app* diff --git a/MacOS_X_InstallGuide.txt b/MacOS_X_InstallGuide.txt deleted file mode 100644 index d56963611..000000000 --- a/MacOS_X_InstallGuide.txt +++ /dev/null @@ -1,66 +0,0 @@ -############################# -###--- QT INSTALLATION ---### -############################# - -###Install Qt via: -http://www.qt.io/download/ - -###Use default options. -###Add to the PATH environment variable with this temporary solution. -export PATH=/users/$USER/Qt/5.5/clang_64/bin:$PATH -###Depends on wich version of Qt you use. - -################################## -###--- MACPORT INSTALLATION ---### -################################## - -###Install MacPort and XCode following this guide: -http://guide.macports.org/#installing.xcode - -###Start XCode to get it updated and to able C compiler to create executables. - -###Install libraries -$sudo port -v selfupdate -$sudo port install openssl -$sudo port install miniupnpc -###For VOIP Plugin: -$sudo port install speex-devel -$sudo port install opencv - -###Get Your OSX SDK if missing: -https://github.com/phracker/MacOSX-SDKs - -########################### -###--- LAST SETTINGS ---### -########################### - -###In QtCreator Option Git add its path: -C:\Program Files\Git\bin -### and select "Pull" with "Rebase" - -###Compil missing libraries -###SQLCipher -cd -git clone https://github.com/sqlcipher/sqlcipher.git -cd sqlcipher -./configure --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC" LDFLAGS="-lcrypto" -make -sudo make install -#NOTE, might be necessary to chmod 000 /usr/local/ssl temporarily during ./configure if -#homebrew uses newer, non-stock ssl dependencies found there. configure might get confused. - -### libMicroHTTPD: The one with port don't have good support. -cd -wget http://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz -tar zxvf libmicrohttpd-0.9.46.tar.gz -cd libmicrohttpd-0.9.46 -bash ./configure -sudo make install - -###You can now compile RS into Qt Creator or with terminal -cd -git clone https://github.com/RetroShare/RetroShare.git retroshare -cd retroshare -qmake;make - -###You can find compiled application on ./retroshare/retroshare-gui/src/RetroShare06.app diff --git a/README.md b/README.md index 4394180f0..f191eb4bb 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ Build Status Compilation on Windows ---------------------------- -Follow this file : [WindowsMSys2_InstallGuide.txt](https://github.com/RetroShare/RetroShare/blob/master/WindowsMSys2_InstallGuide.txt) +Follow this file : [WindowsMSys2_InstallGuide.md](https://github.com/RetroShare/RetroShare/blob/master/WindowsMSys2_InstallGuide.txt) Compilation on MacOSX ---------------------------- -Follow this file : [MacOS_X_InstallGuide.txt](https://github.com/RetroShare/RetroShare/blob/master/MacOS_X_InstallGuide.txt) +Follow this file : [MacOS_X_InstallGuide.md](https://github.com/RetroShare/RetroShare/blob/master/MacOS_X_InstallGuide.txt) Compilation on Linux ---------------------------- From d020d8d8965ad999b52096cbf4213ce6122d0451 Mon Sep 17 00:00:00 2001 From: thunder2 Date: Wed, 5 Oct 2016 06:40:02 +0200 Subject: [PATCH 147/174] Windows build environment: - Added automatic mode for creating git-log - Fixed filename of installer --- build_scripts/Windows/build/git-log.bat | 11 +++++++++++ build_scripts/Windows/installer/retroshare-Qt4.nsi | 2 +- build_scripts/Windows/installer/retroshare-Qt5.nsi | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/build_scripts/Windows/build/git-log.bat b/build_scripts/Windows/build/git-log.bat index 49cfaef1e..8a36c7537 100644 --- a/build_scripts/Windows/build/git-log.bat +++ b/build_scripts/Windows/build/git-log.bat @@ -1,7 +1,13 @@ +:: Usage: +:: call git-log.bat [no-ask] + @echo off setlocal +set NoAsk= +if "%~1"=="no-ask" set NoAsk=1 + :: Initialize environment call "%~dp0..\env.bat" if errorlevel 1 goto error_env @@ -56,9 +62,11 @@ set RsLastRefFile=%BuildPath%\Qt-%QtVersion%-%RsBuildConfig%-LastRef.txt set RsLastRef= if exist "%RsLastRefFile%" set /P RsLastRef=<"%RsLastRefFile%" +if "%NoAsk%"=="1" goto no_ask_for_last_revision if not "%RsLastRef%"=="" echo Last Revision was %RsLastRef% set /P RsLastRefInput=Last Revision: if "%RsLastRefInput%" NEQ "" set RsLastRef=%RsLastRefInput% +:no_ask_for_last_revision :: Get current revision pushd "%SourcePath%" @@ -71,8 +79,11 @@ if "%RsRef%"=="" echo Cannot get git revision.& exit /B 1 echo. echo Creating log from %RsLastRef% echo to %RsRef% + +if "%NoAsk%"=="1" goto no_confirm choice /M "Do you want to proceed?" if %errorlevel%==2 exit /B 1 +:no_confirm if "%RsBuildConfig%" NEQ "release" ( set RsGitLog=%DeployPath%\RetroShare-%RsVersion%-Windows-Portable-%RsDate%-%RsRevision%-Qt-%QtVersion%%RsArchiveAdd%-%RsBuildConfig%.txt diff --git a/build_scripts/Windows/installer/retroshare-Qt4.nsi b/build_scripts/Windows/installer/retroshare-Qt4.nsi index 517c0745a..4efaa22de 100644 --- a/build_scripts/Windows/installer/retroshare-Qt4.nsi +++ b/build_scripts/Windows/installer/retroshare-Qt4.nsi @@ -76,7 +76,7 @@ # Main Install settings Name "${APPNAMEANDVERSION}" InstallDirRegKey HKLM "Software\${APPNAME}" "" -OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-Qt${QTVERSION}${INSTALLERADD}-setup.exe" +OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-Qt-${QTVERSION}${INSTALLERADD}-setup.exe" BrandingText "${APPNAMEANDVERSION}" RequestExecutionlevel highest # Use compression diff --git a/build_scripts/Windows/installer/retroshare-Qt5.nsi b/build_scripts/Windows/installer/retroshare-Qt5.nsi index 916710a5e..cda4aff3a 100644 --- a/build_scripts/Windows/installer/retroshare-Qt5.nsi +++ b/build_scripts/Windows/installer/retroshare-Qt5.nsi @@ -76,7 +76,7 @@ # Main Install settings Name "${APPNAMEANDVERSION}" InstallDirRegKey HKLM "Software\${APPNAME}" "" -OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-Qt${QTVERSION}${INSTALLERADD}-setup.exe" +OutFile "${OUTDIR_}RetroShare-${VERSION}-${Date}-${REVISION}-Qt-${QTVERSION}${INSTALLERADD}-setup.exe" BrandingText "${APPNAMEANDVERSION}" RequestExecutionlevel highest # Use compression From 854eebb2cb4f0411ed6377c5e33cb39bbcb1f385 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Wed, 5 Oct 2016 10:59:51 +0200 Subject: [PATCH 148/174] only update subfiles and subdirs list during dir watching sweep when the TS of the dir has changed. Greatly improve cost of regular directory watching process --- .../src/file_sharing/directory_updater.cc | 98 +++++++++++-------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 0a656417a..41c4a941e 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -137,59 +137,71 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p librs::util::FolderIterator dirIt(cumulated_path); - // collect subdirs and subfiles - - std::map subfiles ; - std::map subdirs ; - - for(;dirIt.isValid();dirIt.next()) + time_t dir_local_mod_time ; + if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time)) { - switch(dirIt.file_type()) - { - case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; - subfiles[dirIt.file_name()].size = dirIt.file_size(); -#ifdef DEBUG_LOCAL_DIR_UPDATER - std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; -#endif - break; - - case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime(); -#ifdef DEBUG_LOCAL_DIR_UPDATER - std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; -#endif - break; - default: - std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; - } + std::cerr << "(EE) Cannot get local mod time for dir index " << indx << std::endl; + return; } - // update folder modificatoin time, which is the only way to detect e.g. removed or renamed files. - mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ; - - // update file and dir lists for current directory. - - mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; - - std::map new_files ; - mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; - - // now go through list of subfiles and request the hash to hashcache - - for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit) + if(dirIt.dir_modtime() != dir_local_mod_time) { - // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. + // collect subdirs and subfiles - RsFileHash hash ; + std::map subfiles ; + std::map subdirs ; - if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) - mSharedDirectories->updateHash(*dit,hash); + for(;dirIt.isValid();dirIt.next()) + { + switch(dirIt.file_type()) + { + case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ; + subfiles[dirIt.file_name()].size = dirIt.file_size(); +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl; +#endif + break; + + case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime(); +#ifdef DEBUG_LOCAL_DIR_UPDATER + std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl; +#endif + break; + default: + std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl; + } + } + // update folder modificatoin time, which is the only way to detect e.g. removed or renamed files. + + mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ; + + // update file and dir lists for current directory. + + mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; + + std::map new_files ; + mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; + + // now go through list of subfiles and request the hash to hashcache + + for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit) + { + // ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it. + + RsFileHash hash ; + + if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash) + mSharedDirectories->updateHash(*dit,hash); + } } +#ifdef DEBUG_LOCAL_DIR_UPDATER + else + std::cerr << " directory is unchanged. Keeping existing files and subdirs list." << std::endl; +#endif // go through the list of sub-dirs and recursively update - DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; - - for(std::map::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it) + for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it) { #ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << " recursing into " << stored_dir_it.name() << std::endl; From fdfd2e92486948bbb3d60f9e0d572e8145059289 Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 24 Sep 2016 16:12:44 +0200 Subject: [PATCH 149/174] Add context menu to show hidden image on lobby. A tooltip show hidden image too. The RSImageBlockWidget is removed. --- retroshare-gui/src/gui/chat/ChatWidget.cpp | 35 +++++--- retroshare-gui/src/gui/chat/ChatWidget.ui | 26 ++---- .../src/gui/common/RSTextBrowser.cpp | 85 ++++++++++++++++++- retroshare-gui/src/gui/common/RSTextBrowser.h | 18 +++- .../gui/gxsforums/GxsForumThreadWidget.cpp | 3 +- retroshare-gui/src/util/imageutil.cpp | 19 ++--- retroshare-gui/src/util/imageutil.h | 2 - 7 files changed, 134 insertions(+), 54 deletions(-) diff --git a/retroshare-gui/src/gui/chat/ChatWidget.cpp b/retroshare-gui/src/gui/chat/ChatWidget.cpp index 725fcb11b..65b27d6e0 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.cpp +++ b/retroshare-gui/src/gui/chat/ChatWidget.cpp @@ -21,18 +21,19 @@ ****************************************************************/ #include -#include -#include -#include +#include #include #include +#include +#include #include -#include -#include -#include -#include -#include +#include #include +#include +#include +#include +#include +#include #include "ChatWidget.h" #include "ui_ChatWidget.h" @@ -157,6 +158,8 @@ ChatWidget::ChatWidget(QWidget *parent) : connect(ui->actionQuote, SIGNAL(triggered()), this, SLOT(quote())); connect(ui->actionDropPlacemark, SIGNAL(triggered()), this, SLOT(dropPlacemark())); connect(ui->actionSave_image, SIGNAL(triggered()), this, SLOT(saveImage())); + connect(ui->actionShow_Hidden_Images, SIGNAL(triggered()), ui->textBrowser, SLOT(showImages())); + ui->actionShow_Hidden_Images->setIcon(ui->textBrowser->getBlockedImage()); connect(ui->hashBox, SIGNAL(fileHashingFinished(QList)), this, SLOT(fileHashingFinished(QList))); @@ -196,9 +199,7 @@ ChatWidget::ChatWidget(QWidget *parent) : ui->actionSendAsPlainText->setChecked(Settings->getChatSendAsPlainTextByDef()); - ui->textBrowser->setImageBlockWidget(ui->imageBlockWidget); - ui->textBrowser->resetImagesStatus(Settings->getChatLoadEmbeddedImages());//Need to be called after setImageBlockWidget - ui->imageBlockWidget->setAutoHide(true); + ui->textBrowser->resetImagesStatus(Settings->getChatLoadEmbeddedImages()); ui->textBrowser->installEventFilter(this); ui->textBrowser->viewport()->installEventFilter(this); ui->chatTextEdit->installEventFilter(this); @@ -581,6 +582,12 @@ bool ChatWidget::eventFilter(QObject *obj, QEvent *event) if (!anchors.isEmpty()){ toolTipText = anchors.at(0); } + if (toolTipText.isEmpty() && !ui->textBrowser->getShowImages()){ + QString imageStr; + if (ui->textBrowser->checkImage(helpEvent->pos(), imageStr)) { + toolTipText = imageStr; + } + } } if (!toolTipText.isEmpty()){ QToolTip::showText(helpEvent->globalPos(), toolTipText); @@ -1016,9 +1023,11 @@ void ChatWidget::contextMenuTextBrowser(QPoint point) contextMnu->addAction(ui->actionQuote); contextMnu->addAction(ui->actionDropPlacemark); - QTextCursor cursor = ui->textBrowser->cursorForPosition(point); - if(ImageUtil::checkImage(cursor)) + if(ui->textBrowser->checkImage(point)) { + if (! ui->textBrowser->getShowImages()) + contextMnu->addAction(ui->actionShow_Hidden_Images); + ui->actionSave_image->setData(point); contextMnu->addAction(ui->actionSave_image); } diff --git a/retroshare-gui/src/gui/chat/ChatWidget.ui b/retroshare-gui/src/gui/chat/ChatWidget.ui index 1b9f57b71..fed47f6ce 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.ui +++ b/retroshare-gui/src/gui/chat/ChatWidget.ui @@ -7,7 +7,7 @@ 0 0 667 - 323 + 334 @@ -206,16 +206,6 @@ border-image: url(:/images/closepressed.png) - - - - - 0 - 0 - - - - @@ -989,6 +979,11 @@ border-image: url(:/images/closepressed.png) Don't replace tag with Emote Icon. + + + Show Hidden Images + + @@ -1017,17 +1012,10 @@ border-image: url(:/images/closepressed.png) QTextEdit
gui/common/MimeTextEdit.h
 - - RSImageBlockWidget - QWidget -
gui/common/RSImageBlockWidget.h
- 1 - - - + diff --git a/retroshare-gui/src/gui/common/RSTextBrowser.cpp b/retroshare-gui/src/gui/common/RSTextBrowser.cpp index cb0634ee6..f57e650a9 100644 --- a/retroshare-gui/src/gui/common/RSTextBrowser.cpp +++ b/retroshare-gui/src/gui/common/RSTextBrowser.cpp @@ -1,8 +1,9 @@ #include -#include #include +#include #include +#include #include "RSTextBrowser.h" #include "RSImageBlockWidget.h" @@ -19,7 +20,7 @@ RSTextBrowser::RSTextBrowser(QWidget *parent) : mImageBlockWidget = NULL; mLinkClickActive = true; - highliter = new RsSyntaxHighlighter(this); + highlighter = new RsSyntaxHighlighter(this); connect(this, SIGNAL(anchorClicked(QUrl)), this, SLOT(linkClicked(QUrl))); } @@ -73,6 +74,23 @@ void RSTextBrowser::paintEvent(QPaintEvent *event) painter.drawText(QRect(QPoint(), vieportWidget->size()), Qt::AlignHCenter | Qt::AlignVCenter | Qt::TextWordWrap, mPlaceholderText); } +#ifdef RSTEXTBROWSER_CHECKIMAGE_DEBUG + QPainter painter(viewport()); + QPen pen = painter.pen(); + pen.setWidth(2); + pen.setColor(QColor(qRgba(255,0,0,128))); + painter.setPen(pen); + painter.drawRect(mCursorRectStart); + pen.setColor(QColor(qRgba(0,255,0,128))); + painter.setPen(pen); + painter.drawRect(mCursorRectLeft); + pen.setColor(QColor(qRgba(0,0,255,128))); + painter.setPen(pen); + painter.drawRect(mCursorRectRight); + pen.setColor(QColor(qRgba(0,0,0,128))); + painter.setPen(pen); + painter.drawRect(mCursorRectEnd); +#endif } QVariant RSTextBrowser::loadResource(int type, const QUrl &name) @@ -103,6 +121,11 @@ QVariant RSTextBrowser::loadResource(int type, const QUrl &name) if (mImageBlockWidget) mImageBlockWidget->show(); + return getBlockedImage(); +} + +QPixmap RSTextBrowser::getBlockedImage() +{ return QPixmap(":/images/imageblocked_24.png"); } @@ -159,3 +182,61 @@ void RSTextBrowser::activateLinkClick(bool active) { mLinkClickActive = active; } + +/** + * @brief RSTextBrowser::checkImage + * @param pos where to check if image is shown in viewport coordinate + * @param imageStr return html source of cursor + * @return True if an image is under cursor + */ +bool RSTextBrowser::checkImage(QPoint pos, QString &imageStr) +{ + //Get text cursor under pos. But if pos is under text browser end line this return last cursor. + QTextCursor cursor = cursorForPosition(pos); + //First get rect of cursor (could be at left or right of image) + QRect cursorRectStart = cursorRect(cursor); + //Second get text + cursor.movePosition(QTextCursor::Left, QTextCursor::MoveAnchor, 1);//To get character just before + QRect cursorRectLeft = cursorRect(cursor); + cursor.movePosition(QTextCursor::Right, QTextCursor::KeepAnchor, 2); + QRect cursorRectRight = cursorRect(cursor); + imageStr = cursor.selection().toHtml(); +#ifdef RSTEXTBROWSER_CHECKIMAGE_DEBUG + mCursorRectStart = cursorRectStart; + mCursorRectLeft = cursorRectLeft; + mCursorRectRight = cursorRectRight; + + std::cerr << "cursorRect LTRB :" << cursorRectStart.left() << ";" << cursorRectStart.top() << ";" << cursorRectStart.right() << ";" << cursorRectStart.bottom() << std::endl; + std::cerr << "cursorRectLeft :" << cursorRectLeft.left() << ";" << cursorRectLeft.top() << ";" << cursorRectLeft.right() << ";" << cursorRectLeft.bottom() << std::endl; + std::cerr << "cursorRectRight :" << cursorRectRight.left() << ";" << cursorRectRight.top() << ";" << cursorRectRight.right() << ";" << cursorRectRight.bottom() << std::endl; + std::cerr << "pos XY :" << pos.x() << ";" << pos.y() << std::endl; +#endif + QRect cursorRectEnd = cursorRectStart; + //Finally set left with right of precedent character. + if (cursorRectEnd.top() < cursorRectLeft.bottom()) + { + cursorRectEnd.setLeft(cursorRectLeft.right()); + } else { + //Image on new line + cursorRectEnd.setLeft(0); + } + //And set Right with left of next character. + if (cursorRectEnd.bottom() > cursorRectRight.top()) + { + cursorRectEnd.setRight(cursorRectRight.left()); + } else { + //New line after Image. + } +#ifdef RSTEXTBROWSER_CHECKIMAGE_DEBUG + mCursorRectEnd = cursorRectEnd; + + std::cerr << "final cursorRect:" << cursorRectEnd.left() << ";" << cursorRectEnd.top() << ";" << cursorRectEnd.right() << ";" << cursorRectEnd.bottom() << std::endl; + viewport()->update(); +#endif + //If pos is on text rect + if (cursorRectEnd.contains(pos)) + { + return imageStr.indexOf("base64,") != -1; + } + return false; +} diff --git a/retroshare-gui/src/gui/common/RSTextBrowser.h b/retroshare-gui/src/gui/common/RSTextBrowser.h index f9e218b77..13e520d00 100644 --- a/retroshare-gui/src/gui/common/RSTextBrowser.h +++ b/retroshare-gui/src/gui/common/RSTextBrowser.h @@ -4,6 +4,8 @@ #include #include "util/RsSyntaxHighlighter.h" +//#define RSTEXTBROWSER_CHECKIMAGE_DEBUG 1 + class RSImageBlockWidget; class RSTextBrowser : public QTextBrowser @@ -18,16 +20,20 @@ public: void setPlaceholderText(const QString &text); void setImageBlockWidget(RSImageBlockWidget *widget); void resetImagesStatus(bool load); + QPixmap getBlockedImage(); + bool checkImage(QPoint pos, QString &imageStr); + bool checkImage(QPoint pos) {QString imageStr; return checkImage(pos, imageStr); } void activateLinkClick(bool active); virtual QVariant loadResource(int type, const QUrl &name); - QColor textColorQuote() const { return highliter->textColorQuote();} + QColor textColorQuote() const { return highlighter->textColorQuote();} + bool getShowImages() const { return mShowImages; } public slots: void showImages(); - void setTextColorQuote(QColor textColorQuote) { highliter->setTextColorQuote(textColorQuote);} + void setTextColorQuote(QColor textColorQuote) { highlighter->setTextColorQuote(textColorQuote);} private slots: void linkClicked(const QUrl &url); @@ -41,7 +47,13 @@ private: bool mShowImages; RSImageBlockWidget *mImageBlockWidget; bool mLinkClickActive; - RsSyntaxHighlighter *highliter; + RsSyntaxHighlighter *highlighter; +#ifdef RSTEXTBROWSER_CHECKIMAGE_DEBUG + QRect mCursorRectStart; + QRect mCursorRectLeft; + QRect mCursorRectRight; + QRect mCursorRectEnd; +#endif }; #endif // RSTEXTBROWSER_H diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 29033ae09..24c4729cd 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -517,8 +517,7 @@ void GxsForumThreadWidget::contextMenuTextBrowser(QPoint point) contextMnu->addSeparator(); - QTextCursor cursor = ui->postText->cursorForPosition(point); - if(ImageUtil::checkImage(cursor)) + if(ui->postText->checkImage(point)) { ui->actionSave_image->setData(point); contextMnu->addAction(ui->actionSave_image); diff --git a/retroshare-gui/src/util/imageutil.cpp b/retroshare-gui/src/util/imageutil.cpp index 01f8c4bcb..1a08ffb07 100644 --- a/retroshare-gui/src/util/imageutil.cpp +++ b/retroshare-gui/src/util/imageutil.cpp @@ -1,13 +1,13 @@ #include "imageutil.h" #include "util/misc.h" -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include ImageUtil::ImageUtil() {} @@ -42,10 +42,3 @@ void ImageUtil::extractImage(QWidget *window, QTextCursor cursor) } } -bool ImageUtil::checkImage(QTextCursor cursor) -{ - cursor.movePosition(QTextCursor::Left, QTextCursor::MoveAnchor, 1); - cursor.movePosition(QTextCursor::Right, QTextCursor::KeepAnchor, 2); - QString imagestr = cursor.selection().toHtml(); - return imagestr.indexOf("base64,") != -1; -} diff --git a/retroshare-gui/src/util/imageutil.h b/retroshare-gui/src/util/imageutil.h index f8ac9016a..30e6ecb67 100644 --- a/retroshare-gui/src/util/imageutil.h +++ b/retroshare-gui/src/util/imageutil.h @@ -4,14 +4,12 @@ #include #include - class ImageUtil { public: ImageUtil(); static void extractImage(QWidget *window, QTextCursor cursor); - static bool checkImage(QTextCursor cursor); }; #endif // IMAGEUTIL_H From 7cd31aa788f0860b926ae5599050fb48e8089b49 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Wed, 5 Oct 2016 14:15:12 +0200 Subject: [PATCH 150/174] fixed bug preventign update of subdirs after last commit --- libretroshare/src/file_sharing/dir_hierarchy.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index a697fa120..3e112ef95 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -188,7 +188,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage de->row = mNodes.size(); de->parent_index = indx; - de->dir_modtime = it->second; + de->dir_modtime = 0;// forces parsing.it->second; de->dir_parent_path = d.dir_parent_path + "/" + d.dir_name ; de->dir_hash = createDirHash(de->dir_name,de->dir_parent_path) ; From 69ecca428180d988187cd2e695792b7aa3ed9376 Mon Sep 17 00:00:00 2001 From: Marco Barbosa Date: Fri, 7 Oct 2016 23:12:40 -0300 Subject: [PATCH 151/174] add uninstall section in README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4394180f0..a90ccb8de 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,11 @@ Compilation on Linux /usr/bin/RetroShare06 /usr/bin/RetroShare06-nogui +5. Uninstall: + ```bash + sudo make uninstall + ``` + Compile only retroshare-nogui ----------------------------- If you want to run RetroShare on a server and don’t need the gui and plugins, From 4b0f82a6d0d2fc39e6c82fbc5dbad57bee9ccb92 Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 8 Oct 2016 12:12:26 +0200 Subject: [PATCH 152/174] Fix Desktop Filename for option page message. --- retroshare-gui/src/gui/settings/rsharesettings.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/settings/rsharesettings.cpp b/retroshare-gui/src/gui/settings/rsharesettings.cpp index 530ffd766..fb7dc9dad 100644 --- a/retroshare-gui/src/gui/settings/rsharesettings.cpp +++ b/retroshare-gui/src/gui/settings/rsharesettings.cpp @@ -789,7 +789,10 @@ bool RshareSettings::getRetroShareProtocol() } } #elif defined(Q_OS_LINUX) - QFile desktop("/usr/share/applications/RetroShare06.desktop"); + QFile desktop("/usr/share/applications/retroshare06.desktop"); + if (!desktop.exists()) { + desktop.setFileName("/usr/share/applications/RetroShare06.desktop"); + } if (desktop.exists()) { desktop.open(QIODevice::ReadOnly | QIODevice::Text); QTextStream in(&desktop); From ada1cee2c8b6a35a6a6edbe860b76ffef7a59b32 Mon Sep 17 00:00:00 2001 From: sehraf Date: Sat, 8 Oct 2016 12:59:27 +0200 Subject: [PATCH 153/174] remove deprecated ssh/protobuf code --- retroshare-nogui/src/menu/menu.cc | 907 ---------- retroshare-nogui/src/menu/menu.h | 249 --- retroshare-nogui/src/menu/menus.cc | 833 ---------- retroshare-nogui/src/menu/menus.h | 346 ---- retroshare-nogui/src/menu/menutest.h | 94 -- retroshare-nogui/src/menu/stdiocomms.cc | 183 -- retroshare-nogui/src/menu/stdiocomms.h | 59 - retroshare-nogui/src/retroshare-nogui.pro | 187 +-- retroshare-nogui/src/retroshare.cc | 282 +--- .../src/rpc/proto/rpcprotochat.cc | 1477 ----------------- retroshare-nogui/src/rpc/proto/rpcprotochat.h | 56 - .../src/rpc/proto/rpcprotofiles.cc | 630 ------- .../src/rpc/proto/rpcprotofiles.h | 45 - .../src/rpc/proto/rpcprotopeers.cc | 610 ------- .../src/rpc/proto/rpcprotopeers.h | 46 - .../src/rpc/proto/rpcprotosearch.cc | 678 -------- .../src/rpc/proto/rpcprotosearch.h | 64 - .../src/rpc/proto/rpcprotostream.cc | 825 --------- .../src/rpc/proto/rpcprotostream.h | 124 -- .../src/rpc/proto/rpcprotosystem.cc | 354 ---- .../src/rpc/proto/rpcprotosystem.h | 47 - .../src/rpc/proto/rpcprotoutils.cc | 59 - .../src/rpc/proto/rpcprotoutils.h | 35 - retroshare-nogui/src/rpc/rpc.cc | 307 ---- retroshare-nogui/src/rpc/rpc.h | 72 - retroshare-nogui/src/rpc/rpcecho.cc | 40 - retroshare-nogui/src/rpc/rpcecho.h | 38 - retroshare-nogui/src/rpc/rpcserver.cc | 538 ------ retroshare-nogui/src/rpc/rpcserver.h | 164 -- retroshare-nogui/src/rpc/rpcsetup.cc | 68 - retroshare-nogui/src/rpc/rpcsetup.h | 36 - retroshare-nogui/src/rpcsystem.h | 64 - retroshare-nogui/src/ssh/rssshd.cc | 913 ---------- retroshare-nogui/src/ssh/rssshd.h | 157 -- rsctrl/src/NOTES.txt | 60 - rsctrl/src/definition/chat.proto | 246 --- rsctrl/src/definition/core.proto | 161 -- rsctrl/src/definition/files.proto | 143 -- rsctrl/src/definition/gxs.proto | 287 ---- rsctrl/src/definition/msgs.proto | 108 -- rsctrl/src/definition/peers.proto | 111 -- rsctrl/src/definition/search.proto | 120 -- rsctrl/src/definition/stream.proto | 152 -- rsctrl/src/definition/system.proto | 121 -- 44 files changed, 5 insertions(+), 12091 deletions(-) delete mode 100644 retroshare-nogui/src/menu/menu.cc delete mode 100644 retroshare-nogui/src/menu/menu.h delete mode 100644 retroshare-nogui/src/menu/menus.cc delete mode 100644 retroshare-nogui/src/menu/menus.h delete mode 100644 retroshare-nogui/src/menu/menutest.h delete mode 100644 retroshare-nogui/src/menu/stdiocomms.cc delete mode 100644 retroshare-nogui/src/menu/stdiocomms.h delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotochat.cc delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotochat.h delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotofiles.cc delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotofiles.h delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotopeers.cc delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotopeers.h delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotosearch.cc delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotosearch.h delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotostream.cc delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotostream.h delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotosystem.cc delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotosystem.h delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotoutils.cc delete mode 100644 retroshare-nogui/src/rpc/proto/rpcprotoutils.h delete mode 100644 retroshare-nogui/src/rpc/rpc.cc delete mode 100644 retroshare-nogui/src/rpc/rpc.h delete mode 100644 retroshare-nogui/src/rpc/rpcecho.cc delete mode 100644 retroshare-nogui/src/rpc/rpcecho.h delete mode 100644 retroshare-nogui/src/rpc/rpcserver.cc delete mode 100644 retroshare-nogui/src/rpc/rpcserver.h delete mode 100644 retroshare-nogui/src/rpc/rpcsetup.cc delete mode 100644 retroshare-nogui/src/rpc/rpcsetup.h delete mode 100644 retroshare-nogui/src/rpcsystem.h delete mode 100644 retroshare-nogui/src/ssh/rssshd.cc delete mode 100644 retroshare-nogui/src/ssh/rssshd.h delete mode 100644 rsctrl/src/NOTES.txt delete mode 100644 rsctrl/src/definition/chat.proto delete mode 100644 rsctrl/src/definition/core.proto delete mode 100644 rsctrl/src/definition/files.proto delete mode 100644 rsctrl/src/definition/gxs.proto delete mode 100644 rsctrl/src/definition/msgs.proto delete mode 100644 rsctrl/src/definition/peers.proto delete mode 100644 rsctrl/src/definition/search.proto delete mode 100644 rsctrl/src/definition/stream.proto delete mode 100644 rsctrl/src/definition/system.proto diff --git a/retroshare-nogui/src/menu/menu.cc b/retroshare-nogui/src/menu/menu.cc deleted file mode 100644 index 523fe8d87..000000000 --- a/retroshare-nogui/src/menu/menu.cc +++ /dev/null @@ -1,907 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "menu/menu.h" -#include -#include -#include - -#include -#include - -#include "util/rsstring.h" - -#define MENU_DEBUG 1 - -/********************************************************** - * Menu Base Interface. - */ - - // RsTermServer Interface. -void MenuInterface::reset(uint32_t /* chan_id */) -{ - mBase->reset(); - mCurrentMenu = mBase; - mInputRequired = false; - mUpdateTime = 0; -} - - - -int MenuInterface::tick() -{ -#ifdef MENU_DEBUG - std::cerr << "MenuInterface::tick()"; - std::cerr << std::endl; -#endif // MENU_DEBUG - - /* try to read a char */ - bool haveInput = false; - uint8_t keypress; - std::string output; - - uint32_t chan_id = 1; // dummy - for menu system. - - int read = mComms->recv(chan_id, &keypress, 1); -#ifdef MENU_DEBUG - std::cerr << "MenuInterface::tick() read " << read << " bytes"; - std::cerr << std::endl; -#endif // MENU_DEBUG - - if (read == 0) - { - haveInput = false; - /* make a harmless key */ - keypress = ' '; - } - else if (read == 1) - { - haveInput = true; - } - else - { - /* error, NON BLOCKING is handled by recv returning 0 */ - mComms->error(chan_id, "Bad Input"); - return -1; - } - - - /**** Main logic bit ****/ - /**** slow down the updates / refresh ****/ - - time_t now = time(NULL); -#define UPDATE_TIME 5 - if (!haveInput) - { - // If Input is Required, - if (mInputRequired) - { - std::cerr << "MenuInterface::tick() No Input & Required-No Output"; - std::cerr << std::endl; - return 0; - } - - // Output will just almost the same, so occasionally. - if (now < mUpdateTime + UPDATE_TIME) - { - std::cerr << "MenuInterface::tick() No Input-Slow Update"; - std::cerr << std::endl; - return 0; - } - - std::cerr << "MenuInterface::tick() No Input - but doing update."; - std::cerr << std::endl; - } - - uint32_t rt = process(keypress, mDrawFlags, output); - mInputRequired = (rt == MENU_PROCESS_NEEDDATA); - mUpdateTime = now; - - if (rt == MENU_PROCESS_QUIT) - { - return -1; - } - - if (output.size() > 0) - { - mComms->send(chan_id, output); - } - - return (haveInput); -} - - -int tailrec_printparents(Menu *m, std::string &buffer) -{ - Menu *p = m->parent(); - if (p) - { - tailrec_printparents(p, buffer); - } - buffer += m->ShortFnDesc(); - buffer += " => "; - return 1; -} - - - -uint32_t MenuInterface::process(char key, uint32_t drawFlags, std::string &buffer) -{ - -#ifdef MENU_DEBUG - std::cout << "MenuInterface::process(" << key << ")"; - std::cout << std::endl; -#endif // MENU_DEBUG - - /* call process on current menu */ - uint32_t rt = mCurrentMenu->process(key); - bool doRedraw = true; - bool showError = false; - bool showHelp = false; - -#ifdef MENU_DEBUG - std::cout << "MenuInterface::process() currentMenu says: " << rt; - std::cout << std::endl; -#endif // MENU_DEBUG - - - uint32_t base_rt = (rt & MENU_PROCESS_MASK); - bool needData = (rt & MENU_PROCESS_NEEDDATA); - switch(base_rt) - { - case MENU_PROCESS_NONE: - if (needData) - { - /* no redraw, this could be called many times */ - doRedraw = false; - } - break; - - case MENU_PROCESS_DONE: - /* no changes - operation performed, or noop */ - break; - - case MENU_PROCESS_ERROR: - /* Show Error at top of Page */ - showError = true; - break; - - case MENU_PROCESS_HELP: - /* Show Help at top of Page */ - showHelp = true; - break; - - case MENU_PROCESS_MENU: - /* new menu to switch to */ - if (mCurrentMenu->selectedMenu()) - { - std::cout << "MenuInterface::process() Switching Menus"; - std::cout << std::endl; - mCurrentMenu = mCurrentMenu->selectedMenu(); - } - else - { - std::cout << "MenuInterface::process() ERROR"; - std::cout << " SelectedMenu == NULL"; - std::cout << std::endl; - } - break; - - case MENU_PROCESS_TOP: - /* switch to Base Menu */ - mCurrentMenu = mBase; - break; - - case MENU_PROCESS_QUIT: - return MENU_PROCESS_QUIT; - break; - } - - if (drawFlags & MENU_DRAW_FLAGS_ECHO) - { - buffer += key; - } - - /* now we redraw, and wait for next data */ - if (!doRedraw) - { - return MENU_PROCESS_NEEDDATA; - } - - /* HEADER */ - for(int i = 0; i < 20; i++) - { - buffer += "\r\n"; - } - - /* ERROR */ - if (showError) - { - mCurrentMenu->showError(); - } - /* HELP */ - else if (showHelp) - { - mCurrentMenu->showHelp(); - } - - /* MENU PAGE */ - drawHeader(drawFlags, buffer); - mCurrentMenu->drawPage(drawFlags, buffer); - - if (needData) - return MENU_PROCESS_NEEDDATA; - - return MENU_PROCESS_NONE; -} - - - - -uint32_t MenuInterface::drawHeader(uint32_t drawFlags, std::string &buffer) -{ - buffer += "=======================================================\r\n"; - buffer += "Retroshare Terminal Menu V2.xxxx ======================\r\n"; - - unsigned int nTotal = 0; - unsigned int nConnected = 0; - rsPeers->getPeerCount(&nTotal, &nConnected, false); - - uint32_t netState = rsConfig->getNetState(); - std::string natState("Unknown"); - switch(netState) - { - default: - case RSNET_NETSTATE_BAD_UNKNOWN: - natState = "YELLOW:Unknown"; - break; - - case RSNET_NETSTATE_BAD_OFFLINE: - natState = "GRAY:Offline"; - break; - - case RSNET_NETSTATE_BAD_NATSYM: - natState = "RED:Nasty Firewall"; - break; - - case RSNET_NETSTATE_BAD_NODHT_NAT: - natState = "RED:NoDht & Firewalled"; - break; - - case RSNET_NETSTATE_WARNING_RESTART: - natState = "YELLOW:Restarting"; - break; - - case RSNET_NETSTATE_WARNING_NATTED: - natState = "YELLOW:Firewalled"; - break; - - case RSNET_NETSTATE_WARNING_NODHT: - natState = "YELLOW:DHT Disabled"; - break; - - case RSNET_NETSTATE_GOOD: - natState = "GREEN:Good!"; - break; - - case RSNET_NETSTATE_ADV_FORWARD: - natState = "GREEN:Forwarded Port"; - break; - } - - float downKb = 0; - float upKb = 0; - rsConfig->GetCurrentDataRates(downKb, upKb); - - rs_sprintf_append(buffer, "Friends %d / %d Network: %s\r\n", - nConnected, nTotal, natState.c_str()); - - rs_sprintf_append(buffer, "Down: %2.2f Up %2.2f\r\n", downKb, upKb); - - std::string menuState; - tailrec_printparents(mCurrentMenu, menuState); - - buffer += "Menu State: "; - buffer += menuState; - buffer += "\r\n"; - - buffer += "=======================================================\r\n"; - - return 1; -} - -/********************************************************** - * Menu (Base) - */ - -Menu::~Menu() -{ - /* cleanup children */ - mParent = NULL; - mSelectedMenu = NULL; - std::map::iterator it; - - for(it = mChildren.begin(); it != mChildren.end(); it++) - { - delete (it->second); - } - - mChildren.clear(); -} - -int Menu::addMenuItem(char key, Menu *child) -{ - std::map::iterator it; - it = mChildren.find(key); - if (it != mChildren.end()) - { - std::cout << "Menu::addMenuItem() ERROR DUPLICATE MENU ITEM"; - std::cout << std::endl; - return 0; - } - mChildren[key] = child; - child->setParent(this); - - return 1; -} - -void Menu::reset() -{ - mSelectedMenu = NULL; - std::map::iterator it; - for(it = mChildren.begin(); it != mChildren.end(); it++) - { - it->second->reset(); - } -} - - -uint32_t Menu::process(char key) -{ - /* try standard list ones */ - uint32_t rt = std_process(key); - if (rt) - { - return rt; - } - - /* now try children */ - rt = process_children(key); - if (rt) - { - return rt; - } - - return MENU_PROCESS_NONE; -} - -uint32_t Menu::std_process(char key) -{ - switch(key) - { - case MENU_KEY_QUIT: - return MENU_PROCESS_QUIT; - break; - case MENU_KEY_HELP: - return MENU_PROCESS_HELP; - break; - case MENU_KEY_TOP: - return MENU_PROCESS_TOP; - break; - case MENU_KEY_UP: - setSelectedMenu(parent()); - return MENU_PROCESS_MENU; - break; - } - - return MENU_PROCESS_NONE; -} - - -uint32_t Menu::process_children(char key) -{ - std::map::iterator it; - it = mChildren.find(key); - - if (it == mChildren.end()) - { - return MENU_PROCESS_NONE; - } - - /* have a child */ - - /* now return, depending on type */ - switch(it->second->op()) - { - case MENU_OP_NEEDDATA: - setSelectedMenu(it->second); - return MENU_PROCESS_MENU | MENU_PROCESS_NEEDDATA; - break; - - case MENU_OP_SUBMENU: - setSelectedMenu(it->second); - return MENU_PROCESS_MENU; - break; - - default: - case MENU_OP_INSTANT: - /* done already! */ - setSelectedMenu(NULL); - return MENU_PROCESS_DONE; - break; - - case MENU_OP_ERROR: - /* done already! */ - setSelectedMenu(NULL); - return MENU_PROCESS_ERROR; - break; - } -} - - -uint32_t Menu::drawPage(uint32_t drawFlags, std::string &buffer) -{ - buffer += "Universal Commands ( "; - if (!(drawFlags & MENU_DRAW_FLAGS_NOQUIT)) - { - buffer += (char) MENU_KEY_QUIT; - buffer += ":Quit "; - } - buffer += (char) MENU_KEY_HELP; - buffer += ":Help "; - buffer += (char) MENU_KEY_TOP; - buffer += ":Top "; - buffer += (char) MENU_KEY_UP; - buffer += ":Up "; - buffer += ")"; - buffer += "\r\n"; - - buffer += "Specific Commands ( "; - std::map::iterator it; - for(it = mChildren.begin(); it != mChildren.end(); it++) - { - buffer += (char) it->first; - buffer += ":"; - buffer += it->second->ShortFnDesc(); - buffer += " "; - } - buffer += ")"; - buffer += "\r\n"; - - return 1; -} - - -uint32_t Menu::drawHelpPage(uint32_t drawFlags, std::string &buffer) -{ - std::cout << "Menu Help: Universal Commands are:"; - std::cout << std::endl; - std::cout << "\tKey: " << (char) MENU_KEY_QUIT << " => Quit"; - std::cout << std::endl; - std::cout << "\tKey: " << (char) MENU_KEY_HELP << " => Help"; - std::cout << std::endl; - std::cout << "\tKey: " << (char) MENU_KEY_TOP << " => Top Menu"; - std::cout << std::endl; - std::cout << "\tKey: " << (char) MENU_KEY_UP << " => Up a Menu"; - std::cout << std::endl; - - std::cout << "Specific Commands are:"; - std::cout << std::endl; - std::map::iterator it; - for(it = mChildren.begin(); it != mChildren.end(); it++) - { - std::cout << "\tKey: " << (char) it->first << " => "; - std::cout << it->second->ShortFnDesc(); - std::cout << std::endl; - } - - return 1; -} - - -/********************************************************** - * Menu List (Base) - */ - - -uint32_t MenuList::drawPage(uint32_t drawFlags, std::string &buffer) -{ - Menu::drawPage(drawFlags, buffer); - - buffer += "Navigation Commands ("; - buffer += (char) MENULIST_KEY_NEXT; - buffer += ":Next "; - buffer += (char) MENULIST_KEY_PREV; - buffer += ":Prev "; - buffer += ")"; - buffer += "\r\n"; - - rs_sprintf_append(buffer, "MenuList::Internals ListSize: %d, SelectIdx: %d Cursor: %d\r\n", - getListCount(), mSelectIdx, mCursor); - - int i = 0; - - int listCount = getListCount(); - int startCount = mCursor + 1; - int endCount = mCursor + 10; - if (endCount > listCount) - { - endCount = listCount; - } - - if (mSelectIdx >= 0) - { - - rs_sprintf_append(buffer, "Current Selection Idx: %d : ", mSelectIdx); - std::string desc; - if (getEntryDesc(mSelectIdx, desc) & (desc != "")) - { - buffer += desc; - } - else - { - buffer += "Missing Description"; - } - buffer += "\r\n"; - } - else - { - buffer += "No Current Selection: Use 0 - 9 to choose an Entry"; - buffer += "\r\n"; - } - - - rs_sprintf_append(buffer, "Showing %d to %d of %d Entries\r\n", - startCount, endCount, listCount); - - std::list::iterator it; - for (it = mList.begin(); it != mList.end(); it++, i++) - { - int curIdx = i - mCursor; - if ((curIdx >= 0) && (curIdx < 10)) - { - if (i == mSelectIdx) - { - rs_sprintf_append(buffer, "SELECTED (%d) ", curIdx); - } - else - { - rs_sprintf_append(buffer, "\t(%d) ", curIdx); - } - std::string desc; - if (getEntryDesc(i, desc) & (desc != "")) - { - buffer += desc; - } - else - { - buffer += *it; - buffer += " => "; - buffer += "Missing Description"; - } - buffer += "\r\n"; - } - } - buffer += "\r\n"; - buffer += "Make Your Choice > "; - - return 1; -} - -uint32_t MenuList::drawHelpPage(uint32_t drawFlags, std::string &buffer) -{ - Menu::drawHelpPage(drawFlags, buffer); - - std::cout << "MenuList Help: Navigation Commands are:"; - std::cout << std::endl; - //std::cout << "\tKey: " << (char) MENULIST_KEY_LIST << " => List"; - //std::cout << std::endl; - std::cout << "\tKey: " << (char) MENULIST_KEY_NEXT << " => Next Page"; - std::cout << std::endl; - std::cout << "\tKey: " << (char) MENULIST_KEY_PREV << " => Prev Page"; - std::cout << std::endl; - - std::cout << "MenuList::drawPage() Internal Details"; - std::cout << std::endl; - std::cout << "List Size: " << getListCount(); - std::cout << std::endl; - std::cout << "SelectIdx: " << mSelectIdx; - std::cout << std::endl; - std::cout << "Cursor: " << mCursor; - std::cout << std::endl; - return 1; -} - - -void MenuList::reset() -{ - Menu::reset(); // clears children too. - - mList.clear(); - mSelectIdx = -1; - mCursor = 0; -} - -uint32_t MenuList::op() -{ - mList.clear(); - mSelectIdx = -1; - mCursor = 0; - - return MENU_OP_ERROR; // SUBMENU -> only for inherited classes; -} - -uint32_t MenuList::getListCount() -{ - return mList.size(); -} - -int MenuList::getCurrentKey(std::string &key) -{ - return getListEntry(mSelectIdx, key); -} - -int MenuList::getCurrentIdx(int &idx) -{ - idx = mSelectIdx; - return 1; -} - -int MenuList::getListEntry(int idx, std::string &key) -{ - if (idx < 0) - { - return MENU_ENTRY_NONE; - } - - std::list::iterator it; - int i = 0; - for (it = mList.begin(); (i < idx) && (it != mList.end()); it++, i++) ; - - if (it != mList.end()) - { - key = *it; - return MENU_ENTRY_OKAY; - } - return MENU_ENTRY_NONE; -} - -int MenuList::getEntryDesc(int idx, std::string &desc) -{ - desc = "Entry Description"; - return MENU_ENTRY_OKAY; -} - -uint32_t MenuList::process(char key) -{ - /* try standard list ones */ - uint32_t rt = Menu::process(key); - if (rt) - { - return rt; - } - - rt = list_process(key); - return rt; -} - -uint32_t MenuList::list_process(char key) -{ - if (((key >= '0') && (key <= '9')) || - ((key >= 'a') && (key <= 'f'))) - { - int idx = 0; - /* select index */ - if ((key >= '0') && (key <= '9')) - { - idx = key - '0'; - } - else - { - idx = key - 'a' + 9; - } - - /* now change selection, dependent on pagination */ - if (mCursor + idx < getListCount()) - { - mSelectIdx = mCursor + idx; - std::cout << "MenuList::list_process() Selected Idx: " << mSelectIdx; - std::cout << std::endl; - } - else - { - std::cout << "MenuList::list_process() Idx Out of Range"; - std::cout << std::endl; - } - - return MENU_PROCESS_DONE; /* ready for next key stroke */ - } - - - switch(key) - { - case MENULIST_KEY_LIST: - /* send a list to output */ - - return MENU_PROCESS_DONE; /* ready for next key stroke */ - break; - case MENULIST_KEY_NEXT: - /* shift to next page */ - if (mCursor + 10 < getListCount()) - { - mCursor += 10; - } - - return MENU_PROCESS_DONE; - break; - case MENULIST_KEY_PREV: - /* shift to prev page */ - if (((int) mCursor) - 10 >= 0) - { - mCursor -= 10; - } - - return MENU_PROCESS_DONE; - break; - } - - return MENU_PROCESS_NONE; -} - - -uint32_t MenuOpBasicKey::op_basic(std::string key) -{ - parent()->setErrorMessage("MenuOpBasicKey Not Overloaded Correctly"); - return MENU_OP_ERROR; -} - - -uint32_t MenuOpBasicKey::op() -{ - std::string key; - Menu *Parent=parent(); - MenuList *p = dynamic_cast(Parent); - - if (!p) - { - if (Parent) - { - Parent->setErrorMessage("Invalid (Basic) Menu Structure"); - } - return MENU_OP_ERROR; - } - - if (p->getCurrentKey(key)) - { - return op_basic(key); - } - - if (Parent) - { - Parent->setErrorMessage("Invalid Current Keys"); - } - return MENU_OP_ERROR; -} - - - -uint32_t MenuOpTwoKeys::op_twokeys(std::string parentkey, std::string key) -{ - parent()->setErrorMessage("MenuOpTwoKeys Not Overloaded Correctly"); - return MENU_OP_ERROR; -} - - -uint32_t MenuOpTwoKeys::op() -{ - std::string parentkey; - std::string key; - Menu *Parent=parent(); - MenuList *p = dynamic_cast(Parent); - - Menu *grandParent=parent()->parent(); - MenuList *gp = dynamic_cast(grandParent); - - if ((!gp) || (!p)) - { - if (Parent) - { - Parent->setErrorMessage("Invalid (TwoKeys) Menu Structure"); - } - return MENU_OP_ERROR; - } - - if ((gp->getCurrentKey(parentkey)) && (p->getCurrentKey(key))) - { - return op_twokeys(parentkey, key); - } - - if (Parent) - { - Parent->setErrorMessage("Invalid Current Keys"); - } - return MENU_OP_ERROR; -} - - - -/********************************************************** - * Menu Op Line Input (Base) - */ - - -uint32_t MenuOpLineInput::op() -{ - return MENU_OP_NEEDDATA; -} - - -uint32_t MenuOpLineInput::process_lines(std::string input) -{ - std::cout << "MenuOpLineInput::process_lines() => SHOULD BE OVERLOADED"; - std::cout << "Input Was: "; - std::cout << std::endl; - std::cout << "=================================================="; - std::cout << std::endl; - std::cout << input; - std::cout << "=================================================="; - std::cout << std::endl; - - return MENU_PROCESS_ERROR; -} - - -uint32_t MenuOpLineInput::process(char key) -{ - /* read data in and add to buffer */ - mInput += key; - if ((key != '\n') && (key != '\r')) - { - return MENU_PROCESS_NEEDDATA; - } - - uint32_t rt = process_lines(mInput); - - switch(rt) - { - case MENU_PROCESS_NEEDDATA: - break; - case MENU_PROCESS_ERROR: - std::cout << "MenuOpLineInput::process() => ERROR"; - std::cout << std::endl; - case MENU_PROCESS_DONE: - /* cleanup for next command */ - std::cout << "MenuOpLineInput::process() Clearing Buffer"; - std::cout << std::endl; - mInput.clear(); - - /* go back to parent menu */ - rt = MENU_PROCESS_MENU; - setSelectedMenu(parent()); - break; - } - - return rt; -} - - diff --git a/retroshare-nogui/src/menu/menu.h b/retroshare-nogui/src/menu/menu.h deleted file mode 100644 index c667e64dd..000000000 --- a/retroshare-nogui/src/menu/menu.h +++ /dev/null @@ -1,249 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - - -#ifndef RSNOGUI_MENU_H -#define RSNOGUI_MENU_H - -#include - -#include -#include -#include - -#include "rpcsystem.h" // generic processing command. - -#define MENU_PROCESS_MASK 0x0fff - -#define MENU_PROCESS_NONE 0x0000 -#define MENU_PROCESS_TOP 0x0001 -#define MENU_PROCESS_MENU 0x0002 -#define MENU_PROCESS_DONE 0x0004 -#define MENU_PROCESS_QUIT 0x0008 -#define MENU_PROCESS_SHUTDOWN 0x0010 -#define MENU_PROCESS_HELP 0x0020 -#define MENU_PROCESS_ERROR 0x0040 - -#define MENU_PROCESS_NEEDDATA 0x1000 // Able to be OR'd with ANOTHER CASE. - -#define MENU_KEY_QUIT 'Q' -#define MENU_KEY_HELP 'h' -#define MENU_KEY_TOP 't' -#define MENU_KEY_REPEAT 'r' -#define MENU_KEY_UP 'u' - -#define MENULIST_KEY_LIST 'l' // Don't need this. -#define MENULIST_KEY_NEXT 'n' -#define MENULIST_KEY_PREV 'p' - -#define MENU_OP_ERROR 0 -#define MENU_OP_INSTANT 1 -#define MENU_OP_SUBMENU 2 -#define MENU_OP_NEEDDATA 3 - -#define MENU_ENTRY_NONE 0 -#define MENU_ENTRY_OKAY 1 - -#define MENU_DRAW_FLAGS_STD 0 -#define MENU_DRAW_FLAGS_HTML 1 -#define MENU_DRAW_FLAGS_ECHO 2 -#define MENU_DRAW_FLAGS_NOQUIT 4 - -class Menu; -class Screen; - -class Menu -{ -public: - Menu(std::string shortDesc): mParent(NULL), mShortDesc(shortDesc) { return; } -virtual ~Menu(); - - void setParent(Menu *p) { mParent = p; } - Menu *parent() { return mParent; } - Menu *selectedMenu() { return mSelectedMenu; } - int addMenuItem(char key, Menu *child); - -virtual void reset(); -virtual uint32_t op() { return MENU_OP_SUBMENU; } /* what type is it? returns SUBMENU, INSTANT, NEEDINPUT */ -virtual uint32_t process(char key); - - // THE BIT STILL TO BE DEFINED! - -std::string ShortFnDesc() { return mShortDesc; }// Menu Text (for Help). -virtual uint32_t drawPage(uint32_t drawFlags, std::string &buffer); -virtual uint32_t drawHelpPage(uint32_t drawFlags, std::string &buffer); - -//virtual std::string menuText() = 0; -//virtual std::string menuHelp() = 0; -virtual void setOpMessage(std::string msg) { return; } -virtual void setErrorMessage(std::string msg) { return; } -virtual uint32_t showError() { return 1; } //= 0; -virtual uint32_t showHelp() { return 1; } //= 0; - -protected: -virtual uint32_t std_process(char key); /* for H/T/R/Q */ -virtual uint32_t process_children(char key); /* check for children ops */ - - void setSelectedMenu(Menu *m) { mSelectedMenu = m; } - -private: - Menu *mParent; - Menu *mSelectedMenu; - std::string mShortDesc; - std::map mChildren; -}; - - -/********************************************************** - * Generic MenuList... provides a list of KEYS, which - * can be iterated through. - * - * Maintains: List + Current + Next / Previous. - * Single Child, which is called for all. - */ - -class MenuList: public Menu -{ -public: - MenuList(std::string shortDesc): Menu(shortDesc) { return; } - -virtual void reset(); -virtual uint32_t op(); -virtual uint32_t process(char key); - - // List Info. - uint32_t getListCount(); - int getCurrentIdx(int &idx); - int getCurrentKey(std::string &key); - int getListEntry(int idx, std::string &key); -virtual int getEntryDesc(int idx, std::string &desc); - - // Output. -virtual uint32_t drawPage(uint32_t drawFlags, std::string &buffer); -virtual uint32_t drawHelpPage(uint32_t drawFlags, std::string &buffer); - -protected: - virtual uint32_t list_process(char key); - - int32_t mSelectIdx; /* -1 => none */ - uint32_t mCursor; /* offset for list display */ - std::list mList; - -private: -}; - - - -class MenuOpBasicKey: public Menu -{ -public: - MenuOpBasicKey(std::string shortDesc): Menu(shortDesc) { return; } - virtual uint32_t op(); - -protected: - virtual uint32_t op_basic(std::string key); -}; - - - -class MenuOpTwoKeys: public Menu -{ - -public: - MenuOpTwoKeys(std::string shortDesc): Menu(shortDesc) { return; } - virtual uint32_t op(); - -protected: - virtual uint32_t op_twokeys(std::string parentkey, std::string key); - -}; - -/* Read input, line by line... - */ - -class MenuOpLineInput: public Menu -{ - -public: - MenuOpLineInput(std::string shortDesc): Menu(shortDesc) { return; } - virtual uint32_t op(); - virtual uint32_t process(char key); - -protected: - virtual uint32_t process_lines(std::string input); - - std::string mInput; -}; - - -#if 0 -class MenuInterface: public RsTermServer -{ -public: - - MenuInterface(Menu *b, uint32_t drawFlags) :mCurrentMenu(b), mBase(b), mDrawFlags(drawFlags), mInputRequired(false) { return; } - uint32_t process(char key, uint32_t drawFlags, std::string &buffer); - uint32_t drawHeader(uint32_t drawFlags, std::string &buffer); - - // RsTermServer Interface. - virtual void reset(); - virtual int tick(bool haveInput, char keypress, std::string &output); - - -private: - Menu *mCurrentMenu; - Menu *mBase; - uint32_t mDrawFlags; - bool mInputRequired; -}; - -#endif - - -class MenuInterface: public RpcSystem -{ -public: - - MenuInterface(RpcComms *c, Menu *b, uint32_t drawFlags) - :mComms(c), mCurrentMenu(b), mBase(b), mDrawFlags(drawFlags), mInputRequired(false) { return; } - - uint32_t process(char key, uint32_t drawFlags, std::string &buffer); - uint32_t drawHeader(uint32_t drawFlags, std::string &buffer); - - // RsSystem Interface. - virtual void reset(uint32_t chan_id); - virtual int tick(); - - -private: - RpcComms *mComms; - Menu *mCurrentMenu; - Menu *mBase; - uint32_t mDrawFlags; - bool mInputRequired; - time_t mUpdateTime; -}; - - -#endif diff --git a/retroshare-nogui/src/menu/menus.cc b/retroshare-nogui/src/menu/menus.cc deleted file mode 100644 index 299aeb665..000000000 --- a/retroshare-nogui/src/menu/menus.cc +++ /dev/null @@ -1,833 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include -#include -#include -#include "util/rsstring.h" -#include "util/rsdir.h" - -#include "menu/menus.h" - -// Can't use: Q, H, T, U (universal) -// or L, N, P (list ops). -// or 0-9,a-f (list selection). - -#define MENU_FRIENDS_KEY_ADD 'a' -#define MENU_FRIENDS_KEY_VIEW 'v' -#define MENU_FRIENDS_KEY_REMOVE 'd' -#define MENU_FRIENDS_KEY_CHAT 'c' - -#define MENU_TRANSFER_KEY_STOP 's' -#define MENU_TRANSFER_KEY_CANCEL 'c' - -#define MENU_SEARCH_KEY_ADD 'a' -#define MENU_SEARCH_KEY_REMOVE 'd' -#define MENU_SEARCH_KEY_VIEW 'v' -#define MENU_SEARCH_KEY_DOWNLOAD 'g' - -#define MENU_FORUMS_KEY_ADD 'a' -#define MENU_FORUMS_KEY_VIEW 'v' -#define MENU_FORUMS_KEY_REMOVE 'd' -#define MENU_FORUMS_KEY_CHAT 'c' - -#define MENU_SHARED_KEY_EXPAND 'e' -#define MENU_SHARED_KEY_UNSHARE 's' -#define MENU_SHARED_KEY_ADD 'a' -#define MENU_SHARED_KEY_PUBLIC 'c' -#define MENU_SHARED_KEY_BROWSABLE 'b' - -#define MENU_TOPLEVEL_KEY_FRIENDS 'f' -#define MENU_TOPLEVEL_KEY_NETWORK 'w' -#define MENU_TOPLEVEL_KEY_TRANSFER 'd' -#define MENU_TOPLEVEL_KEY_SEARCH 's' -#define MENU_TOPLEVEL_KEY_FORUMS 'o' -#define MENU_TOPLEVEL_KEY_SHARED 'k' //If you know of a key which fits better, just change it ;-) -#define MENU_TOPLEVEL_KEY_UPLOADS 'e' - - -Menu *CreateMenuStructure(NotifyTxt *notify) -{ - /* construct Friends Menu */ - MenuList *friends = new MenuListFriends(); - // No Friends Operations Completed Yet! - //friends->addMenuItem(MENU_FRIENDS_KEY_ADD, new MenuOpFriendsAdd()); - //friends->addMenuItem(MENU_FRIENDS_KEY_VIEW, new MenuOpFriendsViewDetail()); - //friends->addMenuItem(MENU_FRIENDS_KEY_REMOVE, new MenuOpFriendsRemove()); - //friends->addMenuItem(MENU_FRIENDS_KEY_CHAT, new MenuOpFriendsChat()); - - MenuList *network = new MenuListNetwork(); - - MenuList *transfers = new MenuListTransfer(); - //transfers->addMenuItem(MENU_TRANSFER_KEY_STOP, new MenuOpTransferStop()); - transfers->addMenuItem(MENU_TRANSFER_KEY_CANCEL, new MenuOpTransferCancel()); - - MenuList *search = new MenuListSearch(notify); - MenuList *searchlist = new MenuListSearchList(notify); - search->addMenuItem(MENU_SEARCH_KEY_ADD, new MenuOpSearchNew(notify)); - //search->addMenuItem(MENU_SEARCH_KEY_REMOVE, new MenuOpSearchDelete()); - search->addMenuItem(MENU_SEARCH_KEY_VIEW, searchlist); - searchlist->addMenuItem(MENU_SEARCH_KEY_DOWNLOAD, new MenuOpSearchListDownload()); - - // Forums - TODO. - //MenuList *forums = new MenuListForums(); - //forums->addMenuItem(MENU_FRIENDS_KEY_ADD, new MenuOpFriendsAdd()); - //forums->addMenuItem(MENU_FRIENDS_KEY_VIEW, new MenuOpFriendsViewDetail()); - //forums->addMenuItem(MENU_FRIENDS_KEY_REMOVE, new MenuOpFriendsRemove()); - //forums->addMenuItem(MENU_FRIENDS_KEY_CHAT, new MenuOpFriendsChat()); - - // Shared folders Menu - MenuList *shared = new MenuListShared(); - shared->addMenuItem(MENU_SHARED_KEY_ADD , new MenuListSharedAddShare()); - shared->addMenuItem(MENU_SHARED_KEY_UNSHARE , new MenuListSharedUnshare()); - shared->addMenuItem(MENU_SHARED_KEY_PUBLIC , new MenuListSharedTogglePublic()); - shared->addMenuItem(MENU_SHARED_KEY_BROWSABLE , new MenuListSharedToggleBrowsable()); - - /* Top Level Menu */ - Menu *tlm = new Menu("Top Level Menu"); - - tlm->addMenuItem(MENU_TOPLEVEL_KEY_FRIENDS, friends); - tlm->addMenuItem(MENU_TOPLEVEL_KEY_NETWORK, network); - tlm->addMenuItem(MENU_TOPLEVEL_KEY_TRANSFER, transfers); - tlm->addMenuItem(MENU_TOPLEVEL_KEY_SEARCH, search); - //tlm->addMenuItem(MENU_TOPLEVEL_KEY_FORUMS, forums); - tlm->addMenuItem(MENU_TOPLEVEL_KEY_SHARED, shared); - - return tlm; -} - - -/************ - * Friends Menu - */ - - -uint32_t MenuListFriends::op() -{ - MenuList::op(); - - rsPeers->getGPGAcceptedList(mList); - //rsPeers->getGPGValidList(mList); - - return MENU_OP_SUBMENU; -} - -int MenuListFriends::getEntryDesc(int idx, std::string &desc) -{ - std::string gpgId; - - if (!getListEntry(idx, gpgId)) - { - /* error */ - return 0; - } - - RsPeerDetails details; - if (rsPeers->getPeerDetails(gpgId, details)) - { - if (details.accept_connection) - { - desc = "Friend: "; - } - else - { - desc = "Neighbour: "; - } - - desc += "<" + gpgId + "> "; - desc += details.name; - } - return 1; -} - - -uint32_t MenuListNetwork::op() -{ - MenuList::op(); - - rsPeers->getGPGValidList(mList); - //rsPeers->getGPGAcceptedList(mList); - - return MENU_OP_SUBMENU; -} - - -uint32_t MenuOpFriendsAdd::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - -uint32_t MenuOpFriendsViewDetail::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -uint32_t MenuOpFriendsRemove::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -uint32_t MenuOpFriendsChat::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -/************ - * Transfer Menu - */ - - -uint32_t MenuListTransfer::op() -{ - MenuList::op(); - - /* load friend list*/ - rsFiles->FileDownloads(mList); - - return MENU_OP_SUBMENU; -} - -int MenuListTransfer::getEntryDesc(int idx, std::string &desc) -{ - std::string hash; - if (!getListEntry(idx, hash)) - { - std::cout << "MenuListTransfer::getEntryDesc() No ListEntry"; - std::cout << std::endl; - return 0; - } - - FileInfo info; - if (!rsFiles->FileDetails(hash, RS_FILE_HINTS_DOWNLOAD, info)) - { - std::cout << "MenuListTransfer::getEntryDesc() No FileDetails"; - std::cout << std::endl; - return 0; - } - - float frac = 100.0 * (float) info.transfered / info.size; - - if (frac != 1.0) - { - if (info.tfRate > 0) - { - rs_sprintf(desc, "<%s> Downloading %3.2f%% from %d Peers @%3.1fKB/s. Name: %s", - info.hash.c_str(), frac, info.peers.size(), info.tfRate, info.fname.c_str()); - } - else - { - rs_sprintf(desc, "<%s> Stalled %3.2f%% Name: %s", - info.hash.c_str(), frac, info.fname.c_str()); - } - } - else - { - rs_sprintf(desc, "<%s> Done! Name: %s", info.hash.c_str(), info.fname.c_str()); - } - return MENU_OP_INSTANT; -} - -uint32_t MenuOpTransferStop::op_basic(std::string key) -{ - rsFiles->FileCancel(key); - parent()->setOpMessage("Stopped Transfer"); - return MENU_OP_INSTANT; -} - -uint32_t MenuOpTransferCancel::op_basic(std::string key) -{ - rsFiles->FileCancel(key); - parent()->setOpMessage("Stopped Transfer"); - return MENU_OP_INSTANT; -} - - -/************ - * Search Menu - */ - - - -uint32_t MenuListSearch::op() -{ - mSelectIdx = -1; - mCursor = 0; - - /* Don't reset List -> done dynamically */ - - return MENU_OP_SUBMENU; -} - -int MenuListSearch::getEntryDesc(int idx, std::string &desc) -{ - std::string strSearchId; - - if (!getListEntry(idx, strSearchId)) - { - /* error */ - return 0; - } - - std::map::iterator it; - std::map::iterator sit; - it = mSearchTerms.find(strSearchId); - sit = mSearchIds.find(strSearchId); - if ((it == mSearchTerms.end()) || (sit == mSearchIds.end())) - { - /* error */ - return 0; - } - - rs_sprintf(desc, "Search(\"%s\") Found %d matches so far", - it->second.c_str(), mNotify->getSearchResultCount(sit->second)); - return 1; -} - - -int MenuListSearch::getCurrentSearchId(uint32_t &id) -{ - std::string strSearchId; - - if (!getListEntry(mSelectIdx, strSearchId)) - { - /* error */ - return 0; - } - - std::map::iterator sit; - sit = mSearchIds.find(strSearchId); - if (sit == mSearchIds.end()) - { - /* error */ - return 0; - } - - id = sit->second; - return 1; -} - -int MenuListSearch::storeSearch(uint32_t searchId, std::string match_string) -{ - std::cout << "MenuListSearch::storeSearch(" << searchId << " => "; - std::cout << match_string; - std::cout << std::endl; - - std::string strSearchId; - rs_sprintf(strSearchId, "%lu", searchId); - mList.push_back(strSearchId); - - mSearchTerms[strSearchId] = match_string; - mSearchIds[strSearchId] = searchId; - - return 1; -} - -int MenuListSearch::removeSearch(std::string strSearchId) -{ - std::cout << "MenuListSearch::removeSearch(" << strSearchId << ")"; - std::cout << std::endl; - - std::map::iterator it; - it = mSearchIds.find(strSearchId); - if (it != mSearchIds.end()) - { - - /* cancel search */ - // CAN'T DO!!! - - /* clear results from Notify Collector */ - mNotify->clearSearchId(it->second); - - /* cleanup local maps */ - mSearchIds.erase(it); - - /* cleanup terms maps (TODO) */ - } - - return 1; -} - -uint32_t MenuOpSearchNew::drawPage(uint32_t drawFlags, std::string &buffer) -{ - buffer += "Enter New Search Term > "; - return 1; -} - - - -uint32_t MenuOpSearchNew::process_lines(std::string input) -{ - /* launch search */ - if (input.size() < 4) - { - std::cout << "MenuOpSearchNew::process_lines() ERROR Input too small"; - std::cout << std::endl; - return MENU_PROCESS_ERROR; - } - - std::string search = input.substr(0, input.size() - 1); // remove \n. - uint32_t searchId = (uint32_t) rsTurtle->turtleSearch(search); - mNotify->collectSearchResults(searchId); - - /* store request in parent */ - MenuListSearch *ms = dynamic_cast(parent()); - if (ms) - { - ms->storeSearch(searchId, search); - } - - return MENU_PROCESS_DONE; -} - -uint32_t MenuOpSearchDelete::op_basic(std::string key) -{ - - return MENU_OP_INSTANT; -} - - -uint32_t MenuListSearchList::op() -{ - MenuList::op(); - return refresh(); -} - -uint32_t MenuListSearchList::refresh() -{ - Menu* p = parent(); - MenuListSearch *mls = dynamic_cast(p); - if (!mls) - { - std::cout << "MenuListSearchList::refresh() mls not there"; - std::cout << std::endl; - return MENU_OP_ERROR; - } - - /* load friend list*/ - mList.clear(); - - uint32_t searchId; - if (!mls->getCurrentSearchId(searchId)) - { - std::cout << "MenuListSearchList::refresh() currentIdx invalid"; - std::cout << std::endl; - return MENU_OP_ERROR; - } - - std::cout << "MenuListSearchList::refresh() searchId: " << searchId; - std::cout << std::endl; - - std::list::iterator it; - mNotify->getSearchResults(searchId, mSearchResults); - - /* convert into useful list */ - for(it = mSearchResults.begin(); it != mSearchResults.end(); it++) - { - mList.push_back(it->hash); - } - - mSelectIdx = -1; - mCursor = 0; - return MENU_OP_SUBMENU; -} - -int MenuListSearchList::getEntryDesc(int idx, std::string &desc) -{ - std::list::iterator it; - int i = 0; - for (it = mSearchResults.begin(); - (i < idx) && (it != mSearchResults.end()); it++, i++) ; - - if (it != mSearchResults.end()) - { - rs_sprintf(desc, "<%s> Size: %llu Name:%s", - it->hash.c_str(), it->size, it->name.c_str()); - return MENU_ENTRY_OKAY; - } - return MENU_ENTRY_NONE; -} - - -int MenuListSearchList::downloadSelected() -{ - if (mSelectIdx < 0) - { - std::cout << "MenuListSearchList::downloadSelected() Invalid Selection"; - std::cout << std::endl; - return MENU_ENTRY_NONE; - } - - std::list::iterator it; - int i = 0; - for (it = mSearchResults.begin(); - (i < mSelectIdx) && (it != mSearchResults.end()); it++, i++) ; - - if (it != mSearchResults.end()) - { - std::list srcIds; - if (rsFiles -> FileRequest(it->name, it->hash, it->size, - "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) - { - std::cout << "MenuListSearchList::downloadSelected() Download Started"; - std::cout << std::endl; - } - else - { - std::cout << "MenuListSearchList::downloadSelected() Error Starting Download"; - std::cout << std::endl; - } - return MENU_ENTRY_OKAY; - } - return MENU_ENTRY_NONE; -} - - - -uint32_t MenuOpSearchListDownload::op_basic(std::string key) -{ - Menu* p = parent(); - MenuListSearchList *mlsl = dynamic_cast(p); - if (!mlsl) - { - std::cout << "MenuOpSearchListDownload::op_basic() ERROR"; - std::cout << std::endl; - return MENU_OP_ERROR; - } - - mlsl->downloadSelected(); - - return MENU_OP_INSTANT; -} - -/************ - * Forums Menu - */ - - - -uint32_t MenuListForums::op() -{ - /* load friend list*/ - mList.clear(); - rsPeers->getGPGAcceptedList(mList); - mSelectIdx = 0; - - return MENU_OP_SUBMENU; -} - - -int MenuListForums::getEntryDesc(int idx, std::string &desc) -{ - - - return MENU_OP_INSTANT; -} - -uint32_t MenuOpForumDetails::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -uint32_t MenuOpForumSubscribe::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -uint32_t MenuOpForumUnsubscribe::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -uint32_t MenuOpForumCreate::op_basic(std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -uint32_t MenuListForumMsgs::op() -{ - /* load friend list*/ - mList.clear(); - rsPeers->getGPGAcceptedList(mList); - mSelectIdx = 0; - - return MENU_OP_SUBMENU; -} - -int MenuListForumMsgs::getEntryDesc(int idx, std::string &desc) -{ - - - return MENU_OP_INSTANT; -} - -uint32_t MenuOpForumMsgView::op_twokeys(std::string parentkey, std::string key) -{ - - - return MENU_OP_INSTANT; -} - -uint32_t MenuOpForumMsgReply::op_twokeys(std::string parentkey, std::string key) -{ - - - return MENU_OP_INSTANT; -} - - -uint32_t MenuOpForumMsgWrite::op_twokeys(std::string parentkey, std::string key) -{ - - - return MENU_OP_INSTANT; -} - -/************ - * Shared folders Menu - */ - - -uint32_t MenuListShared::op() -{ - MenuList::op(); - - mList.clear(); - std::list dirs; - rsFiles->getSharedDirectories(dirs); - std::list::iterator it; - for(it = dirs.begin(); it != dirs.end(); it++) - { - mList.push_back ((*it).virtualname) ; - } - - return MENU_OP_SUBMENU; -} - -int MenuListShared::getEntryDesc(int idx, std::string &desc) -{ - std::list dirs; - rsFiles->getSharedDirectories(dirs); - std::list::iterator it; - std::string shareflag; - int i=0; - for (it = dirs.begin(); (i < idx) && (it != dirs.end()); it++, i++); - if (it != dirs.end()) - { - bool networkwide = (it->shareflags & DIR_FLAGS_NETWORK_WIDE_OTHERS); - bool browsable = (it->shareflags & DIR_FLAGS_BROWSABLE_OTHERS); - if (networkwide && browsable) - { - shareflag = "networkwide - browsable"; - } - else if (browsable) - { - shareflag = "private - browsable"; - } - else if (networkwide) - { - shareflag = "networkwide - anonymous"; - } - else - { - shareflag = "not shared"; - } - - rs_sprintf(desc, "Path: %s Share Type:%s", it->filename.c_str(), shareflag.c_str()); - return MENU_ENTRY_OKAY; - } - return MENU_ENTRY_NONE; -} - -int MenuListShared::unshareSelected() -{ - if (mSelectIdx < 0) - { - std::cout << "MenuListSearchList::unshareSelected() Invalid Selection"; - std::cout << std::endl; - return MENU_ENTRY_NONE; - } - std::list dirs; - rsFiles->getSharedDirectories(dirs); - std::list::iterator it; - int i=0; - for (it = dirs.begin(); (i < mSelectIdx) && (it != dirs.end()); it++, i++); - if (it != dirs.end()) - { - rsFiles->removeSharedDirectory(it->filename); - return MENU_ENTRY_OKAY; - } - return MENU_ENTRY_NONE; -} - -int MenuListShared::toggleFlagSelected(FileStorageFlags shareflags) -{ - if (mSelectIdx < 0) - { - std::cout << "MenuListSearchList::unshareSelected() Invalid Selection"; - std::cout << std::endl; - return MENU_ENTRY_NONE; - } - std::list dirs; - rsFiles->getSharedDirectories(dirs); - std::list::iterator it; - int i=0; - for (it = dirs.begin(); (i < mSelectIdx) && (it != dirs.end()); it++, i++); - if (it != dirs.end()) - { - if(FileStorageFlags(it->shareflags & shareflags) == shareflags) - { - it->shareflags = FileStorageFlags(it->shareflags & ~shareflags); //disable shareflags - rsFiles->updateShareFlags(*it); - } - else - { - it->shareflags = FileStorageFlags(it->shareflags | shareflags); //anable shareflags - rsFiles->updateShareFlags(*it); - } - return MENU_ENTRY_OKAY; - } - return MENU_ENTRY_NONE; -} - -uint32_t MenuListSharedUnshare::op_basic(std::string key) -{ - Menu* p = parent(); - MenuListShared *mls = dynamic_cast(p); - if (!mls) - { - std::cout << "MenuListShared::op_basic() ERROR"; - std::cout << std::endl; - return MENU_OP_ERROR; - } - - mls->unshareSelected(); - - return MENU_OP_INSTANT; -} - -uint32_t MenuListSharedTogglePublic::op_basic(std::string key) -{ - Menu* p = parent(); - MenuListShared *mls = dynamic_cast(p); - if (!mls) - { - std::cout << "MenuListShared::op_basic() ERROR"; - std::cout << std::endl; - return MENU_OP_ERROR; - } - - mls->toggleFlagSelected(DIR_FLAGS_NETWORK_WIDE_OTHERS); - - return MENU_OP_INSTANT; -} - -uint32_t MenuListSharedToggleBrowsable::op_basic(std::string key) -{ - Menu* p = parent(); - MenuListShared *mls = dynamic_cast(p); - if (!mls) - { - std::cout << "MenuListShared::op_basic() ERROR"; - std::cout << std::endl; - return MENU_OP_ERROR; - } - - mls->toggleFlagSelected(DIR_FLAGS_BROWSABLE_OTHERS); - - return MENU_OP_INSTANT; -} - -uint32_t MenuListSharedAddShare::drawPage(uint32_t drawFlags, std::string &buffer) -{ - buffer += "Enter New path 'path' 'virtualfolder' > "; - return 1; -} - - - -uint32_t MenuListSharedAddShare::process_lines(std::string input) -{ - /* launch search */ - if (input.size() < 4) - { - std::cout << "MenuOpSearchNew::process_lines() ERROR Input too small"; - std::cout << std::endl; - return MENU_PROCESS_ERROR; - } - - std::string dir = input.substr(0, input.size() - 1); // remove \n. - - // check that the directory exists! - if (!RsDirUtil::checkDirectory(dir)) - { - std::cout << "MenuOpSearchNew::process_lines() ERROR Directory doesn't exist"; - std::cout << std::endl; - return MENU_PROCESS_ERROR; - } - - // extract top level as the virtual name. - std::string topdir = RsDirUtil::getTopDir(input); - if (topdir.size() < 1) - { - std::cout << "MenuOpSearchNew::process_lines() ERROR TopDir is invalid"; - std::cout << std::endl; - return MENU_PROCESS_ERROR; - } - - SharedDirInfo shareddir; - shareddir.filename = dir; - shareddir.virtualname = topdir; - shareddir.shareflags = FileStorageFlags(0x0); - - if (!rsFiles->addSharedDirectory(shareddir)) - { - std::cout << "MenuOpSearchNew::process_lines() ERROR Adding SharedDir"; - std::cout << std::endl; - return MENU_PROCESS_ERROR; - } - - return MENU_PROCESS_DONE; -} diff --git a/retroshare-nogui/src/menu/menus.h b/retroshare-nogui/src/menu/menus.h deleted file mode 100644 index d4c2486c1..000000000 --- a/retroshare-nogui/src/menu/menus.h +++ /dev/null @@ -1,346 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "menu/menu.h" -#include -#include "notifytxt.h" /* needed to access search results */ -#include - -Menu *CreateMenuStructure(NotifyTxt *notify); - -/************ - * Friends Menu - */ - -class MenuListFriends: public MenuList -{ - -public: - MenuListFriends() :MenuList("Friends") { return; } - - virtual uint32_t op(); - int getEntryDesc(int idx, std::string &desc); - -protected: - MenuListFriends(std::string name) :MenuList(name) { return; } // For Network. -}; - - -class MenuListNetwork: public MenuListFriends -{ - -public: - MenuListNetwork() :MenuListFriends("Network") { return; } - - virtual uint32_t op(); -}; - - -class MenuOpFriendsAdd: public MenuOpBasicKey -{ - public: - - MenuOpFriendsAdd() :MenuOpBasicKey("Add") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - -class MenuOpFriendsViewDetail: public MenuOpBasicKey -{ - public: - - MenuOpFriendsViewDetail() :MenuOpBasicKey("View") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - -class MenuOpFriendsRemove: public MenuOpBasicKey -{ - public: - - MenuOpFriendsRemove() :MenuOpBasicKey("Remove") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - -class MenuOpFriendsChat: public MenuOpBasicKey -{ - public: - - MenuOpFriendsChat() :MenuOpBasicKey("Chat") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - - -/************ - * Transfer Menu - */ - - -class MenuListTransfer: public MenuList -{ - public: - - MenuListTransfer() :MenuList("Downloads") { return; } - virtual uint32_t op(); - int getEntryDesc(int idx, std::string &desc); -}; - - -class MenuOpTransferStop: public MenuOpBasicKey -{ - public: - - MenuOpTransferStop() :MenuOpBasicKey("Stop") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - - -class MenuOpTransferCancel: public MenuOpBasicKey -{ - public: - - MenuOpTransferCancel() :MenuOpBasicKey("Cancel") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - - -/************ - * Search Menu - */ - - -class MenuListSearch: public MenuList -{ - public: - - MenuListSearch(NotifyTxt *notify) - :MenuList("Search"), mNotify(notify) { return; } - - virtual uint32_t op(); - int getEntryDesc(int idx, std::string &desc); - - /* specific Search Functions */ - int getCurrentSearchId(uint32_t &id); - int storeSearch(uint32_t searchId, std::string match_string); - int removeSearch(std::string strSearchId); - -private: - std::map mSearchIds; - std::map mSearchTerms; - NotifyTxt *mNotify; -}; - - -class MenuOpSearchNew: public MenuOpLineInput -{ - public: - - MenuOpSearchNew(NotifyTxt *notify) - :MenuOpLineInput("New"), mNotify(notify) { return; } - virtual uint32_t process_lines(std::string input); - virtual uint32_t drawPage(uint32_t drawFlags, std::string &buffer); - -private: - NotifyTxt *mNotify; -}; - - -class MenuOpSearchDelete: public MenuOpBasicKey -{ - public: - - MenuOpSearchDelete() :MenuOpBasicKey("Delete") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - -class MenuListSearchList: public MenuList -{ - public: - - MenuListSearchList(NotifyTxt *notify) - :MenuList("Results"), mNotify(notify) { return; } - virtual uint32_t op(); - uint32_t refresh(); - int getEntryDesc(int idx, std::string &desc); - int downloadSelected(); - - private: - NotifyTxt *mNotify; - std::list mSearchResults; // local cache for consistency. -}; - - -class MenuOpSearchListDownload: public MenuOpBasicKey -{ - public: - - MenuOpSearchListDownload() :MenuOpBasicKey("Download") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - - -/************ - * Forums Menu - */ - - - -class MenuListForums: public MenuList -{ - public: - - MenuListForums() :MenuList("Forums SubMenu") { return; } - virtual uint32_t op(); - int getEntryDesc(int idx, std::string &desc); -}; - - -class MenuOpForumDetails: public MenuOpBasicKey -{ - public: - - MenuOpForumDetails() :MenuOpBasicKey("Show Forum Details") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - -class MenuOpForumSubscribe: public MenuOpBasicKey -{ - public: - - MenuOpForumSubscribe() :MenuOpBasicKey("Subscribe To Forum") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - -class MenuOpForumUnsubscribe: public MenuOpBasicKey -{ - public: - - MenuOpForumUnsubscribe() :MenuOpBasicKey("Unsubscribe To Forum") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - -class MenuOpForumCreate: public MenuOpBasicKey -{ - public: - - MenuOpForumCreate() :MenuOpBasicKey("Create Forum") { return; } - virtual uint32_t op_basic(std::string hash); -}; - - - -class MenuListForumMsgs: public MenuList -{ - public: - - MenuListForumMsgs() :MenuList("List Forum Msgs") { return; } - virtual uint32_t op(); - int getEntryDesc(int idx, std::string &desc); -}; - -class MenuOpForumMsgView: public MenuOpTwoKeys -{ - public: - - MenuOpForumMsgView() :MenuOpTwoKeys("View Message") { return; } - virtual uint32_t op_twokeys(std::string parentkey, std::string key); -}; - - -class MenuOpForumMsgReply: public MenuOpTwoKeys -{ - public: - - MenuOpForumMsgReply() :MenuOpTwoKeys("Reply to Message") { return; } - virtual uint32_t op_twokeys(std::string parentkey, std::string key); -}; - - -class MenuOpForumMsgWrite: public MenuOpTwoKeys -{ - public: - - MenuOpForumMsgWrite() :MenuOpTwoKeys("Write Message") { return; } - virtual uint32_t op_twokeys(std::string parentkey, std::string key); -}; - -/************ - * Shared folders Menu - */ - - -class MenuListShared: public MenuList -{ - public: - - MenuListShared() :MenuList("My Shared Directories") { return; } - virtual uint32_t op(); - int getEntryDesc(int idx, std::string &desc); - int unshareSelected(); - int toggleFlagSelected(FileStorageFlags shareflags); -}; - - - -class MenuListSharedUnshare: public MenuOpBasicKey -{ - public: - - MenuListSharedUnshare() :MenuOpBasicKey("Stop Sharing Selected") { return; } - virtual uint32_t op_basic(std::string key); -}; - -class MenuListSharedTogglePublic: public MenuOpBasicKey -{ - public: - - MenuListSharedTogglePublic() :MenuOpBasicKey("Enable/Disable Networkwide Sharing") { return; } - virtual uint32_t op_basic(std::string key); -}; - -class MenuListSharedToggleBrowsable: public MenuOpBasicKey -{ - public: - - MenuListSharedToggleBrowsable() :MenuOpBasicKey("Enable/Disable Browsing Of Selected") { return; } - virtual uint32_t op_basic(std::string key); -}; - -class MenuListSharedAddShare: public MenuOpLineInput -{ - public: - - MenuListSharedAddShare() :MenuOpLineInput("Add new Share") { return; } - virtual uint32_t process_lines(std::string input); - virtual uint32_t drawPage(uint32_t drawFlags, std::string &buffer); -}; - diff --git a/retroshare-nogui/src/menu/menutest.h b/retroshare-nogui/src/menu/menutest.h deleted file mode 100644 index 94c58d4ee..000000000 --- a/retroshare-nogui/src/menu/menutest.h +++ /dev/null @@ -1,94 +0,0 @@ - -#include -#include "menu/menu.h" - -class MenuTest -{ -public: - MenuTest(MenuInterface *i, std::istream &in, std::ostream &out) - :mMenus(i), mIn(in), mOut(out) - { - return; - } - -int tick() - { - int c = mIn.get(); - uint8_t key = (uint8_t) c; - uint32_t drawFlags = 0; - std::string buffer; - mMenus->process(key, drawFlags, buffer); - - return 1; - } - -private: - - - MenuInterface *mMenus; - std::istream &mIn; - std::ostream &mOut; -}; - -#include -#include - -#include "rstermserver.h" - -class RsConsole -{ - public: - - RsConsole(RsTermServer *s, int infd, int outfd) - :mServer(s), mIn(infd), mOut(outfd) - { - const int fcflags = fcntl(mIn,F_GETFL); - if (fcflags < 0) - { - std::cerr << "RsConsole() ERROR getting fcntl FLAGS"; - std::cerr << std::endl; - exit(1); - } - if (fcntl(mIn,F_SETFL,fcflags | O_NONBLOCK) <0) - { - std::cerr << "RsConsole() ERROR setting fcntl FLAGS"; - std::cerr << std::endl; - exit(1); - } - } - - int tick() - { - char buf; - std::string output; - int size = read(mIn, &buf, 1); - - bool haveInput = (size > 0); - - int rt = mServer->tick(haveInput, buf, output); - - if (output.size() > 0) - { - write(mOut, output.c_str(), output.size()); - } - - if (rt < 0) - { - std::cerr << "Exit Request"; - exit(1); - } - - if (!haveInput) - { - return 0; - } - - return 1; - } - -private: - - RsTermServer *mServer; - int mIn, mOut; -}; - diff --git a/retroshare-nogui/src/menu/stdiocomms.cc b/retroshare-nogui/src/menu/stdiocomms.cc deleted file mode 100644 index 132b8e89d..000000000 --- a/retroshare-nogui/src/menu/stdiocomms.cc +++ /dev/null @@ -1,183 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "menu/stdiocomms.h" - -#include -#include -#include -#include -#include - - -StdioComms::StdioComms(int infd, int outfd) - :mIn(infd), mOut(outfd) -{ -// fcntl not available on Windows, search alternative when needed -#ifndef WINDOWS_SYS -#if 1 -// THIS Code is strange... -// It seems to mess up stderr. -// But if you redirect it -> is comes out fine. Very Weird. -// HELP!!! - - std::cerr << "StdioComms() STDERR msg 0"; - std::cerr << std::endl; - const int fcflags = fcntl(mIn,F_GETFL); - if (fcflags < 0) - { - std::cerr << "StdioComms() ERROR getting fcntl FLAGS"; - std::cerr << std::endl; - exit(1); - } - if (fcntl(mIn,F_SETFL,fcflags | O_NONBLOCK) <0) - { - std::cerr << "StdioComms() ERROR setting fcntl FLAGS"; - std::cerr << std::endl; - exit(1); - } - std::cerr << "StdioComms() STDERR msg 1"; - std::cerr << std::endl; -#endif -#endif // WINDOWS_SYS -} - - -int StdioComms::isOkay() -{ - return 1; -} - - -int StdioComms::active_channels(std::list &chan_ids) -{ - if (isOkay()) - { - chan_ids.push_back(1); // only one possible here (stdin/stdout) - } - return 1; -} - -int StdioComms::error(uint32_t chan_id, std::string msg) -{ - std::cerr << "StdioComms::error(" << msg << ")"; - std::cerr << std::endl; - return 1; -} - - - -int StdioComms::recv_ready(uint32_t chan_id) -{ - /* should be proper poll / select! - but we don't use this at the moment */ - return 1; -} - - -int StdioComms::recv(uint32_t chan_id, uint8_t *buffer, int bytes) -{ - int size = read(mIn, buffer, bytes); - std::cerr << "StdioComms::recv() returned: " << size; - std::cerr << std::endl; - - if (size == -1) - { - std::cerr << "StdioComms::recv() ERROR: " << errno; - std::cerr << std::endl; - if (errno == EAGAIN) - { - return 0; // OKAY; - } - } - - return size; -} - - -int StdioComms::recv(uint32_t chan_id, std::string &buffer, int bytes) -{ - uint8_t tmpbuffer[bytes]; - int size = read(mIn, tmpbuffer, bytes); - for(int i = 0; i < size; i++) - { - buffer += tmpbuffer[i]; - } - return size; -} - - // these make it easier... -int StdioComms::recv_blocking(uint32_t chan_id, uint8_t *buffer, int bytes) -{ - int totalread = 0; - while(totalread < bytes) - { - int size = read(mIn, &(buffer[totalread]), bytes - totalread); - if (size < 0) - { - if (totalread) - break; // partial read. - else - return size; // error. - } - totalread += size; - usleep(1000); // minisleep - so we don't 100% CPU. - std::cerr << "StdioComms::recv_blocking() read so far: " << size; - std::cerr << " / " << totalread; - std::cerr << std::endl; - } - return totalread; -} - - -int StdioComms::recv_blocking(uint32_t chan_id, std::string &buffer, int bytes) -{ - uint8_t tmpbuffer[bytes]; - int size = recv_blocking(chan_id, tmpbuffer, bytes); - - if (size < 0) - return size; // error. - - for(int i = 0; i < size; i++) - buffer += tmpbuffer[i]; - - return size; -} - - -int StdioComms::send(uint32_t chan_id, uint8_t *buffer, int bytes) -{ - write(mOut, buffer, bytes); - return bytes; -} - - -int StdioComms::send(uint32_t chan_id, const std::string &output) -{ - if (output.size() > 0) - { - write(mOut, output.c_str(), output.size()); - } - return output.size(); -} - - diff --git a/retroshare-nogui/src/menu/stdiocomms.h b/retroshare-nogui/src/menu/stdiocomms.h deleted file mode 100644 index bc559af4c..000000000 --- a/retroshare-nogui/src/menu/stdiocomms.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - - -#ifndef RS_STDIO_COMMS_H -#define RS_STDIO_COMMS_H - -#include "rpcsystem.h" - - - -class StdioComms: public RpcComms -{ -public: - StdioComms(int infd, int outfd); - - virtual int isOkay(); - virtual int error(uint32_t chan_id, std::string msg); - - virtual int active_channels(std::list &chan_ids); - - virtual int recv_ready(uint32_t chan_id); - virtual int recv(uint32_t chan_id, uint8_t *buffer, int bytes); - virtual int recv(uint32_t chan_id, std::string &buffer, int bytes); - - // these make it easier... - virtual int recv_blocking(uint32_t chan_id, uint8_t *buffer, int bytes); - virtual int recv_blocking(uint32_t chan_id, std::string &buffer, int bytes); - - virtual int send(uint32_t chan_id, uint8_t *buffer, int bytes); - virtual int send(uint32_t chan_id, const std::string &buffer); - -private: - int mIn, mOut; -}; - -#endif - diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index c66ae37cd..358cb65f4 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -4,7 +4,6 @@ TEMPLATE = app TARGET = RetroShare06-nogui CONFIG += bitdht #CONFIG += introserver -#CONFIG += sshserver CONFIG -= qt xml gui CONFIG += link_prl @@ -84,8 +83,6 @@ win32 { LIBS += -luuid -lole32 -liphlpapi -lcrypt32 LIBS += -lole32 -lwinmm - PROTOCPATH=$$BIN_DIR - RC_FILE = resources/retroshare_win.rc DEFINES *= WINDOWS_SYS _USE_32BIT_TIME_T @@ -97,11 +94,11 @@ win32 { ##################################### MacOS ###################################### macx { - # ENABLE THIS OPTION FOR Univeral Binary BUILD. - # CONFIG += ppc x86 + # ENABLE THIS OPTION FOR Univeral Binary BUILD. + # CONFIG += ppc x86 LIBS += -Wl,-search_paths_first - LIBS += -lssl -lcrypto -lz + LIBS += -lssl -lcrypto -lz for(lib, LIB_DIR):exists($$lib/libminiupnpc.a){ LIBS += $$lib/libminiupnpc.a} LIBS += -framework CoreFoundation LIBS += -framework Security @@ -111,13 +108,7 @@ macx { DEPENDPATH += . $$INC_DIR INCLUDEPATH += . $$INC_DIR - sshserver { - LIBS += -L../../../lib - #LIBS += -L../../../lib/libssh-0.6.0 - } - - QMAKE_CXXFLAGS *= -Dfseeko64=fseeko -Dftello64=ftello -Dstat64=stat -Dstatvfs64=statvfs -Dfopen64=fopen - + QMAKE_CXXFLAGS *= -Dfseeko64=fseeko -Dftello64=ftello -Dstat64=stat -Dstatvfs64=statvfs -Dfopen64=fopen } ##################################### FreeBSD ###################################### @@ -193,173 +184,3 @@ libresapihttpserver { SOURCES += \ TerminalApiClient.cpp } - -sshserver { - - # This Requires libssh-0.5.* to compile. - # Please use this path below. - # (You can modify it locally if required - but dont commit it!) - - #LIBSSH_DIR = $PWD/../../../lib/libssh-0.5.2 - LIBSSH_DIR = $PWD/../../../libssh-0.6.0rc1 - - # - # Use the following commend to generate a Server RSA Key. - # Key should be in current directory - when run/ - # ssh-keygen -t rsa -f rs_ssh_host_rsa_key - # - # You can connect from a standard ssh, eg: ssh -p 7022 127.0.0.1 - # - # The Menu system is available from the command-line (-T) and SSH (-S) - # if it get covered by debug gunk, just press to refresh. - # - # ./retroshare-nogui -h provides some more instructions. - # - - win32 { - DEFINES *= LIBSSH_STATIC - } - - DEPENDPATH += $$LIBSSH_DIR/include/ - INCLUDEPATH += $$LIBSSH_DIR/include/ - - win32 { - LIBS += -lssh - LIBS += -lssh_threads - } else { - SSH_OK = $$system(pkg-config --atleast-version 0.5.4 libssh && echo yes) - isEmpty(SSH_OK) { - exists($$LIBSSH_DIR/build/src/libssh.a):exists($$LIBSSH_DIR/build/src/threads/libssh_threads.a) { - LIBS += $$LIBSSH_DIR/build/src/libssh.a - LIBS += $$LIBSSH_DIR/build/src/threads/libssh_threads.a - } - else { - ! exists($$LIBSSH_DIR/build/src/libssh.a):message($$LIBSSH_DIR/build/src/libssh.a does not exist) - ! exists($$LIBSSH_DIR/build/src/threads/libssh_threads.a):message($$LIBSSH_DIR/build/src/threads/libssh_threads.a does not exist) - message(You need to download and compile libssh) - message(See http://sourceforge.net/p/retroshare/code/6163/tree/trunk/) - } - } else { - LIBS += -lssh - LIBS += -lssh_threads - } - } - - HEADERS += ssh/rssshd.h - SOURCES += ssh/rssshd.cc - - # For the Menu System - HEADERS += menu/menu.h \ - menu/menus.h \ - menu/stdiocomms.h \ - - SOURCES += menu/menu.cc \ - menu/menus.cc \ - menu/stdiocomms.cc \ - - # For the RPC System - HEADERS += rpc/rpc.h \ - rpc/rpcserver.h \ - rpc/rpcsetup.h \ - rpc/rpcecho.h \ - rpcsystem.h \ - - SOURCES += rpc/rpc.cc \ - rpc/rpcserver.cc \ - rpc/rpcsetup.cc \ - rpc/rpcecho.cc \ - - # Actual protocol files to go here... - #HEADERS += rpc/proto/rpcecho.h \ - - #SOURCES += rpc/proto/rpcecho.cc \ - - DEFINES *= RS_SSH_SERVER - - # Include Protobuf classes. - CONFIG += protorpc -} - -protorpc { - # Proto Services - PROTOS = core.proto peers.proto system.proto chat.proto search.proto files.proto stream.proto - DESTPATH = $$PWD/rpc/proto/gencc - PROTOPATH = $$PWD/../../rsctrl/src/definition - CMD = echo Building protobuf files - for(pf, PROTOS):CMD += && $${PROTOCPATH}protoc --cpp_out=$${DESTPATH} --proto_path=$${PROTOPATH} $${PROTOPATH}/$${pf} - protobuf_gen.commands = $${CMD} - QMAKE_EXTRA_TARGETS += protobuf_gen - PRE_TARGETDEPS += protobuf_gen - - HEADERS += rpc/proto/rpcprotopeers.h \ - rpc/proto/rpcprotosystem.h \ - rpc/proto/rpcprotochat.h \ - rpc/proto/rpcprotosearch.h \ - rpc/proto/rpcprotofiles.h \ - rpc/proto/rpcprotostream.h \ - rpc/proto/rpcprotoutils.h \ - - SOURCES += rpc/proto/rpcprotopeers.cc \ - rpc/proto/rpcprotosystem.cc \ - rpc/proto/rpcprotochat.cc \ - rpc/proto/rpcprotosearch.cc \ - rpc/proto/rpcprotofiles.cc \ - rpc/proto/rpcprotostream.cc \ - rpc/proto/rpcprotoutils.cc \ - - # Offical Generated Code (protobuf 2.4.1) - HEADERS += rpc/proto/gencc/core.pb.h \ - rpc/proto/gencc/peers.pb.h \ - rpc/proto/gencc/system.pb.h \ - rpc/proto/gencc/chat.pb.h \ - rpc/proto/gencc/search.pb.h \ - rpc/proto/gencc/files.pb.h \ - rpc/proto/gencc/stream.pb.h \ - - SOURCES += rpc/proto/gencc/core.pb.cc \ - rpc/proto/gencc/peers.pb.cc \ - rpc/proto/gencc/system.pb.cc \ - rpc/proto/gencc/chat.pb.cc \ - rpc/proto/gencc/search.pb.cc \ - rpc/proto/gencc/files.pb.cc \ - rpc/proto/gencc/stream.pb.cc \ - - # Generated ProtoBuf Code the RPC System - # If you are developing, or have a different version of protobuf - # you can use these ones (run make inside rsctrl/src/ to generate) - #HEADERS += ../../rsctrl/src/gencc/core.pb.h \ - # ../../rsctrl/src/gencc/peers.pb.h \ - # ../../rsctrl/src/gencc/system.pb.h \ - # ../../rsctrl/src/gencc/chat.pb.h \ - # ../../rsctrl/src/gencc/search.pb.h \ - # ../../rsctrl/src/gencc/files.pb.h \ - # ../../rsctrl/src/gencc/stream.pb.h \ - - #SOURCES += ../../rsctrl/src/gencc/core.pb.cc \ - # ../../rsctrl/src/gencc/peers.pb.cc \ - # ../../rsctrl/src/gencc/system.pb.cc \ - # ../../rsctrl/src/gencc/chat.pb.cc \ - # ../../rsctrl/src/gencc/search.pb.cc \ - # ../../rsctrl/src/gencc/files.pb.cc \ - # ../../rsctrl/src/gencc/stream.pb.cc \ - - DEPENDPATH *= rpc/proto/gencc - INCLUDEPATH *= rpc/proto/gencc - - !win32 { - # unrecognized option - QMAKE_CFLAGS += -pthread - QMAKE_CXXFLAGS += -pthread - } - LIBS += -lprotobuf -lpthread - - win32 { - DEPENDPATH += $$LIBS_DIR/include/protobuf - INCLUDEPATH += $$LIBS_DIR/include/protobuf - } - - macx { - PROTOPATH = ../../../protobuf-2.4.1 - INCLUDEPATH += $${PROTOPATH}/src - } -} diff --git a/retroshare-nogui/src/retroshare.cc b/retroshare-nogui/src/retroshare.cc index 75b9e17d8..011e03873 100644 --- a/retroshare-nogui/src/retroshare.cc +++ b/retroshare-nogui/src/retroshare.cc @@ -40,20 +40,6 @@ #include "introserver.h" #endif -#ifdef RS_SSH_SERVER -#include "ssh/rssshd.h" - -#include "menu/menus.h" -#include "menu/stdiocomms.h" - -#include "rpc/rpcsetup.h" - -// NASTY GLOBAL VARIABLE HACK - NEED TO THINK OF A BETTER SYSTEM. -#include "rpc/proto/rpcprotosystem.h" - -void generatePasswordHash() ; -#endif - #ifdef ENABLE_WEBUI #include #include "api/ApiServerMHD.h" @@ -151,123 +137,6 @@ int main(int argc, char **argv) **/ bool strictCheck = true; - -#ifdef RS_SSH_SERVER - /* parse commandline for additional nogui options */ - - int c; - // libretroshare's getopt str - so don't use any of these: "hesamui:p:c:w:l:d:U:r:R:" - // need options for - // enable SSH. (-S) - // set user/password for SSH. -L "user:pwdhash" - // accept RSA Key Auth. -K "RsaPubKeyFile" - // Terminal mode. -T - bool enableRpc = false; - bool enableSsh = false; - bool enableSshHtml = false; - bool enableTerminal = false; - bool enableSshRsa = false; - bool genPwdHash = false; - std::string sshUser = "user"; - std::string sshPwdHash = ""; - std::string sshRsaFile = ""; - - uint16_t extPort = 0; - uint16_t sshPort = 7022; - bool extPortSet = false; - bool displayRPCInfo = false ; - - argstream as(argc,argv) ; - - as >> option('X',"enable-ssh" ,enableSsh ,"Enable SSH" ) - >> option('T',"enable-terminal",enableTerminal ,"Enable terminal interface." ) - >> option('C',"enable-rpc" ,enableRpc ,"Enable RPC protocol. To be used with e.g. -X (SSH).") - >> option('G',"gen-password" ,genPwdHash ,"Generate password hash (to supply to option -P)") -#if 0 - >> option('H',"enable-ssh-html",enableSshHtml ,"Enable SSH html." ) -#endif - >> parameter('S',"ssh-port" ,sshPort ,"port" ,"SSH port to contact the interface.",false) - >> parameter('E',"ext-port" ,extPort ,"port" ,"Specify Alternative External Port (provided to Clients)",false) - >> parameter('L',"ssh-user" ,sshUser ,"name" ,"Ssh login user",false) - >> parameter('P',"ssh-p-hash" ,sshPwdHash ,"hash" ,"Ssh login password hash (Generated by retroshare-nogui -G)",false) - >> parameter('K',"ssh-key-file" ,sshRsaFile ,"RSA key file", "RSA key file for SSH login (not yet implemented).",false )// NOT FINISHED YET. - -#ifdef __APPLE__ - >> help('h',"help","Display this Help"); -#else - >> help() ; -#endif - // Normally argstream would handle this by itself, if we called - // as.defaultErrorHandling() ; - // - // but we have other parameter handling functions after, so we don't want to quit if help is requested. - // - if (as.helpRequested()) - { - std::cerr << "\nSpecific Help Options:" << std::endl; - std::cerr << as.usage() << std::endl; - std::cerr << "\t To setup rs-nogui as a SSH Server is a three step process: " << std::endl; - std::cerr << "\t 1) Generate a RSA keypair in the current directory: \"ssh-keygen -t rsa -f rs_ssh_host_rsa_key\" " << std::endl; - std::cerr << "\t 2) Generate a password hash for the RPC login: \"./retroshare-nogui -G\" " << std::endl; - std::cerr << "\t 3) Launch the RS with remote control enabled: \"./retroshare-nogui -X/-T [-C] -S [port] -L -P \" " << std::endl; - - std::cerr << "\nAdditional options: \n" << std::endl; - } - if (!as.isOk()) - { - std::cerr << as.errorLog(); - return 1; - } - - if (genPwdHash) - { - generatePasswordHash() ; - return 0 ; - } - - /* enforce conditions */ - if ((!sshRsaFile.empty() || !sshPwdHash.empty()) && (!enableSsh)) - { - std::cerr << "ERROR: SSH Server (-X) must be enabled to specify SSH Pwd (-P) or SSH RSA (-K)"; - std::cerr << std::endl; - return 1 ; - } - - if (enableSsh && (!enableSshRsa) && sshPwdHash.empty()) - { - std::cerr << "ERROR: One of (or both) SSH Pwd (-P) and SSH RSA (-K) must be specified with SSH Server (-X)"; - std::cerr << std::endl; - return 1 ; - } - - if (enableRpc && (!enableSsh)) - { - std::cerr << "ERROR: RPC Mode (-C) requires SSH Server (-X) enabled"; - std::cerr << std::endl; - return 1 ; - } - - - /* parse -S, -L & -K parameters */ - if (enableSshRsa) - { - /* check the file exists */ - /* TODO */ - - } - - if (enableSsh) - { - /* try parse it */ - /* TODO */ - - } - -#else - std::cerr << "\nRetroshare command line interface." << std::endl; -#endif - - RsInit::InitRsConfig(); int initResult = RsInit::InitRetroShare(argc, argv, strictCheck); @@ -318,38 +187,6 @@ int main(int argc, char **argv) return 1; } -#ifdef RS_SSH_SERVER - // Says it must be called before all the threads are launched! */ - // NB: this port number is not currently used. - RsSshd *ssh = NULL; - - if (enableSsh) - { - std::ostringstream os ; - os << sshPort ; - ssh = RsSshd::InitRsSshd(os.str(), "rs_ssh_host_rsa_key"); - - // TODO Parse Option - if (enableSshRsa) - { - //ssh->adduser("anrsuser", "test"); - } - - if (!sshPwdHash.empty()) - { - ssh->adduserpwdhash(sshUser, sshPwdHash); - } - - if (!extPortSet) - { - extPort = sshPort; - } - - // NASTY GLOBAL VARIABLE HACK - NEED TO THINK OF A BETTER SYSTEM. - RpcProtoSystem::mExtPort = extPort; - } -#endif - /* Start-up libretroshare server threads */ RsControl::instance() -> StartupRetroShare(); @@ -357,47 +194,6 @@ int main(int argc, char **argv) RsIntroServer rsIS; #endif -#ifdef RS_SSH_SERVER - uint32_t baseDrawFlags = 0; - if (enableSshHtml) - { - baseDrawFlags = MENU_DRAW_FLAGS_HTML; - } - - if (enableSsh) - { - if (enableRpc) - { - /* Build RPC Server */ - RpcMediator *med = CreateRpcSystem(ssh, notify); - ssh->setRpcSystem(med); - ssh->setSleepPeriods(0.01, 0.1); - } - else - { - /* create menu system for SSH */ - Menu *baseMenu = CreateMenuStructure(notify); - MenuInterface *menuInterface = new MenuInterface(ssh, baseMenu, baseDrawFlags | MENU_DRAW_FLAGS_ECHO); - ssh->setRpcSystem(menuInterface); - ssh->setSleepPeriods(0.05, 0.5); - } - - ssh->start(); - } - - MenuInterface *terminalMenu = NULL; - if (enableTerminal) - { - /* Terminal Version */ - RpcComms *stdioComms = new StdioComms(fileno(stdin), fileno(stdout)); - Menu *baseMenu = CreateMenuStructure(notify); - terminalMenu = new MenuInterface(stdioComms, baseMenu, baseDrawFlags | MENU_DRAW_FLAGS_NOQUIT); - //menuTerminal = new RsConsole(menuInterface, fileno(stdin), fileno(stdout)); - } - - -#endif - /* pass control to the GUI */ while(1) { @@ -408,13 +204,6 @@ int main(int argc, char **argv) #endif int rt = 0; -#ifdef RS_SSH_SERVER - if (terminalMenu) - { - rt = terminalMenu->tick(); - } -#endif - // If we have a MenuTerminal ... // only want to sleep if there is no input. (rt == 0). if (rt == 0) @@ -430,73 +219,4 @@ int main(int argc, char **argv) } return 1; -} - -#ifdef RS_SSH_SERVER -void generatePasswordHash() -{ - std::string saltBin; - std::string pwdHashRadix64; - std::string sshPwdForHash = ""; - - std::string passwd1,passwd2 ; - bool cancel ; - - if(!NotifyTxt().askForPassword("","Type your password (at least 8 chars) : ",false,passwd1,cancel)) exit(1) ; - - if(passwd1.length() < 8) - { - std::cerr << "Password must be at least 8 characters long." << std::endl; - exit(1); - } - - if(!NotifyTxt().askForPassword("","Type your password (checking) : ",false,passwd2,cancel)) exit(1) ; - - if(passwd1 != passwd2) - { - std::cerr << "Passwords differ. Please retry." << std::endl; - exit(1); - } - - sshPwdForHash = passwd1 ; - - //std::cerr << "Chosen Password : " << sshPwdForHash; - std::cerr << std::endl; - - GenerateSalt(saltBin); - if (!GeneratePasswordHash(saltBin, sshPwdForHash, pwdHashRadix64)) - { - std::cerr << "Error Generating Password Hash, password probably too short"; - std::cerr << pwdHashRadix64; - std::cerr << std::endl; - exit(1); - } - - std::cout << "Generated Password Hash for rs-nogui: "; - std::cout << pwdHashRadix64; - std::cout << std::endl; - std::cout << std::endl; - - /* checking match */ - if (CheckPasswordHash(pwdHashRadix64, sshPwdForHash)) - { - std::cerr << "Passed Check Okay!"; - std::cerr << std::endl; - } - else - { - std::cerr << "ERROR: Failed CheckPassword!"; - std::cerr << std::endl; - exit(1); - } - - - std::cerr << "Usage:"; - std::cerr << std::endl; - std::cerr << " - for SSH access: ./retroshare-nogui -X -S [port] -L -P " << pwdHashRadix64; - std::cerr << std::endl; - std::cerr << " - for RPC access: ./retroshare-nogui -C -X -S [port] -L -P " << pwdHashRadix64; - std::cerr << std::endl; -} -#endif - +} diff --git a/retroshare-nogui/src/rpc/proto/rpcprotochat.cc b/retroshare-nogui/src/rpc/proto/rpcprotochat.cc deleted file mode 100644 index 388685c5d..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotochat.cc +++ /dev/null @@ -1,1477 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/proto/rpcprotochat.h" -#include "rpc/proto/gencc/chat.pb.h" - -#include -#include -#include - -#include "util/rsstring.h" - -#include - -#include -#include - -#include - - -// Helper Functions -> maybe move to libretroshare/utils ?? -bool convertUTF8toWString(const std::string &msg_utf8, std::wstring &msg_wstr); -bool convertWStringToUTF8(const std::wstring &msg_wstr, std::string &msg_utf8); - -bool convertStringToLobbyId(const std::string &chat_id, ChatLobbyId &lobby_id); -bool convertLobbyIdToString(const ChatLobbyId &lobby_id, std::string &chat_id); - -bool fillLobbyInfoFromChatLobbyInfo(const ChatLobbyInfo &cfi, rsctrl::chat::ChatLobbyInfo *lobby); -bool fillLobbyInfoFromVisibleChatLobbyRecord(const VisibleChatLobbyRecord &pclr, rsctrl::chat::ChatLobbyInfo *lobby); -bool fillLobbyInfoFromChatLobbyInvite(const ChatLobbyInvite &cli, rsctrl::chat::ChatLobbyInfo *lobby); - -bool fillChatMessageFromHistoryMsg(const HistoryMsg &histmsg, rsctrl::chat::ChatMessage *rpcmsg); - -bool createQueuedEventSendMsg(const ChatInfo &chatinfo, rsctrl::chat::ChatType ctype, - std::string chat_id, const RpcEventRegister &ereg, RpcQueuedMsg &qmsg); - - -RpcProtoChat::RpcProtoChat(uint32_t serviceId) - :RpcQueueService(serviceId) -{ - return; -} - -//RpcProtoChat::msgsAccepted(std::list &msgIds); /* not used at the moment */ - -int RpcProtoChat::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - /* check the msgId */ - uint8_t topbyte = getRpcMsgIdExtension(msg_id); - uint16_t service = getRpcMsgIdService(msg_id); - uint8_t submsg = getRpcMsgIdSubMsg(msg_id); - bool isResponse = isRpcMsgIdResponse(msg_id); - - - std::cerr << "RpcProtoChat::processMsg() topbyte: " << (int32_t) topbyte; - std::cerr << " service: " << (int32_t) service << " submsg: " << (int32_t) submsg; - std::cerr << std::endl; - - if (isResponse) - { - std::cerr << "RpcProtoChat::processMsg() isResponse() - not processing"; - std::cerr << std::endl; - return 0; - } - - if (topbyte != (uint8_t) rsctrl::core::CORE) - { - std::cerr << "RpcProtoChat::processMsg() Extension Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (service != (uint16_t) rsctrl::core::CHAT) - { - std::cerr << "RpcProtoChat::processMsg() Service Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (!rsctrl::chat::RequestMsgIds_IsValid(submsg)) - { - std::cerr << "RpcProtoChat::processMsg() SubMsg Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - switch(submsg) - { - case rsctrl::chat::MsgId_RequestChatLobbies: - processReqChatLobbies(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::chat::MsgId_RequestCreateLobby: - processReqCreateLobby(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::chat::MsgId_RequestJoinOrLeaveLobby: - processReqJoinOrLeaveLobby(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::chat::MsgId_RequestSetLobbyNickname: - processReqSetLobbyNickname(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::chat::MsgId_RequestRegisterEvents: - processReqRegisterEvents(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::chat::MsgId_RequestSendMessage: - processReqSendMessage(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::chat::MsgId_RequestChatHistory: - processReqChatHistory(chan_id, msg_id, req_id, msg); - break; - - - default: - std::cerr << "RpcProtoChat::processMsg() ERROR should never get here"; - std::cerr << std::endl; - return 0; - } - - /* must have matched id to get here */ - return 1; -} - - -// Registrations. -//#define REGISTRATION_EVENT_CHAT 1 - -int RpcProtoChat::processReqChatLobbies(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoChat::processReqChatLobbies()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::chat::RequestChatLobbies req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoChat::processReqChatLobbies() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::chat::ResponseChatLobbies resp; - bool success = true; - std::string errorMsg; - - // set these flags from request. - bool fetch_chatlobbylist = true; - bool fetch_invites = true; - bool fetch_visiblelobbies = true; - - switch(req.lobby_set()) - { - case rsctrl::chat::RequestChatLobbies::LOBBYSET_ALL: - std::cerr << "RpcProtoChat::processReqChatLobbies() LOBBYSET_ALL"; - std::cerr << std::endl; - fetch_chatlobbylist = true; - fetch_invites = true; - fetch_visiblelobbies = true; - break; - case rsctrl::chat::RequestChatLobbies::LOBBYSET_JOINED: - std::cerr << "RpcProtoChat::processReqChatLobbies() LOBBYSET_JOINED"; - std::cerr << std::endl; - fetch_chatlobbylist = true; - fetch_invites = false; - fetch_visiblelobbies = false; - break; - case rsctrl::chat::RequestChatLobbies::LOBBYSET_INVITED: - std::cerr << "RpcProtoChat::processReqChatLobbies() LOBBYSET_INVITED"; - std::cerr << std::endl; - fetch_chatlobbylist = false; - fetch_invites = true; - fetch_visiblelobbies = false; - break; - case rsctrl::chat::RequestChatLobbies::LOBBYSET_VISIBLE: - std::cerr << "RpcProtoChat::processReqChatLobbies() LOBBYSET_VISIBLE"; - std::cerr << std::endl; - fetch_chatlobbylist = false; - fetch_invites = false; - fetch_visiblelobbies = true; - break; - default: - std::cerr << "RpcProtoChat::processReqChatLobbies() LOBBYSET ERROR"; - std::cerr << std::endl; - success = false; - errorMsg = "Invalid Lobby Set"; - } - - - std::set done_lobbies; // list of ones we've added already (to avoid duplicates). - - if (fetch_chatlobbylist) - { - std::cerr << "RpcProtoChat::processReqChatLobbies() Fetching chatlobbylist"; - std::cerr << std::endl; - - std::list cl_info; - std::list::iterator it; - - rsMsgs->getChatLobbyList(cl_info); - - for(it = cl_info.begin(); it != cl_info.end(); it++) - { - rsctrl::chat::ChatLobbyInfo *lobby = resp.add_lobbies(); - fillLobbyInfoFromChatLobbyInfo(*it, lobby); - - done_lobbies.insert(it->lobby_id); - - std::cerr << "\t Added Lobby: " << it->lobby_id; - std::cerr << std::endl; - } - } - - /* This is before Public Lobbies - as knowing you have been invited is - * more important, than the full info that might be gleaned from lobby. - * In the future, we might try to merge the info. - */ - if (fetch_invites) - { - std::cerr << "RpcProtoChat::processReqChatLobbies() Fetching invites"; - std::cerr << std::endl; - - std::list invites; - std::list::iterator it; - rsMsgs->getPendingChatLobbyInvites(invites); - - for(it = invites.begin(); it != invites.end(); it++) - { - if (done_lobbies.find(it->lobby_id) == done_lobbies.end()) - { - rsctrl::chat::ChatLobbyInfo *lobby = resp.add_lobbies(); - fillLobbyInfoFromChatLobbyInvite(*it, lobby); - - done_lobbies.insert(it->lobby_id); - - std::cerr << "\t Added Lobby: " << it->lobby_id; - std::cerr << std::endl; - } - else - { - std::cerr << "\t Skipping Already Added Lobby: " << it->lobby_id; - std::cerr << std::endl; - } - } - } - - if (fetch_visiblelobbies) - { - std::cerr << "RpcProtoChat::processReqChatLobbies() Fetching public lobbies"; - std::cerr << std::endl; - - std::vector public_lobbies; - std::vector::iterator it; - - rsMsgs->getListOfNearbyChatLobbies(public_lobbies); - - for(it = public_lobbies.begin(); it != public_lobbies.end(); it++) - { - if (done_lobbies.find(it->lobby_id) == done_lobbies.end()) - { - rsctrl::chat::ChatLobbyInfo *lobby = resp.add_lobbies(); - fillLobbyInfoFromVisibleChatLobbyRecord(*it, lobby); - - done_lobbies.insert(it->lobby_id); - - std::cerr << "\t Added Lobby: " << it->lobby_id; - std::cerr << std::endl; - } - else - { - std::cerr << "\t Skipping Already Added Lobby: " << it->lobby_id; - std::cerr << std::endl; - } - } - } - - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::processReqChatLobbies() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_ResponseChatLobbies, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - -int RpcProtoChat::processReqCreateLobby(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoChat::processReqCreateLobby()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::chat::RequestCreateLobby req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoChat::processReqCreateLobby() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::chat::ResponseChatLobbies resp; - bool success = true; - std::string errorMsg; - - /* convert msg parameters into local ones */ - - std::string lobby_name = req.lobby_name(); - std::string lobby_topic = req.lobby_topic(); - std::list invited_friends; - uint32_t lobby_privacy_type = 0; - - switch(req.privacy_level()) - { - case rsctrl::chat::PRIVACY_PRIVATE: - std::cerr << "\tCreating Private Lobby"; - std::cerr << std::endl; - lobby_privacy_type = RS_CHAT_LOBBY_PRIVACY_LEVEL_PRIVATE; - break; - case rsctrl::chat::PRIVACY_PUBLIC: - std::cerr << "\tCreating Public Lobby"; - std::cerr << std::endl; - lobby_privacy_type = RS_CHAT_LOBBY_PRIVACY_LEVEL_PUBLIC; - break; - default: - std::cerr << "ERROR invalid Privacy Level"; - std::cerr << std::endl; - - success = false; - errorMsg = "Invalid Privacy Level"; - } - - int no_invites = req.invited_friends_size(); - for(int i = 0; i < no_invites; i++) - { - std::string peer = req.invited_friends(i); - /* check that they are a valid friend */ - if (!rsPeers->isFriend(peer)) // checks SSL ID. - { - std::cerr << "ERROR invalid SSL Friend ID: " << peer; - std::cerr << std::endl; - - success = false; - errorMsg = "Invalid SSL Friend ID"; - break; - } - - std::cerr << "Adding Valid Friend to Invites: " << peer; - std::cerr << std::endl; - - invited_friends.push_back(peer); - } - - ChatLobbyId created_lobby_id = 0; - if (success) - { - created_lobby_id = rsMsgs->createChatLobby(lobby_name,lobby_topic, - invited_friends,lobby_privacy_type); - - std::cerr << "Created Lobby Id: " << created_lobby_id; - std::cerr << std::endl; - - std::list cl_info; - std::list::iterator it; - - rsMsgs->getChatLobbyList(cl_info); - - bool found_entry = false; - for(it = cl_info.begin(); it != cl_info.end(); it++) - { - if (it->lobby_id == created_lobby_id) - { - std::cerr << "Found Created Lobby Id: " << created_lobby_id; - std::cerr << std::endl; - - rsctrl::chat::ChatLobbyInfo *lobby = resp.add_lobbies(); - fillLobbyInfoFromChatLobbyInfo(*it, lobby); - found_entry = true; - break; - } - } - - if (!found_entry) - { - std::cerr << "FAILED TO FIND Created Lobby Id: " << created_lobby_id; - std::cerr << std::endl; - - success = false; - errorMsg = "Chat Lobby Creation appears to have failed"; - } - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::processReqCreateLobbies() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_ResponseChatLobbies, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - -int RpcProtoChat::processReqJoinOrLeaveLobby(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoChat::processReqJoinOrLeaveLobby()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::chat::RequestJoinOrLeaveLobby req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoChat::processReqJoinOrLeaveLobby() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::chat::ResponseChatLobbies resp; - bool success = false; - std::string errorMsg; - - /* convert msg parameters into local ones */ - ChatLobbyId lobby_id = 0; - convertStringToLobbyId(req.lobby_id(), lobby_id); - - std::cerr << "Requested Lobby is: " << lobby_id; - std::cerr << std::endl; - - /* now must work out if its an invite or an existing lobby */ - /* look for a pending invitation */ - bool isPendingInvite = false; - { - std::list invites; - std::list::iterator it; - rsMsgs->getPendingChatLobbyInvites(invites); - - for(it = invites.begin(); it != invites.end(); it++) - { - if (it->lobby_id == lobby_id) - { - std::cerr << "It is a Pending Invite" << lobby_id; - std::cerr << std::endl; - - isPendingInvite = true; - break; - } - } - } - - switch(req.action()) - { - case rsctrl::chat::RequestJoinOrLeaveLobby::JOIN_OR_ACCEPT: - { - std::cerr << "Request to JOIN_OR_ACCEPT"; - std::cerr << std::endl; - - if (isPendingInvite) - { - if (!rsMsgs->acceptLobbyInvite(lobby_id)) - { - std::cerr << "ERROR acceptLobbyInvite FAILED"; - std::cerr << std::endl; - - success = false; - errorMsg = "AcceptLobbyInvite returned False"; - } - } - else - { - if (!rsMsgs->joinVisibleChatLobby(lobby_id)) - { - std::cerr << "ERROR joinPublicChatLobby FAILED"; - std::cerr << std::endl; - - success = false; - errorMsg = "joinPublicChatLobby returned False"; - } - } - - break; - } - case rsctrl::chat::RequestJoinOrLeaveLobby::LEAVE_OR_DENY: - { - std::cerr << "Request to LEAVE_OR_DENY (No fail codes!)"; - std::cerr << std::endl; - - if (isPendingInvite) - { - // return void - so can't check. - rsMsgs->denyLobbyInvite(lobby_id); - } - else - { - // return void - so can't check. - rsMsgs->unsubscribeChatLobby(lobby_id); - } - - break; - } - default: - success = false; - errorMsg = "Unknown Action"; - } - - - if (success && (req.action() == rsctrl::chat::RequestJoinOrLeaveLobby::JOIN_OR_ACCEPT)) - { - std::list cl_info; - std::list::iterator it; - - rsMsgs->getChatLobbyList(cl_info); - - bool found_entry = false; - for(it = cl_info.begin(); it != cl_info.end(); it++) - { - if (it->lobby_id == lobby_id) - { - rsctrl::chat::ChatLobbyInfo *lobby = resp.add_lobbies(); - fillLobbyInfoFromChatLobbyInfo(*it, lobby); - found_entry = true; - break; - } - } - - if (!found_entry) - { - success = false; - errorMsg = "Chat Lobby JOIN/ACCEPT appears to have failed"; - } - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::processReqCreateLobbies() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_ResponseChatLobbies, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - -int RpcProtoChat::processReqSetLobbyNickname(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoChat::processReqSetLobbyNickname()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::chat::RequestSetLobbyNickname req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoChat::processReqSetLobbyNickname() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::chat::ResponseSetLobbyNickname resp; - bool success = true; - std::string errorMsg; - - /* convert msg parameters into local ones */ - std::string nickname = req.nickname(); - - std::cerr << "choosen nickname is: " << nickname; - std::cerr << std::endl; - - int no_lobbyids = req.lobby_ids_size(); - for(int i = 0; i < no_lobbyids; i++) - { - std::string idstr = req.lobby_ids(i); - ChatLobbyId id = 0; - convertStringToLobbyId(idstr, id); - - std::cerr << "setting nickname for lobby: " << id; - std::cerr << std::endl; - - if (!rsMsgs->setNickNameForChatLobby(id, nickname)) - { - std::cerr << "ERROR setting nickname for lobby: " << id; - std::cerr << std::endl; - - success = false; - errorMsg = "Failed to Set one of the nicknames"; - break; - } - - } - - if (no_lobbyids == 0) - { - std::cerr << "setting default nickname"; - std::cerr << std::endl; - - /* just do default instead */ - rsMsgs->setDefaultNickNameForChatLobby(nickname); - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::processReqCreateLobbies() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_ResponseSetLobbyNickname, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; - -} - - - -int RpcProtoChat::processReqRegisterEvents(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoChat::processReqRegisterEvents()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::chat::RequestRegisterEvents req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoChat::processReqRegisterEvents() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::chat::ResponseRegisterEvents resp; - bool success = true; - bool doregister = false; - std::string errorMsg; - - switch(req.action()) - { - case rsctrl::chat::RequestRegisterEvents::REGISTER: - doregister = true; - break; - case rsctrl::chat::RequestRegisterEvents::DEREGISTER: - doregister = false; - break; - default: - std::cerr << "ERROR action is invalid"; - std::cerr << std::endl; - - success = false; - errorMsg = "RegisterEvent.Action is invalid"; - break; - } - - if (success) - { - if (doregister) - { - std::cerr << "Registering for Chat Events"; - std::cerr << std::endl; - - registerForEvents(chan_id, req_id, REGISTRATION_EVENT_CHAT); - } - else - { - std::cerr << "Deregistering for Chat Events"; - std::cerr << std::endl; - - deregisterForEvents(chan_id, req_id, REGISTRATION_EVENT_CHAT); - } - printEventRegister(std::cerr); - } - - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::processReqCreateLobbies() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_ResponseRegisterEvents, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - - -int RpcProtoChat::processReqChatHistory(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoChat::processReqChatHistory()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::chat::RequestChatHistory req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoChat::processReqChatHistory() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::chat::ResponseChatHistory resp; - bool success = true; - std::string errorMsg; - - // Get the Chat History for specified IDs.... - - /* switch depending on type */ - bool private_chat = false; - bool lobby_chat = false; - std::string chat_id; - - // copy the ID over. - rsctrl::chat::ChatId *id = resp.mutable_id(); - *id = req.id(); - - switch(req.id().chat_type()) - { - case rsctrl::chat::TYPE_PRIVATE: - { - // easy one. - chat_id = req.id().chat_id(); - private_chat = true; - - std::cerr << "RpcProtoChat::processReqChatHistory() Getting Private Chat History for: "; - std::cerr << chat_id; - std::cerr << std::endl; - - break; - } - case rsctrl::chat::TYPE_LOBBY: - { - std::cerr << "RpcProtoChat::processReqChatHistory() Lobby Chat History NOT IMPLEMENTED YET"; - std::cerr << std::endl; - success = false; - lobby_chat = true; - errorMsg = "Lobby Chat History Not Implemented"; - -#if 0 - /* convert string->ChatLobbyId */ - ChatLobbyId lobby_id; - if (!convertStringToLobbyId(req.msg().id().chat_id(), lobby_id)) - { - std::cerr << "ERROR Failed conversion of Lobby Id"; - std::cerr << std::endl; - - success = false; - errorMsg = "Failed Conversion of Lobby Id"; - } - /* convert lobby id to virtual peer id */ - else if (!rsMsgs->getVirtualPeerId(lobby_id, chat_id)) - { - std::cerr << "ERROR Invalid Lobby Id"; - std::cerr << std::endl; - - success = false; - errorMsg = "Invalid Lobby Id"; - } - lobby_chat = true; - std::cerr << "RpcProtoChat::processReqChatHistory() Getting Lobby Chat History for: "; - std::cerr << chat_id; - std::cerr << std::endl; -#endif - - break; - } - case rsctrl::chat::TYPE_GROUP: - - std::cerr << "RpcProtoChat::processReqChatHistory() Group Chat History NOT IMPLEMENTED YET"; - std::cerr << std::endl; - success = false; - errorMsg = "Group Chat History Not Implemented"; - - break; - default: - - std::cerr << "ERROR Chat Type invalid"; - std::cerr << std::endl; - - success = false; - errorMsg = "Invalid Chat Type"; - break; - } - - // Should be able to reply using the existing message types. - if (success) - { - if (private_chat) - { - /* extract the history */ - std::list msgs; - std::list::iterator it; - rsHistory->getMessages(chat_id, msgs, 0); - - //rsctrl::chat::ChatId *id = resp.mutable_id(); - //id->set_chat_type(rsctrl::chat::TYPE_PRIVATE); - //id->set_chat_id(chat_id); - - for(it = msgs.begin(); it != msgs.end(); it++) - { - rsctrl::chat::ChatMessage *msg = resp.add_msgs(); - fillChatMessageFromHistoryMsg(*it, msg); - - std::cerr << "\t Message: " << it->message; - std::cerr << std::endl; - } - } -#if 0 - else if (lobby_chat) - { - - - } -#endif - } - - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::processReqChatHistory() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_ResponseChatHistory, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - - -int RpcProtoChat::processReqSendMessage(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoChat::processReqSendMessage()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::chat::RequestSendMessage req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoChat::processReqSendMessage() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::chat::ResponseSendMessage resp; - bool success = true; - std::string errorMsg; - - // Send the message. - bool priv_or_lobby = true; - std::string chat_id; - std::string chat_msg = req.msg().msg() ; - - std::cerr << "Chat Message is: " << req.msg().msg(); - std::cerr << std::endl; - - /* switch depending on type */ - switch(req.msg().id().chat_type()) - { - case rsctrl::chat::TYPE_PRIVATE: - // easy one. - chat_id = req.msg().id().chat_id(); - priv_or_lobby = true; - - std::cerr << "Sending Private Chat"; - std::cerr << std::endl; - - - break; - case rsctrl::chat::TYPE_LOBBY: - { - std::cerr << "Sending Lobby Chat"; - std::cerr << std::endl; - - /* convert string->ChatLobbyId */ - ChatLobbyId lobby_id; - if (!convertStringToLobbyId(req.msg().id().chat_id(), lobby_id)) - { - std::cerr << "ERROR Failed conversion of Lobby Id"; - std::cerr << std::endl; - - success = false; - errorMsg = "Failed Conversion of Lobby Id"; - } - /* convert lobby id to virtual peer id */ - else if (!rsMsgs->getVirtualPeerId(lobby_id, chat_id)) - { - std::cerr << "ERROR Invalid Lobby Id"; - std::cerr << std::endl; - - success = false; - errorMsg = "Invalid Lobby Id"; - } - priv_or_lobby = true; - break; - } - case rsctrl::chat::TYPE_GROUP: - - std::cerr << "Sending Group Chat"; - std::cerr << std::endl; - - priv_or_lobby = false; - break; - default: - - std::cerr << "ERROR Chat Type invalid"; - std::cerr << std::endl; - - success = false; - errorMsg = "Invalid Chat Type"; - break; - } - - if (success) - { - if (priv_or_lobby) - { - rsMsgs->sendPrivateChat(chat_id, chat_msg); - } - else - { - rsMsgs->sendPublicChat(chat_msg); - } - } - - /* DONE - Generate Reply */ - - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::processReqSendMessage() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_ResponseSendMessage, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - // EVENTS. -int RpcProtoChat::locked_checkForEvents(uint32_t event, const std::list ®istered, std::list &events) -{ - /* Wow - here already! */ - //std::cerr << "locked_checkForEvents()"; - //std::cerr << std::endl; - - /* only one event type for now */ - if (event != REGISTRATION_EVENT_CHAT) - { - std::cerr << "ERROR Invalid Chat Event Type"; - std::cerr << std::endl; - - /* error */ - return 0; - } - - /* possible events */ - /* TODO lobby invites. how do we make sure these are only sent once? */ - - // Likewise with Chat Queues -> We'll have to track which items have been sent - // to which Chan Ids.... a bit painful. - - /* public chat queues */ - if (rsMsgs->getPublicChatQueueCount() > 0) - { - std::cerr << "Fetching Public Chat Queue"; - std::cerr << std::endl; - - std::list chats; - rsMsgs->getPublicChatQueue(chats); - std::list::iterator it; - for(it = chats.begin(); it != chats.end(); it++) - { - std::cerr << "Public Chat from : " << it->rsid; - std::cerr << " name: " << it->peer_nickname; - std::cerr << std::endl; - { - std::cerr << " Msg: " << it->msg ; - std::cerr << std::endl; - } - - /* must send to all registered clients */ - std::list::const_iterator rit; - for(rit = registered.begin(); rit != registered.end(); rit++) - { - RpcQueuedMsg qmsg; - rsctrl::chat::ChatType ctype = rsctrl::chat::TYPE_GROUP; - std::string chat_id = ""; // No ID for group. - - if (createQueuedEventSendMsg(*it, ctype, chat_id, *rit, qmsg)) - { - std::cerr << "Created MsgEvent"; - std::cerr << std::endl; - - events.push_back(qmsg); - } - else - { - std::cerr << "ERROR Creating MsgEvent"; - std::cerr << std::endl; - } - } - } - } - - - /* private chat queues */ - bool incoming = true; // ignore outgoing for now. (client can request that some other way) - if (rsMsgs->getPrivateChatQueueCount(incoming) > 0) - { - std::cerr << "Fetching Private Chat Queues"; - std::cerr << std::endl; - - std::list priv_chat_ids; - std::list::iterator cit; - rsMsgs->getPrivateChatQueueIds(incoming, priv_chat_ids); - for(cit = priv_chat_ids.begin(); cit != priv_chat_ids.end(); cit++) - { - std::list chats; - rsMsgs->getPrivateChatQueue(incoming, *cit, chats); - rsMsgs->clearPrivateChatQueue(incoming, *cit); - - // Default to Private. - rsctrl::chat::ChatType ctype = rsctrl::chat::TYPE_PRIVATE; - std::string chat_id = *cit; - - - // Check if its actually a LobbyId. - ChatLobbyId lobby_id; - if (rsMsgs->isLobbyId(*cit, lobby_id)) - { - ctype = rsctrl::chat::TYPE_LOBBY; - chat_id.clear(); - convertLobbyIdToString(lobby_id, chat_id); - - std::cerr << "Lobby Chat Queue: " << chat_id; - std::cerr << std::endl; - } - else - { - std::cerr << "Private Chat Queue: " << *cit; - std::cerr << std::endl; - } - - std::list::iterator it; - for(it = chats.begin(); it != chats.end(); it++) - { - std::cerr << "Private Chat from : " << it->rsid; - std::cerr << " name: " << it->peer_nickname; - std::cerr << std::endl; - { - std::cerr << " Msg: " << it->msg; - std::cerr << std::endl; - } - /* must send to all registered clients */ - std::list::const_iterator rit; - for(rit = registered.begin(); rit != registered.end(); rit++) - { - RpcQueuedMsg qmsg; - if (createQueuedEventSendMsg(*it, ctype, chat_id, *rit, qmsg)) - { - std::cerr << "Created MsgEvent"; - std::cerr << std::endl; - - events.push_back(qmsg); - } - else - { - std::cerr << "ERROR Creating MsgEvent"; - std::cerr << std::endl; - } - } - } - } - } - - return 1; -} - - -/***** HELPER FUNCTIONS *****/ - - - -bool fillLobbyInfoFromChatLobbyInfo(const ChatLobbyInfo &cli, rsctrl::chat::ChatLobbyInfo *lobby) -{ - /* convert info into response */ - std::string chat_id; - convertLobbyIdToString(cli.lobby_id, chat_id); - lobby->set_lobby_id(chat_id); - lobby->set_lobby_name(cli.lobby_name); - lobby->set_lobby_topic(cli.lobby_topic); - - /* see if there's a specific nickname for here */ - std::string nick; - if (!rsMsgs->getNickNameForChatLobby(cli.lobby_id,nick)) - { - rsMsgs->getDefaultNickNameForChatLobby(nick); - } - - lobby->set_lobby_nickname(nick); - - // Could be Private or Public. - if (cli.lobby_privacy_level & RS_CHAT_LOBBY_PRIVACY_LEVEL_PRIVATE) - { - lobby->set_privacy_level(rsctrl::chat::PRIVACY_PRIVATE); - } - else - { - lobby->set_privacy_level(rsctrl::chat::PRIVACY_PUBLIC); - } - lobby->set_lobby_state(rsctrl::chat::ChatLobbyInfo::LOBBYSTATE_JOINED); - - lobby->set_no_peers(cli.nick_names.size()); - - lobby->set_last_report_time(0); - lobby->set_last_activity(cli.last_activity); - - std::set::const_iterator pit; - for(pit = cli.participating_friends.begin(); pit != cli.participating_friends.end(); pit++) - { - lobby->add_participating_friends(*pit); - } - - std::map::const_iterator mit; - for(mit = cli.nick_names.begin(); mit != cli.nick_names.end(); mit++) - { - lobby->add_nicknames(mit->first); - } - return true; -} - - -bool fillLobbyInfoFromVisibleChatLobbyRecord(const VisibleChatLobbyRecord &pclr, rsctrl::chat::ChatLobbyInfo *lobby) -{ - /* convert info into response */ - std::string chat_id; - convertLobbyIdToString(pclr.lobby_id, chat_id); - lobby->set_lobby_id(chat_id); - lobby->set_lobby_name(pclr.lobby_name); - lobby->set_lobby_topic(pclr.lobby_topic); - - /* see if there's a specific nickname for here */ - std::string nick; - if (!rsMsgs->getNickNameForChatLobby(pclr.lobby_id,nick)) - { - rsMsgs->getDefaultNickNameForChatLobby(nick); - } - - lobby->set_lobby_nickname(nick); - - // Could be Private or Public. - if (pclr.lobby_privacy_level & RS_CHAT_LOBBY_PRIVACY_LEVEL_PRIVATE) - { - lobby->set_privacy_level(rsctrl::chat::PRIVACY_PRIVATE); - } - else - { - lobby->set_privacy_level(rsctrl::chat::PRIVACY_PUBLIC); - } - // TODO. - lobby->set_lobby_state(rsctrl::chat::ChatLobbyInfo::LOBBYSTATE_VISIBLE); - - lobby->set_no_peers(pclr.total_number_of_peers); - - lobby->set_last_report_time(pclr.last_report_time); - lobby->set_last_activity(0); // Unknown. - - std::set::const_iterator it; - for(it = pclr.participating_friends.begin(); it != pclr.participating_friends.begin(); it++) - { - lobby->add_participating_friends(*it); - } - - //lobby->add_nicknames(); // Unknown. - return true; -} - - -bool fillLobbyInfoFromChatLobbyInvite(const ChatLobbyInvite &cli, rsctrl::chat::ChatLobbyInfo *lobby) -{ - /* convert info into response */ - std::string chat_id; - convertLobbyIdToString(cli.lobby_id, chat_id); - lobby->set_lobby_id(chat_id); - lobby->set_lobby_name(cli.lobby_name); - lobby->set_lobby_topic(cli.lobby_topic); - - /* see if there's a specific nickname for here */ - std::string nick; - if (!rsMsgs->getNickNameForChatLobby(cli.lobby_id,nick)) - { - rsMsgs->getDefaultNickNameForChatLobby(nick); - } - - lobby->set_lobby_nickname(nick); - - // Can be invited to both Public and Private. - if (cli.lobby_privacy_level & RS_CHAT_LOBBY_PRIVACY_LEVEL_PRIVATE) - { - lobby->set_privacy_level(rsctrl::chat::PRIVACY_PRIVATE); - } - else - { - lobby->set_privacy_level(rsctrl::chat::PRIVACY_PUBLIC); - } - lobby->set_lobby_state(rsctrl::chat::ChatLobbyInfo::LOBBYSTATE_INVITED); - - lobby->set_no_peers(0); // Unknown. - - lobby->set_last_report_time(0); // Unknown - lobby->set_last_activity(0); // Unknown - - // Unknown, but we can fill in the inviting party here (the only one we know). - lobby->add_participating_friends(cli.peer_id); - //lobby->add_nicknames(); // Unknown. - return true; -} - - -bool fillChatMessageFromHistoryMsg(const HistoryMsg &histmsg, rsctrl::chat::ChatMessage *rpcmsg) -{ - rsctrl::chat::ChatId *id = rpcmsg->mutable_id(); - - id->set_chat_type(rsctrl::chat::TYPE_PRIVATE); - id->set_chat_id(histmsg.chatPeerId); - - rpcmsg->set_msg(histmsg.message); - - rpcmsg->set_peer_nickname(histmsg.peerName); - rpcmsg->set_chat_flags(0); - - rpcmsg->set_send_time(histmsg.sendTime); - rpcmsg->set_recv_time(histmsg.recvTime); - - return true; -} - - -bool createQueuedEventSendMsg(const ChatInfo &chatinfo, rsctrl::chat::ChatType ctype, - std::string chat_id, const RpcEventRegister &ereg, RpcQueuedMsg &qmsg) -{ - - rsctrl::chat::EventChatMessage event; - rsctrl::chat::ChatMessage *msg = event.mutable_msg(); - rsctrl::chat::ChatId *id = msg->mutable_id(); - - id->set_chat_type(ctype); - id->set_chat_id(chat_id); - - msg->set_peer_nickname(chatinfo.peer_nickname); - msg->set_chat_flags(chatinfo.chatflags); - msg->set_send_time(chatinfo.sendTime); - msg->set_recv_time(chatinfo.recvTime); - msg->set_msg(chatinfo.msg); - - /* DONE - Generate Reply */ - std::string outmsg; - if (!event.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoChat::createQueuedEventSendMsg() ERROR SerialiseToString()"; - std::cerr << std::endl; - return false; - } - - // Correctly Name Message. - qmsg.mMsgId = constructMsgId(rsctrl::core::CORE, rsctrl::core::CHAT, - rsctrl::chat::MsgId_EventChatMessage, true); - - qmsg.mChanId = ereg.mChanId; - qmsg.mReqId = ereg.mReqId; - qmsg.mMsg = outmsg; - - return true; -} - - - - -bool convertUTF8toWString(const std::string &msg_utf8, std::wstring &msg_wstr) -{ - return librs::util::ConvertUtf8ToUtf16(msg_utf8, msg_wstr); -} - -bool convertWStringToUTF8(const std::wstring &msg_wstr, std::string &msg_utf8) -{ - return librs::util::ConvertUtf16ToUtf8(msg_wstr, msg_utf8); -} - - -/* dependent on ChatLobbyId definition as uint64_t */ -bool convertStringToLobbyId(const std::string &chat_id, ChatLobbyId &lobby_id) -{ - if (1 != sscanf(chat_id.c_str(), UINT64FMT, &lobby_id)) - { - return false; - } - return true; -} - -bool convertLobbyIdToString(const ChatLobbyId &lobby_id, std::string &chat_id) -{ - rs_sprintf(chat_id, UINT64FMT, lobby_id); - return true; -} - - diff --git a/retroshare-nogui/src/rpc/proto/rpcprotochat.h b/retroshare-nogui/src/rpc/proto/rpcprotochat.h deleted file mode 100644 index 08595bf3e..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotochat.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_PROTO_CHAT_H -#define RS_RPC_PROTO_CHAT_H - -#include "rpc/rpcserver.h" - -// Registrations. -#define REGISTRATION_EVENT_CHAT 1 - -class RpcProtoChat: public RpcQueueService -{ -public: - RpcProtoChat(uint32_t serviceId); - virtual int processMsg(uint32_t chan_id, uint32_t msgId, uint32_t req_id, const std::string &msg); - -protected: - - int processReqChatLobbies(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqCreateLobby(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqJoinOrLeaveLobby(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqSetLobbyNickname(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqRegisterEvents(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqSendMessage(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - - int processReqChatHistory(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - - - // EVENTS. - virtual int locked_checkForEvents(uint32_t event, const std::list ®istered, std::list &events); -}; - - -#endif /* RS_PROTO_CHAT_H */ diff --git a/retroshare-nogui/src/rpc/proto/rpcprotofiles.cc b/retroshare-nogui/src/rpc/proto/rpcprotofiles.cc deleted file mode 100644 index c5838ec50..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotofiles.cc +++ /dev/null @@ -1,630 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/proto/rpcprotofiles.h" -#include "rpc/proto/gencc/files.pb.h" - -#include -#include - -#include "util/rsstring.h" -#include "util/rsdir.h" - -#include - -#include -#include - -#include - - -bool fill_file_from_details(rsctrl::core::File *file, DirDetails &details); -bool fill_file_as_dir(rsctrl::core::File *file, const std::string &dir_name); - -RpcProtoFiles::RpcProtoFiles(uint32_t serviceId) - :RpcQueueService(serviceId) -{ - return; -} - -int RpcProtoFiles::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - /* check the msgId */ - uint8_t topbyte = getRpcMsgIdExtension(msg_id); - uint16_t service = getRpcMsgIdService(msg_id); - uint8_t submsg = getRpcMsgIdSubMsg(msg_id); - bool isResponse = isRpcMsgIdResponse(msg_id); - - - std::cerr << "RpcProtoFiles::processMsg() topbyte: " << (int32_t) topbyte; - std::cerr << " service: " << (int32_t) service << " submsg: " << (int32_t) submsg; - std::cerr << std::endl; - - if (isResponse) - { - std::cerr << "RpcProtoFiles::processMsg() isResponse() - not processing"; - std::cerr << std::endl; - return 0; - } - - if (topbyte != (uint8_t) rsctrl::core::CORE) - { - std::cerr << "RpcProtoFiles::processMsg() Extension Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (service != (uint16_t) rsctrl::core::FILES) - { - std::cerr << "RpcProtoFiles::processMsg() Service Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (!rsctrl::files::RequestMsgIds_IsValid(submsg)) - { - std::cerr << "RpcProtoFiles::processMsg() SubMsg Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - switch(submsg) - { - case rsctrl::files::MsgId_RequestTransferList: - processReqTransferList(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::files::MsgId_RequestControlDownload: - processReqControlDownload(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::files::MsgId_RequestShareDirList: - processReqShareDirList(chan_id, msg_id, req_id, msg); - break; - - default: - std::cerr << "RpcProtoFiles::processMsg() ERROR should never get here"; - std::cerr << std::endl; - return 0; - } - - /* must have matched id to get here */ - return 1; -} - - - -int RpcProtoFiles::processReqTransferList(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoFiles::processReqTransferList()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::files::RequestTransferList req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoFiles::processReqTransferList() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::files::ResponseTransferList resp; - bool success = true; - std::string errorMsg; - - std::list file_list; - FileSearchFlags hints(0); - - /* convert msg parameters into local ones */ - switch(req.direction()) - { - case rsctrl::files::DIRECTION_UPLOAD: - { - rsFiles->FileUploads(file_list); - hints = RS_FILE_HINTS_UPLOAD; - break; - } - case rsctrl::files::DIRECTION_DOWNLOAD: - { - rsFiles->FileDownloads(file_list); - hints = RS_FILE_HINTS_DOWNLOAD; - break; - } - default: - std::cerr << "RpcProtoFiles::processReqTransferList() ERROR Unknown Dir"; - std::cerr << std::endl; - success = false; - errorMsg = "Unknown Direction"; - break; - } - - std::list::iterator lit; - for(lit = file_list.begin(); lit != file_list.end(); lit++) - { - rsctrl::files::FileTransfer *transfer = resp.add_transfers(); - transfer->set_direction(req.direction()); - - FileInfo info; - if (!rsFiles->FileDetails(*lit, hints, info)) - { - /* error */ - continue; - } - - /* copy file details */ - rsctrl::core::File *filedetails = transfer->mutable_file(); - filedetails->set_hash(info.hash); - filedetails->set_size(info.size); - filedetails->set_name(info.fname); - - transfer->set_fraction( (float) info.transfered / info.size ); - transfer->set_rate_kbs( info.tfRate ); - - switch(info.downloadStatus) - { - case FT_STATE_FAILED: - transfer->set_state(rsctrl::files::TRANSFER_FAILED); - break; - default: - case FT_STATE_OKAY: - transfer->set_state(rsctrl::files::TRANSFER_OKAY); - break; - case FT_STATE_PAUSED: - transfer->set_state(rsctrl::files::TRANSFER_PAUSED); - break; - case FT_STATE_QUEUED: - transfer->set_state(rsctrl::files::TRANSFER_QUEUED); - break; - case FT_STATE_WAITING: - transfer->set_state(rsctrl::files::TRANSFER_WAITING); - break; - case FT_STATE_DOWNLOADING: - transfer->set_state(rsctrl::files::TRANSFER_DOWNLOADING); - break; - case FT_STATE_CHECKING_HASH: - transfer->set_state(rsctrl::files::TRANSFER_CHECKING_HASH); - break; - case FT_STATE_COMPLETE: - transfer->set_state(rsctrl::files::TRANSFER_COMPLETE); - break; - } - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoFiles::processReqTransferList() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::FILES, - rsctrl::files::MsgId_ResponseTransferList, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - -int RpcProtoFiles::processReqControlDownload(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoFiles::processReqControlDownload()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::files::RequestControlDownload req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoFiles::processReqControlDownload() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::files::ResponseControlDownload resp; - bool success = true; - std::string errorMsg; - - std::string filehash = req.file().hash(); - switch(req.action()) - { - case rsctrl::files::RequestControlDownload::ACTION_START: - { - std::list srcIds; - - std::string filename = req.file().name(); - uint64_t filesize = req.file().size(); - - // We Set NETWORK_WIDE flag here -> as files will be found via search. - // If this changes, we might be adjust flag (or pass it in!) - if (!rsFiles -> FileRequest(filename, filehash, filesize, - "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) - { - success = false; - errorMsg = "FileRequest ERROR"; - } - break; - } - case rsctrl::files::RequestControlDownload::ACTION_CONTINUE: - { - if (!rsFiles->changeQueuePosition(filehash,QUEUE_TOP)) - { - success = false; - errorMsg = "File QueuePosition(Top) ERROR"; - } - break; - } - case rsctrl::files::RequestControlDownload::ACTION_WAIT: - { - if (!rsFiles->changeQueuePosition(filehash,QUEUE_BOTTOM)) - { - success = false; - errorMsg = "File QueuePosition(Bottom) ERROR"; - - } - break; - } - case rsctrl::files::RequestControlDownload::ACTION_PAUSE: - { - if (!rsFiles->FileControl(filehash,RS_FILE_CTRL_PAUSE)) - { - success = false; - errorMsg = "FileControl(Pause) ERROR"; - - } - break; - } - case rsctrl::files::RequestControlDownload::ACTION_RESTART: - { - if (!rsFiles->FileControl(filehash,RS_FILE_CTRL_START)) - { - success = false; - errorMsg = "FileControl(Start) ERROR"; - - } - break; - } - case rsctrl::files::RequestControlDownload::ACTION_CHECK: - { - if (!rsFiles->FileControl(filehash,RS_FILE_CTRL_FORCE_CHECK)) - { - success = false; - errorMsg = "FileControl(Check) ERROR"; - - } - break; - } - case rsctrl::files::RequestControlDownload::ACTION_CANCEL: - { - if (!rsFiles->FileCancel(filehash)) - { - success = false; - errorMsg = "FileCancel ERROR"; - } - break; - } - default: - success = false; - errorMsg = "Invalid Action"; - break; - } - - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoFiles::processReqControlDownload() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::FILES, - rsctrl::files::MsgId_ResponseControlDownload, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -int RpcProtoFiles::processReqShareDirList(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoFiles::processReqShareDirList()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::files::RequestShareDirList req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoFiles::processReqShareDirList() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::files::ResponseShareDirList resp; - bool success = true; - std::string errorMsg; - - - std::string uid = req.ssl_id(); - std::string path = req.path(); - DirDetails details; - - if (uid.empty()) - { - uid = rsPeers->getOwnId(); - } - - std::cerr << "RpcProtoFiles::processReqShareDirList() For uid: " << uid << " & path: " << path; - std::cerr << std::endl; - - if (path.empty()) - { - /* we have to do a nasty hack to get anything useful. - * we do a ref=NULL to get the pointers to People, - * then use the correct one to get root directories - */ - std::cerr << "RpcProtoFiles::processReqShareDirList() Hack to get root Dirs!"; - std::cerr << std::endl; - - FileSearchFlags flags; - if (uid == rsPeers->getOwnId()) - { - flags |= RS_FILE_HINTS_LOCAL; - } - - DirDetails root_details; - if (!rsFiles->RequestDirDetails(NULL, root_details, flags)) - { - std::cerr << "RpcProtoFiles::processReqShareDirList() ref=NULL Hack failed"; - std::cerr << std::endl; - success = false; - errorMsg = "Root Directory Request Failed."; - } - else - { - void *person_ref = NULL; - std::list::iterator sit; - for(sit = root_details.children.begin(); sit != root_details.children.end(); sit++) - { - //std::cerr << "RpcProtoFiles::processReqShareDirList() Root.child->name : " << sit->name; - if (sit->name == uid) - { - person_ref = sit->ref; - break; - } - } - - if (!person_ref) - { - std::cerr << "RpcProtoFiles::processReqShareDirList() Person match failed"; - std::cerr << std::endl; - success = false; - errorMsg = "Missing Person Root Directory."; - } - else - { - // Doing the REAL request! - if (!rsFiles->RequestDirDetails(person_ref, details, flags)) - { - std::cerr << "RpcProtoFiles::processReqShareDirList() Personal Shared Dir Hack failed"; - std::cerr << std::endl; - success = false; - errorMsg = "Missing Person Shared Directories"; - } - } - } - - } - else - { - // Path must begin with / for proper matching. - if (path[0] != '/') - { - path = '/' + path; - } - - if (!rsFiles->RequestDirDetails(uid, path, details)) - { - std::cerr << "RpcProtoFiles::processReqShareDirList() ERROR Unknown Dir"; - std::cerr << std::endl; - success = false; - errorMsg = "Directory Request Failed."; - } - } - - - - - - if (success) - { - // setup basics of response. - resp.set_ssl_id(uid); - resp.set_path(path); - - switch(details.type) - { - case DIR_TYPE_ROOT: - { - std::cerr << "RpcProtoFiles::processReqShareDirList() Details.type == ROOT ??"; - std::cerr << std::endl; - resp.set_list_type(rsctrl::files::ResponseShareDirList::DIRQUERY_ROOT); - rsctrl::core::File *file = resp.add_files(); - fill_file_as_dir(file, details.name); - - } - break; - - case DIR_TYPE_FILE: - { - std::cerr << "RpcProtoFiles::processReqShareDirList() Details.type == FILE"; - std::cerr << std::endl; - resp.set_list_type(rsctrl::files::ResponseShareDirList::DIRQUERY_FILE); - rsctrl::core::File *file = resp.add_files(); - fill_file_from_details(file, details); - } - break; - - default: - std::cerr << "RpcProtoFiles::processReqShareDirList() Details.type == UNKNOWN => default to DIR"; - std::cerr << std::endl; - - case DIR_TYPE_PERSON: - case DIR_TYPE_DIR: - { - std::cerr << "RpcProtoFiles::processReqShareDirList() Details.type == DIR or PERSON"; - std::cerr << std::endl; - - resp.set_list_type(rsctrl::files::ResponseShareDirList::DIRQUERY_DIR); - - //std::string dir_path = RsDirUtil::makePath(details.path, details.name); - std::string dir_path = details.path; - - std::cerr << "RpcProtoFiles::processReqShareDirList() details.path: " << details.path; - std::cerr << " details.name: " << details.name; - std::cerr << std::endl; - - std::list::iterator sit; - for(sit = details.children.begin(); sit != details.children.end(); sit++) - { - std::cerr << "RpcProtoFiles::processReqShareDirList() checking child: " << sit->name; - std::cerr << std::endl; - if (sit->type == DIR_TYPE_FILE) - { - std::cerr << "RpcProtoFiles::processReqShareDirList() is FILE, fetching details."; - std::cerr << std::endl; - - DirDetails child_details; - std::string child_path = RsDirUtil::makePath(dir_path, sit->name); - if (rsFiles->RequestDirDetails(uid, child_path, child_details)) - { - rsctrl::core::File *file = resp.add_files(); - fill_file_from_details(file, child_details); - } - else - { - std::cerr << "RpcProtoFiles::processReqShareDirList() RequestDirDetails(" << child_path << ") Failed!!!"; - std::cerr << std::endl; - } - } - else - { - std::cerr << "RpcProtoFiles::processReqShareDirList() is DIR"; - std::cerr << std::endl; - - rsctrl::core::File *file = resp.add_files(); - fill_file_as_dir(file, sit->name); - } - } - } - break; - } - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoFiles::processReqTransferList() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::FILES, - rsctrl::files::MsgId_ResponseShareDirList, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -/***** HELPER FUNCTIONS *****/ - - -bool fill_file_from_details(rsctrl::core::File *file, DirDetails &details) -{ - file->set_hash(details.hash); - file->set_name(details.name); - file->set_size(details.count); - - return true; -} - - -bool fill_file_as_dir(rsctrl::core::File *file, const std::string &dir_name) -{ - file->set_hash(""); - file->set_name(dir_name); - file->set_size(0); - - return true; -} - diff --git a/retroshare-nogui/src/rpc/proto/rpcprotofiles.h b/retroshare-nogui/src/rpc/proto/rpcprotofiles.h deleted file mode 100644 index 7f0af0c9e..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotofiles.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_PROTO_FILES_H -#define RS_RPC_PROTO_FILES_H - -#include "rpc/rpcserver.h" - -class RpcProtoFiles: public RpcQueueService -{ -public: - RpcProtoFiles(uint32_t serviceId); - - virtual int processMsg(uint32_t chan_id, uint32_t msgId, uint32_t req_id, const std::string &msg); - -protected: - - int processReqTransferList(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqControlDownload(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqShareDirList(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg); - -}; - -#endif /* RS_PROTO_FILES_H */ diff --git a/retroshare-nogui/src/rpc/proto/rpcprotopeers.cc b/retroshare-nogui/src/rpc/proto/rpcprotopeers.cc deleted file mode 100644 index cfe9c9076..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotopeers.cc +++ /dev/null @@ -1,610 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/proto/rpcprotopeers.h" -#include "rpc/proto/gencc/peers.pb.h" - -#include -#include - -#include -#include - -bool load_person_details(std::string pgp_id, rsctrl::core::Person *person, - bool getLocations, bool onlyConnected); - -RpcProtoPeers::RpcProtoPeers(uint32_t serviceId) - :RpcQueueService(serviceId) -{ - return; -} - -//RpcProtoPeers::msgsAccepted(std::list &msgIds); /* not used at the moment */ - -int RpcProtoPeers::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - /* check the msgId */ - uint8_t topbyte = getRpcMsgIdExtension(msg_id); - uint16_t service = getRpcMsgIdService(msg_id); - uint8_t submsg = getRpcMsgIdSubMsg(msg_id); - bool isResponse = isRpcMsgIdResponse(msg_id); - - - std::cerr << "RpcProtoPeers::processMsg() topbyte: " << (int32_t) topbyte; - std::cerr << " service: " << (int32_t) service << " submsg: " << (int32_t) submsg; - std::cerr << std::endl; - - if (isResponse) - { - std::cerr << "RpcProtoPeers::processMsg() isResponse() - not processing"; - std::cerr << std::endl; - return 0; - } - - - if (topbyte != (uint8_t) rsctrl::core::CORE) - { - std::cerr << "RpcProtoPeers::processMsg() Extension Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (service != (uint16_t) rsctrl::core::PEERS) - { - std::cerr << "RpcProtoPeers::processMsg() Service Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (!rsctrl::peers::RequestMsgIds_IsValid(submsg)) - { - std::cerr << "RpcProtoPeers::processMsg() SubMsg Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - switch(submsg) - { - case rsctrl::peers::MsgId_RequestPeers: - processRequestPeers(chan_id, msg_id, req_id, msg); - break; - case rsctrl::peers::MsgId_RequestAddPeer: - processAddPeer(chan_id, msg_id, req_id, msg); - break; - case rsctrl::peers::MsgId_RequestExaminePeer: - processExaminePeer(chan_id, msg_id, req_id, msg); - break; - //case rsctrl::peers::MsgId_RequestModifyPeer: - // processModifyPeer(chan_id, msg_id, req_id, msg); - // break; - default: - std::cerr << "RpcProtoPeers::processMsg() ERROR should never get here"; - std::cerr << std::endl; - return 0; - } - - /* must have matched id to get here */ - return 1; -} - - -int RpcProtoPeers::processAddPeer(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoPeers::processAddPeer()"; - std::cerr << std::endl; - - - // parse msg. - rsctrl::peers::RequestAddPeer req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoPeers::processAddPeer() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::peers::ResponsePeerList resp; - bool success = true; - std::string errorMsg; - - /* check if the gpg_id is valid */ - std::string pgp_id = req.pgp_id(); - std::string ssl_id; - if (req.has_ssl_id()) - { - ssl_id = req.ssl_id(); - } - - RsPeerDetails details; - if (!rsPeers->getGPGDetails(pgp_id, details)) - { - success = false; - errorMsg = "Invalid PGP ID"; - } - else - { - switch(req.cmd()) - { - default: - success = false; - errorMsg = "Invalid AddCmd"; - break; - case rsctrl::peers::RequestAddPeer::ADD: - - // TODO. NEED TO HANDLE SERVICE PERMISSION FLAGS. - success = rsPeers->addFriend(ssl_id,pgp_id, RS_NODE_PERM_DEFAULT); - - break; - case rsctrl::peers::RequestAddPeer::REMOVE: - - success = rsPeers->removeFriend(pgp_id); - break; - } - - if (success) - { - rsctrl::core::Person *person = resp.add_peers(); - load_person_details(pgp_id, person, true, false); - } - } - - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::NO_IMPL_YET); - } - - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoPeers::processAddPeer() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::PEERS, - rsctrl::peers::MsgId_ResponsePeerList, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -int RpcProtoPeers::processExaminePeer(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoPeers::processExaminePeer() NOT FINISHED"; - std::cerr << std::endl; - - - // parse msg. - rsctrl::peers::RequestExaminePeer req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoPeers::processExaminePeer() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::peers::ResponsePeerList resp; - bool success = false; - - if (success) - { - switch(req.cmd()) - { - default: - success = false; - break; - case rsctrl::peers::RequestExaminePeer::IMPORT: - break; - case rsctrl::peers::RequestExaminePeer::EXAMINE: - - // Gets the GPG details, but does not add the key to the keyring. - //virtual bool loadDetailsFromStringCert(const std::string& certGPG, RsPeerDetails &pd,uint32_t& error_code) = 0; - - break; - } - } - - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::NO_IMPL_YET); - } - - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoPeers::processAddPeer() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::PEERS, - rsctrl::peers::MsgId_ResponsePeerList, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -int RpcProtoPeers::processModifyPeer(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoPeers::processModifyPeer() NOT FINISHED"; - std::cerr << std::endl; - - - // parse msg. - rsctrl::peers::RequestModifyPeer req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoPeers::processModifyPeer() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - - // response. - rsctrl::peers::ResponsePeerList resp; - bool success = false; - - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::NO_IMPL_YET); - } - - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoPeers::processModifyPeer() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::PEERS, - rsctrl::peers::MsgId_ResponsePeerList, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - -int RpcProtoPeers::processRequestPeers(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoPeers::processRequestPeers()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::peers::RequestPeers reqp; - if (!reqp.ParseFromString(msg)) - { - std::cerr << "RpcProtoPeers::processRequestPeers() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::peers::ResponsePeerList respp; - bool success = true; - std::string errorMsg; - - // Get the list of gpg_id to generate data for. - std::list ids; - bool onlyConnected = false; - switch(reqp.set()) - { - case rsctrl::peers::RequestPeers::OWNID: - { - std::cerr << "RpcProtoPeers::processRequestPeers() OWNID"; - std::cerr << std::endl; - std::string own_id = rsPeers->getGPGOwnId(); - ids.push_back(own_id); - break; - } - case rsctrl::peers::RequestPeers::LISTED: - { - std::cerr << "RpcProtoPeers::processRequestPeers() LISTED"; - std::cerr << std::endl; - int no_pgp_ids = reqp.pgp_ids_size(); - for (int i = 0; i < no_pgp_ids; i++) - { - std::string listed_id = reqp.pgp_ids(i); - std::cerr << "RpcProtoPeers::processRequestPeers() Adding Id: " << listed_id; - std::cerr << std::endl; - ids.push_back(listed_id); - } - break; - - } - case rsctrl::peers::RequestPeers::ALL: - std::cerr << "RpcProtoPeers::processRequestPeers() ALL"; - std::cerr << std::endl; - rsPeers->getGPGAllList(ids); - break; - case rsctrl::peers::RequestPeers::CONNECTED: - { - std::cerr << "RpcProtoPeers::processRequestPeers() CONNECTED"; - std::cerr << std::endl; - /* this ones a bit hard too */ - onlyConnected = true; - std::list ssl_ids; - std::list::const_iterator sit; - rsPeers->getOnlineList(ssl_ids); - for(sit = ssl_ids.begin(); sit != ssl_ids.end(); sit++) - { - std::string gpg_id = rsPeers->getGPGId(*sit); - if (gpg_id.size() > 0) - { - if (std::find(ids.begin(), ids.end(),gpg_id) == ids.end()) - { - ids.push_back(gpg_id); - } - } - } - break; - } - case rsctrl::peers::RequestPeers::FRIENDS: - std::cerr << "RpcProtoPeers::processRequestPeers() FRIENDS"; - std::cerr << std::endl; - rsPeers->getGPGAcceptedList(ids); - break; - case rsctrl::peers::RequestPeers::SIGNED: - std::cerr << "RpcProtoPeers::processRequestPeers() SIGNED"; - std::cerr << std::endl; - rsPeers->getGPGSignedList(ids); - break; - case rsctrl::peers::RequestPeers::VALID: - std::cerr << "RpcProtoPeers::processRequestPeers() VALID"; - std::cerr << std::endl; - rsPeers->getGPGSignedList(ids); - break; - } - - - // work out what data we need to request. - bool getLocations = false; - switch(reqp.info()) - { - default: - case rsctrl::peers::RequestPeers::NAMEONLY: - case rsctrl::peers::RequestPeers::BASIC: - break; - case rsctrl::peers::RequestPeers::LOCATION: - case rsctrl::peers::RequestPeers::ALLINFO: - getLocations = true; - break; - } - - - /* now iterate through the peers and fill in the response. */ - std::list::const_iterator git; - for(git = ids.begin(); git != ids.end(); git++) - { - rsctrl::core::Person *person = respp.add_peers(); - if (!load_person_details(*git, person, getLocations, onlyConnected)) - { - std::cerr << "RpcProtoPeers::processRequestPeers() ERROR Finding GPGID: "; - std::cerr << *git; - std::cerr << std::endl; - - /* cleanup peers */ - success = false; - errorMsg = "Error Loading PeerID"; - } - } - - if (success) - { - rsctrl::core::Status *status = respp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = respp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - - std::string outmsg; - if (!respp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoPeers::processRequestPeers() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::PEERS, - rsctrl::peers::MsgId_ResponsePeerList, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - - - -bool load_person_details(std::string pgp_id, rsctrl::core::Person *person, - bool getLocations, bool onlyConnected) -{ - RsPeerDetails details; - if (!rsPeers->getGPGDetails(pgp_id, details)) - { - std::cerr << "RpcProtoPeers::processRequestPeers() ERROR Finding GPGID: "; - std::cerr << pgp_id; - std::cerr << std::endl; - return false; - } - - /* fill in key gpg details */ - person->set_gpg_id(pgp_id); - person->set_name(details.name); - - std::cerr << "RpcProtoPeers::processRequestPeers() Adding GPGID: "; - std::cerr << pgp_id << " name: " << details.name; - std::cerr << std::endl; - - //if (details.state & RS_PEER_STATE_FRIEND) - if (pgp_id == rsPeers->getGPGOwnId()) - { - std::cerr << "RpcProtoPeers::processRequestPeers() Relation YOURSELF"; - std::cerr << std::endl; - person->set_relation(rsctrl::core::Person::YOURSELF); - } - else if (rsPeers->isGPGAccepted(pgp_id)) - { - std::cerr << "RpcProtoPeers::processRequestPeers() Relation FRIEND"; - std::cerr << std::endl; - person->set_relation(rsctrl::core::Person::FRIEND); - } - else - { - std::list common_friends; - rsDisc->getDiscPgpFriends(pgp_id, common_friends); - int size = common_friends.size(); - if (size) - { - if (size > 2) - { - std::cerr << "RpcProtoPeers::processRequestPeers() Relation FRIEND_OF_MANY_FRIENDS"; - std::cerr << std::endl; - person->set_relation(rsctrl::core::Person::FRIEND_OF_MANY_FRIENDS); - } - else - { - std::cerr << "RpcProtoPeers::processRequestPeers() Relation FRIEND_OF_FRIENDS"; - std::cerr << std::endl; - person->set_relation(rsctrl::core::Person::FRIEND_OF_FRIENDS); - } - } - else - { - std::cerr << "RpcProtoPeers::processRequestPeers() Relation UNKNOWN"; - std::cerr << std::endl; - person->set_relation(rsctrl::core::Person::UNKNOWN); - } - } - - if (getLocations) - { - std::list ssl_ids; - std::list::const_iterator sit; - - if (!rsPeers->getAssociatedSSLIds(pgp_id, ssl_ids)) - { - std::cerr << "RpcProtoPeers::processRequestPeers() No Locations"; - std::cerr << std::endl; - return true; /* end of this peer */ - } - - for(sit = ssl_ids.begin(); sit != ssl_ids.end(); sit++) - { - RsPeerDetails ssldetails; - if (!rsPeers->getPeerDetails(*sit, ssldetails)) - { - continue; /* uhm.. */ - } - if ((onlyConnected) && - (!(ssldetails.state & RS_PEER_STATE_CONNECTED))) - { - continue; - } - - rsctrl::core::Location *loc = person->add_locations(); - - std::cerr << "RpcProtoPeers::processRequestPeers() \t Adding Location: "; - std::cerr << *sit << " loc: " << ssldetails.location; - std::cerr << std::endl; - - /* fill in ssl details */ - loc->set_ssl_id(*sit); - loc->set_location(ssldetails.location); - - /* set addresses */ - rsctrl::core::IpAddr *laddr = loc->mutable_localaddr(); - laddr->set_addr(ssldetails.localAddr); - laddr->set_port(ssldetails.localPort); - - rsctrl::core::IpAddr *eaddr = loc->mutable_extaddr(); - eaddr->set_addr(ssldetails.extAddr); - eaddr->set_port(ssldetails.extPort); - - /* translate status */ - uint32_t loc_state = 0; - //dont think this state should be here. - //if (ssldetails.state & RS_PEER_STATE_FRIEND) - if (ssldetails.state & RS_PEER_STATE_ONLINE) - { - loc_state |= (uint32_t) rsctrl::core::Location::ONLINE; - } - if (ssldetails.state & RS_PEER_STATE_CONNECTED) - { - loc_state |= (uint32_t) rsctrl::core::Location::CONNECTED; - } - if (ssldetails.state & RS_PEER_STATE_UNREACHABLE) - { - loc_state |= (uint32_t) rsctrl::core::Location::UNREACHABLE; - } - - loc->set_state(loc_state); - } - } - return true; /* end of this peer */ -} - - diff --git a/retroshare-nogui/src/rpc/proto/rpcprotopeers.h b/retroshare-nogui/src/rpc/proto/rpcprotopeers.h deleted file mode 100644 index 7e15e0b07..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotopeers.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_PROTO_PEERS_H -#define RS_RPC_PROTO_PEERS_H - -#include "rpc/rpcserver.h" - -class RpcProtoPeers: public RpcQueueService -{ -public: - RpcProtoPeers(uint32_t serviceId); -// virtual msgsAccepted(std::list &msgIds); /* not used at the moment */ - virtual int processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - - virtual int processRequestPeers(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - virtual int processAddPeer(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - - // these aren't implemented yet. - virtual int processExaminePeer(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - virtual int processModifyPeer(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); -}; - - -#endif /* RS_PROTO_PEERS_H */ diff --git a/retroshare-nogui/src/rpc/proto/rpcprotosearch.cc b/retroshare-nogui/src/rpc/proto/rpcprotosearch.cc deleted file mode 100644 index 77276ae12..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotosearch.cc +++ /dev/null @@ -1,678 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/proto/rpcprotosearch.h" -#include "rpc/proto/gencc/search.pb.h" - -#include "notifytxt.h" - -#include -#include - -#include "util/rsstring.h" - -#include - -#include -#include - -#include - -bool condenseSearchResults(const std::list &searchResults, uint32_t limit, - rsctrl::search::SearchSet *result_set); - - -RpcProtoSearch::RpcProtoSearch(uint32_t serviceId, NotifyTxt *notify) - :RpcQueueService(serviceId), mNotify(notify), searchMtx("RpcProtoSearch") -{ - return; -} - -void RpcProtoSearch::reset(uint32_t chan_id) -{ - RpcQueueService::reset(chan_id); - - /* must clear all searches */ - clear_searches(chan_id); -} - - -int RpcProtoSearch::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - /* check the msgId */ - uint8_t topbyte = getRpcMsgIdExtension(msg_id); - uint16_t service = getRpcMsgIdService(msg_id); - uint8_t submsg = getRpcMsgIdSubMsg(msg_id); - bool isResponse = isRpcMsgIdResponse(msg_id); - - - std::cerr << "RpcProtoSearch::processMsg() topbyte: " << (int32_t) topbyte; - std::cerr << " service: " << (int32_t) service << " submsg: " << (int32_t) submsg; - std::cerr << std::endl; - - if (isResponse) - { - std::cerr << "RpcProtoSearch::processMsg() isResponse() - not processing"; - std::cerr << std::endl; - return 0; - } - - if (topbyte != (uint8_t) rsctrl::core::CORE) - { - std::cerr << "RpcProtoSearch::processMsg() Extension Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (service != (uint16_t) rsctrl::core::SEARCH) - { - std::cerr << "RpcProtoSearch::processMsg() Service Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (!rsctrl::search::RequestMsgIds_IsValid(submsg)) - { - std::cerr << "RpcProtoSearch::processMsg() SubMsg Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - switch(submsg) - { - case rsctrl::search::MsgId_RequestBasicSearch: - processReqBasicSearch(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::search::MsgId_RequestCloseSearch: - processReqCloseSearch(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::search::MsgId_RequestListSearches: - processReqListSearches(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::search::MsgId_RequestSearchResults: - processReqSearchResults(chan_id, msg_id, req_id, msg); - break; - - default: - std::cerr << "RpcProtoSearch::processMsg() ERROR should never get here"; - std::cerr << std::endl; - return 0; - } - - /* must have matched id to get here */ - return 1; -} - - - -int RpcProtoSearch::processReqBasicSearch(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoSearch::processReqBasicSearch()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::search::RequestBasicSearch req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoSearch::processReqBasicSearch() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::search::ResponseSearchIds resp; - bool success = true; - std::string errorMsg; - - /* convert msg parameters into local ones */ - std::list terms; - - int no_terms = req.terms_size(); - for(int i = 0; i < no_terms; i++) - { - std::string term = req.terms(i); - /* check for valid term? */ - if (term.size() > 0) - { - terms.push_back(term); - } - } - - NameExpression nameexp(ContainsAllStrings, terms, true); - LinearizedExpression lexpr; - nameexp.linearize(lexpr); - - uint32_t searchId = (uint32_t) rsTurtle->turtleSearch(lexpr); - mNotify->collectSearchResults(searchId); - - /* add into search array */ - add_search(chan_id, searchId); - - /* add to answer */ - resp.add_search_id(searchId); - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoSearch::processReqBasicSearch() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::SEARCH, - rsctrl::search::MsgId_ResponseSearchIds, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -int RpcProtoSearch::processReqCloseSearch(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoSearch::processReqCloseSearch()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::search::RequestCloseSearch req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoSearch::processReqCloseSearch() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::search::ResponseSearchIds resp; - bool success = true; - std::string errorMsg; - - /* convert msg parameters into local ones */ - uint32_t searchId = req.search_id(); - - - /* remove into search array */ - if (!remove_search(chan_id, searchId)) - { - success = false; - errorMsg = "Unknown SearchId in List"; - } - - /* clear search results - * we cannot cancel a turtle search - * so we tell notify to ignore further results - */ - - if (success) - { - if (!mNotify->clearSearchId(searchId)) - { - success = false; - errorMsg = "Unknown SearchId in Notify"; - } - } - - /* add to answer */ - if (success) - { - resp.add_search_id(searchId); - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoSearch::processReqCloseSearch() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::SEARCH, - rsctrl::search::MsgId_ResponseSearchIds, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -int RpcProtoSearch::processReqListSearches(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoSearch::processReqListSearches()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::search::RequestListSearches req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoSearch::processReqListSearches() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::search::ResponseSearchIds resp; - bool success = true; - std::string errorMsg; - - /* convert msg parameters into local ones */ - // Nothing to do. - - std::list reg_search_ids; - std::list::iterator it; - if (!get_search_list(chan_id, reg_search_ids)) - { - /* warning */ - success = false; - errorMsg = "No Searches Active"; - } - - /* iterate through search array */ - for(it = reg_search_ids.begin(); it != reg_search_ids.end(); it++) - { - /* add to answer */ - resp.add_search_id(*it); - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoSearch::processReqListSearches() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::SEARCH, - rsctrl::search::MsgId_ResponseSearchIds, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -int RpcProtoSearch::processReqSearchResults(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoSearch::processReqSearchResults()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::search::RequestSearchResults req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoSearch::processReqSearchResults() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::search::ResponseSearchResults resp; - bool success = true; - std::string errorMsg; - - /* convert msg parameters into local ones */ - std::list reg_search_ids; - std::list requested_search_ids; - if (!get_search_list(chan_id, reg_search_ids)) - { - /* warning */ - } - - int no_searches = req.search_ids_size(); - if (no_searches) - { - /* painful check that they are our searches */ - for(int i = 0; i < no_searches; i++) - { - uint32_t search_id = req.search_ids(i); - - /* check that its in reg_search_ids */ - if (reg_search_ids.end() != std::find(reg_search_ids.begin(), reg_search_ids.end(), search_id)) - { - /* error */ - continue; - } - requested_search_ids.push_back(search_id); - } - } - else - { - /* all current searches */ - requested_search_ids = reg_search_ids; - } - - - std::list::iterator rit; - for(rit = requested_search_ids.begin(); - rit != requested_search_ids.end(); rit++) - { - rsctrl::search::SearchSet *set = resp.add_searches(); - /* add to answer */ - set->set_search_id(*rit); - /* no search details at the moment */ - - /* add into search array */ - std::list::iterator it; - std::list searchResults; - mNotify->getSearchResults(*rit, searchResults); - - condenseSearchResults(searchResults, req.result_limit(), set); - } - - /* DONE - Generate Reply */ - /* different to others - partial success possible */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoSearch::processReqSearchResults() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::SEARCH, - rsctrl::search::MsgId_ResponseSearchResults, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - -/***** HELPER FUNCTIONS *****/ - - // These private functions use Mutex below and manipulate mActiveSearches. -int RpcProtoSearch::get_search_list(uint32_t chan_id, std::list &search_ids) -{ - std::cerr << "RpcProtoSearch::get_search_list(" << chan_id << ")"; - std::cerr << std::endl; - - RsStackMutex stack(searchMtx); /******* LOCKED *********/ - - std::map >::iterator mit; - - mit = mActiveSearches.find(chan_id); - if (mit == mActiveSearches.end()) - { - return 0; - } - - search_ids = mit->second; - - return 1; -} - -int RpcProtoSearch::add_search(uint32_t chan_id, uint32_t search_id) -{ - std::cerr << "RpcProtoSearch::add_search(" << chan_id << ", " << search_id << ")"; - std::cerr << std::endl; - - RsStackMutex stack(searchMtx); /******* LOCKED *********/ - - std::map >::iterator mit; - - mit = mActiveSearches.find(chan_id); - if (mit == mActiveSearches.end()) - { - std::list emptyList; - mActiveSearches[chan_id] = emptyList; - - mit = mActiveSearches.find(chan_id); - } - - /* sanity check */ - if (mit->second.end() != std::find(mit->second.begin(), mit->second.end(), search_id)) - { - std::cerr << "RpcProtoSearch::add_search() ERROR search_id already exists"; - std::cerr << std::endl; - return 0; - } - - mit->second.push_back(search_id); - return 1; -} - -int RpcProtoSearch::remove_search(uint32_t chan_id, uint32_t search_id) -{ - std::cerr << "RpcProtoSearch::remove_search(" << chan_id << ", " << search_id << ")"; - std::cerr << std::endl; - - RsStackMutex stack(searchMtx); /******* LOCKED *********/ - - std::map >::iterator mit; - - mit = mActiveSearches.find(chan_id); - if (mit == mActiveSearches.end()) - { - std::cerr << "RpcProtoSearch::remove_search() ERROR search set doesn't exist"; - std::cerr << std::endl; - return 0; - } - - bool removed = false; - std::list::iterator lit; - for(lit = mit->second.begin(); lit != mit->second.end();) - { - if (*lit == search_id) - { - lit = mit->second.erase(lit); - if (removed) - { - std::cerr << "RpcProtoSearch::remove_search() ERROR removed multiple"; - std::cerr << std::endl; - } - removed = true; - } - else - { - lit++; - } - } - - if (removed) - return 1; - - std::cerr << "RpcProtoSearch::remove_search() ERROR search_id not found"; - std::cerr << std::endl; - - return 0; -} - -int RpcProtoSearch::clear_searches(uint32_t chan_id) -{ - std::cerr << "RpcProtoSearch::clear_searches(" << chan_id << ")"; - std::cerr << std::endl; - - RsStackMutex stack(searchMtx); /******* LOCKED *********/ - - std::map >::iterator mit; - - mit = mActiveSearches.find(chan_id); - if (mit == mActiveSearches.end()) - { - std::cerr << "RpcProtoSearch::clear_searches() WARNING search set not found"; - std::cerr << std::endl; - return 0; - } - - mActiveSearches.erase(mit); - return 1; -} - - - -class RpcSearchInfo -{ - public: - std::string hash; - std::string name; - uint64_t size; - std::map name_map; - uint32_t hits; -}; - - -bool condenseSearchResults(const std::list &searchResults, uint32_t limit, - rsctrl::search::SearchSet *result_set) -{ - std::map searchMap; - std::map::iterator mit; - - std::list::const_iterator it; - for(it = searchResults.begin(); it != searchResults.end(); it++) - { - mit = searchMap.find(it->hash); - if (mit != searchMap.end()) - { - mit->second.hits++; - - if (mit->second.name_map.find(it->name) == mit->second.name_map.end()) - { - mit->second.name_map[it->name] = 1; - } - else - { - mit->second.name_map[it->name]++; - } - - if (it->size != mit->second.size) - { - // ERROR. - } - } - else - { - RpcSearchInfo info; - info.hash = it->hash; - info.size = it->size; - info.name_map[it->name] = 1; - info.hits = 1; - - searchMap[it->hash] = info; - } - } - - unsigned int i = 0; - for(mit = searchMap.begin(); (mit != searchMap.end()) && (i < limit || limit == 0); mit++, i++) - { - std::map::reverse_iterator nit; - nit = mit->second.name_map.rbegin(); - - /* add to answer */ - rsctrl::search::SearchHit *hit = result_set->add_hits(); - rsctrl::core::File *file = hit->mutable_file(); - - file->set_hash(mit->second.hash); - file->set_name(nit->first); - file->set_size(mit->second.size); - - // Uhm not provided for now. default to NETWORK - hit->set_loc(rsctrl::search::SearchHit::NETWORK); - hit->set_no_hits(mit->second.hits); // No aggregation yet. - - // guarenteed to have one item here. - for(nit++; nit != mit->second.name_map.rend(); nit++) - { - hit->add_alt_names(nit->first); - } - } - - return true; -} - - - - - - - - - - - - - - - - diff --git a/retroshare-nogui/src/rpc/proto/rpcprotosearch.h b/retroshare-nogui/src/rpc/proto/rpcprotosearch.h deleted file mode 100644 index 3ad77ce75..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotosearch.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_PROTO_SEARCH_H -#define RS_RPC_PROTO_SEARCH_H - -#include "rpc/rpcserver.h" - -class NotifyTxt; - -class RpcProtoSearch: public RpcQueueService -{ -public: - RpcProtoSearch(uint32_t serviceId, NotifyTxt *notify); - virtual void reset(uint32_t chan_id); - - virtual int processMsg(uint32_t chan_id, uint32_t msgId, uint32_t req_id, const std::string &msg); - -protected: - - int processReqBasicSearch(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqCloseSearch(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqListSearches(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqSearchResults(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - - -private: - // These private functions use Mutex below and manipulate mActiveSearches. - int get_search_list(uint32_t chan_id, std::list &search_ids); - int add_search(uint32_t chan_id, uint32_t search_id); - int remove_search(uint32_t chan_id, uint32_t search_id); - int clear_searches(uint32_t chan_id); - - NotifyTxt *mNotify; - - RsMutex searchMtx; - - /* must store list of active searches per channel */ - std::map > mActiveSearches; - -}; - -#endif /* RS_PROTO_SEARCH_H */ diff --git a/retroshare-nogui/src/rpc/proto/rpcprotostream.cc b/retroshare-nogui/src/rpc/proto/rpcprotostream.cc deleted file mode 100644 index 37980ba2d..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotostream.cc +++ /dev/null @@ -1,825 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/proto/rpcprotostream.h" -#include "rpc/proto/rpcprotoutils.h" - -#include "rpc/proto/gencc/stream.pb.h" -#include "rpc/proto/gencc/core.pb.h" - -#include -#include - -// from libretroshare -#include "util/rsdir.h" - -//#include -//#include -//#include - -#include "util/rsstring.h" - -#include - -#include -#include - -#include - -#define MAX_DESIRED_RATE 1000.0 // 1Mb/s - -#define MIN_STREAM_CHUNK_SIZE 10 -#define MAX_STREAM_CHUNK_SIZE 100000 - -#define STREAM_STANDARD_MIN_DT 0.1 -#define STREAM_BACKGROUND_MIN_DT 0.5 - - -bool fill_stream_details(rsctrl::stream::ResponseStreamDetail &resp, - const std::list &streams); -bool fill_stream_desc(rsctrl::stream::StreamDesc &desc, - const RpcStream &stream); - -bool fill_stream_data(rsctrl::stream::StreamData &data, const RpcStream &stream); - -bool createQueuedStreamMsg(const RpcStream &stream, rsctrl::stream::ResponseStreamData &resp, RpcQueuedMsg &qmsg); - - -RpcProtoStream::RpcProtoStream(uint32_t serviceId) - :RpcQueueService(serviceId) -{ - mNextStreamId = 1; - - return; -} - - -void RpcProtoStream::reset(uint32_t chan_id) -{ - // We should be using a mutex for all stream operations!!!! - // TODO - //RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::list toRemove; - std::map::iterator it; - for(it = mStreams.begin(); it != mStreams.end(); it++) - { - if (it->second.chan_id == chan_id) - { - toRemove.push_back(it->first); - } - } - - std::list::iterator rit; - for(rit = toRemove.begin(); rit != toRemove.end(); rit++) - { - it = mStreams.find(*rit); - if (it != mStreams.end()) - { - mStreams.erase(it); - } - } - - // Call the rest of reset. - RpcQueueService::reset(chan_id); -} - - - - - -//RpcProtoStream::msgsAccepted(std::list &msgIds); /* not used at the moment */ - -int RpcProtoStream::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - /* check the msgId */ - uint8_t topbyte = getRpcMsgIdExtension(msg_id); - uint16_t service = getRpcMsgIdService(msg_id); - uint8_t submsg = getRpcMsgIdSubMsg(msg_id); - bool isResponse = isRpcMsgIdResponse(msg_id); - - - std::cerr << "RpcProtoStream::processMsg() topbyte: " << (int32_t) topbyte; - std::cerr << " service: " << (int32_t) service << " submsg: " << (int32_t) submsg; - std::cerr << std::endl; - - if (isResponse) - { - std::cerr << "RpcProtoStream::processMsg() isResponse() - not processing"; - std::cerr << std::endl; - return 0; - } - - if (topbyte != (uint8_t) rsctrl::core::CORE) - { - std::cerr << "RpcProtoStream::processMsg() Extension Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (service != (uint16_t) rsctrl::core::STREAM) - { - std::cerr << "RpcProtoStream::processMsg() Service Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (!rsctrl::stream::RequestMsgIds_IsValid(submsg)) - { - std::cerr << "RpcProtoStream::processMsg() SubMsg Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - switch(submsg) - { - - case rsctrl::stream::MsgId_RequestStartFileStream: - processReqStartFileStream(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::stream::MsgId_RequestControlStream: - processReqControlStream(chan_id, msg_id, req_id, msg); - break; - - case rsctrl::stream::MsgId_RequestListStreams: - processReqListStreams(chan_id, msg_id, req_id, msg); - break; - -// case rsctrl::stream::MsgId_RequestRegisterStreams: -// processReqRegisterStreams(chan_id, msg_id, req_id, msg); -// break; - - default: - std::cerr << "RpcProtoStream::processMsg() ERROR should never get here"; - std::cerr << std::endl; - return 0; - } - - /* must have matched id to get here */ - return 1; -} - - - -int RpcProtoStream::processReqStartFileStream(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoStream::processReqStartFileStream()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::stream::RequestStartFileStream req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoStream::processReqStartFileStream() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::stream::ResponseStreamDetail resp; - bool success = true; - std::string errorMsg; - - - // SETUP STREAM. - - // FIND the FILE. - std::list hashes; - hashes.push_back(req.file().hash()); - - //HashExpression exp(StringOperator::EqualsString, hashes); - HashExpression exp(EqualsString, hashes); - std::list results; - - FileSearchFlags flags = RS_FILE_HINTS_LOCAL; - int ans = rsFiles->SearchBoolExp(&exp, results, flags); - - // CREATE A STREAM OBJECT. - if (results.size() < 1) - { - success = false; - errorMsg = "No Matching File"; - } - else - { - DirDetails &dirdetail = results.front(); - - RpcStream stream; - stream.chan_id = chan_id; - stream.req_id = req_id; - stream.stream_id = getNextStreamId(); - stream.state = RpcStream::RUNNING; - - // Convert to Full local path. - std::string virtual_path = RsDirUtil::makePath(dirdetail.path, dirdetail.name); - if (!rsFiles->ConvertSharedFilePath(virtual_path, stream.path)) - { - success = false; - errorMsg = "Cannot Match to Shared Directory"; - } - - stream.length = dirdetail.count; - stream.hash = dirdetail.hash; - stream.name = dirdetail.name; - - stream.offset = 0; - stream.start_byte = 0; - stream.end_byte = stream.length; - stream.desired_rate = req.rate_kbs(); - - if (stream.desired_rate > MAX_DESIRED_RATE) - { - stream.desired_rate = MAX_DESIRED_RATE; - } - - // make response - rsctrl::stream::StreamDesc *desc = resp.add_streams(); - if (!fill_stream_desc(*desc, stream)) - { - success = false; - errorMsg = "Failed to Invalid Action"; - } - else - { - // insert. - mStreams[stream.stream_id] = stream; - - // register the stream too. - std::cerr << "RpcProtoStream::processReqStartFileStream() Registering the stream event."; - std::cerr << std::endl; - registerForEvents(chan_id, req_id, REGISTRATION_STREAMS); - } - - std::cerr << "RpcProtoStream::processReqStartFileStream() List of Registered Streams:"; - std::cerr << std::endl; - printEventRegister(std::cerr); - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoStream::processReqStartFileStream() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::STREAM, - rsctrl::stream::MsgId_ResponseStreamDetail, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - return 1; -} - - - -int RpcProtoStream::processReqControlStream(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoStream::processReqControlStream()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::stream::RequestControlStream req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoStream::processReqControlStream() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::stream::ResponseStreamDetail resp; - bool success = true; - std::string errorMsg; - - // FIND MATCHING STREAM. - std::map::iterator it; - it = mStreams.find(req.stream_id()); - if (it != mStreams.end()) - { - // TWEAK - - if (it->second.state == RpcStream::STREAMERR) - { - if (req.action() == rsctrl::stream::RequestControlStream::STREAM_STOP) - { - it->second.state = RpcStream::FINISHED; - } - else - { - success = false; - errorMsg = "Stream Error"; - } - } - else - { - switch(req.action()) - { - case rsctrl::stream::RequestControlStream::STREAM_START: - if (it->second.state == RpcStream::PAUSED) - { - it->second.state = RpcStream::RUNNING; - } - break; - - case rsctrl::stream::RequestControlStream::STREAM_STOP: - it->second.state = RpcStream::FINISHED; - break; - - case rsctrl::stream::RequestControlStream::STREAM_PAUSE: - if (it->second.state == RpcStream::RUNNING) - { - it->second.state = RpcStream::PAUSED; - it->second.transfer_time = 0; // reset timings. - } - break; - - case rsctrl::stream::RequestControlStream::STREAM_CHANGE_RATE: - it->second.desired_rate = req.rate_kbs(); - break; - - case rsctrl::stream::RequestControlStream::STREAM_SEEK: - if (req.seek_byte() < it->second.end_byte) - { - it->second.offset = req.seek_byte(); - } - break; - default: - success = false; - errorMsg = "Invalid Action"; - } - } - - // FILL IN REPLY. - if (success) - { - rsctrl::stream::StreamDesc *desc = resp.add_streams(); - if (!fill_stream_desc(*desc, it->second)) - { - success = false; - errorMsg = "Invalid Action"; - } - } - - // Cleanup - TODO, this is explicit at the moment. - should be automatic after finish. - if (it->second.state == RpcStream::FINISHED) - { - deregisterForEvents(it->second.chan_id, it->second.req_id, REGISTRATION_STREAMS); - mStreams.erase(it); - } - } - else - { - success = false; - errorMsg = "No Matching Stream"; - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoStream::processReqControlStream() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::STREAM, - rsctrl::stream::MsgId_ResponseStreamDetail, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - -int RpcProtoStream::processReqListStreams(uint32_t chan_id, uint32_t /*msg_id*/, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoStream::processReqListStreams()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::stream::RequestListStreams req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoStream::processReqListStreams() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // response. - rsctrl::stream::ResponseStreamDetail resp; - bool success = false; - std::string errorMsg; - - - - std::map::iterator it; - for(it = mStreams.begin(); it != mStreams.end(); it++) - { - bool match = true; - - // TODO fill in match! - /* check that it matches */ - if (! match) - { - continue; - } - - rsctrl::stream::StreamDesc *desc = resp.add_streams(); - if (!fill_stream_desc(*desc, it->second)) - { - success = false; - errorMsg = "Some Details Failed to Fill"; - } - } - - /* DONE - Generate Reply */ - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg(errorMsg); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoStream::processReqListStreams() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::STREAM, - rsctrl::stream::MsgId_ResponseStreamDetail, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - return 1; -} - - - // EVENTS. (STREAMS) -int RpcProtoStream::locked_checkForEvents(uint32_t event, const std::list & /* registered */, std::list &stream_msgs) -{ - /* only one event type for now */ - if (event != REGISTRATION_STREAMS) - { - std::cerr << "ERROR Invalid Stream Event Type"; - std::cerr << std::endl; - - /* error */ - return 0; - } - - /* iterate through streams, and get next chunk of data. - * package up and send it. - * NOTE we'll have to something more complex for VoIP! - */ - - double ts = getTimeStamp(); - double dt = ts - mStreamRates.last_ts; - uint32_t data_sent = 0; - -#define FILTER_K (0.75) - - if (dt < 5.0) //mStreamRates.last_ts != 0) - { - mStreamRates.avg_dt = FILTER_K * mStreamRates.avg_dt - + (1.0 - FILTER_K) * dt; - } - else - { - std::cerr << "RpcProtoStream::locked_checkForEvents() Large dT - resetting avg"; - std::cerr << std::endl; - mStreamRates.avg_dt = 0.0; - } - - mStreamRates.last_ts = ts; - - - std::map::iterator it; - for(it = mStreams.begin(); it != mStreams.end(); it++) - { - RpcStream &stream = it->second; - - if (!(stream.state == RpcStream::RUNNING)) - { - continue; - } - - double stream_dt = ts - stream.transfer_time; - - switch(stream.transfer_type) - { - case RpcStream::REALTIME: - // let it go through always. - break; - - case RpcStream::STANDARD: - if (stream_dt < STREAM_STANDARD_MIN_DT) - { - continue; - } - break; - - - case RpcStream::BACKGROUND: - if (stream_dt < STREAM_BACKGROUND_MIN_DT) - { - continue; - } - break; - } - - if (!stream.transfer_time) - { - std::cerr << "RpcProtoStream::locked_checkForEvents() Null stream.transfer_time .. resetting"; - std::cerr << std::endl; - stream.transfer_avg_dt = STREAM_STANDARD_MIN_DT; - } - else - { - std::cerr << "RpcProtoStream::locked_checkForEvents() stream.transfer_avg_dt: " << stream.transfer_avg_dt; - std::cerr << " stream_dt: " << stream_dt; - std::cerr << std::endl; - stream.transfer_avg_dt = FILTER_K * stream.transfer_avg_dt - + (1.0 - FILTER_K) * stream_dt; - - std::cerr << "RpcProtoStream::locked_checkForEvents() ==> stream.transfer_avg_dt: " << stream.transfer_avg_dt; - std::cerr << std::endl; - } - - uint32_t size = stream.desired_rate * 1000.0 * stream.transfer_avg_dt; - stream.transfer_time = ts; - - - if (size < MIN_STREAM_CHUNK_SIZE) - { - size = MIN_STREAM_CHUNK_SIZE; - } - if (size > MAX_STREAM_CHUNK_SIZE) - { - size = MAX_STREAM_CHUNK_SIZE; - } - - - /* get data */ - uint64_t remaining = stream.end_byte - stream.offset; - if (remaining < size) - { - size = remaining; - stream.state = RpcStream::FINISHED; - std::cerr << "RpcProtoStream::locked_checkForEvents() Sending Remaining: " << size; - std::cerr << std::endl; - } - - std::cerr << "RpcProtoStream::locked_checkForEvents() Handling Stream: " << stream.stream_id << " state: " << stream.state; - std::cerr << std::endl; - std::cerr << "path: " << stream.path; - std::cerr << std::endl; - std::cerr << "offset: " << stream.offset; - std::cerr << " avg_dt: " << stream.transfer_avg_dt; - std::cerr << " x desired_rate: " << stream.desired_rate; - std::cerr << " => chunk_size: " << size; - std::cerr << std::endl; - - /* fill in the answer */ - - rsctrl::stream::ResponseStreamData resp; - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - - rsctrl::stream::StreamData *data = resp.mutable_data(); - data->set_stream_id(stream.stream_id); - - // convert state. - switch(stream.state) - { - case RpcStream::RUNNING: - data->set_stream_state(rsctrl::stream::STREAM_STATE_RUN); - break; - // This case cannot happen. - case RpcStream::PAUSED: - data->set_stream_state(rsctrl::stream::STREAM_STATE_PAUSED); - break; - // This can only happen at last chunk. - default: - case RpcStream::FINISHED: - data->set_stream_state(rsctrl::stream::STREAM_STATE_FINISHED); - break; - } - - - rsctrl::core::Timestamp *ts = data->mutable_send_time(); - setTimeStamp(ts); - - data->set_offset(stream.offset); - data->set_size(size); - - if (fill_stream_data(*data, stream)) - { - - /* increment seek_location - for next request */ - stream.offset += size; - - RpcQueuedMsg qmsg; - if (createQueuedStreamMsg(stream, resp, qmsg)) - { - std::cerr << "Created Stream Msg."; - std::cerr << std::endl; - - stream_msgs.push_back(qmsg); - } - else - { - std::cerr << "ERROR Creating Stream Msg"; - std::cerr << std::endl; - } - } - else - { - stream.state = RpcStream::STREAMERR; - std::cerr << "ERROR Filling Stream Data"; - std::cerr << std::endl; - } - - } - return 1; -} - - - -// TODO -int RpcProtoStream::cleanup_checkForEvents(uint32_t /* event */, const std::list & /* registered */) -{ - std::list to_remove; - std::list::iterator rit; - for(rit = to_remove.begin(); rit != to_remove.end(); rit++) - { - /* kill the stream! */ - std::map::iterator it; - it = mStreams.find(*rit); - if (it != mStreams.end()) - { - mStreams.erase(it); - } - } - return 1; -} - - -/***** HELPER FUNCTIONS *****/ - -bool createQueuedStreamMsg(const RpcStream &stream, rsctrl::stream::ResponseStreamData &resp, RpcQueuedMsg &qmsg) -{ - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoStream::createQueuedEventSendMsg() ERROR SerialiseToString()"; - std::cerr << std::endl; - return false; - } - - // Correctly Name Message. - qmsg.mMsgId = constructMsgId(rsctrl::core::CORE, rsctrl::core::STREAM, - rsctrl::stream::MsgId_ResponseStreamData, true); - - qmsg.mChanId = stream.chan_id; - qmsg.mReqId = stream.req_id; - qmsg.mMsg = outmsg; - - return true; -} - - - - -/****************** NEW HELPER FNS ******************/ - -bool fill_stream_details(rsctrl::stream::ResponseStreamDetail &resp, - const std::list &streams) -{ - std::cerr << "fill_stream_details()"; - std::cerr << std::endl; - - bool val = true; - std::list::const_iterator it; - for (it = streams.begin(); it != streams.end(); it++) - { - rsctrl::stream::StreamDesc *desc = resp.add_streams(); - val &= fill_stream_desc(*desc, *it); - } - - return val; -} - -bool fill_stream_desc(rsctrl::stream::StreamDesc &desc, const RpcStream &stream) -{ - std::cerr << "fill_stream_desc()"; - std::cerr << std::endl; - - return true; -} - -bool fill_stream_data(rsctrl::stream::StreamData &data, const RpcStream &stream) -{ - /* fill the StreamData from stream */ - - /* open file */ - FILE *fd = RsDirUtil::rs_fopen(stream.path.c_str(), "rb"); - if (!fd) - { - std::cerr << "fill_stream_data() Failed to open file: " << stream.path; - std::cerr << std::endl; - return false; - } - - uint32_t data_size = data.size(); - uint64_t base_loc = data.offset(); - void *buffer = malloc(data_size); - - /* seek to correct spot */ - fseeko64(fd, base_loc, SEEK_SET); - - /* copy data into bytes */ - if (1 != fread(buffer, data_size, 1, fd)) - { - std::cerr << "fill_stream_data() Failed to get data. data_size=" << data_size << ", base_loc=" << base_loc << " !"; - std::cerr << std::endl; - free(buffer); - return false; - } - - data.set_stream_data(buffer, data_size); - free(buffer); - - fclose(fd); - - return true; -} - - - - -uint32_t RpcProtoStream::getNextStreamId() -{ - return mNextStreamId++; -} - - diff --git a/retroshare-nogui/src/rpc/proto/rpcprotostream.h b/retroshare-nogui/src/rpc/proto/rpcprotostream.h deleted file mode 100644 index 698e9a52f..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotostream.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_PROTO_STREAM_H -#define RS_RPC_PROTO_STREAM_H - -#include "rpc/rpcserver.h" - -// Registrations. -#define REGISTRATION_STREAMS 1 - -class RpcStream -{ - public: - RpcStream(): chan_id(0), req_id(0), stream_id(0), state(0), - offset(0), length(0), start_byte(0), end_byte(0), desired_rate(0), - transfer_type(0), transfer_time(0), transfer_avg_dt(0) - { return; } - -static const uint32_t STREAMERR = 0x00000; -static const uint32_t RUNNING = 0x00001; -static const uint32_t PAUSED = 0x00002; -static const uint32_t FINISHED = 0x00003; - - uint32_t chan_id; - uint32_t req_id; - uint32_t stream_id; - uint32_t state; - - std::string name; - std::string hash; - std::string path; - - uint64_t offset; // where we currently are. - uint64_t length; // filesize. - - uint64_t start_byte; - uint64_t end_byte; - - float desired_rate; // Kb/s - - - // Transfer Type -static const uint32_t STANDARD = 0x00000; -static const uint32_t REALTIME = 0x00001; -static const uint32_t BACKGROUND = 0x00002; - - uint32_t transfer_type; - double transfer_time; - double transfer_avg_dt; - - -}; - - -class RpcStreamRates -{ - public: - RpcStreamRates(): avg_data_rate(0), avg_dt(1), last_data_rate(0), last_ts(0) { return; } - - double avg_data_rate; - double avg_dt; - - double last_data_rate; - double last_ts; -}; - - - - -class RpcProtoStream: public RpcQueueService -{ -public: - RpcProtoStream(uint32_t serviceId); - virtual int processMsg(uint32_t chan_id, uint32_t msgId, uint32_t req_id, const std::string &msg); - virtual void reset(uint32_t chan_id); - - uint32_t getNextStreamId(); - -protected: - - - int processReqStartFileStream(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqControlStream(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqListStreams(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - int processReqRegisterStreams(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - - - uint32_t mNextStreamId; - - RpcStreamRates mStreamRates; - std::map mStreams; - - // EVENTS. - virtual int locked_checkForEvents(uint32_t event, const std::list ®istered, std::list &events); - - // Not actually used yet. - int cleanup_checkForEvents(uint32_t event, const std::list ®istered); - -}; - - -#endif /* RS_PROTO_STREAM_H */ diff --git a/retroshare-nogui/src/rpc/proto/rpcprotosystem.cc b/retroshare-nogui/src/rpc/proto/rpcprotosystem.cc deleted file mode 100644 index 3bcf885af..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotosystem.cc +++ /dev/null @@ -1,354 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/proto/rpcprotosystem.h" -#include "rpc/proto/gencc/system.pb.h" - -#include -#include -#include -#include - -#include -#include - -// NASTY GLOBAL VARIABLE HACK - NEED TO THINK OF A BETTER SYSTEM. -uint16_t RpcProtoSystem::mExtPort = 0; - -RpcProtoSystem::RpcProtoSystem(uint32_t serviceId) - :RpcQueueService(serviceId) -{ - return; -} - -//RpcProtoSystem::msgsAccepted(std::list &msgIds); /* not used at the moment */ - -int RpcProtoSystem::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - /* check the msgId */ - uint8_t topbyte = getRpcMsgIdExtension(msg_id); - uint16_t service = getRpcMsgIdService(msg_id); - uint8_t submsg = getRpcMsgIdSubMsg(msg_id); - bool isResponse = isRpcMsgIdResponse(msg_id); - - - std::cerr << "RpcProtoSystem::processMsg() topbyte: " << (int32_t) topbyte; - std::cerr << " service: " << (int32_t) service << " submsg: " << (int32_t) submsg; - std::cerr << std::endl; - - if (isResponse) - { - std::cerr << "RpcProtoSystem::processMsg() isResponse() - not processing"; - std::cerr << std::endl; - return 0; - } - - - if (topbyte != (uint8_t) rsctrl::core::CORE) - { - std::cerr << "RpcProtoSystem::processMsg() Extension Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (service != (uint16_t) rsctrl::core::SYSTEM) - { - std::cerr << "RpcProtoSystem::processMsg() Service Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - if (!rsctrl::system::RequestMsgIds_IsValid(submsg)) - { - std::cerr << "RpcProtoSystem::processMsg() SubMsg Mismatch - not processing"; - std::cerr << std::endl; - return 0; - } - - switch(submsg) - { - case rsctrl::system::MsgId_RequestSystemStatus: - processSystemStatus(chan_id, msg_id, req_id, msg); - break; -#if 0 - case rsctrl::system::MsgId_RequestSystemQuit: - processSystemQuit(chan_id, msg_id, req_id, msg); - break; -#endif - case rsctrl::system::MsgId_RequestSystemExternalAccess: - processSystemExternalAccess(chan_id, msg_id, req_id, msg); - break; - default: - std::cerr << "RpcProtoSystem::processMsg() ERROR should never get here"; - std::cerr << std::endl; - return 0; - } - - /* must have matched id to get here */ - return 1; -} - - -int RpcProtoSystem::processSystemStatus(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoSystem::processSystemStatus()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::system::RequestSystemStatus req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoSystem::processSystemStatus() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // NO Options... so go straight to answer. - // response. - rsctrl::system::ResponseSystemStatus resp; - bool success = true; - - unsigned int nTotal = 0; - unsigned int nConnected = 0; - rsPeers->getPeerCount(&nTotal, &nConnected, false); - - float downKb = 0; - float upKb = 0; - rsConfig->GetCurrentDataRates(downKb, upKb); - - // set the data. - resp.set_no_peers(nTotal); - resp.set_no_connected(nConnected); - - rsctrl::core::Bandwidth *bw = resp.mutable_bw_total(); - bw->set_up(upKb); - bw->set_down(downKb); - bw->set_name("Total Connection Bandwidth"); - - uint32_t netState = rsConfig->getNetState(); - std::string natState("Unknown"); - rsctrl::system::ResponseSystemStatus_NetCode protoCode; - - switch(netState) - { - default: - case RSNET_NETSTATE_BAD_UNKNOWN: - protoCode = rsctrl::system::ResponseSystemStatus::BAD_UNKNOWN; - break; - - case RSNET_NETSTATE_BAD_OFFLINE: - protoCode = rsctrl::system::ResponseSystemStatus::BAD_OFFLINE; - break; - - case RSNET_NETSTATE_BAD_NATSYM: - protoCode = rsctrl::system::ResponseSystemStatus::BAD_NATSYM; - break; - - case RSNET_NETSTATE_BAD_NODHT_NAT: - protoCode = rsctrl::system::ResponseSystemStatus::BAD_NODHT_NAT; - break; - - case RSNET_NETSTATE_WARNING_RESTART: - protoCode = rsctrl::system::ResponseSystemStatus::WARNING_RESTART; - break; - - case RSNET_NETSTATE_WARNING_NATTED: - protoCode = rsctrl::system::ResponseSystemStatus::WARNING_NATTED; - break; - - case RSNET_NETSTATE_WARNING_NODHT: - protoCode = rsctrl::system::ResponseSystemStatus::WARNING_NODHT; - break; - - case RSNET_NETSTATE_GOOD: - protoCode = rsctrl::system::ResponseSystemStatus::GOOD; - break; - - case RSNET_NETSTATE_ADV_FORWARD: - protoCode = rsctrl::system::ResponseSystemStatus::ADV_FORWARD; - break; - } - - resp.set_net_status(protoCode); - - - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg("Unknown ERROR"); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoSystem::processSystemStatus() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::SYSTEM, - rsctrl::system::MsgId_ResponseSystemStatus, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - - -int RpcProtoSystem::processSystemQuit(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoSystem::processSystemQuit()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::system::RequestSystemQuit req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoSystem::processSystemQuit() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // NO Options... so go straight to answer. - // response. - rsctrl::system::ResponseSystemQuit resp; - bool success = true; - - switch(req.quit_code()) - { - default: - case rsctrl::system::RequestSystemQuit::CLOSE_CHANNEL: - { - RpcServer *server = getRpcServer(); - server->error(chan_id, "CLOSE_CHANNEL"); - - break; - } - case rsctrl::system::RequestSystemQuit::SHUTDOWN_RS: - { - RsControl::instance()->rsGlobalShutDown(); - break; - } - } - - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg("Unknown ERROR"); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoSystem::processSystemQuit() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::SYSTEM, - rsctrl::system::MsgId_ResponseSystemQuit, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - -int RpcProtoSystem::processSystemExternalAccess(uint32_t chan_id, uint32_t /* msg_id */, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcProtoSystem::processSystemExternalAccess()"; - std::cerr << std::endl; - - // parse msg. - rsctrl::system::RequestSystemExternalAccess req; - if (!req.ParseFromString(msg)) - { - std::cerr << "RpcProtoSystem::processSystemExternalAccess() ERROR ParseFromString()"; - std::cerr << std::endl; - return 0; - } - - // NO Options... so go straight to answer. - // response. - rsctrl::system::ResponseSystemExternalAccess resp; - bool success = true; - - - std::string dhtKey; - if (!rsDht->getOwnDhtId(dhtKey)) - { - success = false; - } - // have to set something anyway!. - resp.set_dht_key(dhtKey); - // NASTY GLOBAL VARIABLE HACK - NEED TO THINK OF A BETTER SYSTEM. - resp.set_ext_port(mExtPort); - - if (success) - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::SUCCESS); - } - else - { - rsctrl::core::Status *status = resp.mutable_status(); - status->set_code(rsctrl::core::Status::FAILED); - status->set_msg("Unknown ERROR"); - } - - std::string outmsg; - if (!resp.SerializeToString(&outmsg)) - { - std::cerr << "RpcProtoSystem::processSystemExternalAccess() ERROR SerialiseToString()"; - std::cerr << std::endl; - return 0; - } - - // Correctly Name Message. - uint32_t out_msg_id = constructMsgId(rsctrl::core::CORE, rsctrl::core::SYSTEM, - rsctrl::system::MsgId_ResponseSystemExternalAccess, true); - - // queue it. - queueResponse(chan_id, out_msg_id, req_id, outmsg); - - return 1; -} - - diff --git a/retroshare-nogui/src/rpc/proto/rpcprotosystem.h b/retroshare-nogui/src/rpc/proto/rpcprotosystem.h deleted file mode 100644 index 1c2ff36be..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotosystem.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_PROTO_SYSTEM_H -#define RS_RPC_PROTO_SYSTEM_H - -#include "rpc/rpcserver.h" - -class RpcProtoSystem: public RpcQueueService -{ -public: - RpcProtoSystem(uint32_t serviceId); -// virtual msgsAccepted(std::list &msgIds); /* not used at the moment */ - virtual int processMsg(uint32_t chan_id, uint32_t msgId, uint32_t req_id, const std::string &msg); - - virtual int processSystemStatus(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - virtual int processSystemQuit(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - virtual int processSystemExternalAccess(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - -// NASTY GLOBAL VARIABLE HACK - NEED TO THINK OF A BETTER SYSTEM. -static uint16_t mExtPort; - -}; - - -#endif /* RS_PROTO_SYSTEM_H */ diff --git a/retroshare-nogui/src/rpc/proto/rpcprotoutils.cc b/retroshare-nogui/src/rpc/proto/rpcprotoutils.cc deleted file mode 100644 index 4568560dd..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotoutils.cc +++ /dev/null @@ -1,59 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/proto/rpcprotoutils.h" -#include - - -double getTimeStamp() -{ - struct timeval tv; - double ts = 0; - if (0 == gettimeofday(&tv, NULL)) - { - ts = tv.tv_sec + (tv.tv_usec / 1000000.0); - } - return ts; -} - -bool setTimeStamp(rsctrl::core::Timestamp *ts) -{ - struct timeval tv; - if (0 != gettimeofday(&tv, NULL)) - { - ts->set_secs(tv.tv_sec); - ts->set_microsecs(tv.tv_usec); - return true; - } - else - { - ts->set_secs(0); - ts->set_microsecs(0); - return false; - } - return false; -} - - - - diff --git a/retroshare-nogui/src/rpc/proto/rpcprotoutils.h b/retroshare-nogui/src/rpc/proto/rpcprotoutils.h deleted file mode 100644 index 9c9ecf647..000000000 --- a/retroshare-nogui/src/rpc/proto/rpcprotoutils.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_PROTO_UTILS_H -#define RS_RPC_PROTO_UTILS_H - - -#include "rpc/proto/gencc/core.pb.h" - -double getTimeStamp(); -bool setTimeStamp(rsctrl::core::Timestamp *ts); - - -#endif /* RS_RPC_PROTO_UTILS_H */ diff --git a/retroshare-nogui/src/rpc/rpc.cc b/retroshare-nogui/src/rpc/rpc.cc deleted file mode 100644 index a1aa1683b..000000000 --- a/retroshare-nogui/src/rpc/rpc.cc +++ /dev/null @@ -1,307 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/rpc.h" -#include "rpc/rpcserver.h" - -// This one is inside libretroshare (BAD)!!! -#include "serialiser/rsbaseserial.h" - -#include - -const uint32_t kMsgHeaderSize = 16; -const uint32_t kMsgMagicCode = 0x137f0001; // Arbitary + 0x0001 - -RpcMediator::RpcMediator(RpcComms *c) - :mComms(c), mServer(NULL) -{ - return; -} - -void RpcMediator::reset(uint32_t chan_id) -{ - mServer->reset(chan_id); -} - -int RpcMediator::error(uint32_t chan_id, std::string msg) -{ - return mComms->error(chan_id, msg); -} - - -int RpcMediator::tick() -{ - bool worked = false; - if (recv()) - { - worked = true; - } - - if (mServer->checkPending()) - { - worked = true; - } - - if (mServer->checkEvents()) - { - worked = true; - } - - if (worked) - return 1; - else - return 0; - return 0; -} - - -int RpcMediator::recv() -{ - int recvd = 0; - - std::list chan_ids; - std::list::iterator it; - mComms->active_channels(chan_ids); - for(it = chan_ids.begin(); it != chan_ids.end(); it++) - { - while(recv_msg(*it)) - { - recvd = 1; - } - } - return recvd; -} - - -int RpcMediator::recv_msg(uint32_t chan_id) -{ - /* nothing in here needs a Mutex... */ - - if (!mComms->recv_ready(chan_id)) - { - return 0; - } - - std::cerr << "RpcMediator::recv_msg() Data Ready"; - std::cerr << std::endl; - - /* read in data */ - uint8_t buffer[kMsgHeaderSize]; - uint32_t bufsize = kMsgHeaderSize; - uint32_t msg_id; - uint32_t req_id; - uint32_t msg_size; - std::string msg_body; - - std::cerr << "RpcMediator::recv_msg() get Header: " << bufsize; - std::cerr << " bytes" << std::endl; - - int read = mComms->recv_blocking(chan_id, buffer, bufsize); - if (read != bufsize) - { - /* error */ - std::cerr << "RpcMediator::recv_msg() Error Reading Header: " << bufsize; - std::cerr << " bytes" << std::endl; - - mComms->error(chan_id, "Failed to Recv Header"); - return 0; - } - - if (!MsgPacker::deserialiseHeader(msg_id, req_id, msg_size, buffer, bufsize)) - { - /* error */ - std::cerr << "RpcMediator::recv_msg() Error Deserialising Header"; - std::cerr << std::endl; - - mComms->error(chan_id, "Failed to Deserialise Header"); - return 0; - } - - std::cerr << "RpcMediator::recv_msg() ChanId: " << chan_id; - std::cerr << " MsgId: " << msg_id; - std::cerr << " ReqId: " << req_id; - std::cerr << std::endl; - - std::cerr << "RpcMediator::recv_msg() get Body: " << msg_size; - std::cerr << " bytes" << std::endl; - - /* grab real size */ - read = mComms->recv_blocking(chan_id, msg_body, msg_size); - if (read != msg_size) - { - /* error */ - std::cerr << "RpcMediator::recv_msg() Error Reading Body: " << bufsize; - std::cerr << " bytes" << std::endl; - - mComms->error(chan_id, "Failed to Recv MsgBody"); - return 0; - } - mServer->processMsg(chan_id, msg_id, req_id, msg_body); - - return 1; -} - - -int RpcMediator::send(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) - -{ - std::cerr << "RpcMediator::send(" << msg_id << "," << req_id << ", len("; - std::cerr << msg.size() << ")) on chan_id: " << chan_id; - std::cerr << std::endl; - - uint8_t buffer[kMsgHeaderSize]; - uint32_t bufsize = kMsgHeaderSize; - uint32_t msg_size = msg.size(); - - bool okay = MsgPacker::serialiseHeader(msg_id, req_id, msg_size, buffer, bufsize); - if (!okay) - { - std::cerr << "RpcMediator::send() SerialiseHeader Failed"; - std::cerr << std::endl; - /* error */ - return 0; - } - - if (!mComms->send(chan_id, buffer, bufsize)) - { - std::cerr << "RpcMediator::send() Send Header Failed"; - std::cerr << std::endl; - /* error */ - mComms->error(chan_id, "Failed to Send Header"); - return 0; - } - - /* now send the body */ - if (!mComms->send(chan_id, msg)) - { - std::cerr << "RpcMediator::send() Send Body Failed"; - std::cerr << std::endl; - /* error */ - mComms->error(chan_id, "Failed to Send Msg"); - return 0; - } - return 1; -} - - - - -/* Msg Packing */ -int MsgPacker::headersize() -{ - return kMsgHeaderSize; -} - -#if 0 -int MsgPacker::msgsize(Message *msg) -{ - /* */ - return 0; -} - -int MsgPacker::pktsize(Message *msg) -{ - /* */ - return headersize() + msgsize(); -} -#endif - -bool MsgPacker::serialiseHeader(uint32_t msg_id, uint32_t req_id, uint32_t msg_size, uint8_t *buffer, uint32_t bufsize) -{ - /* check size */ - if (bufsize < kMsgHeaderSize) - { - return false; - } - - /* pack the data (using libretroshare serialiser for now */ - void *data = buffer; - uint32_t offset = 0; - uint32_t size = bufsize; - - bool ok = true; - - /* 4 x uint32_t for header */ - ok &= setRawUInt32(data, size, &offset, kMsgMagicCode); - ok &= setRawUInt32(data, size, &offset, msg_id); - ok &= setRawUInt32(data, size, &offset, req_id); - ok &= setRawUInt32(data, size, &offset, msg_size); - - return ok; -} - - -bool MsgPacker::deserialiseHeader(uint32_t &msg_id, uint32_t &req_id, uint32_t &msg_size, uint8_t *buffer, uint32_t bufsize) -{ - /* check size */ - if (bufsize < kMsgHeaderSize) - { - return false; - } - - /* pack the data (using libretroshare serialiser for now */ - void *data = buffer; - uint32_t offset = 0; - uint32_t size = bufsize; - uint32_t magic_code; - - bool ok = true; - - /* 4 x uint32_t for header */ - ok &= getRawUInt32(data, size, &offset, &magic_code); - if (!ok) - { - std::cerr << "Failed to deserialise uint32_t(0)"; - std::cerr << std::endl; - } - ok &= getRawUInt32(data, size, &offset, &msg_id); - if (!ok) - { - std::cerr << "Failed to deserialise uint32_t(1)"; - std::cerr << std::endl; - } - ok &= getRawUInt32(data, size, &offset, &req_id); - if (!ok) - { - std::cerr << "Failed to deserialise uint32_t(2)"; - std::cerr << std::endl; - } - ok &= getRawUInt32(data, size, &offset, &msg_size); - if (!ok) - { - std::cerr << "Failed to deserialise uint32_t(3)"; - std::cerr << std::endl; - } - - ok &= (magic_code == kMsgMagicCode); - if (!ok) - { - std::cerr << "Failed to Match MagicCode"; - std::cerr << std::endl; - } - - return ok; -} - - - diff --git a/retroshare-nogui/src/rpc/rpc.h b/retroshare-nogui/src/rpc/rpc.h deleted file mode 100644 index bdc2e8a18..000000000 --- a/retroshare-nogui/src/rpc/rpc.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef RPC_MEDIATOR_H -#define RPC_MEDIATOR_H - -/* - * Interface between RpcServer and RpcComms. - */ - -#include -#include - -#include "rpcsystem.h" - -class RpcServer; - -class RpcMediator: public RpcSystem -{ -public: - - RpcMediator(RpcComms *c); - void setRpcServer(RpcServer *s) { mServer = s; } /* Must only be called during setup */ - - // Overloaded from RpcSystem. -virtual void reset(uint32_t chan_id); -virtual int tick(); - - int recv(); - int recv_msg(uint32_t chan_id); - int send(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - - int error(uint32_t chan_id, std::string msg); // pass an error up to comms level. -private: - RpcComms *mComms; - RpcServer *mServer; - -}; - - -/* Msg Packing */ -class MsgPacker -{ -public: - static int headersize(); - static bool serialiseHeader(uint32_t msg_id, uint32_t req_id, uint32_t msg_size, uint8_t *buffer, uint32_t bufsize); - static bool deserialiseHeader(uint32_t &msg_id, uint32_t &req_id, uint32_t &msg_size, uint8_t *buffer, uint32_t bufsize); -}; - - - -#endif /* RPC_MEDIATOR_H */ diff --git a/retroshare-nogui/src/rpc/rpcecho.cc b/retroshare-nogui/src/rpc/rpcecho.cc deleted file mode 100644 index b11220bb8..000000000 --- a/retroshare-nogui/src/rpc/rpcecho.cc +++ /dev/null @@ -1,40 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/rpcecho.h" - -RpcEcho::RpcEcho(uint32_t serviceId) -:RpcQueueService(serviceId) -{ - return; -} - -int RpcEcho::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - /* */ - queueResponse(chan_id, msg_id, req_id, msg); - return 1; -} - - - diff --git a/retroshare-nogui/src/rpc/rpcecho.h b/retroshare-nogui/src/rpc/rpcecho.h deleted file mode 100644 index b34446310..000000000 --- a/retroshare-nogui/src/rpc/rpcecho.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_ECHO_H -#define RS_RPC_ECHO_H - -#include "rpc/rpcserver.h" - -class RpcEcho: public RpcQueueService -{ -public: - RpcEcho(uint32_t serviceId); - virtual int processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); -}; - - -#endif /* RS_RPC_ECHO_H */ diff --git a/retroshare-nogui/src/rpc/rpcserver.cc b/retroshare-nogui/src/rpc/rpcserver.cc deleted file mode 100644 index a879e08df..000000000 --- a/retroshare-nogui/src/rpc/rpcserver.cc +++ /dev/null @@ -1,538 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/rpcserver.h" - -#include "rpc/rpc.h" - -#include - - - -bool operator<(const RpcUniqueId &a, const RpcUniqueId &b) -{ - if (a.mChanId == b.mChanId) - return (a.mReqId < b.mReqId); - return (a.mChanId < b.mChanId); -} - - -RpcServer::RpcServer(RpcMediator *med) - :mMediator(med), mRpcMtx("RpcMtx") -{ - -} - -void RpcServer::reset(uint32_t chan_id) -{ - std::cerr << "RpcServer::reset(" << chan_id << ")" << std::endl; - { - RsStackMutex stack(mRpcMtx); /********** LOCKED MUTEX ***************/ - - std::list::iterator it; - for(it = mAllServices.begin(); it != mAllServices.end(); it++) - { - /* in mutex, but should be okay */ - (*it)->reset(chan_id); - } - - // clear existing queue. - mRpcQueue.clear(); - } - - return; -} - -int RpcServer::error(uint32_t chan_id, std::string msg) -{ - return mMediator->error(chan_id, msg); -} - -int RpcServer::addService(RpcService *service) -{ - RsStackMutex stack(mRpcMtx); /********** LOCKED MUTEX ***************/ - - service->setRpcServer(this); - mAllServices.push_back(service); - - return 1; -} - - -int RpcServer::processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - std::cerr << "RpcServer::processMsg(" << msg_id << "," << req_id; - std::cerr << ", len(" << msg.size() << ")) from channel: " << chan_id; - std::cerr << std::endl; - - { - RsStackMutex stack(mRpcMtx); /********** LOCKED MUTEX ***************/ - - std::list::iterator it; - for(it = mAllServices.begin(); it != mAllServices.end(); it++) - { - int rt = (*it)->processMsg(chan_id, msg_id, req_id, msg); - if (!rt) - continue; - - /* remember request */ - queueRequest_locked(chan_id, msg_id, req_id, (*it)); - return 1; - } - } - - std::cerr << "RpcServer::processMsg() No service to accepted it - discard"; - std::cerr << std::endl; - return 0; -} - -int RpcServer::queueRequest_locked(uint32_t chan_id, uint32_t /* msgId */, uint32_t req_id, RpcService *service) -{ - std::cerr << "RpcServer::queueRequest_locked() req_id: " << req_id; - std::cerr << std::endl; - - RpcQueuedObj obj; - obj.mChanId = chan_id; - obj.mReqId = req_id; - obj.mService = service; - - mRpcQueue.push_back(obj); - - return 1; -} - - -bool RpcServer::checkPending() -{ - std::list msgsToSend; - bool someRemaining = false; - bool someToSend = false; - - { - RsStackMutex stack(mRpcMtx); /********** LOCKED MUTEX ***************/ - - std::list::iterator it; - for(it = mRpcQueue.begin(); it != mRpcQueue.end();) - { - uint32_t out_chan_id = it->mChanId; - uint32_t out_msg_id = 0; - uint32_t out_req_id = it->mReqId; - std::string out_msg; - if (it->mService->getResponse(out_chan_id, out_msg_id, out_req_id, out_msg)) - { - std::cerr << "RpcServer::checkPending() Response: ("; - std::cerr << out_msg_id << "," << out_req_id; - std::cerr << ", len(" << out_msg.size() << "))"; - std::cerr << " for chan_id: " << out_chan_id; - std::cerr << std::endl; - - /* store and send after queue is processed */ - RpcQueuedMsg msg; - msg.mChanId = out_chan_id; - msg.mMsgId = out_msg_id; - msg.mReqId = out_req_id; - msg.mMsg = out_msg; - - msgsToSend.push_back(msg); - - it = mRpcQueue.erase(it); - someToSend = true; - } - else - { - it++; - someRemaining = true; - } - } - } - - if (someToSend) - { - sendQueuedMsgs(msgsToSend); - } - - return someRemaining; -} - - -bool RpcServer::checkEvents() -{ - std::list msgsToSend; - bool someToSend = false; - - { - RsStackMutex stack(mRpcMtx); /********** LOCKED MUTEX ***************/ - - std::list::iterator it; - for(it = mAllServices.begin(); it != mAllServices.end(); it++) - { - if ((*it)->getEvents(msgsToSend)) - someToSend = true; - } - } - - if (someToSend) - { - sendQueuedMsgs(msgsToSend); - } - return someToSend; -} - -bool RpcServer::sendQueuedMsgs(std::list &msgs) -{ - /* No need for lock, as mOut is the only accessed item - and that has own protection */ - - std::cerr << "RpcServer::sendQueuedMsg() " << msgs.size() << " to send"; - std::cerr << std::endl; - - std::list::iterator it; - for (it = msgs.begin(); it != msgs.end(); it++) - { - mMediator->send(it->mChanId, it->mMsgId, it->mReqId, it->mMsg); - } - return true; -} - - - - - - - -RpcQueueService::RpcQueueService(uint32_t serviceId) -:RpcService(serviceId), mQueueMtx("RpcQueueService") -{ - return; -} - - -void RpcQueueService::reset(uint32_t chan_id) -{ - clearEventsForChannel(chan_id); - - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::list toRemove; - - // iterate through and remove only chan_id items. - std::map::iterator mit; - for(mit = mResponses.begin(); mit != mResponses.end(); mit++) - { - if (mit->second.mChanId == chan_id) - toRemove.push_back(mit->first); - } - - /* remove items */ - std::list::iterator rit; - for(rit = toRemove.begin(); rit != toRemove.end(); rit++) - { - mit = mResponses.find(*rit); - mResponses.erase(mit); - } - - - return; -} - -int RpcQueueService::getResponse(uint32_t &chan_id, uint32_t &msg_id, uint32_t &req_id, std::string &msg) -{ - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::map::iterator it; - - RpcUniqueId uid(chan_id, req_id); - it = mResponses.find(uid); - if (it == mResponses.end()) - { - return 0; - } - - // chan_id & req_id are already set. - msg_id = it->second.mMsgId; - msg = it->second.mMsg; - - mResponses.erase(it); - - return 1; -} - -int RpcQueueService::queueResponse(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) -{ - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - RpcQueuedMsg qmsg; - qmsg.mChanId = chan_id; - qmsg.mMsgId = msg_id; - qmsg.mReqId = req_id; - qmsg.mMsg = msg; - - RpcUniqueId uid(chan_id, req_id); - mResponses[uid] = qmsg; - - return 1; -} - - - - -/********* Events & Registration ******/ - - -int RpcQueueService::getEvents(std::list &events) -{ - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::map >::iterator it; - for(it = mEventRegister.begin(); it != mEventRegister.end(); it++) - { - locked_checkForEvents(it->first, it->second, events); - } - - if (events.empty()) - { - return 0; - } - return 1; -} - -int RpcQueueService::locked_checkForEvents(uint32_t event, const std::list ®istered, std::list & /* events */) -{ - std::cerr << "RpcQueueService::locked_checkForEvents() NOT IMPLEMENTED"; - std::cerr << std::endl; - std::cerr << "\tRegistered for Event Type: " << event; - std::cerr << std::endl; - - std::list::const_iterator it; - for(it = registered.begin(); it != registered.end(); it++) - { - std::cerr << "\t\t Channel ID: " << it->mChanId; - std::cerr << " Request ID: " << it->mReqId; - std::cerr << std::endl; - } - - return 1; -} - -int RpcQueueService::registerForEvents(uint32_t chan_id, uint32_t req_id, uint32_t event_id) -{ - std::cerr << "RpcQueueService::registerForEvents(ChanId: " << chan_id; - std::cerr << ", ReqId: " << req_id; - std::cerr << ", EventId: " << event_id; - std::cerr << ")"; - std::cerr << std::endl; - - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::map >::iterator mit; - mit = mEventRegister.find(event_id); - if (mit == mEventRegister.end()) - { - std::list emptyList; - mEventRegister[event_id] = emptyList; - - mit = mEventRegister.find(event_id); - } - - RpcEventRegister reg; - reg.mChanId = chan_id; - reg.mReqId = req_id; - reg.mEventId = event_id; - - mit->second.push_back(reg); - - return 1; -} - -int RpcQueueService::deregisterForEvents(uint32_t chan_id, uint32_t req_id, uint32_t event_id) -{ - std::cerr << "RpcQueueService::deregisterForEvents(ChanId: " << chan_id; - std::cerr << ", ReqId: " << req_id; - std::cerr << ", EventId: " << event_id; - std::cerr << ")"; - std::cerr << std::endl; - - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::map >::iterator mit; - mit = mEventRegister.find(event_id); - if (mit == mEventRegister.end()) - { - std::cerr << "RpcQueueService::deregisterForEvents() "; - std::cerr << "ERROR no EventId: " << event_id; - std::cerr << std::endl; - - return 0; - } - - bool removed = false; - std::list::iterator lit; - for(lit = mit->second.begin(); lit != mit->second.end();) - { - /* remove all matches */ - if ((lit->mReqId == req_id) && (lit->mChanId == chan_id)) - { - lit = mit->second.erase(lit); - if (removed == true) - { - std::cerr << "RpcQueueService::deregisterForEvents() "; - std::cerr << "WARNING REMOVING MULTIPLE MATCHES"; - std::cerr << std::endl; - } - removed = true; - } - else - { - lit++; - } - } - - if (mit->second.empty()) - { - std::cerr << "RpcQueueService::deregisterForEvents() "; - std::cerr << " Last Registrant for Event, removing List from Map"; - std::cerr << std::endl; - - mEventRegister.erase(mit); - } - - return 1; -} - - -int RpcQueueService::clearEventsForChannel(uint32_t chan_id) -{ - std::cerr << "RpcQueueService::clearEventsForChannel(" << chan_id; - std::cerr << ")"; - std::cerr << std::endl; - - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::list toMapRemove; - - std::map >::iterator mit; - for(mit = mEventRegister.begin(); mit != mEventRegister.end(); mit++) - { - std::list::iterator lit; - for(lit = mit->second.begin(); lit != mit->second.end();) - { - /* remove all matches */ - if (lit->mChanId == chan_id) - { - std::cerr << "RpcQueueService::clearEventsForChannel() "; - std::cerr << " Removing ReqId: " << lit->mReqId; - std::cerr << " for EventId: " << lit->mEventId; - std::cerr << std::endl; - - lit = mit->second.erase(lit); - } - else - { - lit++; - } - } - if (mit->second.empty()) - { - toMapRemove.push_back(mit->first); - } - } - - /* remove any empty lists now */ - std::list::iterator rit; - for(rit = toMapRemove.begin(); rit != toMapRemove.end(); rit++) - { - mit = mEventRegister.find(*rit); - mEventRegister.erase(mit); - } - - return 1; -} - - -int RpcQueueService::printEventRegister(std::ostream &out) -{ - out << "RpcQueueService::printEventRegister()"; - out << std::endl; - - RsStackMutex stack(mQueueMtx); /********** LOCKED MUTEX ***************/ - - std::list toMapRemove; - - std::map >::iterator mit; - for(mit = mEventRegister.begin(); mit != mEventRegister.end(); mit++) - { - out << "Event: " << mit->first; - out << std::endl; - - std::list::iterator lit; - for(lit = mit->second.begin(); lit != mit->second.end(); lit++) - { - out << "\tRegistrant: ReqId: " << lit->mReqId; - out << "\t ChanId: " << lit->mChanId; - out << std::endl; - } - } - return 1; -} - - - - -// Lower 8 bits. -uint8_t getRpcMsgIdSubMsg(uint32_t msg_id) -{ - return msg_id & 0xFF; -} - -// Middle 16 bits. -uint16_t getRpcMsgIdService(uint32_t msg_id) -{ - return (msg_id >> 8) & 0xFFFF; -} - -// Top 8 bits. -uint8_t getRpcMsgIdExtension(uint32_t msg_id) -{ - return (msg_id >> 24) & 0xFE; // Bottom Bit is for Request / Response -} - -bool isRpcMsgIdResponse(uint32_t msg_id) -{ - return (msg_id >> 24) & 0x01; -} - - -uint32_t constructMsgId(uint8_t ext, uint16_t service, uint8_t submsg, bool is_response) -{ - if (is_response) - ext |= 0x01; // Set Bottom Bit. - else - ext &= 0xFE; // Clear Bottom Bit. - - uint32_t msg_id = (ext << 24) + (service << 8) + (submsg); - return msg_id; -} - - - - - diff --git a/retroshare-nogui/src/rpc/rpcserver.h b/retroshare-nogui/src/rpc/rpcserver.h deleted file mode 100644 index 341086234..000000000 --- a/retroshare-nogui/src/rpc/rpcserver.h +++ /dev/null @@ -1,164 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_SERVER_H -#define RS_RPC_SERVER_H - -#include -#include -#include -#include - -#include "util/rsthreads.h" - -// Dividing up MsgIds into components: - -uint8_t getRpcMsgIdSubMsg(uint32_t msg_id); -uint16_t getRpcMsgIdService(uint32_t msg_id); // Middle 16 bits. -uint8_t getRpcMsgIdExtension(uint32_t msg_id); // Top 7 of 8 bits. Bottom Bit is for Request / Response -bool isRpcMsgIdResponse(uint32_t msg_id); - -uint32_t constructMsgId(uint8_t ext, uint16_t service, uint8_t submsg, bool is_response); - -/*** This can be overloaded for plugins - * Also allows a natural seperation of the full interface into sections. - */ - -// The Combination of ChanId & ReqId must be unique for each RPC call. -// This is used as an map index, so failure to make it unique, will lead to lost entries. -class RpcUniqueId -{ - public: - RpcUniqueId():mChanId(0), mReqId(0) {return;} - RpcUniqueId(uint32_t chan_id, uint32_t req_id):mChanId(chan_id), mReqId(req_id) {return;} - uint32_t mChanId; - uint32_t mReqId; -}; - -bool operator<(const RpcUniqueId &a, const RpcUniqueId &b); - -class RpcQueuedMsg -{ -public: - uint32_t mChanId; - uint32_t mMsgId; - uint32_t mReqId; - std::string mMsg; -}; - -class RpcServer; - -class RpcService -{ -public: - RpcService(uint32_t /* serviceId */ ):mRpcServer(NULL) { return; } - virtual void reset(uint32_t /* chan_id */) { return; } - virtual int msgsAccepted(std::list & /* msgIds */) { return 0; } /* not used at the moment */ - virtual int processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg) = 0; /* returns 0 - not handled, > 0, accepted */ - virtual int getResponse(uint32_t &chan_id, uint32_t &msgId, uint32_t &req_id, std::string &msg) = 0; /* 0 - not ready, > 0 heres the response */ - - virtual int getEvents(std::list & /* events */) { return 0; } /* 0 = none, optional feature */ - - void setRpcServer(RpcServer *server) {mRpcServer = server; } - RpcServer *getRpcServer() { return mRpcServer; } -private: - RpcServer *mRpcServer; -}; - - -class RpcEventRegister -{ - public: - uint32_t mChanId; - uint32_t mReqId; - uint32_t mEventId; // THIS IS A LOCAL PARAMETER, Service Specific -}; - -/* Implements a Queue for quick implementation of Instant Response Services */ -class RpcQueueService: public RpcService -{ -public: - RpcQueueService(uint32_t serviceId); -virtual void reset(uint32_t chan_id); -virtual int getResponse(uint32_t &chan_id, uint32_t &msg_id, uint32_t &req_id, std::string &msg); -virtual int getEvents(std::list &events); - -protected: - int queueResponse(uint32_t chan_id, uint32_t msgId, uint32_t req_id, const std::string &msg); - -virtual int locked_checkForEvents(uint32_t event, const std::list ®istered, std::list &events); // Overload for functionality. - - int registerForEvents(uint32_t chan_id, uint32_t req_id, uint32_t event_id); - int deregisterForEvents(uint32_t chan_id, uint32_t req_id, uint32_t event_id); - - int clearEventsForChannel(uint32_t chan_id); - int printEventRegister(std::ostream &out); - -private: - RsMutex mQueueMtx; - - std::map mResponses; - std::map > mEventRegister; -}; - - -/* For Tracking responses */ -class RpcQueuedObj -{ -public: - uint32_t mChanId; - uint32_t mReqId; - RpcService *mService; -}; - - -class RpcMediator; - -class RpcServer -{ - -public: - RpcServer(RpcMediator *med); - int addService(RpcService *service); - int processMsg(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, const std::string &msg); - bool checkPending(); - bool checkEvents(); - - void reset(uint32_t chan_id); - int error(uint32_t chan_id, std::string msg); // pass an error to mediator. - -private: - bool sendQueuedMsgs(std::list &msgs); - int queueRequest_locked(uint32_t chan_id, uint32_t msg_id, uint32_t req_id, RpcService *service); - - RpcMediator *mMediator; - - RsMutex mRpcMtx; - - std::list mRpcQueue; - std::list mAllServices; -}; - - -#endif /* RS_RPC_SERVER_H */ diff --git a/retroshare-nogui/src/rpc/rpcsetup.cc b/retroshare-nogui/src/rpc/rpcsetup.cc deleted file mode 100644 index 0db9a786d..000000000 --- a/retroshare-nogui/src/rpc/rpcsetup.cc +++ /dev/null @@ -1,68 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "rpc/rpcsetup.h" -#include "rpc/rpcserver.h" - -#include "rpc/proto/rpcprotopeers.h" -#include "rpc/proto/rpcprotosystem.h" -#include "rpc/proto/rpcprotochat.h" -#include "rpc/proto/rpcprotosearch.h" -#include "rpc/proto/rpcprotofiles.h" -#include "rpc/proto/rpcprotostream.h" - -#include "rpc/rpcecho.h" - -RpcMediator *CreateRpcSystem(RpcComms *comms, NotifyTxt *notify) -{ - RpcMediator *med = new RpcMediator(comms); - RpcServer *server = new RpcServer(med); - - /* add services */ - RpcProtoPeers *peers = new RpcProtoPeers(1); - server->addService(peers); - - RpcProtoSystem *system = new RpcProtoSystem(1); - server->addService(system); - - RpcProtoChat *chat = new RpcProtoChat(1); - server->addService(chat); - - RpcProtoSearch *search = new RpcProtoSearch(1, notify); - server->addService(search); - - RpcProtoFiles *files = new RpcProtoFiles(1); - server->addService(files); - - RpcProtoStream *streamer = new RpcProtoStream(1); - server->addService(streamer); - - /* Finally an Echo Service - which will echo back any unprocesses commands. */ - RpcEcho *echo = new RpcEcho(1); - server->addService(echo); - - med->setRpcServer(server); - - return med; -} - diff --git a/retroshare-nogui/src/rpc/rpcsetup.h b/retroshare-nogui/src/rpc/rpcsetup.h deleted file mode 100644 index b95193f87..000000000 --- a/retroshare-nogui/src/rpc/rpcsetup.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RPC_SETUP_H -#define RPC_SETUP_H - - -#include "rpc/rpc.h" - -class NotifyTxt; - -RpcMediator *CreateRpcSystem(RpcComms *comms, NotifyTxt *notify); - -#endif - diff --git a/retroshare-nogui/src/rpcsystem.h b/retroshare-nogui/src/rpcsystem.h deleted file mode 100644 index dd1ccfc43..000000000 --- a/retroshare-nogui/src/rpcsystem.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * RetroShare External Interface. - * - * Copyright 2012-2012 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - - -#ifndef RS_RPC_SYSTEM_H -#define RS_RPC_SYSTEM_H - -#include -#include -#include - -class RpcComms -{ -public: - virtual int isOkay() = 0; - virtual int error(uint32_t chan_id, std::string msg) = 0; - - virtual int active_channels(std::list &chan_ids) = 0; - - virtual int recv_ready(uint32_t chan_id) = 0; - virtual int recv(uint32_t chan_id, uint8_t *buffer, int bytes) = 0; - virtual int recv(uint32_t chan_id, std::string &buffer, int bytes) = 0; - - // these make it easier... - virtual int recv_blocking(uint32_t chan_id, uint8_t *buffer, int bytes) = 0; - virtual int recv_blocking(uint32_t chan_id, std::string &buffer, int bytes) = 0; - - virtual int send(uint32_t chan_id, uint8_t *buffer, int bytes) = 0; - virtual int send(uint32_t chan_id, const std::string &buffer) = 0; - - virtual int setSleepPeriods(float /* busy */, float /* idle */) { return 0; } -}; - - -class RpcSystem -{ -public: - /* this must be regularly ticked to update the display */ - virtual void reset(uint32_t chan_id) = 0; - virtual int tick() = 0; -}; - - -#endif // RS_RPC_SERVER_H diff --git a/retroshare-nogui/src/ssh/rssshd.cc b/retroshare-nogui/src/ssh/rssshd.cc deleted file mode 100644 index c4d8082aa..000000000 --- a/retroshare-nogui/src/ssh/rssshd.cc +++ /dev/null @@ -1,913 +0,0 @@ -/* This is a sample implementation of a libssh based SSH server */ -/* -Copyright 2003-2009 Aris Adamantiadis - -This file is part of the SSH Library - -You are free to copy this file, modify it in any way, consider it being public -domain. This does not apply to the rest of the library though, but it is -allowed to cut-and-paste working code from this file to any license of -program. -The goal is to show the API in action. It's not a reference on how terminal -clients must be made or how a client should react. -*/ - -/***** - * Heavily Modified by Robert Fernie 2012... for retroshare project! - * - */ - - -#include - -#include "ssh/rssshd.h" - -#include - -#define RSSSHD_STATE_NULL 0 -#define RSSSHD_STATE_INIT_OK 1 -#define RSSSHD_STATE_CONNECTED 2 -#define RSSSHD_STATE_ERROR 3 - -RsSshd *rsSshd = NULL; // External Reference Variable. - -// NB: This must be called EARLY before all the threads are launched. -RsSshd *RsSshd::InitRsSshd(const std::string &portStr, const std::string &rsakeyfile) -{ -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - ssh_threads_set_callbacks(ssh_threads_get_pthread()); -#endif - ssh_init(); - - rsSshd = new RsSshd(portStr); - if (rsSshd->init(rsakeyfile)) - { - return rsSshd; - } - - rsSshd = NULL; - return rsSshd; -} - - -RsSshd::RsSshd(std::string portStr) -:mSshMtx("sshMtx"), mPortStr(portStr), mChannel(0) -{ - - mState = RSSSHD_STATE_NULL; - mBindState = 0; - mRpcSystem = NULL; - - mSession = NULL; - - setSleepPeriods(0.01, 0.1); - return; -} - - - -int RsSshd::init(const std::string &pathrsakey) -{ - - mBind=ssh_bind_new(); - - //ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_DSAKEY, KEYS_FOLDER "ssh_host_dsa_key"); - //ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_RSAKEY, KEYS_FOLDER "ssh_host_rsa_key"); - - //ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_BINDPORT_STR, arg); - ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_BINDPORT_STR, mPortStr.c_str()); - //ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_DSAKEY, arg); - //ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_HOSTKEY, arg); - ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_RSAKEY, pathrsakey.c_str()); - //ssh_bind_options_set(mBind, SSH_BIND_OPTIONS_LOG_VERBOSITY_STR, "3"); - - mState = RSSSHD_STATE_INIT_OK; - mBindState = 0; - - return 1; -} - - -void RsSshd::run() -{ - /* main loop */ - - /* listen */ - bool sshOk = true; - while(sshOk) - { - std::cerr << "RsSshd::run() starting sshd cycle"; - std::cerr << std::endl; - - if (listenConnect()) - { - std::cerr << "RsSshd::run() success connect => setup mSession"; - std::cerr << std::endl; - - if (setupSession()) - { - std::cerr << "RsSshd::run() setup mSession => interactive"; - std::cerr << std::endl; - - mState = RSSSHD_STATE_CONNECTED; - interactive(); - } - else - { - std::cerr << "RsSshd::run() setup mSession failed"; - std::cerr << std::endl; - } - } - cleanupSession(); -#ifndef WINDOWS_SYS - sleep(5); // have a break; -#else - Sleep(5000); // have a break; -#endif - } -} - - -int RsSshd::listenConnect() -{ - std::cerr << "RsSshd::listenConnect()"; - std::cerr << std::endl; - - if (!mBindState) - { - if(ssh_bind_listen(mBind)<0) - { - printf("Error listening to socket: %s\n",ssh_get_error(mBind)); - return 0; - } - mBindState = 1; - } - else - { - std::cerr << "RsSshd::listenConnect() Already Bound..."; - std::cerr << std::endl; - } - - mSession=ssh_new(); - int r=ssh_bind_accept(mBind,mSession); - if(r==SSH_ERROR) - { - printf("error accepting a connection : %s\n",ssh_get_error(mBind)); - return 0; - } - -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - if (ssh_handle_key_exchange(mSession)) -#else - if (ssh_accept(mSession)) -#endif - { - printf("ssh_handle_key_exchange: %s\n", ssh_get_error(mSession)); - return 0; - } - return 1; -} - - -int RsSshd::setupSession() -{ - std::cerr << "RsSshd::listenConnect()"; - std::cerr << std::endl; - - if (authUser()) - { - std::cerr << "RsSshd::listenConnect() authUser SUCCESS"; - std::cerr << std::endl; - - if (setupChannel()) - { - std::cerr << "RsSshd::listenConnect() setupChannel SUCCESS"; - std::cerr << std::endl; - - if (setupShell()) - { - std::cerr << "RsSshd::listenConnect() setupShell SUCCESS"; - std::cerr << std::endl; - - return 1; - } - } - } - std::cerr << "RsSshd::listenConnect() Failed"; - std::cerr << std::endl; - - return 0; -} - - -int RsSshd::interactive() -{ - std::cerr << "RsSshd::interactive()"; - std::cerr << std::endl; - - doRpcSystem(); - //doEcho(); - return 1; -} - - - -int RsSshd::authUser() -{ - std::cerr << "RsSshd::authUser()"; - std::cerr << std::endl; - - - ssh_message message; - int auth = 0; - - do { - message=ssh_message_get(mSession); - if(!message) - break; - switch(ssh_message_type(message)){ - case SSH_REQUEST_AUTH: - switch(ssh_message_subtype(message)){ - case SSH_AUTH_METHOD_PASSWORD: - printf("User %s wants to auth with pass %s\n", - ssh_message_auth_user(message), - ssh_message_auth_password(message)); - if(auth_password(ssh_message_auth_user(message), - ssh_message_auth_password(message))){ - auth=1; - ssh_message_auth_reply_success(message,0); - break; - } - // not authenticated, send default message - case SSH_AUTH_METHOD_NONE: - default: - ssh_message_auth_set_methods(message,SSH_AUTH_METHOD_PASSWORD); - ssh_message_reply_default(message); - break; - } - break; - default: - ssh_message_reply_default(message); - } - ssh_message_free(message); - } while (!auth); - - if(!auth){ - printf("auth error: %s\n",ssh_get_error(mSession)); - ssh_disconnect(mSession); - return 0; - } - - std::cerr << "RsSshd::authuser() Success"; - std::cerr << std::endl; - - return 1; -} - - -int RsSshd::setupChannel() -{ - std::cerr << "RsSshd::setupChannel()"; - std::cerr << std::endl; - - ssh_message message; - - - do { - message=ssh_message_get(mSession); - if(message){ - switch(ssh_message_type(message)){ - case SSH_REQUEST_CHANNEL_OPEN: - if(ssh_message_subtype(message)==SSH_CHANNEL_SESSION){ - mChannel=ssh_message_channel_request_open_reply_accept(message); - break; - } - default: - ssh_message_reply_default(message); - } - ssh_message_free(message); - } - } while(message && !mChannel); - if(!mChannel){ - printf("error : %s\n",ssh_get_error(mSession)); - ssh_finalize(); - return 0; - } - - return 1; -} - - -int RsSshd::setupShell() -{ - std::cerr << "RsSshd::setupShell()"; - std::cerr << std::endl; - - int shell = 0; - ssh_message message; - - do { - message=ssh_message_get(mSession); - if(message && ssh_message_type(message)==SSH_REQUEST_CHANNEL && - ssh_message_subtype(message)==SSH_CHANNEL_REQUEST_SHELL){ - shell = 1; - ssh_message_channel_request_reply_success(message); - break; - } - if(!shell){ - ssh_message_reply_default(message); - } - ssh_message_free(message); - } while (message && !shell); - - if(!shell){ - printf("error : %s\n",ssh_get_error(mSession)); - return 0; - } - - return 1; -} - -// CLEANUP -int RsSshd::cleanupSession() -{ - std::cerr << "RsSshd::cleanupSession()"; - std::cerr << std::endl; - - ssh_disconnect(mSession); - ssh_free(mSession); - return 1; -} - - -int RsSshd::cleanupAll() -{ - std::cerr << "RsSshd::cleanupAll()"; - std::cerr << std::endl; - - cleanupSession(); - if (mBindState) - { - ssh_bind_free(mBind); - mBindState = 0; - } - ssh_finalize(); - return 1; -} - - - -// Various Operating Modes. -int RsSshd::doEcho() -{ - std::cerr << "RsSshd::doEcho()"; - std::cerr << std::endl; - int i = 0; - char buf[2048]; - - do{ -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - i=ssh_channel_read(mChannel,buf, 2048, 0); -#else - i=channel_read(mChannel,buf, 2048, 0); -#endif - if(i>0) { -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - ssh_channel_write(mChannel, buf, i); - ssh_channel_write(mChannel, buf, i); -#else - channel_write(mChannel, buf, i); - channel_write(mChannel, buf, i); -#endif - if (write(1,buf,i) < 0) { - printf("error writing to buffer\n"); - return 0; - } - } - } while (i>0); - - std::cerr << "RsSshd::doEcho() Finished"; - std::cerr << std::endl; - - return 1; -} - - -int RsSshd::setRpcSystem(RpcSystem *s) -{ - mRpcSystem = s; - return 1; -} - - -#if 0 - -int RsSshd::doTermServer() -{ - std::cerr << "RsSshd::doTermServer()"; - std::cerr << std::endl; - - if (!mTermServer) - { - std::cerr << "RsSshd::doTermServer() ERROR Not Set"; - std::cerr << std::endl; - return 0; - } - - mTermServer->reset(); // clear everything for new user. - - bool okay = true; - while(okay) - { - char buf; -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - int size = ssh_channel_read_nonblocking(mChannel, &buf, 1, 0); -#else - int size = channel_read_nonblocking(mChannel, &buf, 1, 0); -#endif - bool haveInput = (size > 0); - std::string output; - - int rt = mTermServer->tick(haveInput, buf, output); - - if (output.size() > 0) - { -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - ssh_channel_write(mChannel, output.c_str(), output.size()); -#else - channel_write(mChannel, output.c_str(), output.size()); -#endif - } - - if (!haveInput) - { - /* have a little sleep */ - sleep(1); //0000000); // 1/10th sec. - //usleep(10000000); // 1/10th sec. - } - else - { - usleep(10000); // 1/100th sec. - } - - if (rt < 0) - { - okay = false; // exit. - } - } - - std::cerr << "RsSshd::doTermServer() Finished"; - std::cerr << std::endl; - - return 1; -} - -#endif - - -int RsSshd::doRpcSystem() -{ - std::cerr << "RsSshd::doRpcSystem()"; - std::cerr << std::endl; - - if (!mRpcSystem) - { - std::cerr << "RsSshd::doRpcSystem() ERROR Not Set"; - std::cerr << std::endl; - return 0; - } - - uint32_t dummy_chan_id = 1; - mRpcSystem->reset(dummy_chan_id); // clear everything for new user. - - bool okay = true; - while(okay) - { - int rt = mRpcSystem->tick(); - if (rt) - { - // Working - so small sleep, - usleep(mBusyUSleep); - } - else - { - // No work cycle, longer break. - usleep(mIdleUSleep); - } - - if (rt < 0) - { - okay = false; // exit. - } - - if (!isOkay()) - { - okay = false; - } - } - - mRpcSystem->reset(dummy_chan_id); // cleanup old channel items. - - std::cerr << "RsSshd::doRpcSystem() Finished"; - std::cerr << std::endl; - - return 1; -} - -// RpcComms Interface.... -int RsSshd::isOkay() -{ - return (mState == RSSSHD_STATE_CONNECTED); -} - - std::list::iterator it; -int RsSshd::active_channels(std::list &chan_ids) -{ - if (isOkay()) - { - chan_ids.push_back(1); // dummy for now. - } - - return 1; -} - -int RsSshd::error(uint32_t chan_id, std::string msg) -{ - std::cerr << "RsSshd::error(" << msg << ")"; - std::cerr << std::endl; - - mState = RSSSHD_STATE_ERROR; - return 1; -} - - -int RsSshd::recv_ready(uint32_t chan_id) -{ - int bytes = ssh_channel_poll(mChannel, 0); - return bytes; -} - - -int RsSshd::recv(uint32_t chan_id, uint8_t *buffer, int bytes) -{ -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - int size = ssh_channel_read_nonblocking(mChannel, buffer, bytes, 0); -#else - int size = channel_read_nonblocking(mChannel, buffer, bytes, 0); -#endif - return size; -} - - -int RsSshd::recv(uint32_t chan_id, std::string &buffer, int bytes) -{ - char input[bytes]; -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - int size = ssh_channel_read_nonblocking(mChannel, input, bytes, 0); -#else - int size = channel_read_nonblocking(mChannel, input, bytes, 0); -#endif - for(int i = 0; i < size; i++) - { - buffer += input[i]; - } - return size; -} - - -int RsSshd::recv_blocking(uint32_t chan_id, uint8_t *buffer, int bytes) -{ -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - int size = ssh_channel_read(mChannel, buffer, bytes, 0); -#else - int size = channel_read(mChannel, buffer, bytes, 0); -#endif - return size; -} - - -int RsSshd::recv_blocking(uint32_t chan_id, std::string &buffer, int bytes) -{ - char input[bytes]; -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - int size = ssh_channel_read(mChannel, input, bytes, 0); -#else - int size = channel_read(mChannel, input, bytes, 0); -#endif - for(int i = 0; i < size; i++) - { - buffer += input[i]; - } - return size; -} - -int RsSshd::send(uint32_t chan_id, uint8_t *buffer, int bytes) -{ -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - ssh_channel_write(mChannel, buffer, bytes); -#else - channel_write(mChannel, buffer, bytes); -#endif - return 1; -} - -int RsSshd::send(uint32_t chan_id, const std::string &buffer) -{ -#if LIBSSH_VERSION_INT >= SSH_VERSION_INT(0,5,0) - ssh_channel_write(mChannel, buffer.c_str(), buffer.size()); -#else - channel_write(mChannel, buffer.c_str(), buffer.size()); -#endif - return 1; -} - -int RsSshd::setSleepPeriods(float busy, float idle) -{ - mBusyUSleep = busy * 1000000; - mIdleUSleep = idle * 1000000; - return 1; -} - - - -/***********************************************************************************/ - /* PASSWORDS */ -/***********************************************************************************/ - -int RsSshd::auth_password(const char *name, const char *pwd) -{ -#ifdef ALLOW_CLEARPWDS - if (auth_password_basic(name, pwd)) - return 1; -#endif // ALLOW_CLEARPWDS - - if (auth_password_hashed(name, pwd)) - return 1; - return 0; -} - -#define RSSSHD_MIN_PASSWORD 4 -#define RSSSHD_MIN_USERNAME 3 - -#ifdef ALLOW_CLEARPWDS -int RsSshd::adduser(std::string username, std::string password) -{ - if (password.length() < RSSSHD_MIN_PASSWORD) - { - std::cerr << "RsSshd::adduser() Password Too Short"; - return 0; - } - - if (username.length() < RSSSHD_MIN_USERNAME) - { - std::cerr << "RsSshd::adduser() Password Too Short"; - return 0; - } - - std::map::iterator it; - it = mPasswords.find(username); - if (it != mPasswords.end()) - { - std::cerr << "RsSshd::adduser() Warning username already exists"; - } - - mPasswords[username] = password; - - return 1; -} - - -int RsSshd::auth_password_basic(char *name, char *pwd) -{ - std::string username(name); - std::string password(pwd); - - std::map::iterator it; - it = mPasswords.find(username); - if (it == mPasswords.end()) - { - std::cerr << "RsSshd::auth_password() Unknown username"; - return 0; - } - - if (it->second == password) - { - std::cerr << "RsSshd::auth_password() logged in " << username; - return 1; - } - - std::cerr << "RsSshd::auth_password() Invalid pwd for " << username; - return 0; -} -#endif // ALLOW_CLEARPWDS - -//#define RSSSHD_HASH_PWD_LENGTH 40 - -int RsSshd::adduserpwdhash(std::string username, std::string hash) -{ -#if 0 - if (hash.length() != RSSSHD_HASH_PWD_LENGTH) - { - std::cerr << "RsSshd::adduserpwdhash() Hash Wrong Length"; - return 0; - } -#endif - - if (username.length() < RSSSHD_MIN_USERNAME) - { - std::cerr << "RsSshd::adduserpwdhash() Username Too Short"; - return 0; - } - - std::map::iterator it; - it = mPwdHashs.find(username); - if (it != mPwdHashs.end()) - { - std::cerr << "RsSshd::adduser() Warning username already exists"; - } - - mPwdHashs[username] = hash; - - return 1; -} - - -int RsSshd::auth_password_hashed(const char *name, const char *pwd) -{ - std::string username(name); - std::string password(pwd); - - std::map::iterator it; - it = mPwdHashs.find(username); - if (it == mPwdHashs.end()) - { - std::cerr << "RsSshd::auth_password_hashed() Unknown username"; - return 0; - } - - if (CheckPasswordHash(it->second, password)) - { - std::cerr << "RsSshd::auth_password_hashed() logged in " << username; - return 1; - } - - std::cerr << "RsSshd::auth_password_hashed() Invalid pwd for " << username; - return 0; -} - -#include "util/radix64.h" -#include "util/rsrandom.h" -#include - -#define RSSSHD_PWD_SALT_LEN 16 -#define RSSSHD_PWD_MIN_LEN 8 - - -#if 0 -int printHex(const char *data, int len) -{ - for(int i = 0; i < len; i++) - { - fprintf(stderr, "%02x", (uint8_t) data[i]); - } - return 1; -} -#endif - - - -int GenerateSalt(std::string &saltBin) -{ - /* get from random */ - for(int i = 0; i < RSSSHD_PWD_SALT_LEN / 4; i++) - { - uint32_t rnd = RSRandom::random_u32(); - saltBin += ((char *) &rnd)[0]; - saltBin += ((char *) &rnd)[1]; - saltBin += ((char *) &rnd)[2]; - saltBin += ((char *) &rnd)[3]; - } - -#if 0 - std::cerr << "HexSalt: "; - printHex(saltBin.c_str(), saltBin.size()); - std::cerr << std::endl; -#endif - - return 1; -} - -int GeneratePasswordHash(std::string saltBin, std::string password, std::string &pwdHashRadix64) -{ -#if 0 - std::cerr << "GeneratePasswordHash()"; - std::cerr << std::endl; - - std::cerr << "HexSalt: "; - printHex(saltBin.c_str(), saltBin.size()); - std::cerr << std::endl; -#endif - - if (saltBin.size() != RSSSHD_PWD_SALT_LEN) - { - return 0; - } - - if (password.size() < RSSSHD_PWD_MIN_LEN) - { - return 0; - } - - EVP_MD_CTX *ctx = EVP_MD_CTX_create(); - EVP_DigestInit(ctx, EVP_sha256()); - - EVP_DigestUpdate(ctx, saltBin.c_str(), saltBin.size()); - EVP_DigestUpdate(ctx, password.c_str(), password.size()); - - - unsigned char hash[1024]; - unsigned int s = 1024 - RSSSHD_PWD_SALT_LEN; - - for(int i = 0; i < RSSSHD_PWD_SALT_LEN; i++) - { - hash[i] = saltBin[i]; - } - - EVP_DigestFinal(ctx, &(hash[RSSSHD_PWD_SALT_LEN]), &s); - - Radix64::encode((char *)hash, s + RSSSHD_PWD_SALT_LEN, pwdHashRadix64); - -#if 0 - std::cerr << "Salt Length: " << RSSSHD_PWD_SALT_LEN; - std::cerr << std::endl; - std::cerr << "Hash Length: " << s; - std::cerr << std::endl; - std::cerr << "Total Length: " << s + RSSSHD_PWD_SALT_LEN; - std::cerr << std::endl; - - - std::cerr << "Encoded Length: " << pwdHashRadix64.size(); - std::cerr << std::endl; - - std::cerr << "GeneratePasswordHash() Output: " << pwdHashRadix64; - std::cerr << std::endl; -#endif - - return 1; -} - - -int CheckPasswordHash(std::string pwdHashRadix64, std::string password) -{ - char output[1024]; - std::vector buf = Radix64::decode(pwdHashRadix64); - size_t len = buf.size(); - for(unsigned int i = 0; (i < len) && (i < 1024); i++) - { - output[i] = buf[i]; - } - -#if 0 - std::cerr << "CheckPasswordHash() Input: " << pwdHashRadix64; - std::cerr << std::endl; - std::cerr << "Decode Length: " << len; - std::cerr << std::endl; - std::cerr << "HexDecoded: "; - printHex(output, len); - std::cerr << std::endl; -#endif - - /* first N bytes are SALT */ - EVP_MD_CTX *ctx = EVP_MD_CTX_create(); - EVP_DigestInit(ctx, EVP_sha256()); - - EVP_DigestUpdate(ctx, output, RSSSHD_PWD_SALT_LEN); - EVP_DigestUpdate(ctx, password.c_str(), password.size()); - -#if 0 - std::cerr << "HexSalt: "; - printHex(output, RSSSHD_PWD_SALT_LEN); - std::cerr << std::endl; -#endif - - unsigned char hash[128]; - unsigned int s = 128; - EVP_DigestFinal(ctx, hash, &s); - - /* Final Comparison */ - if (s != len - RSSSHD_PWD_SALT_LEN) - { - std::cerr << "Length Mismatch"; - return 0; - } - - if (0 == strncmp(&(output[RSSSHD_PWD_SALT_LEN]), (char *) hash, s)) - { - return 1; - } - - return 0; -} - - - diff --git a/retroshare-nogui/src/ssh/rssshd.h b/retroshare-nogui/src/ssh/rssshd.h deleted file mode 100644 index 0f28ddffd..000000000 --- a/retroshare-nogui/src/ssh/rssshd.h +++ /dev/null @@ -1,157 +0,0 @@ -/* This is a sample implementation of a libssh based SSH server */ -/* -Copyright 2003-2009 Aris Adamantiadis - -This file is part of the SSH Library - -You are free to copy this file, modify it in any way, consider it being public -domain. This does not apply to the rest of the library though, but it is -allowed to cut-and-paste working code from this file to any license of -program. -The goal is to show the API in action. It's not a reference on how terminal -clients must be made or how a client should react. -*/ - -/***** - * Heavily Modified by Robert Fernie 2012... for retroshare project! - * - */ - - -#ifndef RS_SSHD_INTERFACE_H -#define RS_SSHD_INTERFACE_H - -#include -#include - -#include -#include -#include - -// From inside libretroshare.a -#include "util/rsthreads.h" - -#include -#include - -#include "rpcsystem.h" - -#ifndef KEYS_FOLDER -#ifdef _WIN32 -#define KEYS_FOLDER -#else -#define KEYS_FOLDER "/etc/ssh/" -#endif -#endif - - - -/****** - * - * Minimal Options to start with - * - */ - - - -//#define ALLOW_CLEARPWDS 1 - -class RsSshd; -extern RsSshd *rsSshd; - - - // TODO: NB: THIS FN DOES NOT USE A "SLOW" HASH FUNCTION. - // THE FIRST HALF OF THE HASH STRING IS THE SALT -int CheckPasswordHash(std::string pwdHashRadix64, std::string password); -int GeneratePasswordHash(std::string saltBin, std::string password, std::string &pwdHashRadix64); -int GenerateSalt(std::string &saltBin); - -class RsSshd: public RsThread, public RpcComms -{ -public: - -// NB: This must be called EARLY before all the threads are launched. -static RsSshd *InitRsSshd(const std::string& portStr, const std::string &rsakeyfile); - - - // Interface. -int setRpcSystem(RpcSystem *s); -int adduserpwdhash(std::string username, std::string hash); - - // RsThreads Interface. - virtual void run(); /* called once the thread is started */ - - // RsComms Interface. - virtual int isOkay(); - virtual int error(uint32_t chan_id, std::string msg); - - virtual int active_channels(std::list &chan_ids); - virtual int recv_ready(uint32_t chan_id); - - virtual int recv(uint32_t chan_id, uint8_t *buffer, int bytes); - virtual int recv(uint32_t chan_id, std::string &buffer, int bytes); - virtual int recv_blocking(uint32_t chan_id, uint8_t *buffer, int bytes); - virtual int recv_blocking(uint32_t chan_id, std::string &buffer, int bytes); - - virtual int send(uint32_t chan_id, uint8_t *buffer, int bytes); - virtual int send(uint32_t chan_id, const std::string &buffer); - - virtual int setSleepPeriods(float busy, float idle); - -private: - RsSshd(std::string portStr); /* private constructor => so can only create with */ - -int init(const std::string &pathrsakey); - - // High level operations. -int listenConnect(); -int setupSession(); -int interactive(); - - // Lower Level Operations. -int authUser(); -int setupChannel(); -int setupShell(); -int doEcho(); - - // Terminal Handling! -//int doTermServer(); -int doRpcSystem(); - -int cleanupSession(); -int cleanupAll(); - - /* Password Checking */ -int auth_password(const char *name, const char *pwd); -int auth_password_hashed(const char *name, const char *pwd); -#ifdef ALLOW_CLEARPWDS -int auth_password_basic(char *name, char *pwd); -#endif // ALLOW_CLEARPWDS - - // DATA. - - RsMutex mSshMtx; - - uint32_t mBusyUSleep; - uint32_t mIdleUSleep; - - uint32_t mState; - uint32_t mBindState; - - std::string mPortStr; - ssh_session mSession; - ssh_bind mBind; - ssh_channel mChannel; - - RpcSystem *mRpcSystem; - -#ifdef ALLOW_CLEARPWDS - std::map mPasswords; -#endif // ALLOW_CLEARPWDS - std::map mPwdHashs; - -}; - - -#endif - diff --git a/rsctrl/src/NOTES.txt b/rsctrl/src/NOTES.txt deleted file mode 100644 index a06f35ce2..000000000 --- a/rsctrl/src/NOTES.txt +++ /dev/null @@ -1,60 +0,0 @@ - -This document proposes the finer details of how to communicate with RS, -using SSH and protobuf classes. -========================================================================== - -Message Format -==================== - -Protobuf serialisation does not identify Message Types or Message Size. -So we need to have an encapsulation header system - -These headers describe the actual message via a MsgId and MsgSize. -Care must be taken to ensure these IDS are correct, as this is NOT enforced -by protobuf, and will lead to incorrect deserialisation! - -In Each .proto there is a list of MsgIds as an ENUM. - -The protocol message format is as follows: - [HEADER: 16 bytes: 4 x Network Order uint32_t][ VARIABLE LENGTH BODY ] - - [ MAGIC_CODE ] [ MSG_ID ] [ REQ_ID ] [ BODY_SIZE ] [ ..... BODY ..... ] - MagicCode = 0x137f0001. will be incremented for new versions of the protocol. - MsgID = Corresponds to the format of the Body. - ReqID = Generated by Requester, Returned in Response, - make sure its unique. (undefined behaviour for duplicates) - BodySize = Byte Length of Body. - - The Body will consist of a protobuf encoded message. - - -Usage -================= - * Create SSH connection to retroshare-nogui. - * Create Request Message(s), and send over SSH channel - You can send as meny requests as you want. - * They will processed, and responses sent back (potentially in an arbitary order). - -MsgIDs -================= -In Each .proto there is a list of MsgIds as an ENUM. - -0x00 XX XX xx => Reserved for libretroshare Requests. (XX - area, xxxx specific msg) -0x01 XX XX xx => Reserved for libretroshare Responses - -eg. 0x00 (fa bc) [01] -> Request, 0x01 (fa bc) [01/02/03] -> responses - -0x10 YY YY yy => Extensions for project YY (16k), with sub commands yy (256) -0x11 YY YY yy - -... if we run out of YY's will use 0x20 etc. - - -Extensions -================= - -These are welcome and encouraged. -We just need to make sure that MsgIDs don't clash. - - - - diff --git a/rsctrl/src/definition/chat.proto b/rsctrl/src/definition/chat.proto deleted file mode 100644 index 0c0b8465b..000000000 --- a/rsctrl/src/definition/chat.proto +++ /dev/null @@ -1,246 +0,0 @@ -package rsctrl.chat; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// Private, Group and Chat Lobby RPC. -// -// OTHER COMMANDS TO ADD? -// Status Strings. -// Invite Friends to Private Lobbies. -// Chat History. -// ??? -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestChatLobbies = 1; - MsgId_RequestCreateLobby = 2; - MsgId_RequestJoinOrLeaveLobby = 3; - MsgId_RequestSetLobbyNickname = 4; - MsgId_RequestRegisterEvents = 5; - MsgId_RequestSendMessage = 6; - MsgId_RequestChatHistory = 7; -} - -enum ResponseMsgIds { - // STANDARD RESPONSES. - MsgId_ResponseChatLobbies = 1; - MsgId_ResponseSetLobbyNickname = 4; - MsgId_ResponseRegisterEvents = 5; - MsgId_ResponseSendMessage = 6; - MsgId_ResponseChatHistory = 7; - - // EVENTS - MsgId_EventLobbyInvite = 101; - MsgId_EventChatMessage = 102; -} - -/////////////////////////////////////////////////////////////// -// BUILDING BLOCKS. - -// This is a combination of ChatLobbyInfo & PublicChatLobbyRecord. -// Which seem very similar?? - -enum LobbyPrivacyLevel { - PRIVACY_PRIVATE = 1; - PRIVACY_PUBLIC = 2; -} - -enum ChatType { - TYPE_PRIVATE = 1; - TYPE_LOBBY = 2; - TYPE_GROUP = 3; -} - -message ChatLobbyInfo { - required string lobby_id = 1; - required string lobby_topic = 2; - required string lobby_name = 3; - - required string lobby_nickname = 4; // empty for none set. - - enum LobbyState { - LOBBYSTATE_JOINED = 1; - LOBBYSTATE_INVITED = 2; - LOBBYSTATE_VISIBLE = 3; - } - - required LobbyPrivacyLevel privacy_level = 5; - required LobbyState lobby_state = 6; - - required uint32 no_peers = 7; - required uint32 last_report_time = 8; - required uint32 last_activity = 9; - - repeated string participating_friends = 10; // SSL_IDS? - repeated string nicknames = 11; -} - - -message ChatId { - - required ChatType chat_type = 1; - required string chat_id = 2; - -} - -message ChatMessage { - - required ChatId id = 1; - required string msg = 2; - - optional string peer_nickname = 3; - optional uint32 chat_flags = 4; - - optional uint32 send_time = 5; - optional uint32 recv_time = 6; - -} - - -// RESPONSE: ResponseChatLobbies -// This is a generic Response - used often. -// lobbies, will contain a list of affected / requested Lobbies. -message ResponseChatLobbies { - - required rsctrl.core.Status status = 1; - repeated ChatLobbyInfo lobbies = 2; -} - - - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestChatLobbies -message RequestChatLobbies { - - enum LobbySet { - LOBBYSET_ALL = 1; - LOBBYSET_JOINED = 2; - LOBBYSET_INVITED = 3; - LOBBYSET_VISIBLE = 4; - } - - required LobbySet lobby_set = 1; - -} - -// RESPONSE: ResponseChatLobbies - -/////////////////////////////////////////////////////////////// - - -// REQUEST: RequestCreateLobby -message RequestCreateLobby { - - required string lobby_name = 1; - required string lobby_topic = 2; - - required LobbyPrivacyLevel privacy_level = 4; - - repeated string invited_friends = 3; // SSL_IDS -} - -// RESPONSE: ResponseChatLobbies - -/////////////////////////////////////////////////////////////// - -// Accept / Deny Invite, Join / Leave Lobby (these can be combined?) - -// REQUEST: RequestJoinOrLeaveLobby -message RequestJoinOrLeaveLobby { - - enum LobbyAction { - JOIN_OR_ACCEPT = 1; - LEAVE_OR_DENY = 2; - } - - required string lobby_id = 1; - required LobbyAction action = 2; -} - -// RESPONSE: ResponseChatLobbies - -/////////////////////////////////////////////////////////////// - -// Set Nickname. -// Get is done via requesting ChatLobby Info. -// Empty lobby_ids => default Nickname. - -// REQUEST: RequestSetLobbyNickname -message RequestSetLobbyNickname { - required string nickname = 1; - repeated string lobby_ids = 2; -} - -// RESPONSE: ResponseSetLobbyNickname -// Didnt think the whole Lobby status was necessary. -message ResponseSetLobbyNickname { - required rsctrl.core.Status status = 1; -} - - -/////////////////////////////////////////////////////////////// - -// Request Chat Events. -// This is done by registering for events. - -// REQUEST: RequestRegisterEvents -message RequestRegisterEvents { - - enum RegisterAction { - REGISTER = 1; - DEREGISTER = 2; - } - - required RegisterAction action = 1; -} - -// RESPONSE: ResponseRegisterEvents -message ResponseRegisterEvents { - required rsctrl.core.Status status = 1; -} - -// RESPONSE: EventLobbyInvite -message EventLobbyInvite { - required ChatLobbyInfo lobby = 1; -} - -// RESPONSE: EventChatMessage -message EventChatMessage { - required ChatMessage msg = 1; -} - -/////////////////////////////////////////////////////////////// -// Send Message. - -// REQUEST: RequestSendMessage -message RequestSendMessage { - required ChatMessage msg = 1; -} - -// RESPONSE: ResponseSendMessage -message ResponseSendMessage { - required rsctrl.core.Status status = 1; -} - -/////////////////////////////////////////////////////////////// -// Chat History. -// INITIALLY THIS WILL ONLY WORK WITH PRIVATE CHATS. -// NEED TO EXTEND INTERNALS TO HANDLE LOBBIES. - -// REQUEST: RequestChatHistory -message RequestChatHistory { - required ChatId id = 1; /* lobby or chat, group id */ -} - -// RESPONSE: ResponseChatHistory -message ResponseChatHistory { - required rsctrl.core.Status status = 1; - required ChatId id = 2; /* lobby or chat, group id */ - repeated ChatMessage msgs = 3; -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - diff --git a/rsctrl/src/definition/core.proto b/rsctrl/src/definition/core.proto deleted file mode 100644 index ffa051124..000000000 --- a/rsctrl/src/definition/core.proto +++ /dev/null @@ -1,161 +0,0 @@ -package rsctrl.core; - -/////////////////////////////////////////////////////////////// -// These are basic Messages, which are used as building blocks -// throughout the rest of the interface. -// They should not be sent RAW, but should be wrapped in another msg. -/////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////// -// Expected PackageIds. - -enum ExtensionId { CORE = 0; } - -enum PackageId { - PEERS = 1; - SYSTEM = 2; - CHAT = 3; - SEARCH = 4; - FILES = 5; - STREAM = 6; - - // BELOW HERE IS STILL BEING DESIGNED. - //MSGS = 7; - - // THEORETICAL ONES. - GXS = 1000; -} - - -/////////////////////////////////////////////////////////////// -// Basic Status Response, should be in all responses - -message Status { - enum StatusCode { - FAILED = 0; - NO_IMPL_YET = 1; - INVALID_QUERY = 2; - PARTIAL_SUCCESS = 3; - SUCCESS = 4; - READMSG = 5; - } - - required StatusCode code = 1; - optional string msg = 2; -} - -/////////////////////////////////////////////////////////////// - -message IpAddr { - required string addr = 1 [default = ""]; - required uint32 port = 2 [default = 0]; // must be 16 bit, 0 for unknown. -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -// Peer structures, mainly rsctrl.peers related. - -message Location { - - enum StateFlags { // ORd together... - ONLINE = 1; - CONNECTED = 2; - UNREACHABLE = 4; - } - - required string ssl_id = 1; - required string location = 2; - - required IpAddr localaddr = 3; - required IpAddr extaddr = 4; - - required uint32 state = 5; // Not an ENUM as ORd together. -} - -message Person { - - enum Relationship { - YOURSELF = 1; - FRIEND = 2; - FRIEND_OF_MANY_FRIENDS = 3; // 3+ at the moment. - FRIEND_OF_FRIENDS = 4; // 1 or 2. - UNKNOWN = 5; - } - - required string gpg_id = 1; - required string name = 2; - required Relationship relation = 3; - - repeated Location locations = 4; -} - -/////////////////////////////////////////////////////////////// -// File structures, mainly rsctrl.files related. - -message File { - required string name = 1; - required string hash = 2; - required uint64 size = 3; - - // THINK WE DONT WANT THESE HERE... - // BETTER TO KEEP File simple. - //optional string path = 4; - //optional string avail = 5; -} - -message Dir { - required string name = 1; - required string path = 2; - - repeated Dir subdirs = 3; - repeated File files = 4; -} - - -/////////////////////////////////////////////////////////////// - -message Timestamp { - required uint64 secs = 1; - required uint32 microsecs = 2; -} - - -/////////////////////////////////////////////////////////////// -// System Status - -message SystemStatus { - enum NetCode { - BAD_UNKNOWN = 0; - BAD_OFFLINE = 1; - BAD_NATSYM = 2; - BAD_NODHT_NAT = 3; - WARNING_RESTART = 4; - WARNING_NATTED = 5; - WARNING_NODHT = 6; - GOOD = 7; - ADV_FORWARD = 8; - } - - required NetCode net_status = 1; - optional string msg = 2; -} - - -/////////////////////////////////////////////////////////////// -// Bandwidth Measurements. - -message Bandwidth { - required float up = 1; // kB/s - required float down = 2; // kB/s - optional string name = 3; // e.g. DHT, UDP, TCP, Stun, or Total. -} - -message BandwidthSet { - repeated Bandwidth bandwidths = 1; -} - - - -/////////////////////////////////////////////////////////////// - - diff --git a/rsctrl/src/definition/files.proto b/rsctrl/src/definition/files.proto deleted file mode 100644 index 4a12be782..000000000 --- a/rsctrl/src/definition/files.proto +++ /dev/null @@ -1,143 +0,0 @@ -package rsctrl.files; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// List Transfers. -// Control Transfers. -// -// TODO: -// Share Directories. -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestTransferList = 1; - MsgId_RequestControlDownload = 2; - MsgId_RequestShareDirList = 3; -} - -enum ResponseMsgIds { - MsgId_ResponseTransferList = 1; - MsgId_ResponseControlDownload = 2; - MsgId_ResponseShareDirList = 3; -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -// Building Blocks - -enum Direction { - DIRECTION_UPLOAD = 1; - DIRECTION_DOWNLOAD = 2; -} - - -enum TransferState { - TRANSFER_FAILED = 1; - TRANSFER_OKAY = 2; - TRANSFER_PAUSED = 3; - TRANSFER_QUEUED = 4; - TRANSFER_WAITING = 5; - TRANSFER_DOWNLOADING = 6; - TRANSFER_CHECKING_HASH = 7; - TRANSFER_COMPLETE = 8; -} - - -message FileTransfer { - - required rsctrl.core.File file = 1; - required Direction direction = 2; - required float fraction = 3; - required float rate_kBs = 4; - required TransferState state = 5; -} - -/////////////////////////////////////////////////////////////// -// Transfer List. - -// REQUEST: RequestTransferList -message RequestTransferList { - required Direction direction = 1; -} - -// RESPONSE: ResponseTransferList -message ResponseTransferList { - - required rsctrl.core.Status status = 1; - repeated FileTransfer transfers = 2; - -} - -/////////////////////////////////////////////////////////////// -// Download. - -// REQUEST: RequestControlDownload -// START requires name, hash & size. -// other actions only require file hash. -message RequestControlDownload { - - enum Action { - ACTION_START = 1; // start download. - ACTION_CONTINUE = 2; // move to top of queue. - ACTION_WAIT = 3; // send to bottom of queue. - ACTION_PAUSE = 4; // hold indefinitely. - ACTION_RESTART = 5; // end pause, restart download. - ACTION_CHECK = 6; // force check. - ACTION_CANCEL = 7; // remove permenantly. - } - - required rsctrl.core.File file = 1; - required Action action = 2; - -} - -// RESPONSE: ResponseControlDownload -message ResponseControlDownload { - required rsctrl.core.Status status = 1; -} - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -// SHARED FILES - -// REQUEST: RequestShareDirList -message RequestShareDirList { - required string ssl_id = 1; - required string path = 2; -} - - -// RESPONSE: ResponseShareDirList -message ResponseShareDirList { - required rsctrl.core.Status status = 1; - - enum ListType { - DIRQUERY_ROOT = 1; // the query refers to root. - DIRQUERY_PERSON = 2; // the query refers to person - DIRQUERY_FILE = 3; // the query refers to a file. - DIRQUERY_DIR = 4; // move to top of queue. - } - - required string ssl_id = 2; - required string path = 3; - required ListType list_type = 4; - repeated rsctrl.core.File files = 5; -} - - -//// REQUEST: RequestChangeShares -// -//// REQUEST: RequestLiCloseSearch -//// REQUEST: RequestCloseSearch -// - - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - diff --git a/rsctrl/src/definition/gxs.proto b/rsctrl/src/definition/gxs.proto deleted file mode 100644 index 57bcd5a0f..000000000 --- a/rsctrl/src/definition/gxs.proto +++ /dev/null @@ -1,287 +0,0 @@ -package rsctrl.gxs; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// Base Messages for GXS Interface. -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestPeers = 1; - MsgId_RequestAddPeer = 2; - MsgId_RequestModifyPeer = 3; -} - -enum ResponseMsgIds { - MsgId_ResponsePeerList = 1; - MsgId_ResponseAddPeer = 2; - MsgId_ResponseModifyPeer = 3; -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -// BASE DATA TYPES - -message Service { - // - required uint32 service_id = 1; - optional string service_name = 2; -} - -message GroupMeta { - - // FLAGS as ENUMs. - enum GroupFlags { - GF_FLAG1 = 1; - GF_FLAG2 = 2; - GF_FLAG3 = 4; - GF_FLAG4 = 8; - } - - enum SignFlags { - SF_FLAG1 = 1; - SF_FLAG2 = 2; - } - - enum SubscribeFlags { - SUBF_FLAG1 = 1; - SUBF_FLAG2 = 2; - } - - enum StatusFlags { - STATF_FLAG1 = 1; - STATF_FLAG2 = 2; - } - - // THIS IS FIXED: From NETWORK. - required string group_id = 1; - required string group_name = 2; - required uint32 group_flags = 3; - required uint32 sign_flags = 4; - - required uint32 publish_ts = 5; - optional string author_id = 6; - - // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG. - - required uint32 subscribe_flags = 7; - required uint32 group_status = 8; - - optional string service_string = 9; - - optional uint32 pop = 10; // USED? - optional uint32 msg_count = 11; // ??? - optional int32 last_post = 12; // ??? -} - - -message GroupGenericData { - required GroupMeta meta = 1; - required string encoded_data = 2; -} - - -message MsgMeta { - - // FLAGS as ENUMs. - enum GroupFlags { - GF_FLAG1 = 1; - GF_FLAG2 = 2; - GF_FLAG3 = 4; - GF_FLAG4 = 8; - } - - enum SignFlags { - SF_FLAG1 = 1; - SF_FLAG2 = 2; - } - - enum SubscribeFlags { - SUBF_FLAG1 = 1; - SUBF_FLAG2 = 2; - } - - enum StatusFlags { - STATF_FLAG1 = 1; - STATF_FLAG2 = 2; - } - - // THIS IS FIXED: From NETWORK. - required string group_id = 1; - required string msg_id = 2; - - required string thread_id = 3; - required uint32 parent_id = 4; - required uint32 origmsg_id = 5; - - optional string author_id = 6; - - required uint32 publish_ts = 7; - required uint32 msg_name = 8; - - required uint32 msg_flags = 9; - - // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG. - - required uint32 msg_status = 10; - required uint32 child_ts = 11; - optional string service_string = 12; -} - -message MsgGenericData { - required MsgMeta meta = 1; - required string encoded_data = 2; -} - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -message Filter { - required uint32 options = 1; - - optional uint32 status_filter = 2; - optional uint32 status_mask = 3; - - optional uint32 req_type = 4; - - optional uint32 subscribe_filter = 5; - - optional int32 before = 6; - optional int32 after = 7; -} - -message MsgSet { - required string group_id = 1; - repeated string msg_id = 2; -} - -message GroupMsgSet { - repeated MsgSet groups = 1; -} - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestGroupInfo -message RequestGroupInfo { - - required Service service = 1; - - required uint32 ans_type = 2; - required Filter options = 3; - repeated string group_ids = 4; -} - - -// RESPONSE: ResponseGroupList -message ResponseGroupList { - - required rsctrl.core.Status status = 1; - - required Service service = 2; - repeated string group_ids = 3; -} - - -// RESPONSE: ResponseGroupMeta -message ResponseGroupMeta { - - required rsctrl.core.Status status = 1; - - required Service service = 2; - repeated GroupMeta meta = 3; -} - - -// RESPONSE: ResponseGroupData -message ResponseGroupData { - - required rsctrl.core.Status status = 1; - - required Service service = 2; - repeated GroupGenericData data = 3; -} - - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestMsgInfo -message RequestMsgInfo { - - required Service service = 1; - - required uint32 ans_type = 2; - required Filter options = 3; - repeated GroupMsgSet msgs = 4; -} - -// REQUEST: RequestMsgRelatedInfo -message RequestMsgRelatedInfo { - - required Service service = 1; - - required uint32 ans_type = 2; - required Filter options = 3; - repeated GroupMsgSet msgs = 4; -} - -// RESPONSE: ResponseMsgList -message ResponseMsgList { - - required rsctrl.core.Status status = 1; - - required Service service = 2; - repeated GroupMsgSet msgs = 3; -} - -// RESPONSE: ResponseMsgMeta -message ResponseMsgMeta { - - required rsctrl.core.Status status = 1; - - required Service service = 2; - repeated MsgMeta meta = 3; -} - -// RESPONSE: ResponseMsgData -message ResponseMsgData { - - required rsctrl.core.Status status = 1; - - required Service service = 2; - repeated MsgGenericData data = 3; -} - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - - -// REQUEST: RequestCreateGroup -message RequestCreateGroup { - - required Service service = 1; - repeated GroupGenericData data = 2; -} - - -// RESPONSE: ResponseMsgMeta -// As defined above. - - -// REQUEST: RequestCreateMsg -message RequestCreateMsg { - - required Service service = 1; - repeated MsgGenericData data = 2; -} - -// RESPONSE: ResponseMsgMeta -// As defined above. - - - - diff --git a/rsctrl/src/definition/msgs.proto b/rsctrl/src/definition/msgs.proto deleted file mode 100644 index e011dfb33..000000000 --- a/rsctrl/src/definition/msgs.proto +++ /dev/null @@ -1,108 +0,0 @@ -package rsctrl.msgs; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// Access, and Control your Friends / Peers and related Settings. -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestPeers = 1; - MsgId_RequestAddPeer = 2; - MsgId_RequestModifyPeer = 3; -} - -enum ResponseMsgIds { - MsgId_ResponsePeerList = 1; - MsgId_ResponseAddPeer = 2; - MsgId_ResponseModifyPeer = 3; -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestPeers -message RequestPeers { - - // About Who? - enum SetOption { - OWNID = 1; - LISTED = 2; - ONLINE = 3; - FRIENDS = 4; - VALID = 5; - SIGNED = 6; - ALL = 7; - } - - // What do you want? - enum InfoOption { - NAMEONLY = 1; - BASIC = 2; - LOCATION = 3; - ALLINFO = 4; - } - - required SetOption set = 1; - required InfoOption info = 2; - repeated string gpg_ids = 3; -} - - -// RESPONSE: ResponsePeerList -message ResponsePeerList { - required rsctrl.core.Status status = 1; - repeated rsctrl.core.Person peers = 2; -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestAddPeer -message RequestAddPeer { - - enum AddCmd { - NOOP = 0; // No op. - ADD = 1; // Add existing from gpg_id. - REMOVE = 2; // Remove existing from gpg_id. - IMPORT = 3; // Import from cert, with gpg_id. - EXAMINE = 4; // Examine cert, but no action. - } - - required string gpg_id = 1; - required AddCmd cmd = 2; - optional string cert = 3; -} - -// RESPONSE: ResponseAddPeer -message ResponseAddPeer { - required rsctrl.core.Status status = 1; - repeated rsctrl.core.Person peers = 2; -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestModifyPeer -message RequestModifyPeer { - - enum ModCmd { - NOOP = 0; - ADDRESS = 1; - DYNDNS = 2; - //SOMETHING_ELSE = 0x0000010; - //SOMETHING_ELSE = 0x0000020; - //SOMETHING_ELSE = 0x0000040; - //SOMETHING_ELSE = 0x0000080; - } - - required ModCmd cmd = 1; - //required int64 cmd = 1; // Could we OR the Cmds together? - repeated rsctrl.core.Person peers = 2; -} - -// RESPONSE: ResponseModifyPeer -message ResponseModifyPeer { - required rsctrl.core.Status status = 1; - repeated rsctrl.core.Person peers = 2; -} - -/////////////////////////////////////////////////////////////// - diff --git a/rsctrl/src/definition/peers.proto b/rsctrl/src/definition/peers.proto deleted file mode 100644 index 5cc38be27..000000000 --- a/rsctrl/src/definition/peers.proto +++ /dev/null @@ -1,111 +0,0 @@ -package rsctrl.peers; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// Access, and Control your Friends / Peers and related Settings. -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestPeers = 1; - MsgId_RequestAddPeer = 2; - MsgId_RequestExaminePeer = 3; - MsgId_RequestModifyPeer = 4; -} - -enum ResponseMsgIds { - MsgId_ResponsePeerList = 1; -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestPeers -message RequestPeers { - - // About Who? - enum SetOption { - OWNID = 1; - LISTED = 2; - CONNECTED = 3; - FRIENDS = 4; - VALID = 5; - SIGNED = 6; - ALL = 7; - } - - // What do you want? - enum InfoOption { - NAMEONLY = 1; - BASIC = 2; - LOCATION = 3; - ALLINFO = 4; - } - - required SetOption set = 1; - required InfoOption info = 2; - repeated string pgp_ids = 3; -} - - -// RESPONSE: ResponsePeerList -message ResponsePeerList { - required rsctrl.core.Status status = 1; - repeated rsctrl.core.Person peers = 2; -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestAddPeer -message RequestAddPeer { - - enum AddCmd { - ADD = 1; // Add existing from gpg_id. - REMOVE = 2; // Remove existing from gpg_id. - } - - required AddCmd cmd = 1; - required string pgp_id = 2; - optional string ssl_id = 3; - -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestExaminePeer -message RequestExaminePeer { - - enum ExamineCmd { - IMPORT = 3; // Import from cert, with gpg_id. - EXAMINE = 4; // Examine cert, but no action. - } - - // Must have GPG ID to import. Proves you've looked at it. - required string pgp_id = 1; - required ExamineCmd cmd = 2; - required string cert = 3; - -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestModifyPeer -// THIS IS INCOMPLETE... DON'T USE. -message RequestModifyPeer { - - enum ModCmd { - NOOP = 0; - ADDRESS = 1; - DYNDNS = 2; - //SOMETHING_ELSE = 0x0000010; - //SOMETHING_ELSE = 0x0000020; - //SOMETHING_ELSE = 0x0000040; - //SOMETHING_ELSE = 0x0000080; - } - - required ModCmd cmd = 1; - //required int64 cmd = 1; // Could we OR the Cmds together? - repeated rsctrl.core.Person peers = 2; -} - -/////////////////////////////////////////////////////////////// - diff --git a/rsctrl/src/definition/search.proto b/rsctrl/src/definition/search.proto deleted file mode 100644 index d058a6195..000000000 --- a/rsctrl/src/definition/search.proto +++ /dev/null @@ -1,120 +0,0 @@ -package rsctrl.search; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// Searches -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestBasicSearch = 1; - //MsgId_RequestAdvSearch = 2; // NOT IMPLEMENTED YET. - MsgId_RequestCloseSearch = 3; - MsgId_RequestListSearches = 4; - MsgId_RequestSearchResults = 5; -} - -enum ResponseMsgIds { - MsgId_ResponseSearchIds = 1; - MsgId_ResponseSearchResults = 5; -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -// Building Block -message SearchHit { - - enum LocFlag { - LOCAL = 1; // We Have it. - FRIEND = 2; // Browsable - NETWORK = 4; // Network. - } - - required rsctrl.core.File file = 1; - required uint32 loc = 2; // OR of LocFlag so uint field - required uint32 no_hits = 3; // NOT USED YET. - repeated string alt_names = 4; -} - -message SearchSet { - - required uint32 search_id = 1; - repeated SearchHit hits = 2; - -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -// SEARCH (start). - -// REQUEST: RequestBasicSearch -message RequestBasicSearch { - - repeated string terms = 1; -} - -// REQUEST: RequestAdvSearch -message RequestAdvSearch { - - repeated string terms = 1; -} - - -// RESPONSE: ResponseSearchIds -message ResponseSearchIds { - - required rsctrl.core.Status status = 1; - repeated uint32 search_id = 2; -} - - -/////////////////////////////////////////////////////////////// -// SEARCH (cancel) - -// REQUEST: RequestCloseSearch -message RequestCloseSearch { - - required uint32 search_id = 1; -} - - -// RESPONSE: ResponseSearchIds -// As before. - -/////////////////////////////////////////////////////////////// -// SEARCH (list) - -// REQUEST: RequestListSearches -message RequestListSearches { - // Nothing here. -} - -// RESPONSE: ResponseSearchIds -// As before. - -/////////////////////////////////////////////////////////////// -// SEARCH (list) - - -// REQUEST: RequestSearchResults -// Empty search_ids => all results. -message RequestSearchResults { - - optional uint32 result_limit = 1; - repeated uint32 search_ids = 2; -} - -// RESPONSE: ResponseSearchResults -message ResponseSearchResults { - - required rsctrl.core.Status status = 1; - repeated SearchSet searches = 2; -} - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - diff --git a/rsctrl/src/definition/stream.proto b/rsctrl/src/definition/stream.proto deleted file mode 100644 index 9fcbf8636..000000000 --- a/rsctrl/src/definition/stream.proto +++ /dev/null @@ -1,152 +0,0 @@ -package rsctrl.stream; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// This protocol defines how to stream data from retroshare. -// It can be used for VoIP or Files, or whatever. -// -// There are two parts. -// Control and actual streaming. -// -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestStartFileStream = 1; - MsgId_RequestControlStream = 2; - MsgId_RequestListStreams = 3; -} - -enum ResponseMsgIds { - MsgId_ResponseStreamDetail = 1; // RESPONSE to List and Control. - MsgId_ResponseStreamData = 101; -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -// Building Blocks - -enum StreamType { - STREAM_TYPE_ALL = 1; // all streams - STREAM_TYPE_FILES = 2; // files stream - STREAM_TYPE_VOIP = 3; // VoIP stream - STREAM_TYPE_OTHER = 4; // Who knows what else. -} - -enum StreamState { - STREAM_STATE_ERROR = 0; - STREAM_STATE_RUN = 1; - STREAM_STATE_PAUSED = 2; - STREAM_STATE_FINISHED = 3; -} - -message StreamFileDetail { - - required rsctrl.core.File file = 1; - required uint64 offset = 5; - -} - - -message StreamVoipDetail { - - // THIS NEEDS MORE DEFINITION. - required string peer_id = 1; - required uint64 duration = 2; - required uint64 offset = 3; - -} - - -message StreamDesc { - - required uint32 stream_id = 1; - required StreamType stream_type = 2; - required StreamState stream_state = 3; - required float rate_kbs = 4; - optional StreamFileDetail file = 5; - optional StreamVoipDetail voip = 6; - -} - - -message StreamData { - - required uint32 stream_id = 1; - required StreamState stream_state = 2; - required rsctrl.core.Timestamp send_time = 3; - required uint64 offset = 4; - required uint32 size = 5; - required bytes stream_data = 6; - -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - - -// REQUEST: RequestStartFileStream -message RequestStartFileStream { - - required rsctrl.core.File file = 1; - required float rate_kbs = 2; - - // byte range. allows to restart transfer! - optional uint64 start_byte = 3; - optional uint64 end_byte = 4; - -} - -// RESPONSE: ResponseStreamDetail -message ResponseStreamDetail { - - required rsctrl.core.Status status = 1; - repeated StreamDesc streams = 2; - -} - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestControlStream -message RequestControlStream { - - enum StreamAction { - STREAM_START = 1; // start stream - STREAM_STOP = 2; // stop stream - STREAM_PAUSE = 3; // pause stream - STREAM_CHANGE_RATE = 4; // rate of the stream - STREAM_SEEK = 5; // move streaming position. - } - - required uint32 stream_id = 1; - required StreamAction action = 2; - optional float rate_kbs = 3; - optional uint64 seek_byte = 4; -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestListStreams -message RequestListStreams { - - required StreamType request_type = 1; - -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - -// RESPONSE: ResponseStreamData -message ResponseStreamData { - required rsctrl.core.Status status = 1; - required StreamData data = 2; -} - - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - diff --git a/rsctrl/src/definition/system.proto b/rsctrl/src/definition/system.proto deleted file mode 100644 index d979b2696..000000000 --- a/rsctrl/src/definition/system.proto +++ /dev/null @@ -1,121 +0,0 @@ -package rsctrl.system; - -import "core.proto"; - -/////////////////////////////////////////////////////////////// -// Configuration and Stats. -/////////////////////////////////////////////////////////////// - -enum RequestMsgIds { - MsgId_RequestSystemStatus = 1; - MsgId_RequestSystemQuit = 2; - MsgId_RequestSystemExternalAccess = 3; - MsgId_RequestSystemAccount = 4; -} - -enum ResponseMsgIds { - MsgId_ResponseSystemStatus = 1; - MsgId_ResponseSystemQuit = 2; - MsgId_ResponseSystemExternalAccess = 3; - MsgId_ResponseSystemAccount = 4; -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestSystemStatus -message RequestSystemStatus { - // Nothing here? -} - -// RESPONSE: ResponseSystemStatus -message ResponseSystemStatus { - - enum NetCode { - BAD_UNKNOWN = 0; - BAD_OFFLINE = 1; - BAD_NATSYM = 2; - BAD_NODHT_NAT = 3; - WARNING_RESTART = 4; - WARNING_NATTED = 5; - WARNING_NODHT = 6; - GOOD = 7; - ADV_FORWARD = 8; - } - - // Status of response. - required rsctrl.core.Status status = 1; - - // Peers. - required uint32 no_peers = 2; - required uint32 no_connected = 3; - - // Basic Network Information. - required NetCode net_status = 4; - - required rsctrl.core.Bandwidth bw_total = 5; -} - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestSystemQuit -message RequestSystemQuit { - - enum QuitCode { - CLOSE_CHANNEL = 1; - SHUTDOWN_RS = 2; // NOT RECOMMENDED (but some people might like it) - } - - required QuitCode quit_code = 1; -} - -// RESPONSE: ResponseSystemQuit -// Effect potentially immediate (with loss of connection) - only expect a response error. -// Shutdown takes longer - so you should get a response. -message ResponseSystemQuit { - - // Status of response. - required rsctrl.core.Status status = 1; -} - - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestSystemExternalAccess -message RequestSystemExternalAccess { - // Nothing here? -} - -// RESPONSE: ResponseSystemExternalAccess -message ResponseSystemExternalAccess { - - // Status of response. - required rsctrl.core.Status status = 1; - - required uint32 ext_port = 2; - required string dht_key = 3; -} - - -/////////////////////////////////////////////////////////////// - -// REQUEST: RequestSystemAccount -message RequestSystemAccount { - // Nothing here? -} - -// RESPONSE: ResponseSystemAccount -message ResponseSystemAccount { - - // Status of response. - required rsctrl.core.Status status = 1; - - required string pgp_name = 2; - required string location = 3; - required string pgp_id = 4; - required string ssl_id = 5; -} - -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// -/////////////////////////////////////////////////////////////// - From 3dbf9163325a3094cd73a2a189382e16887e0032 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 8 Oct 2016 14:54:06 +0200 Subject: [PATCH 154/174] updated ubuntu changelog --- build_scripts/Debian+Ubuntu/changelog | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/build_scripts/Debian+Ubuntu/changelog b/build_scripts/Debian+Ubuntu/changelog index fa6a37703..2f476393e 100644 --- a/build_scripts/Debian+Ubuntu/changelog +++ b/build_scripts/Debian+Ubuntu/changelog @@ -1,5 +1,33 @@ retroshare06 (0.6.1-1.XXXXXX~YYYYYY) YYYYYY; urgency=low + 9ed759d csoler Sat, 8 Oct 2016 13:38:18 +0200 Merge pull request #527 from PhenomRetroShare/Fix_DesktopFileName + 4b0f82a Phenom Sat, 8 Oct 2016 12:12:26 +0200 Fix Desktop Filename for option page message. + 7cd31aa csoler Wed, 5 Oct 2016 14:15:12 +0200 fixed bug preventign update of subdirs after last commit + 854eebb csoler Wed, 5 Oct 2016 10:59:51 +0200 only update subfiles and subdirs list during dir watching sweep when the TS of the dir has changed. Greatly improve cost of regular directory watching process + d020d8d thunder2 Wed, 5 Oct 2016 06:40:02 +0200 Windows build environment: - Added automatic mode for creating git-log - Fixed filename of installer + d367491 thunder2 Tue, 4 Oct 2016 21:45:59 +0200 Fixed env.bat for Windows build + 20cd123 csoler Tue, 4 Oct 2016 21:41:38 +0200 removed annoying ERROR output that is more a debug info than a real error, in p3filelists + e42dae8 csoler Tue, 4 Oct 2016 21:34:23 +0200 continue sending packet slicing probes in case the peer has restarted + b08a62a csoler Tue, 4 Oct 2016 21:25:48 +0200 restored delay between location cleaning to 5 mins + 133676f csoler Tue, 4 Oct 2016 21:24:53 +0200 simplified the code to remove old locations and fixed bug that in some situation would let a PGP key with no locations + a35985e thunder2 Tue, 6 Sep 2016 21:31:41 +0200 Windows build environment: - Added build script - Added build-installer script - Added pack script - Added gitlog script - Use shadow build + ebfc82c csoler Mon, 3 Oct 2016 21:44:34 +0200 save/restore selection in shared dir list tree view + ed8d786 csoler Mon, 3 Oct 2016 20:49:49 +0200 added missing mChanged=false that caused remote directories to save far too often + c968490 csoler Sat, 1 Oct 2016 22:14:50 +0200 switched to floating point time for bandwidth estimation in pqistreamer. Helps a lot RTTs since rounding to int prveviously caused packets to be delayed up to 1 sec (improvement by Jollavilette) + 20397c3 csoler Sat, 1 Oct 2016 18:09:39 +0200 Merge pull request #523 from bryongloden/patch-1 + 9ae0511 Bryon Gloden Sat, 1 Oct 2016 12:01:09 -0400 uninitialized variable: size + a751155 csoler Sat, 1 Oct 2016 15:46:32 +0200 do not send un-hashed files to friends, and allow to re-send the directory content when all files are hashed + 84341f2 csoler Thu, 29 Sep 2016 22:30:26 +0200 changed _outgoing_file_hashes into a new map that connects services to tunnel ID instead of file hash, hence avoiding to remove a hash on server side when a tunnel is closed (bug found by Jolavillette) + b62b66a csoler Thu, 29 Sep 2016 20:55:16 +0200 replace std::vector for std::set to represent _hashes_to_remove, so as to avoid duplicates (suggested by Jo) + 1fd624c csoler Thu, 29 Sep 2016 12:47:14 +0200 Merge pull request #519 from PhenomRetroShare/Fix_WinCompil + 0ea909d Phenom Thu, 29 Sep 2016 12:45:01 +0200 Fix Windows Compilation after 6e4ba76b + e185bcd csoler Wed, 28 Sep 2016 23:02:24 +0200 fixed removal of directory when removing a friend + c276165 csoler Wed, 28 Sep 2016 22:28:43 +0200 updated clean.sh script to 0.6.1 + + -- Cyril Soler Sat, 8 Oct 2016 15:00:00 +0100 + +retroshare06 (0.6.1-1.20160928.e185bcda~trusty) trusty; urgency=low + Warning: new file list sharing system. * Old list of shares is not preserved and should be re-defined. * The hash cache however is imported from the old format, meaning that Retroshare should not re-hash new files that were already shared before. From 381384534373e13dd73a437988ef3a73871a0c46 Mon Sep 17 00:00:00 2001 From: cave beat Date: Sat, 8 Oct 2016 16:06:42 +0200 Subject: [PATCH 155/174] patch from sss rus translation --- retroshare-gui/src/lang/retroshare_ru.qm | Bin 543541 -> 558394 bytes retroshare-gui/src/lang/retroshare_ru.ts | 481 ++++++++++++----------- 2 files changed, 242 insertions(+), 239 deletions(-) diff --git a/retroshare-gui/src/lang/retroshare_ru.qm b/retroshare-gui/src/lang/retroshare_ru.qm index cba101be1f2c3541a498f766c298f8e647b69a7d..9d00e79d8709eb1c50f00c5369d8cccd6827b822 100644 GIT binary patch delta 45381 zcmX7w1ymJH6vyw(PV7Dl#SRR>#02aX14P8Yz(NJ=#*U|A0AhoREsBW=b|VIue1wew zcDJJ9cbWAZ&u{nf-oD+PnH&Fm_Z?nU@kCf9Ytv)?C6hm_kGx*7>{-7prysTk7(WyM zYJl)OkE{bUnJ999J7g=M<@4AY$eC7(^2u;yd(dt6AUlCxxF)hY@+Wcx=taT+Xlu|5 zaRAK|<009M?PzwQDsJA7fXqUePO!hQKG7wHY;hc9v^FxSt>DB%7r73IH& z$U}G{S53ZtsVJW;L#6?E9W!~!O_2nxRit4>6txN!z?%0)?M(o_@EFL|Kt(DWBR}JT zznXkwt0;&0!e*Lb!*^7~#0cNySl-c-wd=xPK zw-q<`1F=g|WL^84TzU?P|GU{MiiUj@<$pLl`~&bIZxnUu2p~0}`yL?&0o{KeiN|}3 zci!O;-ttPw&q(~7V-b*J@lGN2LW7WVk=D5YbqfMFUYm^Qswh113tZ{|xZ|yAdmo2j z7C^hE08equ+HV6gHb;?h&19h~CQCb*>``4&l*v|<HaE zE_LysIF?A#<%J@XqZQ@%&B!F2L#F|{?f}uD7cPQ%Al@}WUIu!7jmd{NM{u79;9To| z*$RBc9rQLiz#C%u?hdo+-*( zIQs`&00_qSBjvM;07Kn??yU^8mnV*OG|<5JK$1!UjlT@!-AttQGO(I6Kx^3&Ps9ea zteLouX9HWf4xsQ{MXkvvV4-=pPy@C&6r}e7V5@MP_cc&tE;WFy--WL~0=97;z;Zk$ zlEMx}mR!qZE*=A47wcLA+ddUo;TOQHJJNtPs0wUn4A4DCOg^+V`3JukUuV-J6?p`H zD7H6Hau3)(j$=Cq*paqC&P_yS;_c5+6bX}}ue|Ceik#yTppCVfrz&r}rs+MBc%SCq%Cn}NNp1FRgL08(Z*0`>ui z;>r?}*KjW6?Z0hf@;#1a-u}NWimcp7laWmoIp7_@zw<$OJfw(>GPz+puy1)wc!0@H zV}N~k1?UXxjsmaK5qQ_xz+0{a^5>u;_0Iy{!4<&UFgc@y z$xA7UWa1f@;hY$-)#f=kD@Eqnn3>Ksi3tf1qEYg0B>Lm1#fQzzROw$3f1`q z`-t>!>jK+037g&d2%pR-lYL~^Z+X3KC?cn4OLwZfvB?| z`5E~bs=5{dQMWKu^%)E#AsMRYIg9Ggp*q@f@~({{t6>k-6J0?#O;MC3-Jv>~ZdPNr z$-zaSMh)~WH5)*UMQFxrt%VwStEAK^leLB@TG`-PP$O?y)Gh%v67h|-yF!hdc%`)$ zf)j>H(AG;)4!H_WZqLxRN105!q9|q&Md{iQYC0tXpVu7flAj%oPd&q!;wm&u^+O|YaRM}%ToGu)GSDQr7HBK)K#LDJ;d}~1>-1EBhDV_F z-3TBpE1JBp3)+lB(^Pp<$^8?r3^hQU!2##buP#6+EhL1#&R~JZfM} zuq9hjOf7Em@dEIu9|^EON0E$Qqo_6Ar^xD0QRMdTz++^03}hOaocs+utdm=V95x(0 zF7E_+zb|;)9fn6{?P~`jWf#=CMplOT2^ICV1E~#Mkyp7AoW)^rposCQG5O`+ggGZ?xw=nHcA73gw3?*a}%*RTYTqeG!vuPBhtzR;~-F36Tuq1%xqK;rX5w=3x& z`b{x8bQ^RJ4g*51XU&a?A56yWQDnzoC~`4KJ)%7LtZoL-v<~=eLB};>81x8R1LS8k^azi{6Rl;^dVUM^IEA+T za$QA|aRhpN-w)h!2YL>h2I9j-=s7$Nw5koD=Q4Ccb(tdb>tZtSfXR6W75Qj?=(!3F zTE-jbl@JQ#&J0E2wjFw1iUBw#6`98h=ykaf#{Yb#+F%_MO%4q(J{e-s0>p(Ch0kAn7Zhx7|U2*K?qEyM4ej=>6ao z(7COk_e3=UG%ZqGOQuOamHo`!$F z1ATiZ0{ig-`sSPk^6Rmp2x$WSEZLyFI|coUe*y767Y10Z1AtVi3jX$(RQ6p7{$J3L zoZkllP64134FXr8X&#sjLpufmAAA^whG1ZE;}#6JTmbU@5)98XJajmWC|wU={W%y> zVIGj%i($kLOfW;*!^n7h5OXtO+{D&E*M~z8PF|Av6DGz)fYe8Wb>h8TV0-t$l--zQ zglsdp>LE;f(jWc!K$t#!2FM5dVTSJ?^xF?%#(E4!PacC=UC`OwSqZadPe6;yVD|0z zAZ9dGlrO&S+OI&w=fM!JvKH0o#A90NUNUA9hUH2&BR?*!i{? zp1>2>wQVu5?)70;>PF|g|qTEbUu5L^BNXqE0lY&Q=OV+z6EXv}=^-@eubK^!>% z`%{xatjU1`S$H8OZ58>FL^$Y>3gp!vI8-AB}0*IKYwWSwOsA zz>$VqfiBB}W45zF*iC|C)iDWKn;(wVz)AOI7@RC@0lB^toVIrX$l9(bJW}9{%Qs9Y zN5PrCxEcnXS7fD1Daxn$;8KA$z+NqZOAXS1&PahI8V2;gNJtu22=l*I6Ct_Vdytm~ z!?lY}AZiD}jlyWzrhbK+kL*BuWQDY(_W;r3;NI;B&?fsqAWHK{$c2){pw7}(QP2Ctq3jq2*lq8gx5+1h)*ZFU4PID_aON$ zSt|iO_mULAAX5B!MQq#+G&EVn_C^5E0^TNzGE($MDKScU=G z%k7GcuO`L&;&L5eAtjtpRjcevN^W!jI=eh6{c1GOYL`jbA$UT)?vQfZ-UDrUg4kK- zqk3gCjMyFj0<=&tDc{K-$C{D~{fmN3{6H$ULLtGw9I4n5=g6t`q~dii5NlmX#qXFQ zclITfiarDFQ$J!K`W~~L5v0~EG%zmhNbQEXK$hN8)EXBhbt7;{!!k)dcQjD;sl;Ut zX4V%M601v+Es&44q<*3Sba_qE;MPhI?M{$JQ#+xz>P8xU#V`19f;4uE1G0Q9X?(LJ zK$95Kq@)9|lA}qJTNqHRsZW|_;)PtdAuU?OVG*y)kfWE0=bqufH*_YRFYPeM3@4px;jJAqh;$nH4#@f>(m4P{O7E$}yBNj|UH_4; zbusf<`i^w%RTk*2y`)>^7&JU(Nskpo_s|7yW@%OeLw=fq5&FHgA6HvC)6XB412X31*jA< zGWrn6l0V6)<~Y_Ldy;Vn4+BifPo|Gf2krI|MP|*vkW8P9rtn$-nT}t?-n}4J`->po z#u00`B>;Wxi1iT$p*O0MU{}=q^qwSmwlm1zm&yEFm}pF~k_A5dfW0V87T&)L8XHd* zKK~7*j7CB$Z^PVCkkB9QAb*S}VF$f|CC(+`1#SY{U6d@&7l8Wjx}Rk6@4G-N6(dXQ z-or7xrpNTU$=*U};U-@od#7Pabz=(IJG%hBUY_i4z65w( z8*;Q(3~2ubk>hP~-}5jz!E-?bO(&-sqTvbdPfktIKr4BhoVpN+fyi@m#=biUcW-ho z*jg4?zvATbprUAtHx-39l=~Z!>oH!K#HNs2%Sr%s=uA?aa8lQ~LsFMu5L#;iNpoQUkz+{O%usZ4 z9wg10xAEZ{$$%t)iSCYrAieZ+_$&>BhKz6%Lp4a^a%q5+?aIyn3 z%a&MQ+u~&U)t!hK?dXxNq8;222W%75fGmxCmiZcByg$gSHa!MiN z@PfZv)rM$;Chu=SfO8iqRe#t?*)dG3-wdN-b19JQ5Y_i}23F=XwFyDX=$t?c4;%}; zXD}`N%K_CAL2bX6!b$py7T@fKQtdQa!lggJpBc3DH}wBA=P1hW4z%3BaUi}{qZLrh zgdf%kw91?q9HXYRY7vxOI|i5>*o0Q=+6c5(Q)%_un4GdyT4OkV@s`cB=DBSE8H1?v zx@-)m8_))u?}6|?LmM{tM_K(JZM?V^Dq&@4<5aYSs}9g+uIn+f>O-4(M*8`>pw6ljHxP#@GA ztkAs`^_gX0K3N+`4w;t*yj?ds^wUOwtN+pA(^7$+K1N3rnuv3y2_2Dx z!RpcebY#KNsQ>J0LPu_P2kmBx$tUS_Y#R(3b7B-_dUHB%Qx4wpopgLyYhaa2D9X^l5Qf~)5&=U`nriuevjez*B5k3fdC9L%hM@& zck0C$I?eVm(A#|!H4#p&6DfMPIBH#uN$AnTbl#w6K*NX7kmi1%8?9(q3}(eWb-Kt( z3j?|Hn=U$E4(OY{baC%YAladGNixpPl0)gTxHBMoJ*Sba+JOAClSZz|#!^fOjp~kG zahM%lH*z$vtv%`bqFF#Ub*Afo1mS9!L^n(=23oa6bi<8qD94_pn|c(*&$-e~RtEgo zSh_i+JHW~f)EdpwF;?@ZTc6(pvd5lon>7!h=yOH+{Vv^&2?igvj_%za46M;fy3YZ} zbir790Ml~jG@BlL?TQtX%k)q|7huoM(0C_2!9xe>kuZ#QYrdyPwyXphU5y@nj}rZg z9D3|8I=Nam>2U+!f3ugB9=|*qtD&3e=`#0FMlDCrSn!I94O0~DyP52L+~m}=it=DT zdS)@ka5b{%*&bi<3&+rN9kOtaJXa*?&*-`SXtF0t8l+}4S;NRCY!6LtjI}#&7n*!@0%)Zw z(rbgtgQ##{QI>B`uj3t}q7l8pssmrJh~BK+8^G%ky#@aPDKvSz%Ib7}es4AXN) z(c4u}9k({hrME{_1K~KtG4N-Op`u5c}(AuQX zPp)l&+D9l_`Jxi^TXIhn7MjuD-k6!hh0}lTb^!N0H8v?5$fGEYhlYWc;cfE!eNDeL z4;cTX6&^AHV8u(VNLw6=KVLLk)CbA07Fw~}Sd?)1p%u5s0OfVOR-$A-fXxTA68leJ z$>zIO#{LCxJxD8?Py+q`;Rwy{I0}QKPH5#D?gQE{N2{o#p=ohYa|m?>S-iJaeFl0v z;i=V(%*Cj+mC4MDTFq;1(CK{C>Q)~NMtG}`e==nNo4eMf}@%gLP@LNG(KQC#GR-(5`=&Uu~ofrEJ z)LJ^@iA=hywcPU&Gajk6D(`|}do`_9SSki69kkYg&4Ki9q_rOH3hZFG)+R6V*>X^G z8?zEKrXO1SpS?j=ucvi*cLPX~{aObrZqVQ}ntS?kAW^L~kLCS=t)8!SJl_WBOn=R* zX9PNxO9~8s>%1e6g8-4veY$`wWpbM zd8=9T4y4upp ziE+c$W17DchTXj%YXKLwfINOf8wX})f--B2_QkxNo%cPB$HoN3PAm839%1f2C*>})BTUWDH0kjZv;#FT z7Od1)JJ1Y$h_!r@+F(D1Y6qUS0KiFea4N=XEh4o;&1VCBKS(>Y`aSSAQCeIY-hoBq zwZn`2fwtqC=K;Jgt z;N)2?c{R?Vh=qz)n!Z47uwJ7~_8DsO)>K959HL$Sp@F<}QoGsc8~XJf+RZ+(=-(u}uB^Qvj3Ty9N=YgirQDk9K`?$Ou zK#{qMy!$zT0m;hJer98`y0EI2bFUCKqNHlSL-Edh zJ+A#3!GWa@)^e?`8jv<9famejeeGXZP2lb&8FdK;d3!O_`k}>Y)}Dz9weSNcnOK5j z=zW)om`R|0@57{f9CkKbWO}h6fX}0tZvP%wv$D(>5eB537t43P2uiUYtYr9X(B4{D z$;Bw!r)VbMTw>OeXP4p&9;{TkiRk4zvrUU%_ml|+85M{UNZX?Zm4W{vnn}um`BWE)h>Jk(rXo~5fy=xunMc` zg2IGv6IQqHK7dC}t*q`t4cO*zR`<&pfQ3C+J-ctfI$dKfFEMTIUQUtE|Hn0q5Hmc5i+Li{| zQ)hjLRsmA55*zS32n!i~cCf*PpJJDa7YiU?vDM-n3&7?8vMPrKjK+qe&`T^JB>;PM zENs{^TQpEx6ty2wY}na!p!LQgzX5fw!iG~UUKK8)C^5ulZzXipEbkdIFR+7)J@#h(H*DzQ+!LKYpx7EN~nLL^(X z17&#Uo-EuzF)DHc3lEzM+~XJv-#-ZGgY_nV+bL>gve=RZ7cj8d!j@bw0`!GDTUG@t zE2}cu@)hH;qv8=;5p2cD@zlyzc8)+b>l}+D7;60}!y?P%0(INTR-Nnza_!_j*g00z%-E|dgS(J5jB*4plEDA}U zL@CPR&rQ1YHJLS?tzCdp%=1QUZ8#c|Uia9#wm2C-o@eW}Bw|z9H?}_L7HFGOP2PFL zHf7{VGF92;37NpVlxADr^~X#ppQ6?{M^RQN&a7K&H3wF)A=}n20ayu(qL{lzQJ$;B zwl`@H5VfD}sD%l`^HXfcEDCTak?lMdgW+`@MSAgy$yX;7xt|T&mG1)Pd?VPNCEr17 zHWT?18OLI)1)wayg~d9>04w5PH8*;snGEP?a(W_*-Ghw+`2^cL7Tx85{%n76PgJpA zu>*^?p}^3C9a@_OH2+C<==??y<0i4V(QcsCsm6|u3d4?vo9x(3)RJb5VaMGYp_CiT zP8Gd~L25B}I<6|-v7w66)A}Dff9N%^UA8P?Om$2mOCWDxpH~=582cF~S0#3#L?Xzc z$JqsI4whgWvkObp&;ovAiH^3wk6dMmQ?Z6r^EbQfjJaO-;p_^H2a^1sqCDcquC&Cw z!M3O(UDi=ieml*s?8btn^~frg)QF-tE5?$rEl#==XGy-;NN}PMOS+0e;xoanKEID* z(h5a(%vn*q8_cd9a{?{(h{-RW?Apa?AYB=|?uT}veh#}a1w*coE9|CU9LRgF>}Fm! z+!b$=*9s_F>6=653-5NC%q_0SB8AC+^%VIasVE}-Ol}y#Zf<`D-03B|c>$G?kMZnQ zV=OKO4P>{x*I+MMHcP3@fdzapIV%L|2yB)uORa$R;*^cm=-JVvmL z>IoobJ!2WO!_cPRGU-^tWTcBC{qL2c{5F$itVh>b(od1ytim$Tz-T2`v5Y&Ipv?Nq z?wrOGI2O(B{Bc7)VWJ`x`4!o}VMy%%k@a4w7i4-tc6YoFkfP_m0fGk?b9_2^Vy!s`3^f1qWZD&u{reJHv3-;VQ2}IelCNm4N z=OL(y->RU<10S&G)_a&BL~HD&cQeo|UD&H)=YS5%Wv?1y2V>45_G%=4KZ;sJ--MZ>Hf;{4B@bmYfApGe?n^s?Od%PXO(Xp~%Q- zX3cwn6$)XWKVuM?-da(-W$ass9gyyX<;3m-Vz0A56JBA5Tuqj{WDP*0B=!%hTJmum z`_}{G{1dA=x%~>@K`TzaVl>_P5vP^kVq5M)PWR^m?{uEilU)GFbFLM{3zXZq)(Hj5 z^JT1@W08oAisgLVMW9U|a@jNEt z1k~dN#6A$S4=75fuO`n7;|14k1u?%Yw~gRHde-GdixmUfAetBbG#1Fe`@C4UQ^1~B zZFmWZv$(+rUZO()X!MuK22FX%qXmISb>yX@`-7JCPLZ=7y!0(}qYK9Jvhiua#0_p2 zQVf{wNM4~W#tlz)@rw11qa9hzD}MF|ejuLPZ)uK6?O^T@j|qr>qN2-#r%J1 zirUcX*Wivee?hZZ!z-`t4eW4rUbUPL0G#4gG4_*>%kkVCfTc%y)jq@U@qGfo~ zh6z~z|CGv`U0(vs-_PWvRlK>^dyto}@aDm&<+v>8En7ze8Rx>=6m$fV7R=i`#xVQe zYToWM#&Du5Z(jj#v84oWe;vQTa)Nhg&=_DtG2UU04bY+^xW`%aG2x$i$I}>=->Jwu zK1K`L#y5{x|GPX{y&${f=bjE2w&*_Ga~$3(&+dxySrGS(NCWN5Jns3uIf%-4c^5}y ztS#@-0T0w|Jn!1u3#j8v-gS#J3KG%0+Z=2#UGkp$G{k6k-EZ!L z?CBWIMg$@(LZkKBb_GV_A?Xis0z9Mkw1Hw%y} ztNEA_g8|%@@iE^=1KoClTgRqi;xR3Sk2~*$vwVjlXZiVr=MKOJ9pFKoeSz83=Mzt$ zSL{%RPg>O&}*|NJkWSy(l92>>cb-z*rNX5Wi*e>ISgX!R=&y>O{Dz? zzUBQfvK6?Q6e>DvEid73Jk7CjZXl8|sz9 zSTCAyn1#pnc+WST4*+r`o^Q&=_@7K$!8g03zxUk9w~oS)=tgB8W7{3%C_BFW3P#2E zL-`IHoQ$s5`Hm+zi+4>{q#Hm{E4z;Gz=krdN_W0{(FK4je-vrrT18g1k)rr=T~Xc| zqNs~He0PF17|6wieD`H6n>)DjJRHsAN8E65V}`2Ned|6e#iu=_NsV+$2ocsM^8f`yKsUHGB) ziCEH&;fJiaA-A;Qan(YBxwhhQtx-65oT4boPUZ1Fxc=R|`H?0VZ1(%ik95P?+ryO~ z8H08pxFtWbBL~Cj`ii1m7(cQTEnt^T{8%_9ETxw!%3lUQH32&+B0cz-mj1w!X7jU+ z?SR*h=GL>Gvp_^`=Lt=dK>ON^Uo6%fwAXw1C4BIJOb_Chi);cJyPsdl*Bei)0Kc*o zL#!4WPr8(by5L)${BRr4mx281C``BWpW;{Nv;{^y`8D6UctMBx^*Y7SM4vO6_Fj># z4N{aoxc;x#aYd228ox1XE@;2Y^VC)tNci^WcgN*o4tSK`D;b1dZ5n?Nj=7unS^n@Q zN-xGvo_Pi5OqrD?t1MCE`JH*@uV7Tq8}jVkn}N0+#-H7e0(N}~e^K`^MoRVh+g`tb zZGX+*=3o+QZ@tbxtR4w;jywPO51-+;u7+KvwX2y$x*KpQx7FX2Dn&kdpM4QKGii`M(_EkB^gBik|VT82e4VL(^ynkeraR2W#(AV9C`+ZE| zR#X)poz4N7;fu6lG|CE#j*Z^~*@h(sXH3UWbrfC;v6_AOp71(%4A-@{=v*8PPfE7% zE`zr`V;9mHOS8R1m#f`@ZSfXez2~E7RaJBywI9IIM|5K-RHhFQ-QrPf>bOk!Or8(e z9wB-h#Pr?1py-h@AE4h8(Zh-xoW$S@F+lUZR@DA<5j_{{nEjp$zh`R zen;$bU#BRm*_iY!DtxDXK@Dl4@STNsEMmMOtN+$ykAA`z@1)kOmFPoI(9HTP`gX+& z`p-vL`*p)JdwO5=n}xT$>u}LOZ?d)IVt{rRWVu6PfOi^pw*-g*4^g}E+b#w?!#h=A zi12IC9>nL4!f*6StRIXKeoIyY{Z(60d*UhluAn_QQB@2t{}1@y@?xaz8LS8H6C*ET zheeb{jB4Qkn(Gj&82xnuDxUFT>@9Dg2ZR{=C<$amdogkKR7|h!#iWCH<_?p@lp@$j z@NJoxTB8xb_7E{O?jC6WT`>8+o0u-pi^{=@%>Jdx#49G>S5=h7518!qN6dJ?2m1l7 z_sor}eN6tDrl{50s3@kIZQT!Pi9B_S9|;jl zBk;h#$BE^KQbE_##flbRfo(gg$YX~qiuD`Cign+CHT)zZE~DnV^OuNv@(YR2l}*BA zc1(s?Ukq=pTNSasO%%XDXR&!tEez52ip_@)W8)Dq`Ln3lI&>ab>AdP<>k&){sw@+E z8W2qg6x*J(1@bpQk@X%Uw!On7b!Anty=ilRkQHM4dQ>zGDz+av407LNvGYP{Oe#GU z8Tln*v9XBOZ7mK|#}usYWO3*YW=MZSMBLVH*a_KJ#22UpeBQo1V*mf*o8qW%7SMYy z#nHHEfQg~v_<*IrTDKLa?4xjU?GdNkFrSZoEKaq<2*!vK3EFEQGuw#-Os&XikRnUWRTRM=#DyUZfO%~dSJqPN&A z>lrku|C)&>f6#>HRu@_RxEiz#A}a$m-OMWDS=~gS=eL-=dP|Y@%~TZa*O{C?PEpqA zXtG{klS4m<7d>&gR`U=q_PGFYuPI*D!%(kWYw_wXrg*E?3+vl1wNVo7BR+gRf#tF% z;zKU>cBDKLA9v0}@6=L!ZHq#}=jY;ki9`7CXixDy3Y*aUyhYBbbQBn_D~jOi;&%b; zki1=5l8q|?V$MmeMmDx)pOP#XYdyzuCBL5MHEpDO?K<^Heg(egN^0KS6*@)qE)EQa$ z@k5|D!%cput;ho%WsxH9L5vTRwxc|-FQAgN9i57gWbKv3T+v{a$tQ~sLFqTiI#w1R zh5^H(P(^t+-Q@kXviSHV__)nyS^VZ;{Gf}XEE#7qqnRxE4!vfry0Yw_VZhoIlI22h z*4Mis%QtWV@nDCn;2Z;D%57QknjJ8$yR5XT7Klp)WR<+*eYh#BjmEiTomWg&|FH=p z85db2q%MHVL+Qlq(R*!|b#VPMmm#t~en8q^kWE^?1vWI^(PYX@bpY zwz7pk_H=4Ow%8MlQS%qs;@1rj6Q9U7)3NCEp_ptNhDzv}Lb8L+8KCq3NRI|2)dy^?FTq{#gUG^%GXDemjRD7^(SP9uL9d&_s%jKZ!8?n-{P7cah3sCHy96T)oSHUYewD1O?XZOgV zE$5+-QB4l(u>|{nx~9uvKa1ic5xwN_-fMx^x*>-L;#h5olOy(HIDUMuq9!vGS<#7d zz;2JsV13p>d_e74lV%-PwQ6o7f z=^S#m9Q*7dFy~ToT>a`84Kq3J7_Nd-P9{_9$_a_jup)9+PE7obnNgaYoKp$3R}JLU zVK}Qx9g$P7pw0jLNKQ>I2r$2|oSHfd3!Crd%ukuvBi2>sC8wHmFO$zk$obZp8$q+* zD(8Q|jwRoJGNjiW5Tos2Y#}sj95?xBny<0FKc3} zhAypn8zOk9TwNj?gomG8duJbrQ{UyfF0ZlB&_!+-iY9kbFNwJtwoDh3Tbfg#w^J20 znNRNC%s^xY$lW=pob-As_w>sJ_?ad5OvcD4x{Qq7R~yLMuX5kzEX-JY$o*sD0RA+T z2jm=JO~S0Wfr&zs>qzX!X=;N+Ti>)B5<{w{tB?zjcjdt^(*W*vlZQq|qHgCa4<*b2 ztx-N1R|*4*Jw;{w4Gg^qlSdM84&3yXNB8W*4C$6UR{Soow$tVDkX|7F)>PDaZFwFu zC05OPPbO5vBr|idqU?E0Cib2NeC=v^u`>GiUVG#v11;L%NhT+qQ)J0Kk$5GOek!u$ zBzf8B2K3rdnY0-nE^mKSUYlAQv=_(ZwI%4DJ5Q6>?zaHGX0E*cCmH01!t%!FZDQIjeNYOGl+ouCPxmGkCW1X+$t!uz47rI_vZ51O?1PL2FMqUM+5sj zRlaDw5v(FRTE56dHS3wJe0i`1N+@#_#rn7MRc{}RRGP~-6TSf(Fjl^u;DZUpIr+{r z&*H6?pYk#w>L5Rz_Xli`mfzj*X=&O-em{W?Nj2WeAC0i^SfPf@`G?VRM3Vgd2cH$& zkWc=(#4$iQDgWf6{s-HI%nf`6UqmvGp{ib!+iHAMV{zr9M z_Zd(aqbR+n=yV143GW=DGwcu6QeK;U`&#E+ynyAut&8=gfhYRwdI4PKahLQ0Pz%KF z=6azwctQg^=r&H)8$i26={A4%g4n!9FI*6-Tg@)%wr3gx*4Oo-&6)#qkJO9)S03oO zLVBs0I8<+9_3}|(Af}B_l+|CDjPutkxVHx3;-**liT%Ou=k$uZF&2D!U9YqdlTX*D zy2I>$K$az#j5?)Pu?|5MI;xjm^;IuE@XYB=nJx9(ICmtFg2dOdHfS``S_>&?NY zv<*je=TTqLdj;w)#qo~q8KAr7U<9<{yQ0>_S8wnTV?R3wy$O5)t?XUB>9SowcP`Ld z>#jiW#ObYfd1At|!ep+GZq0i^9x1K671##+;BdWd_IqH5s_5-|qj&4HMei{B7_iD- zin6nx-eG)AV38wr&pO@!pF(x7LuiPa{M9@6cn@&;s_w00P}=F9?%i+#fK6|`Yk|(7 zt&P)t674~&`a|!*;sIOX^5hdcZ|T%xo&^LvUX@t!KB?+X?3nnjPbuY%&yHGy^=Vc2Vh5vxJ{?e(3;du@$H!x!@DY7_n>5f$ z-`1zk#7JfJcYWq*?2gaaqR(21rI}++`mD^Cz}ipNXZ^;w{@HUqxLpJai{JElDQ7@7 zY^cxo!ZuqQcayH!`hp^vKn~u~7gR!@68TDBFrzW{|5hERhZfogWcX!0^ym*}gubZiHoTy& z`l46pely?diwEPA4@LGUvLgNTCBzd5Nl>)1Hy`yS<+lRen4m9>4F&Ca2|bdu!smca zDaxw7^~lA}_~`X=eNDkcR5+{ZYcaxM|L*JS%2o%M9;~l>vJ%+)LHdSbj_9|PgkfH1J=pDC!VI$7glWlPlcGb77#8k~@s~%IvAJ_kYQF_c(oYhZ1>)Smr54e=5 z?^;#|qtsXW?&IigyM^n!uh;>pI7#1q(;fG9eRt+CWJi5>RyH;oBvkvY z^b>!huu#}iQIy|fa$2&YwCk^*4!?q7`9wwe=ZAidHU_ee<;yUIlz`)~Rk9ls~0wf@krD>f!dJTMS??FnK^63utx5uHGxV3EQ45M&uD_~>Lv(Y3$tSb)S9wJ$w+Z^|{{2ym zs;|F}L+SOMi~joAFqG9Z^^fh)kX7odf5Im$$=%)hr?^T0?O*AiKAr(B-wXY7|FYO; zJjtqm+1eeT&kX%*AUc_r_w`?H*hW`6pPpL*L#GZ_L+c-jb-V5c?>h@vt*Rymw^HPz zZW(+MzCJ(R5STSnan~>!GMt@@B_jfGN59f>m<8A;sJXw)1 zu~s#TOcAb&?p+#9~H0mqq~7`WW>B(E0rFFzQ|1iBCvQHk@Oypx`vt zaA|@=wCI)5U<-!p)1Mm+wjyVoRFqEnkeNV-t}q%r%K~_~$!L7W4Q0CsqsgFgARl@e zO;-2E-<%j@G;?#o{J-L8quBry8us5en%%%JIuK*DsO|;yNdcn;z5(|4Hd=1MFO1q@ zwCah=C?L*g{ca;j#{x#1=QDv&GRuzDkCa;e&94qr`n{9*D#iX!jc#N_ZACYQ`J zd7{6fY-8@rOTUax&1M39d&KB8#EM>S`%>IM{;jO2m3wY$GYcvM9eLkbx@WOaY&l}w?YWTQrg3-GI_K)^MpY(1ihn6;Q-lx}w-( zH3kR90C~O97@89W*Z^z{!}2@58*dDoiKhFWjWM!Q2++_}V@&y#sB~r-W6zD? zLKiU27RJ1Fn?Zg(Z7l4Imdv-FvG4${3I}^dZV5CN9&x~Bw$KO(EDB^qPb1`^Gy3~d zM%YmgAaKfL+%QEBD@}$MRurrDSj~-%rHn<-dIL;xGM3G3gz@}eBVzq)tm_XpA_H!q zQ_4`}e6X?FCL6usY(<$;(}+6dix$zxSld4iv}?F;eNUWhPu?D!0v)8+*Gs0L)#WC`YX}4(byymD*(-^xK4Sf}?S0DSFG-21i}5eP8Jk*$Tfkb*AD_p$W`SBafagDNfjX#qJ0GwR zd?VX8!a^?Z1M0uPqK(W2`0t!Wj}8WCv)p9DRz)rQi^Uj+Qtami7R%6BoMam;1-7`O zH!Wu=n4bdce9=;9JEmlPd@MExFvz{R-%_OSGhj1MTC7Ez*#ZqXYAL=d8N}!aOUaTe z0bbltl=8l%Y}>vVk_k&${1Fmpzr|ASVs#*GHdyR_<1+G2wAc^AJLopX;yAj&HLx-Q5l7E+0$tr)hWsBP=cZp;k2R zm!(w^9a}a%EUnx(0(E(6aSL1ty!|pu+vosbCDJTyV?N^#5InK8J%N&}r;fx)+TPm2 z($3BU_?)*E?+Uwt@@PxfdGA5iFJ|fa0LOg4m!;3WbfD#*TlyBP1#qjAr7yN-v#X(& zzW30U*WPLwFftK<{k8bb#1iZFKbFDx(=9TvEpj|^ie+%3Ex_bcit^wFi$4Y^K)+f1 zn-)cdHF=7;@gPN!Th^KM4^R}3PgwjXr30(K-(-M|$?0V*{<*asU=`NM#YX9mVj9NqCfQ&+4P>4fPHA-Cib-iB&UIuT&rsdxQi5dW~+f${LeCW_ha<`S2kJ3O?E?>ZMVs^e3l7S z3!)SGW(f)&j!kLHEI}!F1?lOQiTQ9y9*?n1qAO4tX=0gla1;Lhnq@M+&W8-NOm1xe zv`w>2IgkK)fn}EIHHrb*eAqI3*cH?P49lE5I2WG#Sr*2jBKmEtB{Um<|Aj~1vxL=u zj}~veC9J`0;IUOKi_T%{bhor+(FH%y{;pNzbs{a{Vc0AB`l@B|L|nG64K2&hUI4ms zqh)31B%t?uDryBET2>}?!<6i%B@*4Ps1#&bwH6iA^Sv#r@9f8hq?(p>@4tX9y)5gk zcDOz(t+uR3%1*m2>!VlW7hbolPsd*q+cv|pb4@m|a2w0cb<2PbscqSLsxfeCQfxV6 z*?A@upZC3Q+2geqwA3Lczc5Q|IzAUXvW6u#=P7E(6D|9TXJX~U({jKG@8r^Pmbm8g zF<|Lqu^#?~N=EmSmgD)*lx{7ns9ByVGVkjq!+kBshoeC`a?Eo44;HKc{$JahfJIgP z|KsP*y)y%IHE~ROy)kkZx zOfxmLEOTkIOmnHM)XZ|vey?*b18DaBe4hXF{2w294>R|k<$d1E>%H7D-L`#xIgy4g z<dA2>B4iM7p6;93aleRt6CKHm_#zTjvE81AsQAcdw!a<+GCI9R zCe7~voTkVm2VU^vSXuaSA+cPVC5y+lq5tR0W3u=?9+7%yvMl}xFW2EO+58-ULhJWs z%jkVX8n!^Tybw!FN*CGZ9SnTnG1<4$ftKq=vaJlPHuW{=x=%Tk9J37i&@I`vJ^*RD zJF@L<P93G&2MrlEqg-NUacf#^BbI+ z?6+mz+!z(o_2fQgyRa+%fYdV(Hgc?zjOv?Y&#>cqkfvy|dh%9?(#Je@pH= z`7=DJ{IuNnYgDO^FP8f~evX*s{sy(rkh8YEh({>8b1DtHA!qIV1KIEqa`wSvguMKa zJYaVY5lnx`gPw<%{Cko-WLG0}P(C6LIgtzcxjb|~zToWBb8h}ZT6#ks7VIFVUPt6% zZ@B=qvgF|kpx3X<|15(ekWv;BaK>i;0mnJ=QLZKXVSCGds+Wci6^Ku8TPacXs6*W@Sm zA4BUzu{^)mITReX$_qp+D1N!TAUg~2d7iw`JeZK@hRX|gVu2z3Ih6(wgXYfR)co$- z@}j0lFb;WFu9|~vS^p98l5a*3^Ar2z6$vxo%s!T%dwC&zM-Hc!)qlv(ZJv$(pMXyC z^YTSvYW{`1ay`;&d*;e3_wT^j^^sS-2T1nRTzTD_y&>g$<&8^QK#fY|O^fmUzvDla zU!S>_NQKi3I!~0h$eG0S*zfX|3LHgI8+psKpTe@07>d5b+uL z+y}c+gbL$S3YoyE_4I@CkB=7;;cSBZRC}Eu2cf43uyAO+iezLca9{zAq?1ly99}LM%%U%C|j`mCxejzb-aH zR@^TCJpl_e#Ve$74?xcM_1Cb|)&nQ(q&}@|%k9UrmX1>t~AX`2=DuY^tauCm_hY z4T|prJ@Bleoq>s+(N(E`1K4m>zTz*z(*3L`{zrx&i5VyY54Z7mT3 zdnn=E@==WHs6=!IXZjviBBp*utas`w5!3yM)qUrV5?KnY789%3rvR}WdR4K{EJmjC znqq%^6V5C{iPC^-^KL3pq0nQWO$H77ONp9;hfKyTR-$gA%r^dQr4jBY&vxZhI&@KK za^Wi?`7~6TpmHi~J*PBHt-#-Nm1eAxy7Et@#i%#&{a+_#rOl@&2$|QHQ`3P=gC0sU z=&!d7dhsl$(z0|;%|f0*^ClS7agtLD{nVhVJ1Q|2SjLT;ITf4#&Z)V#Q)zqZB_a%% zp~P((?IxzH&6Ex;A0^V@=}O0XVEKogl}=+Igdc6^)SSOs>HPd?Lb_{8mrpW@Igcnw zhY-PdhAPQ-V7OLqP*U_M#A?~6q-^O;NX{cl>VHsler}G^YwSBj%FI#HZT*3KJW6`~ zN6|m@g_6D)!RLVdg#9tqYQNal|{^( z9m?QY&l73LGs@5bRwB^^Wn|g}Lj12O4~D^AHm1s`2~|+frpnk>cVH(jD$f3p?#f3M z=lk%M^N%U6UUPsSURGSg&JgR7Hx$?T03rlEskr_+KrEyFqm(Wlkgv*Vn6*Io&~z|Dnv?3aGX8Kg!(e5(*DbDf2B$ zPyq>2=BvmBwa8Q!U<1O2x82IZlrLbL+batPW5qjnD2qR|6Vhq8QuW4c6dvE@)EeYt z(5$1%(p*$XZpA3i^i3k<_%Y>~ZSWNX+9=EMt?1^b3zX&Jabi7sPFbE4POK$cmFI2) zVGW$5yzqmASVj|r7A7mJ(lUv4hx-#{)%WmjAGB9i4?`lMU#7Bp7L(2(W%V!k8s7MA z%1h0Ff{$%cUMfbA`tlZKZSZLn689-z2 zeE(%ngWk+gHjhM1_`+i4wUI!_eL5L*>>OolLM9PUB`R;OhxEVx7^kL2CzZDfN{DIB zvx=v2JSwH-il;*a-1&9I(`N$_S{~%oZGL){vMnr)Ncry>bn+j{I}VMQMvhb7tzQg- zu~T_33;Dfs`;_-C#S_aj4rNC%0*yhPl${Hc;Ur#GK6vW|AfpeI4+k72Vca$%51UQ`^SH2={Af1aFl&@Ywarn6a<#+*{ z%zwr!-=2PekkKoY?|z0!{cfmovddbu-<(!X^`3)batr0mEyVY~T~y9yP9WBhUdp+% zdw>}Ol%ERG_qX6<<>#(Qv#lysem)5ywQN*=>0CynwHD>qF0GMm^H(nR1_vMBs9bax zqeXINL*+_>FOkBpD1XVo=Y8?zo$3~3LQSgh%R6wt7gSRdfY0b9s_6q*%JDB5^wFNG znAww%VQW=!`O8Fb>`}#yp-APRq5vwbkLA=d<26+p^c|qvHP!3`j9FBwnp+Ga*0^1& z+nRZlSbkWps?Pw)g!Na0HbVb5R;a;WmJsvf+tlC-e?oW~b1HTnp*C0tY}aD3+Nl2> zIF${YnimXJ8^5{);4@2Yih-qoeQMLOd(a~6FXM~#15|`T}Rfhx6&YjUub^Mx%zMXpN_iuu?7jY7gwd zEKViIJL-h68R&@kSe;OTT+pfA>Vzr4^W$Gti?-nVJcm}O#i1jJPT|9{X|o!S-O|4Phm^`Tw6iS@T=_2D)F#Ps8C_2J%k zP|3WY&bSbd+R$#*ZFzwR-+!&TA1g;%ZiqUw1LFA2v(=gGOK5fveo1|5&OG%0y)a6B8r^PWU=rvR zoaHQo?sai0O!w#1wD3Qin#Go!T2epY)cWToPJQHg>Z1RcVIdEwRb4NlFrVIZ-bnWk=>4bJs6sheMDNX*Tw>K1p^Y_wih4i@k!eJ_^F61Q6+6}44_qXc5ALXY8mdG(H%8qvaR;$x{Gjf6b}zmIYKgjMe^(SN z9yjPSi#QeDKg+2pFOO48S~qpyD_4m1illz@4H6#q-m3fKS%Ikc1*(in>5=MZA<)x| z9_o>SFs)1asYez`MEE33{jx1&;=nWN*GpsYG|W}?n|gM%(`{5wcyaQ;BGe^->H5 z-v73GxnMAMyogiFiix1*NV9iTue^o_4cZ2(e@3Gko_E`zljGED>F@z>3{211OeZ90nMMYmCPJnA zLrrY%PXr-blk#^EX=QIsItxJ2IY_g-1B6o2L$k2%wyz>Js~A9p2JdOUGZvx~@`~m= z7ln&=2Whr|Cy1rt4Nd+HJG#_e(>^&*q;_v>ew|~8X~bczo-_o=>V_6@WFQeQZq$N& zkaJ$PTnp;-5%P?;=4-(!Y_V@kEi?>~N%Jf%{0h|O*2`Mthp>Q0u4#>5L1idJH~76$cx1EUURMe z@8gMdVUiZ_Mi0r}P%XhXl?eMz~@V?WA8J>)M|^?u>h<+zF6xP zg#k{aX-VCH;};#*l13wia;I2J`U?rf-n+H#)}M*kah}$FGA!E1PiQGWv?rE}FKX$( zAt70HSWEvC@&79gw7%U7iRmZ1);A5_Fr=f_HxC(W6HD_ zqhUIiKd-&`u?q<2xb_nM$0Q+ozqU3K&E>DO)z)shK`f6n*VYY0;Ib%DdwuB>MEv}g zL9Z^*JP>AcqXO;ir~Tn&)7{#(zpfJ>+fnVE@GpsJ*8|%2hM7cK@s_sz!41UpY76bX zNf(h+eolMuU>|&oO*d^Pz^LGwsO_GF+Rolf+V0s<*G|6L?w{roORqNCo-=7g7<)n6 zXT3ql&Mw+M8zQBh?`a1n08y1+(+)fn0E1SceUv%}%`Wcl+9%;C6nYkDpS?Vth+i$$ zj?O>>LW}v@v5?`wfE~4C_#ZHY%y!z>QNa6cMrbEqMFnHWPuh1ctw9jFLHp^Ovj8xA zwe!u8ZhQHl_S@mDP_NC}V-G}`qM-S*bki07y4>SBaYt6Ca#Ln5crkrukn-!Inv zKD|TCzPet2UL4f%cin&3Ubyp8J-`LioA`l2NB7c$f^L2Xo1bACNAH895OZ5Mz z&C(mX69{=LM{nHzCOVS}^~Mu{ewQcct@i~YXq=(9SqFo2e6-$XFJisOC-t_iS`zb< zyY#k|h-yoR>#^ctgx$aEu{#bRnjWUd3CB=m&ePl1hp?@Z_4c3n1K#h};}cLdZ?ajB zfA~8>{u-`#SpO{a|FxCu1u;c=^iC6Y5bK0uz4M0+iRn+Pp8CKV4ixf?Hpt_p8W3%=eX^eYGpW_c`hV&z~pc z+AsQ$U}#snP<`ZPH*o(0xAokQ-Y3EjpXs?b4*-C?!>RS^c-;{RA?jYN7Y^S?tmj(l zg_8lVgOc@$(*ZCRoz_c)L=+m2>7_#vXk^USrW}oALCnro`Xe-z zSU>5Y&mJ|1n8#%5bA+Y{JeKH>MHUjV$1(k}0cLo?0R8czYlK|6tk1o@5&47D`jh@Z zLO-q6=aUvFHZ{@b3n~%ySbhEwGm#!k(ihN%gq&%lyPxuhbMYx=sXw7U zO=s$h8$FEj+X;il4>9N?dkp%4l~ZBVPX>KEm{ZdSw>TABzrd;F>*Uld?=WcO;|7iW zNMAhLMM!9CxACG`e@?B}Gxe%%^N7&=S$!EjNW{pK`tmO_iD~5*`iju8L^#z~U-201 z!PxKg=h1jzi94!4{~Nxd$ueDE`Sm>b^-KCHZ6t!vpY&C4BeSyUtiF2tAmn_yaB6eQhz+Z)Lc?F7PI?IG@tjq0P=b*P*|> z1Tr#ojQ;Y46~yxBG5uu#4`G?EuP+`<%#S{$Z=3~R@z*x}jcF~2&}NSQ#+k2)WqLh* zi;CpdN51-2X*m(zjM29q8Vm3(KB&L7Z8I@Fen9t_0U$o!t$T+2L4-pu>N^Jcq9f`j z{e#&6oiFDY^lM*EMc;6J_m3hWZ*&FJB z5%U{g>7Sqcln7_q=->QVK}1WS{_TV`QXqWtmwsyfO<1&4{d7GD-=)-w?!##*{N3(sgcw}>MwOBb}3FukUD~Pmy)D- zDOpOCVx^Aw-3bT-ku!dR(+XWVxkV29pi0-^vYbl$V5hTaaA9dx+`0x^I;@>0CN`29 zN-4NILFyv)le$TLrLGt;8Dng#UFG)P<&I{>e2mxEp9qPVDFtKZ zIQ(?OT`5vOjL;u6Nop@8QDL_bY9V$KEv3@9{no&-2~r9!=!PlVdBfzo?iQ!(yplnklTF-ymmMxjnbio}7mp3(G6n9YQp_ zIFkP8FE**l*d4It&VdU-A+04ptSyOetSf%^l@hR(-Xx8ldRUAI4oWU@mQ)ryD_r*8 zX)b$CUY?`0+>u`ubs=oag;Qr52>q5u3HUA->?Dm|UMTs~(mql+UGFO>jTQ_8R{A^qlM{t8|uB3Ag+`GX%u(5sDOnr6n4- z)m&)FbmltqD?MQrVV$JLOOg0I8C&lH27dId6z3l?$WfAS&%r?y9#X@&bF$VrRL--aF4(LJ!vy z0zFw_LcY+kVfCV!9JRw14WYy&c$Fxn)@)TvD|43Gv)PcvIb{>+Zw^j%_*PK8>EWdC4v$$nC1x<~`~9^}Hty78RBZW(Un!Bxl@xVYCl8!Oan! z^|UG2Zuj0+Dd34MPWjutR<>6A>;+5#z|zt(=j6hC9CyIgA!y>?Ohb zgFUCByuewONH6~+gwynFcrkV@jbCM|Pm^LrEu;st@ii-{ms;jv5|ZhdLQ78zjiddf z0I(7E0Str_s3Te`a~x%bxfSKGGu0E=&j>^5PiKT>bmMQP0Bg0Mqgu470e!uh3Gq?y z5;#g2l1g85a7(znDprNo}?aF!O)oS%gVy8XH}gf>5il?RLF zaGLP55K1o&mBKuDFH{)Vx@MKB)$3$J!VGOcW>|6ji8y@b%u?&@AS|n-sM0>U&{fEw z4O0q!1TJW5^?s6n5H`>eKjL}%T|YH3IJd2>PjedilQ6hy@|F7bf1af}&{6D!KvZv# z^ZlU=p^b63dS2*EpF1y{rc)DqLM@gdrTIDKe2eCmWTfOrcP$VbG{bsBtJfP#@I+=c z3e!ebpMG+Fabd|Qex{xoKMNfMbA65_COsww+kDf@oE4?O5s7r=FJSB17@t`B;x9ry zv56E=<@Z%^BjzWW8q{^_>R=cthdtjhuCTX^{phh;&fLk|ERx+5`jyYE@3e1mj zPzDpqFwFO_;p9v5qCN0gJKB3JWZ1vESdaQ0l%!V70vYg-;lMQ9 zz>c^pF)Tn5tPxu!)Bhyu?qvz~t-b*GF?*&M;Kof}sQ@}p2=IZUv*u{dUi3`W-R z^9M4WC4i#i3_Aj|Qc_-6UI@1gipE!nn5S#bHM6~;uypKMlUP_9Yr~nbXJ-v!1H&i zXwhe7lWqk;Mg4PY0hqQ7#%yqWT984nnGw+E5TQYp_eL5;T~nO z$<)yE_A{nQ3H8R3DE3cki`%*Ywc5O^kBKW9M1?ZKh7_1O6d@k0~GYmCuA3@fTYs z@MIl1?#Dc2K8M`gPdZAR72^x+WrgDl%46&UD@y9Hi$wz5v@xLsvMm|!Pz}j;Im*ky zHM;VUDX1U&YL41X{+A8bFe|9WRo7htAJKFAbJO)m8hzEYlJ-o2JDPpX6i)YDH9c%? zQXPUYlSIE>Aj+EB+t9^IJK_)rbzd{tY3eo8gSkBbJDA=hV!`s5j(33tNU!A=B3EX~ z*m-2&GbZ-Qj9a*>n-pIsKn)(~$aAnrw7js`VUKn>@}TqMoh^*X!UBAH(pJ|^VZL3x zGt#WFLU59I`2R9EgVam{?>D&G4$8%Xe|FS8Xz@vNFufe;)4oNViO_G;=Sw^H@42PYR*-3_+?oJiCF=j-F~S1_niO#p2RY;wZA0 zR^%2Hx(euPHn9Mbg`AM z*HO6W8@f2cH?N|s3=X5Hl8#cu20Bxvy3CrTh>bl1{lv}^xXP0iKa3RdrQ3BfG~Ucc zw$nv-1&U$x$pG=Vc{xMY%r1n(HA0*j;x~*!R?R*cM>}#8#3BN=bHmwtxymey{gye< zmuU&VnSPe&6Gl({EJ*3KNt&)qU0HmP!?2D$y4aatIIhxO0-b|0fCn(Fd1ZyA41C%v zxJm7q59D3!D5+>czppQbM6|vf2-U0lY<5s`jdAE$V;piFjuQL$ zG9*MG?JP542pC*ZbR|}lhtv`Gxdo0gN0c#<=i5Nh5*geKrkicvtMP2>p)7BeDTZij zs{jRYcJNv&Eo{e;ErR)ri<@&Lrx;jo659%=2S+aha(eivSEyRqg$#+#M#Hx%SWy zE=!OZoS-)dwU;;{L3{*&%Iw)4!UTRLeY3ZzC1;9fdWtwb zLeAs?ssSo)KMyFOb(MUCG`cI@6qE`+fmb9JUN*jF*fw`(`jCdIC3kh^> zz8LE1l`GB@)il8SSg2zLep2bAVlgDMC)ZK7nG_5P(}0XwEZ-~x9Gz<>XxhIzuL76B z5CNd!IH!FgvZV}?+T&vF`8ky?&l?W$tT3tra@=)eK!&yBt5X9kROrZtHkbrnvUj@U zUoEZ-=kZ~6lEe@g21Up7_{!Me((z(*fvXDN-}VAg7laOUzX7)hNbx*reT^@Gu8i~S8Oi&aiN%gPqNR)o9v^9$3vNKPd8~E`vkFx zD0E_pwn!TIuH+XFsEP@Z^Q@inu9Vmmm{n&ki!oqgS$Z02F@u|F#o|@d|E7*?iJrt# zu|R0m0D&`;66TFq{*(bx26JlzXWx<4qE1(rA~he>5817ECyC$s(rE#YSK~XH8u9+wx%`q-#GUU3Vq&leT zoknwJi6KJvJvRp91Y#Bn!QJ6??i!-)eTFa`C@VB@?$e!yt>q{2 z=4|m@cP0x(p-*vAPi_~O9fipu*(4o5-YDIGEvyp7#rj?$V>b96jyK6p&$hGU$|P$)6AbnbnVnd;m9-a+6 zJZ!EwkroY?I&gLIT$(FxlOW{p53~l;nM18oJS)iX(hf!j?!xa{^xUsTg+h#Yp&G_{ zep?_ugNKx=E>Dl7>%X^Z?%Gs&P%=`bEX~4F)oEKGikY_&cpIr^q>8 zK+%_kGeyI+DLjBw@} z1_d~jTO9eLJWIMr|FOud{K>K_-B8es>yJWOJGv-Z@NeCqx_n!mf-7YCaSjOCrhR(n z*Acj8G%Cm%+>L>NZcL@1(J4}2FYk!XstB;F)v15ZwYvvcTpcWESba1=7|V_#MM-}{ zz0S%qlyIHPW(&b?hKg9&W@wa=voMgq7sxOt3;jc9OKokk+g)*oJvtS^l7j_hd{YkO z99`wDdGN-}AmRmY!o8rdv_(vv{nF2K5#WqYk+!nx)~O$%@*^KI+2V}EsbP}qwkI^e zOnJx#xGHj4P>4Ks^#pIFNr^%)Ksa7&|@QOWIo1esM#ZbOu?Lq@ z>PN;qM$*13QX#cxl3FMK)wO0B-msge4p~=(i@ym0(8Ep19&wd+>YGEby?$w(n zta-wHx8&BgGx(*mXupmq))B;NOV0wZ9@rs6mnlg5cO|)`j3h~Iu>l4i()m^lywd^K z8TCb$f@MipR(S4AX#ROqNEBkh>YHnr-Z7)Fb=Taoq~7)^SO@)PmDE&;VQ+XvZDlJY zofo!3b+=xI)DMNP(HT;Z1lA|gr&>y3^yv&qGdJOd7?!3CgI3mhwc2zglW@P8c9&IJGpcVZkzn;v-F>U-en<@9NGzuPh<@d3dO}LHh#0ydpbu4W?+{Fr% zwPhL4r2f)eA>_g8@;8m$aJEPTP>jW=zJ$a z(Q>2e{J*M%p%L^_rqo96gR~e+L_%S!{MH0Mc&}k#I_uRHBYoywRBg7W`Ms6XabClL zwiA|&toCLUYzc@7s15{-bf9;%+LjzO3&5+a`W_dxWl3}F7K8OQr>kd8Vc{ZVs<;we zDdXt7S<+aPyu-7lCWbEH|2xg4lT*#1CQCM5lm!2VJvl4(frb8vk3JfSQ_3JM~GcB1`+ z?~m2o4Xyq0Jxe5AI6%^x1|WCN+7Mw@Fc`qdT_LUSAV*GFUIC37Bn8V++fpuF5S$~k>x{X>_ZAK^n zor3>p6#S>9Y{G|RnSYk5F&ZNOZBR5{`x~QHAYWVL?LPeHppCdQ?g&xwwmSaXkc9Ac zB2SHyG0I5Q4UPXCQiDmRzx5Wv+==#oe#%-pnKv`sdEL_WKM`2KAOm_C`S?q?=KRbg=(q>?4(yrb5dnee+t^@7IxjZU9=9_86%Jpphp4UpEU46^cGI_BHx(yLNr zfd7z^yaGoab0#&>ITP|smIh#Kt<4?P$Q*OBHKL<M&^L$X8*~^Y}Qi~JPt$F1@ho!UBvDMB3=zNFniKnn7uLU zs5GEDw`N`@SGU4&WKK&5NkNI#J_C$pKPnU>mTEy#e~b2G*oz#1PauNLrL0zzaJWJmzNy4F#*S8kr~V%bZ1y z`zRRJiPpKopW91FgskhZw0wwaB+bN9J3IDYq;S7%=e@qsrh z#|rm#gSY-Q!Bf6Hy1D?}7R#NIQ&!{9APwHgsycV*WS-l|CmPO=2YD@GnB@(kuk|p8 zA$Q5MU;pInaEL&4-qW8&oRH*ucY1FZ7n31Y_o#uNP}{7#wh@g~sW%G5MmSDEGJ%)V z?^@yA>+#$kBo%AI1bTUu1s--0x_8my8dR?}IEGc?w_#1XQ9r>H)uTBAP6onUy!8*& zSl0iFUPsK3Q&fnoHo85$(V}l}_Sul~!XlnEie+Yk<|LT|c@u53I@J?}13kb>b^}{q z?XC2wC{tK%*KL$H5&w@1`5fq4rrjo5n|T)JN*y9-&t##QrXo9l0+Wj$P$K>Q1w;-X zxuw+fXlMjdHP|BS3S&BoqBcJb)RaQt>&5*)9g;Z*96Z?S^-^W<}>Fj+bfBO1# ztC^l^Y?kQz)6qrt{vs1B5i3^IRvnvFr(^!vO<YY0C5ZH*L2pIeP)qBUr%r+=)G z>Uj>WmO2QuSBm84>+O+uunRv8v*FNLe>}p$8ZK6M$McaqJ6@F5#|s5UyK>uxf2eQwccR1b4#WNoUY+2FcO#^aqhhH|H;>sOM)EKFm~Pwu)svW&Yd676B2 zDI8}iHqH)}N}bT3GJch3 z^$BT5b_mP2O(ZaV#8q>h**-C)Egc(Y4)%`(DFJmbiCu@l{Etn@I^Q5l0Lb*kMKQkur20UVgHMhtW#qU9NO#fii@3fZn0d$PUJ z{TrqmK?=}fh{_M|q;O2lD;U7qyjWS)S(XnZ!s7fot?4kSXnADCaTYE(?2W3CeIpiR zI&$&~P%JOSBM^wjyz`^qFwPw08Qa#p(j<(dPmAa$G{BeTi~+~P zYR#k#9v8UO+Z5cUuC258&zXw;wJ9Y<&Kv+oG$q2T#CEGFscxWb!Dkaj^Leb*^Hg~T;RZS1x}9k;r&ZX5eGNI(J@)@o&(PVurqTZZOR1-#UXYw^E;Whn82|~RYI6A zlSCxwQU1JBs@_yT_!PCpT5lK2oDh zH5NuQ(gro*z`yTHwALPpmqgSd>d2LHWQX5|6=@!CBu$QDJ;pSuRU#U?0I9pIVCXZUpmGh?~Jd^ zvNP4qsyVKEn#i(WGzU=6YO~QqM(u0NWt~|vgGIhZe&5KTuu#i8Sm!!{(%p?^e6+cP z#15YQYs_0?#6HAESB9AzgtPkDzdpS=9M zb)&0RT7uo}z%--nnVn}cJ1YbX>{7-JA{_p35^TKP<;2rU#;I{16i*G_?GkMWb>T)H12P+zvuOD%wGr7_#$9} zyhy>H{_>64*V=#;>;aj_{%UT_KC3Te(Xo-}^L+1Dv&|Y({plyanwy9Lczmt=3OyonLMvvHMbQ#58pJ;xA-pvif6iL$PP=^7~Sff*+UC)_<-30X5n~u z71s#n85oDWX#&Ge7+4Sk&`l2X_QnQUBhpf9o^7k+)*;Y2x$HqaX7^najdl)KB{o*f z-f|*MV~9NqhGOh*UbC2&wQhj3=Xp+c9<|3;2S@t$Vrh6K)q zJ!21t#4$^T`xwAUr86J22KpKeKfq9Q=F?!|2Pu{Sp)Z{lD7K_GWKqsc9OxL&dLh|k zy1anYv#iWn76Zp#i9#WRA|=eJ-(9;j`qP>4Y6JOmYDm$selwRR@pa3y+wl-Wn8DW? zYn%y9(zlANr`MsZ=XVNT==F0}HM}+&s#nv5=j}6~et|xAp|!U$ WL5-{#IPkuG3H*A`34fpA9seKe=vPDl delta 35055 zcmXV&cR)@58^@o|IiGRPxymSetL&{rwnBC!WREgJRyJKDRJOABrn0k1R45~(BAe{_ zrLyJs^f^EOyv{i{_n!M1&+~r9xu?S`9a~+|+VrGfiImS9S6#1I=KP>-=N`2N7#$A) zHGr22L)Hc5a(hYUyb{?8lro{n)v=7xE9#oTUI(JP=7U0cbhAa6*z-o+v4jwjsTMoswjVyQC6cRZ=bD0#=|;fh6mA;VCHeF!C$D;Tn@~ z!X?H2egLcpUXM5V=9r||KOfly)c8dvpO==Tt5->iLv@i|0jA-|A;lrQAvOlo_oa{r zfmB#+l?Sw5h{-WHlng(lJB|R7a|cPW`5^K$FxUJW_zdFq6#$`v5>o{^1eE>wcLLw= z?GymVBRJGa$gjwk08T|fG;3t?0YlCO@xTe7o-NP^*6Q+r`QaPk1zzeXfHU5;=MaE4 z4{+#n0opbNc!l4h-A*7AE=#JfE1CTA+T^b{lGHg)lD&#x~06&$1ybMap z0+V-f5#l*>2{qXWr@qH!D~Nk*0eXi6t==2JcR2`$zW{wk0~F(Jk)0iY5{s6B{12u;O-IE5iIBqWM-d&*V%>$axjEg37 zK1)(xoM*hw{JH>*@dfg)8_)v`=k^oO<86S%l|Vkm;jb*o*?UPb8+Y{?3e@G4$v%4} z`Nv|CBIy#)tFb_%JAsv^QedB_0ln!2yxLkxv8@c!1JoK-fo9-5J6AK=In!jf0g}r4 zktWZNGkL+uczjFTa|6{mA5ZFG)48t|T45)MR>sB&&Z+k`KWf!1v<; zt$^kZ#ocqzWT*2$zqx=~t0~amPM}!*a^-=j*aH|%1+~aEV0AkJtCRz*Wdx94RV9^S zxKKS^a7+4{oQ7L8|NMMCNiyZP$<%6+N}(8&ts9x#w_Z{$ddy_DE@`Eq4)TC`zmgP- z9s|QI#~K|5=CcdP;d_z_8*9=7EpSg)OgYCcu<1SfT0NAiN-1ny?)na!g zg)4695qoip&;}sIlGVUQUCf{VwfP4imQDgT8qIIf_P{2DfpWEpq;P#=a_N3xlOqA{ zzn4@zPXb$vtDLow6s2uV9vT5`O%9N|CxER*KhUwFB&{(}lGi#8Y$JY)1&@$cJcunU zFOaP*B$Y+@2KWbRu9qaO)fw3Xl)OYqTHC|qNPl2khNBPPAV~$jz7@A?!v(;$F(5y_ zN-9zKMetm?uve1SId8HhzP}Y;;I;9aNNe-tH7Sqa@H*odPm#C>Wn}K>ZypDN5Xd0yXhVOzj8-PN1t-n?QjSG{JQa zf^E!9V4Bq%Z0~LcHl-&Ntosu+T^+Ey-U>hHKJpo`RnbuD;!;os&xbO8zCh>PhBE8W zm^8m{(&L%QAz4taaAP21CX`=Z1?c!ZsMKX5@bNpK(zi)KqDF#aS~+0l?|>tIVYap- zRNkiou|D#Ks?JA%S)4gfk2b33PFt;=wxhuLybjfdYc zK7bmD_=kmxL5)=WhJ`vpO_WaHu~||$H-efTFF}caX7c7qN#0?jq#(PXR!!UiJ?x+! z`2pPfxyip>!8wlt{9Obt?k|Av&4NZ}U!ox^4Naz0w1VPR2b#>Q4eIPbXz>{rl+%1@ zoqh+PQEzCS840A-CzBUCf%|wgoh38CJpv6-F&k*RHxbmbQ=#4JL;#p0sV;Cg`Rjlr z{rgE$6#oXE)lg43bOX=R|4`cPL|Xr$q!CfGUb<@nw0FU6wQDDIsJa74 zfX zGEMqbH+jB}B%4(fI-NNR%FS8O=^6$iq$PB^i@W0MCg}8hHYzN9->iJ4wG^zKl>|_d z8-e7_Ka%_>krdlHKxaA~b-*j=Jn|fn4I@qd43kt|^o7o&J%C)f2c1`=3oqqhvT1SX z68;wG6%D$qjt52k3|(S^(NLX-F1MP4vidi8xuUDj9cl%ywrzl_d%&yx2=w!Vp_^+z z5UwuJ?Rx$l--7Pp=o{R=fp`Br5Cv9&_wl7bj@W?r6_nA{f0%6O2tDS71L0vN564K- z)s0PN1WK}gBPIFhNzmgOs;yA#O7Q992h^}Z;u{!8z$YL9m_r@#nbQ%NTMqban}A}o zIrI!)3*=i#=(%_m-p~ed8g5Jrv8#)9?^5-K>{vH9nudc%oY&G=G842W$3;32hjQao6T<~ps z09e>>@NE|lEW9T8?w*e_dl2|Od<)9lZs7X{&o}&#4>r5E%GM)h2GvB{Rlq0in(bUqCF#AEo??0!Bm2$^tjSzSk@*Aq4$H=Ppda)NML|2}B=qCq zsFHu0oP9x3O)QPHhJ*TYA@uXYV556g=;xaV^oA4k`*j}3_g<2`M+WpSmJR&OIv8jj z2&8H*_&H!KIA9R?W#j%oZ43T2{XtyY4Z~KWC2jE>Msy6qFufX#2t^%mZ6X91321Pd zK|sE}Py%37srmqGQ(=^~LI{xbx-e=FM!xeSVRWnm@UGKfJQ@pS{Z0tNRZ1R?g-N?2 zfv4**=|08^_+_R>tpYl1yUB@lVEQwZTkAh#KI*U_nYr zwC#^zS&19K-`9eO@i{_?;|KQ|KAL{ea@Z^OovWdJQbB-!5t z*fjVPsDo@^b4m2N<2qSkb96bNrx(JOF@Zon+=uO(gMbD$fgPb8@CHi2t_El@5|d%~ z_m!aZOn^O8Hv_5EANGDI3Z(Y|*tc^D&<6cr-xD-s&$A)M8^uM_Ww3uc2D13?Kx+)^ z=XHdGcT#}+uYyBac*CU?NjAW`4h}os0rGkx9I1iZYi>(8+F>d1R^1>r^#)E=Avj(= z3y4=N9B;G(l%>Dnq}^;_ADY9->ZtuE4}g<3a6Y~Vz?s4Zh!LmZoP#sK>phaZ#5#y` zMu|W7JjC_GRX-?5QvLZzQdsvNfJ-*+KrhaPORj04%zOsP;|l_<*cnp1KY>_z6RsuI z1opQn+$fBOs@GRYOa26~DGMIX%K}z=D?D~VRlNETJl^>P6!8z9WUfa8)CQi1I0LyK z1X-m}>lK~~S?y5PM@)kk4d;NE`4V1ld5imhP%(HN;{YVn89ww&1f>Im&!zAKLc7D~ zpi@Be*27oauj+;0kX=0(lzx@qTU%Qoe%Im0%`QL(JcXYp4uI(M3;uPsLrWMh$)A@e zit6|UmqvM`r)=~rjZhx9YL90hm?AYv44#Rq|9)< zfy!G+*`1$2X?=r~n~ymon>(c3sccXRd?e*NTm5kER8nDpJ%|d;%oRMjLma|TF%LUWY6rgn)nPcP(`N>GXmA`T+(=2XCS#Lq;U>@ zz~gY@=5Z9r%JIZ4wFE$ucBDxOM^Jy(CrxgmUtisYG<}TUBsq_?${7pl3qsuQpvwIn zK|ESD2RgVO@mRGPL-^IC?XM(UQ^QERpOXM){YO04=K`5NlXM)9Q)t_lSUY|S0CMaK z=@cD+;nxn*=}kG{J*tq-qdx-KltQ}tpF*vcOuUMwqgR|ry4S-%VvHN<-n$Ga*FA}M zWsGv)ogh7by+tMTQBo=Yj`X_K1yyrL(&tGf+$FzB-=Mpo{!1i%*I{TToJs!`>wsQn zRx;>&KTu9pA$~pZ4)<*${yAtES^ybtgE!(BL`J@i0_Yz?MsGg?{AC0g(;VmW9Vg=t z9|M?po=j?r-)K^6G9xe@)Wl9EU-dTmX*ZcM85D7&|_+iWmY)vDJZBl_Q7)F*@{L!`xvg9}Zzw%bHJRScY4wA?fqktc2L^id5 z47C1MvT3R-s8%wKY&!J|SEGe&pN@ICZ$V_oF%>`fHQBu{4Pd(?*)t*$h(3YrOA7%u z{S%3D843JL0TKg8f%f($F*di+E-WW8H64NWoJaN-M1$jVf$X1-fycGdWdCf;cJwSu z4mMv3Oz|S8+91p3lG7{?ti084a<&l~2JZ&s>{J!h0;kB?gjMMGzmYhH9>9y2Ao24s zAJoL2TpnVN8H+j8&=a)dgVBN^)~~aoqnEPmxZCBTuq|FLLXHrCxpIuHe zAQ|A?5OQzL15h(5xfh4xvBr0k?H-ZLh8`IE#gYdNFn7?c8+qtc8{o(!@^E%JfSrfP zrbHe`^oE?S2L)FnWE7JrQS=zrwb`gu4qapZU1G$5BJ zkl!DU0zZ9${GHnX^8q6z#rB#CREWTcC`$org`)s*J@EqmVc|0h`D+Vs%1Ke=bxols zF{pg=RN-4o;-c%UXa~B2`scV(C=?AA)KLl#9tW(-cct)8M_^-HDRwAO`H>K%*j5ir zTuxPrI}ZQ?ZIx2F9Y9>Ttdt!*9(a7BQUS#Y{BEXLtIXMjL+_gf04tab*#bY|U z%|hEH#d%kyO*iz0?Yk@O8{iv1Nl?0;#pv~8mg2=Qc-&V&k}SDka$O}!S}0BF7KQ}| z^||7M(KL9kP<(=Qtb~kLdXIYyyt#+cd$tSkqZ5?gyHSiDvL-9OE3w2fV!6_1QghT* ziZW=e2S&X&C57{F#V@5cYQB2Pu<7{v(u06ROjUwhJAj%Us07W%deGjvixWq2q$l4RV#N8S2`-I*1iBXH$hp~!`2EU;Jvb8 zG^Sgv`;?9LS)gnhqip<+@_FoGWz!9BVD;-LTYB2#e;B814ebH2e4w(OrlVkapzL^c zAILsOWoK{*fc+~;adD;+g)*Bp8>j5wJr8Kf2<3nyPSLz`%Ax$A)5b$N{LTf$!vN)o zt<@Rmg{MkvO}wMoAC%+ar~z${D95)&;A&s3ocL54=(u*u$-n443PvcWbo~41DCJyf zOw|RBSK^F7G}*NzdC}4)tIRRkwvVKkm7<*Qi57NtloIcmg;U&1lH3VZ;;pBBK+xVw zds8HqNO#p@8Qxdy`W8D5oxm>Fxz{yQYii-X|++Rs?YmDwVL`gX@5mXyhxi+Lc zuunTA`TI=eI(}nCJ*3>A)q(XKq@-5%#noR&xe2#{6zQhi(a_E~2P^5PQH(HnRZ^ZdO$TxH zvGOb(6PNq8D9;a4fb~Slia!Qad{0-?ZBH7%7?c*L9IJn`Qp+BlnQy0 ztj`CllAF>C!{TAeZ!eVHv#Tio+Lr@(+*zfQUjTVjNM&IdqFsDtGOL-Y-3$TsO(C`L z@QDD+qEtIfN|A5J)uMNcV{qxJ7IQ$ieWI&cyhMM1jmy>I2eD4{FJ3Kku{emiL@jsf z1@IE)_>vyO2LVee7vqU92<<9 zv5eXK2tRk$S2v8{iF1eXh2Q{txwi zu-dA;GwOv1wN>~X^z&`i*29_u8N}7rfi766daSzV>xNA$RFAO{!0L=r+x_qb{_VQz z`SAvjqMId^c{kMd?d)*>PhP9GPhSCKy}R0B#Q>la@2VXyxPvk)NbS-q5^Z~s+QS~D zSnO7{*Yx>7`nFPg9V!mWC0mmzo+fX(n0!^$Xy5+dwR6q7E;BYiM_Ob)+pSpxiR*n5+T7Yg|#s7s3>5$&qT%lurN|4UyJQKocsd zlfHcdK0v55hvmEBV(RP?3xQ-0l@u!@)YP zqlWrmQOaSiy66@z;$}P4CDqVi&F`fy@4gX~+m7mTs{_EyGHT>y48xDrRU;oY0iI@2 z*I1X}e>hh_T|XfkAikrzX+(8^pjqnX0rP=+3^qAnrOExy>ejCAK>DvzcenTsYGDU; zZwuUpnTOP<=((69c31aHF>0v$&!Xu)n64hGfr`qpntG@in)HgRCF%9u>Y?W?0QN_z z*2B|sF|DSkN1D$D@(G)Y50x%wo{KS@&m;^Ks~lK5qPm^^%T~MA$^Y| zKUhvZ*9oQ8k{Rl`G#}t44fV3yEg%;+tCz3t!^+BQHDwL1nN^{Z%B{(gwBZJmjfYvy zgP5z5{9mqm{ksZcS5-B&aW1;o*J^6tyFkj0Rd01`1GMNbH6sH92+v~by~eSiWS&#+ z#g+oPsW2gK0UC}| zKm1z?FrcOSv63_JUoq-O7mOYM&XuGinEKy}vH(TqO42U{)Gy;ON7SH%nk}$e{$ih+ zeX%I+|7HEv?8oTkYFAWqT^Oj(52@e$ar;GdP`|f91rwDiNmu=m6!pic-}~+Z@~n^g z;{}FnJ^!k|?ia*vhdb(@Q4HweV``p@3dG~O$;4pwUwAEGrFv7vc^-%zzo^8m-sw06rxsj;;2BYRL^G3pi&0>u9>ZHlQH zDAI&B?PCo<@AZPVSco6=b~kNZBM?_}CF&WNg7x`Fw7vBZ@FKfthq7p3&Mu%G#$ZGg z+lzLXj{+p{A?>gn<+j~E+Nl68M%xTY9(b8{s)yURPzdcbH515xnxvT7l6GzN5@_2( zCjDMgFDr=y7FnHktBP5xp_i#QcLJqJbL!Kk6ezxB=)iYDShw3nhZaUX6}6iB6U>U& z=t=!Mj|H;kH}wz1=9GnY)c+O+ogMqok<0DSju1NXd^#x3A;?@%T$<1TMa9O3B9h|# z4mzq!7UmiLP;20=bl}v7js*o=~i^?+ONQ0^)Q*UfsWm^8AydIbbS7esz>OA z*=ZnxXV3``AA@?_&17mcol@Tx*w>+SS{-z|kt6B!LrFl+zNOPo)djLHfzImX3rgDp zbaoKR@#^KQ=7CEBoqIDI=$F}a?)OQ+`ZTAZJy2+z^D&w6oreDR99wb*)39RCL4Dtf zhT%7)!SCs!8O}iHExKqA=6dbw(8W4tPF8QGi^JyvEB%8mK8XE6868b#myuKp+S8@i zi-7Xf`hza7au9Pl_34V06M+2cNmtIpHSlUSjp!PQDVGqsieNhJx-VT-IuGysEM0vD zE0@!2(6#N)0*1#XJs+D~V4l-eXXH5>(pHjJaW>f?Nm9%pCankim>kam{>THM%ORK{oiXK-OvUX-R0wS!?r}+J!|R4 zAQUP~Mw+~Fg>IR21#Nz3x+Mei4VIO3>%_-c|E;)`Zu>X@1B?QaYL(@Z{KH|oqjqzk zMoO}} zA#|T50hH56Y4p->*nnh*{DJD(wJx* z`iy;a|2Xt&gKN-(^Lk5ZwVe0+w|)In(4 zcWUCP~uqy-lWFmt^&4Nbi*GWzXeDfseXm&oeF3dKA*AD^dZ}U>a}V5 zOmr@#qiIIZ5%%S)GcmVnSK~hoYNz!yHl?VJ^h@{vygJw?f0aE-Ry?-QK+_vppFVk%Z>YY}=j(4_dHx`M<&_NV z?F&gBwU53Eoe0XcGm^~O@GyPj)eMw>-Raw+@u2wirEeQ0fVkvC-;TyVq~#^~X9r2K z?FW75jcR!5XZqf)4LT(kNiw-LeLo$CJ~xtnC=m=$>z5>b7eGI}!tI;5#pJ7S+GUZ?%u=0c{XSxB9S4_3VFTzSQwKJyO;^G*?0t6X5jj{1bpfvx&MAKNz zC4XRg`vKVhQLh-&k2nHpx0e}nv#~(1SdvyO&I&A|K!*KeHv9na&K*tuTqY^PGnnm$ z9l*PtV0MuVYe9L;zGzWg^~ah0mvKP;c4tMs&jLL+gcTPrf$4QvaZi6xeif5cEB9w5 zPFT@fjo89UZpVtm-4s^pCVD|{3o8?whC${>R-q00_X8fRVts5f$v)01e#LB8@Ml)Z zzB#baU(8`!b5I6OVUAd~gCQoxffLNh_AjVToml1dzCc&ZWL3-h0L~Swijqr2$Fu5t zgYbQ#SPg4RBB%vlvRYwLz!tf&S}&Zijp{qA*WC}p?IW!IA6$&SYnk&VbVlW#GnYs- zFa`WsqtGs>^+q5uWIJ4(HM_nPXzKwcdqlG4T|R+`bYacsVPb>bV(zw1KyFuN?oZKj zs)JeEuc)YacV=x@0f)}!8EbbPN8m#y^K^9sSm($*=M(~^a60R79(_Q-f2`v@|{R@U7Y3k8*4v+mm(U_9@~yyx@_u^_EwVDmj%mI0z37={D1KRY^;ZY-ph@R9W@ld{V5y!EfAC)li0XBsN-k;W#cb& z!NnQECcbh6*1(Ykb;bYq)rUrE>=`!wBks@ar)XV1o1FPp zlFTS2sg^h>DK0)>)@EBkfk|xc4Q#D?-IImn&BpEM&lbL~4`O6}7FwkTkPbsxXufx> zMS?&V zw-kGSh`T#mHWO9v3TL+b&rS@%Il4NEbNk!b0RO?txzKD?I-E>J2vDajB1-7YvNmNi5 z*rs57-!iq?mJ9wsVvn&cFJf^m#j&mJ(W`d+!FG&6-LXD`S$Em>0MU36+kIsfrc|+C zun;aXVZ-)3!$rAgs-&`^grr)i6WjB}4%Ff+S=6EgfD8R3l?%@$srFZr$B&Q{8`DhQ zy2GL_&I6KAokd-K0cz3bEP8oS%nzKlvgrIq0jJiI;N7^M~S?{}~7;}gn zt%kiKMN6`ytucCj>?z6KSzEDKAKbU@TiNj@sO4%8W5>PoSLt?kd@SzY*-O~*J-<+# z&XeT!-P!TI4R9_$vy+Qa|Cd@ODH3DY*@>rs`E6!#E&YJ*y3NkJmBW#I&dzrV#u{=f zcDcwF5Yx)AD;8f&Y=*EaJ5Z=tYpi3*m(oCN@ntEGc47xZQFe6<2BU=*uxowh;vFwx z*XtHVQ~7`1ULnuv-}SpWO+h5b491C$|m+25hn(9g%PytP@wv=K`DTeD=-XNlS@j_*-69Mj4;>FJ5oI9T7W$Wf(&E~SCQYo01{ert- z#wuQ}8Ri2bV|h7jp@N^6c||Xb1+Smsl_+Mt|BK?4276#V;5K(Sfv?A}<4(144IB#Q z)%^d%md!D|)|zcVBG;STRFu~X!W_`8J-oiPC+hW4C%MbuJb)98xa({5(>Gr5M%7U` zIEM1ZU=MUp7w*(ST=`G9lx(ak|@9LC)*pn-8*B}vXzlvJ)z zNwr`*N#;_NyFW$CS+px}SCwJ-?#`|4DGHAv@w~kk)_7N*=k0Hw1LcLv+dsfaXX!iM zp>sTtStb<{$~(G!0(}AvU%5HIAW;_xmRhl6A3MN zx2rvX&f3nqd(FpGO&8vM%s~JrAMQ;tyLLyha_?A7v6Sr1eWuI@vULXUc^IQv2V36r z)_j0|!zAh5mPiaL^&XPyi+J8^i3Z~C0N(3TNz@HhB;)T(tx#a;{zXM1FKhx4}6IuQn(%;)S?|aomYHNU<7Kv<$Tc62vEL0 zmQ>T1@jCWqlfln=hMdfV4r`2c-uaYb1s9Nt3~+ z`K+Lw*fDXC2RlCoQDY6Chr0=KZu5}3<55M2@P#Ya1N#`wLziK;{NV>4x}hR~Qjdp~ z#QuM!(;gnS$N@x??>s-3!$K1HvdH{5wC5|1VBnE9m9K1(19Xlq$)@j;qNzLISQJON?m52E9k=O# zSA1))DCB%*IIF5-#tq0G~fGFn=@SV@vV2F62`QC(57A+^S=II&b@PUQT{PD7`I7N;4(-eF{a)$$d z7Kaw`Pb`1-2QA?5ZMgpYCgQU_jd)fDW-9My@R#)xL5X{7@=BZ}b$ckuiw-sES-|A? zS0?kmOA42M{BmLL5-r#RDF(MkO^AFwX0P{b{Kj)kVw!m73 zf6l{tf7)>V-(H+^s~68{gJJZ4{rR`zM?lH;;osI_U(f)R|2mtFg+>oa-ZhE;w!x;f zv~z-NjsV#DQK&Uu05mTk=sXLE`J)88p6}&$3sDF4!p!kPbUy%Wno-B}BoA1%bRw7KJvWnw?xo6n^>$ zlxtN?zIiOkTz-inMLq#Q_;0EQ<6GWxewSlkPE2`wb&&Rf+S|F~SA$Fqr_bn(qoJEaLe4fzx zk*G->tmu4Zh&pd?iILLq;iAjgAmG)m zimpFufEuQYZjRPI81s3GZZQRc_C73n90&%{u)XN@(GkR)aM9Os5(bO4MZY_)z#4rJ z{nM8Md|4@mT;Gi8`GaD}uk`>$?~9?+BQc(L6e9|60_AMB7|}8W&;7*6o)}wJ*d#{& zum|CmBm#WV5`wJ=7=}}`C02~KX5*asa4|Zce}74eEhoenykj^TCP|9-H+igxB(1sA zWQ)~e%x7#euRU0dy>b8*%|kIZIUYoGXEDAZCNQ5p7voRjqCOR5GIgq$nD`QlPBq1( z#NQZiY!udMBXM<>JT9hPc?^W?6Vp;`0p=p_1Y?=(xR{PD`fTxEG3(1?Ox4^LGRCVk z&E&m)V*aensOP=J{Lk2o((jrG?L7y}>x)HLA_k?Snu$R1AbzDS_U4f~Vo+5HVY2der zSoNkBZbPdi_fo~0;xB-gs4CXqI{-Xvh1k&T9YD_vv1tTa#;Jove&nOl?;>g|MJIzTq$iUXIkfc2{(4vswv1O@XUF$ZYrAxMA*(57fF@T z%{rA3mvl5dgR7YgvXP{__aTo01htW*drFJTx;H4vyF~IP$N5ih*(L4%In#miLmhG!0o*KUD8lLN);)|=51HWRP&Fo$#afOvBl z8$!g;AOi=8 zZ>JfspRwY5V{}puxaF~rqp|h3C35FqRdN8S&1!}A*VEot(R)tY?BYWX{;NjZa#g}_{LJ$ zdR-ZlKvfZQzqLwSw074Nyd$)e6!p>Pg;Vp^K`Xzm3-ET_u!Uckqg^vK8X&XmUyT2q_IFI z*3)`t;r?H8(PU~VZD8d)7$p}*;v$`6rP>gF2-KyvCQ}D$er?)g*}aOSQm&BZw<#7Q zp+lPA7k`Wox@rD%UxV_ktLC5Ngy)Ica6IQf8)|`n(FZKFXk!=P(36td-7IG^Nd&YKY^SfZDBb{W@iD?Uppb0;isV%64Hb0VR zVFeEW86ByGo%oKczoN-|san{p=D^qW(Zc?lkNrP6ljH&SIH`sG^#I<@4T*PruZI?1 zaxsu)XS79CQI0>z)fT-)Z}A5%4wb*qSJ~t|>oxn~g>guMQ9QGZ==$0lo z8QSR`SObb}tDXM44vS7FB-zJ_lDwV2$#*}sbJoRIK)L6rokNN%>$P~r4MW)ojzFL-Z4{XUSs%5opfND2U zdsF`>C_`syZ(5ZA*fK+V6Nnj+IoGte^>aWytD8)Z*WTt=L>#I1ZomK#`!cQCyQ7#$ z{Nkd$dpQ!5!jHB8+PYxAzk>F~iredMMeWPcN&xK|YhV700~Lm8Uk8){u%E1D*F{tN zw7r(SqX+i!6D?;L`hW^YwVxi?8D;ZP%d3C_rsR2D9k2>(LL+q6FBqup2a~N@NU~ZR+AC@#V}x}_Wr{pI(%JE2j@50(-vDxSswC_G zS1&qwC8p<{^^z|qfm$ikq+=VsOmmEYT7J{Z_PLDPccNa_ie7K#2)%p{PvGu9^okx1 z*#CQXtX}akW-z8s&?~*&46I0d-LX&w;J0h(jyEvq+~A`-4afww{u$lr#YP~?E4@lk z8+;rmSg+L$pPrubs%=jg6GP!_lEY4XQz-SuS_z_YEoTbu`GuPl0#A>%>p{h~Kn zGXTZrXT6yR)(Ohp)0+*%uzYbXz1aqi!puF?n%Kulj&6%!DjIr4Ov?hijpuKF}4nr7=by>`Va`_K~D_mr3%#Qu@$gyYRVzqWXwm znCT$C`baF{Druwik+aY=r(M=ZcMe5~_Exu!EgymTzLolT%UuwgGW7}Vb^$%OU7xrz z7~tzAJ?PGSP#pW{Gwa*|n!8O8t`Z6O;D>Ily9tw2C3NedouK><)#nvVz#w&&9TxFdE(RM$g?*#jBlXw^d>H9(7XP!B)R0afw> zlZOK&X+~{H*7ubpAKKPrK&-y#r7yswY5MY6je&-C)gw2)1Kuf5U*&%T)XXXRnnEv7 zyndGyo7U^=&i27N!4G}?fTN(=srveK7f>r&*T@6W_?*6R`wgINTIyTtU{*WkhQ8x3 zN~|)I^_^ILqYlUQT_{YHO11Re3vo!#pwDOnz*q?}_dLr0jqCes4$g z?{_4H8`TeM6VVq`&<_vV0&Lhc{m8Ng`1-7A`q5~dL*J45(eJ45`>oYuH~qo(+Rpkh z6;mYpKIkW1QLz*&sh=K(ma<1@No8GJ1TM?Vv2~C`s?tm9%odqw;`{&6X5<{_63Qv$58DTaW*;3v)a| zQrt__FU)uWY%A!+|v)rC-V)+DeX+!mX`- z`6p&n;@9g}YB^)=sGXkd7mGRJ{(ADVO!WUwJM^0#u4sZk>$jU?=7YY_@09IQ;Q?V%zlW{F=r2hky-@%*pe-|5I!-J~<*4j8XkBI?wF}F+qGYI|x z4L@g)%LhR5&ob1}dDw27YiQf&0knQ*@{DTe<1xV)SIsa+#Nb+aY1nLQ543!)VOv1K z2NmBN1$Sdxezmejp+hJ(H@Q}L;$=`HS)>R zCL3ki^aCZKi&17Kn%uTujIv49fwT!Q%KgUO(0zsBFa$@e=t0A&Cq_m|H4LZWj_Btb z8BUw+F{ispl0NV@s@Hi5YSm*#tzW?)nw&6d?~4UCyPr{KPa6Jxh*97FHIVbRR-@te z5}<^}8m`@wFxT_lq<^a6I^hR44BR(d-=hHv{bRV@F9&=<3!`b2HwGB{jponO@P-;0 zE&JaA5>(k}RYb$~ya7h5_L!P!_|5Pb76GipO{2|re^7s{HQMa@iVrg98f{Kv>{r$k ziE%_*>oTKlxema(T`;^VM1i7~G`fd;0-?+`dOgHBk6CT>y`K(BSudlXZEb+tsYX9+ zqosQu82#?w0sg1C(Z5kJC|fQY14kzU(7(o@S*L(aay5qHPdbSDN01YcFO8vzc4)A4 zNfB%qeyB45pMNm?nqriCIo@Q(Y)Sgyfh2Rzl;rz;48O_gSh-ke(%sgiZ%M;1uQulQ z#u@$|j-Y%GlT__D8vbGUYbkek8U7pb=U1{f8~!o)IXTXf)Yr%GKkyan|K5EJ{}dd` zv>@QwTgEF!iP$a8H?gE;(26iEJ_%J1}NQF9FF~g3FnO^ll*}3sm6-) z381VvZmdFY$^L6>^~!A$1)zb2nPH)7K9NynyO z#Qb`WnbJbW!D5dwuODd~s)-}CM@iC$SFJ|k z^vPJWO*N8A`2o1qGLok5z?yGSBPk6n;J#YMCAaQ)eUXujO=RHq)=2J-!DDciB#F9X zBnL4pL_U+G^~)G3Rn@NOw`utrnM5p`4N7nFnm1tC1106W!|; zZzB&BmMBLFnb$8XrFR13epIe0us8HDXWWznMirnRVC5u8Ioh z{v{*3IhyQf!}u1BZx}Sk`1YX`@S)uJ_6b*cdX6Lym~8wwRU4FE{f%D^zd>O;jX%@f zF#f+e(fGR%mC4hR7O0YkqHwDP2BWunf7e1@F9&|Mj73d5g+Ed;#iBmLC!JScv8b=n zDK-3SVH;zC+Jsp6fXhH#-dgzPx}fO0EZS)dVlP*-=+p4V^M+WA2`FU#U6fRIg-KF( zYd&%R`~I{Td+jlcwa#Lk#FXfPf0hFCL$H@)u*IfVZw$wU#U>^flt0BJvD4gQ8$AtE zG!rD1a>FbInFE&THd_k*RzU2UWw9G|9!+aiOR+*pxP9A65~q8TJfnoAY^Ceix^c@= zb|U`%qZ-!)50DS-EKWyP0rzybR536=S2NX8r3ps8)9fr&-eBUO!b(fk9oRK@;GCs~ z!yphzxhB)>EHxRXUMA&Q>X$qKFuj7swaZT6rR*%O`|yhuUu1Djt%~l~)zYMyJ3vOd zBtLk~($u;Eg~owQON$|BcpQINTJ8G`s{JR6M~M(nAGETx!y6!bA6PtnZGnI4X6ZQj zCW!R!}@B>p3n-8&)dM#ArHmk7d+KRL!Z2|JT-+z{hmG@1MKO zOp?1rR*~(d#8R>#A%q}7YA3PQI!PunBr`EdY$a2BwJ1W5UDQsfmUe2beQ8T;*D5Ww ztEE*{wf&!aFVTK~e?F7<-ZST(^BCa-nqgW2}zv7u@FR@M@z4 z_s0<1?bn6z5zrem&j}M`D5~HrVS?{Q!e6N=Oq>SKYN05+w-jdG_ZQ*)ngA$O@9EUC z?^EIZsVUH`ZZBb4+pFOAy25l0ElOA*Oz+wSXtka&)6|`id4q+SU!j3v&2-Ag*EDF$ z@j5k4Ju1wK#@zRMPbiy+pi-Aw!kqJO;{}FA!otK-FyFVrBJTvey17SKw0b5~`%|5o z9UlsdHbPOk1CoTr!d=2fw-lDHK@e)@Lt*LRU6``~p?oKh(e(bps?XaIrsiy+Vs0Hs z$8KT6?D~YU&l5JiyOQwfBMmy35k4)fj;+__!e*f}VW+Ud<`TpUlNJb@=O073ElSvY zFNIj zJMBm0V}o#{=S0FhTPl1L;#jiPfoP!wcd16gy!qRS4JbtR}4y<86;emVhO2YniO!zAiZu;1A;qN~oR%gG{ zDMJn$bki1{^84EgPkcwfb-E)wIfL^;g8mSm{5*%4vjz)KJ>ZRs3Bt3xZxBn=bm7Gi zH1OzIkvRS&?A!{Gty4lsS*plZj3<_zOGNgmgP6}u7TsJ4F4Eo<`5AD(PaYKc1#JlX z^#h%9kH+iN!d5kC<0S^2)?PFvf+ub_7EQ%>39~XpG-KBs`|eoL>v`W!!(uVzI}QJrtr1g|5r|r4iK(005i&4QO#2Kgp^GPpZ3k_`=8-94 zy5A&hGAS3+XTx`+USg-Fg?M+vDR%mx4q;hsvD*kt!O1mZMz{~*_srMudY*enn|a81FJ#HhH4b{|L9F4;p9=BnuBla)Uu?|pcg!#8wvB%Suq2L37W=~ zi4nH9} zcf+WCuvRQ;I}uV6AQtt)=5Wi>2(jp90AWH#i$%|{)v9g-v6!wTmL_e*;Z4xvC)LH_ zeNMq`z9$aH8m_tJcyZLKxrA-F0TgrXZYPdzp91EbE{^@El&~$s#L~mqGuC~I=-zq+ zJ19GfKzJ3tX)JP<(F-5YMcq;(LGd2(xb&rt*<(rk~o=Iu;Ep4Af9*1;-!m0NCc?Z|C}F@3L7 zOa05B4+uY?zd<)};yUvRVv4?G(57kP`Z*Jc`8pBT-?>Bhh<0MdUmUjCv==wjfET>) zyg{$LEpF@!7i)n_{G=~HYMmnnO&KqKz6MNQk*-r##oT?7=fGaCwW6nb0@nE&iJtar z2~%&rPE7+##I2#N37@*%pxyrvx7lRE_8%>7_rvnrt&8H$dkKgU5ph=m;CBy`_|?o- z(B-Sey!>oR&?v&{5*SuSx+@Pm71Az@>BV{@l2@oFX2XkweVR ze&UhxSVBmK_zez;J2p7*NfrvjOFBH#_T{!PU z70<1}%4Oas;*WW-1)K83i?^WpF830DPKGpmKS})cNhaZb&JnM49zx7lFNs&Lz@9(- zM7&{_5Lhf1Z#6@(DgT0a>t`^Vxk9|%#6|dVy~VrjFfygh#JdH{5u)8HK1lQ?+%u2( zOn4WYOTw;8MA}RkzuzV1_BQzWZ%Ay78pQlCPGa{$gSGkCpxJ2>*SQV$cDyNZ3sw_m z@GOa|sD@zjN}ci($LrMGp}oXsUPKVPon-O?v~x6qb*4oSgh9mLdYh7^3~AtBSR>XfT}U#hwaM&_>=>5WbR8lSDxsi}Wmsrvdk zzN-G*Uju+350nyW&qiP|K}tBEfVCU5 z)aWNLlYe_DX>DUHyVa7C*7*?gmoZYSkS*A>woQ5~VG`b;+9_q4;j(QoNj*w|jCSlX z=$Ro>?{Q#WJ1_Od+q7o?EXm!+r<9mmxTUwToJOY1(Yss7>pt9%@*&)t???m300UO{m$E9bKG1iPlym4agkaW5Ia+vbn=09CKveBR zB-?MDiMhcnX>dm{^GDf|y=@u}h?(V`Tp~z62CTBL1m=j%f2T3n!eJ=W)4D3 zZt0yp`-o-xH`16GC6?ZAD)q>3*Z&j zASz4_| z6F#n$w601K;s5wns%V5)FE#~A6%#O0ZNHN?E&;xOlrC*qSxm_CrP6j($hP`M+CxSY z{?r(qn)BY4_Vu|-%pcB|_DA4AvOVLZ{X=)*{hz>0Y5)8Kh*mY0_8-RE>=|bbI;UKx z%&sXqWpj4x)a+-H4y{92ZP#k)>+=r@=Qlw5W?UfN>+|T8uenM3J_O>|zPofH1DbKr zb?L+`9&^1*Ive*K%x98*m>Ua6CtEsS#jOH7c9AZ4z9QzP?WC&{p#kr&kgoa0Ap#R3 z-C6{$t@f*Qy8`g~i$LkOd5GXf-}Abo$Mtk- z&NG7+BUU_Bdhkg)F+IH?J*=HV_*VZIw0oTN$ej)y{z<0vXc}~WW4HA8KR~S|-KEFc zI{$=1>G5qSmL8u-Pih>;66`eT$;WAk>E4lEgp9?S&_^;?%OBRWn#||yBK$iGW&R56 zd(xM(c^mLTp(LBNolmFt$`&qwFrgK)cj-*RnNG{z?;-rY73VMc1iVj7zwMEQ`{>EG z%d&5iSe)^ARj$GVekVwBz=;gPT^J(=d1;Z-yK+#{*Mz+2DF;hXsizOf)k2?ReQ>KB z_5jjwJXMZ72t~GZwp@K3mf56Uat*Zy4jQn?H3OFr{=_`F&Y;J{^3HE^o%wK7rk!)k z@e4``v%^PjK(I2o%_%qhV=&{uu}E+=}Y;h3Fo<-|_#lv;c)H`Xh5$&K6n zO4yh~a^pOV;5l!(c@&;M^FU5%4s`tCA9Bio3GfGe<&a3A!QGs75|;|rh&y^L=uceh+Z z7_Y6|eFzkfuwL%I6ujXs?`&LbD3!Z^h0y8a<#JE$i}IOr&&SIVLfs(ujvhp~IwR!X zsaFX;r?Y zUMvrsjFr);g|hV{M9YV^m2*<7WBvcDd^y()96x`#oLi+jmeKmlL(c*Ub($jQU%?u# zl}IXOuYYlr=L8oS0Lf4 ze<_cjH3JIgfk7KvNWe%6C|bFRE{!((FZyjNb80sp`;MP73h{=uSd@}{}k`M-N-;R5vXJ=p`U zc2=24NiKKq+k&J35gg`Fkru4VF$h|Yu`=_T(Rc9*bbe|hJ(?Xh5KlJ^b6 zLc_k<^1cb0_<76wZoG%*AIkfGZB3Z0ZSo<@V?sO&Q8JkdnjL6k1+f6rSioU z9}~XpiG1Vy6=I2OAm6Nspwn=ReE;|syoS?F{=FuUlXydZko*>qPG|Yim)!|F^Jn?_ z;LU_YKa*cXLOS+$mEA9{R3oN7_Y_kfygrwFLhF6lpdbkwrcRT`^dv{Kj__ zMNjXf`1gX!-r=tV6hSjqTVT*;t(BnQ(}WAYqJ#~B)${dIA_@>$tsksJV5bH7G)k%7 z@E^jD+OAX|+J}&jTDz5chXN6$x~;^lf`a&BlM-_PE>hS-C9dw9#MJjAC2kaaKUcI8 z&mD(zx>AYXbp%Vb$CUcaDa7}_R~q_(XO@Uc!|(kGnR7x(Nc@@bCYO>h<{~yUzo9f* zgCSo(NT+OMl#(=L7q)o+s5ChkLD;+QFO;-CL!m%E&?*0ZiIO%-hRW@xymd_j7?X9% z_y1dIkCO{bcf53J-Y`pPzb%EBx7AZRmRJ!h?yYqFs~KVUmna!GZxZt4bEQWxq$4U> z>AMj&rPh8W>+9WwxstAA{c{93V4F@Y?!7*WE%Gt^@|}vkH_m?8bX2hq2dWK;SB8#- zogdIeaWL3Q{lf^QuqPaipzg{Dmx3kOTFU4sG_+@_GP?7hggsQGyz}4+4lJs#Q}*;y zWenDSc<;r^*l!96d25DJ+M<{+bB`$Fo`YF?ZdJy&ErtDWQKF2eFe>wxD--%>5>wn> zWpbpQa4|!a$=ysaPJzml{6~a5`a^l|X$AKGCMX~H1Bm?gu`-R+!OBPtWf~(9j_apP z>tVtswPwn68bQeQ>dFj%=#A$NWyYFBL{^R~Gaf=3YIRa(zcGf8nin*R_5Xw(M#8&Q z47zQwPMLw98}y5+I%RjK>6E)YL#O;t&kcJ2kWNjkk1~5g5#svw3|bQv;oh?EccpCW zRKnEBRX(EM60XXh%7QbU3A^}DWnnelU0?a zKTL(0y{DAReF+yuQ(nsa;&n*WpXF=9f@$a*^EZpq5W zm+*rYT%fYD0MfPKsj@2YA7XAUE34etdS&X}R9SszAu(q-mDRBS4E0sk6m%!19@$F8 zIOy_+c}m4hY|s6%z4GbkI)sVqrhNJNXhtzVqnZ6rJoSN2`!;2u9# zzH2{$aK49?lXyeQG9g(x^~NngtN}XZt7j>vGFoFj-(NZX^D(R)tx(QCEWr}ZaOI~V znB#BCm0#BUgZBn{E0?Q)+17ujTrGxC^Sh;7O8}NE+^5_My-CO|d!lmZ^Z@wt-IV(? zdI3znq5KsMe}C*D<*#E2#4=`v^5}g4jVFI9PX{`%Ikgxe@{8?gs}4+q&0Ab=CT(uE zel)k)>yry1n>QU9X4(AU_?@7#(wkK`54d@>s^<%qN#$vbg*og=@?uu7^yX={h9_BI zKHxnM6=n|ac@)4*V?Bq1nfbh>BZ(&Ui8{B6DLk+hpTZ~biF`}G5ue1T5*6Pypohos z;cwQ7uHQY+UZfV-ii)g*ZECjDQDiSFE>fMjYIFlNJJ0H}W*6IBMX{8LWCA?aFy>QB zWIaBDPeq~iQDi*dSSyt8NK~XW<(u;<^u=v+u)kkxdr@J&b(ETEv%0eL)Kuq4y$tGC zgZa*r5zX{t=;NAM^PNP;p>@DoAF^iwd*nh;~j)LUY92L{BJ3f>oJL*6*0^P zmgbM(e5j`p8%X=M<-EPUdRbi#yJK)uqsas6Gt`IW?$f9gzHXW4SrpB7`Ulgtv-1;`z7uBswdd z52S!K@o9W#y#w0t zChNnShH~SPkbtc)i9%+^5jozKm){-Xy*;{l2RyoX$?? z{iypQqtIaqmROeAOP3{B0_Uz~s<<<3#V%(NSlFf(+FW*Ljy>C&pFav*ZOa~7q>gfy zxYS&i-R8(q^L3`rb-L6nmvcmsH7nny=GyaZkPK^ab{@FN$X@KSI*M{35B1byr)n!I zMoxQCo@!Nd>@Hh&vHB`QtlBQcp%!bhm2EAu)%#CvLmj9?OR0sXtN-KiG3Y{P916s;AOWy`Y;x1*g}TlJKeJ#aemN7YkH9D3(o=5>yY zRr@$g&;o~ARBX@B*Tw-47X07#s`gy1L8@(JVZJ@vUW{R~77bOKYSq|_>#13zR9o!e z*#ERP*X1nucfV>C>0_o3W~@qwg|I=iZ70T`WzT5B7fOE#vjl`kV(waki8ZE9!$)I9 zO_?B#Z$tN-VS;F#aPZ~I3iF%cTBg^Glf<{deLR#|OD&bEeJ#PXd;-JM%%i3tI^!pc znf|%m9PFbrZH!hdjoi+L(4$*SKJ@P==9*k2-;&P!21%+f&->$NfKZ@Te^Z^~n?oqn zsZ3yCeK1}EKAJ?eK!#*or&IM3IuT*#EWTttlGDJgP4%`npzaD&XhW?t@w(I|Lp-+Q z*#`I+I#N@>$-1nDqr3+Aw8EoF)Sk&yPfydOB_7GikOD)Q*4H2Cz(b66<H1Oxx@#g3p>ZuHAFOA0 zy-LusSaWh*unH=ePRnZ7xY?$T$g>yQG--)db;D8Uau%Zmotq&{UdWsF53{~wB>*awT6w*_%X|dZ3qtsj* zl2Knlk@3Z=Ms&78eA;SXBCFOPI;}es=^vMsZ_OU6HST3^+I6LcA#AYDgr0}V%swxA zt+#0@J=2K^pgS6{_2_{PFlv9dW|DZCG}ThmxQ}heFbq9oH97U1p1AhRI)+Q(Bk8dr zrYH{lK+89ntI{WFyf^ixOdTUzw~kCwsAuTQ06KVPCpp!Qu2CW%R)@9tqjy!{{ti7=YYX|vl*s84WZ+vuh%%$xN3HRdCx26Z1}0_oldrm9tZ zJ3!ky)OI=0q4r|Ct%%mYj^?lb4884tgQ@CSdxL4r(9XA*wXocJ6`@d3>8K`!rrc&i ze1a>xVpQ?y7PEx*8p_wAf8S=hC#gUxnyJzx-@p;gby01CnN5H>Yg*YwIRh5MW!HtE z*h%Z(LOT}x#*_*w#w8xMGKIv@``?-ZgAyyBQ8I1$xiQ9rboWh0rB&}Rm&?2+`FYmd zWoj^{0KO{UjCQ%lga|(AE@w%h+SyuQ^Gv+YNRggder)w{k6%3dN2n*hBRjP|(~l}+ z*=Q~a&omj!HfQ2!Y$+S&X;;c}5tWL%Jkb(bn{+2m;sRRiLL9`7YX>@1seEzW0Nuq* z(x=DR<}I3$EM$%&8b(RN#A-O|Uzwqv8j!BqL-@1?6SPEk6G>AP-59HA##<9#KeciQ&Ru^vp<$L`_5iIR34c0nl}{0BcU?e}~`ZXqM%QOF=d~(yl?**XrBQ+nUvH zs`gF%H>U3&o7J#Eti3^@3b>KdtT3gWLv^}xYycKcb+D~ibvg{-q85}ur@~m|+6R{a zve@jX+NC!N6>7B=JKbuwp)ahe0~#aGo&$k{es&GFXWP^`{Yopj1QsVB=*WN@IZoZU zj6!9)jjF8iHDnp=L_6xCTRFgDLv64Js>?n&uUO58b^N(5C z`fC$zcm^6XX|N@k{#}@e3&MbJ;&DM)Y=&|ewitN&OOCe@iafju4g=> z&$E9=O4WExDjI9N40OJ^saLw_!UHVEQ^o9H|S)-QmkkUL5juN+%Zw#%+_RIJ*?r9)($ zH8rO}lRs-tf!#p|1aWogo~m3BT@!@iz2VOVQz@A9GU;D?!h*SeJUt)E{m7)yE(@7p zdgz9^R`{z%kU$Nj>-?|h*ZG_74&_2U%_2B+Bz@itC4YF<6znTNhW2Hl57Mo3YId@+v8V?22ixQE_l`BSRV_0zdl)INhN*6&*8 zdR6oGO>-6$z?K%#VbeL4iKneHErAwI-9w$x=DA)G;;S5*Z0ot`WCb(Y;X`mz{!A{# z(_|J`llL5&%iUyX^gM36=g0Y6IAhpU`$BFVwa!J4Pu*hz0{s7DGgaNf(QY%is-A<3 zxaADpx`Z1_kKAEGXwOAlfXA_v+sea^fBKFkfcCG#MFduka&;Pg;~QM}=t>-5V0GO($E4vNO-3$^tG1 z&|`a8A3?K7hHcR01Il5l#S0ObM5up&0gMqWFaCDGSX~MM!t0~*LEiAOjRI-JWG-0w zpL)xzKLye0qL+nk>E~6ArVeAON^Mb&Hs*e)?=2eD!{iSZuS`I72Kq?U+zDF#nF$)pmxR{Ala=*oGXF1Ot!!T^iRD|+ zu*n>LKI~;~{(nXaPih*bA+$lN*2k(|Xn}?w{OE_jveo}J&iCe+0&SX3Nyb3xSWj;& z0v39jX7kY#Tx*l|DiWkktk!<$7yQ+1f!5>K`bmq3X+T?h)`-oZcKZF-rbt>l)+E!^ z0hWNMmvg3P(~2 zy|jWGz|O?$>}G`jn&>^$o_@)s^tlTE`PzJQNKJG^i}S!c{y$v;$G&O{_1$8M4Ax51 z?3adKz{q$^8-?aHUgW}Qx3QK03+$s-Ki!b%72Zd8b8-6$iD{Kzl>p)lX6J&)eJE05f`g1RgjegKSZ^&AF(b{0>z-~FoL3aB`EiwzE4NL}DzWDm z#{lNo3as{gwP;ik9O8OqRIO7NhxhRf&@^u4MW!qfEcDVJxT(QOwXBrt_kD5L-HfxbxgLzKG0W{E< zRZ?7;j)BUzJBAu#rhn**QX=TU1v$kJ@xN zFcOGcBgRmir#f}8iaA3R8$k;j0)U3^jNa12lUZwB8j9iJFz=45D| z^vWWY9u6&>=}^0-cU6blMp+P?@M@RS+HTEv4rVyq)u!VZt}0zKk`EE(SGkdswsfF} zDed9G<)WKqwYEjj4efbfIy?uRb{F%ho~a{vyX@&Sg&!DBAFtrscrL8uuP7d`FZhub z&+VOjLm#^7FyGJIU<4{vufleN2S{@~1;4}?~Km&AWlgj>j zy^1$b>uAIr^G2D5m(AH9?s8sYBt9wkTffVPJ>UeD`9b)Agc@~~CZHe`KZ!stM z(EHWRess`bo~OUO#VMXtfAf}L&%L_lE^PQ1J{VJ{g%zPHb%ogqFd-Qu4%d}_dWs39 z*8{x*sY|s44l?5J5quOJ%3ko!9V7=H`Uq0=8rzcO%wntSr4U#Ronk;BO$xQRo)$A! zi%JS#2H><Gv;HIXV-TX_1{H!m`KZX}ui foa?FaojEPU)A6Qx&JK^f%Hp%ybGegO@5cWR=CXT? diff --git a/retroshare-gui/src/lang/retroshare_ru.ts b/retroshare-gui/src/lang/retroshare_ru.ts index 1506404c3..fdc6378c1 100644 --- a/retroshare-gui/src/lang/retroshare_ru.ts +++ b/retroshare-gui/src/lang/retroshare_ru.ts @@ -1,6 +1,6 @@ - + AWidget @@ -560,7 +560,7 @@ p, li { white-space: pre-wrap; } On List Ite&m - + Отдельным списком @@ -660,7 +660,7 @@ p, li { white-space: pre-wrap; } Disable SysTray ToolTip - + Трэй без подсказки @@ -1058,7 +1058,7 @@ p, li { white-space: pre-wrap; } Ban this person (Sets negative opinion) - + Забанить этого пользователя (устанавливает негативную репутацию) @@ -1068,12 +1068,12 @@ p, li { white-space: pre-wrap; } Sort by Name - Сортировать по имени + Сортировать по имени Sort by Activity - + Сортировать по последней активности @@ -1310,7 +1310,7 @@ p, li { white-space: pre-wrap; } Anonymous IDs accepted - + Анонимные ID разрешены @@ -1556,27 +1556,27 @@ Double click lobbies to enter and chat. Distant Chat - + Удаленный чат Everyone - + Кого угодно Contacts - Контакты + Контакты Nobody - + Никого Accept encrypted distant chat from - + Разрешить шифрованный удаленный чат от @@ -1631,7 +1631,7 @@ Double click lobbies to enter and chat. Send as plain text by default - + Посылать как простой текст @@ -1775,7 +1775,7 @@ Double click lobbies to enter and chat. UserName - + Имя пользователя @@ -2028,17 +2028,17 @@ Double click lobbies to enter and chat. Quote - Цитата + Цитата Quotes the selected text - + Цитировать выбранный текст Drop Placemark - + Поставить метку @@ -2048,23 +2048,23 @@ Double click lobbies to enter and chat. Save image - + Сохранить изображение Send as PlainText - + Послать как простой текст Send as plain text without font. - + Послать как простой текст без шрифта Don't replace tag with Emote Icon. - + Не заметь текст смайлами @@ -2075,13 +2075,15 @@ Double click lobbies to enter and chat. It remains %1 characters after HTML conversion. - + %1 символов +после конвертирования в HTML. Warning: This message is too big of %1 characters after HTML conversion. - + Внимание: Это сообщение слишком большое, %1 символов +после конвертирования в HTML. @@ -2441,7 +2443,7 @@ after HTML conversion. Use this certificate to make friends: - + Используемый сертификат: @@ -2659,7 +2661,7 @@ after HTML conversion. Please, paste your friend's Retroshare certificate into the box below - + Пожалуйста введите сертификат вашего друга в поле внизу @@ -2784,7 +2786,7 @@ after HTML conversion. Email - Email + Email @@ -3211,7 +3213,7 @@ resources. The text below is your Retroshare certificate. You have to provide it to your friend - + Последущий текст это ваш сертификат RetroShare. Вы должны передать его вашему другу @@ -3881,7 +3883,7 @@ p, li { white-space: pre-wrap; } Invited Members - + Приглашенные Пользователи @@ -3891,7 +3893,7 @@ p, li { white-space: pre-wrap; } Known People - + Известные Пользователи @@ -3906,7 +3908,7 @@ p, li { white-space: pre-wrap; } Name: - Имя: + Имя: @@ -3916,7 +3918,7 @@ p, li { white-space: pre-wrap; } Contact author: - + Идентификатор Автора @@ -3926,12 +3928,12 @@ p, li { white-space: pre-wrap; } [Circle Admin] - + [Администратор Круга] Distribution: - + Достпно: @@ -3946,7 +3948,7 @@ p, li { white-space: pre-wrap; } Private - Частный + Частный @@ -3956,7 +3958,7 @@ p, li { white-space: pre-wrap; } Only visible to members of: - + Доступно только кругу: @@ -4051,28 +4053,28 @@ p, li { white-space: pre-wrap; } Circle name - + Имя Круга Update - + Обновить Close - Закрыть + Закрыть Create New Circle - + Создать Новый Круг Create - Создать + Создать @@ -4082,12 +4084,12 @@ p, li { white-space: pre-wrap; } Add Member - + Добавить члена Remove Member - + Удалить члена @@ -4101,12 +4103,12 @@ p, li { white-space: pre-wrap; } Group Name: - Имя группы: + Имя группы: Group ID: - + ID Группы Group Name @@ -4120,7 +4122,7 @@ p, li { white-space: pre-wrap; } To be defined - + Должно быть задано @@ -4367,12 +4369,12 @@ p, li { white-space: pre-wrap; } No compatible ID for this forum - + Нет совместимого ID для этого форума None of your identities is allowed to post in this forum. This could be due to the forum being limited to a circle that contains none of your identities, or forum flags requiring a PGP-signed identity. - + Ни один из ваших идентификаторов не имеет прав для отправки сообщений в этот форум. Возможно доступ к форуму ограничен кругом в котором вы не состоите, или форум требует наличие подписанного ID. @@ -4679,7 +4681,7 @@ Do you want to reject this message? No peer found in DHT - + Не найдено пиров в DHT @@ -4934,7 +4936,7 @@ Do you want to reject this message? Copy %1 to clipboard - + Копировать %1 в буфер обмена @@ -5190,7 +5192,7 @@ Do you want to reject this message? Relays - + Реле @@ -5981,7 +5983,7 @@ at least one peer was not added to a group Select file for importing your friendlist from - + Выберите файл из которого импортировать список друзей Select file for importing yoour friendlist from @@ -6714,7 +6716,7 @@ Fill in your PGP password when asked, to sign your new key. Register retroshare:// as URL protocol - + Заргеистрировать retroshare:// как URI протокол Register retroshare:// as URL protocol (Restart required) @@ -6743,7 +6745,7 @@ Fill in your PGP password when asked, to sign your new key. !!!The RetroShare's desktop file is missing or wrong!!! - + !!!Ярлык RetroShare отсутствует или поврежден!!! @@ -6768,12 +6770,12 @@ Fill in your PGP password when asked, to sign your new key. You have enough right. - + У вас достаточно прав. You don't have enough right. Run RetroShare as Admin to change this setting. - + У вас недостаточно прав. Запустите RetroShare от имени администратора для изменения этих настроек. @@ -7037,7 +7039,7 @@ p, li { white-space: pre-wrap; } Identity Name - + Имя идентификатора @@ -7057,17 +7059,17 @@ p, li { white-space: pre-wrap; } Stored data size - + Размер сохраненных данных Receive time (secs ago) - + Время получения (просшло секунд) Sending time (secs ago) - + Время отправки (прошло секунд) Data size @@ -7094,7 +7096,7 @@ p, li { white-space: pre-wrap; } Details - Подробности + Подробности @@ -7127,7 +7129,7 @@ p, li { white-space: pre-wrap; } [Unknown identity] - + [Неизвестный идентификатор] @@ -7148,7 +7150,7 @@ p, li { white-space: pre-wrap; } [Unknown] - [Неизвестный] + [Неизвестный] @@ -7286,12 +7288,12 @@ p, li { white-space: pre-wrap; } Share channel publish permissions - + Дать разрешение публиковать в канал You can allow your friends to publish in your channel, or send the publish permissions to another Retroshare instance of yours. Select the friends which you want to be allowed to publish in this channel. Note: it is currently not possible to revoke channel publish permissions. - + Вы можете разрешить вашим друзьям писать в канал или разрешать другим пользователям писать в канал. Выберите друзей которым вы хотите разрешить писать в канал. Имейте ввиду: на данный момент невозможно отменить разрешения записи в канал. @@ -7367,17 +7369,17 @@ p, li { white-space: pre-wrap; } Sort by Posts - + Сортировать по сообщениям You are admin (modify names and description using Edit menu) - + Вы администратор (изменяйте название и описание используя меню Редактировать) You have been granted as publisher (you can post here!) - + Вам разрешено публиковать @@ -7735,12 +7737,12 @@ p, li { white-space: pre-wrap; } New Comment: - + Новый Коментарий Comment Value - + Значение Коментария @@ -7911,27 +7913,27 @@ p, li { white-space: pre-wrap; } Public - Публичный + Публичный Restricted to members of circle " - + Только для членов круга " Restricted to members of circle - + Только для членов круга Your eyes only - + Только для вас You and your friend nodes - + Для вас и ваших друзей @@ -7956,18 +7958,18 @@ p, li { white-space: pre-wrap; } Administrator: - + Администратор: unknown - Неизвестно + Неизвестно Distribution: - + Распространение: @@ -8321,7 +8323,7 @@ before you can comment Save image - + Сохранить изображение @@ -8450,7 +8452,7 @@ before you can comment This message was obtained from %1 - + Это сообщение получено от %1 @@ -8460,58 +8462,58 @@ before you can comment Anonymous/unknown node IDs reputation threshold set to 0.4 - + Порог для агонимных/неизвестных id установлен в 0.4 Anonymous IDs reputation threshold set to 0.4 - + Порог для анонимных ID установлен в 0.4 Message routing info kept for 10 days - + Маршрут сообщения хранится 10 дней [unknown] - + [неизвестно] Public - Публичный + Публичный Restricted to members of circle " - + ограничено для членов круга " Restricted to members of circle - + Ограничено для членов круга Only friends nodes in group - + Только ноды друзей в группе Your eyes only - + Только для вас Distribution - Распределение + Распределение Anti-spam - + Анти-спам @@ -8841,29 +8843,29 @@ before you can comment Spam-protection - + Защита от спама Comments: - Комментарии: + Комментарии: TextLabel - Текстовая метка + Текстовая метка Distribution: - + Распространение: Anti Spam: - + Анти-Спам: @@ -8879,23 +8881,23 @@ before you can comment Restricted to circle: - + Ограничено кругом: Limited to your friends - + Только для ваших другей Allowed - + Разрешено Disallowed - + Запрещено @@ -8907,12 +8909,12 @@ before you can comment PGP signature required - + Необзодима PGP подпись Only friends nodes in group - + Только ноды друзей в группе @@ -8922,7 +8924,7 @@ before you can comment PGP signature from known ID required - + Необходима PGP подпись от известного ID @@ -8986,7 +8988,7 @@ before you can comment Restricted to Circle - + Ограничено кругом @@ -8996,7 +8998,7 @@ before you can comment Posts permissions: - + Разрешения: @@ -9006,17 +9008,17 @@ before you can comment All allowed - + Все Defavor unsigned IDs - + Запретить неподписанные ID Defavor unsigned IDs and IDs from unknown nodes - + Запретить неподписанные и неизвестные ID @@ -9098,7 +9100,7 @@ before you can comment Share publish permissions - + Поделиться правами на запись @@ -9217,42 +9219,42 @@ before you can comment Authenticated tunnels: - + Авторизованные туннели: Tunnel ID: %1 - + ID туннеля: %1 from: %1 - + от: %1 to: %1 - + к: %1 status: %1 - + статус: %1 total sent: %1 bytes - + всего отправленно: %1 байт total recv: %1 bytes - + всего получено: %1 байт Unknown Peer - Неизвестный пир + Неизвестный пир @@ -9605,7 +9607,7 @@ p, li { white-space: pre-wrap; } Owner node ID : - ID владельца узла + ID владельца узла: @@ -9615,17 +9617,17 @@ p, li { white-space: pre-wrap; } Owner node name : - Имя владельца узла + Имя владельца узла: Identity name : - Псевдоним + Псевдоним: Identity ID : - ID личности + ID личности: @@ -9726,7 +9728,7 @@ p, li { white-space: pre-wrap; } Overall: - Общая + Общая: @@ -9836,17 +9838,17 @@ p, li { white-space: pre-wrap; } Create new circle - + Создать новы круг Persons - + Участники Person - + Участник @@ -9865,27 +9867,27 @@ p, li { white-space: pre-wrap; } Circles - Круги + Круги Circle name - + Имя круга Membership - Членство + Членство Public Circles - Публичные Круги + Публичные Круги Personal Circles - Личные Круги + Личные Круги @@ -9911,12 +9913,12 @@ p, li { white-space: pre-wrap; } Owner node ID : - ID владельца узла + ID владельца узла : Identity name : - Псевдоним + Псевдоним : @@ -9941,12 +9943,12 @@ p, li { white-space: pre-wrap; } Identity ID : - ID личности + ID личности : Owner node name : - Имя владельца узла + Имя владельца узла : @@ -9956,7 +9958,7 @@ p, li { white-space: pre-wrap; } Send Invite - + Пригласить @@ -9996,7 +9998,7 @@ p, li { white-space: pre-wrap; } Overall: - Общая + Общая: Owned by you @@ -10010,12 +10012,12 @@ p, li { white-space: pre-wrap; } ID - + ID Search ID - + Искомый ID @@ -10033,22 +10035,22 @@ p, li { white-space: pre-wrap; } My own identities - + Мои идентификаторы My contacts - + Мои контакты Owned by myself - + Мои Linked to my node - + Связанные с моей нодой @@ -10058,162 +10060,162 @@ p, li { white-space: pre-wrap; } Other circles - + Другие круги Circles I belong to - + Круги со мной Circle ID: - + ID круга: Visibility: - + Видимость: Private (only visible to invited members) - + Частный (видим только членам) Only visible to full members of circle - + Видим только полноценным членам круга Public - Публичный + Публичный Your role: - + Ваша роль: Administrator (Can edit invite list, and request membership). - + Администратор (Может редактировать лист приглашений и запросов членства). User (Can only request membership). - + Пользователь (Может запросить членство). Distribution: - + Распространение: subscribed (Receive/forward membership requests from others and invite list). - + подписан (получает/передает запросы членства от других). unsubscribed (Only receive invite list). - + неподписанный (получает только праглашения). Your status: - + Ваш статус: Full member (you have access to data limited to this circle) - + Полноценный член (вы имеете доступ к данным достпным этому кругу) Not a member (do not have access to data limited to this circle) - + Не член (нет доступа к данным доступным кругу) Unknown ID : - + Незивестный ID : Identity ID: - + ID идентификатора: Status: - Статус: + Статус: Full member - + Полноценный член Invited by admin - + Пригглашен администратором Subscription request pending - + Запрошена подписка unknown - Неизвестно + Неизвестно Invited - + Приглашен Subscription pending - + Ожидание подписки Member - + Член Edit Circle - + Редактировать Круг See details - + Смотреть детали Request subscription - + Запросить подписку Accept circle invitation - + ПРинять приглашение в круг Quit this circle - + Покинуть круг Cancel subscribe request - + Отменить запрос подписки @@ -10221,17 +10223,17 @@ p, li { white-space: pre-wrap; } for identity - + для идентификатора Revoke this member - + Выгнать члена Grant membership - + Разрешить членство @@ -10245,24 +10247,24 @@ These identities will soon be not supported anymore. [Unknown node] - + [Неизвестная нода] Unverified signature from node - + Неподтвержденная подпись от ноды Unchecked signature - + Нероверенная подпись [unverified] - + [неподтверждено] @@ -10292,27 +10294,27 @@ These identities will soon be not supported anymore. Add to Contacts - + Добавить в контакты Remove from Contacts - + Удалить из контактов Set positive opinion - + Позитивное мнение Set neutral opinion - + Нейтральное мнение Set negative opinion - + Негативное мнение @@ -10327,22 +10329,22 @@ These identities will soon be not supported anymore. Hi,<br>I want to be friends with you on RetroShare.<br> - + Привет,<br>Я хочу дружить с тобой в RetroShare.<br> You have a friend invite - + Вы получили предложение дружбы Respond now: - + Ответить сейчас: Thanks, <br> - Спасибо, <br> + Спасибо, <br> @@ -10404,7 +10406,7 @@ These identities will soon be not supported anymore. Distant chat refused with this person. - В удалённом чате с этим человеком отказано + В удалённом чате с этим человеком отказано. @@ -10694,17 +10696,17 @@ These identities will soon be not supported anymore. Save image - + Сохранить изображение Cannot save the image, invalid filename - + Невозможно сохранить изображение, не верное имя файла Not an image - + Не является изображением @@ -11120,12 +11122,12 @@ RetroShare безопасно приостановит доступ диска Address list: - + Список адресов Recommend this friend - + Рекомендовать этого друга @@ -11235,17 +11237,17 @@ RetroShare безопасно приостановит доступ диска All addresses (mixed) - + Все адреса (смешано) All people - + Все люди My contacts - + Мои контакты @@ -11631,22 +11633,22 @@ Do you want to save message ? Everyone - + Все Contacts - Контакты + Контакты Nobody - + Никто Accept encrypted distant messages from - + Принимать зашифрованное сообщение от @@ -12382,7 +12384,7 @@ Do you want to save message ? This message goes to a distant person. - + Это сообщение будет отправлено удаленному участнику. @@ -12448,12 +12450,12 @@ Do you want to save message ? Spoiler - + Спойлер Select text to hide, then push this button - + Выберите текст который нужно скрыть, затем нажмите эту кнопку @@ -13466,7 +13468,7 @@ p, li { white-space: pre-wrap; } Below is the node's PGP key. It identifies the node and all its locations. A "Retroshare certificate" that you can exchange in order to make friends, is in the the "details" of each separate location. - + Ниже указан PGP ключ ноды, он определяет ноду и все её местоположения, "Сертификат RetroShare" который может быть использован для создания дружбы находится на вкладки "детали" в каждой отдельной локации @@ -13481,12 +13483,12 @@ p, li { white-space: pre-wrap; } Options - Параметры + Параметры These options apply to all locations of the same node: - + Эти опции применяются для всех локаций этой ноды @@ -13496,7 +13498,7 @@ p, li { white-space: pre-wrap; } Use as direct source, when available - + Использовать как прямой источник, если возможно @@ -13506,7 +13508,7 @@ p, li { white-space: pre-wrap; } Auto-download recommended files from this node - Авто-загрузка рекомендованных файлов с этого узла + Авто-загрузка рекомендованных файлов с этого узла @@ -13516,23 +13518,23 @@ p, li { white-space: pre-wrap; } Require white list clearance - Требуется разрешение белого списка + Требуется разрешение белого списка Max upload speed (0=unlimited) - + Максимальная скорость отдачи (0 - неограничено) Max download speed (0=unlimited) - + Максимальная скорость скачивания (0 - неограничено) kB/s - + kB/s @@ -13802,37 +13804,37 @@ p, li { white-space: pre-wrap; } Chat with this person - Начать чат + Начать чат Chat with this person as... - Инициировать чат с этим человеком от имени... + Инициировать чат с этим человеком от имени... Send message to this person - + Послать сообщение Person details - + Детали Distant chat cannot work - Удалённый чат не может работать + Удалённый чат не может работать Distant chat refused with this person. - В удалённом чате с этим человеком отказано + В удалённом чате с этим человеком отказано. Error code - Код ошибки + Код ошибки @@ -13840,22 +13842,22 @@ p, li { white-space: pre-wrap; } Identities handling - + Обработка идентификаторов ban all identities of a node when more than - + Банить все идентификаторы ноды если более чем 1 of them have a negative opinion - + из них имеет негативную репутацию People - Участники + Участники @@ -14402,7 +14404,7 @@ malicious behavior of crafted plugins. Chat remotely closed. Please close this window. - + Чат закрыт другой стороной. Пожалуйста закройте это окно. @@ -14659,12 +14661,12 @@ malicious behavior of crafted plugins. New Comment: - + Новый коментарий: Comment Value - + Значение Комантария @@ -15196,12 +15198,12 @@ and open the Make Friend Wizard. Warning: Retroshare is about to ask your system to open this file. - + Внимание: RetroShare собирается запросить открытие этого файла системой. Before you do so, please make sure that this file does not contain malicious executable code. - + Перед тем как это делать убедитесь в том что файл не содержит вредоносного кода. @@ -15316,17 +15318,17 @@ and open the Make Friend Wizard. Posted not found - + Не найдено Posted message not found - + Сообщение не найдено Posted messages not found - + Сообщения не найдены @@ -15645,7 +15647,8 @@ Reported error is: Security: no anonymous IDs - + +Безопасность: Анонимные ID запрещены @@ -16349,7 +16352,7 @@ Characters <b>",|,/,\,&lt;,&gt;,*,?</b> will be replace File Path - + Путь к файлу @@ -16621,7 +16624,7 @@ Reducing image to %1x%2 pixels? Registry Access Error. Maybe you need Administrator right. - + Ошибка доуступа к реестру, возможно необходимы права администратора. @@ -18120,7 +18123,7 @@ Select the Friends with which you want to Share your Channel. Lobby - + Лоби @@ -19704,12 +19707,12 @@ Try to be patient! Anonymous tunnels - + Анонимный туннели Authenticated tunnels - + Авторизованные туннели From 9e1f21b94158283f34ae21018ba2c7593b6b4f4f Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 8 Oct 2016 19:19:28 +0200 Subject: [PATCH 156/174] updated ubuntu changelog --- build_scripts/Debian+Ubuntu/changelog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build_scripts/Debian+Ubuntu/changelog b/build_scripts/Debian+Ubuntu/changelog index 2f476393e..666e580fb 100644 --- a/build_scripts/Debian+Ubuntu/changelog +++ b/build_scripts/Debian+Ubuntu/changelog @@ -1,10 +1,20 @@ retroshare06 (0.6.1-1.XXXXXX~YYYYYY) YYYYYY; urgency=low + 9a3eb1c csoler Sat, 8 Oct 2016 14:30:58 +0200 Merge pull request #528 from sehraf/pr-drop-ssh-protobuf + c55fb0d csoler Sat, 8 Oct 2016 14:11:43 +0200 Merge pull request #524 from cavebeat/compile + 97f2589 csoler Sat, 8 Oct 2016 14:09:34 +0200 Merge pull request #513 from grennvyper/patch-1 + aa200bf csoler Sat, 8 Oct 2016 14:06:54 +0200 Merge pull request #526 from marcobarbosa16/master + 2591818 csoler Sat, 8 Oct 2016 14:04:31 +0200 Merge pull request #515 from PhenomRetroShare/Add_ContextMenuToShowImageInLobby 9ed759d csoler Sat, 8 Oct 2016 13:38:18 +0200 Merge pull request #527 from PhenomRetroShare/Fix_DesktopFileName + ada1cee sehraf Sat, 8 Oct 2016 12:59:27 +0200 remove deprecated ssh/protobuf code 4b0f82a Phenom Sat, 8 Oct 2016 12:12:26 +0200 Fix Desktop Filename for option page message. + 69ecca4 MarcoBarbosa Fri, 7 Oct 2016 23:12:40 -0300 add uninstall section in README.md 7cd31aa csoler Wed, 5 Oct 2016 14:15:12 +0200 fixed bug preventign update of subdirs after last commit + fdfd2e9 Phenom Sat, 24 Sep 2016 16:12:44 +0200 Add context menu to show hidden image on lobby. 854eebb csoler Wed, 5 Oct 2016 10:59:51 +0200 only update subfiles and subdirs list during dir watching sweep when the TS of the dir has changed. Greatly improve cost of regular directory watching process d020d8d thunder2 Wed, 5 Oct 2016 06:40:02 +0200 Windows build environment: - Added automatic mode for creating git-log - Fixed filename of installer + 891e61e cave beat Tue, 4 Oct 2016 23:27:40 +0200 MacOS to .md, + README update + 9177c82 cave beat Tue, 4 Oct 2016 23:14:50 +0200 updated MSys2 Install Guide to .md d367491 thunder2 Tue, 4 Oct 2016 21:45:59 +0200 Fixed env.bat for Windows build 20cd123 csoler Tue, 4 Oct 2016 21:41:38 +0200 removed annoying ERROR output that is more a debug info than a real error, in p3filelists e42dae8 csoler Tue, 4 Oct 2016 21:34:23 +0200 continue sending packet slicing probes in case the peer has restarted From 3d9fb600f00f89eedf283d4f8db5331f4cd38f96 Mon Sep 17 00:00:00 2001 From: Phenom Date: Sun, 9 Oct 2016 12:37:09 +0200 Subject: [PATCH 157/174] Fix Missing ID-Avatars in Create Circle Menu Fix issue#530 --- .../src/gui/Circles/CreateCircleDialog.cpp | 144 ++++++++++-------- .../src/gui/Circles/CreateCircleDialog.h | 1 + .../src/gui/People/IdentityWidget.h | 1 + .../src/gui/People/PeopleDialog.cpp | 10 +- 4 files changed, 90 insertions(+), 66 deletions(-) diff --git a/retroshare-gui/src/gui/Circles/CreateCircleDialog.cpp b/retroshare-gui/src/gui/Circles/CreateCircleDialog.cpp index 79e85d1cc..8ed0bfc47 100644 --- a/retroshare-gui/src/gui/Circles/CreateCircleDialog.cpp +++ b/retroshare-gui/src/gui/Circles/CreateCircleDialog.cpp @@ -25,16 +25,16 @@ #include #include -#include "gui/Circles/CreateCircleDialog.h" -#include "gui/gxs/GxsIdDetails.h" #include #include + +#include "gui/common/AvatarDefs.h" +#include "gui/Circles/CreateCircleDialog.h" +#include "gui/gxs/GxsIdDetails.h" #include "gui/Identity/IdDialog.h" #include "gui/Identity/IdEditDialog.h" -#include - //#define DEBUG_CREATE_CIRCLE_DIALOG 1 #define CREATECIRCLEDIALOG_CIRCLEINFO 2 @@ -258,8 +258,9 @@ void CreateCircleDialog::addMember() QString keyId = item->text(RSCIRCLEID_COL_KEYID); QString idtype = item->text(RSCIRCLEID_COL_IDTYPE); QString nickname = item->text(RSCIRCLEID_COL_NICKNAME); + QIcon icon = item->icon(RSCIRCLEID_COL_NICKNAME); - addMember(keyId, idtype, nickname); + addMember(keyId, idtype, nickname, icon); } void CreateCircleDialog::addMember(const RsGxsIdGroup &idGroup) @@ -267,15 +268,27 @@ void CreateCircleDialog::addMember(const RsGxsIdGroup &idGroup) QString keyId = QString::fromStdString(idGroup.mMeta.mGroupId.toStdString()); QString nickname = QString::fromUtf8(idGroup.mMeta.mGroupName.c_str()); QString idtype = tr("Anon Id"); + + QPixmap pixmap ; + + if(idGroup.mImage.mSize == 0 || !pixmap.loadFromData(idGroup.mImage.mData, idGroup.mImage.mSize, "PNG")) + pixmap = QPixmap::fromImage(GxsIdDetails::makeDefaultIcon(RsGxsId(idGroup.mMeta.mGroupId))); + if (idGroup.mPgpKnown){ RsPeerDetails details; rsPeers->getGPGDetails(idGroup.mPgpId, details); idtype = QString::fromUtf8(details.name.c_str()); }//if (idGroup.mPgpKnown) - addMember(keyId, idtype, nickname); + addMember(keyId, idtype, nickname, QIcon(pixmap)); } void CreateCircleDialog::addMember(const QString& keyId, const QString& idtype, const QString& nickname ) +{ + QIcon icon; + addMember(keyId, idtype, nickname, icon); +} + +void CreateCircleDialog::addMember(const QString& keyId, const QString& idtype, const QString& nickname, const QIcon& icon) { QTreeWidget *tree = ui.treeWidget_membership; @@ -293,9 +306,9 @@ void CreateCircleDialog::addMember(const QString& keyId, const QString& idtype, QTreeWidgetItem *member = new QTreeWidgetItem(); member->setText(RSCIRCLEID_COL_NICKNAME, nickname); + member->setIcon(RSCIRCLEID_COL_NICKNAME, icon); member->setText(RSCIRCLEID_COL_KEYID, keyId); member->setText(RSCIRCLEID_COL_IDTYPE, idtype); - //member->setIcon(RSCIRCLEID_COL_NICKNAME, pixmap); tree->addTopLevelItem(member); @@ -317,14 +330,19 @@ void CreateCircleDialog::addCircle(const RsGxsCircleDetails &cirDetails) QString nickname = QString::fromUtf8(gxs_details.mNickname.c_str()); QString idtype = tr("Anon Id"); - addMember(keyId, idtype, nickname); + QPixmap pixmap ; + + if(gxs_details.mAvatar.mSize == 0 || !pixmap.loadFromData(gxs_details.mAvatar.mData, gxs_details.mAvatar.mSize, "PNG")) + pixmap = QPixmap::fromImage(GxsIdDetails::makeDefaultIcon(gxs_details.mId)); + + addMember(keyId, idtype, nickname, QIcon(pixmap)); }//if(!gxs_id.isNull() && rsIdentity->getIdDetails(gxs_id,gxs_details)) }//for (itUnknownPeers it = cirDetails.mUnknownPeers.begin() typedef std::set::const_iterator itAllowedPeers; for (itAllowedPeers it = cirDetails.mAllowedNodes.begin() ; it != cirDetails.mAllowedNodes.end() ; ++it ) - { + { RsPgpId gpg_id = *it; RsPeerDetails details ; if(!gpg_id.isNull() && rsPeers->getGPGDetails(gpg_id,details)) { @@ -333,7 +351,10 @@ void CreateCircleDialog::addCircle(const RsGxsCircleDetails &cirDetails) QString nickname = QString::fromUtf8(details.name.c_str()); QString idtype = tr("PGP Identity"); - addMember(keyId, idtype, nickname); + QPixmap avatar; + AvatarDefs::getAvatarFromGpgId(gpg_id, avatar); + + addMember(keyId, idtype, nickname, QIcon(avatar)); }//if(!gpg_id.isNull() && rsPeers->getGPGDetails(gpg_id,details)) }//for (itAllowedPeers it = cirDetails.mAllowedPeers.begin() @@ -756,7 +777,7 @@ void CreateCircleDialog::loadIdentities(uint32_t token) bool acceptAllPGP = ui.radioButton_ListAllPGP->isChecked(); //bool acceptKnownPGP = ui.radioButton_ListKnownPGP->isChecked(); - RsGxsIdGroup data; + RsGxsIdGroup idGroup; std::vector datavector; std::vector::iterator vit; if (!rsIdentity->getGroupData(token, datavector)) { @@ -765,67 +786,68 @@ void CreateCircleDialog::loadIdentities(uint32_t token) return; } - for(vit = datavector.begin(); vit != datavector.end(); ++vit) - { - data = (*vit); + for(vit = datavector.begin(); vit != datavector.end(); ++vit) + { + idGroup = (*vit); - /* do filtering */ - bool ok = false; - if (acceptAnonymous) - ok = true; - else if (acceptAllPGP) - ok = data.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID ; - else if (data.mPgpKnown) - ok = data.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID ; + /* do filtering */ + bool ok = false; + if (acceptAnonymous) + ok = true; + else if (acceptAllPGP) + ok = idGroup.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID ; + else if (idGroup.mPgpKnown) + ok = idGroup.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID ; - if (!ok) { + if (!ok) { #ifdef DEBUG_CREATE_CIRCLE_DIALOG - std::cerr << "CreateCircleDialog::insertIdentities() Skipping ID: " << data.mMeta.mGroupId; - std::cerr << std::endl; + std::cerr << "CreateCircleDialog::insertIdentities() Skipping ID: " << data.mMeta.mGroupId; + std::cerr << std::endl; #endif - continue; - } + continue; + } - QString keyId = QString::fromStdString(data.mMeta.mGroupId.toStdString()); - QString nickname = QString::fromUtf8(data.mMeta.mGroupName.c_str()); - QString idtype = tr("Anon Id"); - - QPixmap pixmap ; + QString keyId = QString::fromStdString(idGroup.mMeta.mGroupId.toStdString()); + QString nickname = QString::fromUtf8(idGroup.mMeta.mGroupName.c_str()); + QString idtype = tr("Anon Id"); - if(data.mImage.mSize == 0 || !pixmap.loadFromData(data.mImage.mData, data.mImage.mSize, "PNG")) - pixmap = QPixmap::fromImage(GxsIdDetails::makeDefaultIcon(RsGxsId(data.mMeta.mGroupId))) ; + QPixmap pixmap ; - if (data.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID) - { - if (data.mPgpKnown) { - RsPeerDetails details; - rsPeers->getGPGDetails(data.mPgpId, details); - idtype = QString::fromUtf8(details.name.c_str()); - } - else - idtype = tr("PGP Linked Id"); - - } + if(idGroup.mImage.mSize == 0 || !pixmap.loadFromData(idGroup.mImage.mData, idGroup.mImage.mSize, "PNG")) + pixmap = QPixmap::fromImage(GxsIdDetails::makeDefaultIcon(RsGxsId(idGroup.mMeta.mGroupId))) ; - QTreeWidgetItem *item = new QTreeWidgetItem(); - item->setText(RSCIRCLEID_COL_NICKNAME, nickname); - item->setIcon(RSCIRCLEID_COL_NICKNAME, QIcon(pixmap)); - item->setText(RSCIRCLEID_COL_KEYID, keyId); - item->setText(RSCIRCLEID_COL_IDTYPE, idtype); - tree->addTopLevelItem(item); + if (idGroup.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID) + { + if (idGroup.mPgpKnown) { + RsPeerDetails details; + rsPeers->getGPGDetails(idGroup.mPgpId, details); + idtype = QString::fromUtf8(details.name.c_str()); + } + else + idtype = tr("PGP Linked Id"); - // External Circle. - if (mIsExistingCircle) - { - // check if its in the circle. + } - // We use an explicit cast - // + QTreeWidgetItem *item = new QTreeWidgetItem(); + item->setText(RSCIRCLEID_COL_NICKNAME, nickname); + item->setIcon(RSCIRCLEID_COL_NICKNAME, QIcon(pixmap)); + item->setText(RSCIRCLEID_COL_KEYID, keyId); + item->setText(RSCIRCLEID_COL_IDTYPE, idtype); + tree->addTopLevelItem(item); - if ( mCircleGroup.mInvitedMembers.find(RsGxsId(data.mMeta.mGroupId)) != mCircleGroup.mInvitedMembers.end()) /* found it */ - addMember(keyId, idtype, nickname); - } - } + // External Circle. + if (mIsExistingCircle) + { + // check if its in the circle. + + // We use an explicit cast + // + + if ( mCircleGroup.mInvitedMembers.find(RsGxsId(idGroup.mMeta.mGroupId)) != mCircleGroup.mInvitedMembers.end()) /* found it */ + addMember(keyId, idtype, nickname, QIcon(pixmap)); + + } + } } void CreateCircleDialog::loadRequest(const TokenQueue *queue, const TokenRequest &req) diff --git a/retroshare-gui/src/gui/Circles/CreateCircleDialog.h b/retroshare-gui/src/gui/Circles/CreateCircleDialog.h index 2cb050c04..2a6ce94de 100644 --- a/retroshare-gui/src/gui/Circles/CreateCircleDialog.h +++ b/retroshare-gui/src/gui/Circles/CreateCircleDialog.h @@ -41,6 +41,7 @@ public: void editNewId(bool isExternal); void editExistingId(const RsGxsGroupId &circleId, const bool &clearList = true, bool readonly=true); + void addMember(const QString &keyId, const QString &idtype, const QString &nickname, const QIcon &icon); void addMember(const QString &keyId, const QString &idtype, const QString &nickname); void addMember(const RsGxsIdGroup &idGroup); void addCircle(const RsGxsCircleDetails &cirDetails); diff --git a/retroshare-gui/src/gui/People/IdentityWidget.h b/retroshare-gui/src/gui/People/IdentityWidget.h index 8daebc6c1..c9ade98e3 100644 --- a/retroshare-gui/src/gui/People/IdentityWidget.h +++ b/retroshare-gui/src/gui/People/IdentityWidget.h @@ -42,6 +42,7 @@ public: const QString idtype() const { return _idtype; } const QString nickname() const { return _nickname; } const QString gxsId() const { return _gxsId; } + const QImage avatar() const { return _avatar; } signals: void addButtonClicked(); diff --git a/retroshare-gui/src/gui/People/PeopleDialog.cpp b/retroshare-gui/src/gui/People/PeopleDialog.cpp index 842627338..5098da953 100644 --- a/retroshare-gui/src/gui/People/PeopleDialog.cpp +++ b/retroshare-gui/src/gui/People/PeopleDialog.cpp @@ -557,7 +557,7 @@ void PeopleDialog::addToCircleInt() std::map::iterator itIdFound; if((itIdFound=_pgp_identity_widgets.find(pgp_id)) != _pgp_identity_widgets.end()) { IdentityWidget *idWidget = itIdFound->second; - dlg.addMember(idWidget->keyId(), idWidget->idtype(), idWidget->nickname()); + dlg.addMember(idWidget->keyId(), idWidget->idtype(), idWidget->nickname(), QIcon(QPixmap::fromImage(idWidget->avatar())) ); }//if((itFound=_pgp_identity_widgets.find(pgp_id)) != _pgp_identity_widgets.end()) dlg.editExistingId(circle->groupInfo().mGroupId, false,false); @@ -769,7 +769,7 @@ void PeopleDialog::fl_flowLayoutItemDroppedInt(QListflListItem IdentityWidget* idDest = qobject_cast(dest); if (idDest) { if (idDest->havePGPDetail()){ - dlg.addMember(idDest->keyId(), idDest->idtype(), idDest->nickname()); + dlg.addMember(idDest->keyId(), idDest->idtype(), idDest->nickname(), QIcon(QPixmap::fromImage(idDest->avatar())) ); }//if (idDest->havePGPDetail()) }//if (idDest) @@ -790,7 +790,7 @@ void PeopleDialog::fl_flowLayoutItemDroppedInt(QListflListItem } else {//if (cirDropped) IdentityWidget* idDropped = qobject_cast(flCurs); if (idDropped){ - dlg.addMember(idDropped->keyId(), idDropped->idtype(), idDropped->nickname()); + dlg.addMember(idDropped->keyId(), idDropped->idtype(), idDropped->nickname(), QIcon(QPixmap::fromImage(idDropped->avatar())) ); }//if (idDropped) }//else (cirDropped) @@ -980,7 +980,7 @@ void PeopleDialog::pf_dropEventOccursInt(QDropEvent *event) IdentityWidget* idDropped = qobject_cast(flCurs); if (idDropped){ if (idDropped->havePGPDetail()){ - dlg.addMember(idDropped->keyId(), idDropped->idtype(), idDropped->nickname()); + dlg.addMember(idDropped->keyId(), idDropped->idtype(), idDropped->nickname(), QIcon(QPixmap::fromImage(idDropped->avatar())) ); atLeastOne = true; }//if (idDropped->havePGPDetail()) }//if (idDropped) @@ -1015,7 +1015,7 @@ void PeopleDialog::pf_dropEventOccursInt(QDropEvent *event) IdentityWidget* idDropped = qobject_cast(flCurs); if (idDropped){ if (idDropped->havePGPDetail()){ - dlg.addMember(idDropped->keyId(), idDropped->idtype(), idDropped->nickname()); + dlg.addMember(idDropped->keyId(), idDropped->idtype(), idDropped->nickname(), QIcon(QPixmap::fromImage(idDropped->avatar())) ); atLeastOne = true; }//if (idDropped->havePGPDetail()) From 957d48b5fc4c68b9239eb6accb60ade28bbc1ec9 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 9 Oct 2016 21:03:01 +0200 Subject: [PATCH 158/174] removed folderIterator::d_name() because it duplicates file_name() --- libretroshare/src/plugins/pluginmanager.cc | 3 +-- libretroshare/src/rsserver/rsaccounts.cc | 3 +-- libretroshare/src/util/folderiterator.cc | 21 +-------------------- libretroshare/src/util/folderiterator.h | 2 -- retroshare-nogui/src/introserver.cc | 3 +-- 5 files changed, 4 insertions(+), 28 deletions(-) diff --git a/libretroshare/src/plugins/pluginmanager.cc b/libretroshare/src/plugins/pluginmanager.cc index 98335ea5a..19980c812 100644 --- a/libretroshare/src/plugins/pluginmanager.cc +++ b/libretroshare/src/plugins/pluginmanager.cc @@ -147,8 +147,7 @@ void RsPluginManager::loadPlugins(const std::vector& plugin_directo for(;dirIt.isValid();dirIt.next()) { - std::string fname; - dirIt.d_name(fname); + std::string fname = dirIt.file_name(); char lc = plugin_directories[i][plugin_directories[i].length()-1] ; // length cannot be 0 here. diff --git a/libretroshare/src/rsserver/rsaccounts.cc b/libretroshare/src/rsserver/rsaccounts.cc index 9bb8b49ed..c55fadd53 100644 --- a/libretroshare/src/rsserver/rsaccounts.cc +++ b/libretroshare/src/rsserver/rsaccounts.cc @@ -536,8 +536,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map & for(;dirIt.isValid();dirIt.next()) { /* check entry type */ - std::string fname; - dirIt.d_name(fname); + std::string fname = dirIt.file_name(); std::string fullname = mBaseDirectory + "/" + fname; #ifdef FIM_DEBUG std::cerr << "calling stats on " << fullname <d_name ; if(mFileName == "." || mFileName == "..") continue ; @@ -145,25 +145,6 @@ bool FolderIterator::readdir() #endif } -bool FolderIterator::d_name(std::string& dest) -{ - if(!validity) - return false; - -#ifdef WINDOWS_SYS - if(! ConvertUtf16ToUtf8(fileInfo.cFileName, dest)) { - validity = false; - return false; - } -#else - if(ent == 0) - return false; - dest = ent->d_name; -#endif - - return true; -} - time_t FolderIterator::dir_modtime() const { return mFolderModTime ; } const std::string& FolderIterator::file_fullpath() { return mFullPath ; } diff --git a/libretroshare/src/util/folderiterator.h b/libretroshare/src/util/folderiterator.h index 21ebacd06..f31e555f8 100644 --- a/libretroshare/src/util/folderiterator.h +++ b/libretroshare/src/util/folderiterator.h @@ -39,8 +39,6 @@ public: bool readdir(); void next(); -#warning this one should go, as it reports the same information than file_name() - bool d_name(std::string& dest); bool closedir(); const std::string& file_name() ; diff --git a/retroshare-nogui/src/introserver.cc b/retroshare-nogui/src/introserver.cc index 3734154b2..bc373c2da 100644 --- a/retroshare-nogui/src/introserver.cc +++ b/retroshare-nogui/src/introserver.cc @@ -293,8 +293,7 @@ int RsIntroServer::checkForNewCerts() for(;dirIt.isValid();dirIt.next()) { /* check entry type */ - std::string fname; - dirIt.d_name(fname); + std::string fname = dirIt.file_name(); std::string fullname = mCertLoadPath + "/" + fname; #ifdef RSIS_DEBUG std::cerr << "calling stats on " << fullname < Date: Sun, 9 Oct 2016 21:43:00 +0200 Subject: [PATCH 159/174] replaced explicit old code in cleanupDirectory() by new code based on FolderIterator --- libretroshare/src/util/rsdir.cc | 196 +------------------------------- 1 file changed, 5 insertions(+), 191 deletions(-) diff --git a/libretroshare/src/util/rsdir.cc b/libretroshare/src/util/rsdir.cc index dbf3c5bd6..c2ae4c916 100644 --- a/libretroshare/src/util/rsdir.cc +++ b/libretroshare/src/util/rsdir.cc @@ -34,6 +34,7 @@ #include "util/rsstring.h" #include "util/rsrandom.h" #include "util/rsmemory.h" +#include "util/folderiterator.h" #include "retroshare/rstypes.h" #include "rsthreads.h" #include @@ -437,200 +438,13 @@ bool RsDirUtil::checkCreateDirectory(const std::string& dir) bool RsDirUtil::cleanupDirectory(const std::string& cleandir, const std::set &keepFiles) { + for(librs::util::FolderIterator it(cleandir);it.isValid();it.next()) + if(it.file_type() == librs::util::FolderIterator::TYPE_FILE && (keepFiles.end() == std::find(keepFiles.begin(), keepFiles.end(), it.file_name()))) + remove( (cleandir + "/" + it.file_name()).c_str() ) ; - /* check for the dir existance */ -#ifdef WINDOWS_SYS - std::wstring wcleandir; - librs::util::ConvertUtf8ToUtf16(cleandir, wcleandir); - _WDIR *dir = _wopendir(wcleandir.c_str()); -#else - DIR *dir = opendir(cleandir.c_str()); -#endif - - - if (!dir) - { - return false; - } - -#ifdef WINDOWS_SYS - struct _wdirent *dent; - struct _stat buf; - - while(NULL != (dent = _wreaddir(dir))) -#else - struct dirent *dent; - struct stat buf; - - while(NULL != (dent = readdir(dir))) -#endif - { - /* check entry type */ -#ifdef WINDOWS_SYS - const std::wstring &wfname = dent -> d_name; - std::wstring wfullname = wcleandir + L"/" + wfname; -#else - const std::string &fname = dent -> d_name; - std::string fullname = cleandir + "/" + fname; -#endif - -#ifdef WINDOWS_SYS - if (-1 != _wstat(wfullname.c_str(), &buf)) -#else - if (-1 != stat(fullname.c_str(), &buf)) -#endif - { - /* only worry about files */ - if (S_ISREG(buf.st_mode)) - { -#ifdef WINDOWS_SYS - std::string fname; - librs::util::ConvertUtf16ToUtf8(wfname, fname); -#endif - /* check if we should keep it */ - if (keepFiles.end() == std::find(keepFiles.begin(), keepFiles.end(), fname)) - { - /* can remove */ -#ifdef WINDOWS_SYS - _wremove(wfullname.c_str()); -#else - remove(fullname.c_str()); -#endif - } - } - } - } - - /* close directory */ -#ifdef WINDOWS_SYS - _wclosedir(dir); -#else - closedir(dir); -#endif - - return true; + return true; } - - -/* faster cleanup - first construct two sets - then iterate over together */ -bool RsDirUtil::cleanupDirectoryFaster(const std::string& cleandir, const std::set &keepFiles) -{ - - /* check for the dir existance */ -#ifdef WINDOWS_SYS - std::map fileMap; - std::map::const_iterator fit; - - std::wstring wcleandir; - librs::util::ConvertUtf8ToUtf16(cleandir, wcleandir); - _WDIR *dir = _wopendir(wcleandir.c_str()); -#else - std::map fileMap; - std::map::const_iterator fit; - - DIR *dir = opendir(cleandir.c_str()); -#endif - - if (!dir) - { - return false; - } - -#ifdef WINDOWS_SYS - struct _wdirent *dent; - struct _stat buf; - - while(NULL != (dent = _wreaddir(dir))) - { - const std::wstring &wfname = dent -> d_name; - std::wstring wfullname = wcleandir + L"/" + wfname; - - if (-1 != _wstat(wfullname.c_str(), &buf)) - { - /* only worry about files */ - if (S_ISREG(buf.st_mode)) - { - std::string fname; - librs::util::ConvertUtf16ToUtf8(wfname, fname); - fileMap[fname] = wfullname; - } - } - } -#else - struct dirent *dent; - struct stat buf; - - while(NULL != (dent = readdir(dir))) - { - const std::string &fname = dent -> d_name; - std::string fullname = cleandir + "/" + fname; - - if (-1 != stat(fullname.c_str(), &buf)) - { - /* only worry about files */ - if (S_ISREG(buf.st_mode)) - { - fileMap[fname] = fullname; - } - } - } -#endif - - - - std::set::const_iterator kit; - - fit = fileMap.begin(); - kit = keepFiles.begin(); - - while(fit != fileMap.end() && kit != keepFiles.end()) - { - if (fit->first < *kit) // fit is not in keep list; - { -#ifdef WINDOWS_SYS - _wremove(fit->second.c_str()); -#else - remove(fit->second.c_str()); -#endif - ++fit; - } - else if (*kit < fit->first) // keepitem doesn't exist. - { - ++kit; - } - else // in keep list. - { - ++fit; - ++kit; - } - } - - // cleanup extra that aren't in keep list. - while(fit != fileMap.end()) - { -#ifdef WINDOWS_SYS - _wremove(fit->second.c_str()); -#else - remove(fit->second.c_str()); -#endif - ++fit; - } - - /* close directory */ -#ifdef WINDOWS_SYS - _wclosedir(dir); -#else - closedir(dir); -#endif - - return true; -} - - - - - /* slightly nicer helper function */ bool RsDirUtil::hashFile(const std::string& filepath, std::string &name, RsFileHash &hash, uint64_t &size) From 3c5e12ae8424582c0e0f6980ad9ff39e1f7aa484 Mon Sep 17 00:00:00 2001 From: cyril soler Date: Mon, 10 Oct 2016 11:29:03 +0200 Subject: [PATCH 160/174] fixed compilation on windows --- libretroshare/src/util/folderiterator.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/util/folderiterator.cc b/libretroshare/src/util/folderiterator.cc index 0918cf4a9..03edf3b13 100644 --- a/libretroshare/src/util/folderiterator.cc +++ b/libretroshare/src/util/folderiterator.cc @@ -63,7 +63,11 @@ void FolderIterator::next() { while(readdir()) { - mFileName = ent->d_name ; +#ifdef WINDOWS_SYS + ConvertUtf16ToUtf8(fileInfo.cFileName, mFileName) ; +#else + mFileName = ent->d_name ; +#endif if(mFileName == "." || mFileName == "..") continue ; From bd9a464d11796316146cdd694771e3952d4bea85 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 12 Oct 2016 23:20:38 +0200 Subject: [PATCH 161/174] added deterministic but unredictable hash generation for dir hashes, which should be preserved accross reboots. Should fix the msg from friends sending requests for the "wrong" dir hashes. --- .../src/file_sharing/dir_hierarchy.cc | 20 ++++++++++++---- .../src/file_sharing/dir_hierarchy.h | 4 ++-- .../src/file_sharing/directory_storage.cc | 4 ++-- .../src/file_sharing/directory_storage.h | 2 +- .../src/file_sharing/directory_updater.cc | 22 ++++++++++------- .../src/file_sharing/directory_updater.h | 5 ++++ .../src/file_sharing/file_sharing_defaults.h | 1 + libretroshare/src/file_sharing/p3filelists.cc | 24 +++++++++++++++++-- 8 files changed, 63 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 3e112ef95..26de88688 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -149,7 +149,7 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e) return e < mNodes.size() && mNodes[e] != NULL ; } -bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs) +bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs,const RsFileHash& random_hash_seed) { if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return false; @@ -190,7 +190,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage de->parent_index = indx; de->dir_modtime = 0;// forces parsing.it->second; de->dir_parent_path = d.dir_parent_path + "/" + d.dir_name ; - de->dir_hash = createDirHash(de->dir_name,de->dir_parent_path) ; + de->dir_hash = createDirHash(de->dir_name,d.dir_hash,random_hash_seed) ; mDirHashes[de->dir_hash] = mNodes.size() ; @@ -201,7 +201,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage return true; } -RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string &/*dir_name*/, const std::string &/*dir_parent_path*/) +RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string& dir_name, const RsFileHash& dir_parent_hash, const RsFileHash& random_hash_salt) { // What we need here: a unique identifier // - that cannot be bruteforced to find the real directory name and path @@ -215,7 +215,19 @@ RsFileHash InternalFileHierarchyStorage::createDirHash(const std::string &/*dir_ // Option 3: just compute something random, but then we need to store it so as to not // confuse friends when restarting. - return RsFileHash::random(); + RsTemporaryMemory mem(dir_name.size() + 2*RsFileHash::SIZE_IN_BYTES) ; + + memcpy( mem, random_hash_salt.toByteArray(),RsFileHash::SIZE_IN_BYTES) ; + memcpy(&mem[ RsFileHash::SIZE_IN_BYTES], dir_parent_hash.toByteArray(),RsFileHash::SIZE_IN_BYTES) ; + memcpy(&mem[2*RsFileHash::SIZE_IN_BYTES],dir_name.c_str(), dir_name.size()) ; + + RsFileHash res = RsDirUtil::sha1sum( mem,mem.size() ) ; + +#ifdef DEBUG_DIRECTORY_STORAGE + std::cerr << "Creating new dir hash for dir " << dir_name << ", parent dir hash=" << dir_parent_hash << " seed=[hidden]" << " result is " << res << std::endl; +#endif + + return res ; } bool InternalFileHierarchyStorage::removeDirectory(DirectoryStorage::EntryIndex indx) // no reference here! Very important. Otherwise, the messign we do inside can change the value of indx!! diff --git a/libretroshare/src/file_sharing/dir_hierarchy.h b/libretroshare/src/file_sharing/dir_hierarchy.h index fe937fac9..364b8f6d8 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.h +++ b/libretroshare/src/file_sharing/dir_hierarchy.h @@ -95,7 +95,7 @@ public: int parentRow(DirectoryStorage::EntryIndex e); bool isIndexValid(DirectoryStorage::EntryIndex e) const; bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const; - bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map& subdirs); + bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::map& subdirs, const RsFileHash &random_hash_seed); bool removeDirectory(DirectoryStorage::EntryIndex indx) ; bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const; bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map& subfiles,std::map& new_files); @@ -153,7 +153,7 @@ public: private: void recursPrint(int depth,DirectoryStorage::EntryIndex node) const; static bool nodeAccessError(const std::string& s); - static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ; + static RsFileHash createDirHash(const std::string& dir_name, const RsFileHash &dir_parent_hash, const RsFileHash &random_hash_salt) ; // Allocates a new entry in mNodes, possible re-using an empty slot and returns its index. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 7780717ca..2c8f48743 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -127,10 +127,10 @@ bool DirectoryStorage::setDirectoryUpdateTime (EntryIndex index,time_t update bool DirectoryStorage::setDirectoryRecursModTime(EntryIndex index,time_t rec_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); } bool DirectoryStorage::setDirectoryLocalModTime (EntryIndex index,time_t loc_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); } -bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs) +bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map& subdirs,const RsFileHash& hash_salt) { RS_STACK_MUTEX(mDirStorageMtx) ; - bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ; + bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs,hash_salt) ; locked_check() ; return res ; } diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index 2d8119445..c920a2df4 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -133,7 +133,7 @@ class DirectoryStorage // Sets the subdirectory/subfiles list of entry indx the supplied one, possible adding and removing directories (resp.files). New directories are set empty with // just a name and need to be updated later on. New files are returned in a list so that they can be sent to hash cache. // - bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs) ; + bool updateSubDirectoryList(const EntryIndex& indx, const std::map &subdirs, const RsFileHash &random_hash_salt) ; bool updateSubFilesList(const EntryIndex& indx, const std::map &subfiles, std::map &new_files) ; bool removeDirectory(const EntryIndex& indx) ; diff --git a/libretroshare/src/file_sharing/directory_updater.cc b/libretroshare/src/file_sharing/directory_updater.cc index 41c4a941e..de92c6ee1 100644 --- a/libretroshare/src/file_sharing/directory_updater.cc +++ b/libretroshare/src/file_sharing/directory_updater.cc @@ -55,7 +55,7 @@ void LocalDirectoryUpdater::setEnabled(bool b) return ; if(b) - start("fs dir updater") ; + start("fs dir updater") ; else shutdown(); @@ -88,17 +88,23 @@ void LocalDirectoryUpdater::forceUpdate() void LocalDirectoryUpdater::sweepSharedDirectories() { + if(mHashSalt.isNull()) + { + std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl; + return ; + } + RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); #ifdef DEBUG_LOCAL_DIR_UPDATER std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl; #endif - // recursive update algorithm works that way: - // - the external loop starts on the shared directory list and goes through sub-directories + // recursive update algorithm works that way: + // - the external loop starts on the shared directory list and goes through sub-directories // - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure. - // - the information that is costly to compute (the hash) is store externally into a separate structure. - // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. - // + // - the information that is costly to compute (the hash) is store externally into a separate structure. + // - doing so, changing directory names or moving files between directories does not cause a re-hash of the content. + // std::list shared_directory_list ; mSharedDirectories->getSharedDirectoryList(shared_directory_list); @@ -109,7 +115,7 @@ void LocalDirectoryUpdater::sweepSharedDirectories() // make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order. - mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ; + mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ; // now for each of them, go recursively and match both files and dirs @@ -177,7 +183,7 @@ void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_p // update file and dir lists for current directory. - mSharedDirectories->updateSubDirectoryList(indx,subdirs) ; + mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt) ; std::map new_files ; mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ; diff --git a/libretroshare/src/file_sharing/directory_updater.h b/libretroshare/src/file_sharing/directory_updater.h index d0f920cb0..85df00d6c 100644 --- a/libretroshare/src/file_sharing/directory_updater.h +++ b/libretroshare/src/file_sharing/directory_updater.h @@ -41,6 +41,9 @@ public: void forceUpdate(); bool inDirectoryCheck() const ; + void setHashSalt(const RsFileHash& hash) { mHashSalt = hash; } + const RsFileHash& hashSalt() const { return mHashSalt; } + void setFileWatchPeriod(int seconds) ; uint32_t fileWatchPeriod() const ; @@ -60,6 +63,8 @@ private: HashStorage *mHashCache ; LocalDirectoryStorage *mSharedDirectories ; + RsFileHash mHashSalt ; + time_t mLastSweepTime; time_t mLastTSUpdateTime; diff --git a/libretroshare/src/file_sharing/file_sharing_defaults.h b/libretroshare/src/file_sharing/file_sharing_defaults.h index 94e2981f7..520fc61ea 100644 --- a/libretroshare/src/file_sharing/file_sharing_defaults.h +++ b/libretroshare/src/file_sharing/file_sharing_defaults.h @@ -33,6 +33,7 @@ static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 se static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch +static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc. static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache. diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index f8f87c701..a35de0e79 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -326,7 +326,14 @@ cleanup = true; rskv->tlvkvs.pairs.push_back(kv); } + { + RsTlvKeyValue kv; + kv.key = WATCH_HASH_SALT_SS; + kv.value = mLocalDirWatcher->hashSalt().toStdString(); + + rskv->tlvkvs.pairs.push_back(kv); + } /* Add KeyValue to saveList */ sList.push_back(rskv); @@ -373,6 +380,11 @@ bool p3FileDatabase::loadList(std::list& load) { setWatchEnabled(kit->value == "YES") ; } + else if(kit->key == WATCH_HASH_SALT_SS) + { + std::cerr << "Initing directory watcher with saved secret salt..." << std::endl; + mLocalDirWatcher->setHashSalt(RsFileHash(kit->value)) ; + } delete *it ; continue ; } @@ -398,6 +410,14 @@ bool p3FileDatabase::loadList(std::list& load) delete *it ; } + if(mLocalDirWatcher->hashSalt().isNull()) + { + std::cerr << "(WW) Initialising directory watcher salt to some random value " << std::endl; + mLocalDirWatcher->setHashSalt(RsFileHash::random()) ; + + IndicateConfigChanged(); + } + /* set directories */ mLocalSharedDirs->setSharedDirectoryList(dirList); @@ -642,8 +662,8 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); return true; } - else - return false; + else + return false; uint32_t fi; DirectoryStorage::EntryIndex e ; From 10c269ec5a5e9c9ea0aa252de227f914cfc68a3a Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 12 Oct 2016 23:31:32 +0200 Subject: [PATCH 162/174] removed warning when friend directory list cannot be found --- libretroshare/src/file_sharing/dir_hierarchy.cc | 2 ++ libretroshare/src/file_sharing/directory_storage.cc | 4 ++-- libretroshare/src/file_sharing/directory_storage.h | 2 +- libretroshare/src/file_sharing/filelist_io.cc | 2 ++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 26de88688..513ead3a5 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -1144,7 +1144,9 @@ bool InternalFileHierarchyStorage::load(const std::string& fname) } catch(read_error& e) { +#ifdef DEBUG_DIRECTORY_STORAGE std::cerr << "Error while reading: " << e.what() << std::endl; +#endif if(buffer != NULL) free(buffer) ; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 2c8f48743..6a4d3e93e 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -174,10 +174,10 @@ int DirectoryStorage::searchHash(const RsFileHash& hash, std::list & return mFileHierarchy->searchHash(hash,results); } -void DirectoryStorage::load(const std::string& local_file_name) +bool DirectoryStorage::load(const std::string& local_file_name) { RS_STACK_MUTEX(mDirStorageMtx) ; - mFileHierarchy->load(local_file_name); + return mFileHierarchy->load(local_file_name); } void DirectoryStorage::save(const std::string& local_file_name) { diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index c920a2df4..195573266 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -152,7 +152,7 @@ class DirectoryStorage void cleanup(); protected: - void load(const std::string& local_file_name) ; + bool load(const std::string& local_file_name) ; void save(const std::string& local_file_name) ; private: diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc index 8a1693c46..7be770c82 100644 --- a/libretroshare/src/file_sharing/filelist_io.cc +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -190,7 +190,9 @@ bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned cha if(!RsDirUtil::checkFile( fname,file_size,false ) ) { +#ifdef FIM_DEBUG std::cerr << "Encrypted file " << fname << " not available." << std::endl; +#endif return false; } From 633a6cf8c2fc40b4eb40b8dcb2778b5dfa78b550 Mon Sep 17 00:00:00 2001 From: Gio Date: Wed, 12 Oct 2016 20:43:38 +0200 Subject: [PATCH 163/174] Fixed a bunch of warnings in safe ways --- libresapi/src/api/IdentityHandler.cpp | 2 +- libresapi/src/api/json.cpp | 2 - libretroshare/src/file_sharing/hash_cache.cc | 2 +- libretroshare/src/file_sharing/p3filelists.cc | 39 ++++++++++--------- libretroshare/src/ft/ftcontroller.cc | 6 +-- libretroshare/src/ft/ftserver.cc | 2 +- libretroshare/src/gxs/rsgenexchange.cc | 10 ++--- libretroshare/src/rsserver/rsaccounts.cc | 2 +- libretroshare/src/serialiser/rsbaseserial.cc | 2 +- libretroshare/src/services/p3banlist.cc | 16 ++++---- retroshare-gui/src/gui/FriendsDialog.cpp | 2 +- .../src/gui/gxs/GxsMessageFramePostWidget.cpp | 2 +- retroshare-nogui/src/TerminalApiClient.cpp | 2 +- 13 files changed, 41 insertions(+), 48 deletions(-) diff --git a/libresapi/src/api/IdentityHandler.cpp b/libresapi/src/api/IdentityHandler.cpp index 6d017fad8..f707e3596 100644 --- a/libresapi/src/api/IdentityHandler.cpp +++ b/libresapi/src/api/IdentityHandler.cpp @@ -29,7 +29,7 @@ private: std::vector mIds; StateToken mStateToken; protected: - virtual void gxsDoWork(Request &req, Response &resp) + virtual void gxsDoWork(Request& /*req*/, Response &resp) { resp.mDataStream.getStreamToMember(); for(std::vector::iterator vit = mIds.begin(); vit != mIds.end(); ++vit) diff --git a/libresapi/src/api/json.cpp b/libresapi/src/api/json.cpp index d921e6b80..2bcec7e81 100644 --- a/libresapi/src/api/json.cpp +++ b/libresapi/src/api/json.cpp @@ -824,7 +824,6 @@ static Value DeserializeValue(std::string& str, bool* had_error, std::stack::iterator it(mFiles.begin());it!=mFiles.end();) - if(it->second.time_stamp + duration < (uint64_t)now) + if((uint64_t)(it->second.time_stamp + duration) < (uint64_t)now) { #ifdef HASHSTORAGE_DEBUG std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ; diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index f8f87c701..0b1335410 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -621,29 +621,30 @@ void p3FileDatabase::requestDirUpdate(void *ref) } } -bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const +bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result, + FileSearchFlags flags ) const { - RS_STACK_MUTEX(mFLSMtx) ; + RS_STACK_MUTEX(mFLSMtx); - result = NULL ; + result = NULL; - if (ref == NULL) - if(flags & RS_FILE_HINTS_LOCAL) - { - if(row != 0) - return false ; + if (ref == NULL) + { + if(flags & RS_FILE_HINTS_LOCAL) + { + if(row != 0) return false; - convertEntryIndexToPointer(0,0,result); + convertEntryIndexToPointer(0,0,result); - return true ; - } - else if((uint32_t)row < mRemoteDirectories.size()) - { - convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); - return true; - } - else - return false; + return true; + } + else if((uint32_t)row < mRemoteDirectories.size()) + { + convertEntryIndexToPointer(mRemoteDirectories[row]->root(), row+1, result); + return true; + } + else return false; + } uint32_t fi; DirectoryStorage::EntryIndex e ; @@ -667,8 +668,8 @@ bool p3FileDatabase::findChildPointer(void *ref, int row, void *& result, FileSe return res; } -// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files. +// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files. int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const { RS_STACK_MUTEX(mFLSMtx) ; diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index ae90dfe50..821bd1ea0 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -727,14 +727,13 @@ bool ftController::completeFile(const RsFileHash& hash) std::string path; std::string name; uint64_t size = 0; - uint32_t state = 0; uint32_t period = 0; TransferRequestFlags flags ; TransferRequestFlags extraflags ; uint32_t completeCount = 0; { - RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ + RS_STACK_MUTEX(ctrlMutex); #ifdef CONTROL_DEBUG std::cerr << "ftController:completeFile(" << hash << ")"; @@ -816,7 +815,6 @@ bool ftController::completeFile(const RsFileHash& hash) name = fc->mName; //hash = fc->mHash; size = fc->mSize; - state = fc->mState; period = 30 * 24 * 3600; /* 30 days */ extraflags.clear() ; @@ -837,7 +835,7 @@ bool ftController::completeFile(const RsFileHash& hash) if(flags & RS_FILE_REQ_ANONYMOUS_ROUTING) mTurtle->stopMonitoringTunnels(hash_to_suppress) ; - } /******* UNLOCKED ********/ + } // UNLOCK: RS_STACK_MUTEX(ctrlMutex); /******************** NO Mutex from Now ******************** diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 832530ade..f259b388f 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -543,7 +543,7 @@ int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags { return mFileDatabase->RequestDirDetails(ref,details,flags) ; } -uint32_t ftServer::getType(void *ref, FileSearchFlags flags) +uint32_t ftServer::getType(void *ref, FileSearchFlags /*flags*/) { return mFileDatabase->getType(ref) ; } diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index d2ebedd85..3ff86cdf6 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1691,7 +1691,7 @@ void RsGenExchange::processMsgMetaChanges() { MsgLocMetaData& m = mit->second; - int32_t value, mask; + int32_t value, mask; bool ok = true; bool changed = false; @@ -1717,7 +1717,7 @@ void RsGenExchange::processMsgMetaChanges() { RsGxsMsgMetaData* meta = *(msgMetaV.begin()); value = (meta->mMsgStatus & ~mask) | (mask & value); - changed = (meta->mMsgStatus != value); + changed = (static_cast(meta->mMsgStatus) != value); m.val.put(RsGeneralDataService::MSG_META_STATUS, value); delete meta; ok = true; @@ -2822,10 +2822,8 @@ void RsGenExchange::processRecvdMessages() mNetService->rejectMessage(*it) ; } -bool RsGenExchange::acceptNewGroup(const RsGxsGrpMetaData *grpMeta) -{ - return true; -} +bool RsGenExchange::acceptNewGroup(const RsGxsGrpMetaData* /*grpMeta*/ ) +{ return true; } void RsGenExchange::processRecvdGroups() { diff --git a/libretroshare/src/rsserver/rsaccounts.cc b/libretroshare/src/rsserver/rsaccounts.cc index c55fadd53..09c3e7d63 100644 --- a/libretroshare/src/rsserver/rsaccounts.cc +++ b/libretroshare/src/rsserver/rsaccounts.cc @@ -1059,7 +1059,7 @@ bool RsAccountsDetail::GenerateSSLCertificate(const RsPgpId& pgp_id, const s X509_print_ex(bio_out, x509, nmflag, reqflag); - BIO_flush(bio_out); + (void) BIO_flush(bio_out); BIO_free(bio_out); /* Save cert to file */ diff --git a/libretroshare/src/serialiser/rsbaseserial.cc b/libretroshare/src/serialiser/rsbaseserial.cc index 283c4f50e..b34c9f1fd 100644 --- a/libretroshare/src/serialiser/rsbaseserial.cc +++ b/libretroshare/src/serialiser/rsbaseserial.cc @@ -231,7 +231,7 @@ uint32_t getRawStringSize(const std::string &outStr) bool getRawString(void *data, uint32_t size, uint32_t *offset, std::string &outStr) { -#warning I had to change this. It seems like a bug to not clear the string. Should make sure it's not introducing any side effect. +#warning "I had to change this. It seems like a bug to not clear the string. Should make sure it's not introducing any side effect." outStr.clear(); uint32_t len = 0; diff --git a/libretroshare/src/services/p3banlist.cc b/libretroshare/src/services/p3banlist.cc index 9595510bd..fd4540f0a 100644 --- a/libretroshare/src/services/p3banlist.cc +++ b/libretroshare/src/services/p3banlist.cc @@ -206,27 +206,25 @@ void p3BanList::autoFigureOutBanRanges() { RS_STACK_MUTEX(mBanMtx) ; - bool changed = false ; - // clear automatic ban ranges - for(std::map::iterator it(mBanRanges.begin());it!=mBanRanges.end();) + for(std::map::iterator it(mBanRanges.begin()); + it!=mBanRanges.end(); ) + { if(it->second.reason == RSBANLIST_REASON_AUTO_RANGE) { std::map::iterator it2=it ; ++it2 ; mBanRanges.erase(it) ; it=it2 ; - - changed = true ; } - else - ++it; + else ++it; + } IndicateConfigChanged(); - if(!mAutoRangeIps) - return ; + if(!mAutoRangeIps) return; + #ifdef DEBUG_BANLIST std::cerr << "Automatically figuring out IP ranges from banned IPs." << std::endl; #endif diff --git a/retroshare-gui/src/gui/FriendsDialog.cpp b/retroshare-gui/src/gui/FriendsDialog.cpp index 4590ccdc5..52d25d5e3 100644 --- a/retroshare-gui/src/gui/FriendsDialog.cpp +++ b/retroshare-gui/src/gui/FriendsDialog.cpp @@ -54,7 +54,7 @@ #define IMAGE_NETWORK2 ":/icons/png/netgraph.png" #define IMAGE_PEERS ":/icons/png/keyring.png" #define IMAGE_IDENTITY ":/images/identity/identities_32.png" -#define IMAGE_CIRCLES ":/icons/png/circles.png" +//#define IMAGE_CIRCLES ":/icons/png/circles.png" /****** * #define FRIENDS_DEBUG 1 diff --git a/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.cpp b/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.cpp index f9808a4d9..df5442bcd 100644 --- a/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.cpp +++ b/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.cpp @@ -59,7 +59,7 @@ void GxsMessageFramePostWidget::groupIdChanged() fillComplete(); } -QString GxsMessageFramePostWidget::groupName(bool withUnreadCount) +QString GxsMessageFramePostWidget::groupName(bool /*withUnreadCount*/) { QString name = groupId().isNull () ? tr("No name") : mGroupName; diff --git a/retroshare-nogui/src/TerminalApiClient.cpp b/retroshare-nogui/src/TerminalApiClient.cpp index 6822004b4..6a08218c1 100644 --- a/retroshare-nogui/src/TerminalApiClient.cpp +++ b/retroshare-nogui/src/TerminalApiClient.cpp @@ -246,7 +246,7 @@ void TerminalApiClient::data_tick() if(!ask_for_password && runstate == "waiting_account_select" && last_char >= '0' && last_char <= '9' - && (last_char-'0') < accounts.size()) + && static_cast(last_char-'0') < accounts.size()) { std::string acc = accounts[last_char-'0']; JsonStream reqs; From 4db6d6b56f303fa963a7e3a607fe19ecd549b637 Mon Sep 17 00:00:00 2001 From: Phenom Date: Thu, 13 Oct 2016 13:26:44 +0200 Subject: [PATCH 164/174] Fix the "Desktop file is missing" shown in option page. --- retroshare-gui/src/gui/settings/rsharesettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/settings/rsharesettings.cpp b/retroshare-gui/src/gui/settings/rsharesettings.cpp index fb7dc9dad..1d0507337 100644 --- a/retroshare-gui/src/gui/settings/rsharesettings.cpp +++ b/retroshare-gui/src/gui/settings/rsharesettings.cpp @@ -802,7 +802,7 @@ bool RshareSettings::getRetroShareProtocol() } desktop.close(); if (lines.contains("Exec=" + getAppPathForProtocol())) - if (lines.contains("MimeType=x-scheme-handler/retroshare")) + if (lines.contains("MimeType=x-scheme-handler/retroshare;")) return true; } #else From 8483c0d0caa759efa20a2a8530c66cf776d569ae Mon Sep 17 00:00:00 2001 From: Phenom Date: Sun, 16 Oct 2016 15:48:50 +0200 Subject: [PATCH 165/174] Add ElidedLabel to GroupTreeWidget --- retroshare-gui/src/gui/common/ElidedLabel.cpp | 14 +- retroshare-gui/src/gui/common/ElidedLabel.h | 5 + retroshare-gui/src/gui/common/FriendList.cpp | 1 + .../src/gui/common/GroupTreeWidget.cpp | 296 +++++++++++------- 4 files changed, 195 insertions(+), 121 deletions(-) diff --git a/retroshare-gui/src/gui/common/ElidedLabel.cpp b/retroshare-gui/src/gui/common/ElidedLabel.cpp index 934756dc4..0dd4f69b6 100644 --- a/retroshare-gui/src/gui/common/ElidedLabel.cpp +++ b/retroshare-gui/src/gui/common/ElidedLabel.cpp @@ -52,7 +52,9 @@ ElidedLabel::ElidedLabel(const QString &text, QWidget *parent) , mElided(false) , mOnlyPlainText(false) , mContent(text) + , mTextColor(QColor()) { + setStyleSheet("background-color: rgba(0,0,0,0%)"); mRectElision = QRect(); setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred); } @@ -62,7 +64,9 @@ ElidedLabel::ElidedLabel(QWidget *parent) , mElided(false) , mOnlyPlainText(false) , mContent("") + , mTextColor(QColor()) { + setStyleSheet("background-color: rgba(0,0,0,0%)"); mRectElision = QRect(); setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred); } @@ -94,7 +98,7 @@ void ElidedLabel::paintEvent(QPaintEvent *event) QFontMetrics fontMetrics = painter.fontMetrics(); QRect cr = contentsRect(); cr.adjust(margin(), margin(), -margin(), -margin()); - + bool didElide = false; QChar ellipsisChar(0x2026);//= "…" int lineSpacing = fontMetrics.lineSpacing(); @@ -225,3 +229,11 @@ void ElidedLabel::mousePressEvent(QMouseEvent *ev) } QLabel::mousePressEvent(ev); } + +void ElidedLabel::setTextColor(const QColor &color) +{ + QPalette tmpPalette = palette(); + tmpPalette.setColor(foregroundRole(), color); + setPalette(tmpPalette); + mTextColor = color; +} diff --git a/retroshare-gui/src/gui/common/ElidedLabel.h b/retroshare-gui/src/gui/common/ElidedLabel.h index db51aef79..a8110a857 100644 --- a/retroshare-gui/src/gui/common/ElidedLabel.h +++ b/retroshare-gui/src/gui/common/ElidedLabel.h @@ -53,6 +53,7 @@ class ElidedLabel : public QLabel Q_PROPERTY(QString text READ text WRITE setText) Q_PROPERTY(bool isElided READ isElided) Q_PROPERTY(bool isOnlyPlainText READ isOnlyPlainText WRITE setOnlyPlainText) + Q_PROPERTY(QColor textColor READ textColor WRITE setTextColor) public: ElidedLabel(const QString &text, QWidget *parent = 0); @@ -62,6 +63,9 @@ public: bool isElided() const { return mElided; } bool isOnlyPlainText() const { return mOnlyPlainText; } + QColor textColor() const { return mTextColor; } + void setTextColor(const QColor &color); + public slots: void setText(const QString &text); void setOnlyPlainText(const bool &value); @@ -79,6 +83,7 @@ private: bool mOnlyPlainText; QString mContent; QRect mRectElision; + QColor mTextColor; }; #endif // ELIDEDLABEL_H diff --git a/retroshare-gui/src/gui/common/FriendList.cpp b/retroshare-gui/src/gui/common/FriendList.cpp index e61e8a568..643e9d482 100644 --- a/retroshare-gui/src/gui/common/FriendList.cpp +++ b/retroshare-gui/src/gui/common/FriendList.cpp @@ -510,6 +510,7 @@ static void getNameWidget(QTreeWidget *treeWidget, QTreeWidgetItem *item, Elided if (!widget) { widget = new QWidget; + widget->setAttribute(Qt::WA_TranslucentBackground); nameLabel = new ElidedLabel(widget); textLabel = new ElidedLabel(widget); diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index b396083b5..12408b92a 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -18,20 +18,22 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. ****************************************************************/ - -#include -#include -#include -#include - -#include "retroshare/rsgxsflags.h" #include "GroupTreeWidget.h" #include "ui_GroupTreeWidget.h" -#include "RSItemDelegate.h" +#include +#include +#include +#include +#include + +#include "retroshare/rsgxsflags.h" + #include "PopularityDefs.h" -#include "gui/settings/rsharesettings.h" +#include "RSItemDelegate.h" #include "RSTreeWidgetItem.h" +#include "gui/common/ElidedLabel.h" +#include "gui/settings/rsharesettings.h" #include "util/QtVersion.h" #include @@ -55,6 +57,9 @@ #define FILTER_NAME_INDEX 0 #define FILTER_DESC_INDEX 1 +Q_DECLARE_METATYPE(ElidedLabel*) +Q_DECLARE_METATYPE(QLabel*) + GroupTreeWidget::GroupTreeWidget(QWidget *parent) : QWidget(parent), ui(new Ui::GroupTreeWidget) { @@ -116,6 +121,38 @@ GroupTreeWidget::~GroupTreeWidget() delete ui; } +static void getNameWidget(QTreeWidget *treeWidget, QTreeWidgetItem *item, ElidedLabel *&nameLabel, QLabel *&waitLabel) +{ + QWidget *widget = treeWidget->itemWidget(item, COLUMN_NAME); + + if (!widget) { + widget = new QWidget; + widget->setAttribute(Qt::WA_TranslucentBackground); + nameLabel = new ElidedLabel(widget); + waitLabel = new QLabel(widget); + QMovie *movie = new QMovie(":/images/loader/circleball-16.gif"); + waitLabel->setMovie(movie); + waitLabel->setHidden(true); + + widget->setProperty("nameLabel", qVariantFromValue(nameLabel)); + widget->setProperty("waitLabel", qVariantFromValue(waitLabel)); + + QHBoxLayout *layout = new QHBoxLayout; + layout->setSpacing(0); + layout->setContentsMargins(0, 0, 0, 0); + + layout->addWidget(nameLabel); + layout->addWidget(waitLabel); + + widget->setLayout(layout); + + treeWidget->setItemWidget(item, COLUMN_NAME, widget); + } else { + nameLabel = widget->property("nameLabel").value(); + waitLabel = widget->property("waitLabel").value(); + } +} + void GroupTreeWidget::changeEvent(QEvent *e) { QWidget::changeEvent(e); @@ -229,7 +266,7 @@ void GroupTreeWidget::initDisplayMenu(QToolButton *toolButton) actionSortByLastPost = displayMenu->addAction(QIcon(), tr("Sort by Last Post"), this, SLOT(sort())); actionSortByLastPost->setCheckable(true); actionSortByLastPost->setActionGroup(actionGroup); - + actionSortByPosts = displayMenu->addAction(QIcon(), tr("Sort by Posts"), this, SLOT(sort())); actionSortByPosts->setCheckable(true); actionSortByPosts->setActionGroup(actionGroup); @@ -255,6 +292,11 @@ void GroupTreeWidget::updateColors() } item->setForeground(COLUMN_NAME, brush); + + ElidedLabel *nameLabel = NULL; + QLabel *waitLabel = NULL; + getNameWidget(ui->treeWidget, item, nameLabel, waitLabel); + nameLabel->setTextColor(brush.color()); } } @@ -293,21 +335,27 @@ QTreeWidgetItem *GroupTreeWidget::addCategoryItem(const QString &name, const QIc { QFont font; QTreeWidgetItem *item = new QTreeWidgetItem(); + ui->treeWidget->addTopLevelItem(item); + + ElidedLabel *nameLabel = NULL; + QLabel *waitLabel = NULL; + getNameWidget(ui->treeWidget, item, nameLabel, waitLabel); + + nameLabel->setText(name); + item->setData(COLUMN_DATA, ROLE_NAME, name); font = item->font(COLUMN_NAME); font.setBold(true); - item->setText(COLUMN_NAME, name); - item->setData(COLUMN_DATA, ROLE_NAME, name); item->setFont(COLUMN_NAME, font); + nameLabel->setFont(font); item->setIcon(COLUMN_NAME, icon); - int S = QFontMetricsF(font).height(); + int S = QFontMetricsF(font).height(); - item->setSizeHint(COLUMN_NAME, QSize(S*1.1, S*1.1)); + item->setSizeHint(COLUMN_NAME, QSize(S*1.1, S*1.1)); item->setForeground(COLUMN_NAME, QBrush(textColorCategory())); + nameLabel->setTextColor(textColorCategory()); item->setData(COLUMN_DATA, ROLE_COLOR, GROUPTREEWIDGET_COLOR_CATEGORY); - ui->treeWidget->addTopLevelItem(item); - item->setExpanded(expand); return item; @@ -334,114 +382,119 @@ QString GroupTreeWidget::itemIdAt(QPoint &point) void GroupTreeWidget::fillGroupItems(QTreeWidgetItem *categoryItem, const QList &itemList) { - if (categoryItem == NULL) { - return; - } + if (categoryItem == NULL) { + return; + } - QString filterText = ui->filterLineEdit->text(); + QString filterText = ui->filterLineEdit->text(); - /* Iterate all items */ - QList::const_iterator it; - for (it = itemList.begin(); it != itemList.end(); ++it) { - const GroupItemInfo &itemInfo = *it; + /* Iterate all items */ + QList::const_iterator it; + for (it = itemList.begin(); it != itemList.end(); ++it) { + const GroupItemInfo &itemInfo = *it; - QTreeWidgetItem *item = NULL; + QTreeWidgetItem *item = NULL; - /* Search exisiting item */ - int childCount = categoryItem->childCount(); - for (int child = 0; child < childCount; ++child) { - QTreeWidgetItem *childItem = categoryItem->child(child); - if (childItem->data(COLUMN_DATA, ROLE_ID).toString() == itemInfo.id) { - /* Found child */ - item = childItem; - break; - } - } + /* Search exisiting item */ + int childCount = categoryItem->childCount(); + for (int child = 0; child < childCount; ++child) { + QTreeWidgetItem *childItem = categoryItem->child(child); + if (childItem->data(COLUMN_DATA, ROLE_ID).toString() == itemInfo.id) { + /* Found child */ + item = childItem; + break; + } + } - if (item == NULL) { - item = new RSTreeWidgetItem(compareRole); - item->setData(COLUMN_DATA, ROLE_ID, itemInfo.id); - categoryItem->addChild(item); - } + if (item == NULL) { + item = new RSTreeWidgetItem(compareRole); + item->setData(COLUMN_DATA, ROLE_ID, itemInfo.id); + categoryItem->addChild(item); + } - item->setText(COLUMN_NAME, itemInfo.name); - item->setData(COLUMN_DATA, ROLE_NAME, itemInfo.name); - item->setData(COLUMN_DATA, ROLE_DESCRIPTION, itemInfo.description); + ElidedLabel *nameLabel = NULL; + QLabel *waitLabel = NULL; + getNameWidget(ui->treeWidget, item, nameLabel, waitLabel); - /* Set last post */ - qlonglong lastPost = itemInfo.lastpost.toTime_t(); - item->setData(COLUMN_DATA, ROLE_LASTPOST, -lastPost); // negative for correct sorting + nameLabel->setText(itemInfo.name); + item->setData(COLUMN_DATA, ROLE_NAME, itemInfo.name); + item->setData(COLUMN_DATA, ROLE_DESCRIPTION, itemInfo.description); - /* Set visible posts */ - item->setData(COLUMN_DATA, ROLE_POSTS, -itemInfo.max_visible_posts);// negative for correct sorting + /* Set last post */ + qlonglong lastPost = itemInfo.lastpost.toTime_t(); + item->setData(COLUMN_DATA, ROLE_LASTPOST, -lastPost); // negative for correct sorting - /* Set icon */ - if (ui->treeWidget->itemWidget(item, COLUMN_NAME)) { - /* Item is waiting, save icon in role */ - item->setData(COLUMN_DATA, ROLE_SAVED_ICON, itemInfo.icon); - } else { - item->setIcon(COLUMN_NAME, itemInfo.icon); - } + /* Set visible posts */ + item->setData(COLUMN_DATA, ROLE_POSTS, -itemInfo.max_visible_posts);// negative for correct sorting - /* Set popularity */ - QString tooltip = PopularityDefs::tooltip(itemInfo.popularity); + /* Set icon */ + if (waitLabel->isVisible()) { + /* Item is waiting, save icon in role */ + item->setData(COLUMN_DATA, ROLE_SAVED_ICON, itemInfo.icon); + } else { + item->setIcon(COLUMN_NAME, itemInfo.icon); + } - item->setIcon(COLUMN_POPULARITY, PopularityDefs::icon(itemInfo.popularity)); - item->setData(COLUMN_DATA, ROLE_POPULARITY, -itemInfo.popularity); // negative for correct sorting + /* Set popularity */ + QString tooltip = PopularityDefs::tooltip(itemInfo.popularity); - /* Set tooltip */ - if (itemInfo.adminKey) - tooltip += "\n" + tr("You are admin (modify names and description using Edit menu)"); - else if (itemInfo.publishKey) - tooltip += "\n" + tr("You have been granted as publisher (you can post here!)"); + item->setIcon(COLUMN_POPULARITY, PopularityDefs::icon(itemInfo.popularity)); + item->setData(COLUMN_DATA, ROLE_POPULARITY, -itemInfo.popularity); // negative for correct sorting - if(!IS_GROUP_SUBSCRIBED(itemInfo.subscribeFlags)) - { - tooltip += "\n" + QString::number(itemInfo.max_visible_posts) + " messages available" ; - tooltip += "\n" + tr("Subscribe to download and read messages") ; - } + /* Set tooltip */ + if (itemInfo.adminKey) + tooltip += "\n" + tr("You are admin (modify names and description using Edit menu)"); + else if (itemInfo.publishKey) + tooltip += "\n" + tr("You have been granted as publisher (you can post here!)"); - item->setToolTip(COLUMN_NAME, tooltip); - item->setToolTip(COLUMN_POPULARITY, tooltip); + if(!IS_GROUP_SUBSCRIBED(itemInfo.subscribeFlags)) + { + tooltip += "\n" + QString::number(itemInfo.max_visible_posts) + " messages available" ; + tooltip += "\n" + tr("Subscribe to download and read messages") ; + } - item->setData(COLUMN_DATA, ROLE_SUBSCRIBE_FLAGS, itemInfo.subscribeFlags); + item->setToolTip(COLUMN_NAME, tooltip); + item->setToolTip(COLUMN_POPULARITY, tooltip); - /* Set color */ - QBrush brush; - if (itemInfo.publishKey) { - brush = QBrush(textColorPrivateKey()); - item->setData(COLUMN_DATA, ROLE_COLOR, GROUPTREEWIDGET_COLOR_PRIVATEKEY); - } else { - brush = ui->treeWidget->palette().color(QPalette::Text); - item->setData(COLUMN_DATA, ROLE_COLOR, GROUPTREEWIDGET_COLOR_STANDARD); - } - item->setForeground(COLUMN_NAME, brush); + item->setData(COLUMN_DATA, ROLE_SUBSCRIBE_FLAGS, itemInfo.subscribeFlags); - /* Calculate score */ - calculateScore(item, filterText); - } + /* Set color */ + QBrush brush; + if (itemInfo.publishKey) { + brush = QBrush(textColorPrivateKey()); + item->setData(COLUMN_DATA, ROLE_COLOR, GROUPTREEWIDGET_COLOR_PRIVATEKEY); + } else { + brush = ui->treeWidget->palette().color(QPalette::Text); + item->setData(COLUMN_DATA, ROLE_COLOR, GROUPTREEWIDGET_COLOR_STANDARD); + } + item->setForeground(COLUMN_NAME, brush); + nameLabel->setTextColor(brush.color()); - /* Remove all items not in list */ - int child = 0; - int childCount = categoryItem->childCount(); - while (child < childCount) { - QString id = categoryItem->child(child)->data(COLUMN_DATA, ROLE_ID).toString(); + /* Calculate score */ + calculateScore(item, filterText); + } - for (it = itemList.begin(); it != itemList.end(); ++it) { - if (it->id == id) { - break; - } - } + /* Remove all items not in list */ + int child = 0; + int childCount = categoryItem->childCount(); + while (child < childCount) { + QString id = categoryItem->child(child)->data(COLUMN_DATA, ROLE_ID).toString(); - if (it == itemList.end()) { - delete(categoryItem->takeChild(child)); - childCount = categoryItem->childCount(); - } else { - ++child; - } - } + for (it = itemList.begin(); it != itemList.end(); ++it) { + if (it->id == id) { + break; + } + } - resort(categoryItem); + if (it == itemList.end()) { + delete(categoryItem->takeChild(child)); + childCount = categoryItem->childCount(); + } else { + ++child; + } + } + + resort(categoryItem); } void GroupTreeWidget::setUnreadCount(QTreeWidgetItem *item, int unreadCount) @@ -449,19 +502,22 @@ void GroupTreeWidget::setUnreadCount(QTreeWidgetItem *item, int unreadCount) if (item == NULL) { return; } + ElidedLabel *nameLabel = NULL; + QLabel *waitLabel = NULL; + getNameWidget(ui->treeWidget, item, nameLabel, waitLabel); QString name = item->data(COLUMN_DATA, ROLE_NAME).toString(); - QFont font = item->font(COLUMN_NAME); + QFont font = nameLabel->font(); if (unreadCount) { - name += QString(" (%1)").arg(unreadCount); + name = QString("(%1) ").arg(unreadCount) + name; font.setBold(true); } else { font.setBold(false); } - item->setText(COLUMN_NAME, name); - item->setFont(COLUMN_NAME, font); + nameLabel->setText(name); + nameLabel->setFont(font); } QTreeWidgetItem *GroupTreeWidget::getItemFromId(const QString &id) @@ -507,9 +563,11 @@ bool GroupTreeWidget::setWaiting(const QString &id, bool wait) return false; } - QWidget *w = ui->treeWidget->itemWidget(item, COLUMN_NAME); + ElidedLabel *nameLabel = NULL; + QLabel *waitLabel = NULL; + getNameWidget(ui->treeWidget, item, nameLabel, waitLabel); if (wait) { - if (w) { + if (waitLabel->isVisible()) { /* Allready waiting */ } else { /* Save icon in role */ @@ -519,21 +577,19 @@ bool GroupTreeWidget::setWaiting(const QString &id, bool wait) /* Create empty icon of the same size */ QPixmap pixmap(ui->treeWidget->iconSize()); pixmap.fill(Qt::transparent); - item->setIcon(COLUMN_NAME, QIcon(pixmap)); - QLabel *label = new QLabel(this); - QMovie *movie = new QMovie(":/images/loader/circleball-16.gif"); - label->setMovie(movie); - - ui->treeWidget->setItemWidget(item, COLUMN_NAME, label); - - movie->start(); + /* Show waitLabel and hide nameLabel */ + nameLabel->setHidden(true); + waitLabel->setVisible(true); + waitLabel->movie()->start(); } } else { - if (w) { - ui->treeWidget->setItemWidget(item, COLUMN_NAME, NULL); - delete(w); + if (waitLabel->isVisible()) { + /* Show nameLabel and hide waitLabel */ + waitLabel->movie()->stop(); + waitLabel->setHidden(true); + nameLabel->setVisible(true); /* Set icon saved in role */ item->setIcon(COLUMN_NAME, item->data(COLUMN_DATA, ROLE_SAVED_ICON).value()); From ecaea051fae1cd0652c24cfde8acfe81e9088c81 Mon Sep 17 00:00:00 2001 From: hopetech Date: Wed, 19 Oct 2016 15:41:25 +0200 Subject: [PATCH 166/174] Fix some compilation warnings --- retroshare-gui/src/gui/RemoteDirModel.cpp | 2 +- retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp | 2 +- retroshare-gui/src/gui/common/RSGraphWidget.cpp | 2 ++ retroshare-gui/src/gui/gxs/GxsIdChooser.cpp | 2 +- retroshare-gui/src/gui/settings/ServerPage.cpp | 4 ++-- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index bfe7e2dbe..67dfb6d27 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -497,7 +497,7 @@ QVariant TreeStyle_RDM::sortRole(const QModelIndex& /*index*/,const DirDetails& } return QVariant(); } -QVariant FlatStyle_RDM::sortRole(const QModelIndex& index,const DirDetails& details,int coln) const +QVariant FlatStyle_RDM::sortRole(const QModelIndex& /*index*/,const DirDetails& details,int coln) const { /* * Person: name, id, 0, 0; diff --git a/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp b/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp index 12911efc9..bfc961dad 100644 --- a/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp +++ b/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp @@ -167,7 +167,7 @@ void PopupDistantChatDialog::closeEvent(QCloseEvent *e) PopupChatDialog::closeEvent(e) ; } -QString PopupDistantChatDialog::getPeerName(const ChatId &id) const +QString PopupDistantChatDialog::getPeerName(const ChatId &/*id*/) const { DistantChatPeerInfo tinfo; diff --git a/retroshare-gui/src/gui/common/RSGraphWidget.cpp b/retroshare-gui/src/gui/common/RSGraphWidget.cpp index fa7d00bdc..2d586ae38 100644 --- a/retroshare-gui/src/gui/common/RSGraphWidget.cpp +++ b/retroshare-gui/src/gui/common/RSGraphWidget.cpp @@ -393,10 +393,12 @@ void RSGraphWidget::paintData() paintLine(points, getColor(i)); } if(_maxValue > 0.0f) + { if(_flags & RSGRAPH_FLAGS_LOG_SCALE_Y) _y_scale = _rec.height()*0.8 / log(_maxValue) ; else _y_scale = _rec.height()*0.8/_maxValue ; + } } /** Returns a list of points on the bandwidth graph based on the supplied set diff --git a/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp b/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp index 9ce07b77b..48ec40074 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp @@ -195,7 +195,7 @@ bool GxsIdChooser::isInConstraintSet(const RsGxsId& id) const return mConstraintIdsSet.find(id) != mConstraintIdsSet.end() ; } -void GxsIdChooser::setEntryEnabled(int indx,bool enabled) +void GxsIdChooser::setEntryEnabled(int indx,bool /*enabled*/) { removeItem(indx) ; diff --git a/retroshare-gui/src/gui/settings/ServerPage.cpp b/retroshare-gui/src/gui/settings/ServerPage.cpp index 581a23183..9773570bb 100755 --- a/retroshare-gui/src/gui/settings/ServerPage.cpp +++ b/retroshare-gui/src/gui/settings/ServerPage.cpp @@ -486,7 +486,7 @@ void ServerPage::addPeerToIPTable(QTableWidget *table,int row,const BanListPeer& void ServerPage::toggleGroupIps(bool b) { rsBanList->enableAutoRange(b) ; } void ServerPage::setGroupIpLimit(int n) { rsBanList->setAutoRangeLimit(n) ; } -void ServerPage::ipFilterContextMenu(const QPoint& point) +void ServerPage::ipFilterContextMenu(const QPoint& /*point*/) { QMenu contextMenu(this) ; int row = ui.filteredIpsTable->currentRow(); @@ -604,7 +604,7 @@ void ServerPage::ipWhiteListContextMenu(const QPoint& /* point */) if(item == NULL) return ; - bool status = item->data(Qt::UserRole).toBool(); + //bool status = item->data(Qt::UserRole).toBool(); contextMenu.addAction(tr("Remove"),this,SLOT(removeWhiteListedIp())); From f20ee21ebe20a53a8ef4dcea59020ed9705c23af Mon Sep 17 00:00:00 2001 From: cave Date: Fri, 21 Oct 2016 19:56:02 +0200 Subject: [PATCH 167/174] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba5435faf..dea1e4b8b 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ Build Status Compilation on Windows ---------------------------- -Follow this file : [WindowsMSys2_InstallGuide.md](https://github.com/RetroShare/RetroShare/blob/master/WindowsMSys2_InstallGuide.txt) +Follow this file : [WindowsMSys2_InstallGuide.md](https://github.com/RetroShare/RetroShare/blob/master/WindowsMSys2_InstallGuide.md) Compilation on MacOSX ---------------------------- -Follow this file : [MacOS_X_InstallGuide.md](https://github.com/RetroShare/RetroShare/blob/master/MacOS_X_InstallGuide.txt) +Follow this file : [MacOS_X_InstallGuide](https://github.com/RetroShare/RetroShare/blob/master/MacOS_X_InstallGuide.md) Compilation on Linux ---------------------------- From 58a4d39b0f93902b1b49fe5ed8cd89c4c73f1d1c Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 22 Oct 2016 14:48:23 +0200 Subject: [PATCH 168/174] Fix handle of url in EmbedInHtmlAhref. The url cannot contains space. Query and hash are now handled. --- retroshare-gui/src/util/HandleRichText.cpp | 96 +++++++++++++++------- 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/retroshare-gui/src/util/HandleRichText.cpp b/retroshare-gui/src/util/HandleRichText.cpp index 5f494224c..191016589 100644 --- a/retroshare-gui/src/util/HandleRichText.cpp +++ b/retroshare-gui/src/util/HandleRichText.cpp @@ -71,39 +71,73 @@ class EmbedInHtmlAhref : public EmbedInHtml public: EmbedInHtmlAhref() : EmbedInHtml(Ahref) { - // myRE.setPattern("(\\bretroshare://[^\\s]*)|(\\bhttps?://[^\\s]*)|(\\bfile://[^\\s]*)|(\\bwww\\.[^\\s]*)"); + // The following regular expressions for finding URLs in + // plain text are borrowed from https://regex101.com/r/eR9yG2/4 + //Modified to: (Adding \s to stop when query have space char else don't stop at end.) + // (([\s]+)?(([a-z0-9.+-]+):)((\/\/)(\/|(((([^\/:@?&#\s]+)(:([^\/@?&#\s]+))?)@)?([^:\/?&#\s]+)(:([1-9][0-9]*))?)(?=[\/#$?]))))(([^#?\s]+)?(\?([^#\s]+))?(#([^\s]+))?([\s])?) + // regAddress .|| .| || | .| | . . .| .| ..| .../regPath .| | . .| .| ./ + // regBefo/eChar .|| .| || | .| | . . .| .| ..| ... regPathnam/| | . .regHash /regEnd/har + // regScheme /regGrp5| || | .| | . . .| .| ..| ../ regSearch . / + // |regS/ash || | .| | . . .| .| ..| .. regQuery/ + // regFileAuthHost .| | . . .| .| ..| ./ + // regAuthHost .| | . . .| .| ./regPosLk / + // regUserPass .| | . . /regHost /regPort / + // regUser /| | . . + // regGrp12 . / + // regPassCharse/ + // + // to get all group captured. + // Test patern: " https://user:password@example.com:8080/./api/api/../users/./get/22iohoife.extension?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3#test " - // The following regular expressions for finding URLs in - // plain text are borrowed from *gnome-terminal*: + QString regBeforeChar = "([\\s]+)?";//2nd Group: " " + QStringList regSchemes; + // regSchemes.append("news:"); + // regSchemes.append("telnet:"); + // regSchemes.append("nntp:"); + // regSchemes.append("file:/"); + regSchemes.append("https?:"); + // regSchemes.append("ftps?:"); + // regSchemes.append("sftp:"); + // regSchemes.append("webcal:"); + regSchemes.append("retroshare:"); - QString regPassCharset = "[-\\w,?;\\.:/!%$^*&~\\\"#']"; - QString regHost = "[-\\w]+(\\.[-\\w]+)*"; - QString regPort = "(?:\\:\\d{1,5})?"; - QString regPathCharset = "[-\\w_$\\.+!*,;@&=?/~#%]"; - QString regPathTermSet = "[^\\]'.}<>) \\t\\r\\n,\\\"]"; - QStringList regSchemes; -// regSchemes.append("news:"); -// regSchemes.append("telnet:"); -// regSchemes.append("nntp:"); -// regSchemes.append("file:/"); - regSchemes.append("https?:"); -// regSchemes.append("ftps?:"); -// regSchemes.append("sftp:"); -// regSchemes.append("webcal:"); - regSchemes.append("retroshare:"); - QString regScheme = "((?:" + regSchemes.join(")|(?:") + "))"; - QString regUserPass = "[-\\w]+(?:%s+)?" % regPassCharset; - QString regUrlPath = "(?:(/" + regPathCharset + "+(?:[(]" + regPathCharset +"*[)])*" + regPathCharset + "*)*" + regPathTermSet + ")?"; - QStringList regHotLinkFinders; - regHotLinkFinders.append(regScheme + "//(?:" + regUserPass + "@)?"+ regHost + regPort + regUrlPath); -// regHotLinkFinders.append("(?:(?:www)|(?:ftp))[-\\w]*\\." + regHost + regPort + regUrlPath); -// regHotLinkFinders.append("(?:(?:callto:)|(?:h323:)|(?:sip:))[-\\w][-\\w\\.]*(?:" + regPort + "/[a-z0-9]+)?@" + regHost); -// regHotLinkFinders.append("(?:mailto:)?[-\\w][-\\w\\.]*@[-\\w]+\\." + regHost); -// regHotLinkFinders.append("news:[\\w^_{|}~!\\\"#$%&'()*+,\\./;:=?`]+"); - while (!regHotLinkFinders.isEmpty()) { - myREs.append(QRegExp(regHotLinkFinders.takeFirst(), Qt::CaseInsensitive)); - }; - } + QString regScheme = "(?:" + regSchemes.join(")|(?:") + ")";//3rd Group: "https:" //4th group inside + + QString regSlash = "(\\/\\/)";//6th Group: "//" + + QString regUser = "([^\\/:@?&#\\s]+)";//11th Group: "user" + QString regPassCharset = "([^\\/@?&#\\s]+)";//13th Group "password" + QString regGrp12 = "(:" + regPassCharset + ")?"; // 12th Group: ":password" + QString regUserPass = "((" + regUser + regGrp12 + ")@)?"; //9th Group: "user:password@" with 10th inside + + QString regHost = "([^:\\/?&#\\s]+)"; //14th Group: "example.com" + QString regPort = "(:([1-9][0-9]*))?"; //15th Group: ":8080" with 16th inside + + QString regAuthHost = regUserPass + regHost + regPort; //8th Group: "user:password@example.com:8080" + QString regPosLk = ""; //Positive Lookahead + + QString regFileAuthHost = "(\\/|" + regAuthHost + regPosLk + ")"; //7th Group: "user:password@example.com:8080" Could be "/" with "file:///" + QString regGrp5 = "(" + regSlash + regFileAuthHost + ")"; //5th Group: "//user:password@example.com:8080" + QString regAddress = "(" + regBeforeChar + regScheme + regGrp5 + ")"; //1rst Group: "https://user:password@example.com:8080" + + QString regPathName = "([^#?\\s]+)?"; //18th Group: "/./api/api/../users/./get/22iohoife.extension" + + QString regQuery = "([^#\\s]+)"; //20th Group: "return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3" + QString regSearch = "(\\?" + regQuery + ")?"; //19th Group: "?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3" + + QString regHash = "(#([^\\s]+))?"; //21th Group: "#test" 22th inside + QString regEndChar = "";//"([\\s])?"; //23th Group: " " + QString regPath = "(" + regPathName + regSearch + regHash + regEndChar +")"; //17th Group: "/./api/api/../users/./get/22iohoife.extension?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3#test" + + QString regUrlPath = regAddress + regPath; + + QStringList regHotLinkFinders; + regHotLinkFinders.append(regUrlPath); + + while (!regHotLinkFinders.isEmpty()) { + myREs.append(QRegExp(regHotLinkFinders.takeFirst(), Qt::CaseInsensitive)); + }; + } }; From 9eafdd30f742e7118bcdeb7ee49534fe18914c1f Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 22 Oct 2016 16:02:38 +0200 Subject: [PATCH 169/174] Add ability to past RsLink in some more text fields. --- .../src/gui/Posted/PostedCreatePostDialog.ui | 38 ++++++---- .../src/gui/gxs/GxsCreateCommentDialog.ui | 19 +++-- .../gui/gxschannels/CreateGxsChannelMsg.ui | 76 ++++++++++++------- 3 files changed, 86 insertions(+), 47 deletions(-) diff --git a/retroshare-gui/src/gui/Posted/PostedCreatePostDialog.ui b/retroshare-gui/src/gui/Posted/PostedCreatePostDialog.ui index aca174f1c..d8128b199 100644 --- a/retroshare-gui/src/gui/Posted/PostedCreatePostDialog.ui +++ b/retroshare-gui/src/gui/Posted/PostedCreatePostDialog.ui @@ -13,8 +13,17 @@ Submit Post - - + + + 0 + + + 0 + + + 0 + + 0 @@ -31,14 +40,14 @@ - + QFrame::StyledPanel QFrame::Raised - + @@ -120,7 +129,7 @@ - + @@ -144,9 +153,9 @@ - + - + Qt::Horizontal @@ -162,7 +171,7 @@ - + Signed by: @@ -183,12 +192,12 @@ - + - + Qt::Horizontal @@ -235,6 +244,11 @@
gui/common/HeaderFrame.h
1 + + MimeTextEdit + QTextEdit +
gui/common/MimeTextEdit.h
+ GxsIdChooser QComboBox @@ -246,9 +260,7 @@
gui/common/StyledLabel.h
 - - - + buttonBox diff --git a/retroshare-gui/src/gui/gxs/GxsCreateCommentDialog.ui b/retroshare-gui/src/gui/gxs/GxsCreateCommentDialog.ui index 374949716..f989ccb6e 100644 --- a/retroshare-gui/src/gui/gxs/GxsCreateCommentDialog.ui +++ b/retroshare-gui/src/gui/gxs/GxsCreateCommentDialog.ui @@ -13,9 +13,9 @@ Make Comment - + - + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> <html><head><meta name="qrichtext" content="1" /><style type="text/css"> @@ -26,11 +26,11 @@ p, li { white-space: pre-wrap; } - + - + - + Qt::Horizontal @@ -43,7 +43,7 @@ p, li { white-space: pre-wrap; } - + Signed by @@ -55,7 +55,7 @@ p, li { white-space: pre-wrap; } - + @@ -72,6 +72,11 @@ p, li { white-space: pre-wrap; } + + MimeTextEdit + QTextEdit +
gui/common/MimeTextEdit.h
+ GxsIdChooser QComboBox diff --git a/retroshare-gui/src/gui/gxschannels/CreateGxsChannelMsg.ui b/retroshare-gui/src/gui/gxschannels/CreateGxsChannelMsg.ui index 5c603cb93..c9e4c3346 100644 --- a/retroshare-gui/src/gui/gxschannels/CreateGxsChannelMsg.ui +++ b/retroshare-gui/src/gui/gxschannels/CreateGxsChannelMsg.ui @@ -20,8 +20,17 @@ :/images/logo/logo_16.png:/images/logo/logo_16.png - - + + + 0 + + + 0 + + + 0 + + 0 @@ -31,14 +40,14 @@ - + QFrame::StyledPanel QFrame::Raised - + @@ -53,15 +62,15 @@ 0 - + Channel Post - + - + - + 75 @@ -86,9 +95,9 @@ - + - + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"> <html><head><meta name="qrichtext" content="1" /><style type="text/css"> @@ -157,15 +166,15 @@ p, li { white-space: pre-wrap; } - + Message - + - + - + 75 @@ -183,14 +192,14 @@ p, li { white-space: pre-wrap; } - + - + :/images/attachment.png:/images/attachment.png @@ -198,9 +207,9 @@ p, li { white-space: pre-wrap; } Attachments - + - + @@ -210,7 +219,7 @@ p, li { white-space: pre-wrap; } - + 16777215 @@ -229,7 +238,7 @@ p, li { white-space: pre-wrap; } - + Qt::Horizontal @@ -281,7 +290,7 @@ p, li { white-space: pre-wrap; } - + Qt::ScrollBarAlwaysOn @@ -293,7 +302,7 @@ p, li { white-space: pre-wrap; } 0 0 - 98 + 523 24 @@ -303,11 +312,20 @@ p, li { white-space: pre-wrap; } 0 - + 3 - + + 3 + + + 3 + + + 3 + + 3 @@ -324,7 +342,7 @@ p, li { white-space: pre-wrap; } Drag and Drop Files from Search Results - + @@ -336,7 +354,7 @@ p, li { white-space: pre-wrap; } - + @@ -378,10 +396,14 @@ p, li { white-space: pre-wrap; }
gui/common/HeaderFrame.h
1 + + MimeTextEdit + QTextEdit +
gui/common/MimeTextEdit.h
+ - From a67cf921922ca820bd0e7798cdf132407a6689bc Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 22 Oct 2016 16:15:29 +0200 Subject: [PATCH 170/174] Add Tooltip on comments column to see too long text. --- retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp b/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp index 3e7ec7b8f..7631ad23f 100644 --- a/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp +++ b/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp @@ -418,19 +418,21 @@ void GxsCommentTreeWidget::service_loadThread(const uint32_t &token) std::cerr << "GxsCommentTreeWidget::service_loadThread() Got Comment: " << comment.mMeta.mMsgId; std::cerr << std::endl; - GxsIdRSTreeWidgetItem *item = new GxsIdRSTreeWidgetItem(NULL); + GxsIdRSTreeWidgetItem *item = new GxsIdRSTreeWidgetItem(NULL) ; QString text; { - QDateTime qtime; - qtime.setTime_t(comment.mMeta.mPublishTs); + QDateTime qtime ; + qtime.setTime_t(comment.mMeta.mPublishTs) ; - text = qtime.toString("yyyy-MM-dd hh:mm:ss"); - item->setText(PCITEM_COLUMN_DATE, text); + text = qtime.toString("yyyy-MM-dd hh:mm:ss") ; + item->setText(PCITEM_COLUMN_DATE, text) ; + item->setToolTip(PCITEM_COLUMN_DATE, text) ; } text = QString::fromUtf8(comment.mComment.c_str()); item->setText(PCITEM_COLUMN_COMMENT, text); + item->setToolTip(PCITEM_COLUMN_COMMENT, text); RsGxsId authorId = comment.mMeta.mAuthorId; item->setId(authorId, PCITEM_COLUMN_AUTHOR, false); From 46cf25d3aa4502b4996b66150976f96b300732f7 Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 22 Oct 2016 15:17:49 +0200 Subject: [PATCH 171/174] Fix Copy RsLink in Clipboard. Spaces and specials characters are now encoded. --- retroshare-gui/src/gui/RetroShareLink.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/RetroShareLink.cpp b/retroshare-gui/src/gui/RetroShareLink.cpp index 0923c7864..5aa195373 100644 --- a/retroshare-gui/src/gui/RetroShareLink.cpp +++ b/retroshare-gui/src/gui/RetroShareLink.cpp @@ -790,9 +790,10 @@ QString RetroShareLink::toString() const #if QT_VERSION >= QT_VERSION_CHECK(5,0,0) url.setQuery(urlQuery); + return url.toString(QUrl::EncodeSpaces | QUrl::EncodeUnicode); +#else + return url.toString().replace(" ","%20");//Seems to be already done with Qt4 but to be sure. #endif - - return url.toString(); } QString RetroShareLink::niceName() const From 54ba617229531e3be218362f71525b96070c9aeb Mon Sep 17 00:00:00 2001 From: hunbernd Date: Sun, 23 Oct 2016 20:42:59 +0200 Subject: [PATCH 172/174] Fix: unable to upload files on Windows Caused by multiple unneeded / character in file path. --- libretroshare/src/file_sharing/dir_hierarchy.cc | 6 +++--- libretroshare/src/file_sharing/directory_storage.cc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 513ead3a5..07e395455 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -189,7 +189,7 @@ bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage de->row = mNodes.size(); de->parent_index = indx; de->dir_modtime = 0;// forces parsing.it->second; - de->dir_parent_path = d.dir_parent_path + "/" + d.dir_name ; + de->dir_parent_path = RsDirUtil::makePath(d.dir_parent_path, d.dir_name) ; de->dir_hash = createDirHash(de->dir_name,d.dir_hash,random_hash_seed) ; mDirHashes[de->dir_hash] = mNodes.size() ; @@ -446,7 +446,7 @@ bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryI mNodes[dir_index] = de ; - de->dir_parent_path = d.dir_parent_path + "/" + dir_name ; + de->dir_parent_path = RsDirUtil::makePath(d.dir_parent_path, dir_name) ; de->dir_hash = subdirs_hash[i]; mDirHashes[subdirs_hash[i]] = dir_index ; @@ -696,7 +696,7 @@ public: inline virtual uint64_t file_size() const { return mFe.file_size ; } inline virtual const RsFileHash& file_hash() const { return mFe.file_hash ; } inline virtual time_t file_modtime() const { return mFe.file_modtime ; } - inline virtual std::string file_parent_path()const { return mDe.dir_parent_path + "/" + mDe.dir_name ; } + inline virtual std::string file_parent_path()const { return RsDirUtil::makePath(mDe.dir_parent_path, mDe.dir_name) ; } inline virtual uint32_t file_popularity() const { NOT_IMPLEMENTED() ; return 0; } private: diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 6a4d3e93e..983846d9d 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -244,7 +244,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) d.min_age = now - dir_entry->dir_most_recent_time ; d.age = now - dir_entry->dir_modtime ; d.name = dir_entry->dir_name; - d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ; + d.path = RsDirUtil::makePath(dir_entry->dir_parent_path, dir_entry->dir_name) ; d.parent = (void*)(intptr_t)dir_entry->parent_index ; if(indx == 0) @@ -268,7 +268,7 @@ bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d) const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index); if(parent_dir_entry != NULL) - d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ; + d.path = RsDirUtil::makePath(parent_dir_entry->dir_parent_path, parent_dir_entry->dir_name) ; else d.path = "" ; } From fd5a5cc804db34cdaf39b44bb8a96a31616dd0d3 Mon Sep 17 00:00:00 2001 From: Phenom Date: Mon, 24 Oct 2016 20:11:47 +0200 Subject: [PATCH 173/174] Fix PR#540 --- retroshare-gui/src/util/HandleRichText.cpp | 58 +++++++++++----------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/retroshare-gui/src/util/HandleRichText.cpp b/retroshare-gui/src/util/HandleRichText.cpp index 191016589..d5088d2df 100644 --- a/retroshare-gui/src/util/HandleRichText.cpp +++ b/retroshare-gui/src/util/HandleRichText.cpp @@ -74,22 +74,22 @@ public: // The following regular expressions for finding URLs in // plain text are borrowed from https://regex101.com/r/eR9yG2/4 //Modified to: (Adding \s to stop when query have space char else don't stop at end.) - // (([\s]+)?(([a-z0-9.+-]+):)((\/\/)(\/|(((([^\/:@?&#\s]+)(:([^\/@?&#\s]+))?)@)?([^:\/?&#\s]+)(:([1-9][0-9]*))?)(?=[\/#$?]))))(([^#?\s]+)?(\?([^#\s]+))?(#([^\s]+))?([\s])?) - // regAddress .|| .| || | .| | . . .| .| ..| .../regPath .| | . .| .| ./ - // regBefo/eChar .|| .| || | .| | . . .| .| ..| ... regPathnam/| | . .regHash /regEnd/har - // regScheme /regGrp5| || | .| | . . .| .| ..| ../ regSearch . / - // |regS/ash || | .| | . . .| .| ..| .. regQuery/ - // regFileAuthHost .| | . . .| .| ..| ./ - // regAuthHost .| | . . .| .| ./regPosLk / - // regUserPass .| | . . /regHost /regPort / - // regUser /| | . . - // regGrp12 . / - // regPassCharse/ + // ((?<=\s)(([a-z0-9.+-]+):)((\/\/)(\/|(((([^\/:@?&#\s]+)(:([^\/@?&#\s]+))?)@)?([^:\/?&#\s]+)(:([1-9][0-9]*))?)(?=[\/#$?]))))(([^#?\s]+)?(\?([^#\s]+))?(#([^\s]+))?([\s])?) + // regAddress .|| .| || | .| | . . .| .| ..| .../regPath .| | . .| .| ./ + // regBef/reChar .|| .| || | .| | . . .| .| ..| ... regPathnam/| | . .regHash /regEnd/har + // regScheme /regGrp5| || | .| | . . .| .| ..| ../ regSearch . / + // |regS/ash || | .| | . . .| .| ..| .. regQuery/ + // regFileAuthHost .| | . . .| .| ..| ./ + // regAuthHost .| | . . .| .| ./regPosLk / + // regUserPass .| | . . /regHost /regPort / + // regUser /| | . . + // regGrp12 . / + // regPassCharse/ // // to get all group captured. // Test patern: " https://user:password@example.com:8080/./api/api/../users/./get/22iohoife.extension?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3#test " - QString regBeforeChar = "([\\s]+)?";//2nd Group: " " + QString regBeforeChar = ""; //"(?<=\\s)";//WARNING, Look Behind not supported by Qt QStringList regSchemes; // regSchemes.append("news:"); // regSchemes.append("telnet:"); @@ -101,33 +101,33 @@ public: // regSchemes.append("webcal:"); regSchemes.append("retroshare:"); - QString regScheme = "(?:" + regSchemes.join(")|(?:") + ")";//3rd Group: "https:" //4th group inside + QString regScheme = "(?:" + regSchemes.join(")|(?:") + ")";//2nd Group: "https:" //3rd group inside - QString regSlash = "(\\/\\/)";//6th Group: "//" + QString regSlash = "(\\/\\/)";//5th Group: "//" - QString regUser = "([^\\/:@?&#\\s]+)";//11th Group: "user" - QString regPassCharset = "([^\\/@?&#\\s]+)";//13th Group "password" - QString regGrp12 = "(:" + regPassCharset + ")?"; // 12th Group: ":password" - QString regUserPass = "((" + regUser + regGrp12 + ")@)?"; //9th Group: "user:password@" with 10th inside + QString regUser = "([^\\/:@?&#\\s]+)";//10th Group: "user" + QString regPassCharset = "([^\\/@?&#\\s]+)";//12th Group "password" + QString regGrp12 = "(:" + regPassCharset + ")?"; // 11th Group: ":password" + QString regUserPass = "((" + regUser + regGrp12 + ")@)?"; //8th Group: "user:password@" with 9th inside - QString regHost = "([^:\\/?&#\\s]+)"; //14th Group: "example.com" - QString regPort = "(:([1-9][0-9]*))?"; //15th Group: ":8080" with 16th inside + QString regHost = "([^:\\/?&#\\s]+)"; //13th Group: "example.com" + QString regPort = "(:([1-9][0-9]*))?"; //14th Group: ":8080" with 15th inside - QString regAuthHost = regUserPass + regHost + regPort; //8th Group: "user:password@example.com:8080" + QString regAuthHost = regUserPass + regHost + regPort; //7th Group: "user:password@example.com:8080" QString regPosLk = ""; //Positive Lookahead - QString regFileAuthHost = "(\\/|" + regAuthHost + regPosLk + ")"; //7th Group: "user:password@example.com:8080" Could be "/" with "file:///" - QString regGrp5 = "(" + regSlash + regFileAuthHost + ")"; //5th Group: "//user:password@example.com:8080" + QString regFileAuthHost = "(\\/|" + regAuthHost + regPosLk + ")"; //6th Group: "user:password@example.com:8080" Could be "/" with "file:///" + QString regGrp5 = "(" + regSlash + regFileAuthHost + ")"; //4th Group: "//user:password@example.com:8080" QString regAddress = "(" + regBeforeChar + regScheme + regGrp5 + ")"; //1rst Group: "https://user:password@example.com:8080" - QString regPathName = "([^#?\\s]+)?"; //18th Group: "/./api/api/../users/./get/22iohoife.extension" + QString regPathName = "([^#?\\s]+)?"; //17th Group: "/./api/api/../users/./get/22iohoife.extension" - QString regQuery = "([^#\\s]+)"; //20th Group: "return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3" - QString regSearch = "(\\?" + regQuery + ")?"; //19th Group: "?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3" + QString regQuery = "([^#\\s]+)"; //19th Group: "return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3" + QString regSearch = "(\\?" + regQuery + ")?"; //18th Group: "?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3" - QString regHash = "(#([^\\s]+))?"; //21th Group: "#test" 22th inside - QString regEndChar = "";//"([\\s])?"; //23th Group: " " - QString regPath = "(" + regPathName + regSearch + regHash + regEndChar +")"; //17th Group: "/./api/api/../users/./get/22iohoife.extension?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3#test" + QString regHash = "(#([^\\s]+))?"; //20th Group: "#test" 21th inside + QString regEndChar = "";//"([\\s])?"; //22th Group: " " + QString regPath = "(" + regPathName + regSearch + regHash + regEndChar +")"; //16th Group: "/./api/api/../users/./get/22iohoife.extension?return=name&return=email&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3&a[]=3#test" QString regUrlPath = regAddress + regPath; From 32e54e53b7b7aba804f234345d6949c72a63cdad Mon Sep 17 00:00:00 2001 From: Phenom Date: Tue, 25 Oct 2016 14:58:06 +0200 Subject: [PATCH 174/174] Fix PR#536 revealing of bad factor management in StyledElidedLabel. --- .../src/gui/common/StyledElidedLabel.cpp | 17 +++++++++++------ .../src/gui/common/StyledElidedLabel.h | 3 +++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/retroshare-gui/src/gui/common/StyledElidedLabel.cpp b/retroshare-gui/src/gui/common/StyledElidedLabel.cpp index 68c84d0be..3a196dc24 100644 --- a/retroshare-gui/src/gui/common/StyledElidedLabel.cpp +++ b/retroshare-gui/src/gui/common/StyledElidedLabel.cpp @@ -25,19 +25,24 @@ /** Constructor */ StyledElidedLabel::StyledElidedLabel(QWidget *parent) - : ElidedLabel(parent) + : ElidedLabel(parent), _lastFactor(-1) { } StyledElidedLabel::StyledElidedLabel(const QString &text, QWidget *parent) - : ElidedLabel(text, parent) + : ElidedLabel(text, parent), _lastFactor(-1) { } void StyledElidedLabel::setFontSizeFactor(int factor) { - QFont f = font(); - qreal fontSize = factor * f.pointSizeF() / 100; - f.setPointSizeF(fontSize); - setFont(f); + int newFactor = factor; + if (factor > 0) { + if (_lastFactor > 0) newFactor = 100 + factor - _lastFactor; + _lastFactor = factor; + QFont f = font(); + qreal fontSize = newFactor * f.pointSizeF() / 100; + f.setPointSizeF(fontSize); + setFont(f); + } } diff --git a/retroshare-gui/src/gui/common/StyledElidedLabel.h b/retroshare-gui/src/gui/common/StyledElidedLabel.h index c871cb9c8..dea9ccd51 100644 --- a/retroshare-gui/src/gui/common/StyledElidedLabel.h +++ b/retroshare-gui/src/gui/common/StyledElidedLabel.h @@ -34,6 +34,9 @@ public: StyledElidedLabel(const QString &text, QWidget *parent = NULL); void setFontSizeFactor(int factor); + +private: + int _lastFactor; }; #endif