From d35dd76ca56bf2a786a33588bae6609caca7f6de Mon Sep 17 00:00:00 2001 From: jolavillette Date: Sat, 2 May 2020 09:47:24 +0200 Subject: [PATCH 01/72] try to isolate pqistreamer mutex from pqissl mutex, step 1: pqistreamer::tick_recv() --- libretroshare/src/pqi/pqistreamer.cc | 41 ++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index dc9ceaabc..6c3c0d180 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -288,16 +288,26 @@ int pqistreamer::tick_bio() int pqistreamer::tick_recv(uint32_t timeout) { - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ +// Only our thread manipulates mIncoming queue and related counters. +// The lock of pqistreamer mutex is thus not needed here. +// For the moment and for safety reasons it is still kept in 3 places: +// - in pqistreamer::tick_recv before calling free_pend_locked, as I dont know what this method actually does +// - in pqistreamer::handleincomingitem_locked, as it modifies mIncoming queue +// - in pqistreamer::inReadBytes_locked, as it modifies related counters +// +// RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ if (mBio->moretoread(timeout)) { handleincoming_locked(); } - if(!(mBio->isactive())) - { - free_pend_locked(); - } + if(!(mBio->isactive())) + { + // lock only now, see comment above + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + + free_pend_locked(); + } return 1; } @@ -414,16 +424,20 @@ int pqistreamer::handleincomingitem_locked(RsItem *pqi,int len) // Use overloaded Contact function pqi -> PeerId(PeerId()); - mIncoming.push_back(pqi); - ++mIncomingSize ; - /*******************************************************************************************/ - // keep info for stats for a while. Only keep the items for the last two seconds. sec n is ongoing and second n-1 - // is a full statistics chunk that can be used in the GUI + // lock only now, see comment in pqistreamer::tick_recv + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - locked_addTrafficClue(pqi,len,mCurrentStatsChunk_In) ; + mIncoming.push_back(pqi); + ++mIncomingSize ; - /*******************************************************************************************/ + /*******************************************************************************************/ + // keep info for stats for a while. Only keep the items for the last two seconds. sec n is ongoing and second n-1 + // is a full statistics chunk that can be used in the GUI + + locked_addTrafficClue(pqi,len,mCurrentStatsChunk_In) ; + + /*******************************************************************************************/ return 1; } @@ -1241,6 +1255,9 @@ void pqistreamer::inReadBytes_locked(uint32_t inb) } #endif + // lock only now, see comment in pqistreamer::tick_recv + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + mTotalRead += inb; mCurrRead += inb; mAvgReadCount += inb; From d8470869118607af7dc3ad247fa11a7236063ccd Mon Sep 17 00:00:00 2001 From: jolavillette Date: Sun, 3 May 2020 20:32:17 +0200 Subject: [PATCH 02/72] finish emoving unneeded pqistreamer locks in recv part --- libretroshare/src/pqi/pqistreamer.cc | 106 +++++++++++++-------------- libretroshare/src/pqi/pqistreamer.h | 14 ++-- 2 files changed, 59 insertions(+), 61 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 6c3c0d180..45b48228e 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -159,7 +159,7 @@ pqistreamer::~pqistreamer() if (mRsSerialiser) delete mRsSerialiser; - free_pend_locked() ; + free_pend() ; // clean up incoming. while (!mIncoming.empty()) @@ -199,14 +199,12 @@ RsItem *pqistreamer::GetItem() pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::GetItem()"); #endif - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - if(mIncoming.empty()) return NULL; RsItem *osr = mIncoming.front() ; - mIncoming.pop_front() ; - --mIncomingSize; + mIncoming.pop_front() ; + --mIncomingSize; return osr; } @@ -288,25 +286,26 @@ int pqistreamer::tick_bio() int pqistreamer::tick_recv(uint32_t timeout) { -// Only our thread manipulates mIncoming queue and related counters. -// The lock of pqistreamer mutex is thus not needed here. -// For the moment and for safety reasons it is still kept in 3 places: -// - in pqistreamer::tick_recv before calling free_pend_locked, as I dont know what this method actually does -// - in pqistreamer::handleincomingitem_locked, as it modifies mIncoming queue -// - in pqistreamer::inReadBytes_locked, as it modifies related counters +// Apart from a few exceptions that are atomic (mLastIncomingTs, mIncomingSize), only this pqi thread reads/writes mIncoming queue and related counters. +// The lock of pqistreamer mutex is thus not needed. +// The mutex lock is still needed before calling locked_addTrafficClue because this method is also used by the thread pushing packets in mOutPkts // -// RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ +// The following methods have been renamed by removing the 'locked' part of the name: +// - handleincoming_locked +// - handleincomingitem_locked +// - inReadBytes_locked +// - inAllowedBytes_locked +// - addPartialPacket_locked +// - allocate_rpend_locked +// - free_pend_locked; if (mBio->moretoread(timeout)) { - handleincoming_locked(); + handleincoming(); } if(!(mBio->isactive())) { - // lock only now, see comment above - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - - free_pend_locked(); + free_pend(); } return 1; } @@ -319,7 +318,7 @@ int pqistreamer::tick_send(uint32_t timeout) /* short circuit everything is bio isn't active */ if (!(mBio->isactive())) { - free_pend_locked(); + free_pend(); return 0; } @@ -413,11 +412,11 @@ int pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize) return 1; // keep error internal. } -int pqistreamer::handleincomingitem_locked(RsItem *pqi,int len) +int pqistreamer::handleincomingitem(RsItem *pqi,int len) { #ifdef DEBUG_PQISTREAMER - pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincomingitem_locked()"); + pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincomingitem()"); #endif // timestamp last received packet. mLastIncomingTs = time(NULL); @@ -425,9 +424,6 @@ int pqistreamer::handleincomingitem_locked(RsItem *pqi,int len) // Use overloaded Contact function pqi -> PeerId(PeerId()); - // lock only now, see comment in pqistreamer::tick_recv - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - mIncoming.push_back(pqi); ++mIncomingSize ; @@ -435,6 +431,8 @@ int pqistreamer::handleincomingitem_locked(RsItem *pqi,int len) // keep info for stats for a while. Only keep the items for the last two seconds. sec n is ongoing and second n-1 // is a full statistics chunk that can be used in the GUI + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + locked_addTrafficClue(pqi,len,mCurrentStatsChunk_In) ; /*******************************************************************************************/ @@ -470,8 +468,8 @@ void pqistreamer::locked_addTrafficClue(const RsItem *pqi,uint32_t pktsize,std:: rstime_t pqistreamer::getLastIncomingTS() { - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - +// This is the only case where another thread (rs main for pqiperson) will access our data +// Still a mutex lock is not needed because the operation is atomic return mLastIncomingTs; } @@ -707,23 +705,23 @@ int pqistreamer::handleoutgoing_locked() /* Handles reading from input stream. */ -int pqistreamer::handleincoming_locked() +int pqistreamer::handleincoming() { int readbytes = 0; static const int max_failed_read_attempts = 2000 ; #ifdef DEBUG_PQISTREAMER - pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincoming_locked()"); + pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincoming()"); #endif if(!(mBio->isactive())) { mReading_state = reading_state_initial ; - free_pend_locked(); + free_pend(); return 0; } else - allocate_rpend_locked(); + allocate_rpend(); // enough space to read any packet. uint32_t maxlen = mPkt_rpend_size; @@ -732,7 +730,7 @@ int pqistreamer::handleincoming_locked() // initial read size: basic packet. int blen = getRsPktBaseSize(); // this is valid for both packet slices and normal un-sliced packets (same header size) - int maxin = inAllowedBytes_locked(); + int maxin = inAllowedBytes(); #ifdef DEBUG_PQISTREAMER std::cerr << "[" << (void*)pthread_self() << "] " << "reading state = " << mReading_state << std::endl ; @@ -981,19 +979,19 @@ continue_packet: std::cerr << "Inputing partial packet " << RsUtil::BinToHex((char*)block,8) << std::endl; #endif uint32_t packet_length = 0 ; - pkt = addPartialPacket_locked(block,pktlen,slice_packet_id,is_packet_starting,is_packet_ending,packet_length) ; + pkt = addPartialPacket(block,pktlen,slice_packet_id,is_packet_starting,is_packet_ending,packet_length) ; pktlen = packet_length ; } else pkt = mRsSerialiser->deserialise(block, &pktlen); - if ((pkt != NULL) && (0 < handleincomingitem_locked(pkt,pktlen))) + if ((pkt != NULL) && (0 < handleincomingitem(pkt,pktlen))) { #ifdef DEBUG_PQISTREAMER pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "Successfully Read a Packet!"); #endif - inReadBytes_locked(pktlen); // only count deserialised packets, because that's what is actually been transfered. + inReadBytes(pktlen); // only count deserialised packets, because that's what is actually been transfered. } else if (!is_partial_packet) { @@ -1026,7 +1024,7 @@ continue_packet: return 0; } -RsItem *pqistreamer::addPartialPacket_locked(const void *block, uint32_t len, uint32_t slice_packet_id, bool is_packet_starting, bool is_packet_ending, uint32_t &total_len) +RsItem *pqistreamer::addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id, bool is_packet_starting, bool is_packet_ending, uint32_t &total_len) { #ifdef DEBUG_PACKET_SLICING std::cerr << "Receiving partial packet. size=" << len << ", ID=" << std::hex << slice_packet_id << std::dec << ", starting:" << is_packet_starting << ", ending:" << is_packet_ending ; @@ -1170,7 +1168,7 @@ int pqistreamer::outAllowedBytes_locked() return quota; } -int pqistreamer::inAllowedBytes_locked() +int pqistreamer::inAllowedBytes() { double t = getCurrentTS(); // in sec, with high accuracy @@ -1208,7 +1206,7 @@ int pqistreamer::inAllowedBytes_locked() #ifdef DEBUG_PQISTREAMER uint64_t t_now = 1000 * getCurrentTS(); - std::cerr << std::dec << t_now << " DEBUG_PQISTREAMER pqistreamer::inAllowedBytes_locked PeerId " << this->PeerId().toStdString() << " dt " << (int)(1000 * dt) << "ms, mAvgDtIn " << (int)(1000 * mAvgDtIn) << "ms, maxin " << (int)(maxin) << " bytes/s, mCurrRead " << mCurrRead << " bytes, quota " << (int)(quota) << " bytes" << std::endl; + std::cerr << std::dec << t_now << " DEBUG_PQISTREAMER pqistreamer::inAllowedBytes PeerId " << this->PeerId().toStdString() << " dt " << (int)(1000 * dt) << "ms, mAvgDtIn " << (int)(1000 * mAvgDtIn) << "ms, maxin " << (int)(maxin) << " bytes/s, mCurrRead " << mCurrRead << " bytes, quota " << (int)(quota) << " bytes" << std::endl; #endif return quota; @@ -1245,7 +1243,7 @@ void pqistreamer::outSentBytes_locked(uint32_t outb) return; } -void pqistreamer::inReadBytes_locked(uint32_t inb) +void pqistreamer::inReadBytes(uint32_t inb) { #ifdef DEBUG_PQISTREAMER { @@ -1255,9 +1253,6 @@ void pqistreamer::inReadBytes_locked(uint32_t inb) } #endif - // lock only now, see comment in pqistreamer::tick_recv - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - mTotalRead += inb; mCurrRead += inb; mAvgReadCount += inb; @@ -1265,7 +1260,7 @@ void pqistreamer::inReadBytes_locked(uint32_t inb) return; } -void pqistreamer::allocate_rpend_locked() +void pqistreamer::allocate_rpend() { if(mPkt_rpending) return; @@ -1288,17 +1283,17 @@ int pqistreamer::reset() #ifdef DEBUG_PQISTREAMER std::cerr << "pqistreamer::reset()" << std::endl; #endif - free_pend_locked(); + free_pend(); return 1 ; } -void pqistreamer::free_pend_locked() +void pqistreamer::free_pend() { if(mPkt_rpending) { #ifdef DEBUG_PQISTREAMER - std::cerr << "pqistreamer::free_pend_locked(): pending input packet buffer" << std::endl; + std::cerr << "pqistreamer::free_pend(): pending input packet buffer" << std::endl; #endif free(mPkt_rpending); mPkt_rpending = 0; @@ -1308,7 +1303,7 @@ void pqistreamer::free_pend_locked() if (mPkt_wpending) { #ifdef DEBUG_PQISTREAMER - std::cerr << "pqistreamer::free_pend_locked(): pending output packet buffer" << std::endl; + std::cerr << "pqistreamer::free_pend(): pending output packet buffer" << std::endl; #endif free(mPkt_wpending); mPkt_wpending = NULL; @@ -1317,7 +1312,7 @@ void pqistreamer::free_pend_locked() #ifdef DEBUG_PQISTREAMER if(!mPartialPackets.empty()) - std::cerr << "pqistreamer::free_pend_locked(): " << mPartialPackets.size() << " pending input partial packets" << std::endl; + std::cerr << "pqistreamer::free_pend(): " << mPartialPackets.size() << " pending input partial packets" << std::endl; #endif // also delete any incoming partial packet for(std::map::iterator it(mPartialPackets.begin());it!=mPartialPackets.end();++it) @@ -1335,23 +1330,27 @@ int pqistreamer::gatherStatistics(std::list& outqueue_lst,std return locked_gatherStatistics(outqueue_lst,inqueue_lst); } + int pqistreamer::getQueueSize(bool in) { - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - if (in) - return mIncomingSize; - else - return locked_out_queue_size(); +// no mutex is needed here because this is atomic + return mIncomingSize; + else + { + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + return locked_out_queue_size(); + } } void pqistreamer::getRates(RsBwRates &rates) { RateInterface::getRates(rates); - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ +// no mutex is needed here because this is atomic + rates.mQueueIn = mIncomingSize; - rates.mQueueIn = mIncomingSize; + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ rates.mQueueOut = locked_out_queue_size(); } @@ -1417,4 +1416,3 @@ void *pqistreamer::locked_pop_out_data(uint32_t /*max_slice_size*/, uint32_t &si return res ; } - diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index c5b2bc983..1589448c9 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -104,12 +104,12 @@ class pqistreamer: public PQInterface private: int queue_outpqi_locked(RsItem *i,uint32_t& serialized_size); - int handleincomingitem_locked(RsItem *i, int len); + int handleincomingitem(RsItem *i, int len); // ticked regularly (manages out queues and sending // via above interfaces. virtual int handleoutgoing_locked(); - virtual int handleincoming_locked(); + virtual int handleincoming(); // Bandwidth/Streaming Management. float outTimeSlice_locked(); @@ -117,11 +117,11 @@ class pqistreamer: public PQInterface int outAllowedBytes_locked(); void outSentBytes_locked(uint32_t ); - int inAllowedBytes_locked(); - void inReadBytes_locked(uint32_t ); + int inAllowedBytes(); + void inReadBytes(uint32_t ); // cleans up everything that's pending / half finished. - void free_pend_locked(); + void free_pend(); // RsSerialiser - determines which packets can be serialised. RsSerialiser *mRsSerialiser; @@ -129,7 +129,7 @@ class pqistreamer: public PQInterface void *mPkt_wpending; // storage for pending packet to write. uint32_t mPkt_wpending_size; // ... and its size. - void allocate_rpend_locked(); // use these two functions to allocate/free the buffer below + void allocate_rpend(); // use these two functions to allocate/free the buffer below int mPkt_rpend_size; // size of pkt_rpending. void *mPkt_rpending; // storage for read in pending packets. @@ -177,7 +177,7 @@ class pqistreamer: public PQInterface bool mAcceptsPacketSlicing ; rstime_t mLastSentPacketSlicingProbe ; void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); - RsItem *addPartialPacket_locked(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending,uint32_t& total_len); + RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending,uint32_t& total_len); std::map mPartialPackets ; }; From 28458bf10a1b1cc7c30738d0b9570babfa5943fc Mon Sep 17 00:00:00 2001 From: jolavillette Date: Mon, 4 May 2020 08:11:57 +0200 Subject: [PATCH 03/72] tick_send: remove unneeded lock around mBio->cansend --- libretroshare/src/pqi/pqistreamer.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 45b48228e..393f2ac70 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -313,17 +313,16 @@ int pqistreamer::tick_recv(uint32_t timeout) int pqistreamer::tick_send(uint32_t timeout) { - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - - /* short circuit everything is bio isn't active */ + /* short circuit everything if bio isn't active */ if (!(mBio->isactive())) { - free_pend(); + free_pend(); return 0; } if (mBio->cansend(timeout)) { + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ handleoutgoing_locked(); } From 757e922c4f7626a475586ca807e8031681a81776 Mon Sep 17 00:00:00 2001 From: sehraf Date: Mon, 4 May 2020 15:13:39 +0200 Subject: [PATCH 04/72] Fix unix_fcntl_nonblock() to not overwrite any existing flags. --- libretroshare/src/pqi/pqinetwork.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/pqi/pqinetwork.cc b/libretroshare/src/pqi/pqinetwork.cc index e2251ed6a..18fe71fc0 100644 --- a/libretroshare/src/pqi/pqinetwork.cc +++ b/libretroshare/src/pqi/pqinetwork.cc @@ -413,9 +413,10 @@ int unix_fcntl_nonblock(int fd) { int ret; -/******************* WINDOWS SPECIFIC PART ******************/ +/******************* OS SPECIFIC PART ******************/ #ifndef WINDOWS_SYS // ie UNIX - ret = fcntl(fd, F_SETFL, O_NONBLOCK); + int flags = fcntl(fd, F_GETFL); + ret = fcntl(fd, F_SETFL, flags | O_NONBLOCK); #ifdef NET_DEBUG std::cerr << "unix_fcntl_nonblock():" << ret << " errno:" << errno << std::endl; From 7113d1935fa9b21506413c8f0c789078836d19be Mon Sep 17 00:00:00 2001 From: jolavillette Date: Mon, 4 May 2020 22:42:18 +0200 Subject: [PATCH 05/72] finish removing pqistreamer locks, clean rates methods --- libretroshare/src/pqi/pqistreamer.cc | 155 ++++++++++++++++----------- libretroshare/src/pqi/pqistreamer.h | 39 ++++--- 2 files changed, 113 insertions(+), 81 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 393f2ac70..62b4d7f84 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -1,5 +1,5 @@ /******************************************************************************* - * libretroshare/src/pqi: pqistreamer.h * + * libretroshare/src/pqi: pqistreamer.cc * * * * libretroshare: retroshare core library * * * @@ -102,38 +102,39 @@ pqistreamer::pqistreamer(RsSerialiser *rss, const RsPeerId& id, BinInterface *bi mAvgDtOut(0), mAvgDtIn(0) { - // 100 B/s (minimal) - setMaxRate(true, 0.1); - setMaxRate(false, 0.1); - setRate(true, 0); // needs to be off-mutex - setRate(false, 0); + // 100 B/s (minimal) + setMaxRate(true, 0.1); + setMaxRate(false, 0.1); + setRate(true, 0); // needs to be off-mutex + setRate(false, 0); - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - mAcceptsPacketSlicing = false ; // by default. Will be turned into true when everyone's ready. - mLastSentPacketSlicingProbe = 0 ; + mAcceptsPacketSlicing = false ; // by default. Will be turned into true when everyone's ready. + mLastSentPacketSlicingProbe = 0 ; - mAvgLastUpdate = mCurrSentTS = mCurrReadTS = getCurrentTS(); + mAvgLastUpdate = mCurrSentTS = mCurrReadTS = getCurrentTS(); - mIncomingSize = 0 ; + mIncomingSize = 0 ; + mIncomingSize_bytes = 0; - mStatisticsTimeStamp = 0 ; - /* allocated once */ - mPkt_rpend_size = 0; - mPkt_rpending = 0; - mReading_state = reading_state_initial ; + mStatisticsTimeStamp = 0 ; + /* allocated once */ + mPkt_rpend_size = 0; + mPkt_rpending = 0; + mReading_state = reading_state_initial ; - pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::pqistreamer() Initialisation!"); + pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::pqistreamer() Initialisation!"); - if (!bio_in) - { - pqioutput(PQL_ALERT, pqistreamerzone, "pqistreamer::pqistreamer() NULL bio, FATAL ERROR!"); - exit(1); - } + if (!bio_in) + { + pqioutput(PQL_ALERT, pqistreamerzone, "pqistreamer::pqistreamer() NULL bio, FATAL ERROR!"); + exit(1); + } - mFailed_read_attempts = 0; // reset failed read, as no packet is still read. + mFailed_read_attempts = 0; // reset failed read, as no packet is still read. - return; + return; } pqistreamer::~pqistreamer() @@ -177,6 +178,7 @@ pqistreamer::~pqistreamer() // Get/Send Items. +// This is the entry poing for methods willing to send items through our out queue int pqistreamer::SendItem(RsItem *si,uint32_t& out_size) { #ifdef RSITEM_DEBUG @@ -205,10 +207,24 @@ RsItem *pqistreamer::GetItem() RsItem *osr = mIncoming.front() ; mIncoming.pop_front() ; --mIncomingSize; +// for future use +// mIncomingSize_bytes -= return osr; } + +float pqistreamer::getMaxRate(bool b) +{ + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + return getMaxRate_locked(b); +} + +float pqistreamer::getMaxRate_locked(bool b) +{ + return RateInterface::getMaxRate(b) ; +} + float pqistreamer::getRate(bool b) { RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ @@ -217,26 +233,28 @@ float pqistreamer::getRate(bool b) void pqistreamer::setMaxRate(bool b,float f) { - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - RateInterface::setMaxRate(b,f) ; + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + setMaxRate_locked(b,f); } + +void pqistreamer::setMaxRate_locked(bool b,float f) +{ + RateInterface::setMaxRate(b,f) ; +} + void pqistreamer::setRate(bool b,float f) { RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ RateInterface::setRate(b,f) ; } + void pqistreamer::updateRates() { - // update rates both ways. + // update actual rates both ways. double t = getCurrentTS(); // get current timestamp. - double diff ; - - { - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - diff = t - mAvgLastUpdate ; - } + double diff = t - mAvgLastUpdate; if (diff > PQISTREAM_AVG_PERIOD) { @@ -261,10 +279,11 @@ void pqistreamer::updateRates() setRate(false, 0); } + mAvgLastUpdate = t; + mAvgReadCount = 0; + { RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - mAvgLastUpdate = t; - mAvgReadCount = 0; mAvgSentCount = 0; } } @@ -275,7 +294,7 @@ int pqistreamer::tick_bio() RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ mBio->tick(); - /* short circuit everything is bio isn't active */ + /* short circuit everything if bio isn't active */ if (!(mBio->isactive())) { return 0; @@ -283,21 +302,12 @@ int pqistreamer::tick_bio() return 1; } - int pqistreamer::tick_recv(uint32_t timeout) { // Apart from a few exceptions that are atomic (mLastIncomingTs, mIncomingSize), only this pqi thread reads/writes mIncoming queue and related counters. -// The lock of pqistreamer mutex is thus not needed. -// The mutex lock is still needed before calling locked_addTrafficClue because this method is also used by the thread pushing packets in mOutPkts -// -// The following methods have been renamed by removing the 'locked' part of the name: -// - handleincoming_locked -// - handleincomingitem_locked -// - inReadBytes_locked -// - inAllowedBytes_locked -// - addPartialPacket_locked -// - allocate_rpend_locked -// - free_pend_locked; +// The lock of pqistreamer mutex is thus not needed here. +// The mutex lock is still needed before calling locked_addTrafficClue because this method is also used by the thread pushing packets in mOutPkts. +// Locks around rates are provided internally. if (mBio->moretoread(timeout)) { @@ -310,7 +320,6 @@ int pqistreamer::tick_recv(uint32_t timeout) return 1; } - int pqistreamer::tick_send(uint32_t timeout) { /* short circuit everything if bio isn't active */ @@ -348,12 +357,11 @@ int pqistreamer::status() return 0; } +// this method is overloaded by pqiqosstreamer void pqistreamer::locked_storeInOutputQueue(void *ptr,int,int) { mOutPkts.push_back(ptr); } -// -/**************** HANDLE OUTGOING TRANSLATION + TRANSMISSION ******/ int pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize) { @@ -362,7 +370,6 @@ int pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize) std::cerr << "pqistreamer::queue_outpqi() called." << std::endl; #endif - /* decide which type of packet it is */ pktsize = mRsSerialiser->size(pqi); @@ -370,7 +377,6 @@ int pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize) if(ptr == NULL) return 0 ; - #ifdef DEBUG_PQISTREAMER std::cerr << "pqistreamer::queue_outpqi() serializing packet with packet size : " << pktsize << std::endl; @@ -424,16 +430,17 @@ int pqistreamer::handleincomingitem(RsItem *pqi,int len) pqi -> PeerId(PeerId()); mIncoming.push_back(pqi); - ++mIncomingSize ; + ++mIncomingSize; + // for future use + // mIncomingSize_bytes += len; /*******************************************************************************************/ // keep info for stats for a while. Only keep the items for the last two seconds. sec n is ongoing and second n-1 // is a full statistics chunk that can be used in the GUI - - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - - locked_addTrafficClue(pqi,len,mCurrentStatsChunk_In) ; - + { + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + locked_addTrafficClue(pqi,len,mCurrentStatsChunk_In) ; + } /*******************************************************************************************/ return 1; @@ -467,8 +474,8 @@ void pqistreamer::locked_addTrafficClue(const RsItem *pqi,uint32_t pktsize,std:: rstime_t pqistreamer::getLastIncomingTS() { -// This is the only case where another thread (rs main for pqiperson) will access our data -// Still a mutex lock is not needed because the operation is atomic + // This is the only case where another thread (rs main for pqiperson) will access our data + // Still a mutex lock is not needed because the operation is atomic return mLastIncomingTs; } @@ -1145,7 +1152,7 @@ int pqistreamer::outAllowedBytes_locked() // low pass filter on mAvgDtOut mAvgDtOut = PQISTREAM_AVG_DT_FRAC * mAvgDtOut + (1 - PQISTREAM_AVG_DT_FRAC) * dt; - double maxout = getMaxRate(false) * 1024.0; + double maxout = getMaxRate_locked(false) * 1024.0; // this is used to take into account a possible excess of data sent during the previous round mCurrSent -= int(dt * maxout); @@ -1330,6 +1337,7 @@ int pqistreamer::gatherStatistics(std::list& outqueue_lst,std return locked_gatherStatistics(outqueue_lst,inqueue_lst); } +// this method is overloaded by pqiqosstreamer int pqistreamer::getQueueSize(bool in) { if (in) @@ -1342,6 +1350,19 @@ int pqistreamer::getQueueSize(bool in) } } +int pqistreamer::getQueueSize_bytes(bool in) +{ + if (in) +// no mutex is needed here because this is atomic +// for future use, mIncomingSize_bytes is not updated yet + return mIncomingSize_bytes; + else + { + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + return locked_compute_out_pkt_size(); + } +} + void pqistreamer::getRates(RsBwRates &rates) { RateInterface::getRates(rates); @@ -1349,10 +1370,13 @@ void pqistreamer::getRates(RsBwRates &rates) // no mutex is needed here because this is atomic rates.mQueueIn = mIncomingSize; - RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - rates.mQueueOut = locked_out_queue_size(); + { + RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + rates.mQueueOut = locked_out_queue_size(); + } } +// this method is overloaded by pqiqosstreamer int pqistreamer::locked_out_queue_size() const { // Warning: because out_pkt is a list, calling size @@ -1362,6 +1386,7 @@ int pqistreamer::locked_out_queue_size() const return mOutPkts.size() ; } +// this method is overloaded by pqiqosstreamer void pqistreamer::locked_clear_out_queue() { for(std::list::iterator it = mOutPkts.begin(); it != mOutPkts.end(); ) @@ -1376,6 +1401,7 @@ void pqistreamer::locked_clear_out_queue() } } +// this method is overloaded by pqiqosstreamer int pqistreamer::locked_compute_out_pkt_size() const { int total = 0 ; @@ -1394,6 +1420,7 @@ int pqistreamer::locked_gatherStatistics(std::list& out_lst,std:: return 1 ; } +// this method is overloaded by pqiqosstreamer void *pqistreamer::locked_pop_out_data(uint32_t /*max_slice_size*/, uint32_t &size, bool &starts, bool &ends, uint32_t &packet_id) { size = 0 ; diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index 1589448c9..0e7f6b815 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -38,8 +38,8 @@ class RsSerialiser; struct PartialPacketRecord { - void *mem ; - uint32_t size ; + void *mem ; + uint32_t size ; }; /** @@ -65,18 +65,23 @@ class pqistreamer: public PQInterface virtual RsItem *GetItem(); virtual int status(); - rstime_t getLastIncomingTS(); // Time of last data packet, for checking a connection is alive. + rstime_t getLastIncomingTS(); // Time of last data packet, for checking a connection is alive. virtual void getRates(RsBwRates &rates); virtual int getQueueSize(bool in); // extracting data. + virtual int getQueueSize_bytes(bool in); // size of incoming queue in bytes virtual int gatherStatistics(std::list& outqueue_stats,std::list& inqueue_stats); // extracting data. // mutex protected versions of RateInterface calls. virtual void setRate(bool b,float f) ; virtual void setMaxRate(bool b,float f) ; - virtual float getRate(bool b) ; + virtual void setMaxRate_locked(bool b,float f) ; - protected: - virtual int reset() ; + virtual float getRate(bool b) ; + virtual float getMaxRate(bool b) ; + virtual float getMaxRate_locked(bool b); + + protected: + virtual int reset() ; int tick_bio(); int tick_send(uint32_t timeout); @@ -129,13 +134,12 @@ class pqistreamer: public PQInterface void *mPkt_wpending; // storage for pending packet to write. uint32_t mPkt_wpending_size; // ... and its size. - void allocate_rpend(); // use these two functions to allocate/free the buffer below + void allocate_rpend(); // use these two functions to allocate/free the buffer below int mPkt_rpend_size; // size of pkt_rpending. void *mPkt_rpending; // storage for read in pending packets. - enum {reading_state_packet_started=1, - reading_state_initial=0 } ; + enum {reading_state_packet_started=1, reading_state_initial=0 } ; int mReading_state ; int mFailed_read_attempts ; @@ -144,7 +148,8 @@ class pqistreamer: public PQInterface std::list mOutPkts; // Cntrl / Search / Results queue std::list mIncoming; - uint32_t mIncomingSize; // size of mIncoming. To avoid calling linear cost std::list::size() + uint32_t mIncomingSize; // size of mIncoming. To avoid calling linear cost std::list::size() + uint32_t mIncomingSize_bytes; // size of Incoming in btyes // data for network stats. int mTotalRead; @@ -154,8 +159,8 @@ class pqistreamer: public PQInterface int mCurrRead; int mCurrSent; - double mCurrReadTS; // TS from which these are measured. - double mCurrSentTS; + double mCurrReadTS; // TS from which these are measured. + double mCurrSentTS; double mAvgLastUpdate; // TS from which these are measured. uint32_t mAvgReadCount; @@ -174,12 +179,12 @@ class pqistreamer: public PQInterface std::list mCurrentStatsChunk_Out ; rstime_t mStatisticsTimeStamp ; - bool mAcceptsPacketSlicing ; - rstime_t mLastSentPacketSlicingProbe ; - void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); - RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending,uint32_t& total_len); + bool mAcceptsPacketSlicing ; + rstime_t mLastSentPacketSlicingProbe ; + void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); + RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending,uint32_t& total_len); - std::map mPartialPackets ; + std::map mPartialPackets ; }; #endif //MRK_PQI_STREAMER_HEADER From 2c1238db2c397adf4539893480e9e0500716e478 Mon Sep 17 00:00:00 2001 From: jolavillette Date: Wed, 6 May 2020 12:11:54 +0200 Subject: [PATCH 06/72] remove unneeded lock of p3ServiceServer mutex --- libretroshare/src/pqi/pqiservice.cc | 120 +++++++++++----------------- 1 file changed, 48 insertions(+), 72 deletions(-) diff --git a/libretroshare/src/pqi/pqiservice.cc b/libretroshare/src/pqi/pqiservice.cc index 35a4fac64..86336631a 100644 --- a/libretroshare/src/pqi/pqiservice.cc +++ b/libretroshare/src/pqi/pqiservice.cc @@ -1,5 +1,5 @@ /******************************************************************************* - * libretroshare/src/pqi: pqiservice.h * + * libretroshare/src/pqi: pqiservice.cc * * * * libretroshare: retroshare core library * * * @@ -23,6 +23,22 @@ #include "util/rsdebug.h" #include "util/rsstring.h" +#include +#include +static double getCurrentTS() +{ +#ifndef WINDOWS_SYS + struct timeval cts_tmp; + gettimeofday(&cts_tmp, NULL); + double cts = (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0; +#else + struct _timeb timebuf; + _ftime( &timebuf); + double cts = (timebuf.time) + ((double) timebuf.millitm) / 1000.0; +#endif + return cts; +} + #ifdef SERVICE_DEBUG const int pqiservicezone = 60478; #endif @@ -44,7 +60,7 @@ bool pqiService::send(RsRawItem *item) p3ServiceServer::p3ServiceServer(pqiPublisher *pub, p3ServiceControl *ctrl) : mPublisher(pub), mServiceControl(ctrl), srvMtx("p3ServiceServer") { - RsStackMutex stack(srvMtx); /********* LOCKED *********/ + RS_STACK_MUTEX(srvMtx); /********* LOCKED *********/ #ifdef SERVICE_DEBUG pqioutput(PQL_DEBUG_BASIC, pqiservicezone, @@ -56,7 +72,7 @@ p3ServiceServer::p3ServiceServer(pqiPublisher *pub, p3ServiceControl *ctrl) : mP int p3ServiceServer::addService(pqiService *ts, bool defaultOn) { - RsStackMutex stack(srvMtx); /********* LOCKED *********/ + RS_STACK_MUTEX(srvMtx); /********* LOCKED *********/ #ifdef SERVICE_DEBUG pqioutput(PQL_DEBUG_BASIC, pqiservicezone, @@ -84,7 +100,7 @@ int p3ServiceServer::addService(pqiService *ts, bool defaultOn) bool p3ServiceServer::getServiceItemNames(uint32_t service_type,std::map& names) { - RsStackMutex stack(srvMtx); /********* LOCKED *********/ + RS_STACK_MUTEX(srvMtx); /********* LOCKED *********/ std::map::iterator it=services.find(service_type) ; @@ -99,7 +115,7 @@ bool p3ServiceServer::getServiceItemNames(uint32_t service_type,std::map PacketId(), (item -> PacketId() & 0xffffff00)); - item -> print_string(out); - std::cerr << out; - std::cerr << std::endl; - } -#endif - // Packet Filtering. // This doesn't need to be in Mutex. if (!mServiceControl->checkFilter(item->PacketId() & 0xffffff00, item->PeerId())) { -#ifdef SERVICE_DEBUG - std::cerr << "p3ServiceServer::recvItem() Fails Filtering " << std::endl; -#endif delete item; return false; } + pqiService *s = NULL; - std::map::iterator it; - it = services.find(item -> PacketId() & 0xffffff00); - if (it == services.end()) + // access the service map under mutex lock { -#ifdef SERVICE_DEBUG - std::cerr << "p3ServiceServer::incoming() Service: No Service - deleting"; - std::cerr << std::endl; -#endif - delete item; - return false; + RS_STACK_MUTEX(srvMtx); + auto it = services.find(item -> PacketId() & 0xffffff00); + if (it == services.end()) + { + delete item; + return false; + } + s = it->second; } - { -#ifdef SERVICE_DEBUG - std::cerr << "p3ServiceServer::incoming() Sending to : " << (void *) it -> second; - std::cerr << std::endl; -#endif - - return (it->second) -> recv(item); - } - - delete item; - return false; + // then call recv off mutex + bool result = s->recv(item); + return result; } - - bool p3ServiceServer::sendItem(RsRawItem *item) { #ifdef SERVICE_DEBUG @@ -204,40 +193,27 @@ bool p3ServiceServer::sendItem(RsRawItem *item) } mPublisher->sendItem(item); + return true; } - - int p3ServiceServer::tick() { - mServiceControl->tick(); - RsStackMutex stack(srvMtx); /********* LOCKED *********/ - -#ifdef SERVICE_DEBUG - pqioutput(PQL_DEBUG_ALL, pqiservicezone, - "p3ServiceServer::tick()"); -#endif - - std::map::iterator it; - - // from the beginning to where we started. - for(it = services.begin();it != services.end(); ++it) - { - -#ifdef SERVICE_DEBUG - std::string out; - rs_sprintf(out, "p3ServiceServer::service id: %u -> Service: %p", it -> first, it -> second); - pqioutput(PQL_DEBUG_ALL, pqiservicezone, out); -#endif - - // now we should actually tick the service. - (it -> second) -> tick(); + // make a copy of the service map + std::map local_map; + { + RS_STACK_MUTEX(srvMtx); + local_map=services; } + + // tick all services off mutex + for(auto it(local_map.begin());it!=local_map.end();++it) + { + (it->second)->tick(); + } + return 1; + } - - - From 4ead639e8cabdc51e4bdd7db540d7cca1123b112 Mon Sep 17 00:00:00 2001 From: jolavillette Date: Wed, 6 May 2020 21:29:46 +0200 Subject: [PATCH 07/72] remove unneeded pqithreadstreamer and pqissl mutex locks --- libretroshare/src/pqi/pqissl.cc | 9 ++++++--- libretroshare/src/pqi/pqithreadstreamer.cc | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/pqi/pqissl.cc b/libretroshare/src/pqi/pqissl.cc index 0ef930fc2..07993a4eb 100644 --- a/libretroshare/src/pqi/pqissl.cc +++ b/libretroshare/src/pqi/pqissl.cc @@ -372,9 +372,11 @@ int pqissl::status() // tick...... int pqissl::tick() { - RsStackMutex stack(mSslMtx); /**** LOCKED MUTEX ****/ + // there is no reason to lock pqissl mutex now + // we will lock the mutex later if we actually need to call to ConnectAttempt + // RsStackMutex stack(mSslMtx); /**** LOCKED MUTEX ****/ - //pqistreamer::tick(); + // pqistreamer::tick(); // continue existing connection attempt. if (!active) @@ -385,7 +387,8 @@ int pqissl::tick() #ifdef PQISSL_LOG_DEBUG rslog(RSL_DEBUG_BASIC, pqisslzone, "pqissl::tick() Continuing Connection Attempt!"); #endif - + // now lock pqissl mutex, that will take up to 10 ms + RsStackMutex stack(mSslMtx); /**** LOCKED MUTEX ****/ ConnectAttempt(); return 1; } diff --git a/libretroshare/src/pqi/pqithreadstreamer.cc b/libretroshare/src/pqi/pqithreadstreamer.cc index 857075ef3..6a84a2e0b 100644 --- a/libretroshare/src/pqi/pqithreadstreamer.cc +++ b/libretroshare/src/pqi/pqithreadstreamer.cc @@ -43,8 +43,10 @@ bool pqithreadstreamer::RecvItem(RsItem *item) int pqithreadstreamer::tick() { - RsStackMutex stack(mThreadMutex); - tick_bio(); +// pqithreadstreamer mutex lock is not needed here +// we are only checking if the connection is active, and if not active we will try to establish it +// RsStackMutex stack(mThreadMutex); + tick_bio(); return 0; } From 7c2efbc630d6ac0a0035d514b67cd1bb6b8cb7b7 Mon Sep 17 00:00:00 2001 From: jolavillette Date: Sat, 9 May 2020 20:35:20 +0200 Subject: [PATCH 08/72] reduce cpu load by increasing sleep before ticking pqi and core --- libretroshare/src/pqi/pqithreadstreamer.cc | 90 +++++++++++---------- libretroshare/src/rsserver/p3face-server.cc | 19 +++-- 2 files changed, 58 insertions(+), 51 deletions(-) diff --git a/libretroshare/src/pqi/pqithreadstreamer.cc b/libretroshare/src/pqi/pqithreadstreamer.cc index 6a84a2e0b..6745dc864 100644 --- a/libretroshare/src/pqi/pqithreadstreamer.cc +++ b/libretroshare/src/pqi/pqithreadstreamer.cc @@ -23,17 +23,17 @@ #include "pqi/pqithreadstreamer.h" #include -#define DEFAULT_STREAMER_TIMEOUT 10000 // 10 ms. -#define DEFAULT_STREAMER_SLEEP 1000 // 1 ms. +#define DEFAULT_STREAMER_TIMEOUT 10000 // 10 ms +#define DEFAULT_STREAMER_SLEEP 30000 // 30 ms #define DEFAULT_STREAMER_IDLE_SLEEP 1000000 // 1 sec -//#define PQISTREAMER_DEBUG +// #define PQISTREAMER_DEBUG pqithreadstreamer::pqithreadstreamer(PQInterface *parent, RsSerialiser *rss, const RsPeerId& id, BinInterface *bio_in, int bio_flags_in) :pqistreamer(rss, id, bio_in, bio_flags_in), mParent(parent), mTimeout(0), mThreadMutex("pqithreadstreamer") { - mTimeout = DEFAULT_STREAMER_TIMEOUT; - mSleepPeriod = DEFAULT_STREAMER_SLEEP; + mTimeout = DEFAULT_STREAMER_TIMEOUT; + mSleepPeriod = DEFAULT_STREAMER_SLEEP; } bool pqithreadstreamer::RecvItem(RsItem *item) @@ -43,9 +43,8 @@ bool pqithreadstreamer::RecvItem(RsItem *item) int pqithreadstreamer::tick() { -// pqithreadstreamer mutex lock is not needed here -// we are only checking if the connection is active, and if not active we will try to establish it -// RsStackMutex stack(mThreadMutex); + // pqithreadstreamer mutex lock is not needed here + // we will only check if the connection is active, and if not we will try to establish it tick_bio(); return 0; @@ -53,47 +52,50 @@ int pqithreadstreamer::tick() void pqithreadstreamer::threadTick() { - uint32_t recv_timeout = 0; - uint32_t sleep_period = 0; - bool isactive = false; - { - RsStackMutex stack(mStreamerMtx); - recv_timeout = mTimeout; - sleep_period = mSleepPeriod; - isactive = mBio->isactive(); - } + uint32_t recv_timeout = 0; + uint32_t sleep_period = 0; + bool isactive = false; + + { + RsStackMutex stack(mStreamerMtx); + recv_timeout = mTimeout; + sleep_period = mSleepPeriod; + isactive = mBio->isactive(); + } - updateRates() ; + // update the connection rates + updateRates() ; - if (!isactive) - { - rstime::rs_usleep(DEFAULT_STREAMER_IDLE_SLEEP); - return ; - } + // if the connection est not active, long sleep then return + if (!isactive) + { + rstime::rs_usleep(DEFAULT_STREAMER_IDLE_SLEEP); + return ; + } - { - RsStackMutex stack(mThreadMutex); - tick_recv(recv_timeout); - } + // fill incoming queue with items from SSL + { + RsStackMutex stack(mThreadMutex); + tick_recv(recv_timeout); + } - // Push Items, Outside of Mutex. - RsItem *incoming = NULL; - while((incoming = GetItem())) - { - RecvItem(incoming); - } + // move items to appropriate service queue or shortcut to fast service + RsItem *incoming = NULL; + while((incoming = GetItem())) + { + RecvItem(incoming); + } - { - RsStackMutex stack(mThreadMutex); - tick_send(0); - } + // parse the outgoing queue and send items to SSL + { + RsStackMutex stack(mThreadMutex); + tick_send(0); + } - if (sleep_period) - { - rstime::rs_usleep(sleep_period); - } + // sleep + if (sleep_period) + { + rstime::rs_usleep(sleep_period); + } } - - - diff --git a/libretroshare/src/rsserver/p3face-server.cc b/libretroshare/src/rsserver/p3face-server.cc index 0ed2f698c..9426b0471 100644 --- a/libretroshare/src/rsserver/p3face-server.cc +++ b/libretroshare/src/rsserver/p3face-server.cc @@ -143,13 +143,14 @@ void RsServer::threadTick() // if there is time left, we sleep double timeToSleep = mTickInterval - mAvgRunDuration; - if (timeToSleep > 0) - { +// never sleep less than 50 ms + if (timeToSleep < 0.050) + timeToSleep = 0.050; + #ifdef TICK_DEBUG - RsDbg() << "TICK_DEBUG will sleep " << timeToSleep << " ms" << std::endl; + RsDbg() << "TICK_DEBUG will sleep " << (int) (1000 * timeToSleep) << " ms" << std::endl; #endif - rstime::rs_usleep(timeToSleep * 1000000); - } + rstime::rs_usleep(timeToSleep * 1000000); double ts = getCurrentTS(); mLastts = ts; @@ -229,12 +230,16 @@ void RsServer::threadTick() // ticking is done, now compute new values of mLastRunDuration, mAvgRunDuration and mTickInterval ts = getCurrentTS(); mLastRunDuration = ts - mLastts; + +// low-pass filter and don't let mAvgRunDuration exceeds maxTickInterval mAvgRunDuration = 0.1 * mLastRunDuration + 0.9 * mAvgRunDuration; + if (mAvgRunDuration > maxTickInterval) + mAvgRunDuration = maxTickInterval; #ifdef TICK_DEBUG RsDbg() << "TICK_DEBUG new mLastRunDuration " << mLastRunDuration << " mAvgRunDuration " << mAvgRunDuration << std::endl; if (mLastRunDuration > WARN_BIG_CYCLE_TIME) - RsDbg() << "TICK_DEBUG excessively long lycle time " << mLastRunDuration << std::endl; + RsDbg() << "TICK_DEBUG excessively long cycle time " << mLastRunDuration << std::endl; #endif // if the core has returned that there is more to tick we decrease the ticking interval, else we increse it @@ -250,7 +255,7 @@ void RsServer::threadTick() RsDbg() << "TICK_DEBUG new tick interval " << mTickInterval << std::endl; #endif -// keep the tick interval within allowed limits +// keep the tick interval target within allowed limits if (mTickInterval < minTickInterval) mTickInterval = minTickInterval; else if (mTickInterval > maxTickInterval) From 1f547aaefecefbf6a470f1f59a53b9a9fcd65a5d Mon Sep 17 00:00:00 2001 From: jolavillette Date: Thu, 14 May 2020 23:13:36 +0200 Subject: [PATCH 09/72] bandwidth control improvement --- libretroshare/src/ft/fttransfermodule.cc | 6 +- libretroshare/src/pqi/pqihandler.cc | 173 +++++++++-------------- 2 files changed, 71 insertions(+), 108 deletions(-) diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index 8c6ae1f08..ab9900762 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -789,16 +789,16 @@ bool ftTransferModule::locked_tickPeerTransfer(peerInfo &info) std::cerr << std::endl; #endif - if (next_req > info.desiredRate * 1.1) + // cap next_req to desiredRate in order to respect the bandwidth limit and to avoid clogging our outqueue + if (next_req > info.desiredRate) { - next_req = info.desiredRate * 1.1; + next_req = info.desiredRate; #ifdef FT_DEBUG std::cerr << "locked_tickPeerTransfer() Reached MaxRate: next_req: " << next_req; std::cerr << std::endl; #endif } - if (next_req > FT_TM_MAX_PEER_RATE) { next_req = FT_TM_MAX_PEER_RATE; diff --git a/libretroshare/src/pqi/pqihandler.cc b/libretroshare/src/pqi/pqihandler.cc index 1e86fa9d1..72d7dbf9d 100644 --- a/libretroshare/src/pqi/pqihandler.cc +++ b/libretroshare/src/pqi/pqihandler.cc @@ -42,39 +42,19 @@ using std::dec; #include #endif -//#define PQI_HDL_DEBUG_UR 1 - -#ifdef PQI_HDL_DEBUG_UR -static double getCurrentTS() -{ - -#ifndef WINDOWS_SYS - struct timeval cts_tmp; - gettimeofday(&cts_tmp, NULL); - double cts = (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0; -#else - struct _timeb timebuf; - _ftime( &timebuf); - double cts = (timebuf.time) + ((double) timebuf.millitm) / 1000.0; -#endif - return cts; -} -#endif - struct RsLog::logInfo pqihandlerzoneInfo = {RsLog::Default, "pqihandler"}; #define pqihandlerzone &pqihandlerzoneInfo //static const int PQI_HANDLER_NB_PRIORITY_LEVELS = 10 ; //static const float PQI_HANDLER_NB_PRIORITY_RATIO = 2 ; -/**** -#define DEBUG_TICK 1 -#define RSITEM_DEBUG 1 -****/ +//#define UPDATE_RATES_DEBUG 1 +// #define DEBUG_TICK 1 +// #define RSITEM_DEBUG 1 pqihandler::pqihandler() : coreMtx("pqihandler") { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // setup minimal total+individual rates. rateIndiv_out = 0.01; @@ -97,7 +77,7 @@ int pqihandler::tick() int moreToTick = 0; { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // tick all interfaces... std::map::iterator it; @@ -127,9 +107,13 @@ int pqihandler::tick() if(now > mLastRateCapUpdate + 5) { + std::map rateMap; + std::map::iterator it; + + // every 5 secs, update the max rates for all modules - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ for(std::map::iterator it = mods.begin(); it != mods.end(); ++it) { // This is rather inelegant, but pqihandler has searchModules that are dynamically allocated, so the max rates @@ -149,7 +133,7 @@ int pqihandler::tick() bool pqihandler::queueOutRsItem(RsItem *item) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ uint32_t size ; locked_HandleRsItem(item, size); @@ -166,7 +150,7 @@ bool pqihandler::queueOutRsItem(RsItem *item) int pqihandler::status() { std::map::iterator it; - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ { // for output std::string out = "pqihandler::status() Active Modules:\n"; @@ -192,7 +176,7 @@ int pqihandler::status() bool pqihandler::AddSearchModule(SearchModule *mod) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // if peerid used -> error. //std::map::iterator it; if (mod->peerid != mod->pqi->PeerId()) @@ -223,7 +207,7 @@ bool pqihandler::AddSearchModule(SearchModule *mod) bool pqihandler::RemoveSearchModule(SearchModule *mod) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) { @@ -313,7 +297,7 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat total.mQueueOut = 0; /* Lock once rates have been retrieved */ - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) @@ -340,10 +324,6 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat // internal fn to send updates int pqihandler::UpdateRates() { -#ifdef PQI_HDL_DEBUG_UR - uint64_t t_now; -#endif - std::map::iterator it; float avail_in = getMaxRate(true); @@ -353,18 +333,15 @@ int pqihandler::UpdateRates() float used_bw_out = 0; /* Lock once rates have been retrieved */ - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ int num_sm = mods.size(); float used_bw_in_table[num_sm]; /* table of in bandwidth currently used by each module */ float used_bw_out_table[num_sm]; /* table of out bandwidth currently used by each module */ - int effectiveUploadsSm = 0; - int effectiveDownloadsSm = 0; - - // loop through modules to get the used bandwith and the number of modules that are affectively transfering -#ifdef PQI_HDL_DEBUG_UR - std::cerr << "Looping through modules" << std::endl; + // loop through modules to get the used bandwidth +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Looping through modules" << std::endl; #endif int index = 0; @@ -372,49 +349,33 @@ int pqihandler::UpdateRates() for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - float crate_in = mod -> pqi -> getRate(true); traffInSum += mod -> pqi -> getTraffic(true); traffOutSum += mod -> pqi -> getTraffic(false); -#ifdef PQI_HDL_DEBUG_UR - if(crate_in > 0.0) - std::cerr << " got in rate for peer " << it->first << " : " << crate_in << std::endl; -#endif - - if ((crate_in > 0.01 * avail_in) || (crate_in > 0.1)) - { - ++effectiveDownloadsSm; - } - + float crate_in = mod -> pqi -> getRate(true); float crate_out = mod -> pqi -> getRate(false); - if ((crate_out > 0.01 * avail_out) || (crate_out > 0.1)) - { - ++effectiveUploadsSm; - } used_bw_in += crate_in; used_bw_out += crate_out; - /* fill the table of bandwidth */ + /* fill the table of used bandwidths */ used_bw_in_table[index] = crate_in; used_bw_out_table[index] = crate_out; + ++index; } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; #endif /* Sort the used bw in/out table in ascending order */ std::sort(used_bw_in_table, used_bw_in_table + num_sm); std::sort(used_bw_out_table, used_bw_out_table + num_sm); -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): Done." << std::endl; - std::cerr << dec << t_now << " pqihandler::UpdateRates(): used_bw_out " << used_bw_out << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates used_bw_out " << used_bw_out << std::endl; #endif /* Calculate the optimal out_max value, taking into account avail_out and the out bw requested by modules */ @@ -441,9 +402,8 @@ int pqihandler::UpdateRates() } } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; #endif /* Allocate only half the remaining out bw, if any, to make it smoother */ @@ -473,67 +433,70 @@ int pqihandler::UpdateRates() } } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; #endif /* Allocate only half the remaining in bw, if any, to make it smoother */ in_max_bw = in_max_bw + in_remaining_bw / 2; - -#ifdef DEBUG_QOS -// std::cerr << "Totals (In) Used B/W " << used_bw_in; -// std::cerr << " Available B/W " << avail_in; -// std::cerr << " Effective transfers " << effectiveDownloadsSm << std::endl; -// std::cerr << "Totals (Out) Used B/W " << used_bw_out; -// std::cerr << " Available B/W " << avail_out; -// std::cerr << " Effective transfers " << effectiveUploadsSm << std::endl; -#endif - + // store current total in and ou used bw locked_StoreCurrentRates(used_bw_in, used_bw_out); - //computing average rates for effective transfers - float max_in_effective = avail_in / num_sm; - if (effectiveDownloadsSm != 0) { - max_in_effective = avail_in / effectiveDownloadsSm; - } - float max_out_effective = avail_out / num_sm; - if (effectiveUploadsSm != 0) { - max_out_effective = avail_out / effectiveUploadsSm; - } - - //modify the in and out limit -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; #endif + // retrieve down (from peer point of view) bandwidth limits set by peers in their own settings + std::map rateMap; + rsConfig->getAllBandwidthRates(rateMap); + std::map::iterator rateMap_it; + +#ifdef UPDATE_RATES_DEBUG + // Dump RsConfigurationDataRates + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates RsConfigDataRates dump" << std::endl; + for (rateMap_it = rateMap.begin(); rateMap_it != rateMap.end(); rateMap_it++) + RsDbg () << "UPDATE_RATES pqihandler::UpdateRates PeerId " << rateMap_it->first.toStdString() << " mAllowedOut " << rateMap_it->second.mAllowedOut << std::endl; +#endif + + // update max rates taking into account the limits set by peers in their own settings for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - - mod -> pqi -> setMaxRate(true, in_max_bw); - mod -> pqi -> setMaxRate(false, out_max_bw); + + // for our down bandwidth we set the max to the calculated value without taking into account the max set by peers: they will control their up bw on their side + mod -> pqi -> setMaxRate(true, in_max_bw); + + // for our up bandwidth we limit to the maximum down bw provided by peers via BwCtrl because we don't want to clog our outqueues, the SSL buffers, and our friends inbound queues + if ((rateMap_it = rateMap.find(mod->pqi->PeerId())) != rateMap.end()) + { + if (rateMap_it->second.mAllowedOut > 0) + { + if (out_max_bw > rateMap_it->second.mAllowedOut) + mod -> pqi -> setMaxRate(false, rateMap_it->second.mAllowedOut); + else + mod -> pqi -> setMaxRate(false, out_max_bw); + } + else + mod -> pqi -> setMaxRate(false, out_max_bw); + } } - - //cap the rates +#ifdef UPDATE_RATES_DEBUG + // dump maxRates for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - if (mod -> pqi -> getMaxRate(false) < max_out_effective) mod -> pqi -> setMaxRate(false, max_out_effective); - if (mod -> pqi -> getMaxRate(false) > avail_out) mod -> pqi -> setMaxRate(false, avail_out); - if (mod -> pqi -> getMaxRate(true) < max_in_effective) mod -> pqi -> setMaxRate(true, max_in_effective); - if (mod -> pqi -> getMaxRate(true) > avail_in) mod -> pqi -> setMaxRate(true, avail_in); + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates PeerID " << (mod ->pqi -> PeerId()).toStdString() << " new bandwidth limits up " << mod -> pqi -> getMaxRate(false) << " down " << mod -> pqi -> getMaxRate(true) << std::endl; } +#endif return 1; } void pqihandler::getCurrentRates(float &in, float &out) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ in = rateTotal_in; out = rateTotal_out; From 51ba879bb72f272679f8545a2a2a19ed667444ff Mon Sep 17 00:00:00 2001 From: jolavillette Date: Thu, 14 May 2020 23:28:20 +0200 Subject: [PATCH 10/72] bandwidth control improvement --- libretroshare/src/ft/fttransfermodule.cc | 6 +- libretroshare/src/pqi/pqihandler.cc | 173 +++++++++-------------- 2 files changed, 71 insertions(+), 108 deletions(-) diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index 8c6ae1f08..ab9900762 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -789,16 +789,16 @@ bool ftTransferModule::locked_tickPeerTransfer(peerInfo &info) std::cerr << std::endl; #endif - if (next_req > info.desiredRate * 1.1) + // cap next_req to desiredRate in order to respect the bandwidth limit and to avoid clogging our outqueue + if (next_req > info.desiredRate) { - next_req = info.desiredRate * 1.1; + next_req = info.desiredRate; #ifdef FT_DEBUG std::cerr << "locked_tickPeerTransfer() Reached MaxRate: next_req: " << next_req; std::cerr << std::endl; #endif } - if (next_req > FT_TM_MAX_PEER_RATE) { next_req = FT_TM_MAX_PEER_RATE; diff --git a/libretroshare/src/pqi/pqihandler.cc b/libretroshare/src/pqi/pqihandler.cc index 1e86fa9d1..72d7dbf9d 100644 --- a/libretroshare/src/pqi/pqihandler.cc +++ b/libretroshare/src/pqi/pqihandler.cc @@ -42,39 +42,19 @@ using std::dec; #include #endif -//#define PQI_HDL_DEBUG_UR 1 - -#ifdef PQI_HDL_DEBUG_UR -static double getCurrentTS() -{ - -#ifndef WINDOWS_SYS - struct timeval cts_tmp; - gettimeofday(&cts_tmp, NULL); - double cts = (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0; -#else - struct _timeb timebuf; - _ftime( &timebuf); - double cts = (timebuf.time) + ((double) timebuf.millitm) / 1000.0; -#endif - return cts; -} -#endif - struct RsLog::logInfo pqihandlerzoneInfo = {RsLog::Default, "pqihandler"}; #define pqihandlerzone &pqihandlerzoneInfo //static const int PQI_HANDLER_NB_PRIORITY_LEVELS = 10 ; //static const float PQI_HANDLER_NB_PRIORITY_RATIO = 2 ; -/**** -#define DEBUG_TICK 1 -#define RSITEM_DEBUG 1 -****/ +//#define UPDATE_RATES_DEBUG 1 +// #define DEBUG_TICK 1 +// #define RSITEM_DEBUG 1 pqihandler::pqihandler() : coreMtx("pqihandler") { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // setup minimal total+individual rates. rateIndiv_out = 0.01; @@ -97,7 +77,7 @@ int pqihandler::tick() int moreToTick = 0; { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // tick all interfaces... std::map::iterator it; @@ -127,9 +107,13 @@ int pqihandler::tick() if(now > mLastRateCapUpdate + 5) { + std::map rateMap; + std::map::iterator it; + + // every 5 secs, update the max rates for all modules - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ for(std::map::iterator it = mods.begin(); it != mods.end(); ++it) { // This is rather inelegant, but pqihandler has searchModules that are dynamically allocated, so the max rates @@ -149,7 +133,7 @@ int pqihandler::tick() bool pqihandler::queueOutRsItem(RsItem *item) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ uint32_t size ; locked_HandleRsItem(item, size); @@ -166,7 +150,7 @@ bool pqihandler::queueOutRsItem(RsItem *item) int pqihandler::status() { std::map::iterator it; - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ { // for output std::string out = "pqihandler::status() Active Modules:\n"; @@ -192,7 +176,7 @@ int pqihandler::status() bool pqihandler::AddSearchModule(SearchModule *mod) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // if peerid used -> error. //std::map::iterator it; if (mod->peerid != mod->pqi->PeerId()) @@ -223,7 +207,7 @@ bool pqihandler::AddSearchModule(SearchModule *mod) bool pqihandler::RemoveSearchModule(SearchModule *mod) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) { @@ -313,7 +297,7 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat total.mQueueOut = 0; /* Lock once rates have been retrieved */ - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) @@ -340,10 +324,6 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat // internal fn to send updates int pqihandler::UpdateRates() { -#ifdef PQI_HDL_DEBUG_UR - uint64_t t_now; -#endif - std::map::iterator it; float avail_in = getMaxRate(true); @@ -353,18 +333,15 @@ int pqihandler::UpdateRates() float used_bw_out = 0; /* Lock once rates have been retrieved */ - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ int num_sm = mods.size(); float used_bw_in_table[num_sm]; /* table of in bandwidth currently used by each module */ float used_bw_out_table[num_sm]; /* table of out bandwidth currently used by each module */ - int effectiveUploadsSm = 0; - int effectiveDownloadsSm = 0; - - // loop through modules to get the used bandwith and the number of modules that are affectively transfering -#ifdef PQI_HDL_DEBUG_UR - std::cerr << "Looping through modules" << std::endl; + // loop through modules to get the used bandwidth +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Looping through modules" << std::endl; #endif int index = 0; @@ -372,49 +349,33 @@ int pqihandler::UpdateRates() for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - float crate_in = mod -> pqi -> getRate(true); traffInSum += mod -> pqi -> getTraffic(true); traffOutSum += mod -> pqi -> getTraffic(false); -#ifdef PQI_HDL_DEBUG_UR - if(crate_in > 0.0) - std::cerr << " got in rate for peer " << it->first << " : " << crate_in << std::endl; -#endif - - if ((crate_in > 0.01 * avail_in) || (crate_in > 0.1)) - { - ++effectiveDownloadsSm; - } - + float crate_in = mod -> pqi -> getRate(true); float crate_out = mod -> pqi -> getRate(false); - if ((crate_out > 0.01 * avail_out) || (crate_out > 0.1)) - { - ++effectiveUploadsSm; - } used_bw_in += crate_in; used_bw_out += crate_out; - /* fill the table of bandwidth */ + /* fill the table of used bandwidths */ used_bw_in_table[index] = crate_in; used_bw_out_table[index] = crate_out; + ++index; } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; #endif /* Sort the used bw in/out table in ascending order */ std::sort(used_bw_in_table, used_bw_in_table + num_sm); std::sort(used_bw_out_table, used_bw_out_table + num_sm); -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): Done." << std::endl; - std::cerr << dec << t_now << " pqihandler::UpdateRates(): used_bw_out " << used_bw_out << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates used_bw_out " << used_bw_out << std::endl; #endif /* Calculate the optimal out_max value, taking into account avail_out and the out bw requested by modules */ @@ -441,9 +402,8 @@ int pqihandler::UpdateRates() } } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; #endif /* Allocate only half the remaining out bw, if any, to make it smoother */ @@ -473,67 +433,70 @@ int pqihandler::UpdateRates() } } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; #endif /* Allocate only half the remaining in bw, if any, to make it smoother */ in_max_bw = in_max_bw + in_remaining_bw / 2; - -#ifdef DEBUG_QOS -// std::cerr << "Totals (In) Used B/W " << used_bw_in; -// std::cerr << " Available B/W " << avail_in; -// std::cerr << " Effective transfers " << effectiveDownloadsSm << std::endl; -// std::cerr << "Totals (Out) Used B/W " << used_bw_out; -// std::cerr << " Available B/W " << avail_out; -// std::cerr << " Effective transfers " << effectiveUploadsSm << std::endl; -#endif - + // store current total in and ou used bw locked_StoreCurrentRates(used_bw_in, used_bw_out); - //computing average rates for effective transfers - float max_in_effective = avail_in / num_sm; - if (effectiveDownloadsSm != 0) { - max_in_effective = avail_in / effectiveDownloadsSm; - } - float max_out_effective = avail_out / num_sm; - if (effectiveUploadsSm != 0) { - max_out_effective = avail_out / effectiveUploadsSm; - } - - //modify the in and out limit -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; #endif + // retrieve down (from peer point of view) bandwidth limits set by peers in their own settings + std::map rateMap; + rsConfig->getAllBandwidthRates(rateMap); + std::map::iterator rateMap_it; + +#ifdef UPDATE_RATES_DEBUG + // Dump RsConfigurationDataRates + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates RsConfigDataRates dump" << std::endl; + for (rateMap_it = rateMap.begin(); rateMap_it != rateMap.end(); rateMap_it++) + RsDbg () << "UPDATE_RATES pqihandler::UpdateRates PeerId " << rateMap_it->first.toStdString() << " mAllowedOut " << rateMap_it->second.mAllowedOut << std::endl; +#endif + + // update max rates taking into account the limits set by peers in their own settings for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - - mod -> pqi -> setMaxRate(true, in_max_bw); - mod -> pqi -> setMaxRate(false, out_max_bw); + + // for our down bandwidth we set the max to the calculated value without taking into account the max set by peers: they will control their up bw on their side + mod -> pqi -> setMaxRate(true, in_max_bw); + + // for our up bandwidth we limit to the maximum down bw provided by peers via BwCtrl because we don't want to clog our outqueues, the SSL buffers, and our friends inbound queues + if ((rateMap_it = rateMap.find(mod->pqi->PeerId())) != rateMap.end()) + { + if (rateMap_it->second.mAllowedOut > 0) + { + if (out_max_bw > rateMap_it->second.mAllowedOut) + mod -> pqi -> setMaxRate(false, rateMap_it->second.mAllowedOut); + else + mod -> pqi -> setMaxRate(false, out_max_bw); + } + else + mod -> pqi -> setMaxRate(false, out_max_bw); + } } - - //cap the rates +#ifdef UPDATE_RATES_DEBUG + // dump maxRates for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - if (mod -> pqi -> getMaxRate(false) < max_out_effective) mod -> pqi -> setMaxRate(false, max_out_effective); - if (mod -> pqi -> getMaxRate(false) > avail_out) mod -> pqi -> setMaxRate(false, avail_out); - if (mod -> pqi -> getMaxRate(true) < max_in_effective) mod -> pqi -> setMaxRate(true, max_in_effective); - if (mod -> pqi -> getMaxRate(true) > avail_in) mod -> pqi -> setMaxRate(true, avail_in); + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates PeerID " << (mod ->pqi -> PeerId()).toStdString() << " new bandwidth limits up " << mod -> pqi -> getMaxRate(false) << " down " << mod -> pqi -> getMaxRate(true) << std::endl; } +#endif return 1; } void pqihandler::getCurrentRates(float &in, float &out) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ in = rateTotal_in; out = rateTotal_out; From 74646413c4f82db31c1bb5f13fc6c771dd59b59d Mon Sep 17 00:00:00 2001 From: defnax Date: Sat, 23 May 2020 00:16:49 +0200 Subject: [PATCH 11/72] Added mail settings for emoticons --- retroshare-gui/src/gui/msgs/MessageWidget.cpp | 10 +++++--- .../src/gui/settings/MessagePage.cpp | 7 +++++- retroshare-gui/src/gui/settings/MessagePage.h | 1 + .../src/gui/settings/MessagePage.ui | 23 ++++++++++++------- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/retroshare-gui/src/gui/msgs/MessageWidget.cpp b/retroshare-gui/src/gui/msgs/MessageWidget.cpp index 3c085c3c8..a9a004a51 100644 --- a/retroshare-gui/src/gui/msgs/MessageWidget.cpp +++ b/retroshare-gui/src/gui/msgs/MessageWidget.cpp @@ -667,10 +667,14 @@ void MessageWidget::fill(const std::string &msgId) } ui.subjectText->setText(QString::fromUtf8(msgInfo.title.c_str())); + + unsigned int formatTextFlag = RSHTML_FORMATTEXT_EMBED_LINKS ; - // emoticons disabled because of crazy cost. - //text = RsHtmlMsg(msgInfo.msgflags).formatText(ui.msgText->document(), QString::fromUtf8(msgInfo.msg.c_str()), RSHTML_FORMATTEXT_EMBED_SMILEYS | RSHTML_FORMATTEXT_EMBED_LINKS); - text = RsHtmlMsg(msgInfo.msgflags).formatText(ui.msgText->document(), QString::fromUtf8(msgInfo.msg.c_str()), RSHTML_FORMATTEXT_EMBED_LINKS); + // embed smileys ? + if (Settings->valueFromGroup(QString("Messages"), QString::fromUtf8("Emoticons"), true).toBool()) { + formatTextFlag |= RSHTML_FORMATTEXT_EMBED_SMILEYS ; + } + text = RsHtmlMsg(msgInfo.msgflags).formatText(ui.msgText->document(), QString::fromUtf8(msgInfo.msg.c_str()), formatTextFlag); ui.msgText->resetImagesStatus(Settings->getMsgLoadEmbeddedImages() || (msgInfo.msgflags & RS_MSG_LOAD_EMBEDDED_IMAGES)); ui.msgText->setHtml(text); diff --git a/retroshare-gui/src/gui/settings/MessagePage.cpp b/retroshare-gui/src/gui/settings/MessagePage.cpp index eb0eabccc..91a24ab02 100644 --- a/retroshare-gui/src/gui/settings/MessagePage.cpp +++ b/retroshare-gui/src/gui/settings/MessagePage.cpp @@ -39,7 +39,6 @@ MessagePage::MessagePage(QWidget * parent, Qt::WindowFlags flags) connect (ui.editpushButton, SIGNAL(clicked(bool)), this, SLOT (editTag())); connect (ui.deletepushButton, SIGNAL(clicked(bool)), this, SLOT (deleteTag())); connect (ui.defaultTagButton, SIGNAL(clicked(bool)), this, SLOT (defaultTag())); - //connect (ui.encryptedMsgs_CB, SIGNAL(toggled(bool)), this, SLOT (toggleEnableEncryptedDistantMsgs(bool))); connect (ui.tags_listWidget, SIGNAL(currentRowChanged(int)), this, SLOT(currentRowChangedTag(int))); @@ -54,6 +53,7 @@ MessagePage::MessagePage(QWidget * parent, Qt::WindowFlags flags) connect(ui.setMsgToReadOnActivate,SIGNAL(toggled(bool)), this,SLOT(updateMsgToReadOnActivate())); connect(ui.loadEmbeddedImages, SIGNAL(toggled(bool)), this,SLOT(updateLoadEmbededImages() )); connect(ui.openComboBox, SIGNAL(currentIndexChanged(int)),this,SLOT(updateMsgOpen() )); + connect(ui.emoticonscheckBox, SIGNAL(toggled(bool)), this,SLOT(updateLoadEmoticons() )); } MessagePage::~MessagePage() @@ -84,6 +84,7 @@ void MessagePage::updateMsgToReadOnActivate() { Settings->setMsgSetToReadOnActiv void MessagePage::updateLoadEmbededImages() { Settings->setMsgLoadEmbeddedImages(ui.loadEmbeddedImages->isChecked()); } void MessagePage::updateMsgOpen() { Settings->setMsgOpen( static_cast(ui.openComboBox->itemData(ui.openComboBox->currentIndex()).toInt()) ); } void MessagePage::updateDistantMsgs() { Settings->setValue("DistantMessages", ui.comboBox->currentIndex()); } +void MessagePage::updateLoadEmoticons() { Settings->setValueToGroup("Messages", "Emoticons", ui.emoticonscheckBox->isChecked()); } void MessagePage::updateMsgTags() { @@ -110,9 +111,12 @@ void MessagePage::updateMsgTags() void MessagePage::load() { + Settings->beginGroup(QString("Messages")); whileBlocking(ui.setMsgToReadOnActivate)->setChecked(Settings->getMsgSetToReadOnActivate()); whileBlocking(ui.loadEmbeddedImages)->setChecked(Settings->getMsgLoadEmbeddedImages()); whileBlocking(ui.openComboBox)->setCurrentIndex(ui.openComboBox->findData(Settings->getMsgOpen())); + whileBlocking(ui.emoticonscheckBox)->setChecked(Settings->value("Emoticons", true).toBool()); + Settings->endGroup(); // state of filter combobox @@ -267,3 +271,4 @@ void MessagePage::currentRowChangedTag(int row) ui.editpushButton->setEnabled(bEditEnable); ui.deletepushButton->setEnabled(bDeleteEnable); } + diff --git a/retroshare-gui/src/gui/settings/MessagePage.h b/retroshare-gui/src/gui/settings/MessagePage.h index b4c2e9c67..6dd6f87df 100644 --- a/retroshare-gui/src/gui/settings/MessagePage.h +++ b/retroshare-gui/src/gui/settings/MessagePage.h @@ -58,6 +58,7 @@ private slots: void updateMsgOpen() ; void updateDistantMsgs() ; void updateMsgTags() ; + void updateLoadEmoticons(); private: void fillTags(); diff --git a/retroshare-gui/src/gui/settings/MessagePage.ui b/retroshare-gui/src/gui/settings/MessagePage.ui index 57d801bed..58da48b9f 100644 --- a/retroshare-gui/src/gui/settings/MessagePage.ui +++ b/retroshare-gui/src/gui/settings/MessagePage.ui @@ -65,14 +65,7 @@ - - - - Load embedded images - - - - + @@ -86,6 +79,20 @@ + + + + Load embedded images + + + + + + + Load Emoticons + + + From e5e468b083059cbbb091c5e8ffc7605c5ec54e60 Mon Sep 17 00:00:00 2001 From: Phenom Date: Tue, 12 May 2020 19:52:49 +0200 Subject: [PATCH 12/72] Fix AutoLogin --- libretroshare/src/libretroshare.pro | 2 ++ libretroshare/src/rsserver/rsloginhandler.cc | 10 +++++++++- libretroshare/src/use_libretroshare.pri | 5 +++++ retroshare-gui/src/retroshare-gui.pro | 4 ++++ retroshare-service/src/retroshare-service.pro | 10 ++++++++++ 5 files changed, 30 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index c7b38ce0b..495caf4b2 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -327,6 +327,8 @@ INCLUDEPATH *= $${OPENPGPSDK_DIR} PRE_TARGETDEPS *= $${OPENPGPSDK_DIR}/lib/libops.a LIBS *= $${OPENPGPSDK_DIR}/lib/libops.a -lbz2 +################################### HEADERS & SOURCES ############################# + HEADERS += ft/ftchunkmap.h \ ft/ftcontroller.h \ ft/ftdata.h \ diff --git a/libretroshare/src/rsserver/rsloginhandler.cc b/libretroshare/src/rsserver/rsloginhandler.cc index 8a0469e1d..dbe023235 100644 --- a/libretroshare/src/rsserver/rsloginhandler.cc +++ b/libretroshare/src/rsserver/rsloginhandler.cc @@ -26,6 +26,7 @@ #include "rsloginhandler.h" #include "util/rsdir.h" #include "retroshare/rsinit.h" +#include "util/rsdebug.h" //#define DEBUG_RSLOGINHANDLER 1 @@ -497,8 +498,15 @@ bool RsLoginHandler::enableAutoLogin(const RsPeerId& ssl_id,const std::string& s NULL); if (error) { + RsErr() << __PRETTY_FUNCTION__ + << " Could not store passwd using libsecret with" + << " error.code=" << error->code + << " error.domain=" << error->domain + << " error.message=\"" << error->message << "\"" << std::endl; + if (error->code == 2) + RsErr() << "Do have a key wallet installed?" << std::endl + << "Like gnome-keyring or other using \"Secret Service\" by DBus." << std::endl; g_error_free (error); - std::cerr << "Could not store passwd using libsecret" << std::endl; return false; } std::cout << "Stored passwd " << "************************" << " using libsecret" << std::endl; diff --git a/libretroshare/src/use_libretroshare.pri b/libretroshare/src/use_libretroshare.pri index 681805a7f..eb7a4e470 100644 --- a/libretroshare/src/use_libretroshare.pri +++ b/libretroshare/src/use_libretroshare.pri @@ -102,3 +102,8 @@ android-* { CONFIG *= qt QT *= network } + +################################### Pkg-Config Stuff ############################# + +LIBS *= $$system(pkg-config --libs $$PKGCONFIG) + diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index a5d6838ae..68f2987c0 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -344,12 +344,16 @@ openbsd-* { LIBS *= -rdynamic } +################################### COMMON stuff ################################## + wikipoos { PRE_TARGETDEPS *= $$OUT_PWD/../../supportlibs/pegmarkdown/lib/libpegmarkdown.a LIBS *= $$OUT_PWD/../../supportlibs/pegmarkdown/lib/libpegmarkdown.a LIBS *= -lglib-2.0 } +################################### HEADERS & SOURCES ############################# + # Tor controller HEADERS += TorControl/AddOnionCommand.h \ diff --git a/retroshare-service/src/retroshare-service.pro b/retroshare-service/src/retroshare-service.pro index 75cb0b581..ae3b38049 100644 --- a/retroshare-service/src/retroshare-service.pro +++ b/retroshare-service/src/retroshare-service.pro @@ -29,6 +29,8 @@ QT -= gui SOURCES += retroshare-service.cc +################################# Linux ########################################## + android-* { QT += androidextras @@ -56,11 +58,14 @@ appimage { INSTALLS += desktop_files } + unix { target.path = "$${RS_BIN_DIR}" INSTALLS += target } +################################# MacOSX ########################################## + macx { # ENABLE THIS OPTION FOR Univeral Binary BUILD. #CONFIG += ppc x86 @@ -81,6 +86,8 @@ macx { INCLUDEPATH += . $$INC_DIR } +################################# Windows ########################################## + win32-g++ { CONFIG(debug, debug|release) { # show console output @@ -130,3 +137,6 @@ win32-g++ { QMAKE_PRE_LINK = $(CHK_DIR_EXISTS) lib || $(MKDIR) lib } } + +################################### COMMON stuff ################################## + From ea827b180a29806e80f916e437c370f526c48c76 Mon Sep 17 00:00:00 2001 From: jolavillette Date: Sat, 23 May 2020 22:20:49 +0200 Subject: [PATCH 13/72] restore 1.1 factor in ft/fttransfermodule --- libretroshare/src/ft/fttransfermodule.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index ab9900762..8c6ae1f08 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -789,16 +789,16 @@ bool ftTransferModule::locked_tickPeerTransfer(peerInfo &info) std::cerr << std::endl; #endif - // cap next_req to desiredRate in order to respect the bandwidth limit and to avoid clogging our outqueue - if (next_req > info.desiredRate) + if (next_req > info.desiredRate * 1.1) { - next_req = info.desiredRate; + next_req = info.desiredRate * 1.1; #ifdef FT_DEBUG std::cerr << "locked_tickPeerTransfer() Reached MaxRate: next_req: " << next_req; std::cerr << std::endl; #endif } + if (next_req > FT_TM_MAX_PEER_RATE) { next_req = FT_TM_MAX_PEER_RATE; From 735a123a02236d551f95b3f2e641e051a3ba9f10 Mon Sep 17 00:00:00 2001 From: jolavillette Date: Sun, 24 May 2020 00:32:24 +0200 Subject: [PATCH 14/72] restore 1.1 factor in ft/fttransfermodule --- libretroshare/src/ft/fttransfermodule.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index ab9900762..8c6ae1f08 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -789,16 +789,16 @@ bool ftTransferModule::locked_tickPeerTransfer(peerInfo &info) std::cerr << std::endl; #endif - // cap next_req to desiredRate in order to respect the bandwidth limit and to avoid clogging our outqueue - if (next_req > info.desiredRate) + if (next_req > info.desiredRate * 1.1) { - next_req = info.desiredRate; + next_req = info.desiredRate * 1.1; #ifdef FT_DEBUG std::cerr << "locked_tickPeerTransfer() Reached MaxRate: next_req: " << next_req; std::cerr << std::endl; #endif } + if (next_req > FT_TM_MAX_PEER_RATE) { next_req = FT_TM_MAX_PEER_RATE; From 178b37a95f6682026747f931b714a6c8307ff979 Mon Sep 17 00:00:00 2001 From: defnax Date: Sun, 24 May 2020 02:28:54 +0200 Subject: [PATCH 15/72] small ui fixes & changes * Update the context menu icons for attachments * Display the File type icon for attachments * on reply load the tags --- retroshare-gui/src/gui/icons.qrc | 1 + .../src/gui/icons/mail/downloadall.png | Bin 0 -> 2986 bytes retroshare-gui/src/gui/msgs/MessageComposer.cpp | 6 ++++++ retroshare-gui/src/gui/msgs/MessageWidget.cpp | 6 ++++-- retroshare-gui/src/gui/msgs/MessagesDialog.cpp | 3 ++- 5 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 retroshare-gui/src/gui/icons/mail/downloadall.png diff --git a/retroshare-gui/src/gui/icons.qrc b/retroshare-gui/src/gui/icons.qrc index 311530b03..6e4bd7569 100644 --- a/retroshare-gui/src/gui/icons.qrc +++ b/retroshare-gui/src/gui/icons.qrc @@ -301,6 +301,7 @@ icons/png/arrow-left.png icons/png/next-unread.png icons/mail/compose.png + icons/mail/downloadall.png icons/mail/delete.png icons/mail/tags.png icons/mail/quote.png diff --git a/retroshare-gui/src/gui/icons/mail/downloadall.png b/retroshare-gui/src/gui/icons/mail/downloadall.png new file mode 100644 index 0000000000000000000000000000000000000000..b4d31d3d0a1c6242f3c55cbbe56cb96f0d4ed7a7 GIT binary patch literal 2986 zcmZ`*c{r5q7k)B}EvgTxNFg=ZLyGJfYnHKxN{c;FNHmr#DN9nJS*(!?g-8lX3&}oa z42t5T5(!yGgWvOxx~}h^-*vrnp65B|zR$VOdERSgu3B3f3-E2>0{{q^ni$%^H46Fg zprHNrDU}5mw5Ps>J^*DY{Oscxc;0@}#Kr=Eu$>SW3&0|R{|Z2`G5}+40BGI@Ks12* zjGzSo*NF8YJ3ItnNDMTEj-t|0i0$U0bc1X<*Xof%han7=4lEOW%n!-R) z7=Xk83y~RRXnt4G6Rtf1;9}wV}TU7fe%dH4F`fC z{{a5%(eO?=vs(3INKgqP23jnfWcDCZ0=c!8)s|s$aHAij!^RN|rwU^KL|64Ah2YHL zSP(yw4Q`MJz5{II?`IyvqCv9R<5fcj(-Qm3Bk;q`vO49 z;cg9l(Px8fENy{GT-ozN-%Y3QmB#MbQZai875#Yg&VQfq$?!AldP3{hh77zGeDO*0 z%{I-ZbsEa8LhBU-SS|l7RJIE*cZ#UTZ0w3!L!vWo^oogglew<DSZ#{SlWBWeRBNd&;6G|La;x4$}@}@6mkA?M(RyUT4vT=bWZO5#{9R%*he*04NrMrRu?>= z-nz#0{E2*htT|zBYzVGkwwk~LZ-B|+KmY_^BcC;Ca)M&e$sJ^Bf#?1gBf4>qgd1b- zECASeQ$u~b3w;w5Z#@xv;YNSHq3C*{H2em&Kz(cUOc^e1op(Wbc~|qH@;m`^Ir+}+ z>kjf#X*-HzrTkJ)r^y(;k3OyRexsj^!uxdAz`=owkC!}u)T)O3Jjm8<)BeJ4yXZd( z*8yXbl(2tkytQH6n0el_62e_o$E;V@9?^yHr11`l4_|7w68iB&ebaq*RotRLtc<3(>}G9RW~Y?jB=X|5SJ%QLb@)-85D+gCQ=HaMSWBjJCiv z<iW2dUU#l8c-j7r z^=r&qLFv@+h_wd@CAW#flRm1Vw_i5+7Eg)rYsWp@8}dw$CG6@S7x^j~$FNYVqNF+= zm*ppv;Hz>>s=Bs{O23e)nws`Mc4a8Z?QAWoRxB}ndTO$UAN|hx?Wk*wxO-`IyzYkS zNsr)TS5=(STE~8r%s5C#9Ge64Icv=rrOz@35*OyDh;j3;*52r)W^9yPMrjcZB#1od z-7fEk7Jo|DSF#B@I+7yFF^ylZeP>87V~^@crr~0KHhP~CD_Xz88=)gfr&wdH$Op7B zEGw7KLgl>z(e!R|wU#Z)RbAUR*~(;VbNv#v>{WV6{qg1*2UctiK-D#QF5d{&`C{BR zI%<|-A+9T1 z@oiBI{wgkJG>3W zOd)Jfc1Bmui%nho8xPc~mz-9gf1|Ljz)K))bm;CmQ}V}q-}D`BvDkxo3&VF)t1*}+2_{O=trbG#*>Zf{ghsKS?qm7OfSHRT*mdN;0s3g1Pr$~7FLLur{O6sKeQ|F z<#l)vJ}gcMmz0rf&D`r~k+t@ZM{{c%17^|&oR8Y(JvlZwCT2o6$}8Cs@H>>L7pS!K zpkbx{A??kBJ}(J=Kk%Bc;O0mAD(0M@V)R?^m&^9*TNBBZ?qLZliY;;-DV+%&N1oH? z{5{G2ck(KgOWmX9_?mB6Bl=nJ}GT*$uoH- z;b0#;N}%Xu_mro^R>G6i!6I`Ugc3D!sLr-Oc@gEyrO$vfZ2;~weYxYx95);;AJ8J})$*y>5k z>#X1zp~pKsTiyx7@;@#zKYYJFn@Z24=kSb-C>ar-@_0_^&!Cq!cDiuXtCKFk2}#es;$Cv}cYm`_Xmn60f~NI&n`1 zx1-v7n)(Jv`!5JuEn+>c&nuL-R)$Jhy2PIU_8b*v-lKoYRBp>R8`0bd%y*T5@O7-+ z=UHP?c8qo8wXOxL(}{V-J;Wu_xMCcO z;+j9~sHQj_`*h5Xs7I*F3N(&X>$~NVa^K;X%J`Iu`1UgK5hRH8pt|H}054F=BA zolV=KSw_?aCu(oDvEdA__x>c4_(PL|WexMr^Ha%L>a|^4hs0X1{sjF#=U2$7p zJ~Je?v`xCOf8i;Y>Z9IRbH|wF{#xP@k-EkR!=v{X*C&5$-?c-kgSRDNzXZ|XL3l|q zdu;gCydrI`&1_x2&{S2HQ&!egR!+M48UFSC55dp>l($Fd|0gW99ZrIT%}51Xe~+M$ n7lg9(x75&e7QhsP{fgoUtw={fc;1>Nq<1>$H literal 0 HcmV?d00001 diff --git a/retroshare-gui/src/gui/msgs/MessageComposer.cpp b/retroshare-gui/src/gui/msgs/MessageComposer.cpp index 6d33e6ca6..92f4de457 100644 --- a/retroshare-gui/src/gui/msgs/MessageComposer.cpp +++ b/retroshare-gui/src/gui/msgs/MessageComposer.cpp @@ -1206,6 +1206,12 @@ MessageComposer *MessageComposer::replyMsg(const std::string &msgId, bool all) // needed to send system flags with reply msgComposer->msgFlags = (msgInfo.msgflags & RS_MSG_SYSTEM); + MsgTagInfo tagInfo; + rsMail->getMessageTag(msgId, tagInfo); + + msgComposer->m_tagIds = tagInfo.tagIds; + msgComposer->showTagLabels(); + msgComposer->calculateTitle(); /* window will destroy itself! */ diff --git a/retroshare-gui/src/gui/msgs/MessageWidget.cpp b/retroshare-gui/src/gui/msgs/MessageWidget.cpp index 3c085c3c8..bfa94abba 100644 --- a/retroshare-gui/src/gui/msgs/MessageWidget.cpp +++ b/retroshare-gui/src/gui/msgs/MessageWidget.cpp @@ -36,6 +36,7 @@ #include "gui/common/TagDefs.h" #include "gui/common/PeerDefs.h" #include "gui/common/Emoticons.h" +#include "gui/common/FilesDefs.h" #include "gui/settings/rsharesettings.h" #include "MessageComposer.h" #include "MessageWidget.h" @@ -51,8 +52,8 @@ #include /* Images for context menu icons */ -#define IMAGE_DOWNLOAD ":/images/start.png" -#define IMAGE_DOWNLOADALL ":/images/startall.png" +#define IMAGE_DOWNLOAD ":/icons/png/download.png" +#define IMAGE_DOWNLOADALL ":/icons/mail/downloadall.png" #define COLUMN_FILE_NAME 0 #define COLUMN_FILE_SIZE 1 @@ -555,6 +556,7 @@ void MessageWidget::fill(const std::string &msgId) for (it = recList.begin(); it != recList.end(); ++it) { QTreeWidgetItem *item = new QTreeWidgetItem; item->setText(COLUMN_FILE_NAME, QString::fromUtf8(it->fname.c_str())); + item->setIcon(COLUMN_FILE_NAME, FilesDefs::getIconFromFileType(it->fname.c_str())); item->setText(COLUMN_FILE_SIZE, misc::friendlyUnit(it->size)); item->setData(COLUMN_FILE_SIZE, Qt::UserRole, QVariant(qulonglong(it->size)) ); item->setText(COLUMN_FILE_HASH, QString::fromStdString(it->hash.toStdString())); diff --git a/retroshare-gui/src/gui/msgs/MessagesDialog.cpp b/retroshare-gui/src/gui/msgs/MessagesDialog.cpp index 37a6ee319..5d6bc87c5 100644 --- a/retroshare-gui/src/gui/msgs/MessagesDialog.cpp +++ b/retroshare-gui/src/gui/msgs/MessagesDialog.cpp @@ -53,6 +53,7 @@ #include /* Images for context menu icons */ +#define IMAGE_MAIL ":/icons/png/message.png" #define IMAGE_MESSAGE ":/icons/mail/compose.png" #define IMAGE_MESSAGEREMOVE ":/icons/mail/delete.png" #define IMAGE_STAR_ON ":/images/star-on-16.png" @@ -766,7 +767,7 @@ void MessagesDialog::openAsTab() return; } - ui.tabWidget->addTab(msgWidget, msgWidget->subject(true)); + ui.tabWidget->addTab(msgWidget,QIcon(IMAGE_MAIL), msgWidget->subject(true)); ui.tabWidget->setCurrentWidget(msgWidget); connect(msgWidget, SIGNAL(messageRemoved()), this, SLOT(messageRemoved())); From 301def6018c0913728f1b7721c2bdaefb03ec85a Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 24 May 2020 23:40:26 +0200 Subject: [PATCH 16/72] Revert "bandwidth control improvement" --- libretroshare/src/pqi/pqihandler.cc | 173 +++++++++++++++++----------- 1 file changed, 105 insertions(+), 68 deletions(-) diff --git a/libretroshare/src/pqi/pqihandler.cc b/libretroshare/src/pqi/pqihandler.cc index 72d7dbf9d..1e86fa9d1 100644 --- a/libretroshare/src/pqi/pqihandler.cc +++ b/libretroshare/src/pqi/pqihandler.cc @@ -42,19 +42,39 @@ using std::dec; #include #endif +//#define PQI_HDL_DEBUG_UR 1 + +#ifdef PQI_HDL_DEBUG_UR +static double getCurrentTS() +{ + +#ifndef WINDOWS_SYS + struct timeval cts_tmp; + gettimeofday(&cts_tmp, NULL); + double cts = (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0; +#else + struct _timeb timebuf; + _ftime( &timebuf); + double cts = (timebuf.time) + ((double) timebuf.millitm) / 1000.0; +#endif + return cts; +} +#endif + struct RsLog::logInfo pqihandlerzoneInfo = {RsLog::Default, "pqihandler"}; #define pqihandlerzone &pqihandlerzoneInfo //static const int PQI_HANDLER_NB_PRIORITY_LEVELS = 10 ; //static const float PQI_HANDLER_NB_PRIORITY_RATIO = 2 ; -//#define UPDATE_RATES_DEBUG 1 -// #define DEBUG_TICK 1 -// #define RSITEM_DEBUG 1 +/**** +#define DEBUG_TICK 1 +#define RSITEM_DEBUG 1 +****/ pqihandler::pqihandler() : coreMtx("pqihandler") { - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ // setup minimal total+individual rates. rateIndiv_out = 0.01; @@ -77,7 +97,7 @@ int pqihandler::tick() int moreToTick = 0; { - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ // tick all interfaces... std::map::iterator it; @@ -107,13 +127,9 @@ int pqihandler::tick() if(now > mLastRateCapUpdate + 5) { - std::map rateMap; - std::map::iterator it; - - // every 5 secs, update the max rates for all modules - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ for(std::map::iterator it = mods.begin(); it != mods.end(); ++it) { // This is rather inelegant, but pqihandler has searchModules that are dynamically allocated, so the max rates @@ -133,7 +149,7 @@ int pqihandler::tick() bool pqihandler::queueOutRsItem(RsItem *item) { - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ uint32_t size ; locked_HandleRsItem(item, size); @@ -150,7 +166,7 @@ bool pqihandler::queueOutRsItem(RsItem *item) int pqihandler::status() { std::map::iterator it; - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ { // for output std::string out = "pqihandler::status() Active Modules:\n"; @@ -176,7 +192,7 @@ int pqihandler::status() bool pqihandler::AddSearchModule(SearchModule *mod) { - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ // if peerid used -> error. //std::map::iterator it; if (mod->peerid != mod->pqi->PeerId()) @@ -207,7 +223,7 @@ bool pqihandler::AddSearchModule(SearchModule *mod) bool pqihandler::RemoveSearchModule(SearchModule *mod) { - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) { @@ -297,7 +313,7 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat total.mQueueOut = 0; /* Lock once rates have been retrieved */ - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) @@ -324,6 +340,10 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat // internal fn to send updates int pqihandler::UpdateRates() { +#ifdef PQI_HDL_DEBUG_UR + uint64_t t_now; +#endif + std::map::iterator it; float avail_in = getMaxRate(true); @@ -333,15 +353,18 @@ int pqihandler::UpdateRates() float used_bw_out = 0; /* Lock once rates have been retrieved */ - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ int num_sm = mods.size(); float used_bw_in_table[num_sm]; /* table of in bandwidth currently used by each module */ float used_bw_out_table[num_sm]; /* table of out bandwidth currently used by each module */ - // loop through modules to get the used bandwidth -#ifdef UPDATE_RATES_DEBUG - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Looping through modules" << std::endl; + int effectiveUploadsSm = 0; + int effectiveDownloadsSm = 0; + + // loop through modules to get the used bandwith and the number of modules that are affectively transfering +#ifdef PQI_HDL_DEBUG_UR + std::cerr << "Looping through modules" << std::endl; #endif int index = 0; @@ -349,33 +372,49 @@ int pqihandler::UpdateRates() for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); + float crate_in = mod -> pqi -> getRate(true); traffInSum += mod -> pqi -> getTraffic(true); traffOutSum += mod -> pqi -> getTraffic(false); - float crate_in = mod -> pqi -> getRate(true); +#ifdef PQI_HDL_DEBUG_UR + if(crate_in > 0.0) + std::cerr << " got in rate for peer " << it->first << " : " << crate_in << std::endl; +#endif + + if ((crate_in > 0.01 * avail_in) || (crate_in > 0.1)) + { + ++effectiveDownloadsSm; + } + float crate_out = mod -> pqi -> getRate(false); + if ((crate_out > 0.01 * avail_out) || (crate_out > 0.1)) + { + ++effectiveUploadsSm; + } used_bw_in += crate_in; used_bw_out += crate_out; - /* fill the table of used bandwidths */ + /* fill the table of bandwidth */ used_bw_in_table[index] = crate_in; used_bw_out_table[index] = crate_out; - ++index; } -#ifdef UPDATE_RATES_DEBUG - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; +#ifdef PQI_HDL_DEBUG_UR + t_now = 1000 * getCurrentTS(); + std::cerr << dec << t_now << " pqihandler::UpdateRates(): Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; #endif /* Sort the used bw in/out table in ascending order */ std::sort(used_bw_in_table, used_bw_in_table + num_sm); std::sort(used_bw_out_table, used_bw_out_table + num_sm); -#ifdef UPDATE_RATES_DEBUG - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates used_bw_out " << used_bw_out << std::endl; +#ifdef PQI_HDL_DEBUG_UR + t_now = 1000 * getCurrentTS(); + std::cerr << dec << t_now << " pqihandler::UpdateRates(): Done." << std::endl; + std::cerr << dec << t_now << " pqihandler::UpdateRates(): used_bw_out " << used_bw_out << std::endl; #endif /* Calculate the optimal out_max value, taking into account avail_out and the out bw requested by modules */ @@ -402,8 +441,9 @@ int pqihandler::UpdateRates() } } -#ifdef UPDATE_RATES_DEBUG - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; +#ifdef PQI_HDL_DEBUG_UR + t_now = 1000 * getCurrentTS(); + std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; #endif /* Allocate only half the remaining out bw, if any, to make it smoother */ @@ -433,70 +473,67 @@ int pqihandler::UpdateRates() } } -#ifdef UPDATE_RATES_DEBUG - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; +#ifdef PQI_HDL_DEBUG_UR + t_now = 1000 * getCurrentTS(); + std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; #endif /* Allocate only half the remaining in bw, if any, to make it smoother */ in_max_bw = in_max_bw + in_remaining_bw / 2; - // store current total in and ou used bw + +#ifdef DEBUG_QOS +// std::cerr << "Totals (In) Used B/W " << used_bw_in; +// std::cerr << " Available B/W " << avail_in; +// std::cerr << " Effective transfers " << effectiveDownloadsSm << std::endl; +// std::cerr << "Totals (Out) Used B/W " << used_bw_out; +// std::cerr << " Available B/W " << avail_out; +// std::cerr << " Effective transfers " << effectiveUploadsSm << std::endl; +#endif + locked_StoreCurrentRates(used_bw_in, used_bw_out); -#ifdef UPDATE_RATES_DEBUG - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; + //computing average rates for effective transfers + float max_in_effective = avail_in / num_sm; + if (effectiveDownloadsSm != 0) { + max_in_effective = avail_in / effectiveDownloadsSm; + } + float max_out_effective = avail_out / num_sm; + if (effectiveUploadsSm != 0) { + max_out_effective = avail_out / effectiveUploadsSm; + } + + //modify the in and out limit +#ifdef PQI_HDL_DEBUG_UR + t_now = 1000 * getCurrentTS(); + std::cerr << dec << t_now << " pqihandler::UpdateRates(): setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; #endif - // retrieve down (from peer point of view) bandwidth limits set by peers in their own settings - std::map rateMap; - rsConfig->getAllBandwidthRates(rateMap); - std::map::iterator rateMap_it; - -#ifdef UPDATE_RATES_DEBUG - // Dump RsConfigurationDataRates - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates RsConfigDataRates dump" << std::endl; - for (rateMap_it = rateMap.begin(); rateMap_it != rateMap.end(); rateMap_it++) - RsDbg () << "UPDATE_RATES pqihandler::UpdateRates PeerId " << rateMap_it->first.toStdString() << " mAllowedOut " << rateMap_it->second.mAllowedOut << std::endl; -#endif - - // update max rates taking into account the limits set by peers in their own settings for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - - // for our down bandwidth we set the max to the calculated value without taking into account the max set by peers: they will control their up bw on their side - mod -> pqi -> setMaxRate(true, in_max_bw); - - // for our up bandwidth we limit to the maximum down bw provided by peers via BwCtrl because we don't want to clog our outqueues, the SSL buffers, and our friends inbound queues - if ((rateMap_it = rateMap.find(mod->pqi->PeerId())) != rateMap.end()) - { - if (rateMap_it->second.mAllowedOut > 0) - { - if (out_max_bw > rateMap_it->second.mAllowedOut) - mod -> pqi -> setMaxRate(false, rateMap_it->second.mAllowedOut); - else - mod -> pqi -> setMaxRate(false, out_max_bw); - } - else - mod -> pqi -> setMaxRate(false, out_max_bw); - } + + mod -> pqi -> setMaxRate(true, in_max_bw); + mod -> pqi -> setMaxRate(false, out_max_bw); } -#ifdef UPDATE_RATES_DEBUG - // dump maxRates + + //cap the rates for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - RsDbg() << "UPDATE_RATES pqihandler::UpdateRates PeerID " << (mod ->pqi -> PeerId()).toStdString() << " new bandwidth limits up " << mod -> pqi -> getMaxRate(false) << " down " << mod -> pqi -> getMaxRate(true) << std::endl; + if (mod -> pqi -> getMaxRate(false) < max_out_effective) mod -> pqi -> setMaxRate(false, max_out_effective); + if (mod -> pqi -> getMaxRate(false) > avail_out) mod -> pqi -> setMaxRate(false, avail_out); + if (mod -> pqi -> getMaxRate(true) < max_in_effective) mod -> pqi -> setMaxRate(true, max_in_effective); + if (mod -> pqi -> getMaxRate(true) > avail_in) mod -> pqi -> setMaxRate(true, avail_in); } -#endif return 1; } void pqihandler::getCurrentRates(float &in, float &out) { - RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ + RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ in = rateTotal_in; out = rateTotal_out; From b294a34d11055402ea4166f951cc1609f3688cfa Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 25 May 2020 20:28:50 +0200 Subject: [PATCH 17/72] Revert "Revert "bandwidth control improvement"" --- libretroshare/src/pqi/pqihandler.cc | 173 +++++++++++----------------- 1 file changed, 68 insertions(+), 105 deletions(-) diff --git a/libretroshare/src/pqi/pqihandler.cc b/libretroshare/src/pqi/pqihandler.cc index 1e86fa9d1..72d7dbf9d 100644 --- a/libretroshare/src/pqi/pqihandler.cc +++ b/libretroshare/src/pqi/pqihandler.cc @@ -42,39 +42,19 @@ using std::dec; #include #endif -//#define PQI_HDL_DEBUG_UR 1 - -#ifdef PQI_HDL_DEBUG_UR -static double getCurrentTS() -{ - -#ifndef WINDOWS_SYS - struct timeval cts_tmp; - gettimeofday(&cts_tmp, NULL); - double cts = (cts_tmp.tv_sec) + ((double) cts_tmp.tv_usec) / 1000000.0; -#else - struct _timeb timebuf; - _ftime( &timebuf); - double cts = (timebuf.time) + ((double) timebuf.millitm) / 1000.0; -#endif - return cts; -} -#endif - struct RsLog::logInfo pqihandlerzoneInfo = {RsLog::Default, "pqihandler"}; #define pqihandlerzone &pqihandlerzoneInfo //static const int PQI_HANDLER_NB_PRIORITY_LEVELS = 10 ; //static const float PQI_HANDLER_NB_PRIORITY_RATIO = 2 ; -/**** -#define DEBUG_TICK 1 -#define RSITEM_DEBUG 1 -****/ +//#define UPDATE_RATES_DEBUG 1 +// #define DEBUG_TICK 1 +// #define RSITEM_DEBUG 1 pqihandler::pqihandler() : coreMtx("pqihandler") { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // setup minimal total+individual rates. rateIndiv_out = 0.01; @@ -97,7 +77,7 @@ int pqihandler::tick() int moreToTick = 0; { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // tick all interfaces... std::map::iterator it; @@ -127,9 +107,13 @@ int pqihandler::tick() if(now > mLastRateCapUpdate + 5) { + std::map rateMap; + std::map::iterator it; + + // every 5 secs, update the max rates for all modules - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ for(std::map::iterator it = mods.begin(); it != mods.end(); ++it) { // This is rather inelegant, but pqihandler has searchModules that are dynamically allocated, so the max rates @@ -149,7 +133,7 @@ int pqihandler::tick() bool pqihandler::queueOutRsItem(RsItem *item) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ uint32_t size ; locked_HandleRsItem(item, size); @@ -166,7 +150,7 @@ bool pqihandler::queueOutRsItem(RsItem *item) int pqihandler::status() { std::map::iterator it; - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ { // for output std::string out = "pqihandler::status() Active Modules:\n"; @@ -192,7 +176,7 @@ int pqihandler::status() bool pqihandler::AddSearchModule(SearchModule *mod) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ // if peerid used -> error. //std::map::iterator it; if (mod->peerid != mod->pqi->PeerId()) @@ -223,7 +207,7 @@ bool pqihandler::AddSearchModule(SearchModule *mod) bool pqihandler::RemoveSearchModule(SearchModule *mod) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) { @@ -313,7 +297,7 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat total.mQueueOut = 0; /* Lock once rates have been retrieved */ - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ std::map::iterator it; for(it = mods.begin(); it != mods.end(); ++it) @@ -340,10 +324,6 @@ int pqihandler::ExtractRates(std::map &ratemap, RsBwRat // internal fn to send updates int pqihandler::UpdateRates() { -#ifdef PQI_HDL_DEBUG_UR - uint64_t t_now; -#endif - std::map::iterator it; float avail_in = getMaxRate(true); @@ -353,18 +333,15 @@ int pqihandler::UpdateRates() float used_bw_out = 0; /* Lock once rates have been retrieved */ - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ int num_sm = mods.size(); float used_bw_in_table[num_sm]; /* table of in bandwidth currently used by each module */ float used_bw_out_table[num_sm]; /* table of out bandwidth currently used by each module */ - int effectiveUploadsSm = 0; - int effectiveDownloadsSm = 0; - - // loop through modules to get the used bandwith and the number of modules that are affectively transfering -#ifdef PQI_HDL_DEBUG_UR - std::cerr << "Looping through modules" << std::endl; + // loop through modules to get the used bandwidth +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Looping through modules" << std::endl; #endif int index = 0; @@ -372,49 +349,33 @@ int pqihandler::UpdateRates() for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - float crate_in = mod -> pqi -> getRate(true); traffInSum += mod -> pqi -> getTraffic(true); traffOutSum += mod -> pqi -> getTraffic(false); -#ifdef PQI_HDL_DEBUG_UR - if(crate_in > 0.0) - std::cerr << " got in rate for peer " << it->first << " : " << crate_in << std::endl; -#endif - - if ((crate_in > 0.01 * avail_in) || (crate_in > 0.1)) - { - ++effectiveDownloadsSm; - } - + float crate_in = mod -> pqi -> getRate(true); float crate_out = mod -> pqi -> getRate(false); - if ((crate_out > 0.01 * avail_out) || (crate_out > 0.1)) - { - ++effectiveUploadsSm; - } used_bw_in += crate_in; used_bw_out += crate_out; - /* fill the table of bandwidth */ + /* fill the table of used bandwidths */ used_bw_in_table[index] = crate_in; used_bw_out_table[index] = crate_out; + ++index; } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates Sorting used_bw_out_table: " << num_sm << " entries" << std::endl; #endif /* Sort the used bw in/out table in ascending order */ std::sort(used_bw_in_table, used_bw_in_table + num_sm); std::sort(used_bw_out_table, used_bw_out_table + num_sm); -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): Done." << std::endl; - std::cerr << dec << t_now << " pqihandler::UpdateRates(): used_bw_out " << used_bw_out << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates used_bw_out " << used_bw_out << std::endl; #endif /* Calculate the optimal out_max value, taking into account avail_out and the out bw requested by modules */ @@ -441,9 +402,8 @@ int pqihandler::UpdateRates() } } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " out_max_bw " << out_max_bw << " remaining out bw " << out_remaining_bw << std::endl; #endif /* Allocate only half the remaining out bw, if any, to make it smoother */ @@ -473,67 +433,70 @@ int pqihandler::UpdateRates() } } -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates mod_index " << mod_index << " in_max_bw " << in_max_bw << " remaining in bw " << in_remaining_bw << std::endl; #endif /* Allocate only half the remaining in bw, if any, to make it smoother */ in_max_bw = in_max_bw + in_remaining_bw / 2; - -#ifdef DEBUG_QOS -// std::cerr << "Totals (In) Used B/W " << used_bw_in; -// std::cerr << " Available B/W " << avail_in; -// std::cerr << " Effective transfers " << effectiveDownloadsSm << std::endl; -// std::cerr << "Totals (Out) Used B/W " << used_bw_out; -// std::cerr << " Available B/W " << avail_out; -// std::cerr << " Effective transfers " << effectiveUploadsSm << std::endl; -#endif - + // store current total in and ou used bw locked_StoreCurrentRates(used_bw_in, used_bw_out); - //computing average rates for effective transfers - float max_in_effective = avail_in / num_sm; - if (effectiveDownloadsSm != 0) { - max_in_effective = avail_in / effectiveDownloadsSm; - } - float max_out_effective = avail_out / num_sm; - if (effectiveUploadsSm != 0) { - max_out_effective = avail_out / effectiveUploadsSm; - } - - //modify the in and out limit -#ifdef PQI_HDL_DEBUG_UR - t_now = 1000 * getCurrentTS(); - std::cerr << dec << t_now << " pqihandler::UpdateRates(): setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; +#ifdef UPDATE_RATES_DEBUG + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates setting new out_max " << out_max_bw << " in_max " << in_max_bw << std::endl; #endif + // retrieve down (from peer point of view) bandwidth limits set by peers in their own settings + std::map rateMap; + rsConfig->getAllBandwidthRates(rateMap); + std::map::iterator rateMap_it; + +#ifdef UPDATE_RATES_DEBUG + // Dump RsConfigurationDataRates + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates RsConfigDataRates dump" << std::endl; + for (rateMap_it = rateMap.begin(); rateMap_it != rateMap.end(); rateMap_it++) + RsDbg () << "UPDATE_RATES pqihandler::UpdateRates PeerId " << rateMap_it->first.toStdString() << " mAllowedOut " << rateMap_it->second.mAllowedOut << std::endl; +#endif + + // update max rates taking into account the limits set by peers in their own settings for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - - mod -> pqi -> setMaxRate(true, in_max_bw); - mod -> pqi -> setMaxRate(false, out_max_bw); + + // for our down bandwidth we set the max to the calculated value without taking into account the max set by peers: they will control their up bw on their side + mod -> pqi -> setMaxRate(true, in_max_bw); + + // for our up bandwidth we limit to the maximum down bw provided by peers via BwCtrl because we don't want to clog our outqueues, the SSL buffers, and our friends inbound queues + if ((rateMap_it = rateMap.find(mod->pqi->PeerId())) != rateMap.end()) + { + if (rateMap_it->second.mAllowedOut > 0) + { + if (out_max_bw > rateMap_it->second.mAllowedOut) + mod -> pqi -> setMaxRate(false, rateMap_it->second.mAllowedOut); + else + mod -> pqi -> setMaxRate(false, out_max_bw); + } + else + mod -> pqi -> setMaxRate(false, out_max_bw); + } } - - //cap the rates +#ifdef UPDATE_RATES_DEBUG + // dump maxRates for(it = mods.begin(); it != mods.end(); ++it) { SearchModule *mod = (it -> second); - if (mod -> pqi -> getMaxRate(false) < max_out_effective) mod -> pqi -> setMaxRate(false, max_out_effective); - if (mod -> pqi -> getMaxRate(false) > avail_out) mod -> pqi -> setMaxRate(false, avail_out); - if (mod -> pqi -> getMaxRate(true) < max_in_effective) mod -> pqi -> setMaxRate(true, max_in_effective); - if (mod -> pqi -> getMaxRate(true) > avail_in) mod -> pqi -> setMaxRate(true, avail_in); + RsDbg() << "UPDATE_RATES pqihandler::UpdateRates PeerID " << (mod ->pqi -> PeerId()).toStdString() << " new bandwidth limits up " << mod -> pqi -> getMaxRate(false) << " down " << mod -> pqi -> getMaxRate(true) << std::endl; } +#endif return 1; } void pqihandler::getCurrentRates(float &in, float &out) { - RsStackMutex stack(coreMtx); /**************** LOCKED MUTEX ****************/ + RS_STACK_MUTEX(coreMtx); /**************** LOCKED MUTEX ****************/ in = rateTotal_in; out = rateTotal_out; From 3332c32a840f5652a77c12d41a9fcd4c0afed49c Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 21 May 2020 18:17:49 +0200 Subject: [PATCH 18/72] RsBase64 handle correcly 0 lenght buffer encoding --- libretroshare/src/util/rsbase64.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libretroshare/src/util/rsbase64.cc b/libretroshare/src/util/rsbase64.cc index 6b856888e..72ea119e8 100644 --- a/libretroshare/src/util/rsbase64.cc +++ b/libretroshare/src/util/rsbase64.cc @@ -40,6 +40,12 @@ rs_view_ptr data, size_t len, std::string& outString, bool padding, bool urlSafe ) { + if(!data || !len) + { + outString.clear(); + return; + } + const char* sDict = urlSafe ? uDict : bDict; // Workaround if input and output are the same buffer. @@ -137,6 +143,7 @@ /*static*/ size_t RsBase64::encodedSize(size_t decodedSize, bool padding) { + if(!decodedSize) return 0; if(padding) return 4 * (decodedSize + 2) / 3; return static_cast( std::ceil(4L * static_cast(decodedSize) / 3L) ); From ebf32bf955a3a1ab0e80f14a3edea65763aca694 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 27 May 2020 22:27:30 +0200 Subject: [PATCH 19/72] moved parts of the GrpMeta cache into a template to also handle MsgMeta --- libretroshare/src/gxs/rsdataservice.cc | 138 ++++++++----------------- libretroshare/src/gxs/rsdataservice.h | 106 ++++++++++++++++++- 2 files changed, 142 insertions(+), 102 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 6bdc924ed..8bf848a38 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -110,8 +110,6 @@ const std::string RsGeneralDataService::MSG_META_STATUS = KEY_MSG_STATUS; const uint32_t RsGeneralDataService::GXS_MAX_ITEM_SIZE = 1572864; // 1.5 Mbytes -static const uint32_t CACHE_ENTRY_GRACE_PERIOD = 600 ; // 10 minutes - static int addColumn(std::list &list, const std::string &attribute) { list.push_back(attribute); @@ -123,7 +121,6 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d : RsGeneralDataService(), mDbMutex("RsDataService"), mServiceDir(serviceDir), mDbName(dbName), mDbPath(mServiceDir + "/" + dbName), mServType(serviceType), mDb(NULL) { bool isNewDatabase = !RsDirUtil::fileExists(mDbPath); - mGrpMetaDataCache_ContainsAllDatabase = false ; mDb = new RetroDb(mDbPath, RetroDb::OPEN_READWRITE_CREATE, key); @@ -488,8 +485,7 @@ bool RsDataService::finishReleaseUpdate(int release, bool result) RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c, int colOffset,bool use_cache) { #ifdef RS_DATA_SERVICE_DEBUG - std::cerr << "RsDataService::locked_getGrpMeta()"; - std::cerr << std::endl; + std::cerr << "RsDataService::locked_getGrpMeta()" << std::endl; #endif bool ok = true; @@ -507,20 +503,13 @@ RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c, int colOffset RsGxsGroupId grpId(tempId) ; if(use_cache) - { - auto it = mGrpMetaDataCache.find(grpId) ; - - if(it != mGrpMetaDataCache.end()) - grpMeta = it->second ; - else - { - grpMeta = new RsGxsGrpMetaData(); - mGrpMetaDataCache[grpId] = grpMeta ; - } - } + grpMeta = mGrpMetaDataCache.getOrCreateMeta(grpId); else grpMeta = new RsGxsGrpMetaData(); + // if(!grpMeta->mGroupId.isNull()) // the grpMeta is already initialized because it comes from the cache + // return grpMeta; + grpMeta->mGroupId = RsGxsGroupId(tempId); c.getString(mColGrpMeta_NxsIdentity + colOffset, tempId); grpMeta->mAuthorId = RsGxsId(tempId); @@ -653,24 +642,38 @@ RsNxsGrp* RsDataService::locked_getGroup(RetroCursor &c) return NULL; } -RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset) +RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset,bool use_cache) { - RsGxsMsgMetaData* msgMeta = new RsGxsMsgMetaData(); - bool ok = true; uint32_t data_len = 0, offset = 0; char* data = NULL; + RsGxsGroupId group_id; + RsGxsMessageId msg_id; + std::string gId; c.getString(mColMsgMeta_GrpId + colOffset, gId); - msgMeta->mGroupId = RsGxsGroupId(gId); + group_id = RsGxsGroupId(gId); std::string temp; c.getString(mColMsgMeta_MsgId + colOffset, temp); - msgMeta->mMsgId = RsGxsMessageId(temp); + msg_id = RsGxsMessageId(temp); // without these, a msg is meaningless - ok &= (!msgMeta->mGroupId.isNull()) && (!msgMeta->mMsgId.isNull()); + ok &= (!group_id.isNull()) && (!msg_id.isNull()); + + RsGxsMsgMetaData* msgMeta = nullptr; + +// if(use_cache) +// msgMeta = mMsgMetaDataCache[group_id].getOrCreateMeta(msg_id); +// else + msgMeta = new RsGxsMsgMetaData(); + + //if(!msgMeta->mGroupId.isNull()) // we cannot do that because the cursor needs to advance. Is there a method to skip some data in the db? + // return msgMeta; + + msgMeta->mGroupId = group_id; + msgMeta->mMsgId = msg_id; c.getString(mColMsgMeta_OrigMsgId + colOffset, temp); msgMeta->mOrigMsgId = RsGxsMessageId(temp); @@ -704,7 +707,7 @@ RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset if(ok) return msgMeta; - else + else if(!use_cache) delete msgMeta; return NULL; @@ -834,7 +837,7 @@ int RsDataService::storeMessage(const std::list& msg) // This is needed so that mLastPost is correctly updated in the group meta when it is re-loaded. - locked_clearGrpMetaCache(msgMetaPtr->mGroupId); + mGrpMetaDataCache.clear(msgMetaPtr->mGroupId); } // finish transaction @@ -926,7 +929,7 @@ int RsDataService::storeGroup(const std::list& grp) cv.put(KEY_GRP_STATUS, (int32_t)grpMetaPtr->mGroupStatus); cv.put(KEY_GRP_LAST_POST, (int32_t)grpMetaPtr->mLastPost); - locked_updateGrpMetaCache(*grpMetaPtr); + mGrpMetaDataCache.updateMeta(grpMetaPtr->mGroupId,*grpMetaPtr); if (!mDb->sqlInsert(GRP_TABLE_NAME, "", cv)) { @@ -942,54 +945,6 @@ int RsDataService::storeGroup(const std::list& grp) return ret; } -void RsDataService::locked_updateGrpMetaCache(const RsGxsGrpMetaData& meta) -{ - auto it = mGrpMetaDataCache.find(meta.mGroupId) ; - - if(it != mGrpMetaDataCache.end()) - *(it->second) = meta ; - else - mGrpMetaDataCache[meta.mGroupId] = new RsGxsGrpMetaData(meta) ; -} - -void RsDataService::locked_clearGrpMetaCache(const RsGxsGroupId& gid) -{ - rstime_t now = time(NULL) ; - auto it = mGrpMetaDataCache.find(gid) ; - - // We dont actually delete the item, because it might be used by a calling client. - // In this case, the memory will not be used for long, so we keep it into a list for a safe amount - // of time and delete it later. Using smart pointers here would be more elegant, but that would need - // to be implemented thread safe, which is difficult in this case. - - if(it != mGrpMetaDataCache.end()) - { -#ifdef RS_DATA_SERVICE_DEBUG - std::cerr << "(II) moving database cache entry " << (void*)(*it).second << " to dead list." << std::endl; -#endif - - mOldCachedItems.push_back(std::make_pair(now,it->second)) ; - - mGrpMetaDataCache.erase(it) ; - mGrpMetaDataCache_ContainsAllDatabase = false; - } - - // We also take that opportunity to delete old entries. - - auto it2(mOldCachedItems.begin()); - - while(it2!=mOldCachedItems.end() && (*it2).first + CACHE_ENTRY_GRACE_PERIOD < now) - { -#ifdef RS_DATA_SERVICE_DEBUG - std::cerr << "(II) deleting old GXS database cache entry " << (void*)(*it2).second << ", " << now - (*it2).first << " seconds old." << std::endl; -#endif - - delete (*it2).second ; - it2 = mOldCachedItems.erase(it2) ; - } - -} - int RsDataService::updateGroup(const std::list &grp) { @@ -1058,7 +1013,7 @@ int RsDataService::updateGroup(const std::list &grp) mDb->sqlUpdate(GRP_TABLE_NAME, "grpId='" + grpPtr->grpId.toStdString() + "'", cv); - locked_updateGrpMetaCache(*grpMetaPtr); + mGrpMetaDataCache.updateMeta(grpMetaPtr->mGroupId,*grpMetaPtr); } // finish transaction bool ret = mDb->commitTransaction(); @@ -1275,7 +1230,7 @@ void RsDataService::locked_retrieveMessages(RetroCursor *c, std::vectormetaData = locked_getMsgMeta(*c, metaOffset); + m->metaData = locked_getMsgMeta(*c, metaOffset,false); } msgs.push_back(m); } @@ -1357,7 +1312,7 @@ void RsDataService::locked_retrieveMsgMeta(RetroCursor *c, std::vectormoveToFirst(); while(valid){ - RsGxsMsgMetaData* m = locked_getMsgMeta(*c, 0); + RsGxsMsgMetaData* m = locked_getMsgMeta(*c, 0,true); if(m != NULL) msgMeta.push_back(m); @@ -1385,13 +1340,13 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) if(grp.empty()) { - if(mGrpMetaDataCache_ContainsAllDatabase) // grab all the stash from the cache, so as to avoid decryption costs. + if(mGrpMetaDataCache.isCacheUpToDate()) // grab all the stash from the cache, so as to avoid decryption costs. { #ifdef RS_DATA_SERVICE_DEBUG_CACHE std::cerr << (void*)this << ": RsDataService::retrieveGxsGrpMetaData() retrieving all from cache!" << std::endl; #endif - grp = mGrpMetaDataCache ; + grp = mGrpMetaDataCache.getFullMetaMap() ; } else { @@ -1426,9 +1381,8 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) delete c; } - mGrpMetaDataCache_ContainsAllDatabase = true ; + mGrpMetaDataCache.setCacheUpToDate(true); } - } else { @@ -1436,15 +1390,10 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) for(; mit != grp.end(); ++mit) { - std::map::const_iterator itt = mGrpMetaDataCache.find(mit->first) ; + RsGxsGrpMetaData *meta = mGrpMetaDataCache.getMeta(mit->first) ; - if(itt != mGrpMetaDataCache.end()) - { -#ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << mDbName << ": Retrieving Grp metadata grpId=" << mit->first << " from cache!" << std::endl; -#endif - grp[mit->first] = itt->second ; - } + if(meta) + grp[mit->first] = meta; else { #ifdef RS_DATA_SERVICE_DEBUG_CACHE @@ -1538,7 +1487,7 @@ int RsDataService::updateGroupMetaData(GrpLocMetaData &meta) std::cerr << (void*)this << ": erasing old entry from cache." << std::endl; #endif - locked_clearGrpMetaCache(meta.grpId); + mGrpMetaDataCache.clear(meta.grpId); return mDb->sqlUpdate(GRP_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", meta.val) ? 1 : 0; } @@ -1698,23 +1647,18 @@ bool RsDataService::locked_removeGroupEntries(const std::vector& g // start a transaction bool ret = mDb->beginTransaction(); - std::vector::const_iterator vit = grpIds.begin(); - - for(; vit != grpIds.end(); ++vit) + for(auto grpId:grpIds) { - - const RsGxsGroupId& grpId = *vit; mDb->sqlDelete(GRP_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", ""); // also remove the group meta from cache. - locked_clearGrpMetaCache(*vit) ; + mGrpMetaDataCache.clear(grpId) ; } ret &= mDb->commitTransaction(); - - mGrpMetaDataCache_ContainsAllDatabase = false ; return ret; } + uint32_t RsDataService::cacheSize() const { return 0; } diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index 8b4e2aac9..5babb59ce 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -35,6 +35,104 @@ public: ContentValue cv; }; +template class t_MetaDataCache +{ +public: + t_MetaDataCache() : mCache_ContainsAllMetas(false) {} + + bool isCacheUpToDate() const { return mCache_ContainsAllMetas ; } + void setCacheUpToDate(bool b) { mCache_ContainsAllMetas = b; } + const std::map getFullMetaMap() const { return mMetas ; } + + MetaDataClass *getMeta(const ID& id) + { + auto itt = mMetas.find(id); + + if(itt != mMetas.end()) + return itt->second ; + else + return NULL; + } + + MetaDataClass *getOrCreateMeta(const ID& id) + { + MetaDataClass *meta = nullptr; + auto it = mMetas.find(id) ; + + if(it != mMetas.end()) + { +#ifdef RS_DATA_SERVICE_DEBUG + RsDbg() << __PRETTY_FUNCTION__ << ": getting group meta " << grpId << " from cache." << std::endl; +#endif + meta = it->second ; + } + else + { +#ifdef RS_DATA_SERVICE_DEBUG + RsDbg() << __PRETTY_FUNCTION__ << ": group meta " << grpId << " not in cache. Loading it from DB..." << std::endl; +#endif + meta = new MetaDataClass(); + mMetas[id] = meta ; + } + + return meta; + } + + void updateMeta(const ID& id,const MetaDataClass& meta) + { + auto it = mMetas.find(id) ; + + if(it != mMetas.end()) + *(it->second) = meta ; + else + mMetas[id] = new MetaDataClass(meta) ; + } + + void clear(const ID& id) + { + rstime_t now = time(NULL) ; + auto it = mMetas.find(id) ; + + // We dont actually delete the item, because it might be used by a calling client. + // In this case, the memory will not be used for long, so we keep it into a list for a safe amount + // of time and delete it later. Using smart pointers here would be more elegant, but that would need + // to be implemented thread safe, which is difficult in this case. + + if(it != mMetas.end()) + { +#ifdef RS_DATA_SERVICE_DEBUG + std::cerr << "(II) moving database cache entry " << (void*)(*it).second << " to dead list." << std::endl; +#endif + + mOldCachedItems.push_back(std::make_pair(now,it->second)) ; + + mMetas.erase(it) ; + mCache_ContainsAllMetas = false; + } + + // We also take that opportunity to delete old entries. + + auto it2(mOldCachedItems.begin()); + + while(it2!=mOldCachedItems.end() && (*it2).first + CACHE_ENTRY_GRACE_PERIOD < now) + { +#ifdef RS_DATA_SERVICE_DEBUG + std::cerr << "(II) deleting old GXS database cache entry " << (void*)(*it2).second << ", " << now - (*it2).first << " seconds old." << std::endl; +#endif + delete (*it2).second ; + it2 = mOldCachedItems.erase(it2) ; + } + } + +private: + std::map mMetas; + std::list > mOldCachedItems ; // dead list, where items get deleted after being unused for a while. This is due to not using smart ptrs. + + static const uint32_t CACHE_ENTRY_GRACE_PERIOD = 600 ; // 10 minutes + + bool mCache_ContainsAllMetas ; +}; + class RsDataService : public RsGeneralDataService { public: @@ -202,7 +300,7 @@ private: * extracts a msg meta item from a cursor at its * current position */ - RsGxsMsgMetaData* locked_getMsgMeta(RetroCursor& c, int colOffset); + RsGxsMsgMetaData* locked_getMsgMeta(RetroCursor& c, int colOffset, bool use_cache); /*! * extracts a grp meta item from a cursor at its @@ -348,10 +446,8 @@ private: void locked_clearGrpMetaCache(const RsGxsGroupId& gid); void locked_updateGrpMetaCache(const RsGxsGrpMetaData& meta); - std::map mGrpMetaDataCache ; - std::list > mOldCachedItems ; - - bool mGrpMetaDataCache_ContainsAllDatabase ; + t_MetaDataCache mGrpMetaDataCache; + std::map > mMsgMetaDataCache; }; #endif // RSDATASERVICE_H From eb6af15b1a0f67088f90c28b40905b6a6ffe4a7c Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 28 May 2020 15:11:14 +0200 Subject: [PATCH 20/72] enabled MsgMetaData cache in rsgxsdataaccess.cc and witched metas to const, and removed deletion of retrieved metas since now the cache handles it --- libretroshare/src/gxs/rsdataservice.cc | 314 +++++++++++++---------- libretroshare/src/gxs/rsdataservice.h | 21 +- libretroshare/src/gxs/rsgds.h | 4 +- libretroshare/src/gxs/rsgenexchange.cc | 22 +- libretroshare/src/gxs/rsgxs.h | 4 +- libretroshare/src/gxs/rsgxsdata.cc | 2 +- libretroshare/src/gxs/rsgxsdata.h | 2 +- libretroshare/src/gxs/rsgxsdataaccess.cc | 92 ++++--- libretroshare/src/gxs/rsgxsdataaccess.h | 12 +- libretroshare/src/gxs/rsgxsnetservice.cc | 25 +- libretroshare/src/gxs/rsgxsnetservice.h | 2 +- libretroshare/src/gxs/rsgxsutil.cc | 6 +- 12 files changed, 277 insertions(+), 229 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 8bf848a38..b09e62c99 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -507,8 +507,8 @@ RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c, int colOffset else grpMeta = new RsGxsGrpMetaData(); - // if(!grpMeta->mGroupId.isNull()) // the grpMeta is already initialized because it comes from the cache - // return grpMeta; + if(!grpMeta->mGroupId.isNull()) // the grpMeta is already initialized because it comes from the cache + return grpMeta; grpMeta->mGroupId = RsGxsGroupId(tempId); c.getString(mColGrpMeta_NxsIdentity + colOffset, tempId); @@ -664,13 +664,13 @@ RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset RsGxsMsgMetaData* msgMeta = nullptr; -// if(use_cache) -// msgMeta = mMsgMetaDataCache[group_id].getOrCreateMeta(msg_id); -// else + if(use_cache) + msgMeta = mMsgMetaDataCache[group_id].getOrCreateMeta(msg_id); + else msgMeta = new RsGxsMsgMetaData(); - //if(!msgMeta->mGroupId.isNull()) // we cannot do that because the cursor needs to advance. Is there a method to skip some data in the db? - // return msgMeta; + if(!msgMeta->mGroupId.isNull()) // we cannot do that because the cursor needs to advance. Is there a method to skip some data in the db? + return msgMeta; msgMeta->mGroupId = group_id; msgMeta->mMsgId = msg_id; @@ -838,6 +838,7 @@ int RsDataService::storeMessage(const std::list& msg) // This is needed so that mLastPost is correctly updated in the group meta when it is re-loaded. mGrpMetaDataCache.clear(msgMetaPtr->mGroupId); + mMsgMetaDataCache[msgMetaPtr->mGroupId].updateMeta(msgMetaPtr->mMsgId,*msgMetaPtr); } // finish transaction @@ -1240,7 +1241,7 @@ void RsDataService::locked_retrieveMessages(RetroCursor *c, std::vectorfirst; + const std::set& msgIdV = mit->second; // if vector empty then request all messages - const std::set& msgIdV = mit->second; - std::vector metaSet; - if(msgIdV.empty()){ - RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", ""); + t_MetaDataCache& cache(mMsgMetaDataCache[grpId]); - if (c) - { - locked_retrieveMsgMeta(c, metaSet); + if(msgIdV.empty()) + { + if(cache.isCacheUpToDate()) + cache.getFullMetaList(msgMeta[grpId]); + else + { + RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", ""); + + if (c) + { + locked_retrieveMsgMetaList(c, msgMeta[grpId]); #ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << mDbName << ": Retrieving (all) Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; + std::cerr << mDbName << ": Retrieving (all) Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; #endif - } - }else{ - - // request each grp - std::set::const_iterator sit = msgIdV.begin(); - - for(; sit!=msgIdV.end(); ++sit){ - const RsGxsMessageId& msgId = *sit; - RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() - + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", ""); - - if (c) - { - locked_retrieveMsgMeta(c, metaSet); -#ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << mDbName << ": Retrieving Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; -#endif - } - } + cache.setCacheUpToDate(true); + } + delete c; + } } + else + { + // request each msg meta + auto& metaSet(msgMeta[grpId]); -#ifdef RS_DATA_SERVICE_DEBUG_TIME - resultCount += metaSet.size(); + for(auto sit(msgIdV.begin()); sit!=msgIdV.end(); ++sit) + { + const RsGxsMessageId& msgId = *sit; + + RsGxsMsgMetaData *meta = cache.getMeta(msgId); + + if(meta) + metaSet.push_back(meta); + else + { + RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", ""); + + c->moveToFirst(); + RsGxsMsgMetaData* meta = locked_getMsgMeta(*c, 0,true); + + if(meta) + metaSet.push_back(meta); + + delete c; + +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << mDbName << ": Retrieving Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; #endif - - msgMeta[grpId] = metaSet; + } + } + } } #ifdef RS_DATA_SERVICE_DEBUG_TIME @@ -1305,22 +1320,43 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes return 1; } -void RsDataService::locked_retrieveMsgMeta(RetroCursor *c, std::vector &msgMeta) +void RsDataService::locked_retrieveGrpMetaList(RetroCursor *c, std::map& grpMeta) { + if(!c) + { + RsErr() << __PRETTY_FUNCTION__ << ": attempt to retrieve Group Meta data from the DB with null cursor!" << std::endl; + return; + } - if(c) - { - bool valid = c->moveToFirst(); - while(valid){ - RsGxsMsgMetaData* m = locked_getMsgMeta(*c, 0,true); + bool valid = c->moveToFirst(); - if(m != NULL) - msgMeta.push_back(m); + while(valid) + { + RsGxsGrpMetaData* m = locked_getGrpMeta(*c, 0,true); - valid = c->moveToNext(); - } - delete c; - } + if(m) + grpMeta[m->mGroupId] = m; + + valid = c->moveToNext(); + } +} +void RsDataService::locked_retrieveMsgMetaList(RetroCursor *c, std::vector& msgMeta) +{ + if(!c) + { + RsErr() << __PRETTY_FUNCTION__ << ": attempt to retrieve Msg Meta data from the DB with null cursor!" << std::endl; + return; + } + + bool valid = c->moveToFirst(); + while(valid){ + const RsGxsMsgMetaData* m = locked_getMsgMeta(*c, 0,true); + + if(m != NULL) + msgMeta.push_back(m); + + valid = c->moveToNext(); + } } int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) @@ -1346,7 +1382,7 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) std::cerr << (void*)this << ": RsDataService::retrieveGxsGrpMetaData() retrieving all from cache!" << std::endl; #endif - grp = mGrpMetaDataCache.getFullMetaMap() ; + mGrpMetaDataCache.getFullMetaList(grp) ; } else { @@ -1357,87 +1393,99 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "", ""); - if(c) + if(c) { - bool valid = c->moveToFirst(); + locked_retrieveGrpMetaList(c,grp); - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true); - - if(g) - { - grp[g->mGroupId] = g; -#ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << (void *)this << " " << mDbName << ": Retrieving (all) Grp metadata grpId=" << g->mGroupId << std::endl; -#endif - } - valid = c->moveToNext(); - -#ifdef RS_DATA_SERVICE_DEBUG_TIME - ++resultCount; -#endif - } - delete c; + mGrpMetaDataCache.setCacheUpToDate(true); } + delete c; - mGrpMetaDataCache.setCacheUpToDate(true); +// if(c) +// { +// bool valid = c->moveToFirst(); +// +// while(valid) +// { +// RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true); +// +// if(g) +// { +// grp[g->mGroupId] = g; +//#ifdef RS_DATA_SERVICE_DEBUG_CACHE +// std::cerr << (void *)this << " " << mDbName << ": Retrieving (all) Grp metadata grpId=" << g->mGroupId << std::endl; +//#endif +// } +// valid = c->moveToNext(); +// +//#ifdef RS_DATA_SERVICE_DEBUG_TIME +// ++resultCount; +//#endif +// } +// delete c; +// } } } else - { - std::map::iterator mit = grp.begin(); + { + for(auto mit(grp.begin()); mit != grp.end(); ++mit) + { + RsGxsGrpMetaData *meta = mGrpMetaDataCache.getMeta(mit->first) ; - for(; mit != grp.end(); ++mit) - { - RsGxsGrpMetaData *meta = mGrpMetaDataCache.getMeta(mit->first) ; - - if(meta) - grp[mit->first] = meta; - else - { + if(meta) + grp[mit->first] = meta; + else + { #ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << mDbName << ": Retrieving Grp metadata grpId=" << mit->first ; + std::cerr << mDbName << ": Retrieving Grp metadata grpId=" << mit->first ; #endif - const RsGxsGroupId& grpId = mit->first; - RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", ""); + const RsGxsGroupId& grpId = mit->first; + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", ""); - if(c) - { - bool valid = c->moveToFirst(); + c->moveToFirst(); + RsGxsGrpMetaData* meta = locked_getGrpMeta(*c, 0,true); + if(meta) + grp[mit->first] = meta; + + delete c; + + // if(c) + // { + // bool valid = c->moveToFirst(); + // + //#ifdef RS_DATA_SERVICE_DEBUG_CACHE + // if(!valid) + // std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl; + //#endif + // while(valid) + // { + // RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true); + // + // if(g) + // { + // grp[g->mGroupId] = g; + //#ifdef RS_DATA_SERVICE_DEBUG_CACHE + // std::cerr << ". Got it. Updating cache." << std::endl; + //#endif + // } + // valid = c->moveToNext(); + // + //#ifdef RS_DATA_SERVICE_DEBUG_TIME + // ++resultCount; + //#endif + // } + // delete c; + // } #ifdef RS_DATA_SERVICE_DEBUG_CACHE - if(!valid) - std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl; + else + std::cerr << ". not found!" << std::endl; #endif - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0,true); + } + } - if(g) - { - grp[g->mGroupId] = g; -#ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << ". Got it. Updating cache." << std::endl; -#endif - } - valid = c->moveToNext(); - -#ifdef RS_DATA_SERVICE_DEBUG_TIME - ++resultCount; -#endif - } - delete c; - } -#ifdef RS_DATA_SERVICE_DEBUG_CACHE - else - std::cerr << ". not found!" << std::endl; -#endif - } - } - - } + } #ifdef RS_DATA_SERVICE_DEBUG_TIME std::cerr << "RsDataService::retrieveGxsGrpMetaData() " << mDbName << ", Requests: " << requestedGroups << ", Results: " << resultCount << ", Time: " << timer.duration() << std::endl; @@ -1474,14 +1522,14 @@ int RsDataService::resetDataStore() return 1; } -int RsDataService::updateGroupMetaData(GrpLocMetaData &meta) +int RsDataService::updateGroupMetaData(const GrpLocMetaData& meta) { #ifdef RS_DATA_SERVICE_DEBUG_CACHE std::cerr << (void*)this << ": Updating Grp Meta data: grpId = " << meta.grpId << std::endl; #endif RsStackMutex stack(mDbMutex); - RsGxsGroupId& grpId = meta.grpId; + const RsGxsGroupId& grpId = meta.grpId; #ifdef RS_DATA_SERVICE_DEBUG_CACHE std::cerr << (void*)this << ": erasing old entry from cache." << std::endl; @@ -1492,17 +1540,19 @@ int RsDataService::updateGroupMetaData(GrpLocMetaData &meta) return mDb->sqlUpdate(GRP_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", meta.val) ? 1 : 0; } -int RsDataService::updateMessageMetaData(MsgLocMetaData &metaData) +int RsDataService::updateMessageMetaData(const MsgLocMetaData& metaData) { #ifdef RS_DATA_SERVICE_DEBUG_CACHE std::cerr << (void*)this << ": Updating Msg Meta data: grpId = " << metaData.msgId.first << " msgId = " << metaData.msgId.second << std::endl; #endif RsStackMutex stack(mDbMutex); - RsGxsGroupId& grpId = metaData.msgId.first; - RsGxsMessageId& msgId = metaData.msgId.second; - return mDb->sqlUpdate(MSG_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() - + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", metaData.val) ? 1 : 0; + const RsGxsGroupId& grpId = metaData.msgId.first; + const RsGxsMessageId& msgId = metaData.msgId.second; + + mMsgMetaDataCache[grpId].clear(msgId); + + return mDb->sqlUpdate(MSG_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", metaData.val) ? 1 : 0; } int RsDataService::removeMsgs(const GxsMsgReq& msgIds) @@ -1627,13 +1677,13 @@ bool RsDataService::locked_removeMessageEntries(const GxsMsgReq& msgIds) { const RsGxsGroupId& grpId = mit->first; const std::set& msgsV = mit->second; - std::set::const_iterator vit = msgsV.begin(); + auto& cache(mMsgMetaDataCache[grpId]); - for(; vit != msgsV.end(); ++vit) + for(auto& msgId:msgsV) { - const RsGxsMessageId& msgId = *vit; - mDb->sqlDelete(MSG_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() - + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", ""); + mDb->sqlDelete(MSG_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", ""); + + cache.clear(msgId); } } diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index 5babb59ce..35234be9e 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -42,7 +42,9 @@ public: bool isCacheUpToDate() const { return mCache_ContainsAllMetas ; } void setCacheUpToDate(bool b) { mCache_ContainsAllMetas = b; } - const std::map getFullMetaMap() const { return mMetas ; } + + void getFullMetaList(std::map& mp) const { mp = mMetas ; } + void getFullMetaList(std::vector& mp) const { for(auto& m:mMetas) mp.push_back(m.second) ; } MetaDataClass *getMeta(const ID& id) { @@ -128,7 +130,7 @@ private: std::map mMetas; std::list > mOldCachedItems ; // dead list, where items get deleted after being unused for a while. This is due to not using smart ptrs. - static const uint32_t CACHE_ENTRY_GRACE_PERIOD = 600 ; // 10 minutes + static const uint32_t CACHE_ENTRY_GRACE_PERIOD = 600 ; // Unused items are deleted 10 minutes after last usage. bool mCache_ContainsAllMetas ; }; @@ -245,13 +247,13 @@ public: * @param metaData The meta data item to update * @return error code */ - int updateMessageMetaData(MsgLocMetaData& metaData); + int updateMessageMetaData(const MsgLocMetaData& metaData); /*! * @param metaData The meta data item to update * @return error code */ - int updateGroupMetaData(GrpLocMetaData& meta); + int updateGroupMetaData(const GrpLocMetaData &meta); /*! * Completely clear out data stored in @@ -292,9 +294,16 @@ private: /*! * Retrieves all the msg meta results from a cursor * @param c cursor to result set - * @param metaSet message metadata retrieved from cursor are stored here + * @param msgMeta message metadata retrieved from cursor are stored here */ - void locked_retrieveMsgMeta(RetroCursor* c, std::vector& msgMeta); + void locked_retrieveMsgMetaList(RetroCursor* c, std::vector& msgMeta); + + /*! + * Retrieves all the grp meta results from a cursor + * @param c cursor to result set + * @param grpMeta group metadata retrieved from cursor are stored here + */ + void locked_retrieveGrpMetaList(RetroCursor *c, std::map& grpMeta); /*! * extracts a msg meta item from a cursor at its diff --git a/libretroshare/src/gxs/rsgds.h b/libretroshare/src/gxs/rsgds.h index 04c2b17e7..fb985b9f7 100644 --- a/libretroshare/src/gxs/rsgds.h +++ b/libretroshare/src/gxs/rsgds.h @@ -239,12 +239,12 @@ public: /*! * @param metaData */ - virtual int updateMessageMetaData(MsgLocMetaData& metaData) = 0; + virtual int updateMessageMetaData(const MsgLocMetaData& metaData) = 0; /*! * @param metaData */ - virtual int updateGroupMetaData(GrpLocMetaData& meta) = 0; + virtual int updateGroupMetaData(const GrpLocMetaData& meta) = 0; virtual int updateGroupKeys(const RsGxsGroupId& grpId,const RsTlvSecurityKeySet& keys,uint32_t subscribed_flags) = 0 ; diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 8debf99f4..bfc1bbd23 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1274,18 +1274,18 @@ bool RsGenExchange::getMsgMeta(const uint32_t &token, for(; mit != result.end(); ++mit) { - std::vector& metaV = mit->second; + std::vector& metaV = mit->second; std::vector& msgInfoV = msgInfo[mit->first]; - std::vector::iterator vit = metaV.begin(); + std::vector::iterator vit = metaV.begin(); RsMsgMetaData meta; for(; vit != metaV.end(); ++vit) { - RsGxsMsgMetaData& m = *(*vit); + const RsGxsMsgMetaData& m = *(*vit); meta = m; msgInfoV.push_back(meta); - delete *vit; + //delete *vit; } metaV.clear(); } @@ -1302,18 +1302,18 @@ bool RsGenExchange::getMsgRelatedMeta(const uint32_t &token, GxsMsgRelatedMetaMa for(; mit != result.end(); ++mit) { - std::vector& metaV = mit->second; + std::vector& metaV = mit->second; std::vector& msgInfoV = msgMeta[mit->first]; - std::vector::iterator vit = metaV.begin(); + std::vector::iterator vit = metaV.begin(); RsMsgMetaData meta; for(; vit != metaV.end(); ++vit) { - RsGxsMsgMetaData& m = *(*vit); + const RsGxsMsgMetaData& m = *(*vit); meta = m; msgInfoV.push_back(meta); - delete *vit; + //delete *vit; } metaV.clear(); } @@ -2016,15 +2016,15 @@ void RsGenExchange::processMsgMetaChanges() if(mit != result.end()) { - std::vector& msgMetaV = mit->second; + std::vector& msgMetaV = mit->second; if(!msgMetaV.empty()) { - RsGxsMsgMetaData* meta = *(msgMetaV.begin()); + const RsGxsMsgMetaData* meta = *(msgMetaV.begin()); value = (meta->mMsgStatus & ~mask) | (mask & value); changed = (static_cast(meta->mMsgStatus) != value); m.val.put(RsGeneralDataService::MSG_META_STATUS, value); - delete meta; + //delete meta; ok = true; } } diff --git a/libretroshare/src/gxs/rsgxs.h b/libretroshare/src/gxs/rsgxs.h index 3f25b087b..fd7f693f9 100644 --- a/libretroshare/src/gxs/rsgxs.h +++ b/libretroshare/src/gxs/rsgxs.h @@ -29,8 +29,8 @@ /* data types used throughout Gxs from netservice to genexchange */ -typedef std::map > GxsMsgMetaResult; -typedef std::map > MsgRelatedMetaResult; +typedef std::map > GxsMsgMetaResult; +typedef std::map > MsgRelatedMetaResult; // Default values that are used throughout GXS code diff --git a/libretroshare/src/gxs/rsgxsdata.cc b/libretroshare/src/gxs/rsgxsdata.cc index 0ec61de8e..c3c9b9f08 100644 --- a/libretroshare/src/gxs/rsgxsdata.cc +++ b/libretroshare/src/gxs/rsgxsdata.cc @@ -209,7 +209,7 @@ RsGxsMsgMetaData::~RsGxsMsgMetaData(){ return; } -uint32_t RsGxsMsgMetaData::serial_size() +uint32_t RsGxsMsgMetaData::serial_size() const { uint32_t s = 8; // header size diff --git a/libretroshare/src/gxs/rsgxsdata.h b/libretroshare/src/gxs/rsgxsdata.h index 282f063fb..6ef5b0c28 100644 --- a/libretroshare/src/gxs/rsgxsdata.h +++ b/libretroshare/src/gxs/rsgxsdata.h @@ -94,7 +94,7 @@ public: ~RsGxsMsgMetaData(); bool deserialise(void *data, uint32_t *size); bool serialise(void* data, uint32_t *size); - uint32_t serial_size(); + uint32_t serial_size() const; void clear(); void operator =(const RsMsgMetaData& rMeta); diff --git a/libretroshare/src/gxs/rsgxsdataaccess.cc b/libretroshare/src/gxs/rsgxsdataaccess.cc index 6104b0b56..a695672f9 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.cc +++ b/libretroshare/src/gxs/rsgxsdataaccess.cc @@ -1028,7 +1028,7 @@ bool RsGxsDataAccess::getMsgMetaDataList( const GxsMsgReq& msgIds, const RsTokRe //auto& filter( metaFilter[grpId] ); // does the initialization of metaFilter[grpId] and avoids further O(log(n)) calls - std::vector& metaV = meta_it->second; + std::vector& metaV = meta_it->second; if (onlyLatestMsgs) // if we only consider latest messages, we need to first filter out messages with "children" { @@ -1062,7 +1062,7 @@ bool RsGxsDataAccess::getMsgMetaDataList( const GxsMsgReq& msgIds, const RsTokRe for(uint32_t i=0;imParentId.isNull()) { - delete msgMeta; + //delete msgMeta; metaV[i] = nullptr; continue; } if (onlyOrigMsgs && !msgMeta->mOrigMsgId.isNull() && msgMeta->mMsgId != msgMeta->mOrigMsgId) { - delete msgMeta; + //delete msgMeta; metaV[i] = nullptr; continue; } @@ -1187,7 +1187,7 @@ bool RsGxsDataAccess::getMsgIdList( const GxsMsgReq& msgIds, const RsTokReqOptio } // delete meta data - cleanseMsgMetaMap(result); + //cleanseMsgMetaMap(result); return true; } @@ -1296,9 +1296,7 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) return true; } - std::vector::iterator vit_msgIds = req->mMsgIds.begin(); - - for(; vit_msgIds != req->mMsgIds.end(); ++vit_msgIds) + for(auto vit_msgIds(req->mMsgIds.begin()); vit_msgIds != req->mMsgIds.end(); ++vit_msgIds) { MsgMetaFilter filterMap; @@ -1310,8 +1308,8 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) GxsMsgReq msgIds; msgIds.insert(std::make_pair(grpMsgIdPair.first, std::set())); mDataStore->retrieveGxsMsgMetaData(msgIds, result); - std::vector& metaV = result[grpMsgIdPair.first]; - std::vector::iterator vit_meta; + std::vector& metaV = result[grpMsgIdPair.first]; + std::vector::iterator vit_meta; // msg id to relate to const RsGxsMessageId& msgId = grpMsgIdPair.second; @@ -1319,10 +1317,11 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) std::set outMsgIds; - RsGxsMsgMetaData* origMeta = nullptr; + const RsGxsMsgMetaData* origMeta = nullptr; + for(vit_meta = metaV.begin(); vit_meta != metaV.end(); ++vit_meta) { - RsGxsMsgMetaData* meta = *vit_meta; + const RsGxsMsgMetaData* meta = *vit_meta; if(msgId == meta->mMsgId) { @@ -1337,12 +1336,11 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) RsDbg() << "RsGxsDataAccess::getMsgRelatedInfo(): Cannot find meta of msgId (to relate to)!" << std::endl; #endif - cleanseMsgMetaMap(result); return false; } const RsGxsMessageId& origMsgId = origMeta->mOrigMsgId; - std::map& metaMap = filterMap[grpId]; + std::map& metaMap = filterMap[grpId]; if (onlyLatestMsgs) { @@ -1354,7 +1352,7 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) for(vit_meta = metaV.begin(); vit_meta != metaV.end(); ++vit_meta) { - RsGxsMsgMetaData* meta = *vit_meta; + const RsGxsMsgMetaData* meta = *vit_meta; // skip msgs that aren't children. if (onlyChildMsgs) @@ -1422,11 +1420,11 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) /* first guess is potentially better than Orig (can't be worse!) */ rstime_t latestTs = 0; RsGxsMessageId latestMsgId; - RsGxsMsgMetaData* latestMeta=nullptr; + const RsGxsMsgMetaData* latestMeta=nullptr; for(vit_meta = metaV.begin(); vit_meta != metaV.end(); ++vit_meta) { - RsGxsMsgMetaData* meta = *vit_meta; + const RsGxsMsgMetaData* meta = *vit_meta; if (meta->mOrigMsgId == origMsgId) { @@ -1446,7 +1444,7 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) { for(vit_meta = metaV.begin(); vit_meta != metaV.end(); ++vit_meta) { - RsGxsMsgMetaData* meta = *vit_meta; + const RsGxsMsgMetaData* meta = *vit_meta; if (meta->mOrigMsgId == origMsgId) { @@ -1482,8 +1480,6 @@ bool RsGxsDataAccess::getMsgRelatedInfo(MsgRelatedInfoReq *req) outMsgIds.clear(); filteredOutMsgIds.clear(); - - cleanseMsgMetaMap(result); } return true; } @@ -1496,7 +1492,7 @@ bool RsGxsDataAccess::getGroupStatistic(GroupStatisticRequest *req) GxsMsgMetaResult metaResult; mDataStore->retrieveGxsMsgMetaData(metaReq, metaResult); - const std::vector& msgMetaV = metaResult[req->mGrpId]; + const std::vector& msgMetaV = metaResult[req->mGrpId]; req->mGroupStatistic.mGrpId = req->mGrpId; req->mGroupStatistic.mNumMsgs = msgMetaV.size(); @@ -1514,7 +1510,7 @@ bool RsGxsDataAccess::getGroupStatistic(GroupStatisticRequest *req) for(uint32_t i = 0; i < msgMetaV.size(); ++i) { - RsGxsMsgMetaData* m = msgMetaV[i]; + const RsGxsMsgMetaData* m = msgMetaV[i]; req->mGroupStatistic.mTotalSizeOfMsgs += m->mMsgSize + m->serial_size(); if(obsolete_msgs.find(m->mMsgId) != obsolete_msgs.end()) // skip obsolete messages. @@ -1540,7 +1536,7 @@ bool RsGxsDataAccess::getGroupStatistic(GroupStatisticRequest *req) } } - cleanseMsgMetaMap(metaResult); + //cleanseMsgMetaMap(metaResult); return true; } @@ -1595,21 +1591,19 @@ bool RsGxsDataAccess::getMsgIdList(MsgIdReq* req) mDataStore->retrieveGxsMsgMetaData(req->mMsgIds, result); - GxsMsgMetaResult::iterator mit = result.begin(), mit_end = result.end(); for(; mit != mit_end; ++mit) { const RsGxsGroupId grpId = mit->first; - std::vector& metaV = mit->second; - std::vector::iterator vit = metaV.begin(), + std::vector& metaV = mit->second; + std::vector::iterator vit = metaV.begin(), vit_end = metaV.end(); for(; vit != vit_end; ++vit) { - RsGxsMsgMetaData* meta = *vit; + const RsGxsMsgMetaData* meta = *vit; req->mMsgIdResult[grpId].insert(meta->mMsgId); - delete meta; // discard meta data mem } } @@ -1622,24 +1616,24 @@ bool RsGxsDataAccess::getMsgIdList(MsgIdReq* req) return true; } -void RsGxsDataAccess::cleanseMsgMetaMap(GxsMsgMetaResult& result) -{ - GxsMsgMetaResult::iterator mit = result.begin(); - - for(; mit !=result.end(); ++mit) - { - - std::vector& msgMetaV = mit->second; - std::vector::iterator vit = msgMetaV.begin(); - for(; vit != msgMetaV.end(); ++vit) - { - delete *vit; - } - } - - result.clear(); - return; -} +// void RsGxsDataAccess::cleanseMsgMetaMap(GxsMsgMetaResult& result) +// { +// GxsMsgMetaResult::iterator mit = result.begin(); +// +// for(; mit !=result.end(); ++mit) +// { +// +// std::vector& msgMetaV = mit->second; +// std::vector::iterator vit = msgMetaV.begin(); +// for(; vit != msgMetaV.end(); ++vit) +// { +// delete *vit; +// } +// } +// +// result.clear(); +// return; +// } void RsGxsDataAccess::filterMsgIdList( GxsMsgIdResult& resultsMap, const RsTokReqOptions& opts, const MsgMetaFilter& msgMetas ) const { @@ -1659,11 +1653,11 @@ void RsGxsDataAccess::filterMsgIdList( GxsMsgIdResult& resultsMap, const RsTokRe for( std::set::iterator msgIdIt = msgsIdSet.begin(); msgIdIt != msgsIdSet.end(); ) { const RsGxsMessageId& msgId(*msgIdIt); - const std::map& msgsMetaMap = + const std::map& msgsMetaMap = cit->second; bool keep = false; - std::map::const_iterator msgsMetaMapIt; + std::map::const_iterator msgsMetaMapIt; if( (msgsMetaMapIt = msgsMetaMap.find(msgId)) != msgsMetaMap.end() ) { diff --git a/libretroshare/src/gxs/rsgxsdataaccess.h b/libretroshare/src/gxs/rsgxsdataaccess.h index 21cb89b24..727ebe605 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.h +++ b/libretroshare/src/gxs/rsgxsdataaccess.h @@ -28,7 +28,7 @@ #include "rsgds.h" -typedef std::map< RsGxsGroupId, std::map > MsgMetaFilter; +typedef std::map< RsGxsGroupId, std::map > MsgMetaFilter; typedef std::map< RsGxsGroupId, RsGxsGrpMetaData* > GrpMetaFilter; bool operator<(const std::pair& p1,const std::pair& p2); @@ -328,11 +328,11 @@ private: */ void tokenList(std::list &tokens); - /*! - * Convenience function to delete the ids - * @param filter the meta filter to clean - */ - void cleanseMsgMetaMap(GxsMsgMetaResult& result); + // /*! + // * Convenience function to delete the ids + // * @param filter the meta filter to clean + // */ + // void cleanseMsgMetaMap(GxsMsgMetaResult& result); public: diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 3259a060c..484ad466b 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -953,7 +953,7 @@ void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStatsItem *grs) #endif mDataStore->retrieveGxsMsgMetaData(reqIds, result); - const std::vector& vec(result[grs->grpId]) ; + const std::vector& vec(result[grs->grpId]) ; if(vec.empty()) // that means we don't have any, or there isn't any, but since the default is always 0, no need to send. return ; @@ -970,12 +970,9 @@ void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStatsItem *grs) // be used to discard groups that are not used. for(uint32_t i=0;ilast_post_TS < vec[i]->mPublishTs) grs_resp->last_post_TS = vec[i]->mPublishTs; - delete vec[i] ; - } #ifdef NXS_NET_DEBUG_6 GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " sending back statistics item with " << vec.size() << " elements." << std::endl; #endif @@ -2953,21 +2950,19 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) reqIds[grpId] = std::set(); GxsMsgMetaResult result; mDataStore->retrieveGxsMsgMetaData(reqIds, result); - std::vector &msgMetaV = result[grpId]; + std::vector &msgMetaV = result[grpId]; #ifdef NXS_NET_DEBUG_1 GXSNETDEBUG_PG(item->PeerId(),grpId) << " retrieving grp message list..." << std::endl; GXSNETDEBUG_PG(item->PeerId(),grpId) << " grp locally contains " << msgMetaV.size() << " messsages." << std::endl; #endif - std::vector::const_iterator vit = msgMetaV.begin(); + std::vector::const_iterator vit = msgMetaV.begin(); std::set msgIdSet; // put ids in set for each searching for(; vit != msgMetaV.end(); ++vit) - { msgIdSet.insert((*vit)->mMsgId); - delete(*vit); - } + msgMetaV.clear(); #ifdef NXS_NET_DEBUG_1 @@ -4367,7 +4362,7 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_ GxsMsgMetaResult metaResult; mDataStore->retrieveGxsMsgMetaData(req, metaResult); - std::vector& msgMetas = metaResult[item->grpId]; + std::vector& msgMetas = metaResult[item->grpId]; #ifdef NXS_NET_DEBUG_0 GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " retrieving message meta data." << std::endl; @@ -4395,9 +4390,9 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_ if(canSendMsgIds(msgMetas, *grpMeta, peer, should_encrypt_to_this_circle_id)) { - for(std::vector::iterator vit = msgMetas.begin();vit != msgMetas.end(); ++vit) + for(auto vit = msgMetas.begin();vit != msgMetas.end(); ++vit) { - RsGxsMsgMetaData* m = *vit; + const RsGxsMsgMetaData* m = *vit; // Check reputation @@ -4497,8 +4492,8 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsgReqItem *item,bool item_ // release meta resource - for(std::vector::iterator vit = msgMetas.begin(); vit != msgMetas.end(); ++vit) - delete *vit; + // for(std::vector::iterator vit = msgMetas.begin(); vit != msgMetas.end(); ++vit) + // delete *vit; } void RsGxsNetService::locked_pushMsgRespFromList(std::list& itemL, const RsPeerId& sslId, const RsGxsGroupId& grp_id,const uint32_t& transN) @@ -4542,7 +4537,7 @@ void RsGxsNetService::locked_pushMsgRespFromList(std::list& itemL, c } } -bool RsGxsNetService::canSendMsgIds(std::vector& msgMetas, const RsGxsGrpMetaData& grpMeta, const RsPeerId& sslId,RsGxsCircleId& should_encrypt_id) +bool RsGxsNetService::canSendMsgIds(std::vector& msgMetas, const RsGxsGrpMetaData& grpMeta, const RsPeerId& sslId,RsGxsCircleId& should_encrypt_id) { #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendMsgIds() CIRCLE VETTING" << std::endl; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 39fd2982e..e4056bb06 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -395,7 +395,7 @@ private: * @return false, if you cannot send to this peer, true otherwise */ bool canSendGrpId(const RsPeerId& sslId, const RsGxsGrpMetaData& grpMeta, std::vector& toVet, bool &should_encrypt); - bool canSendMsgIds(std::vector& msgMetas, const RsGxsGrpMetaData&, const RsPeerId& sslId, RsGxsCircleId &should_encrypt_id); + bool canSendMsgIds(std::vector& msgMetas, const RsGxsGrpMetaData&, const RsPeerId& sslId, RsGxsCircleId &should_encrypt_id); /*! * \brief checkPermissionsForFriendGroup diff --git a/libretroshare/src/gxs/rsgxsutil.cc b/libretroshare/src/gxs/rsgxsutil.cc index 2396618a8..951476403 100644 --- a/libretroshare/src/gxs/rsgxsutil.cc +++ b/libretroshare/src/gxs/rsgxsutil.cc @@ -85,7 +85,7 @@ bool RsGxsMessageCleanUp::clean() for(; mit != result.end(); ++mit) { - std::vector& metaV = mit->second; + std::vector& metaV = mit->second; // First, make a map of which message have a child message. This allows to only delete messages that dont have child messages. // A more accurate way to go would be to compute the time of the oldest message and possibly delete all the branch, but in the @@ -99,7 +99,7 @@ bool RsGxsMessageCleanUp::clean() for( uint32_t i=0;imMsgId)!=messages_with_kids.end()); @@ -132,7 +132,7 @@ bool RsGxsMessageCleanUp::clean() std::cerr << std::endl; #endif - delete meta; + //delete meta; } } From a17f5a21f822dba9cd7b246358db5a5b0f8bc144 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 28 May 2020 15:55:49 +0200 Subject: [PATCH 21/72] re-enabled resultCOunt --- libretroshare/src/gxs/rsdataservice.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index b09e62c99..6834444ae 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -1400,6 +1400,9 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) mGrpMetaDataCache.setCacheUpToDate(true); } delete c; +#ifdef RS_DATA_SERVICE_DEBUG_TIME + resultCount += grp.size(); +#endif // if(c) // { @@ -1418,9 +1421,6 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) // } // valid = c->moveToNext(); // -//#ifdef RS_DATA_SERVICE_DEBUG_TIME -// ++resultCount; -//#endif // } // delete c; // } @@ -1449,6 +1449,10 @@ int RsDataService::retrieveGxsGrpMetaData(RsGxsGrpMetaTemporaryMap& grp) if(meta) grp[mit->first] = meta; +#ifdef RS_DATA_SERVICE_DEBUG_TIME + ++resultCount; +#endif + delete c; // if(c) From 1a362f1cc00bd1835e93da00d5d94f5115026be5 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 28 May 2020 16:06:31 +0200 Subject: [PATCH 22/72] moved a debug output to the right place --- libretroshare/src/gxs/rsdataservice.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 6834444ae..23d59d231 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -1271,13 +1271,13 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes if (c) { locked_retrieveMsgMetaList(c, msgMeta[grpId]); -#ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << mDbName << ": Retrieving (all) Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; -#endif cache.setCacheUpToDate(true); } delete c; } +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << mDbName << ": Retrieving (all) Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; +#endif } else { @@ -1303,12 +1303,11 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes metaSet.push_back(meta); delete c; - -#ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << mDbName << ": Retrieving Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; -#endif } } +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << mDbName << ": Retrieving Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; +#endif } } From 31e6e02a919debb29e193617ab7689add22477ac Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 28 May 2020 17:07:33 +0200 Subject: [PATCH 23/72] removed MsgMeta delete that was left --- libretroshare/src/gxs/rsgxsnetservice.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 484ad466b..72a078075 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -4604,7 +4604,7 @@ bool RsGxsNetService::canSendMsgIds(std::vector& msgMet GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " deleting MsgMeta entry for msg ID " << msgMetas[i]->mMsgId << " signed by " << msgMetas[i]->mAuthorId << " who is not in group circle " << circleId << std::endl; #endif - delete msgMetas[i] ; + //delete msgMetas[i] ; msgMetas[i] = msgMetas[msgMetas.size()-1] ; msgMetas.pop_back() ; } From d76f397358cbf805aebdf23afcc694bdccccbf4d Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 28 May 2020 17:57:42 +0200 Subject: [PATCH 24/72] RsBase64 calculate size properly and avoid FP math --- libretroshare/src/util/rsbase64.cc | 9 ++++----- libretroshare/src/util/rsbase64.h | 4 ++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/util/rsbase64.cc b/libretroshare/src/util/rsbase64.cc index 72ea119e8..f62c40cef 100644 --- a/libretroshare/src/util/rsbase64.cc +++ b/libretroshare/src/util/rsbase64.cc @@ -21,8 +21,6 @@ * * *******************************************************************************/ -#include - #include "util/rsbase64.h" #include "util/rsdebug.h" @@ -144,9 +142,10 @@ /*static*/ size_t RsBase64::encodedSize(size_t decodedSize, bool padding) { if(!decodedSize) return 0; - if(padding) return 4 * (decodedSize + 2) / 3; - return static_cast( - std::ceil(4L * static_cast(decodedSize) / 3L) ); + + // Thanks https://stackoverflow.com/a/45401395 + if(padding) return ceilDivision(decodedSize, 3) * 4; + return ceilDivision(decodedSize * 8, 6); } /*static*/ std::tuple RsBase64::decodedSize( diff --git a/libretroshare/src/util/rsbase64.h b/libretroshare/src/util/rsbase64.h index 0715b15ae..819870089 100644 --- a/libretroshare/src/util/rsbase64.h +++ b/libretroshare/src/util/rsbase64.h @@ -137,4 +137,8 @@ private: */ static inline bool isBase64Char(char c) { return rDict[static_cast(c)] >= 0; } + + /** Perform ceil division without floating point operations */ + static inline size_t ceilDivision(size_t dividend, size_t divisor) + { return (dividend + divisor - 1) / divisor; } }; From c1c303218cff1d12639cb2d3840e4298a05785a3 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 28 May 2020 23:01:04 +0200 Subject: [PATCH 25/72] Change raw memory JSON representation Fix bug reported by b1rdG The new way permits to add more formats in the future without breaking retro-compatibility again. Add support for RsJson in rsdebug for Android --- .../src/serialiser/rstypeserializer.cc | 89 +++++++++---------- .../src/serialiser/rstypeserializer.h | 17 ++-- libretroshare/src/util/rsdebug.h | 8 +- libretroshare/src/util/rsjson.h | 2 +- 4 files changed, 63 insertions(+), 53 deletions(-) diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index 1ea3add9a..7cf799d0f 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -503,8 +503,16 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/, // Binary blocks // //============================================================================// +#if __cplusplus < 201703L +/* Solve weird undefined reference error with C++ < 17 see: + * https://stackoverflow.com/questions/8016780/undefined-reference-to-static-constexpr-char + */ +/*static*/ decltype(RsTypeSerializer::RawMemoryWrapper::base64_key) constexpr +RsTypeSerializer::RawMemoryWrapper::base64_key; + /*static*/ /* without this Android compilation breaks */ constexpr uint32_t RsTypeSerializer::RawMemoryWrapper::MAX_SERIALIZED_CHUNK_SIZE; +#endif /*static*/ void RsTypeSerializer::RawMemoryWrapper::serial_process( @@ -542,18 +550,7 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( ctx.mOffset += second; break; case RsGenericSerializer::DESERIALIZE: - if(first || second) - { - /* Items are created anew before deserialization so buffer pointer - * must be null and size 0 at this point */ - - RsWarn() << __PRETTY_FUNCTION__ << " DESERIALIZE got uninitialized " - << " or pre-allocated buffer! Buffer pointer: " << first - << " must be null and size: " << second << " must be 0 at " - << "this point. Does your item costructor initialize them " - << "properly?" << std::endl; - print_stacktrace(); - } + freshMemCheck(); RS_SERIAL_PROCESS(second); if(!ctx.mOk) break; @@ -597,44 +594,33 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( if(!ctx.mOk) break; std::string encodedValue; RsBase64::encode(first, second, encodedValue, true, false); - ctx.mJson.SetString( - encodedValue.data(), - static_cast(encodedValue.length()), - ctx.mJson.GetAllocator()); + ctx.mOk = ctx.mOk && + RsTypeSerializer::to_JSON(base64_key, encodedValue, ctx.mJson); break; } case RsGenericSerializer::FROM_JSON: { - const bool yelding = !!( - RsSerializationFlags::YIELDING & ctx.mFlags ); - if(!(ctx.mOk || yelding)) - { - clear(); - break; - } - if(!ctx.mJson.IsString()) - { - RsErr() << __PRETTY_FUNCTION__ << " " - << std::errc::invalid_argument << std::endl; - print_stacktrace(); + freshMemCheck(); - ctx.mOk = false; - clear(); - break; - } - if( ctx.mJson.GetStringLength() > + const auto failure = [&]() -> void { ctx.mOk = false; clear(); }; + const bool yielding = !!( + RsSerializationFlags::YIELDING & ctx.mFlags ); + if(!(ctx.mOk || yielding)) return failure(); + + std::string encodedValue; + if(!RsTypeSerializer::from_JSON( + base64_key, encodedValue, ctx.mJson )) return failure(); + + if( encodedValue.length() > RsBase64::encodedSize(MAX_SERIALIZED_CHUNK_SIZE, true) ) { RsErr() << __PRETTY_FUNCTION__ << " " << std::errc::message_size << std::endl; print_stacktrace(); - ctx.mOk = false; - clear(); - break; + return failure(); } - std::string encodedValue = ctx.mJson.GetString(); std::vector decoded; auto ec = RsBase64::decode(encodedValue, decoded); if(ec) @@ -642,9 +628,7 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( RsErr() << __PRETTY_FUNCTION__ << " " << ec << std::endl; print_stacktrace(); - ctx.mOk = false; - clear(); - break; + return failure(); } const auto decodedSize = decoded.size(); @@ -655,11 +639,8 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process( break; } - if(decodedSize != second) - { - first = reinterpret_cast(realloc(first, decodedSize)); - second = static_cast(decodedSize); - } + first = reinterpret_cast(malloc(decodedSize)); + second = static_cast(decodedSize); memcpy(first, decoded.data(), second); break; @@ -675,6 +656,24 @@ void RsTypeSerializer::RawMemoryWrapper::clear() second = 0; } +bool RsTypeSerializer::RawMemoryWrapper::freshMemCheck() +{ + if(first || second) + { + /* Items are created anew before deserialization so buffer pointer + * must be null and size 0 at this point */ + + RsWarn() << __PRETTY_FUNCTION__ << " got uninitialized " + << " or pre-allocated buffer! Buffer pointer: " << first + << " must be null and size: " << second << " must be 0 at " + << "this point. Does your item costructor initialize them " + << "properly?" << std::endl; + print_stacktrace(); + return false; + } + return true; +} + //============================================================================// // std::error_condition // //============================================================================// diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index d72280bb3..3c8af510e 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -59,12 +59,17 @@ struct RsTypeSerializer /// Maximum supported size 10MB static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024; + /** Key used for JSON serialization. + * @note Changing this value breaks JSON API retro-compatibility */ + static constexpr char base64_key[] = "base64"; + /// @see RsSerializable void serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx ) override; private: void clear(); + bool freshMemCheck(); }; /// Most types are not valid sequence containers @@ -777,9 +782,9 @@ struct RsTypeSerializer { if(!yielding) { - std::cerr << __PRETTY_FUNCTION__ << " \"" << memberName - << "\" not found in JSON:" << std::endl - << jDoc << std::endl << std::endl; + RsErr() << __PRETTY_FUNCTION__ << " \"" << memberName + << "\" not found in JSON:" << std::endl + << jDoc << std::endl << std::endl; print_stacktrace(); } ctx.mOk = false; @@ -790,9 +795,9 @@ struct RsTypeSerializer if(!v.IsObject()) { - std::cerr << __PRETTY_FUNCTION__ << " \"" << memberName - << "\" has wrong type in JSON, object expected, got:" - << std::endl << jDoc << std::endl << std::endl; + RsErr() << __PRETTY_FUNCTION__ << " \"" << memberName + << "\" has wrong type in JSON, object expected, got:" + << std::endl << jDoc << std::endl << std::endl; print_stacktrace(); ctx.mOk = false; break; diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index c333bd255..01a8b94df 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -32,6 +32,8 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err); # include # include +# include "util/rsjson.h" + enum class RsLoggerCategories { DEBUG = ANDROID_LOG_DEBUG, @@ -55,6 +57,10 @@ struct t_RsLogger inline stream_type& operator<<(const T& val) { ostr << val; return *this; } + template + inline stream_type& operator<<(const RsJson& val) + { ostr << val; return *this; } + /// needed for manipulators and things like std::endl stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) { @@ -111,7 +117,7 @@ struct t_RsLogger const auto now = system_clock::now(); const auto sec = time_point_cast(now); const auto msec = duration_cast(now - sec); - std::stringstream tstream; + std::ostringstream tstream; tstream << static_cast(CATEGORY) << " " << sec.time_since_epoch().count() << "." << std::setfill('0') << std::setw(3) << msec.count() diff --git a/libretroshare/src/util/rsjson.h b/libretroshare/src/util/rsjson.h index db864a73f..13073f3b2 100644 --- a/libretroshare/src/util/rsjson.h +++ b/libretroshare/src/util/rsjson.h @@ -29,7 +29,7 @@ typedef rapidjson::Document RsJson; /** - * Print out RsJson to a stream, use std::stringstream to get the string + * Print out RsJson to a stream, use std::ostringstream to get the string * @param[out] out output stream * @param[in] jDoc JSON document to print * @return same output stream passed as out parameter From 1dbc0c1fcff2f23cbdac3691f59de60035ecfb1a Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 28 May 2020 23:58:07 +0200 Subject: [PATCH 26/72] fixed long-term bug due to sending an empty list of destkeys for GXSSecurity to encrypt an item --- libretroshare/src/gxs/gxssecurity.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index 6dedf1dff..d5b377328 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -651,6 +651,9 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u try { + if(keys.empty()) + throw std::runtime_error("EVP_SealInit will not be called with 0 keys. GxsSecurity::encrypt() was called with an empty set of destination keys!") ; + for(uint32_t i=0;i Date: Fri, 29 May 2020 01:09:10 +0200 Subject: [PATCH 27/72] rsdebug supports variadic style for atomic log --- libretroshare/src/util/rsdebug.h | 73 +++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index c333bd255..f242e05cf 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -46,6 +46,18 @@ struct t_RsLogger { inline t_RsLogger() = default; + /** Offer variadic style too, as a benefit this has better atomicity then + * << style, but doesn't supports manipulators and things like std::endl + * @see https://stackoverflow.com/a/27375675 */ + template + inline t_RsLogger(Arg&& arg, Args&&... args) + { + ostr << std::forward(arg); + using expander = int[]; + (void)expander{0, (void(ostr << std::forward(args)), 0)...}; + mFlush(); + } + /** On other platforms expose the type of underlying stream. * On Android it cannot work like that so return the class type itself * just for code compatibility with other platforms */ @@ -60,12 +72,7 @@ struct t_RsLogger { if(pf == static_cast( &std::endl< char, std::char_traits > )) - { - __android_log_write( - static_cast(CATEGORY), - "RetroShare", ostr.str().c_str() ); - ostr.str() = ""; - } + mFlush(); else ostr << pf; return *this; @@ -78,6 +85,14 @@ struct t_RsLogger private: std::ostringstream ostr; + + void mFlush() + { + __android_log_write( + static_cast(CATEGORY), + "RetroShare", ostr.str().c_str() ); + ostr.str() = ""; + } }; #else // def __ANDROID__ @@ -99,38 +114,56 @@ enum class RsLoggerCategories template struct t_RsLogger { - inline t_RsLogger() = default; - /// Expose the type of underlying stream using stream_type = decltype(std::cerr); + /// Return underlying stream to write avoiding additional prefixes + static inline stream_type& uStream() { return std::cerr; } + + inline t_RsLogger() = default; + + /** Offer variadic style too, as a benefit this has better atomicity then + * << style, but doesn't supports manipulators and things like std::endl + * @see https://stackoverflow.com/a/27375675 */ + template + inline t_RsLogger(Arg&& arg, Args&&... args) + { + std::ostringstream ostr; + ostr << getPrefix() << std::forward(arg); + using expander = int[]; + (void)expander{0, (void(ostr << std::forward(args)), 0)...}; + ostr << std::endl; + uStream() << ostr.str(); + } + template inline stream_type& operator<<(const T& val) + { return uStream() << getPrefix() << val; } + + /// needed for manipulators and things like std::endl + stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) + { return uStream() << pf; } + +private: + std::string getPrefix() { using namespace std::chrono; const auto now = system_clock::now(); const auto sec = time_point_cast(now); const auto msec = duration_cast(now - sec); - std::stringstream tstream; + std::ostringstream tstream; tstream << static_cast(CATEGORY) << " " << sec.time_since_epoch().count() << "." << std::setfill('0') << std::setw(3) << msec.count() - << " " << val; - return std::cerr << tstream.str(); + << " "; + return tstream.str(); } - - /// needed for manipulators and things like std::endl - stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) - { return std::cerr << pf; } - - /// Return underlying stream to write avoiding additional prefixes - inline stream_type& uStream() const { return std::cerr; } }; #endif // def __ANDROID__ /** - * Comfortable debug message loggin, supports chaining like std::cerr but can + * Comfortable debug message logging, supports chaining like std::cerr but can * be easly and selectively disabled at compile time to reduce generated binary * size and performance impact without too many \#ifdef around. * @@ -190,6 +223,8 @@ struct RsNoDbg { inline RsNoDbg() = default; + template inline RsNoDbg(T, Args...) {} + /** Defined as the type itself just for code compatibility with other * logging classes */ using stream_type = RsNoDbg; From d1e95f94a8e32066dbf83464042606d5718dd96b Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 29 May 2020 09:49:07 +0200 Subject: [PATCH 28/72] added method to print cache size --- libretroshare/src/gxs/rsdataservice.cc | 35 ++++++++++++++++++++++++++ libretroshare/src/gxs/rsdataservice.h | 12 +++++++++ libretroshare/src/gxs/rsgxsdata.h | 1 + 3 files changed, 48 insertions(+) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 23d59d231..bd22a8aa9 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -1721,3 +1721,38 @@ int RsDataService::setCacheSize(uint32_t /* size */) return 0; } +void RsDataService::debug_printCacheSize() const +{ + uint32_t nb_items,nb_items_on_deadlist; + uint64_t total_size,total_size_of_deadlist; + + mGrpMetaDataCache.debug_computeSize(nb_items, nb_items_on_deadlist, total_size,total_size_of_deadlist); + + RsDbg() << "Cache size: " << std::endl; + RsDbg() << " Groups: " << " total: " << nb_items << " (dead: " << nb_items_on_deadlist << "), size: " << total_size << " (Dead: " << total_size_of_deadlist << ")" << std::endl; + + nb_items = 0,nb_items_on_deadlist = 0; + total_size = 0,total_size_of_deadlist = 0; + + for(auto it:mMsgMetaDataCache) + { + uint32_t tmp_nb_items,tmp_nb_items_on_deadlist; + uint64_t tmp_total_size,tmp_total_size_of_deadlist; + + it.second.debug_computeSize(tmp_nb_items, tmp_nb_items_on_deadlist, tmp_total_size,tmp_total_size_of_deadlist); + + nb_items += tmp_nb_items; + nb_items_on_deadlist += tmp_nb_items_on_deadlist; + total_size += tmp_total_size; + total_size_of_deadlist += tmp_total_size_of_deadlist; + } + RsDbg() << " Msgs: " << " total: " << nb_items << " (dead: " << nb_items_on_deadlist << "), size: " << total_size << " (Dead: " << total_size_of_deadlist << ")" << std::endl; +} + + + + + + + + diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index 35234be9e..56dc8dea6 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -126,6 +126,16 @@ public: } } + void debug_computeSize(uint32_t& nb_items, uint32_t& nb_items_on_deadlist, uint64_t& total_size,uint64_t& total_size_of_deadlist) const + { + nb_items = mMetas.size(); + nb_items_on_deadlist = mOldCachedItems.size(); + total_size = 0; + total_size_of_deadlist = 0; + + for(auto it:mMetas) total_size += it.second->serial_size(); + for(auto it:mOldCachedItems) total_size_of_deadlist += it.second->serial_size(); + } private: std::map mMetas; std::list > mOldCachedItems ; // dead list, where items get deleted after being unused for a while. This is due to not using smart ptrs. @@ -274,6 +284,8 @@ public: int updateGroupKeys(const RsGxsGroupId& grpId,const RsTlvSecurityKeySet& keys, uint32_t subscribe_flags) ; + void debug_printCacheSize() const; + private: /*! diff --git a/libretroshare/src/gxs/rsgxsdata.h b/libretroshare/src/gxs/rsgxsdata.h index 6ef5b0c28..2102a8254 100644 --- a/libretroshare/src/gxs/rsgxsdata.h +++ b/libretroshare/src/gxs/rsgxsdata.h @@ -48,6 +48,7 @@ public: bool deserialise(void *data, uint32_t &pktsize); bool serialise(void* data, uint32_t &pktsize, uint32_t api_version); uint32_t serial_size(uint32_t api_version) const; + uint32_t serial_size() const { return serial_size(RS_GXS_GRP_META_DATA_CURRENT_API_VERSION); } void clear(); void operator =(const RsGroupMetaData& rMeta); From 43f7b6c0ba930f5a1b03d94ad999af4a1481746f Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 29 May 2020 11:03:57 +0200 Subject: [PATCH 29/72] RsFiles::alreadyHaveFile look into extra list too --- libretroshare/src/file_sharing/p3filelists.cc | 10 ++++++++-- libretroshare/src/ft/ftserver.cc | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/file_sharing/p3filelists.cc b/libretroshare/src/file_sharing/p3filelists.cc index c7db78b0f..55a28dec6 100644 --- a/libretroshare/src/file_sharing/p3filelists.cc +++ b/libretroshare/src/file_sharing/p3filelists.cc @@ -1424,9 +1424,15 @@ int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::lis return !results.empty() ; } -bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const + +bool p3FileDatabase::search( + const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const { - RS_STACK_MUTEX(mFLSMtx) ; + RS_STACK_MUTEX(mFLSMtx); + + if( (hintflags & RS_FILE_HINTS_EXTRA) && + mExtraFiles->search(hash, hintflags, info) ) + return true; if(hintflags & RS_FILE_HINTS_LOCAL) { diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 3ae1af491..08e294d2b 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -293,7 +293,8 @@ bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& re bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) { - return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info); + return mFileDatabase->search( + hash, RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_LOCAL, info ); } bool ftServer::FileRequest( From 22618de00c0a878a0b98d9fa69442ae2cc958589 Mon Sep 17 00:00:00 2001 From: defnax Date: Fri, 29 May 2020 18:16:59 +0200 Subject: [PATCH 30/72] * Added missed facepalming emoji & stylesheet fixes * Added missed facepalming emoji * Added default stylesheets for Messages selection & hover * Added default selection color for the Main Window ListView --- retroshare-gui/src/gui/emojione.qrc | 2 ++ retroshare-gui/src/gui/emojione/emotes.acs | 2 ++ .../src/gui/emojione/man-facepalming.png | Bin 0 -> 2166 bytes .../src/gui/emojione/woman-facepalming.png | Bin 0 -> 2212 bytes .../src/gui/qss/stylesheet/Standard.qss | 19 ++++++++++++++++++ 5 files changed, 23 insertions(+) create mode 100644 retroshare-gui/src/gui/emojione/man-facepalming.png create mode 100644 retroshare-gui/src/gui/emojione/woman-facepalming.png diff --git a/retroshare-gui/src/gui/emojione.qrc b/retroshare-gui/src/gui/emojione.qrc index fdbfc58aa..b636f7b09 100644 --- a/retroshare-gui/src/gui/emojione.qrc +++ b/retroshare-gui/src/gui/emojione.qrc @@ -937,5 +937,7 @@ emojione/1F1FF-1F1FC.png emojione/flags.png emojione/flags2.png + emojione/man-facepalming.png + emojione/woman-facepalming.png diff --git a/retroshare-gui/src/gui/emojione/emotes.acs b/retroshare-gui/src/gui/emojione/emotes.acs index 6e9007948..26af35787 100644 --- a/retroshare-gui/src/gui/emojione/emotes.acs +++ b/retroshare-gui/src/gui/emojione/emotes.acs @@ -113,6 +113,8 @@ "emojione/people2.png"|":man_with_gua_pi_mao:":"emojione/1F472.png"; "emojione/people2.png"|":levitate:|:man_in_business_suit_levitating:":"emojione/1F574.png"; "emojione/people2.png"|":dancer:":"emojione/1F483.png"; +"emojione/people2.png"|":man_facepalming:":"emojione/man-facepalming.png"; +"emojione/people2.png"|":woman_facepalming:":"emojione/woman-facepalming.png"; "emojione/people2.png"|":bust_in_silhouette:":"emojione/1F464.png"; "emojione/people2.png"|":busts_in_silhouette:":"emojione/1F465.png"; "emojione/people2.png"|":family:":"emojione/1F46A.png"; diff --git a/retroshare-gui/src/gui/emojione/man-facepalming.png b/retroshare-gui/src/gui/emojione/man-facepalming.png new file mode 100644 index 0000000000000000000000000000000000000000..d0594eefdd610a67acaa7fee4425af8ede29e7b1 GIT binary patch literal 2166 zcmZ{lX*AT27skKVG$i{L85uN|jG2^B%nX?!`x@DYLCM&olzlC`knB69RE(`bV=XCr z6H;V}(NMND!*6=|fAK%}Joh>0x#x54xvy@LnTf$MkOT+-fMZyUo(0pv|HQ?~?1ACo zTTEleBaM*&(3r+ecjfpubjMg213UnaBm{ z+XdlVgFH0d13Z`pz+rHh3JjqFgImGjn(7*w>SvW;FijXNG&Df^e*j-UoVRD#{|7h~ z-39xaXGFVitY29(!vK~X}r zs1nww3JXPR5W{?*&o~bGR4EQ+WS-61M1`1~+u3iVQNm`v%~1T^Zin7jp1$~q@>gTG zeL0v0YVqePN243t7FLSNPIn;PhlaiE8I^&{RF|G(E@SL{z~=jIU*bUW*0HPEyoxWz z6_U$_|7!R7G^{ypY43=*Q^%bTwHW;Bh%ZF`bNViG4UX7eVo~baDiL0r%|B%|kjaJg zGTDvj+SVRd=H~7-Z2a=PC@!uz)zs9XJ2Ds?_W?s)r%g?WnFM)Sqf3AB zX+w0S59Hs$1>ze<))E&lyl!>Kbt)`!niDu7AVz(soq?4j;pKTqt3FvHlMLOxwO{SW z)kOsJXmcZW_f?HWlWFng({4GWrca%L%OSF781?lC-^@;#(S}yapR5nps>mu`ov*xf zBXyu6gr?_qPbMhqex@c)T%65OVwa{-nH2Z$gse_gCDIO#KFI1#dbT3_4IG-6IWNU~ zR}Y-oMSevBgN_Fjt13A|AISY!cJ!`mPpf*Q^Xr;#)C7U# z9G~90$)93}d!vV}9M}20Sxf~^Z?GdX??`^e%QnD!*tx|k0irotz5sZvIFA)Zzt;c0 ze=SHXc;!V*>@Y06i#Iwe+9v+=$k&}jUQ{3M=EhtnMsHecG%1c- zJsPlcNlgfEWG8k+j?SR?MA`H(wgubBZ2CY@8U)I zk&r0qx!a2fMtW8&7@sxEa@8JzrOg{PvgL{0xxrP09xh}W);Q)4WK6T@Vj$v6cp9(F(#Ic`)nJJKj}Ct* z`kL0KZPkOkKDYbq-oIvT@j73x%_p*8rV4ecs)no9ds z`;n_9y?VQTLOv>ephI>6w;iHwOJU13L==-(gA?MV6w>H&R@1CSav2-_VAW%Ts(BGh zQ@5-aYR2N{bZ8|?9~6sa9Nk$Ar*sM6gSkNP% z$==0xKG=;8<4+9ghM!r>^AdVGHWacqDA-aqiTPTdFVpc7mS}-jr9)JPiEp?$C!&uA zixw$TBi37u_VAWQXV1L?`X?4vMyxkqx%ORBFnkZS%#G-{Fcm|)p~X7cB#TUzzQ+{U6T}F@=>-L-7E$ImQdFie~aq z>F`~ly2PF;E`1=oS3r@3fsBM=Dj7@{3L<6Kt&P3ExWyg`elRsOU z+-x_;;qI=+GcZe_U+mc)6T%Tke>SJtOYbl~>rAK(!iMzyG~!IU!%^+SESdfK-vYDBq_p8Jup{+s+Y%_< z%d4fU!wvP!R_$XRc_K6R-rw1lJc^09Zoc89vm9u4qJH~tx={Q^@9l&5sbRClw$zF2 z;(JvEB4H<_@{w)MaSzsI(nAU3lzcj}xi?bz zJkXjCv_W^rEg|=((Y1P$_6B>#Eu0)SifA>16swv8E&7U+*28@s(16ec2oa_rxmZhh z4NM)Clt##;INyAfA@6Hbvk`evS8B-aL+2}NZbHp<#VI}zT>MLjc2n*NW(^uCZMRIa zTZR(OsSbUR%^9*(TpFC(nwI^xejOrabn2+}tzYVgPN8C*B*xwQrB*NvSPqWc>*R_J%4@SQA1;8^H_>ZAxkUd5Ro3-C literal 0 HcmV?d00001 diff --git a/retroshare-gui/src/gui/emojione/woman-facepalming.png b/retroshare-gui/src/gui/emojione/woman-facepalming.png new file mode 100644 index 0000000000000000000000000000000000000000..10346185ef0aad7b3871caa5298379e788259345 GIT binary patch literal 2212 zcmZ{lXEfW57sr2yEfi7fF&k6Q=1U8QG1W52x=F#V$|NNHDWYoRZ+W-)k<5` zXsOzzMbVn&(U<=h|8vjxe$V;b^F8<6S2xkb=nj~Uiw*z)u&$1#>811k6%Ey8$A*L) zTpGwt%}@;hn$qac94P+|opekM0U$&e0HR_6;N%iTZ2$lc1^}D(007Se0CvBk4kY4o zKP!FSJj!9L?}SZ zD!|zxzyMYm_HvSF`WJ$I`7D7-pgDEx%bFGy((+Pz1B-NxV{CiY739Gc_7Cz}fVs0GW zCvSzAyh zAVfo+yKn!I>BxmSVIt=TI(X~LME1#>XH631hu?P=Fk+)va2w7}F1yux3?v#~J6TPw z9z{)tC@n`38%^K7e3N%H{`w--md}`}>!{*Od5!3rSrjqp3m+NSUd*eamx`MzX z5LUC_ZE5#7{e4hByHlC=G+2Sr9X%D9ZzLS3Ux~0$fcU>aLFXlfP+a$N|J{0!r(5|U zi3Jf#s2gMC8?-kgpEwOXEQ~vEEZ-`>n_EwxIJ_mpEGQt6Gwtpz3K4rS+FT#=MC~?* zL?>AxOO#-V&P309lr+_0fXU{LhCr-RrZT!|8z5*xuI+)XjZ~Vzos;?pYXD> zvWSqbWGnK{=2_t>ROI7E3F~#Q#~$`aznjuv3MSelBlhr9R7GZ15h^GARS|^zniCf% zk0F112?<9$3-t6I$MZ^DlLPf%l@q$R5kdPh2OW$F1*;sN;GnHy1vV2V)|3SveGR6h z^zEN$L|}UytKI!{15Jh%O)Y8gP~t7Q96%`JAao;ChJh)3VJOWAwYy=@oRR|fPjMwo zj;{2gsmYuOYar0ZRgv{(I%3*yUvM}DwBjY zpUJ<`rAKQ4=e8lv(xTSb%Q)c+9h1>PL}+8nqvb`O8#Y=r8WWDQgQ30CqA#{Yr{~A} z{s3n3XsfokFfTganuNG8?8ZH%Pc#G2h7~jWNwS@7CVNg4C5rahvB-LCKfX4E8x)W> zQK7nUw~B_{z?FlTItCNdac+SPds?qA@yp+l4slQQjIh!OtWwbAQlF3!6z@2T~=Q!Gho)Hm)3yx*%f`y=akA5JN-v z9za&-OJigjVoC5}WMyT2qEs3l3;)_J zCfZr&#haq|&4?_fcBE?XpHq{8LQEKI&+(Kfnp;UvcUF&9XPt6t@Ok>xR@^yf!@co= z>X2H+@#8OVCC;CBpV$nBMDct1sc(Cm8xQP|wpBeIF5S>jGsR_a#!K zQookod`dyhFmRb~E`&=>@DA!UL7N1HafXm?P(y3o=0FdpL^D~Fu1J$oah=*kkE^&6 zR;`IIYz@Eof;S%fNB0Q-s=Z|gXKJ9hH14#r$@p^9Q*7!Gv=lr)E(Yb}Qm?2m<*LAb&WXDbw!k0n z?!&Y6OEg}s)Nm#j^HTX_ZZ-B%FggCtgtSZqnqF9l&1N)tb9V?Z{d1G!nH^D%HJ@?? z>y#(Tk=He=PYYMQib*C^)pYFa#b++DhR+`@JVGK`)?!@1tEbWvzIB@_8P-q5KckhY z;*e_D^Qz^6^le!A57zKAVyX6fKSH7ZoEf37XFXvq-0i(0dkWD={p8=6y#T>VoMulm7tv(8l zENqdGwjidAn$@Q(@W)~&J2|iG#||_wg7SQ~ZKF+6Rr4hBtbTciaxg_+7|A4T??zw^ kp6YjzJ+i4q{3g4geliN7@!7 Date: Sat, 30 May 2020 12:28:02 +0200 Subject: [PATCH 31/72] Fix qss StyleSheet dark theme item color for gtk users gtk users can't set Qt system theme. --- retroshare-gui/src/gui/common/RSElidedItemDelegate.cpp | 8 ++++---- retroshare-gui/src/qss/blacknight.qss | 10 ++++++++++ retroshare-gui/src/qss/qdarkstyle-v2.qss | 5 ++++- retroshare-gui/src/qss/qdarkstyle.qss | 5 ++++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/retroshare-gui/src/gui/common/RSElidedItemDelegate.cpp b/retroshare-gui/src/gui/common/RSElidedItemDelegate.cpp index 89a2ebe8a..3fa95f898 100644 --- a/retroshare-gui/src/gui/common/RSElidedItemDelegate.cpp +++ b/retroshare-gui/src/gui/common/RSElidedItemDelegate.cpp @@ -39,19 +39,19 @@ //#define DEBUG_EID_PAINT 1 /* To test it you can make an empty.qss file with: -QTreeView::item, QTreeWidget::item{ +QTreeView::item, QTreeWidget::item, QListWidget::item{ color: #AB0000; background-color: #00DC00; } -QTreeView::item:selected, QTreeWidget::item:selected{ +QTreeView::item:selected, QTreeWidget::item:selected, QListWidget::item:selected{ color: #00CD00; background-color: #0000BA; } -QTreeView::item:hover, QTreeWidget::item:hover{ +QTreeView::item:hover, QTreeWidget::item:hover, QListWidget::item:hover{ color: #0000EF; background-color: #FEDCBA; } -QQTreeView::item:selected:hover, TreeWidget::item:selected:hover{ +QQTreeView::item:selected:hover, TreeWidget::item:selected:hover, QListWidget::item:selected:hover{ color: #ABCDEF; background-color: #FE0000; } diff --git a/retroshare-gui/src/qss/blacknight.qss b/retroshare-gui/src/qss/blacknight.qss index 3f08e0aa8..62adfe013 100644 --- a/retroshare-gui/src/qss/blacknight.qss +++ b/retroshare-gui/src/qss/blacknight.qss @@ -51,6 +51,10 @@ background: black; color: lightgray; border-color: transparent; } +QTreeView::item, QTreeWidget::item, QListWidget::item{ + color: lightgray; +} + QDialog, QMainWindow{ background-color: qlineargradient(spread:reflect, x1:0, y1:0, x2:0, y2:1, stop:0 rgb(25, 25, 25), stop:0.05 rgb(0, 0, 0), stop:0.95 rgb(0, 0, 0), stop:1 rgb(25, 25, 25)); } @@ -255,6 +259,12 @@ QTextEdit { color: white; } +RSTextBrowser, MimeTextEdit +{ + /*qproperty-textColorQuote: rgb(125, 125, 255);*/ + qproperty-textColorQuotes: ColorList(#789922 #039bd5 #800000 #800080 #008080 #b10dc9 #85144b #3d9970); +} + /* OpModeStatus need to be at end to overload other values*/ OpModeStatus { qproperty-opMode_Full_Color: #007000; diff --git a/retroshare-gui/src/qss/qdarkstyle-v2.qss b/retroshare-gui/src/qss/qdarkstyle-v2.qss index 97a331ac9..cb573d765 100644 --- a/retroshare-gui/src/qss/qdarkstyle-v2.qss +++ b/retroshare-gui/src/qss/qdarkstyle-v2.qss @@ -51,6 +51,9 @@ QWidget { selection-background-color: #1464A0; selection-color: #F0F0F0; } +QTreeView::item, QTreeWidget::item, QListWidget::item{ + color: #F0F0F0; +} QWidget:disabled { background-color: #19232D; @@ -2152,5 +2155,5 @@ GxsChannelDialog GroupTreeWidget QTreeWidget#treeWidget::item{ RSTextBrowser, MimeTextEdit { /*qproperty-textColorQuote: rgb(125, 125, 255);*/ - qproperty-textColorQuotes: ColorList(#0000ff #00ff00 #00ffff #ff0000 #ff00ff #ffff00 #ffffff); + qproperty-textColorQuotes: ColorList(#789922 #039bd5 #800000 #800080 #008080 #b10dc9 #85144b #3d9970); } diff --git a/retroshare-gui/src/qss/qdarkstyle.qss b/retroshare-gui/src/qss/qdarkstyle.qss index 059c87f9d..b50eb9aa1 100644 --- a/retroshare-gui/src/qss/qdarkstyle.qss +++ b/retroshare-gui/src/qss/qdarkstyle.qss @@ -60,6 +60,9 @@ QWidget border-image: none; outline: 0; } +QTreeView::item, QTreeWidget::item, QListWidget::item{ + color: silver; +} QWidget:item:hover { @@ -1299,7 +1302,7 @@ WireGroupItem QFrame#frame{ RSTextBrowser, MimeTextEdit { /*qproperty-textColorQuote: rgb(125, 125, 255);*/ - qproperty-textColorQuotes: ColorList(#0000ff #00ff00 #00ffff #ff0000 #ff00ff #ffff00 #ffffff); + qproperty-textColorQuotes: ColorList(#789922 #039bd5 #800000 #800080 #008080 #b10dc9 #85144b #3d9970); } ChatWidget QFrame#pluginTitleFrame From d256595aa71f46c32d57b15b2b9fe03c76e17d84 Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 30 May 2020 18:19:43 +0200 Subject: [PATCH 32/72] Fix Message Sent keeps Tags --- libretroshare/src/services/p3msgservice.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/services/p3msgservice.cc b/libretroshare/src/services/p3msgservice.cc index 01a825bb3..83c1c8e88 100644 --- a/libretroshare/src/services/p3msgservice.cc +++ b/libretroshare/src/services/p3msgservice.cc @@ -1187,11 +1187,15 @@ bool p3MsgService::MessageSend(MessageInfo &info) /* use processMsg to get the new msgId */ msg->recvTime = time(NULL); msg->msgId = getNewUniqueMsgId(); - + msg->msgFlags |= RS_MSG_OUTGOING; imsg[msg->msgId] = msg; + // Update info for caller + info.msgId = std::to_string(msg->msgId); + info .msgflags = msg->msgFlags; + RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_ADD); } From df12c72ab88d69ca85618dc6a3ef01a6c482b721 Mon Sep 17 00:00:00 2001 From: Phenom Date: Sat, 30 May 2020 19:38:37 +0200 Subject: [PATCH 33/72] Fix Message Delete Last to clear widget. --- retroshare-gui/src/gui/msgs/MessagesDialog.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/msgs/MessagesDialog.cpp b/retroshare-gui/src/gui/msgs/MessagesDialog.cpp index 5d6bc87c5..40b61739a 100644 --- a/retroshare-gui/src/gui/msgs/MessagesDialog.cpp +++ b/retroshare-gui/src/gui/msgs/MessagesDialog.cpp @@ -1105,7 +1105,10 @@ void MessagesDialog::removemessage() void MessagesDialog::messageRemoved() { - ui.messageTreeWidget->setCurrentIndex(lastSelectedIndex); + if (lastSelectedIndex.isValid()) + ui.messageTreeWidget->setCurrentIndex(lastSelectedIndex); + else + insertMsgTxtAndFiles(QModelIndex()); } void MessagesDialog::undeletemessage() From e7cc74751f18910642ae0bb2260fa674758522e3 Mon Sep 17 00:00:00 2001 From: defnax Date: Sun, 31 May 2020 00:21:19 +0200 Subject: [PATCH 34/72] Removed stylesheet it gets issues on other style --- retroshare-gui/src/gui/qss/stylesheet/Standard.qss | 9 --------- 1 file changed, 9 deletions(-) diff --git a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss index 53ac80a44..00569eef2 100644 --- a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss +++ b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss @@ -917,12 +917,3 @@ MessagesDialog QWidget#messageTreeWidget::item:hover { GxsForumThreadWidget QWidget#threadTreeWidget::item { padding: 2px; } - -MainWindow QListView#listWidget::item:selected { - background-color: #cde8ff; - color: black; -} - -MainWindow QListView#listWidget::item:hover { - background-color: #e5f3ff; -} From 295ecf368ee841392d40191176ac830758369882 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sun, 31 May 2020 15:19:00 +0200 Subject: [PATCH 35/72] ExtraFileHash check for integer overflow When passing large periods 2038 problems was silently triggered due to time being stored as int in FileInfo::age, thus causing erratic behaviour in extra files timeout. Now period is checked and if too large an error is reported. Deprecate FileDetails which is confusing dummy wrapper of FileInfo Remove ftExtraList::cleanupEntry deadcode --- libretroshare/src/ft/ftextralist.cc | 65 +++++++++++--------------- libretroshare/src/ft/ftextralist.h | 9 ++-- libretroshare/src/ft/ftserver.cc | 10 ++++ libretroshare/src/retroshare/rsfiles.h | 3 +- 4 files changed, 46 insertions(+), 41 deletions(-) diff --git a/libretroshare/src/ft/ftextralist.cc b/libretroshare/src/ft/ftextralist.cc index ccefe810d..a4a1245e0 100644 --- a/libretroshare/src/ft/ftextralist.cc +++ b/libretroshare/src/ft/ftextralist.cc @@ -4,7 +4,7 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2008 Robert Fernie * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -21,6 +21,9 @@ * * *******************************************************************************/ +#include +#include + #ifdef WINDOWS_SYS #include "util/rswin.h" #endif @@ -245,12 +248,8 @@ bool ftExtraList::cleanupOldFiles() /* remove items */ for(std::list::iterator rit = toRemove.begin(); rit != toRemove.end(); ++rit) { - if (mFiles.end() != (it = mFiles.find(*rit))) - { - cleanupEntry(it->second.info.path, it->second.info.transfer_info_flags); - mFiles.erase(it); - } - mHashOfHash.erase(makeEncryptedHash(*rit)) ; + if (mFiles.end() != (it = mFiles.find(*rit))) mFiles.erase(it); + mHashOfHash.erase(makeEncryptedHash(*rit)); } IndicateConfigChanged(); @@ -258,46 +257,39 @@ bool ftExtraList::cleanupOldFiles() return true; } - -bool ftExtraList::cleanupEntry(std::string /*path*/, TransferRequestFlags /*flags*/) -{ -// if (flags & RS_FILE_CONFIG_CLEANUP_DELETE) -// { -// /* Delete the file? - not yet! */ -// } - return true; -} - - /*** - * Hash file, and add to the files, - * file is removed after period. - **/ - bool ftExtraList::hashExtraFile( std::string path, uint32_t period, TransferRequestFlags flags ) { -#ifdef DEBUG_ELIST - std::cerr << "ftExtraList::hashExtraFile() path: " << path; - std::cerr << " period: " << period; - std::cerr << " flags: " << flags; + constexpr rstime_t max_int = std::numeric_limits::max(); + const rstime_t now = time(nullptr); + const rstime_t timeOut = now + period; - std::cerr << std::endl; -#endif - - auto failure = [](std::string errMsg) + if(timeOut > max_int) { - RsErr() << __PRETTY_FUNCTION__ << " " << errMsg << std::endl; + /* Under the hood period is stored as int FileInfo::age so we do this + * check here to detect 2038 year problem + * https://en.wikipedia.org/wiki/Year_2038_problem */ + RsErr() << __PRETTY_FUNCTION__ << " period: " << period << " > " + << max_int - now << std::errc::value_too_large << std::endl; return false; - }; + } if(!RsDirUtil::fileExists(path)) - return failure("file: " + path + "not found"); + { + RsErr() << __PRETTY_FUNCTION__ << " path: " << path + << std::errc::no_such_file_or_directory << std::endl; + return false; + } if(RsDirUtil::checkDirectory(path)) - return failure("Cannot add a directory: " + path + "as extra file"); + { + RsErr() << __PRETTY_FUNCTION__ << " path: " << path + << std::errc::is_a_directory << std::endl; + return false; + } FileDetails details(path, period, flags); - details.info.age = static_cast(time(nullptr) + period); + details.info.age = static_cast(timeOut); { RS_STACK_MUTEX(extMutex); @@ -492,8 +484,7 @@ bool ftExtraList::loadList(std::list& load) if (ts > (rstime_t)fi->file.age) { - /* to old */ - cleanupEntry(fi->file.path, TransferRequestFlags(fi->flags)); + /* too old */ delete (*it); continue ; } diff --git a/libretroshare/src/ft/ftextralist.h b/libretroshare/src/ft/ftextralist.h index 52bc87c98..229c0fe3c 100644 --- a/libretroshare/src/ft/ftextralist.h +++ b/libretroshare/src/ft/ftextralist.h @@ -60,7 +60,7 @@ #include "pqi/p3cfgmgr.h" #include "util/rstime.h" -class FileDetails +class RS_DEPRECATED_FOR(FileInfo) FileDetails { public: FileDetails() @@ -130,7 +130,11 @@ public: * file is removed after period. **/ - bool hashExtraFile(std::string path, uint32_t period, TransferRequestFlags flags); + /** + * Hash file, and add to the files, file is removed after period. + */ + bool hashExtraFile( + std::string path, uint32_t period, TransferRequestFlags flags ); bool hashExtraFileDone(std::string path, FileInfo &info); /*** @@ -165,7 +169,6 @@ private: /* Worker Functions */ void hashAFile(); bool cleanupOldFiles(); - bool cleanupEntry(std::string path, TransferRequestFlags flags); mutable RsMutex extMutex; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 08e294d2b..0f42b8437 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -22,6 +22,8 @@ #include #include +#include +#include #include "crypto/chacha20.h" //const int ftserverzone = 29539; @@ -820,6 +822,14 @@ bool ftServer::ExtraFileRemove(const RsFileHash& hash) bool ftServer::ExtraFileHash( std::string localpath, rstime_t period, TransferRequestFlags flags ) { + constexpr rstime_t uintmax = std::numeric_limits::max(); + if(period > uintmax) + { + RsErr() << __PRETTY_FUNCTION__ << " period: " << period << " > " + << uintmax << std::errc::value_too_large << std::endl; + return false; + } + return mFtExtra->hashExtraFile( localpath, static_cast(period), flags ); } diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 64563f621..92c017cef 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -658,7 +658,8 @@ public: * @brief Get file details * @jsonapi{development} * @param[in] hash file identifier - * @param[in] hintflags filtering hint (RS_FILE_HINTS_EXTRA|...|RS_FILE_HINTS_LOCAL) + * @param[in] hintflags filtering hint ( RS_FILE_HINTS_UPLOAD|...| + * RS_FILE_HINTS_EXTRA|RS_FILE_HINTS_LOCAL ) * @param[out] info storage for file information * @return true if file found, false otherwise */ From aef87fb1f89630660f63918c2d599c01d4acfa9b Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 1 Jun 2020 00:43:44 +0200 Subject: [PATCH 36/72] Multiple improvements in debugging helpers Add variadic style macro helpers that make debugging code disappear completely (also the paramethers evaluation) when debugging level is lower Chained style << is now atomic like variadic style, still can't benefit of paramethers eveluation stripping Reduced to minimum the difference between Android and other platform debugging code Port debugging code to new helpers, expecially a few corner cases to test it, and as examples of how to use it. --- libretroshare/src/jsonapi/jsonapi.cpp | 8 +- libretroshare/src/libretroshare.pro | 7 +- .../src/retroshare/rsgxsifacehelper.h | 16 +- .../src/serialiser/rstypeserializer.cc | 3 +- .../src/serialiser/rstypeserializer.h | 41 ++- libretroshare/src/util/rsdebug.h | 285 +++++++----------- libretroshare/src/util/rsdebuglevel0.h | 42 +++ libretroshare/src/util/rsdebuglevel1.h | 42 +++ libretroshare/src/util/rsdebuglevel2.h | 42 +++ libretroshare/src/util/rsdebuglevel3.h | 42 +++ libretroshare/src/util/rsdebuglevel4.h | 42 +++ 11 files changed, 365 insertions(+), 205 deletions(-) create mode 100644 libretroshare/src/util/rsdebuglevel0.h create mode 100644 libretroshare/src/util/rsdebuglevel1.h create mode 100644 libretroshare/src/util/rsdebuglevel2.h create mode 100644 libretroshare/src/util/rsdebuglevel3.h create mode 100644 libretroshare/src/util/rsdebuglevel4.h diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index 338782a01..abb68478a 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -467,18 +467,18 @@ void JsonApiServer::registerHandler( const std::function)>& callback ) { /* Declare outside the lambda to avoid returning a dangling - * reference on Android */ + * reference */ RsWarn tWarn; const auto authFail = - [&](int status) -> RsWarn::stream_type& + [&](int status) -> std::ostream& { /* Capture session by reference as it is cheaper then copying * shared_ptr by value which is not needed in this case */ session->close(status, corsOptionsHeaders); return tWarn << "JsonApiServer authentication handler " - "blocked an attempt to call JSON API " - "authenticated method: " << path; + "blocked an attempt to call JSON API " + "authenticated method: " << path; }; if(session->get_request()->get_method() == "OPTIONS") diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 3f9dde680..e3ff086a1 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -468,7 +468,12 @@ HEADERS += turtle/p3turtle.h \ turtle/turtleclientservice.h HEADERS += util/folderiterator.h \ - util/rsdebug.h \ + util/rsdebug.h \ + util/rsdebuglevel0.h \ + util/rsdebuglevel1.h \ + util/rsdebuglevel2.h \ + util/rsdebuglevel3.h \ + util/rsdebuglevel4.h \ util/rskbdinput.h \ util/rsmemory.h \ util/smallobject.h \ diff --git a/libretroshare/src/retroshare/rsgxsifacehelper.h b/libretroshare/src/retroshare/rsgxsifacehelper.h index d2f588623..7db596873 100644 --- a/libretroshare/src/retroshare/rsgxsifacehelper.h +++ b/libretroshare/src/retroshare/rsgxsifacehelper.h @@ -507,25 +507,27 @@ private: std::map mActiveTokens; +#ifdef DEBUG_GXSIFACEHELPER void locked_dumpTokens() { const uint16_t service_id = mGxs.serviceType(); const auto countSize = static_cast(TokenRequestType::__MAX); uint32_t count[countSize] = {0}; - RsDbg() << __PRETTY_FUNCTION__ << "Service 0x" << std::hex << service_id - << " (" << rsServiceControl->getServiceName( - RsServiceInfo::RsServiceInfoUIn16ToFullServiceId(service_id) ) - << ") this=0x" << static_cast(this) - << ") Active tokens (per type): "; + RsDbg rsdbg; + rsdbg << __PRETTY_FUNCTION__ << " Service 0x" << std::hex << service_id + << " (" << rsServiceControl->getServiceName( + RsServiceInfo::RsServiceInfoUIn16ToFullServiceId(service_id) ) + << ") this=0x" << static_cast(this) + << ") Active tokens (per type): "; // let's count how many token of each type we've got. for(auto& it: mActiveTokens) ++count[static_cast(it.second)]; for(uint32_t i=0; i < countSize; ++i) - RsDbg().uStream() /* << i << ":" */ << count[i] << " "; - RsDbg().uStream() << std::endl; + rsdbg /* << i << ":" */ << count[i] << " "; } +#endif // def DEBUG_GXSIFACEHELPER RS_SET_CONTEXT_DEBUG_LEVEL(1) }; diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index 7cf799d0f..0c7bfc435 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -235,8 +235,7 @@ template<> bool RsTypeSerializer::from_JSON( \ \ if(!ret) \ { \ - Dbg3() << __PRETTY_FUNCTION__ << " " << memberName << " not found" \ - << std::endl; \ + RS_DBG3(memberName, " not found"); \ return false; \ } \ \ diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 3c8af510e..6474f6971 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -39,7 +39,7 @@ #include "serialiser/rsserializer.h" #include "serialiser/rsserializable.h" #include "util/rsjson.h" -#include "util/rsdebug.h" +#include "util/rsdebuglevel1.h" #include "util/cxx14retrocompat.h" @@ -715,12 +715,9 @@ struct RsTypeSerializer E& member, const std::string& memberName ) { -#ifdef RSSERIAL_DEBUG - std::cerr << __PRETTY_FUNCTION__ << " processing enum: " - << typeid(E).name() << " as " - << typeid(typename std::underlying_type::type).name() - << std::endl; -#endif + RS_DBG4( "processing enum: ", typeid(E).name(), " as ", + typeid(typename std::underlying_type::type).name() ); + serial_process( j, ctx, reinterpret_cast::type&>(member), @@ -1004,14 +1001,16 @@ protected: uint8_t data[], uint32_t size, uint32_t &offset, T member ) { std::decay_t backupMember = member; +#if RS_DEBUG_LEVEL >= 3 uint32_t offsetBackup = offset; +#endif bool ok = true; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wbool-compare" /* Check with < and not with <= here as we write last byte after * the loop. Order of && operands very important here! */ - while(member > 127 && (ok = offset < size)) + while(member > 127 && (ok = (offset < size))) { // | 128: Set the next byte flag data[offset++] = (static_cast(member & 127)) | 128; @@ -1036,13 +1035,13 @@ protected: data[offset++] = static_cast(member & 127); - Dbg3() << __PRETTY_FUNCTION__ << " backupMember: " << backupMember - << " offsetBackup: " << offsetBackup << " offeset: " << offset - << " serialized as: "; +#if RS_DEBUG_LEVEL >= 3 + RsDbg tdbg( __PRETTY_FUNCTION__, " backupMember: ", backupMember, + " offsetBackup: ", offsetBackup, " offeset: ", offset, + " serialized as: " ); for(; offsetBackup < offset; ++offsetBackup) - Dbg3().uStream() << " " << std::bitset<8>(data[offsetBackup]); - Dbg3().uStream() << std::endl; - + tdbg << " " << std::bitset<8>(data[offsetBackup]); +#endif return ok; } @@ -1082,13 +1081,13 @@ protected: /* If return is not triggered inside the for loop, either the buffer * ended before we encountered the end of the number, or the number * is VLQ encoded improperly */ - RsErr() << __PRETTY_FUNCTION__ << std::errc::illegal_byte_sequence - << " size: " << size - << " offsetBackup: " << offsetBackup - << " offset: " << offset << " bytes: "; + RsErr rserr; + rserr << __PRETTY_FUNCTION__ << std::errc::illegal_byte_sequence + << " size: " << size + << " offsetBackup: " << offsetBackup + << " offset: " << offset << " bytes: "; for(; offsetBackup < offset; ++offsetBackup) - RsErr().uStream() << " " << std::bitset<8>(data[offsetBackup]); - RsErr().uStream() << std::endl; + rserr << " " << std::bitset<8>(data[offsetBackup]); print_stacktrace(); return false; @@ -1151,7 +1150,7 @@ protected: struct ErrConditionWrapper : RsSerializable { - ErrConditionWrapper(const std::error_condition& ec): mec(ec) {} + explicit ErrConditionWrapper(const std::error_condition& ec): mec(ec) {} /** supports only TO_JSON if a different SerializeJob is passed it will * explode at runtime */ diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index 73d5a2132..4a88d8c90 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -21,19 +21,23 @@ *******************************************************************************/ #pragma once -#include +#include +#include #include -/** Stream helper for std::error_condition */ -std::ostream &operator<<(std::ostream& out, const std::error_condition& err); - #ifdef __ANDROID__ # include -# include -# include +#else // def __ANDROID__ +# include +# include +# include +#endif // def __ANDROID__ -# include "util/rsjson.h" +#include "util/rsjson.h" + + +#ifdef __ANDROID__ enum class RsLoggerCategories { DEBUG = ANDROID_LOG_DEBUG, @@ -42,72 +46,7 @@ enum class RsLoggerCategories ERROR = ANDROID_LOG_ERROR, FATAL = ANDROID_LOG_FATAL }; - -template -struct t_RsLogger -{ - inline t_RsLogger() = default; - - /** Offer variadic style too, as a benefit this has better atomicity then - * << style, but doesn't supports manipulators and things like std::endl - * @see https://stackoverflow.com/a/27375675 */ - template - inline t_RsLogger(Arg&& arg, Args&&... args) - { - ostr << std::forward(arg); - using expander = int[]; - (void)expander{0, (void(ostr << std::forward(args)), 0)...}; - mFlush(); - } - - /** On other platforms expose the type of underlying stream. - * On Android it cannot work like that so return the class type itself - * just for code compatibility with other platforms */ - using stream_type = t_RsLogger; - - template - inline stream_type& operator<<(const T& val) - { ostr << val; return *this; } - - template - inline stream_type& operator<<(const RsJson& val) - { ostr << val; return *this; } - - /// needed for manipulators and things like std::endl - stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) - { - if(pf == static_cast( - &std::endl< char, std::char_traits > )) - mFlush(); - else ostr << pf; - - return *this; - } - - /** On other platforms return underlying stream to write avoiding additional - * prefixes. On Android it cannot work like that so return the object itself - * just for code compatibility with other platforms */ - inline stream_type& uStream() { return *this; } - -private: - std::ostringstream ostr; - - void mFlush() - { - __android_log_write( - static_cast(CATEGORY), - "RetroShare", ostr.str().c_str() ); - ostr.str() = ""; - } -}; - #else // def __ANDROID__ - -#include -#include -#include -#include - enum class RsLoggerCategories { DEBUG = 'D', @@ -116,90 +55,101 @@ enum class RsLoggerCategories ERROR = 'E', FATAL = 'F' }; +#endif // def __ANDROID__ + + +/** Stream helper for std::error_condition */ +std::ostream &operator<<(std::ostream& out, const std::error_condition& err); + template -struct t_RsLogger +struct t_RsLogger : std::ostringstream { - /// Expose the type of underlying stream - using stream_type = decltype(std::cerr); + t_RsLogger() { setPrefix(); } + ~t_RsLogger() { flush(); } - /// Return underlying stream to write avoiding additional prefixes - static inline stream_type& uStream() { return std::cerr; } - - inline t_RsLogger() = default; - - /** Offer variadic style too, as a benefit this has better atomicity then - * << style, but doesn't supports manipulators and things like std::endl - * @see https://stackoverflow.com/a/27375675 */ - template - inline t_RsLogger(Arg&& arg, Args&&... args) + /** Offer variadic style, this doesn't supports things like std::endl as + * paramether but when used toghether with conditional debugging macros + * reduces binary size as paramethers of suppressed calls are not evaluated + * and literally disappear in preprocessing fase @see RsDbg */ + template + explicit inline t_RsLogger(Args&&... args) { - std::ostringstream ostr; - ostr << getPrefix() << std::forward(arg); - using expander = int[]; - (void)expander{0, (void(ostr << std::forward(args)), 0)...}; - ostr << std::endl; - uStream() << ostr.str(); + setPrefix(); + + /* Combine initializer list and comma operator so the compiler unpack + * template arguments and feed our own stream without recursion + * see https://stackoverflow.com/a/27375675 */ + using expander = char[]; + (void) expander {0, (void((*this) << std::forward(args)), 0)...}; } - template - inline stream_type& operator<<(const T& val) - { return uStream() << getPrefix() << val; } - - /// needed for manipulators and things like std::endl - stream_type& operator<<(std::ostream& (*pf)(std::ostream&)) - { return uStream() << pf; } + /** Dump buffer stream to log */ + void flush() + { +#ifdef __ANDROID__ + __android_log_write( + static_cast(CATEGORY), + "RetroShare", str().c_str() ); +#else // def __ANDROID__ + (*this) << std::endl; + std::cerr << str(); +#endif // def __ANDROID__ + str() = ""; + } private: - std::string getPrefix() +#ifdef __ANDROID__ + inline void setPrefix() {} +#else // def __ANDROID__ + void setPrefix() { using namespace std::chrono; const auto now = system_clock::now(); const auto sec = time_point_cast(now); const auto msec = duration_cast(now - sec); - std::ostringstream tstream; - tstream << static_cast(CATEGORY) << " " + (*this) << static_cast(CATEGORY) << " " << sec.time_since_epoch().count() << "." - << std::setfill('0') << std::setw(3) << msec.count() - << " "; - return tstream.str(); + << std::setfill('0') << std::setw(3) << msec.count() << " "; } -}; #endif // def __ANDROID__ +}; /** - * Comfortable debug message logging, supports chaining like std::cerr but can - * be easly and selectively disabled at compile time to reduce generated binary - * size and performance impact without too many \#ifdef around. + * Comfortable debug message logging, supports both variadic style and chaining + * style like std::cerr. + * Can be easly and selectively disabled at compile time. + * To reduce generated binary size and performance impact when debugging is + * disabled without too many \#ifdef around the code combining the variadic + * style with the leveled debugging macros is the way to go. * - * To selectively debug your context you can just add something like this in - * in that context, as an example for a class you can just add a line like this - * inside class declaration: + * To selectively debug your file you just need to include the header of desired + * debugging level (0 to 4) @code{.cpp} -RS_SET_CONTEXT_DEBUG_LEVEL(2) +#include "util/rsdebuglevel2.h" @endcode - * And the you can write debug messages around the code of the class like this: + * Then where you want to print debug messages use @code{.cpp} -Dbg1() << "Level 1 debug message example, this will be compiled and " - << "printed" << std::endl; -Dbg2() << "Level 2 debug message example, this will be compiled and " - << "printed" << std::endl; -Dbg3() << "Level 3 debug message example, this will not be compiled and " - << "printed, and without #ifdef around!!" << std::endl; -Dbg4() << "Level 4 debug message example, this will not be compiled and " - << "printed, and without #ifdef around!!" << std::endl; +RS_DBG0("Hello 0 ", "my debug ", my_variable) << " message " << variable2; +RS_DBG1("Hello 1 ", "my debug ", my_variable) << " message " << variable2; +RS_DBG2("Hello 2 ", "my debug ", my_variable) << " message " << variable2; +RS_DBG3("Hello 3 ", "my debug ", my_variable) << " message " << variable2; +RS_DBG4("Hello 4 ", "my debug ", my_variable) << " message " << variable2; @endcode - * To change the debugging level, for example to completely disable debug - * messages you can change it to 0 -@code{.cpp} -RS_SET_CONTEXT_DEBUG_LEVEL(0) -@endcode - * While to set it to maximim level you have to pass 4. + * To change the debugging level just include a different level header like + * `util/rsdebuglevel1.h`, debug messages with lower or equal level then the + * included header will be printed, the others will not. + * Remember then on messages with debug level higher then the included the + * paramethers you pass as macro arguments (variadic style) will disappear in + * the preprocessing phase, so their evaluation will not be included in the + * final binary and not executed at runtime, instead the paramether passed with + * `<<` (chaining style) will be in the compiled binary and evaluated at runtime + * even if are not printed, due to how C++ is made it is not possible to avoid + * this, so we suggest to use variadic style for debug messages. */ using RsDbg = t_RsLogger; - /** * Comfortable log information reporting helper, supports chaining like * std::cerr. @@ -220,6 +170,7 @@ using RsErr = t_RsLogger; * to terminate) messages */ using RsFatal = t_RsLogger; + /** * Keeps compatible syntax with RsDbg but explicitely do nothing in a way that * any modern compiler should be smart enough to optimize out all the function @@ -228,27 +179,32 @@ using RsFatal = t_RsLogger; struct RsNoDbg { inline RsNoDbg() = default; - - template inline RsNoDbg(T, Args...) {} - - /** Defined as the type itself just for code compatibility with other - * logging classes */ - using stream_type = RsNoDbg; + template inline explicit RsNoDbg(Args...) {} /** This match most of the types, but might be not enough for templated * types */ template - inline stream_type& operator<<(const T&) { return *this; } + inline RsNoDbg& operator<<(const T&) { return *this; } /// needed for manipulators and things like std::endl - inline stream_type& operator<<(std::ostream& (*/*pf*/)(std::ostream&)) + inline RsNoDbg& operator<<(std::ostream& (*/*pf*/)(std::ostream&)) { return *this; } - /** Return the object itself just for code compatibility with other - * logging classes */ - inline stream_type& uStream() { return *this; } + /** Do nothing. Just for code compatibility with other logging classes */ + inline void flush() {} }; + + + +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// +/// All the following lines are DEPRECATED!! + +#include "util/rsdeprecate.h" + /** * Concatenate preprocessor tokens A and B without expanding macro definitions * (however, if invoked from a macro, macro arguments are expanded). @@ -267,41 +223,30 @@ struct RsNoDbg // A bunch of boilerplate, but just in one place #define RS_SET_CONTEXT_DEBUG_LEVEL0 \ - using Dbg1 = RsNoDbg; \ - using Dbg2 = RsNoDbg; \ - using Dbg3 = RsNoDbg; \ - using Dbg4 = RsNoDbg; + using Dbg1 RS_DEPRECATED_FOR(RS_DBG1) = RsNoDbg; \ + using Dbg2 RS_DEPRECATED_FOR(RS_DBG2) = RsNoDbg; \ + using Dbg3 RS_DEPRECATED_FOR(RS_DBG3) = RsNoDbg; \ + using Dbg4 RS_DEPRECATED_FOR(RS_DBG4) = RsNoDbg; #define RS_SET_CONTEXT_DEBUG_LEVEL1 \ - using Dbg1 = RsDbg; \ - using Dbg2 = RsNoDbg; \ - using Dbg3 = RsNoDbg; \ - using Dbg4 = RsNoDbg; + using Dbg1 RS_DEPRECATED_FOR(RS_DBG1) = RsDbg; \ + using Dbg2 RS_DEPRECATED_FOR(RS_DBG2) = RsNoDbg; \ + using Dbg3 RS_DEPRECATED_FOR(RS_DBG3) = RsNoDbg; \ + using Dbg4 RS_DEPRECATED_FOR(RS_DBG4) = RsNoDbg; #define RS_SET_CONTEXT_DEBUG_LEVEL2 \ - using Dbg1 = RsDbg; \ - using Dbg2 = RsDbg; \ - using Dbg3 = RsNoDbg; \ - using Dbg4 = RsNoDbg; + using Dbg1 RS_DEPRECATED_FOR(RS_DBG1) = RsDbg; \ + using Dbg2 RS_DEPRECATED_FOR(RS_DBG2) = RsDbg; \ + using Dbg3 RS_DEPRECATED_FOR(RS_DBG3) = RsNoDbg; \ + using Dbg4 RS_DEPRECATED_FOR(RS_DBG4) = RsNoDbg; #define RS_SET_CONTEXT_DEBUG_LEVEL3 \ - using Dbg1 = RsDbg; \ - using Dbg2 = RsDbg; \ - using Dbg3 = RsDbg; \ - using Dbg4 = RsNoDbg; + using Dbg1 RS_DEPRECATED_FOR(RS_DBG1) = RsDbg; \ + using Dbg2 RS_DEPRECATED_FOR(RS_DBG2) = RsDbg; \ + using Dbg3 RS_DEPRECATED_FOR(RS_DBG3) = RsDbg; \ + using Dbg4 RS_DEPRECATED_FOR(RS_DBG4) = RsNoDbg; #define RS_SET_CONTEXT_DEBUG_LEVEL4 \ - using Dbg1 = RsDbg; \ - using Dbg2 = RsDbg; \ - using Dbg3 = RsDbg; \ - using Dbg4 = RsDbg; - - -//////////////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////////////// -/// All the following lines are DEPRECATED!! - -#include - -#include "util/rsdeprecate.h" + using Dbg1 RS_DEPRECATED_FOR(RS_DBG1) = RsDbg; \ + using Dbg2 RS_DEPRECATED_FOR(RS_DBG2) = RsDbg; \ + using Dbg3 RS_DEPRECATED_FOR(RS_DBG3) = RsDbg; \ + using Dbg4 RS_DEPRECATED_FOR(RS_DBG4) = RsDbg; namespace RsLog { enum RS_DEPRECATED_FOR("RsErr, RsDbg, RsNoDbg") logLvl { diff --git a/libretroshare/src/util/rsdebuglevel0.h b/libretroshare/src/util/rsdebuglevel0.h new file mode 100644 index 000000000..5c68ec0be --- /dev/null +++ b/libretroshare/src/util/rsdebuglevel0.h @@ -0,0 +1,42 @@ +/******************************************************************************* + * RetroShare debugging level * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +// #pragma once // This is commented out on purpose! + +#include + +#undef RS_DEBUG_LEVEL +#define RS_DEBUG_LEVEL 0 + +#undef RS_DBG0 +#define RS_DBG0(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG1 +#define RS_DBG1(...) RsNoDbg("") + +#undef RS_DBG2 +#define RS_DBG2(...) RsNoDbg("") + +#undef RS_DBG3 +#define RS_DBG3(...) RsNoDbg("") + +#undef RS_DBG4 +#define RS_DBG4(...) RsNoDbg("") diff --git a/libretroshare/src/util/rsdebuglevel1.h b/libretroshare/src/util/rsdebuglevel1.h new file mode 100644 index 000000000..7e968e402 --- /dev/null +++ b/libretroshare/src/util/rsdebuglevel1.h @@ -0,0 +1,42 @@ +/******************************************************************************* + * RetroShare debugging level * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +// #pragma once // This is commented out on purpose! + +#include + +#undef RS_DEBUG_LEVEL +#define RS_DEBUG_LEVEL 1 + +#undef RS_DBG0 +#define RS_DBG0(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG1 +#define RS_DBG1(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG2 +#define RS_DBG2(...) RsNoDbg() + +#undef RS_DBG3 +#define RS_DBG3(...) RsNoDbg() + +#undef RS_DBG4 +#define RS_DBG4(...) RsNoDbg() diff --git a/libretroshare/src/util/rsdebuglevel2.h b/libretroshare/src/util/rsdebuglevel2.h new file mode 100644 index 000000000..2cbf1a224 --- /dev/null +++ b/libretroshare/src/util/rsdebuglevel2.h @@ -0,0 +1,42 @@ +/******************************************************************************* + * RetroShare debugging level * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +// #pragma once // This is commented out on purpose! + +#include + +#undef RS_DEBUG_LEVEL +#define RS_DEBUG_LEVEL 2 + +#undef RS_DBG0 +#define RS_DBG0(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG1 +#define RS_DBG1(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG2 +#define RS_DBG2(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG3 +#define RS_DBG3(...) RsNoDbg() + +#undef RS_DBG4 +#define RS_DBG4(...) RsNoDbg() diff --git a/libretroshare/src/util/rsdebuglevel3.h b/libretroshare/src/util/rsdebuglevel3.h new file mode 100644 index 000000000..53bcbfe9f --- /dev/null +++ b/libretroshare/src/util/rsdebuglevel3.h @@ -0,0 +1,42 @@ +/******************************************************************************* + * RetroShare debugging level * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +// #pragma once // This is commented out on purpose! + +#include + +#undef RS_DEBUG_LEVEL +#define RS_DEBUG_LEVEL 3 + +#undef RS_DBG0 +#define RS_DBG0(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG1 +#define RS_DBG1(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG2 +#define RS_DBG2(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG3 +#define RS_DBG3(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG4 +#define RS_DBG4(...) RsNoDbg() diff --git a/libretroshare/src/util/rsdebuglevel4.h b/libretroshare/src/util/rsdebuglevel4.h new file mode 100644 index 000000000..f8697cad8 --- /dev/null +++ b/libretroshare/src/util/rsdebuglevel4.h @@ -0,0 +1,42 @@ +/******************************************************************************* + * RetroShare debugging level * + * * + * Copyright (C) 2020 Gioacchino Mazzurco * + * Copyright (C) 2020 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +// #pragma once // This is commented out on purpose! + +#include + +#undef RS_DEBUG_LEVEL +#define RS_DEBUG_LEVEL 4 + +#undef RS_DBG0 +#define RS_DBG0(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG1 +#define RS_DBG1(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG2 +#define RS_DBG2(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG3 +#define RS_DBG3(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) + +#undef RS_DBG4 +#define RS_DBG4(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) From 3586fa78e62a60bf7bfbfbb2fe5f977e14b6c957 Mon Sep 17 00:00:00 2001 From: Klaus Eisentraut <2080545+keisentraut@users.noreply.github.com> Date: Tue, 2 Jun 2020 23:18:34 +0200 Subject: [PATCH 37/72] cherrypicked missing include from https://github.com/public/OpenPGP-SDK/pull/3 by @agl --- openpgpsdk/src/openpgpsdk/openssl_crypto.c | 1 + 1 file changed, 1 insertion(+) diff --git a/openpgpsdk/src/openpgpsdk/openssl_crypto.c b/openpgpsdk/src/openpgpsdk/openssl_crypto.c index d1573437d..086dad77a 100644 --- a/openpgpsdk/src/openpgpsdk/openssl_crypto.c +++ b/openpgpsdk/src/openpgpsdk/openssl_crypto.c @@ -22,6 +22,7 @@ /** \file */ +#include #include #include #include From 58a4eafcc2e03d59310f53f4b9dd6ed05b424d73 Mon Sep 17 00:00:00 2001 From: defnax Date: Wed, 3 Jun 2020 12:24:58 +0200 Subject: [PATCH 38/72] Added reputation check for board posts * Added reputation check for board posts * Added show author feature for the menu --- .../src/gui/Posted/PostedCardView.cpp | 182 +++++++++------- retroshare-gui/src/gui/Posted/PostedItem.cpp | 202 +++++++++++------- retroshare-gui/src/gui/Posted/PostedItem.h | 1 + 3 files changed, 222 insertions(+), 163 deletions(-) diff --git a/retroshare-gui/src/gui/Posted/PostedCardView.cpp b/retroshare-gui/src/gui/Posted/PostedCardView.cpp index e75c2fb62..75a01557a 100644 --- a/retroshare-gui/src/gui/Posted/PostedCardView.cpp +++ b/retroshare-gui/src/gui/Posted/PostedCardView.cpp @@ -146,6 +146,8 @@ void PostedCardView::setup() QAction *CopyLinkAction = new QAction(QIcon(""),tr("Copy RetroShare Link"), this); connect(CopyLinkAction, SIGNAL(triggered()), this, SLOT(copyMessageLink())); + QAction *showInPeopleAct = new QAction(QIcon(), tr("Show author in people tab"), this); + connect(showInPeopleAct, SIGNAL(triggered()), this, SLOT(showAuthorInPeople())); int S = QFontMetricsF(font()).height() ; @@ -157,6 +159,8 @@ void PostedCardView::setup() QMenu *menu = new QMenu(); menu->addAction(CopyLinkAction); + menu->addSeparator(); + menu->addAction(showInPeopleAct); ui->shareButton->setMenu(menu); ui->clearButton->hide(); @@ -172,90 +176,106 @@ void PostedCardView::fill() // return; // } - QPixmap sqpixmap2 = QPixmap(":/images/thumb-default.png"); + RsReputationLevel overall_reputation = rsReputations->overallReputationLevel(mPost.mMeta.mAuthorId); + bool redacted = (overall_reputation == RsReputationLevel::LOCALLY_NEGATIVE); - mInFill = true; - int desired_height = 1.5*(ui->voteDownButton->height() + ui->voteUpButton->height() + ui->scoreLabel->height()); - int desired_width = sqpixmap2.width()*desired_height/(float)sqpixmap2.height(); - - QDateTime qtime; - qtime.setTime_t(mPost.mMeta.mPublishTs); - QString timestamp = qtime.toString("hh:mm dd-MMM-yyyy"); - QString timestamp2 = misc::timeRelativeToNow(mPost.mMeta.mPublishTs); - ui->dateLabel->setText(timestamp2); - ui->dateLabel->setToolTip(timestamp); - - ui->fromLabel->setId(mPost.mMeta.mAuthorId); - - // Use QUrl to check/parse our URL - // The only combination that seems to work: load as EncodedUrl, extract toEncoded(). - QByteArray urlarray(mPost.mLink.c_str()); - QUrl url = QUrl::fromEncoded(urlarray.trimmed()); - QString urlstr = "Invalid Link"; - QString sitestr = "Invalid Link"; - - bool urlOkay = url.isValid(); - if (urlOkay) - { - QString scheme = url.scheme(); - if ((scheme != "https") - && (scheme != "http") - && (scheme != "ftp") - && (scheme != "retroshare")) - { - urlOkay = false; - sitestr = "Invalid Link Scheme"; - } - } - - if (urlOkay) - { - urlstr = QString(" "); - urlstr += messageName(); - urlstr += QString(" "); - - QString siteurl = url.toEncoded(); - sitestr = QString(" %2 ").arg(siteurl).arg(siteurl); - - ui->titleLabel->setText(urlstr); - }else - { - ui->titleLabel->setText(messageName()); - - } - - if (urlarray.isEmpty()) - { - ui->siteLabel->hide(); - } - - ui->siteLabel->setText(sitestr); - - if(mPost.mImage.mData != NULL) - { - QPixmap pixmap; - GxsIdDetails::loadPixmapFromData(mPost.mImage.mData, mPost.mImage.mSize, pixmap,GxsIdDetails::ORIGINAL); - // Wiping data - as its been passed to thumbnail. - - QPixmap scaledpixmap; - if(pixmap.width() > 800){ - QPixmap scaledpixmap = pixmap.scaledToWidth(800, Qt::SmoothTransformation); - ui->pictureLabel->setPixmap(scaledpixmap); - }else{ - ui->pictureLabel->setPixmap(pixmap); - } - } - else if (mPost.mImage.mData == NULL) - { + if(redacted) { + ui->commentButton->setDisabled(true); + ui->voteUpButton->setDisabled(true); + ui->voteDownButton->setDisabled(true); ui->picture_frame->hide(); - } - else - { - ui->picture_frame->show(); - } + ui->fromLabel->setId(mPost.mMeta.mAuthorId); + ui->titleLabel->setText(tr( "

The author of this message (with ID %1) is banned.").arg(QString::fromStdString(mPost.mMeta.mAuthorId.toStdString()))) ; + QDateTime qtime; + qtime.setTime_t(mPost.mMeta.mPublishTs); + QString timestamp = qtime.toString("hh:mm dd-MMM-yyyy"); + ui->dateLabel->setText(timestamp); + } else { + QPixmap sqpixmap2 = QPixmap(":/images/thumb-default.png"); + + mInFill = true; + int desired_height = 1.5*(ui->voteDownButton->height() + ui->voteUpButton->height() + ui->scoreLabel->height()); + int desired_width = sqpixmap2.width()*desired_height/(float)sqpixmap2.height(); + + QDateTime qtime; + qtime.setTime_t(mPost.mMeta.mPublishTs); + QString timestamp = qtime.toString("hh:mm dd-MMM-yyyy"); + QString timestamp2 = misc::timeRelativeToNow(mPost.mMeta.mPublishTs); + ui->dateLabel->setText(timestamp2); + ui->dateLabel->setToolTip(timestamp); + + ui->fromLabel->setId(mPost.mMeta.mAuthorId); + + // Use QUrl to check/parse our URL + // The only combination that seems to work: load as EncodedUrl, extract toEncoded(). + QByteArray urlarray(mPost.mLink.c_str()); + QUrl url = QUrl::fromEncoded(urlarray.trimmed()); + QString urlstr = "Invalid Link"; + QString sitestr = "Invalid Link"; + + bool urlOkay = url.isValid(); + if (urlOkay) + { + QString scheme = url.scheme(); + if ((scheme != "https") + && (scheme != "http") + && (scheme != "ftp") + && (scheme != "retroshare")) + { + urlOkay = false; + sitestr = "Invalid Link Scheme"; + } + } + + if (urlOkay) + { + urlstr = QString(" "); + urlstr += messageName(); + urlstr += QString(" "); + + QString siteurl = url.toEncoded(); + sitestr = QString(" %2 ").arg(siteurl).arg(siteurl); + + ui->titleLabel->setText(urlstr); + }else + { + ui->titleLabel->setText(messageName()); + + } + + if (urlarray.isEmpty()) + { + ui->siteLabel->hide(); + } + + ui->siteLabel->setText(sitestr); + + if(mPost.mImage.mData != NULL) + { + QPixmap pixmap; + GxsIdDetails::loadPixmapFromData(mPost.mImage.mData, mPost.mImage.mSize, pixmap,GxsIdDetails::ORIGINAL); + // Wiping data - as its been passed to thumbnail. + + QPixmap scaledpixmap; + if(pixmap.width() > 800){ + QPixmap scaledpixmap = pixmap.scaledToWidth(800, Qt::SmoothTransformation); + ui->pictureLabel->setPixmap(scaledpixmap); + }else{ + ui->pictureLabel->setPixmap(pixmap); + } + } + else if (mPost.mImage.mData == NULL) + { + ui->picture_frame->hide(); + } + else + { + ui->picture_frame->show(); + } + } //QString score = "Hot" + QString::number(post.mHotScore); //score += " Top" + QString::number(post.mTopScore); diff --git a/retroshare-gui/src/gui/Posted/PostedItem.cpp b/retroshare-gui/src/gui/Posted/PostedItem.cpp index 751f68ee5..46de8a1e6 100644 --- a/retroshare-gui/src/gui/Posted/PostedItem.cpp +++ b/retroshare-gui/src/gui/Posted/PostedItem.cpp @@ -32,6 +32,8 @@ #include "gui/common/FilesDefs.h" #include "util/qtthreadsutils.h" #include "util/HandleRichText.h" +#include "gui/MainWindow.h" +#include "gui/Identity/IdDialog.h" #include "PhotoView.h" #include "ui_PostedItem.h" @@ -338,6 +340,24 @@ void BasePostedItem::viewPicture() /* window will destroy itself! */ } +void BasePostedItem::showAuthorInPeople() +{ + if(mPost.mMeta.mAuthorId.isNull()) + { + std::cerr << "(EE) GxsForumThreadWidget::loadMsgData_showAuthorInPeople() ERROR Missing Message Data..."; + std::cerr << std::endl; + } + + /* window will destroy itself! */ + IdDialog *idDialog = dynamic_cast(MainWindow::getPage(MainWindow::People)); + + if (!idDialog) + return ; + + MainWindow::showWindow(MainWindow::People); + idDialog->navigate(RsGxsId(mPost.mMeta.mAuthorId)); +} + //======================================================================================== // PostedItem // //======================================================================================== @@ -394,6 +414,8 @@ void PostedItem::setup() QAction *CopyLinkAction = new QAction(QIcon(""),tr("Copy RetroShare Link"), this); connect(CopyLinkAction, SIGNAL(triggered()), this, SLOT(copyMessageLink())); + QAction *showInPeopleAct = new QAction(QIcon(), tr("Show author in people tab"), this); + connect(showInPeopleAct, SIGNAL(triggered()), this, SLOT(showAuthorInPeople())); int S = QFontMetricsF(font()).height() ; @@ -407,6 +429,8 @@ void PostedItem::setup() QMenu *menu = new QMenu(); menu->addAction(CopyLinkAction); + menu->addSeparator(); + menu->addAction(showInPeopleAct); ui->shareButton->setMenu(menu); ui->clearButton->hide(); @@ -438,8 +462,6 @@ void PostedItem::makeUpVote() emit vote(msgId, true); } - - void PostedItem::setComment(const RsGxsComment& cmt) { ui->newCommentLabel->show(); @@ -459,97 +481,115 @@ void PostedItem::setCommentsSize(int comNb) void PostedItem::fill() { - RetroShareLink link = RetroShareLink::createGxsGroupLink(RetroShareLink::TYPE_POSTED, mGroupMeta.mGroupId, groupName()); - ui->nameLabel->setText(link.toHtml()); + RsReputationLevel overall_reputation = rsReputations->overallReputationLevel(mPost.mMeta.mAuthorId); + bool redacted = (overall_reputation == RsReputationLevel::LOCALLY_NEGATIVE); - QPixmap sqpixmap2 = QPixmap(":/images/thumb-default.png"); + if(redacted) { + ui->expandButton->setDisabled(true); + ui->commentButton->setDisabled(true); + ui->voteUpButton->setDisabled(true); + ui->voteDownButton->setDisabled(true); - mInFill = true; - int desired_height = 1.5*(ui->voteDownButton->height() + ui->voteUpButton->height() + ui->scoreLabel->height()); - int desired_width = sqpixmap2.width()*desired_height/(float)sqpixmap2.height(); + ui->thumbnailLabel->setPixmap( QPixmap(":/images/thumb-default.png")); + ui->fromLabel->setId(mPost.mMeta.mAuthorId); + ui->titleLabel->setText(tr( "

The author of this message (with ID %1) is banned.").arg(QString::fromStdString(mPost.mMeta.mAuthorId.toStdString()))) ; + QDateTime qtime; + qtime.setTime_t(mPost.mMeta.mPublishTs); + QString timestamp = qtime.toString("hh:mm dd-MMM-yyyy"); + ui->dateLabel->setText(timestamp); + } else { + RetroShareLink link = RetroShareLink::createGxsGroupLink(RetroShareLink::TYPE_POSTED, mGroupMeta.mGroupId, groupName()); + ui->nameLabel->setText(link.toHtml()); - QDateTime qtime; - qtime.setTime_t(mPost.mMeta.mPublishTs); - QString timestamp = qtime.toString("hh:mm dd-MMM-yyyy"); - QString timestamp2 = misc::timeRelativeToNow(mPost.mMeta.mPublishTs); - ui->dateLabel->setText(timestamp2); - ui->dateLabel->setToolTip(timestamp); + QPixmap sqpixmap2 = QPixmap(":/images/thumb-default.png"); - ui->fromLabel->setId(mPost.mMeta.mAuthorId); + mInFill = true; + int desired_height = 1.5*(ui->voteDownButton->height() + ui->voteUpButton->height() + ui->scoreLabel->height()); + int desired_width = sqpixmap2.width()*desired_height/(float)sqpixmap2.height(); - // Use QUrl to check/parse our URL - // The only combination that seems to work: load as EncodedUrl, extract toEncoded(). - QByteArray urlarray(mPost.mLink.c_str()); - QUrl url = QUrl::fromEncoded(urlarray.trimmed()); - QString urlstr = "Invalid Link"; - QString sitestr = "Invalid Link"; + QDateTime qtime; + qtime.setTime_t(mPost.mMeta.mPublishTs); + QString timestamp = qtime.toString("hh:mm dd-MMM-yyyy"); + QString timestamp2 = misc::timeRelativeToNow(mPost.mMeta.mPublishTs); + ui->dateLabel->setText(timestamp2); + ui->dateLabel->setToolTip(timestamp); - bool urlOkay = url.isValid(); - if (urlOkay) - { - QString scheme = url.scheme(); - if ((scheme != "https") - && (scheme != "http") - && (scheme != "ftp") - && (scheme != "retroshare")) + ui->fromLabel->setId(mPost.mMeta.mAuthorId); + + // Use QUrl to check/parse our URL + // The only combination that seems to work: load as EncodedUrl, extract toEncoded(). + QByteArray urlarray(mPost.mLink.c_str()); + QUrl url = QUrl::fromEncoded(urlarray.trimmed()); + QString urlstr = "Invalid Link"; + QString sitestr = "Invalid Link"; + + bool urlOkay = url.isValid(); + if (urlOkay) { - urlOkay = false; - sitestr = "Invalid Link Scheme"; + QString scheme = url.scheme(); + if ((scheme != "https") + && (scheme != "http") + && (scheme != "ftp") + && (scheme != "retroshare")) + { + urlOkay = false; + sitestr = "Invalid Link Scheme"; + } } - } - if (urlOkay) - { - urlstr = QString(" "); - urlstr += messageName(); - urlstr += QString(" "); + if (urlOkay) + { + urlstr = QString(" "); + urlstr += messageName(); + urlstr += QString(" "); - QString siteurl = url.toEncoded(); - sitestr = QString(" %2 ").arg(siteurl).arg(siteurl); + QString siteurl = url.toEncoded(); + sitestr = QString(" %2 ").arg(siteurl).arg(siteurl); - ui->titleLabel->setText(urlstr); - }else - { - ui->titleLabel->setText(messageName()); + ui->titleLabel->setText(urlstr); + }else + { + ui->titleLabel->setText(messageName()); - } - - if (urlarray.isEmpty()) - { - ui->siteLabel->hide(); - } - - ui->siteLabel->setText(sitestr); - - if(mPost.mImage.mData != NULL) - { - QPixmap pixmap; - GxsIdDetails::loadPixmapFromData(mPost.mImage.mData, mPost.mImage.mSize, pixmap,GxsIdDetails::ORIGINAL); - // Wiping data - as its been passed to thumbnail. - - QPixmap sqpixmap = pixmap.scaled(desired_width,desired_height, Qt::KeepAspectRatioByExpanding, Qt::SmoothTransformation); - ui->thumbnailLabel->setPixmap(sqpixmap); - ui->thumbnailLabel->setToolTip(tr("Click to view Picture")); - - QPixmap scaledpixmap; - if(pixmap.width() > 800){ - QPixmap scaledpixmap = pixmap.scaledToWidth(800, Qt::SmoothTransformation); - ui->pictureLabel->setPixmap(scaledpixmap); - }else{ - ui->pictureLabel->setPixmap(pixmap); } - } - else if (urlOkay && (mPost.mImage.mData == NULL)) - { - ui->expandButton->setDisabled(true); - ui->thumbnailLabel->setPixmap(FilesDefs::getPixmapFromQtResourcePath(LINK_IMAGE)); - } - else - { - ui->expandButton->setDisabled(true); - ui->thumbnailLabel->setPixmap(sqpixmap2); + + if (urlarray.isEmpty()) + { + ui->siteLabel->hide(); + } + + ui->siteLabel->setText(sitestr); + + if(mPost.mImage.mData != NULL) + { + QPixmap pixmap; + GxsIdDetails::loadPixmapFromData(mPost.mImage.mData, mPost.mImage.mSize, pixmap,GxsIdDetails::ORIGINAL); + // Wiping data - as its been passed to thumbnail. + + QPixmap sqpixmap = pixmap.scaled(desired_width,desired_height, Qt::KeepAspectRatioByExpanding, Qt::SmoothTransformation); + ui->thumbnailLabel->setPixmap(sqpixmap); + ui->thumbnailLabel->setToolTip(tr("Click to view Picture")); + + QPixmap scaledpixmap; + if(pixmap.width() > 800){ + QPixmap scaledpixmap = pixmap.scaledToWidth(800, Qt::SmoothTransformation); + ui->pictureLabel->setPixmap(scaledpixmap); + }else{ + ui->pictureLabel->setPixmap(pixmap); + } + } + else if (urlOkay && (mPost.mImage.mData == NULL)) + { + ui->expandButton->setDisabled(true); + ui->thumbnailLabel->setPixmap(FilesDefs::getPixmapFromQtResourcePath(LINK_IMAGE)); + } + else + { + ui->expandButton->setDisabled(true); + ui->thumbnailLabel->setPixmap(sqpixmap2); + } } @@ -701,5 +741,3 @@ void PostedItem::toggleNotes() } } - - diff --git a/retroshare-gui/src/gui/Posted/PostedItem.h b/retroshare-gui/src/gui/Posted/PostedItem.h index 7debd9d75..d726cf830 100644 --- a/retroshare-gui/src/gui/Posted/PostedItem.h +++ b/retroshare-gui/src/gui/Posted/PostedItem.h @@ -55,6 +55,7 @@ private slots: void readAndClearItem(); void copyMessageLink(); void viewPicture(); + void showAuthorInPeople(); signals: void vote(const RsGxsGrpMsgIdPair& msgId, bool up); From bbaad838572b5fba6109bc7d3b5c55c2c68e6cdc Mon Sep 17 00:00:00 2001 From: Phenom Date: Mon, 1 Jun 2020 13:44:43 +0200 Subject: [PATCH 39/72] Fix build with Qt 5.15+ --- retroshare-gui/src/gui/elastic/elnode.h | 8 +++++--- .../src/gui/gxs/GxsCommentTreeWidget.cpp | 16 ++++++++------- retroshare-gui/src/gui/gxs/GxsIdDetails.cpp | 20 ++++++++++--------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/retroshare-gui/src/gui/elastic/elnode.h b/retroshare-gui/src/gui/elastic/elnode.h index 1e0edd1a8..33043ddab 100644 --- a/retroshare-gui/src/gui/elastic/elnode.h +++ b/retroshare-gui/src/gui/elastic/elnode.h @@ -23,6 +23,10 @@ #ifndef ELNODE_H #define ELNODE_H +#include "graphwidget.h" + +#include + #include #if QT_VERSION >= 0x040600 #include @@ -30,9 +34,7 @@ #include #endif #include - -#include -#include "graphwidget.h" +#include class Edge; QT_BEGIN_NAMESPACE diff --git a/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp b/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp index 14413963f..a1f824e9a 100644 --- a/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp +++ b/retroshare-gui/src/gui/gxs/GxsCommentTreeWidget.cpp @@ -18,6 +18,14 @@ * * *******************************************************************************/ +#include "GxsCommentTreeWidget.h" + +#include "gui/common/FilesDefs.h" +#include "gui/common/RSElidedItemDelegate.h" +#include "gui/common/RSTreeWidgetItem.h" +#include "gui/gxs/GxsCreateCommentDialog.h" +#include "gui/gxs/GxsIdTreeWidgetItem.h" + #include #include #include @@ -25,15 +33,9 @@ #include #include #include +#include #include -#include "gui/common/RSElidedItemDelegate.h" -#include "gui/common/FilesDefs.h" -#include "gui/gxs/GxsCommentTreeWidget.h" -#include "gui/gxs/GxsCreateCommentDialog.h" -#include "gui/gxs/GxsIdTreeWidgetItem.h" -#include "gui/common/RSTreeWidgetItem.h" - #include #define PCITEM_COLUMN_COMMENT 0 diff --git a/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp b/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp index 22469575f..701a9153d 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp @@ -18,21 +18,23 @@ * * *******************************************************************************/ -#include -#include -#include -#include - -#include -#include -#include "gui/common/AvatarDialog.h" #include "GxsIdDetails.h" + +#include "gui/common/AvatarDialog.h" #include "retroshare-gui/RsAutoUpdatePage.h" #include +#include + +#include +#include +#include +#include +#include +#include #include -#include +#include /* Images for tag icons */ #define IMAGE_LOADING ":/images/folder-draft.png" From f75a6424b5e04e5b93f4ff39a80ca60eb0d167db Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 28 May 2020 23:58:07 +0200 Subject: [PATCH 40/72] fixed long-term bug due to sending an empty list of destkeys for GXSSecurity to encrypt an item --- libretroshare/src/gxs/gxssecurity.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index 6dedf1dff..d5b377328 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -651,6 +651,9 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u try { + if(keys.empty()) + throw std::runtime_error("EVP_SealInit will not be called with 0 keys. GxsSecurity::encrypt() was called with an empty set of destination keys!") ; + for(uint32_t i=0;i Date: Thu, 4 Jun 2020 19:38:46 +0200 Subject: [PATCH 41/72] Enabled sorting for gxs transport statistics, view details & small changes * Enabled sorting for gxs transport statistics * Context menu to view author details on gxs transport groups * ui layout fixes for view boards details * Display counting for people tree --- retroshare-gui/src/gui/Identity/IdDialog.cpp | 8 + .../src/gui/Posted/PostedListWidget.ui | 387 +++++++++--------- .../gui/statistics/GxsTransportStatistics.cpp | 63 ++- .../gui/statistics/GxsTransportStatistics.h | 3 + .../gui/statistics/GxsTransportStatistics.ui | 15 +- 5 files changed, 270 insertions(+), 206 deletions(-) diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index 2fd243c20..1a585d532 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -1575,6 +1575,14 @@ void IdDialog::loadIdentities(const std::map& ids_set /* count items */ int itemCount = contactsItem->childCount() + allItem->childCount() + ownItem->childCount(); ui->label_count->setText( "(" + QString::number( itemCount ) + ")" ); + + int contactsCount = contactsItem->childCount() ; + int allCount = allItem->childCount() ; + int ownCount = ownItem->childCount(); + + contactsItem->setText(0, tr("My contacts") + " (" + QString::number( contactsCount ) + ")" ); + allItem->setText(0, tr("All") + " (" + QString::number( allCount ) + ")" ); + ownItem->setText(0, tr("My own identities") + " (" + QString::number( ownCount ) + ")" ); navigate(RsGxsId(oldCurrentId)); filterIds(); diff --git a/retroshare-gui/src/gui/Posted/PostedListWidget.ui b/retroshare-gui/src/gui/Posted/PostedListWidget.ui index c4a6d51cd..3fa7069e6 100644 --- a/retroshare-gui/src/gui/Posted/PostedListWidget.ui +++ b/retroshare-gui/src/gui/Posted/PostedListWidget.ui @@ -298,199 +298,10 @@ false - - - - - - - 64 - 64 - - - - - 64 - 64 - - - - :/icons/png/postedlinks.png - - - true - - - - - - - - 14 - - - - TextLabel - - - - - - - - - 6 - - - - - - 75 - true - - - - Popularity - - - - - - - - 75 - true - - - - 0 - - - - - - - - 0 - 0 - - - - - 75 - true - - - - Posts - - - - - - - - 75 - true - - - - 0 - - - - - - - - 75 - true - - - - Created - - - - - - - unknown - - - - - - - - 75 - true - - - - Administrator: - - - - - - - unknown - - - true - - - - - - - - 75 - true - - - - Distribution: - - - - - - - unknown - - - - - - - - 0 - 0 - - - - - 75 - true - - - - Last Post: - - - - - - - unknown - - - - - - + + 6 + + Qt::Horizontal @@ -503,7 +314,7 @@ - + @@ -526,6 +337,194 @@ p, li { white-space: pre-wrap; } + + + + 0 + + + + + + 75 + true + + + + Popularity + + + + + + + + 75 + true + + + + 0 + + + + + + + + 0 + 0 + + + + + 75 + true + + + + Posts + + + + + + + + 75 + true + + + + 0 + + + + + + + + 75 + true + + + + Created + + + + + + + unknown + + + + + + + + 75 + true + + + + Administrator: + + + + + + + unknown + + + true + + + + + + + + 75 + true + + + + Distribution: + + + + + + + unknown + + + + + + + + 0 + 0 + + + + + 75 + true + + + + Last Post: + + + + + + + unknown + + + + + + + + 64 + 64 + + + + + 64 + 64 + + + + :/icons/png/postedlinks.png + + + true + + + + + + + + 14 + + + + TextLabel + + + + + diff --git a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp index 10663a735..4b3ddf5e8 100644 --- a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp +++ b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp @@ -49,6 +49,8 @@ #include "gui/gxs/GxsIdLabel.h" #include "gui/gxs/GxsIdDetails.h" #include "gui/gxs/GxsIdTreeWidgetItem.h" +#include "gui/Identity/IdDialog.h" +#include "gui/MainWindow.h" #define COL_PENDING_ID 0 #define COL_PENDING_DESTINATION 1 @@ -61,11 +63,13 @@ #define COL_PENDING_DESTINATION_ID 8 #define COL_GROUP_GRP_ID 0 -#define COL_GROUP_NUM_MSGS 1 -#define COL_GROUP_SIZE_MSGS 2 -#define COL_GROUP_SUBSCRIBED 3 -#define COL_GROUP_POPULARITY 4 -#define COL_GROUP_UNIQUE_ID 5 +#define COL_GROUP_PUBLISHTS 1 +#define COL_GROUP_NUM_MSGS 2 +#define COL_GROUP_SIZE_MSGS 3 +#define COL_GROUP_SUBSCRIBED 4 +#define COL_GROUP_POPULARITY 5 +#define COL_GROUP_UNIQUE_ID 6 +#define COL_GROUP_AUTHOR_ID 7 //static const int PARTIAL_VIEW_SIZE = 9 ; //static const int MAX_TUNNEL_REQUESTS_DISPLAY = 10 ; @@ -94,8 +98,10 @@ GxsTransportStatistics::GxsTransportStatistics(QWidget *parent) QHeaderView_setSectionResizeMode(groupTreeWidget->header(), QHeaderView::ResizeToContents); connect(treeWidget, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(CustomPopupMenu(QPoint))); - - treeWidget->setColumnHidden(COL_PENDING_DESTINATION_ID,true); + connect(groupTreeWidget, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(CustomPopupMenuGroups(QPoint))); + + treeWidget->setColumnHidden(COL_PENDING_DESTINATION_ID,true); + groupTreeWidget->setColumnHidden(COL_GROUP_AUTHOR_ID,true); // load settings processSettings(true); @@ -146,6 +152,19 @@ void GxsTransportStatistics::CustomPopupMenu( QPoint ) contextMnu.exec(QCursor::pos()); } +void GxsTransportStatistics::CustomPopupMenuGroups( QPoint ) +{ + QMenu contextMnu( this ); + + QTreeWidgetItem *item = groupTreeWidget->currentItem(); + if (item) { + contextMnu.addAction(QIcon(":/images/info16.png"), tr("View details"), this, SLOT(showAuthorInPeople())); + + } + + contextMnu.exec(QCursor::pos()); +} + void GxsTransportStatistics::updateDisplay(bool) { time_t now = time(NULL) ; @@ -275,9 +294,10 @@ void GxsTransportStatistics::updateContent() groupTreeWidget->addTopLevelItem(item); groupTreeWidget->setItemExpanded(item,openned_groups.find(it->first) != openned_groups.end()); - QString msg_time_string = (stat.last_publish_TS>0)?QString(" (Last msg: %1)").arg(QDateTime::fromTime_t((uint)stat.last_publish_TS).toString()):"" ; + QString msg_time_string = (stat.last_publish_TS>0)?QString("(Last msg: %1)").arg(QDateTime::fromTime_t((uint)stat.last_publish_TS).toString()):"" ; - item->setData(COL_GROUP_NUM_MSGS, Qt::DisplayRole, QString::number(stat.mNumMsgs) + msg_time_string) ; + item->setData(COL_GROUP_PUBLISHTS, Qt::DisplayRole, msg_time_string) ; + item->setData(COL_GROUP_NUM_MSGS, Qt::DisplayRole, QString::number(stat.mNumMsgs) ) ; item->setData(COL_GROUP_GRP_ID, Qt::DisplayRole, QString::fromStdString(it->first.toStdString())) ; item->setData(COL_GROUP_SIZE_MSGS, Qt::DisplayRole, QString::number(stat.mTotalSizeOfMsgs)) ; item->setData(COL_GROUP_SUBSCRIBED,Qt::DisplayRole, stat.subscribed?tr("Yes"):tr("No")) ; @@ -308,6 +328,8 @@ void GxsTransportStatistics::updateContent() rsIdentity->getIdDetails(meta.mAuthorId,idDetails); QPixmap pixmap ; + QDateTime qdatetime; + qdatetime.setTime_t(meta.mPublishTs); if(idDetails.mAvatar.mSize == 0 || !GxsIdDetails::loadPixmapFromData(idDetails.mAvatar.mData, idDetails.mAvatar.mSize, pixmap,GxsIdDetails::SMALL)) pixmap = GxsIdDetails::makeDefaultIcon(meta.mAuthorId,GxsIdDetails::SMALL); @@ -315,7 +337,9 @@ void GxsTransportStatistics::updateContent() sitem->setIcon(COL_GROUP_GRP_ID, QIcon(pixmap)); sitem->setData(COL_GROUP_UNIQUE_ID, Qt::DisplayRole,QString::fromStdString(meta.mMsgId.toStdString())); - sitem->setData(COL_GROUP_NUM_MSGS,Qt::DisplayRole, QDateTime::fromTime_t(meta.mPublishTs).toString()); + sitem->setData(COL_GROUP_AUTHOR_ID, Qt::DisplayRole, QString::fromStdString(meta.mAuthorId.toStdString())) ; + sitem->setText(COL_GROUP_PUBLISHTS, QDateTime::fromTime_t(meta.mPublishTs).toString()); + sitem->setData(COL_GROUP_PUBLISHTS, Qt::UserRole, qdatetime); } } } @@ -333,6 +357,25 @@ void GxsTransportStatistics::personDetails() dialog->show(); } +void GxsTransportStatistics::showAuthorInPeople() +{ + QTreeWidgetItem *item = groupTreeWidget->currentItem(); + std::string id = item->text(COL_GROUP_AUTHOR_ID).toStdString(); + + if (id.empty()) { + return; + } + + /* window will destroy itself! */ + IdDialog *idDialog = dynamic_cast(MainWindow::getPage(MainWindow::People)); + + if (!idDialog) + return ; + + MainWindow::showWindow(MainWindow::People); + idDialog->navigate(RsGxsId(id)); +} + #ifdef TO_REMOVE void GxsTransportStatistics::loadGroupMeta(const std::vector& groupInfo) { diff --git a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.h b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.h index 01a31e776..d23fdaa0f 100644 --- a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.h +++ b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.h @@ -51,7 +51,10 @@ public: private slots: /** Create the context popup menu and it's submenus */ void CustomPopupMenu( QPoint point ); + void CustomPopupMenuGroups( QPoint point ) ; + void personDetails(); + void showAuthorInPeople(); private: void updateDisplay(bool complete) ; diff --git a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.ui b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.ui index e2c7449c0..8ce780e81 100644 --- a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.ui +++ b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.ui @@ -41,7 +41,7 @@ true - false + true @@ -108,6 +108,12 @@ Qt::CustomContextMenu + + true + + + true + Group ID / Author @@ -115,7 +121,12 @@ - Number of messages / Publish TS + Publish TS + + + + + Number of messages From 508ff090ba65fa7441ac57c9ecfff79b5a045a3a Mon Sep 17 00:00:00 2001 From: defnax Date: Sat, 6 Jun 2020 00:13:39 +0200 Subject: [PATCH 42/72] update to use FilesDefs --- retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp index 4b3ddf5e8..4f1605c46 100644 --- a/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp +++ b/retroshare-gui/src/gui/statistics/GxsTransportStatistics.cpp @@ -51,6 +51,7 @@ #include "gui/gxs/GxsIdTreeWidgetItem.h" #include "gui/Identity/IdDialog.h" #include "gui/MainWindow.h" +#include "gui/common/FilesDefs.h" #define COL_PENDING_ID 0 #define COL_PENDING_DESTINATION 1 @@ -145,7 +146,7 @@ void GxsTransportStatistics::CustomPopupMenu( QPoint ) QTreeWidgetItem *item = treeWidget->currentItem(); if (item) { - contextMnu.addAction(QIcon(":/images/info16.png"), tr("Details"), this, SLOT(personDetails())); + contextMnu.addAction(FilesDefs::getIconFromQtResourcePath(":/images/info16.png"), tr("View details"), this, SLOT(personDetails())); } @@ -158,7 +159,7 @@ void GxsTransportStatistics::CustomPopupMenuGroups( QPoint ) QTreeWidgetItem *item = groupTreeWidget->currentItem(); if (item) { - contextMnu.addAction(QIcon(":/images/info16.png"), tr("View details"), this, SLOT(showAuthorInPeople())); + contextMnu.addAction(FilesDefs::getIconFromQtResourcePath(":/images/info16.png"), tr("View details"), this, SLOT(showAuthorInPeople())); } From 4148e3c8db79097363d15f8f20abb04a3729b730 Mon Sep 17 00:00:00 2001 From: defnax Date: Sat, 6 Jun 2020 00:31:04 +0200 Subject: [PATCH 43/72] update to use FilesDefs --- retroshare-gui/src/gui/Posted/PostedCardView.cpp | 2 +- retroshare-gui/src/gui/Posted/PostedItem.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/Posted/PostedCardView.cpp b/retroshare-gui/src/gui/Posted/PostedCardView.cpp index 75a01557a..0aafaeeb5 100644 --- a/retroshare-gui/src/gui/Posted/PostedCardView.cpp +++ b/retroshare-gui/src/gui/Posted/PostedCardView.cpp @@ -192,7 +192,7 @@ void PostedCardView::fill() ui->dateLabel->setText(timestamp); } else { - QPixmap sqpixmap2 = QPixmap(":/images/thumb-default.png"); + QPixmap sqpixmap2 = FilesDefs::getPixmapFromQtResourcePath(":/images/thumb-default.png"); mInFill = true; int desired_height = 1.5*(ui->voteDownButton->height() + ui->voteUpButton->height() + ui->scoreLabel->height()); diff --git a/retroshare-gui/src/gui/Posted/PostedItem.cpp b/retroshare-gui/src/gui/Posted/PostedItem.cpp index 46de8a1e6..b4cfa2d4a 100644 --- a/retroshare-gui/src/gui/Posted/PostedItem.cpp +++ b/retroshare-gui/src/gui/Posted/PostedItem.cpp @@ -501,7 +501,7 @@ void PostedItem::fill() RetroShareLink link = RetroShareLink::createGxsGroupLink(RetroShareLink::TYPE_POSTED, mGroupMeta.mGroupId, groupName()); ui->nameLabel->setText(link.toHtml()); - QPixmap sqpixmap2 = QPixmap(":/images/thumb-default.png"); + QPixmap sqpixmap2 = FilesDefs::getPixmapFromQtResourcePath(":/images/thumb-default.png"); mInFill = true; int desired_height = 1.5*(ui->voteDownButton->height() + ui->voteUpButton->height() + ui->scoreLabel->height()); From 878111d22449d31569b222fd8c13ee3a98e5202b Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 5 Jun 2020 22:05:38 +0200 Subject: [PATCH 44/72] remove password from json tokens --- libretroshare/src/jsonapi/jsonapi.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index 338782a01..82d082520 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -180,6 +180,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), std::string errorMessage; bool makeHidden = false; bool makeAutoTor = false; + std::string createToken; // deserialize input parameters from JSON { @@ -189,6 +190,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), RS_SERIAL_PROCESS(password); RS_SERIAL_PROCESS(makeHidden); RS_SERIAL_PROCESS(makeAutoTor); + RS_SERIAL_PROCESS(createToken); } // call retroshare C++ API @@ -196,8 +198,9 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), location, password, errorMessage, makeHidden, makeAutoTor ); - if(retval) - authorizeUser(location.mLocationId.toStdString(),password); + std::string tokenUser, tokenPw; + if(retval && parseToken(createToken, tokenUser, tokenPw)) + authorizeUser(tokenUser,tokenPw); // serialize out parameters and return value to JSON { @@ -238,9 +241,6 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), RsInit::LoadCertificateStatus retval = rsLoginHelper->attemptLogin(account, password); - if( retval == RsInit::OK ) - authorizeUser(account.toStdString(), password); - // serialize out parameters and return value to JSON { RsGenericSerializer::SerializeContext& ctx(cAns); From f99a8a0fc63d8eb404100246b31b1f6ad2e2ba44 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 9 Jun 2020 16:10:07 +0200 Subject: [PATCH 45/72] Improve homogeneity of rsdebug Offer variadic style macros also for RsInfo...RsFatal --- libretroshare/src/util/rsdebug.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index 4a88d8c90..71645e6dc 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -148,7 +148,8 @@ RS_DBG4("Hello 4 ", "my debug ", my_variable) << " message " << variable2; * even if are not printed, due to how C++ is made it is not possible to avoid * this, so we suggest to use variadic style for debug messages. */ -using RsDbg = t_RsLogger; +using RsDbg = t_RsLogger; +#define RS_DBG(...) RsDbg(__PRETTY_FUNCTION__, " ", __VA_ARGS__) /** * Comfortable log information reporting helper, supports chaining like @@ -158,17 +159,21 @@ using RsDbg = t_RsLogger; RsInfo() << __PRETTY_FUNCTION__ << "My information message" << std::cerr; @endcode */ -using RsInfo = t_RsLogger; +using RsInfo = t_RsLogger; +#define RS_INFO(...) RsInfo(__PRETTY_FUNCTION__, " ", __VA_ARGS__) /// Similar to @see RsInfo but for warning messages -using RsWarn = t_RsLogger; +using RsWarn = t_RsLogger; +#define RS_WARN(...) RsWarn(__PRETTY_FUNCTION__, " ", __VA_ARGS__) /// Similar to @see RsInfo but for error messages -using RsErr = t_RsLogger; +using RsErr = t_RsLogger; +#define RS_ERR(...) RsErr(__PRETTY_FUNCTION__, " ", __VA_ARGS__) /** Similar to @see RsInfo but for fatal errors (the ones which cause RetroShare * to terminate) messages */ -using RsFatal = t_RsLogger; +using RsFatal = t_RsLogger; +#define RS_FATAL(...) RsFatal(__PRETTY_FUNCTION__, " ", __VA_ARGS__) /** From 4f0e18cbfd48ba13f5b1cb164d0c9f16931abc31 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 9 Jun 2020 20:02:40 +0200 Subject: [PATCH 46/72] Fix Android armv7 compilation due to narrowing --- libretroshare/src/util/rsdebug.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index 71645e6dc..49aadcd5a 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -80,7 +80,7 @@ struct t_RsLogger : std::ostringstream /* Combine initializer list and comma operator so the compiler unpack * template arguments and feed our own stream without recursion * see https://stackoverflow.com/a/27375675 */ - using expander = char[]; + using expander = int[]; (void) expander {0, (void((*this) << std::forward(args)), 0)...}; } From 275bdba1329e007e781232422fc222c0974f8d3f Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 9 Jun 2020 20:08:23 +0200 Subject: [PATCH 47/72] Unify common error category message into a function --- libretroshare/src/retroshare/rsevents.h | 4 ++-- libretroshare/src/retroshare/rsfiles.h | 4 ++-- libretroshare/src/util/rsdebug.cc | 6 ++++++ libretroshare/src/util/rsdebug.h | 5 ++++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/retroshare/rsevents.h b/libretroshare/src/retroshare/rsevents.h index 47a499497..65e0e8557 100644 --- a/libretroshare/src/retroshare/rsevents.h +++ b/libretroshare/src/retroshare/rsevents.h @@ -33,6 +33,7 @@ #include "serialiser/rsserializable.h" #include "serialiser/rstypeserializer.h" #include "util/rstime.h" +#include "util/rsdebug.h" class RsEvents; @@ -126,8 +127,7 @@ struct RsEventsErrorCategory: std::error_category case RsEventsErrorNum::INVALID_HANDLER_ID: return "Invalid handler id"; default: - return "Error message for error: " + std::to_string(ev) + - " not available in category: " + name(); + return rsErrorNotInCategory(ev, name()); } } diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 92c017cef..fa6b37e29 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -37,6 +37,7 @@ #include "util/rstime.h" #include "retroshare/rsevents.h" #include "util/rsmemory.h" +#include "util/rsdebug.h" class RsFiles; @@ -63,8 +64,7 @@ struct RsFilesErrorCategory: std::error_category case RsFilesErrorNum::FILES_HANDLE_NOT_FOUND: return "Files handle not found"; default: - return "Error message for error: " + std::to_string(ev) + - " not available in category: " + name(); + return rsErrorNotInCategory(ev, name()); } } diff --git a/libretroshare/src/util/rsdebug.cc b/libretroshare/src/util/rsdebug.cc index 93049e1b9..0a4777ab0 100644 --- a/libretroshare/src/util/rsdebug.cc +++ b/libretroshare/src/util/rsdebug.cc @@ -30,6 +30,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err) << " category: " << err.category().name(); } +std::string rsErrorNotInCategory(int errNum, const std::string& categoryName) +{ + return "Error message for error: " + std::to_string(errNum) + + " not available in category: " + categoryName; +} + //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index 49aadcd5a..6409e9492 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -61,6 +61,10 @@ enum class RsLoggerCategories /** Stream helper for std::error_condition */ std::ostream &operator<<(std::ostream& out, const std::error_condition& err); +/** Provide unkown error message for all error categories to avoid duplicating + * the message around */ +std::string rsErrorNotInCategory(int errNum, const std::string& categoryName); + template struct t_RsLogger : std::ostringstream @@ -201,7 +205,6 @@ struct RsNoDbg - //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// From 4a34cfaf11975d2fdb97c83c964629ba601e03bc Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 9 Jun 2020 20:13:22 +0200 Subject: [PATCH 48/72] Improve API to create locations Improve on top of sehraf work, add documentation for API token params Deprecate old version and disable it once RS 0.6.6 is released New method take relevant paramethers explicitely instead of having them incapsualted inside a RsLoginHelper::Location struct New method return a proper error_condition instead of boolean + string RsLoginHelper::attemptLogin doesn't need manual wrapper anymore --- libretroshare/src/jsonapi/jsonapi.cpp | 40 +++++-- libretroshare/src/retroshare/rsinit.h | 128 ++++++++++++++++++++--- libretroshare/src/retroshare/rsjsonapi.h | 4 +- libretroshare/src/rsserver/rsinit.cc | 44 ++++++++ 4 files changed, 194 insertions(+), 22 deletions(-) diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index c219896d4..3dcde2a27 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -165,6 +165,7 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), RsThread::setStopTimeout(10); #endif +#if !RS_VERSION_AT_LEAST(0,6,6) registerHandler("/rsLoginHelper/createLocation", [this](const std::shared_ptr session) { @@ -215,8 +216,9 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), DEFAULT_API_CALL_JSON_RETURN(rb::OK); } ); }, false); +#endif // !RS_VERSION_AT_LEAST(0,6,6) - registerHandler("/rsLoginHelper/attemptLogin", + registerHandler("/rsLoginHelper/createLocationV2", [this](const std::shared_ptr session) { auto reqSize = session->get_request()->get_header("Content-Length", 0); @@ -226,25 +228,51 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"), { INITIALIZE_API_CALL_JSON_CONTEXT; - RsPeerId account; + RsPeerId locationId; + RsPgpId pgpId; + std::string locationName; + std::string pgpName; std::string password; + // JSON API only + std::string apiUser; + std::string apiPass; + // deserialize input parameters from JSON { RsGenericSerializer::SerializeContext& ctx(cReq); RsGenericSerializer::SerializeJob j(RsGenericSerializer::FROM_JSON); - RS_SERIAL_PROCESS(account); + RS_SERIAL_PROCESS(locationId); + RS_SERIAL_PROCESS(pgpId); + RS_SERIAL_PROCESS(locationName); + RS_SERIAL_PROCESS(pgpName); RS_SERIAL_PROCESS(password); + + // JSON API only + RS_SERIAL_PROCESS(apiUser); + RS_SERIAL_PROCESS(apiPass); } - // call retroshare C++ API - RsInit::LoadCertificateStatus retval = - rsLoginHelper->attemptLogin(account, password); + std::error_condition retval; + + if(apiUser.empty()) + retval = RsJsonApiErrorNum::TOKEN_FORMAT_INVALID; + + if(!retval) + retval = badApiCredientalsFormat(apiUser, apiPass); + + if(!retval) // call retroshare C++ API + retval = rsLoginHelper->createLocationV2( + locationId, pgpId, locationName, pgpName, password ); + + if(!retval) retval = authorizeUser(apiUser, apiPass); // serialize out parameters and return value to JSON { RsGenericSerializer::SerializeContext& ctx(cAns); RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON); + RS_SERIAL_PROCESS(locationId); + RS_SERIAL_PROCESS(pgpId); RS_SERIAL_PROCESS(retval); } diff --git a/libretroshare/src/retroshare/rsinit.h b/libretroshare/src/retroshare/rsinit.h index 165649f16..19c0349bb 100644 --- a/libretroshare/src/retroshare/rsinit.h +++ b/libretroshare/src/retroshare/rsinit.h @@ -20,8 +20,7 @@ *******************************************************************************/ #pragma once -/// RetroShare initialization and login API - +/// @file RetroShare initialization and login API // Initialize ok, result >= 0 #define RS_INIT_OK 0 // Initialize ok @@ -32,11 +31,15 @@ #define RS_INIT_NO_KEYRING -3 // Keyring is empty. Need to import it. #define RS_INIT_NO_EXECUTABLE -4 // executable path hasn't been set in config options -#include #include #include #include -#include +#include +#include + +#include "retroshare/rstypes.h" +#include "retroshare/rsversion.h" + class RsLoginHelper; @@ -46,6 +49,71 @@ class RsLoginHelper; */ extern RsLoginHelper* rsLoginHelper; + +enum class RsInitErrorNum : int32_t +{ + ALREADY_LOGGED_IN = 6000, + CANT_ACQUIRE_LOCK = 6001, + INVALID_LOCATION_NAME = 6002, + PGP_NAME_OR_ID_NEEDED = 6003, + PGP_KEY_CREATION_FAILED = 6004, + SSL_KEY_CREATION_FAILED = 6005, + INVALID_SSL_ID = 6006, + LOGIN_FAILED = 6007 +}; + +struct RsInitErrorCategory: std::error_category +{ + const char* name() const noexcept override + { return "RetroShare init"; } + + std::string message(int ev) const override + { + switch (static_cast(ev)) + { + case RsInitErrorNum::ALREADY_LOGGED_IN: + return "Already logged in"; + case RsInitErrorNum::CANT_ACQUIRE_LOCK: + return "Cannot aquire lock on location data. Another instance is " + "already running with this profile?"; + case RsInitErrorNum::INVALID_LOCATION_NAME: + return "Invalid location name"; + case RsInitErrorNum::PGP_NAME_OR_ID_NEEDED: + return "Either PGP name or PGP id is needed"; + case RsInitErrorNum::PGP_KEY_CREATION_FAILED: + return "Failure creating PGP key"; + case RsInitErrorNum::SSL_KEY_CREATION_FAILED: + return "Failure creating SSL key"; + case RsInitErrorNum::INVALID_SSL_ID: + return "Invalid SSL id"; + case RsInitErrorNum::LOGIN_FAILED: + return "Generic login failure"; + default: + return rsErrorNotInCategory(ev, name()); + } + } + + const static RsInitErrorCategory instance; +}; + + +namespace std +{ +/** Register RsJsonApiErrorNum as an error condition enum, must be in std + * namespace */ +template<> struct is_error_condition_enum : true_type {}; +} + +/** Provide RsInitErrorNum conversion to std::error_condition, must be in + * same namespace of RsInitErrorNum */ +inline std::error_condition make_error_condition(RsInitErrorNum e) noexcept +{ + return std::error_condition( + static_cast(e), RsInitErrorCategory::instance ); +}; + + + /** * @brief The RsInitConfig struct * This class contains common configuration options, that executables using libretroshare may want to @@ -85,7 +153,7 @@ struct RsConfigOptions class RsInit { public: - enum LoadCertificateStatus : uint8_t + enum RS_DEPRECATED_FOR(RsInitErrorNum) LoadCertificateStatus : uint8_t { OK, /// Everything go as expected, no error occurred ERR_ALREADY_RUNNING, /// Another istance is running already @@ -317,7 +385,7 @@ public: /** * @brief Normal way to attempt login - * @jsonapi{development,manualwrapper} + * @jsonapi{development,unauthenticated} * @param[in] account Id of the account to which attempt login * @param[in] password Password for the given account * @return RsInit::OK if login attempt success, error code otherwhise @@ -353,6 +421,44 @@ public: void getLocations(std::vector& locations); /** + * @brief Creates a new RetroShare location, and log in once is created + * @jsonapi{development,manualwrapper} + * @param[out] locationId storage for generated location SSL id + * @param[inout] pgpId specify PGP id to use to sign the location, if a null + * id is passed the PGP key is created too and this param is used as + * storage for its id. + * @param[in] password to protect and unlock the associated PGP key + * param[in] apiUser (JSON API only) string containing username for JSON API + * so it can be later used to authenticate JSON API calls. It is passed + * down to @see RsJsonApi::authorizeUser under the hood. + * param[in] apiPass (JSON API only) string containing password for JSON API + * so it can be later used to authenticate JSON API calls. It is passed + * down to @see RsJsonApi::authorizeUser under the hood. + * To improve security we strongly advise to not use the same as the + * password used for the PGP key. + * @return Success or error information + */ + std::error_condition createLocationV2( + RsPeerId& locationId, + RsPgpId& pgpId, + const std::string& locationName, + const std::string& pgpName, + const std::string& password + /* JSON API only + * const std::string& apiUser + * const std::string& apiPass */ ); + + /** + * @brief Check if RetroShare is already logged in, this usually return true + * after a successfull attemptLogin() and before closeSession() + * @jsonapi{development,unauthenticated} + * @return true if already logged in, false otherwise + */ + bool isLoggedIn(); + +#if !RS_VERSION_AT_LEAST(0,6,6) + /** + * @deprecated Use @see createLocationV2 instead * @brief Creates a new RetroShare location, and log in once is created * @jsonapi{development,manualwrapper} * @param[inout] location provide input information to generate the location @@ -365,15 +471,9 @@ public: * Tor hidden location. UNTESTED! * @return true if success, false otherwise */ + RS_DEPRECATED_FOR(createLocationV2) bool createLocation( RsLoginHelper::Location& location, const std::string& password, std::string& errorMessage, bool makeHidden = false, bool makeAutoTor = false ); - - /** - * @brief Check if RetroShare is already logged in, this usually return true - * after a successfull attemptLogin() and before closeSession() - * @jsonapi{development,unauthenticated} - * @return true if already logged in, false otherwise - */ - bool isLoggedIn(); +#endif // !RS_VERSION_AT_LEAST(0,6,6) }; diff --git a/libretroshare/src/retroshare/rsjsonapi.h b/libretroshare/src/retroshare/rsjsonapi.h index 147011787..38e33490e 100644 --- a/libretroshare/src/retroshare/rsjsonapi.h +++ b/libretroshare/src/retroshare/rsjsonapi.h @@ -29,6 +29,7 @@ #include #include +#include "util/rsdebug.h" #include "util/rsmemory.h" class RsJsonApi; @@ -74,8 +75,7 @@ struct RsJsonApiErrorCategory: std::error_category case RsJsonApiErrorNum::NOT_A_MACHINE_GUN: return "Method must not be called in burst"; default: - return "Error message for error: " + std::to_string(ev) + - " not available in category: " + name(); + return rsErrorNotInCategory(ev, name()); } } diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 25391fcbe..5dcccfe0c 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -114,6 +114,8 @@ RsLoginHelper* rsLoginHelper = nullptr; RsAccounts* rsAccounts = nullptr; +const RsInitErrorCategory RsInitErrorCategory::instance; + RsConfigOptions::RsConfigOptions() : autoLogin(false), @@ -1950,6 +1952,47 @@ void RsLoginHelper::getLocations(std::vector& store) } } +std::error_condition RsLoginHelper::createLocationV2( + RsPeerId& locationId, RsPgpId& pgpId, + const std::string& locationName, const std::string& pgpName, + const std::string& password ) +{ + if(isLoggedIn()) return RsInitErrorNum::ALREADY_LOGGED_IN; + if(locationName.empty()) return RsInitErrorNum::INVALID_LOCATION_NAME; + if(pgpId.isNull() && pgpName.empty()) + return RsInitErrorNum::PGP_NAME_OR_ID_NEEDED; + + std::string errorMessage; + if(pgpId.isNull() && !RsAccounts::GeneratePGPCertificate( + pgpName, "", password, pgpId, 4096, errorMessage ) ) + { + RS_ERR("Failure creating PGP key: ", errorMessage); + return RsInitErrorNum::PGP_KEY_CREATION_FAILED; + } + + std::string sslPassword = + RsRandom::random_alphaNumericString(RsInit::getSslPwdLen()); + + rsNotify->cachePgpPassphrase(password); + rsNotify->setDisableAskPassword(true); + + bool ret = RsAccounts::createNewAccount( + pgpId, "", locationName, "", false, false, sslPassword, + locationId, errorMessage ); + if(!ret) + { + RS_ERR("Failure creating SSL key: ", errorMessage); + return RsInitErrorNum::SSL_KEY_CREATION_FAILED; + } + + RsInit::LoadPassword(sslPassword); + ret = (RsInit::OK == attemptLogin(locationId, password)); + rsNotify->setDisableAskPassword(false); + + return (ret ? std::error_condition() : RsInitErrorNum::LOGIN_FAILED); +} + +#if !RS_VERSION_AT_LEAST(0,6,6) bool RsLoginHelper::createLocation( RsLoginHelper::Location& l, const std::string& password, std::string& errorMessage, bool makeHidden, bool makeAutoTor ) @@ -1991,6 +2034,7 @@ bool RsLoginHelper::createLocation( rsNotify->setDisableAskPassword(false); return ret; } +#endif // !RS_VERSION_AT_LEAST(0,6,6) bool RsLoginHelper::isLoggedIn() { From 462f52585b0c59fb509cd285f3b96e8ac6a4bea3 Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 8 May 2020 16:10:17 +0200 Subject: [PATCH 49/72] add helper for i2p related functions --- libretroshare/src/libretroshare.pro | 4 +- libretroshare/src/util/i2pcommon.cpp | 172 +++++++++++++++++++++ libretroshare/src/util/i2pcommon.h | 222 +++++++++++++++++++++++++++ 3 files changed, 397 insertions(+), 1 deletion(-) create mode 100644 libretroshare/src/util/i2pcommon.cpp create mode 100644 libretroshare/src/util/i2pcommon.h diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 76a39b662..837c1f715 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -154,6 +154,7 @@ rs_webui { HEADERS += plugins/pluginmanager.h \ plugins/dlfcn_win32.h \ rsitems/rspluginitems.h \ + util/i2pcommon.h \ util/rsinitedptr.h HEADERS += $$PUBLIC_HEADERS @@ -517,7 +518,8 @@ SOURCES += ft/ftchunkmap.cc \ ft/ftfilesearch.cc \ ft/ftserver.cc \ ft/fttransfermodule.cc \ - ft/ftturtlefiletransferitem.cc + ft/ftturtlefiletransferitem.cc \ + util/i2pcommon.cpp SOURCES += crypto/chacha20.cpp \ crypto/hashstream.cc\ diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp new file mode 100644 index 000000000..e50fc9d83 --- /dev/null +++ b/libretroshare/src/util/i2pcommon.cpp @@ -0,0 +1,172 @@ +#include "i2pcommon.h" + +#include "util/rsbase64.h" +#include "util/rsdebug.h" + +namespace i2p { + +const std::string generateNameSuffix(const size_t len) { + std::vector tmp(len); + RsRandom::random_bytes(tmp.data(), len); + const std::string location = Radix32::encode(tmp.data(), len); + + return location; +} + +std::string keyToBase32Addr(const std::string &key) +{ + std::string copy(key); + + // replace I2P specific chars + std::replace(copy.begin(), copy.end(), '~', '/'); + std::replace(copy.begin(), copy.end(), '-', '+'); + + // decode + std::vector bin; + RsBase64::decode(copy, bin); + + // hash + std::vector sha256 = RsUtil::BinToSha256(bin); + // encode + std::string out = Radix32::encode(sha256); + + // i2p uses lowercase + std::transform(out.begin(), out.end(), out.begin(), ::tolower); + out.append(".b32.i2p"); + + return out; +} + +const std::string makeOption(const std::string &lhs, const int8_t &rhs) { + // convert number to int + std::ostringstream oss; + oss << (int)rhs; + return lhs + "=" + oss.str(); +} + +uint16_t readTwoBytesBE(std::vector::const_iterator &p) +{ + uint16_t val = 0; + val += *p++; + val <<= 8; + val += *p++; + return val; +} + +std::string publicKeyFromPrivate(std::string const &priv) +{ + /* + * https://geti2p.net/spec/common-structures#destination + * https://geti2p.net/spec/common-structures#keysandcert + * https://geti2p.net/spec/common-structures#certificate + */ + if (priv.length() < 884) // base64 ( = 663 bytes = KeyCert + priv Keys) + return std::string(); + + // creat a copy to work on, need to convert it to standard base64 + auto priv_copy(priv); + std::replace(priv_copy.begin(), priv_copy.end(), '~', '/'); + std::replace(priv_copy.begin(), priv_copy.end(), '-', '+'); + + // get raw data + std::vector dataPriv; + RsBase64::decode(priv_copy, dataPriv); + + auto p = dataPriv.cbegin(); + RsDbg() << __PRETTY_FUNCTION__ << " dataPriv.size " << dataPriv.size() << std::endl; + + size_t publicKeyLen = 256 + 128; // default length (bytes) + uint8_t certType = 0; + uint16_t len = 0; + uint16_t signingKeyType = 0; + uint16_t cryptKey = 0; + + // only used for easy break + do { + try { + // jump to certificate + p += publicKeyLen; + // try to read type and length + certType = *p++; + len = readTwoBytesBE(p); + + // only 0 and 5 are used / valid at this point + // check for == 0 + if (certType == static_cast::type>(CertType::Null)) { + /* + * CertType.Null + * type null is followed by 0x00 0x00 + * so has to be 0! + */ + RsDbg() << __PRETTY_FUNCTION__ << " cert is CertType.Null" << std::endl; + publicKeyLen += 3; // add 0x00 0x00 0x00 + + if (len != 0) + // weird + RsDbg() << __PRETTY_FUNCTION__ << " cert is CertType.Null but len != 0" << std::endl; + + break; + } + + // check for != 5 + if (certType != static_cast::type>(CertType::Key)) { + // unsupported + RsDbg() << __PRETTY_FUNCTION__ << " cert type " << certType << " is unsupported" << std::endl; + return std::string(); + } + + RsDbg() << __PRETTY_FUNCTION__ << " cert is CertType.Key" << std::endl; + publicKeyLen += 7; // = 1 + 2 + 2 + 2 = 7 bytes + + /* + * "Key certificates were introduced in release 0.9.12. Prior to that release, all PublicKeys were 256-byte ElGamal keys, and all SigningPublicKeys were 128-byte DSA-SHA1 keys." + * --> there is space for 256+128 bytes, longer keys are splitted and appended to the certificate + * We don't need to bother with the splitting here as only the lenght is important! + */ + + // Signing Public Key + // likely 7 + signingKeyType = readTwoBytesBE(p); + + RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << std::endl; + if (signingKeyType >= 3 && signingKeyType <= 6) { + RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " has oversize" << std::endl; + // calculate oversize + + auto it = signingKeyLengths.find(static_cast(signingKeyType)); + if (it == signingKeyLengths.end()) { + // just in case + RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " cannot be found in size map!" << std::endl; + return std::string(); + } + + if (it->second.first <= 128) { + // just in case, it's supposed to be larger! + RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " is oversize but size calculation would underflow!" << std::endl; + return std::string(); + } + + publicKeyLen += it->second.first - 128; // 128 = default DSA key length = the space than can be used before the key must be splitted + } + + // Crypto Public Key + // likely 0 + cryptKey = readTwoBytesBE(p); + RsDbg() << __PRETTY_FUNCTION__ << " crypto pubkey type " << cryptKey << std::endl; + // info: these are all smaller than the default 256 bytes, so no oversize calculation is needed + + break; + } catch (const std::out_of_range &e) { + RsDbg() << __PRETTY_FUNCTION__ << " hit exception!" << e.what() << std::endl; + return std::string(); + } + } while(false); + + std::string pub; + auto data2 = std::vector(dataPriv.cbegin(), dataPriv.cbegin() + publicKeyLen); + RsBase64::encode(data2.data(), data2.size(), pub, false, false); + + return pub; +} + +} // namespace i2p diff --git a/libretroshare/src/util/i2pcommon.h b/libretroshare/src/util/i2pcommon.h new file mode 100644 index 000000000..98815496b --- /dev/null +++ b/libretroshare/src/util/i2pcommon.h @@ -0,0 +1,222 @@ +#ifndef I2PCOMMON_H +#define I2PCOMMON_H + +#include +#include + +#include "util/rsrandom.h" +#include "util/radix32.h" +#include "util/rsbase64.h" +#include "util/rsprint.h" +#include "util/rsdebug.h" + +/* + * This header provides common code for i2p related code, namely BOB and SAM3 support. + */ + +namespace i2p { + +static constexpr int8_t kDefaultLength = 3; // i2p default +static constexpr int8_t kDefaultQuantity = 3; // i2p default + 1 +static constexpr int8_t kDefaultVariance = 0; +static constexpr int8_t kDefaultBackupQuantity = 0; + +/** + * @brief The address struct + * This structure is a container for any i2p address/key. The public key is used for addressing and can be (optionally) hashed to generate the .b32.i2p address. + */ +struct address { + std::string base32; + std::string publicKey; + std::string privateKey; + + void clear() { + base32.clear(); + publicKey.clear(); + privateKey.clear(); + } +}; + +/** + * @brief The settings struct + * Common structure with all settings that are shared between any i2p backends + */ +struct settings { + bool enable; + struct address address; + + // connection parameter + int8_t inLength; + int8_t inQuantity; + int8_t inVariance; + int8_t inBackupQuantity; + + int8_t outLength; + int8_t outQuantity; + int8_t outVariance; + int8_t outBackupQuantity; + + void initDefault() { + enable = false; + address.clear(); + + inLength = kDefaultLength; + inQuantity = kDefaultQuantity; + inVariance = kDefaultVariance; + inBackupQuantity = kDefaultBackupQuantity; + + outLength = kDefaultLength; + outQuantity = kDefaultQuantity; + outVariance = kDefaultVariance; + outBackupQuantity = kDefaultBackupQuantity; + } +}; + +/* + Type Type Code Payload Length Total Length Notes + Null 0 0 3 + HashCash 1 varies varies Experimental, unused. Payload contains an ASCII colon-separated hashcash string. + Hidden 2 0 3 Experimental, unused. Hidden routers generally do not announce that they are hidden. + Signed 3 40 or 72 43 or 75 Experimental, unused. Payload contains a 40-byte DSA signature, optionally followed by the 32-byte Hash of the signing Destination. + Multiple 4 varies varies Experimental, unused. Payload contains multiple certificates. + Key 5 4+ 7+ Since 0.9.12. See below for details. +*/ +enum class CertType : uint8_t { + Null = 0, + HashCash = 1, + Hidden = 2, + Signed = 3, + Multiple = 4, + Key = 5 +}; + +/* + * public + Type Type Code Total Public Key Length Since Usage + DSA_SHA1 0 128 0.9.12 Legacy Router Identities and Destinations, never explicitly set + ECDSA_SHA256_P256 1 64 0.9.12 Older Destinations + ECDSA_SHA384_P384 2 96 0.9.12 Rarely if ever used for Destinations + ECDSA_SHA512_P521 3 132 0.9.12 Rarely if ever used for Destinations + RSA_SHA256_2048 4 256 0.9.12 Offline only; never used in Key Certificates for Router Identities or Destinations + RSA_SHA384_3072 5 384 0.9.12 Offline only; never used in Key Certificates for Router Identities or Destinations + RSA_SHA512_4096 6 512 0.9.12 Offline only; never used in Key Certificates for Router Identities or Destinations + EdDSA_SHA512_Ed25519 7 32 0.9.15 Recent Router Identities and Destinations + EdDSA_SHA512_Ed25519ph 8 32 0.9.25 Offline only; never used in Key Certificates for Router Identities or Destinations + reserved (GOST) 9 64 Reserved, see proposal 134 + reserved (GOST) 10 128 Reserved, see proposal 134 + RedDSA_SHA512_Ed25519 11 32 0.9.39 For Destinations and encrypted leasesets only; never used for Router Identities + reserved 65280-65534 Reserved for experimental use + reserved 65535 Reserved for future expansion + + * private + Type Length (bytes) Since Usage + DSA_SHA1 20 Legacy Router Identities and Destinations + ECDSA_SHA256_P256 32 0.9.12 Recent Destinations + ECDSA_SHA384_P384 48 0.9.12 Rarely used for Destinations + ECDSA_SHA512_P521 66 0.9.12 Rarely used for Destinations + RSA_SHA256_2048 512 0.9.12 Offline signing, never used for Router Identities or Destinations + RSA_SHA384_3072 768 0.9.12 Offline signing, never used for Router Identities or Destinations + RSA_SHA512_4096 1024 0.9.12 Offline signing, never used for Router Identities or Destinations + EdDSA_SHA512_Ed25519 32 0.9.15 Recent Router Identities and Destinations + EdDSA_SHA512_Ed25519ph 32 0.9.25 Offline signing, never used for Router Identities or Destinations + RedDSA_SHA512_Ed25519 32 0.9.39 For Destinations and encrypted leasesets only, never used for Router Identities + */ +enum class SigningKeyType : uint16_t { + DSA_SHA1 = 0, + ECDSA_SHA256_P256 = 1, + ECDSA_SHA384_P384 = 2, + ECDSA_SHA512_P521 = 3, + RSA_SHA256_2048 = 4, + RSA_SHA384_3072 = 5, + RSA_SHA512_4096 = 6, + EdDSA_SHA512_Ed25519 = 7, + EdDSA_SHA512_Ed25519ph = 8, + RedDSA_SHA512_Ed25519 = 11 +}; + +/* + * public + Type Type Code Total Public Key Length Usage + ElGamal 0 256 All Router Identities and Destinations + P256 1 64 Reserved, see proposal 145 + P384 2 96 Reserved, see proposal 145 + P521 3 132 Reserved, see proposal 145 + X25519 4 32 Not for use in key certs. See proposal 144 + reserved 65280-65534 Reserved for experimental use + reserved 65535 Reserved for future expansion + + * private + Type Length (bytes) Since Usage + ElGamal 256 All Router Identities and Destinations + P256 32 TBD Reserved, see proposal 145 + P384 48 TBD Reserved, see proposal 145 + P521 66 TBD Reserved, see proposal 145 + X25519 32 0.9.38 Little-endian. See proposal 144 +*/ +enum class CryptoKeyType : uint16_t { + ElGamal = 0, + P256 = 1, + P384 = 2, + P521 = 3, + X25519 = 4 +}; + +static const std::map> cryptoKeyLengths { + {CryptoKeyType::ElGamal, {256, 256}}, + {CryptoKeyType::P256, { 64, 32}}, + {CryptoKeyType::P384, { 96, 48}}, + {CryptoKeyType::P521, {132, 66}}, + {CryptoKeyType::X25519, { 32, 32}}, +}; + +static const std::map> signingKeyLengths { + {SigningKeyType::DSA_SHA1, {128, 128}}, + {SigningKeyType::ECDSA_SHA256_P256, { 64, 32}}, + {SigningKeyType::ECDSA_SHA384_P384, { 96, 48}}, + {SigningKeyType::ECDSA_SHA512_P521, {132, 66}}, + {SigningKeyType::RSA_SHA256_2048, {256, 512}}, + {SigningKeyType::RSA_SHA384_3072, {384, 768}}, + {SigningKeyType::RSA_SHA512_4096, {512,1024}}, + {SigningKeyType::EdDSA_SHA512_Ed25519, { 32, 32}}, + {SigningKeyType::EdDSA_SHA512_Ed25519ph,{ 32, 32}}, + {SigningKeyType::RedDSA_SHA512_Ed25519, { 32, 32}}, +}; + + +/** + * @brief generateNameSuffix Generates a base32 name suffix for tunnel identification + * @param len lenght of random bytes, will be expanded by base32 encoding + * @return base32 string + * + * RSRandom::random_alphaNumericString can return very weird looking strings like: ,,@z+M + * -> so use base32 instead + * + * 5 characters = 8 base32 symbols + */ +const std::string generateNameSuffix(const size_t len = 5); + +/** + * @brief makeOption Creates the string "lhs=rhs" used by BOB and SAM. Converts rhs + * @param lhs option to set + * @param rhs value to set + * @return concatenated string + */ +const std::string makeOption(const std::string &lhs, const int8_t &rhs); + +/** + * @brief keyToBase32Addr generated a base32 address (.b32.i2p) from a given public key + * @param key public key + * @return generated base32 address + */ +std::string keyToBase32Addr(const std::string &key); + +/** + * @brief publicKeyFromPrivate parses the private key and calculates the lenght of the public key + * @param priv private key (which includes the public key) to read + * @return public key used for addressing + */ +std::string publicKeyFromPrivate(const std::string &priv); + +} // namespace i2p + +#endif // I2PCOMMON_H From 4b6e12ca454d8673c3f0bdac3f7ff2f890305849 Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 8 May 2020 16:13:27 +0200 Subject: [PATCH 50/72] i2p: bob: use common i2p::keyToBase32Addr() --- .../src/services/autoproxy/p3i2pbob.cc | 24 +------------------ .../src/services/autoproxy/p3i2pbob.h | 9 ++++--- 2 files changed, 5 insertions(+), 28 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index 693570ac2..256a1cf4d 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -250,28 +250,6 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) } } -std::string p3I2pBob::keyToBase32Addr(const std::string &key) -{ - std::string copy(key); - - // replace I2P specific chars - std::replace(copy.begin(), copy.end(), '~', '/'); - std::replace(copy.begin(), copy.end(), '-', '+'); - - // decode - std::vector bin = Radix64::decode(copy); - // hash - std::vector sha256 = RsUtil::BinToSha256(bin); - // encode - std::string out = Radix32::encode(sha256); - - // i2p uses lowercase - std::transform(out.begin(), out.end(), out.begin(), ::tolower); - out.append(".b32.i2p"); - - return out; -} - bool inline isAnswerOk(const std::string &answer) { return (answer.compare(0, 2, "OK") == 0); } @@ -346,7 +324,7 @@ int p3I2pBob::stateMachineBOB() switch (mBOBState) { case bsNewkeysN: key = answer.substr(3, answer.length()-3); - mSetting.addr = keyToBase32Addr(key); + mSetting.addr = i2p::keyToBase32Addr(key); IndicateConfigChanged(); break; case bsGetkeys: diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.h b/libretroshare/src/services/autoproxy/p3i2pbob.h index e6ef60bae..428501a4d 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.h +++ b/libretroshare/src/services/autoproxy/p3i2pbob.h @@ -30,9 +30,10 @@ #include #endif +#include "pqi/p3cfgmgr.h" #include "services/autoproxy/rsautoproxymonitor.h" #include "util/rsthreads.h" -#include "pqi/p3cfgmgr.h" +#include "util/i2pcommon.h" /* * This class implements I2P BOB (BASIC OPEN BRIDGE) communication to allow RS @@ -49,7 +50,7 @@ * * Note 3: * BOB needs a unique name as an ID for each tunnel. - * We use 'RetroShare-' + 8 base32 characters. + * We use 'RetroShare-' + 8 random base32 characters. * * Design: * The service uses three state machines to manage its task: @@ -72,7 +73,7 @@ * mCommands[bobState::quit] = {quit, bobState::cleared}; * * stateMachineController: - * This state machone manages the high level tasks. + * This state machine manages the high level tasks. * It is controlled by mState and mTask. * * mTast: @@ -203,8 +204,6 @@ public: void processTaskAsync(taskTicket *ticket); void processTaskSync(taskTicket *ticket); - static std::string keyToBase32Addr(const std::string &key); - void threadTick() override; /// @see RsTickingThread private: From 6136416b5627a6858eace83d78b406c636059e9f Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 8 May 2020 16:33:47 +0200 Subject: [PATCH 51/72] i2p: bob: convert bobSettings to i2p::settings --- libretroshare/src/rsserver/rsinit.cc | 2 +- .../src/services/autoproxy/p3i2pbob.cc | 40 ++++++++----------- .../src/services/autoproxy/p3i2pbob.h | 14 +------ .../src/gui/settings/ServerPage.cpp | 32 ++++++++------- 4 files changed, 35 insertions(+), 53 deletions(-) diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 5dcccfe0c..1c4e2a062 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1724,7 +1724,7 @@ int RsServer::StartupRetroShare() // now enable bob bobSettings bs; autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::getSettings, &bs); - bs.enableBob = true; + bs.enable = true; autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::setSettings, &bs); } else { std::cerr << "RsServer::StartupRetroShare failed to receive keys" << std::endl; diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index 256a1cf4d..6a52120dc 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -74,15 +74,7 @@ p3I2pBob::p3I2pBob(p3PeerMgr *peerMgr) mProcessing(NULL), mLock("I2P-BOB") { // set defaults - mSetting.enableBob = kDefaultBOBEnable; - mSetting.keys = ""; - mSetting.addr = ""; - mSetting.inLength = kDefaultLength; - mSetting.inQuantity = kDefaultQuantity; - mSetting.inVariance = kDefaultVariance; - mSetting.outLength = kDefaultLength; - mSetting.outQuantity = kDefaultQuantity; - mSetting.outVariance = kDefaultVariance; + mSetting.initDefault(); mCommands.clear(); } @@ -90,7 +82,7 @@ p3I2pBob::p3I2pBob(p3PeerMgr *peerMgr) bool p3I2pBob::isEnabled() { RS_STACK_MUTEX(mLock); - return mSetting.enableBob; + return mSetting.enable; } bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) @@ -151,7 +143,7 @@ bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) { RS_STACK_MUTEX(mLock); - addr = mSetting.addr; + addr = mSetting.address.base32; } std::cout << "p3I2pBob::initialSetup addr '" << addr << "'" << std::endl; @@ -324,12 +316,12 @@ int p3I2pBob::stateMachineBOB() switch (mBOBState) { case bsNewkeysN: key = answer.substr(3, answer.length()-3); - mSetting.addr = i2p::keyToBase32Addr(key); + mSetting.address.base32 = i2p::keyToBase32Addr(key); IndicateConfigChanged(); break; case bsGetkeys: key = answer.substr(3, answer.length()-3); - mSetting.keys = key; + mSetting.address.privateKey = key; IndicateConfigChanged(); break; default: @@ -465,7 +457,7 @@ int p3I2pBob::stateMachineController_locked_idle() mProcessing = mPending.front(); mPending.pop(); - if (!mSetting.enableBob && ( + if (!mSetting.enable && ( mProcessing->task == autoProxyTask::start || mProcessing->task == autoProxyTask::stop || mProcessing->task == autoProxyTask::proxyStatusCheck)) { @@ -539,7 +531,7 @@ int p3I2pBob::stateMachineController_locked_connected() switch (mTask) { case ctRunSetUp: // when we have a key use it for server tunnel! - if(mSetting.keys.empty()) { + if(mSetting.address.privateKey.empty()) { rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_connected: setting mBOBState = setnickC"); mBOBState = bsSetnickC; } else { @@ -750,9 +742,9 @@ bool p3I2pBob::saveList(bool &cleanup, std::list &lst) RsTlvKeyValue kv; RS_STACK_MUTEX(mLock); - addKVS(vitem, kv, kConfigKeyBOBEnable, mSetting.enableBob ? "TRUE" : "FALSE") - addKVS(vitem, kv, kConfigKeyBOBKey, mSetting.keys) - addKVS(vitem, kv, kConfigKeyBOBAddr, mSetting.addr) + addKVS(vitem, kv, kConfigKeyBOBEnable, mSetting.enable ? "TRUE" : "FALSE") + addKVS(vitem, kv, kConfigKeyBOBKey, mSetting.address.privateKey) + addKVS(vitem, kv, kConfigKeyBOBAddr, mSetting.address.base32) addKVSInt(vitem, kv, kConfigKeyInLength, mSetting.inLength) addKVSInt(vitem, kv, kConfigKeyInQuantity, mSetting.inQuantity) addKVSInt(vitem, kv, kConfigKeyInVariance, mSetting.inVariance) @@ -786,11 +778,11 @@ bool p3I2pBob::loadList(std::list &load) RS_STACK_MUTEX(mLock); for(std::list::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) { if (kit->key == kConfigKeyBOBEnable) - mSetting.enableBob = kit->value == "TRUE"; + mSetting.enable = kit->value == "TRUE"; else if (kit->key == kConfigKeyBOBKey) - mSetting.keys = kit->value; + mSetting.address.privateKey = kit->value; else if (kit->key == kConfigKeyBOBAddr) - mSetting.addr = kit->value; + mSetting.address.base32 = kit->value; getKVSUInt(kit, kConfigKeyInLength, mSetting.inLength) getKVSUInt(kit, kConfigKeyInQuantity, mSetting.inQuantity) getKVSUInt(kit, kConfigKeyInVariance, mSetting.inVariance) @@ -958,7 +950,7 @@ void p3I2pBob::finalizeSettings_locked() sockaddr_storage_setport(mI2PProxyAddr, 2827); rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "finalizeSettings_locked using " + sockaddr_storage_tostring(mI2PProxyAddr)); - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "finalizeSettings_locked using " + mSetting.addr); + rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "finalizeSettings_locked using " + mSetting.address.base32); peerState ps; mPeerMgr->getOwnNetStatus(ps); @@ -980,7 +972,7 @@ void p3I2pBob::finalizeSettings_locked() const std::string getnick = "getnick RetroShare-" + location; const std::string newkeys = "newkeys"; const std::string getkeys = "getkeys"; - const std::string setkeys = "setkeys " + mSetting.keys; + const std::string setkeys = "setkeys " + mSetting.address.privateKey; const std::string inhost = "inhost " + sockaddr_storage_iptostring(proxy); const std::string inport = toString("inport ", sockaddr_storage_port(proxy)); const std::string outhost = "outhost " + sockaddr_storage_iptostring(ps.localaddr); @@ -1049,7 +1041,7 @@ void p3I2pBob::updateSettings_locked() peerState ps; mPeerMgr->getOwnNetStatus(ps); - const std::string setkeys = "setkeys " + mSetting.keys; + const std::string setkeys = "setkeys " + mSetting.address.privateKey; const std::string inhost = "inhost " + sockaddr_storage_iptostring(proxy); const std::string inport = toString("inport ", sockaddr_storage_port(proxy)); const std::string outhost = "outhost " + sockaddr_storage_iptostring(ps.localaddr); diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.h b/libretroshare/src/services/autoproxy/p3i2pbob.h index 428501a4d..29bcbcb61 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.h +++ b/libretroshare/src/services/autoproxy/p3i2pbob.h @@ -163,19 +163,7 @@ struct bobStateInfo { bobState nextState; }; -struct bobSettings { - bool enableBob; ///< This field is used by the pqi subsystem to determinine whether SOCKS proxy or BOB is used for I2P connections - std::string keys; ///< (optional) server keys - std::string addr; ///< (optional) hidden service addr. in base32 form - - int8_t inLength; - int8_t inQuantity; - int8_t inVariance; - - int8_t outLength; - int8_t outQuantity; - int8_t outVariance; -}; +struct bobSettings : i2p::settings {}; /// /// \brief The bobStates struct diff --git a/retroshare-gui/src/gui/settings/ServerPage.cpp b/retroshare-gui/src/gui/settings/ServerPage.cpp index 2881acd60..f0f8566e0 100755 --- a/retroshare-gui/src/gui/settings/ServerPage.cpp +++ b/retroshare-gui/src/gui/settings/ServerPage.cpp @@ -23,6 +23,7 @@ #include #include "rshare.h" #include "rsharesettings.h" +#include "util/i2pcommon.h" #include "util/RsNetUtil.h" #include "util/misc.h" @@ -1352,7 +1353,7 @@ void ServerPage::updateInProxyIndicator() ui.iconlabel_service_incoming->setMovie(movie); movie->start(); - if (mHiddenType == RS_HIDDEN_TYPE_I2P && mBobSettings.enableBob) { + if (mHiddenType == RS_HIDDEN_TYPE_I2P && mBobSettings.enable) { QTcpSocket tcpSocket; @@ -1439,15 +1440,16 @@ void ServerPage::getNewKey() void ServerPage::loadKey() { - mBobSettings.keys = ui.pteBobServerKey->toPlainText().toStdString(); - mBobSettings.addr = p3I2pBob::keyToBase32Addr(mBobSettings.keys); + mBobSettings.address.privateKey = ui.pteBobServerKey->toPlainText().toStdString(); + mBobSettings.address.publicKey = i2p::publicKeyFromPrivate(mBobSettings.address.privateKey); + mBobSettings.address.base32 = i2p::keyToBase32Addr(mBobSettings.address.publicKey); rsAutoProxyMonitor::taskSync(autoProxyType::I2PBOB, autoProxyTask::setSettings, &mBobSettings); } void ServerPage::enableBob(bool checked) { - mBobSettings.enableBob = checked; + mBobSettings.enable = checked; rsAutoProxyMonitor::taskSync(autoProxyType::I2PBOB, autoProxyTask::setSettings, &mBobSettings); @@ -1487,7 +1489,7 @@ void ServerPage::toggleBobAdvancedSettings(bool checked) { ui.swBobAdvanced->setCurrentIndex(checked ? 1 : 0); - if (!mBobSettings.keys.empty()) { + if (!mBobSettings.address.privateKey.empty()) { if (checked) { ui.pbBobGenAddr->show(); } else { @@ -1578,9 +1580,9 @@ void ServerPage::loadCommon() whileBlocking(ui.hiddenpage_proxyPort_i2p_2)->setValue(proxyport); // this one is for bob tab // don't use whileBlocking here - ui.cb_enableBob->setChecked(mBobSettings.enableBob); + ui.cb_enableBob->setChecked(mBobSettings.enable); - if (!mBobSettings.keys.empty()) { + if (!mBobSettings.address.privateKey.empty()) { ui.lBobB32Addr->show(); ui.leBobB32Addr->show(); } @@ -1623,13 +1625,13 @@ void ServerPage::saveBob() void ServerPage::updateStatusBob() { - QString addr = QString::fromStdString(mBobSettings.addr); + QString addr = QString::fromStdString(mBobSettings.address.base32); if (ui.leBobB32Addr->text() != addr) { ui.leBobB32Addr->setText(addr); ui.hiddenpage_serviceAddress->setText(addr); - ui.pteBobServerKey->setPlainText(QString::fromStdString(mBobSettings.keys)); + ui.pteBobServerKey->setPlainText(QString::fromStdString(mBobSettings.address.privateKey)); - if (!mBobSettings.keys.empty()) { + if (!mBobSettings.address.privateKey.empty()) { // we have an addr -> show fields ui.lBobB32Addr->show(); ui.leBobB32Addr->show(); @@ -1655,7 +1657,7 @@ void ServerPage::updateStatusBob() QString bobSimpleText = QString(); bobSimpleText.append(tr("RetroShare uses BOB to set up a %1 tunnel at %2:%3 (named %4)\n\n" "When changing options (e.g. port) use the buttons at the bottom to restart BOB.\n\n"). - arg(mBobSettings.keys.empty() ? tr("client") : tr("server"), + arg(mBobSettings.address.privateKey.empty() ? tr("client") : tr("server"), ui.hiddenpage_proxyAddress_i2p_2->text(), ui.hiddenpage_proxyPort_i2p_2->text(), bs.tunnelName.empty() ? tr("unknown") : @@ -1777,15 +1779,15 @@ void ServerPage::updateStatusBob() void ServerPage::setUpBobElements() { - ui.gbBob->setEnabled(mBobSettings.enableBob); - if (mBobSettings.enableBob) { + ui.gbBob->setEnabled(mBobSettings.enable); + if (mBobSettings.enable) { ui.hiddenpage_proxyAddress_i2p->setEnabled(false); ui.hiddenpage_proxyAddress_i2p->setToolTip("Use I2P/BOB settings to change this value"); ui.hiddenpage_proxyPort_i2p->setEnabled(false); ui.hiddenpage_proxyPort_i2p->setToolTip("Use I2P/BOB settings to change this value"); - ui.leBobB32Addr->setText(QString::fromStdString(mBobSettings.addr)); - ui.pteBobServerKey->setPlainText(QString::fromStdString(mBobSettings.keys)); + ui.leBobB32Addr->setText(QString::fromStdString(mBobSettings.address.base32)); + ui.pteBobServerKey->setPlainText(QString::fromStdString(mBobSettings.address.privateKey)); // cast to int to avoid problems int li, lo, qi, qo, vi, vo; From c20d92f04cfcb64d508ea25b60bf1e181da61a93 Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 8 May 2020 16:50:30 +0200 Subject: [PATCH 52/72] i2p: bob: convert to RsDbg --- .../src/services/autoproxy/p3i2pbob.cc | 113 +++++++++--------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index 6a52120dc..a68d85396 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -26,6 +26,7 @@ #include "pqi/p3peermgr.h" #include "rsitems/rsconfigitems.h" +#include "util/rsdebug.h" #include "util/radix32.h" #include "util/radix64.h" #include "util/rsdebug.h" @@ -56,7 +57,7 @@ static const int sleepFactorDefault = 10; // 0.5s static const int sleepFactorFast = 1; // 0.05s static const int sleepFactorSlow = 20; // 1s -static struct RsLog::logInfo i2pBobLogInfo = {RsLog::Default, "p3I2pBob"}; +RS_SET_CONTEXT_DEBUG_LEVEL(0) static const rstime_t selfCheckPeroid = 30; @@ -164,7 +165,7 @@ void p3I2pBob::processTaskAsync(taskTicket *ticket) } break; default: - rslog(RsLog::Warning, &i2pBobLogInfo, "p3I2pBob::processTaskAsync unknown task"); + RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::processTaskAsync unknown task" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -179,7 +180,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::status: // check if everything needed is set if (!data) { - rslog(RsLog::Warning, &i2pBobLogInfo, "p3I2pBob::status autoProxyTask::status data is missing"); + RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::status autoProxyTask::status data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -193,7 +194,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::getSettings: // check if everything needed is set if (!data) { - rslog(RsLog::Warning, &i2pBobLogInfo, "p3I2pBob::data_tick autoProxyTask::getSettings data is missing"); + RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::data_tick autoProxyTask::getSettings data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -207,7 +208,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::setSettings: // check if everything needed is set if (!data) { - rslog(RsLog::Warning, &i2pBobLogInfo, "p3I2pBob::data_tick autoProxyTask::setSettings data is missing"); + RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::data_tick autoProxyTask::setSettings data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -227,7 +228,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) break; case autoProxyTask::getErrorInfo: if (!data) { - rslog(RsLog::Warning, &i2pBobLogInfo, "p3I2pBob::data_tick autoProxyTask::getErrorInfo data is missing"); + RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::data_tick autoProxyTask::getErrorInfo data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); } else { RS_STACK_MUTEX(mLock); @@ -236,7 +237,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) } break; default: - rslog(RsLog::Warning, &i2pBobLogInfo, "p3I2pBob::processTaskSync unknown task"); + RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::processTaskSync unknown task" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -257,7 +258,7 @@ void p3I2pBob::threadTick() RS_STACK_MUTEX(mLock); std::stringstream ss; ss << "data_tick mState: " << mState << " mTask: " << mTask << " mBOBState: " << mBOBState << " mPending: " << mPending.size(); - rslog(RsLog::Debug_All, &i2pBobLogInfo, ss.str()); + Dbg4() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; } sleepTime += stateMachineController(); @@ -298,13 +299,13 @@ int p3I2pBob::stateMachineBOB() while (answer.find("OK Listing done") == std::string::npos) { std::stringstream ss; ss << "stateMachineBOB status check: read loop, counter: " << counter; - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, ss.str()); + Dbg3() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; answer += recv(); counter++; } if (answer.find(mTunnelName) == std::string::npos) { - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineBOB status check: tunnel down!"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineBOB status check: tunnel down!" << std::endl; // signal error *((bool *)mProcessing->data) = true; } @@ -344,8 +345,8 @@ int p3I2pBob::stateMachineBOB_locked_failure(const std::string &answer, const bo return sleepFactorDefault; } - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineBOB FAILED to run command '" + currentState.command + "'"); - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineBOB '" + answer + "'"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineBOB FAILED to run command '" + currentState.command + "'" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineBOB '" + answer + "'" << std::endl; mErrorMsg.append("FAILED to run command '" + currentState.command + "'" + '\n'); mErrorMsg.append("reason '" + answer + "'" + '\n'); @@ -392,14 +393,14 @@ int p3I2pBob::stateMachineController() return stateMachineController_locked_idle(); case csDoConnect: if (!connectI2P()) { - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController doConnect: unable to connect"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController doConnect: unable to connect" << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "unable to connect to BOB port"; return sleepFactorSlow; } - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController doConnect: connected"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController doConnect: connected" << std::endl; mState = csConnected; break; case csConnected: @@ -407,7 +408,7 @@ int p3I2pBob::stateMachineController() case csWaitForBob: // check connection problems if (mSocket == 0) { - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController waitForBob: conection lost"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController waitForBob: conection lost" << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "connection lost to BOB"; @@ -417,21 +418,21 @@ int p3I2pBob::stateMachineController() // check for finished BOB protocol if (mBOBState == bsCleared) { // done - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController waitForBob: mBOBState == bsCleared"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController waitForBob: mBOBState == bsCleared" << std::endl; mState = csDoDisconnect; } break; case csDoDisconnect: if (!disconnectI2P() || mSocket != 0) { // just in case - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController doDisconnect: can't disconnect"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController doDisconnect: can't disconnect" << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "unable to disconnect from BOB"; return sleepFactorDefault; } - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController doDisconnect: disconnected"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController doDisconnect: disconnected" << std::endl; mState = csDisconnected; break; case csDisconnected: @@ -462,7 +463,7 @@ int p3I2pBob::stateMachineController_locked_idle() mProcessing->task == autoProxyTask::stop || mProcessing->task == autoProxyTask::proxyStatusCheck)) { // skip since we are not enabled - rslog(RsLog::Debug_Alert, &i2pBobLogInfo, "stateMachineController_locked_idle: disabled -> skipping ticket"); + Dbg1() << __PRETTY_FUNCTION__ << " stateMachineController_locked_idle: disabled -> skipping ticket" << std::endl; rsAutoProxyMonitor::taskDone(mProcessing, autoProxyStatus::disabled); mProcessing = NULL; } else { @@ -484,7 +485,7 @@ int p3I2pBob::stateMachineController_locked_idle() mTask = ctRunCheck; break; default: - rslog(RsLog::Debug_Alert, &i2pBobLogInfo, "stateMachineController_locked_idle unknown async task"); + Dbg1() << __PRETTY_FUNCTION__ << " stateMachineController_locked_idle unknown async task" << std::endl; rsAutoProxyMonitor::taskError(mProcessing); mProcessing = NULL; break; @@ -532,28 +533,28 @@ int p3I2pBob::stateMachineController_locked_connected() case ctRunSetUp: // when we have a key use it for server tunnel! if(mSetting.address.privateKey.empty()) { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_connected: setting mBOBState = setnickC"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = setnickC" << std::endl; mBOBState = bsSetnickC; } else { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_connected: setting mBOBState = setnickS"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = setnickS" << std::endl; mBOBState = bsSetnickS; } break; case ctRunShutDown: // shut down existing tunnel - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_connected: setting mBOBState = getnick"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = getnick" << std::endl; mBOBState = bsGetnick; break; case ctRunCheck: - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_connected: setting mBOBState = list"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = list" << std::endl; mBOBState = bsList; break; case ctRunGetKeys: - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_connected: setting mBOBState = setnickN"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = setnickN" << std::endl; mBOBState = bsSetnickN; break; case ctIdle: - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_connected: task is idle. This should not happen!"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: task is idle. This should not happen!" << std::endl; break; } @@ -569,7 +570,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() if(errorHappened) { // reset old state mStateOld = csIdel; - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_disconnected: error during process!"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: error during process!" << std::endl; } // answer ticket @@ -598,12 +599,12 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; if (!errorHappened) { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_disconnected: run check result: ok"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: run check result: ok" << std::endl; break; } // switch to error newState = csError; - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_disconnected: run check result: error"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: run check result: error" << std::endl; mErrorMsg = "Connection check failed. Will try to restart tunnel."; break; @@ -626,7 +627,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; break; case ctIdle: - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_disconnected: task is idle. This should not happen!"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: task is idle. This should not happen!" << std::endl; rsAutoProxyMonitor::taskError(mProcessing); } mProcessing = NULL; @@ -642,14 +643,14 @@ int p3I2pBob::stateMachineController_locked_error() { // wait for bob protocoll if (mBOBState != bsCleared) { - rslog(RsLog::Debug_All, &i2pBobLogInfo, "stateMachineController_locked_error: waiting for BOB"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: waiting for BOB" << std::endl; return sleepFactorFast; } #if 0 std::stringstream ss; ss << "stateMachineController_locked_error: mProcessing: " << (mProcessing ? "not null" : "null"); - rslog(RsLog::Debug_All, &i2pBobLogInfo, ss.str()); + Dbg4() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; #endif // try to finish ticket @@ -657,7 +658,7 @@ int p3I2pBob::stateMachineController_locked_error() switch (mTask) { case ctRunCheck: // connection check failed at some point - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_error: failed to check proxy status (it's likely dead)!"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: failed to check proxy status (it's likely dead)!" << std::endl; *((bool *)mProcessing->data) = true; mState = csDoDisconnect; mStateOld = csIdel; @@ -665,7 +666,7 @@ int p3I2pBob::stateMachineController_locked_error() break; case ctRunShutDown: // not a big deal though - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_error: failed to shut down tunnel (it's likely dead though)!"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: failed to shut down tunnel (it's likely dead though)!" << std::endl; mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); @@ -673,14 +674,14 @@ int p3I2pBob::stateMachineController_locked_error() case ctIdle: // should not happen but we need to deal with it // this will produce some error messages in the log and finish the task (marked as failed) - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_error: task is idle. This should not happen!"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: task is idle. This should not happen!" << std::endl; mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); break; case ctRunGetKeys: case ctRunSetUp: - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_error: failed to receive key / start up"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: failed to receive key / start up" << std::endl; mStateOld = csError; mState = csDoDisconnect; // keep the error message @@ -691,7 +692,7 @@ int p3I2pBob::stateMachineController_locked_error() // periodically retry if (mLastProxyCheck < time(NULL) - (selfCheckPeroid >> 1) && mTask == ctRunSetUp) { - rslog(RsLog::Warning, &i2pBobLogInfo, "stateMachineController_locked_error: retrying"); + RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: retrying" << std::endl; mLastProxyCheck = time(NULL); mErrorMsg.clear(); @@ -704,7 +705,7 @@ int p3I2pBob::stateMachineController_locked_error() // check for new tickets if (!mPending.empty()) { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "stateMachineController_locked_error: processing new ticket"); + Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: processing new ticket" << std::endl; // reset and try new task mTask = ctIdle; @@ -735,7 +736,7 @@ RsSerialiser *p3I2pBob::setupSerialiser() bool p3I2pBob::saveList(bool &cleanup, std::list &lst) { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "saveList"); + Dbg4() << __PRETTY_FUNCTION__ << " saveList" << std::endl; cleanup = true; RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet; @@ -770,7 +771,7 @@ bool p3I2pBob::saveList(bool &cleanup, std::list &lst) bool p3I2pBob::loadList(std::list &load) { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "loadList"); + Dbg4() << __PRETTY_FUNCTION__ << " loadList" << std::endl; for(std::list::const_iterator it = load.begin(); it!=load.end(); ++it) { RsConfigKeyValueSet *vitem = dynamic_cast(*it); @@ -790,7 +791,7 @@ bool p3I2pBob::loadList(std::list &load) getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity) getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance) else - rslog(RsLog::Warning, &i2pBobLogInfo, "loadList unknown key: " + kit->key); + RsDbg() << __PRETTY_FUNCTION__ << " loadList unknown key: " + kit->key << std::endl; } } delete vitem; @@ -854,7 +855,7 @@ void p3I2pBob::getStates(bobStates *bs) std::string p3I2pBob::executeCommand(const std::string &command) { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "executeCommand_locked running '" + command + "'"); + Dbg4() << __PRETTY_FUNCTION__ << " executeCommand_locked running '" + command + "'" << std::endl; std::string copy = command; copy.push_back('\n'); @@ -866,7 +867,7 @@ std::string p3I2pBob::executeCommand(const std::string &command) // receive answer (trailing new line is already removed!) std::string ans = recv(); - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "executeCommand_locked answer '" + ans + "'"); + Dbg4() << __PRETTY_FUNCTION__ << " executeCommand_locked answer '" + ans + "'" << std::endl; return ans; } @@ -876,7 +877,7 @@ bool p3I2pBob::connectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket != 0) { - rslog(RsLog::Warning, &i2pBobLogInfo, "connectI2P_locked mSocket != 0"); + RsDbg() << __PRETTY_FUNCTION__ << " connectI2P_locked mSocket != 0" << std::endl; return false; } @@ -884,21 +885,21 @@ bool p3I2pBob::connectI2P() mSocket = unix_socket(PF_INET, SOCK_STREAM, 0); if (mSocket < 0) { - rslog(RsLog::Warning, &i2pBobLogInfo, "connectI2P_locked Failed to open socket! Socket Error: " + socket_errorType(errno)); + RsDbg() << __PRETTY_FUNCTION__ << " connectI2P_locked Failed to open socket! Socket Error: " + socket_errorType(errno) << std::endl; return false; } // connect int err = unix_connect(mSocket, mI2PProxyAddr); if (err != 0) { - rslog(RsLog::Warning, &i2pBobLogInfo, "connectI2P_locked Failed to connect to BOB! Socket Error: " + socket_errorType(errno)); + RsDbg() << __PRETTY_FUNCTION__ << " connectI2P_locked Failed to connect to BOB! Socket Error: " + socket_errorType(errno) << std::endl; return false; } // receive hello msg recv(); - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "connectI2P_locked done"); + Dbg4() << __PRETTY_FUNCTION__ << " connectI2P_locked done" << std::endl; return true; } @@ -907,17 +908,17 @@ bool p3I2pBob::disconnectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket == 0) { - rslog(RsLog::Warning, &i2pBobLogInfo, "disconnectI2P_locked mSocket == 0"); + RsDbg() << __PRETTY_FUNCTION__ << " disconnectI2P_locked mSocket == 0" << std::endl; return true; } int err = unix_close(mSocket); if (err != 0) { - rslog(RsLog::Warning, &i2pBobLogInfo, "disconnectI2P_locked Failed to close socket! Socket Error: " + socket_errorType(errno)); + RsDbg() << __PRETTY_FUNCTION__ << " disconnectI2P_locked Failed to close socket! Socket Error: " + socket_errorType(errno) << std::endl; return false; } - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "disconnectI2P_locked done"); + Dbg4() << __PRETTY_FUNCTION__ << " disconnectI2P_locked done" << std::endl; mSocket = 0; return true; } @@ -938,7 +939,7 @@ std::string toString(const std::string &a, const int8_t b) { void p3I2pBob::finalizeSettings_locked() { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "finalizeSettings_locked"); + Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked" << std::endl; sockaddr_storage_clear(mI2PProxyAddr); // get i2p proxy addr @@ -949,8 +950,8 @@ void p3I2pBob::finalizeSettings_locked() sockaddr_storage_setipv4(mI2PProxyAddr, (sockaddr_in*)&proxy); sockaddr_storage_setport(mI2PProxyAddr, 2827); - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "finalizeSettings_locked using " + sockaddr_storage_tostring(mI2PProxyAddr)); - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "finalizeSettings_locked using " + mSetting.address.base32); + Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked using " + sockaddr_storage_tostring(mI2PProxyAddr) << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked using " + mSetting.address.base32 << std::endl; peerState ps; mPeerMgr->getOwnNetStatus(ps); @@ -965,7 +966,7 @@ void p3I2pBob::finalizeSettings_locked() std::vector tmp(len); RSRandom::random_bytes(tmp.data(), len); const std::string location = Radix32::encode(tmp.data(), len); - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "finalizeSettings_locked using suffix " + location); + Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked using suffix " + location << std::endl; mTunnelName = "RetroShare-" + location; const std::string setnick = "setnick RetroShare-" + location; @@ -1033,7 +1034,7 @@ void p3I2pBob::finalizeSettings_locked() void p3I2pBob::updateSettings_locked() { - rslog(RsLog::Debug_Basic, &i2pBobLogInfo, "updateSettings_locked"); + Dbg4() << __PRETTY_FUNCTION__ << " updateSettings_locked" << std::endl; sockaddr_storage proxy; mPeerMgr->getProxyServerAddress(RS_HIDDEN_TYPE_I2P, proxy); @@ -1095,7 +1096,7 @@ std::string p3I2pBob::recv() #if 0 std::stringstream ss; ss << "recv length: " << length << " (bufferSize: " << bufferSize << ") ans: " << ans.length(); - rslog(RsLog::Debug_All, &i2pBobLogInfo, ss.str()); + Dbg4() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; #endif // clear and resize buffer again From a41e10b178e3903173e535893a0bbaf6f5dbdd26 Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 8 May 2020 16:50:59 +0200 Subject: [PATCH 53/72] i2p: bob: remove unused variables --- libretroshare/src/services/autoproxy/p3i2pbob.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index a68d85396..f63a10b7f 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -44,11 +44,6 @@ static const std::string kConfigKeyOutLength = "OUT_LENGTH"; static const std::string kConfigKeyOutQuantity = "OUT_QUANTITY"; static const std::string kConfigKeyOutVariance = "OUT_VARIANCE"; -static const bool kDefaultBOBEnable = false; -static const int8_t kDefaultLength = 3; -static const int8_t kDefaultQuantity = 4; -static const int8_t kDefaultVariance = 0; - /// Sleep duration for receiving loop static const useconds_t sleepTimeRecv = 10; // times 1000 = 10ms /// Sleep duration for everything else From 1da3d262d94ef68d0289fdfcae243570d842fd85 Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 8 May 2020 16:54:51 +0200 Subject: [PATCH 54/72] i2p: bob: removed function name within debug output (replaced by __PRETTY_FUNCTION__) --- .../src/services/autoproxy/p3i2pbob.cc | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index f63a10b7f..e8ebb5979 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -160,7 +160,7 @@ void p3I2pBob::processTaskAsync(taskTicket *ticket) } break; default: - RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::processTaskAsync unknown task" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " unknown task" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -175,7 +175,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::status: // check if everything needed is set if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::status autoProxyTask::status data is missing" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::status data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -189,7 +189,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::getSettings: // check if everything needed is set if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::data_tick autoProxyTask::getSettings data is missing" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::getSettings data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -203,7 +203,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::setSettings: // check if everything needed is set if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::data_tick autoProxyTask::setSettings data is missing" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::setSettings data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -223,7 +223,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) break; case autoProxyTask::getErrorInfo: if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::data_tick autoProxyTask::getErrorInfo data is missing" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::getErrorInfo data is missing" << std::endl; rsAutoProxyMonitor::taskError(ticket); } else { RS_STACK_MUTEX(mLock); @@ -232,7 +232,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) } break; default: - RsDbg() << __PRETTY_FUNCTION__ << " p3I2pBob::processTaskSync unknown task" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " unknown task" << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -300,7 +300,7 @@ int p3I2pBob::stateMachineBOB() } if (answer.find(mTunnelName) == std::string::npos) { - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineBOB status check: tunnel down!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " status check: tunnel down!" << std::endl; // signal error *((bool *)mProcessing->data) = true; } @@ -340,8 +340,8 @@ int p3I2pBob::stateMachineBOB_locked_failure(const std::string &answer, const bo return sleepFactorDefault; } - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineBOB FAILED to run command '" + currentState.command + "'" << std::endl; - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineBOB '" + answer + "'" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " FAILED to run command '" + currentState.command + "'" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " '" + answer + "'" << std::endl; mErrorMsg.append("FAILED to run command '" + currentState.command + "'" + '\n'); mErrorMsg.append("reason '" + answer + "'" + '\n'); @@ -388,14 +388,14 @@ int p3I2pBob::stateMachineController() return stateMachineController_locked_idle(); case csDoConnect: if (!connectI2P()) { - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController doConnect: unable to connect" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " doConnect: unable to connect" << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "unable to connect to BOB port"; return sleepFactorSlow; } - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController doConnect: connected" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " doConnect: connected" << std::endl; mState = csConnected; break; case csConnected: @@ -403,7 +403,7 @@ int p3I2pBob::stateMachineController() case csWaitForBob: // check connection problems if (mSocket == 0) { - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController waitForBob: conection lost" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " waitForBob: conection lost" << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "connection lost to BOB"; @@ -413,21 +413,21 @@ int p3I2pBob::stateMachineController() // check for finished BOB protocol if (mBOBState == bsCleared) { // done - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController waitForBob: mBOBState == bsCleared" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " waitForBob: mBOBState == bsCleared" << std::endl; mState = csDoDisconnect; } break; case csDoDisconnect: if (!disconnectI2P() || mSocket != 0) { // just in case - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController doDisconnect: can't disconnect" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " doDisconnect: can't disconnect" << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "unable to disconnect from BOB"; return sleepFactorDefault; } - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController doDisconnect: disconnected" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " doDisconnect: disconnected" << std::endl; mState = csDisconnected; break; case csDisconnected: @@ -458,7 +458,7 @@ int p3I2pBob::stateMachineController_locked_idle() mProcessing->task == autoProxyTask::stop || mProcessing->task == autoProxyTask::proxyStatusCheck)) { // skip since we are not enabled - Dbg1() << __PRETTY_FUNCTION__ << " stateMachineController_locked_idle: disabled -> skipping ticket" << std::endl; + Dbg1() << __PRETTY_FUNCTION__ << " disabled -> skipping ticket" << std::endl; rsAutoProxyMonitor::taskDone(mProcessing, autoProxyStatus::disabled); mProcessing = NULL; } else { @@ -480,7 +480,7 @@ int p3I2pBob::stateMachineController_locked_idle() mTask = ctRunCheck; break; default: - Dbg1() << __PRETTY_FUNCTION__ << " stateMachineController_locked_idle unknown async task" << std::endl; + Dbg1() << __PRETTY_FUNCTION__ << " unknown async task" << std::endl; rsAutoProxyMonitor::taskError(mProcessing); mProcessing = NULL; break; @@ -528,28 +528,28 @@ int p3I2pBob::stateMachineController_locked_connected() case ctRunSetUp: // when we have a key use it for server tunnel! if(mSetting.address.privateKey.empty()) { - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = setnickC" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = setnickC" << std::endl; mBOBState = bsSetnickC; } else { - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = setnickS" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = setnickS" << std::endl; mBOBState = bsSetnickS; } break; case ctRunShutDown: // shut down existing tunnel - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = getnick" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = getnick" << std::endl; mBOBState = bsGetnick; break; case ctRunCheck: - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = list" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = list" << std::endl; mBOBState = bsList; break; case ctRunGetKeys: - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: setting mBOBState = setnickN" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = setnickN" << std::endl; mBOBState = bsSetnickN; break; case ctIdle: - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_connected: task is idle. This should not happen!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " task is idle. This should not happen!" << std::endl; break; } @@ -565,7 +565,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() if(errorHappened) { // reset old state mStateOld = csIdel; - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: error during process!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " error during process!" << std::endl; } // answer ticket @@ -594,12 +594,12 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; if (!errorHappened) { - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: run check result: ok" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " run check result: ok" << std::endl; break; } // switch to error newState = csError; - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: run check result: error" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " run check result: error" << std::endl; mErrorMsg = "Connection check failed. Will try to restart tunnel."; break; @@ -622,7 +622,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; break; case ctIdle: - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_disconnected: task is idle. This should not happen!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " task is idle. This should not happen!" << std::endl; rsAutoProxyMonitor::taskError(mProcessing); } mProcessing = NULL; @@ -638,7 +638,7 @@ int p3I2pBob::stateMachineController_locked_error() { // wait for bob protocoll if (mBOBState != bsCleared) { - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: waiting for BOB" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " waiting for BOB" << std::endl; return sleepFactorFast; } @@ -653,7 +653,7 @@ int p3I2pBob::stateMachineController_locked_error() switch (mTask) { case ctRunCheck: // connection check failed at some point - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: failed to check proxy status (it's likely dead)!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " failed to check proxy status (it's likely dead)!" << std::endl; *((bool *)mProcessing->data) = true; mState = csDoDisconnect; mStateOld = csIdel; @@ -661,7 +661,7 @@ int p3I2pBob::stateMachineController_locked_error() break; case ctRunShutDown: // not a big deal though - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: failed to shut down tunnel (it's likely dead though)!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " failed to shut down tunnel (it's likely dead though)!" << std::endl; mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); @@ -669,14 +669,14 @@ int p3I2pBob::stateMachineController_locked_error() case ctIdle: // should not happen but we need to deal with it // this will produce some error messages in the log and finish the task (marked as failed) - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: task is idle. This should not happen!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " task is idle. This should not happen!" << std::endl; mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); break; case ctRunGetKeys: case ctRunSetUp: - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: failed to receive key / start up" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " failed to receive key / start up" << std::endl; mStateOld = csError; mState = csDoDisconnect; // keep the error message @@ -687,7 +687,7 @@ int p3I2pBob::stateMachineController_locked_error() // periodically retry if (mLastProxyCheck < time(NULL) - (selfCheckPeroid >> 1) && mTask == ctRunSetUp) { - RsDbg() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: retrying" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " retrying" << std::endl; mLastProxyCheck = time(NULL); mErrorMsg.clear(); @@ -700,7 +700,7 @@ int p3I2pBob::stateMachineController_locked_error() // check for new tickets if (!mPending.empty()) { - Dbg4() << __PRETTY_FUNCTION__ << " stateMachineController_locked_error: processing new ticket" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " processing new ticket" << std::endl; // reset and try new task mTask = ctIdle; @@ -731,7 +731,7 @@ RsSerialiser *p3I2pBob::setupSerialiser() bool p3I2pBob::saveList(bool &cleanup, std::list &lst) { - Dbg4() << __PRETTY_FUNCTION__ << " saveList" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << std::endl; cleanup = true; RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet; @@ -766,7 +766,7 @@ bool p3I2pBob::saveList(bool &cleanup, std::list &lst) bool p3I2pBob::loadList(std::list &load) { - Dbg4() << __PRETTY_FUNCTION__ << " loadList" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << std::endl; for(std::list::const_iterator it = load.begin(); it!=load.end(); ++it) { RsConfigKeyValueSet *vitem = dynamic_cast(*it); @@ -786,7 +786,7 @@ bool p3I2pBob::loadList(std::list &load) getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity) getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance) else - RsDbg() << __PRETTY_FUNCTION__ << " loadList unknown key: " + kit->key << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " unknown key: " + kit->key << std::endl; } } delete vitem; @@ -850,7 +850,7 @@ void p3I2pBob::getStates(bobStates *bs) std::string p3I2pBob::executeCommand(const std::string &command) { - Dbg4() << __PRETTY_FUNCTION__ << " executeCommand_locked running '" + command + "'" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " running '" + command + "'" << std::endl; std::string copy = command; copy.push_back('\n'); @@ -862,7 +862,7 @@ std::string p3I2pBob::executeCommand(const std::string &command) // receive answer (trailing new line is already removed!) std::string ans = recv(); - Dbg4() << __PRETTY_FUNCTION__ << " executeCommand_locked answer '" + ans + "'" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " answer '" + ans + "'" << std::endl; return ans; } @@ -872,7 +872,7 @@ bool p3I2pBob::connectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket != 0) { - RsDbg() << __PRETTY_FUNCTION__ << " connectI2P_locked mSocket != 0" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " mSocket != 0" << std::endl; return false; } @@ -880,21 +880,21 @@ bool p3I2pBob::connectI2P() mSocket = unix_socket(PF_INET, SOCK_STREAM, 0); if (mSocket < 0) { - RsDbg() << __PRETTY_FUNCTION__ << " connectI2P_locked Failed to open socket! Socket Error: " + socket_errorType(errno) << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " Failed to open socket! Socket Error: " + socket_errorType(errno) << std::endl; return false; } // connect int err = unix_connect(mSocket, mI2PProxyAddr); if (err != 0) { - RsDbg() << __PRETTY_FUNCTION__ << " connectI2P_locked Failed to connect to BOB! Socket Error: " + socket_errorType(errno) << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " Failed to connect to BOB! Socket Error: " + socket_errorType(errno) << std::endl; return false; } // receive hello msg recv(); - Dbg4() << __PRETTY_FUNCTION__ << " connectI2P_locked done" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " done" << std::endl; return true; } @@ -903,17 +903,17 @@ bool p3I2pBob::disconnectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket == 0) { - RsDbg() << __PRETTY_FUNCTION__ << " disconnectI2P_locked mSocket == 0" << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " mSocket == 0" << std::endl; return true; } int err = unix_close(mSocket); if (err != 0) { - RsDbg() << __PRETTY_FUNCTION__ << " disconnectI2P_locked Failed to close socket! Socket Error: " + socket_errorType(errno) << std::endl; + RsDbg() << __PRETTY_FUNCTION__ << " Failed to close socket! Socket Error: " + socket_errorType(errno) << std::endl; return false; } - Dbg4() << __PRETTY_FUNCTION__ << " disconnectI2P_locked done" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " done" << std::endl; mSocket = 0; return true; } @@ -934,7 +934,7 @@ std::string toString(const std::string &a, const int8_t b) { void p3I2pBob::finalizeSettings_locked() { - Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << std::endl; sockaddr_storage_clear(mI2PProxyAddr); // get i2p proxy addr @@ -945,8 +945,8 @@ void p3I2pBob::finalizeSettings_locked() sockaddr_storage_setipv4(mI2PProxyAddr, (sockaddr_in*)&proxy); sockaddr_storage_setport(mI2PProxyAddr, 2827); - Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked using " + sockaddr_storage_tostring(mI2PProxyAddr) << std::endl; - Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked using " + mSetting.address.base32 << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " using " + sockaddr_storage_tostring(mI2PProxyAddr) << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " using " + mSetting.address.base32 << std::endl; peerState ps; mPeerMgr->getOwnNetStatus(ps); @@ -961,7 +961,7 @@ void p3I2pBob::finalizeSettings_locked() std::vector tmp(len); RSRandom::random_bytes(tmp.data(), len); const std::string location = Radix32::encode(tmp.data(), len); - Dbg4() << __PRETTY_FUNCTION__ << " finalizeSettings_locked using suffix " + location << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << " using suffix " + location << std::endl; mTunnelName = "RetroShare-" + location; const std::string setnick = "setnick RetroShare-" + location; @@ -1029,7 +1029,7 @@ void p3I2pBob::finalizeSettings_locked() void p3I2pBob::updateSettings_locked() { - Dbg4() << __PRETTY_FUNCTION__ << " updateSettings_locked" << std::endl; + Dbg4() << __PRETTY_FUNCTION__ << std::endl; sockaddr_storage proxy; mPeerMgr->getProxyServerAddress(RS_HIDDEN_TYPE_I2P, proxy); From f4f08f037922ac292d509c62812789cc06803bb1 Mon Sep 17 00:00:00 2001 From: sehraf Date: Sun, 10 May 2020 13:37:53 +0200 Subject: [PATCH 55/72] i2p: bob: rework recv() to check for new line character as EOL indicator --- .../src/services/autoproxy/p3i2pbob.cc | 72 ++++++++++++------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index e8ebb5979..ef8e1ad48 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -44,8 +44,8 @@ static const std::string kConfigKeyOutLength = "OUT_LENGTH"; static const std::string kConfigKeyOutQuantity = "OUT_QUANTITY"; static const std::string kConfigKeyOutVariance = "OUT_VARIANCE"; -/// Sleep duration for receiving loop -static const useconds_t sleepTimeRecv = 10; // times 1000 = 10ms +/// Sleep duration for receiving loop in error/no-data case +static const useconds_t sleepTimeRecv = 250; // times 1000 = 250ms /// Sleep duration for everything else static const useconds_t sleepTimeWait = 50; // times 1000 = 50ms or 0.05s static const int sleepFactorDefault = 10; // 0.5s @@ -1069,38 +1069,62 @@ void p3I2pBob::updateSettings_locked() std::string p3I2pBob::recv() { + // BOB works line based + // -> \n indicates and of the line + + constexpr uint16_t bufferSize = 128; + char buffer[bufferSize]; + std::string ans; - ssize_t length; - const uint16_t bufferSize = 128; - std::vector buffer(bufferSize); + uint16_t retry = 10; do { - doSleep(sleepTimeRecv); + memset(buffer, 0, bufferSize); - // there is only one thread that touches mSocket - no need for a lock - length = ::recv(mSocket, buffer.data(), buffer.size(), 0); - if (length < 0) + // peek at data + auto length = ::recv(mSocket, buffer, bufferSize, MSG_PEEK); + if (length <= 0) { + if (length < 0) { + // error + perror(__PRETTY_FUNCTION__); + } + retry--; + doSleep(sleepTimeRecv); continue; + } - ans.append(buffer.begin(), buffer.end()); + // at least one byte was read - // clean received string - ans.erase(std::remove(ans.begin(), ans.end(), '\0'), ans.end()); - ans.erase(std::remove(ans.begin(), ans.end(), '\n'), ans.end()); + // search for new line + auto bufferStr = std::string(buffer); + size_t pos = bufferStr.find('\n'); -#if 0 - std::stringstream ss; - ss << "recv length: " << length << " (bufferSize: " << bufferSize << ") ans: " << ans.length(); - Dbg4() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; -#endif + if (pos == std::string::npos) { + // no new line found -> more to read - // clear and resize buffer again - buffer.clear(); - buffer.resize(bufferSize); + // sanity check + if (length != bufferSize) { + // expectation: a full buffer was peeked) + Dbg1() << __PRETTY_FUNCTION__ << " peeked less than bufferSize but also didn't found a new line character" << std::endl; + } + // this should never happen + assert(length <= bufferSize); + } else { + // new line found -> end of message - if (this->shouldStop()) - break; - } while(length == bufferSize || ans.size() < 4); + // calculate how much there is to read, read the \n, too! + length = pos + 1; + + // end loop + retry = 0; + } + + // now read for real + memset(buffer, 0, bufferSize); + length = ::recv(mSocket, buffer, length, 0); + bufferStr = std::string(buffer); + ans.append(bufferStr); + } while(retry > 0); return ans; } From b6a550b8f5509ce9e97da65614e2cfbd87cab32b Mon Sep 17 00:00:00 2001 From: sehraf Date: Thu, 21 May 2020 10:12:59 +0200 Subject: [PATCH 56/72] autoproxy: make use of rsdebug.h --- .../services/autoproxy/rsautoproxymonitor.cc | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc index d198bd207..263663a4e 100644 --- a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc +++ b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc @@ -22,6 +22,7 @@ #include "rsautoproxymonitor.h" #include /* for usleep() */ +#include "util/rsdebug.h" #include "util/rstime.h" rsAutoProxyMonitor *rsAutoProxyMonitor::mInstance = NULL; @@ -42,8 +43,10 @@ rsAutoProxyMonitor *rsAutoProxyMonitor::instance() void rsAutoProxyMonitor::addProxy(autoProxyType::autoProxyType_enum type, autoProxyService *service) { RS_STACK_MUTEX(mLock); - if (mProxies.find(type) != mProxies.end()) - std::cerr << "sAutoProxyMonitor::addProxy type " << type << " already added - OVERWRITING" << std::endl; + if (mProxies.find(type) != mProxies.end()) { + RsErr() << __PRETTY_FUNCTION__<< " type " << type << " already added - OVERWRITING" << std::endl; + print_stacktrace(); + } mProxies[type] = service; } @@ -117,7 +120,7 @@ void rsAutoProxyMonitor::stopAllRSShutdown() do { rstime::rs_usleep(1000 * 1000); RS_STACK_MUTEX(mLock); - std::cout << "(II) waiting for auto proxy service(s) to shut down " << t << "/" << timeout << " (remaining: " << mProxies.size() << ")" << std::endl; + RsDbg() << __PRETTY_FUNCTION__<< " waiting for auto proxy service(s) to shut down " << t << "/" << timeout << " (remaining: " << mProxies.size() << ")" << std::endl; if (mProxies.empty()) break; t++; @@ -146,13 +149,16 @@ void rsAutoProxyMonitor::task(taskTicket *ticket) { // sanity checks if (!ticket->async && ticket->types.size() > 1) { - std::cerr << "(WW) rsAutoProxyMonitor::task synchronous call to multiple services. This can cause problems!" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " synchronous call to multiple services. This can cause problems!" << std::endl; + print_stacktrace(); } if (ticket->async && !ticket->cb && ticket->data) { - std::cerr << "(WW) rsAutoProxyMonitor::task asynchronous call with data but no callback. This will likely causes memory leak!" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " asynchronous call with data but no callback. This will likely causes memory leak!" << std::endl; + print_stacktrace(); } if (ticket->types.size() > 1 && ticket->data) { - std::cerr << "(WW) rsAutoProxyMonitor::task call with data to multiple services. This will likely causes memory leak!" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " call with data to multiple services. This will likely causes memory leak!" << std::endl; + print_stacktrace(); } std::vector::const_iterator it; @@ -187,7 +193,8 @@ void rsAutoProxyMonitor::taskAsync(std::vector if (isAsyncTask(task)) { // Usually the services will reject this ticket. // Just print a warning - maybe there is some special case where this is a good idea. - std::cerr << "(WW) rsAutoProxyMonitor::taskSync called with an asynchronous task!" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " called with an asynchronous task!" << std::endl; + print_stacktrace(); } taskTicket *tt = getTicket(); @@ -244,7 +252,8 @@ void rsAutoProxyMonitor::taskDone(taskTicket *t, autoProxyStatus::autoProxyStatu t->cb->taskFinished(t); if (t != NULL) { // callack did not clean up properly - std::cerr << "(WW) rsAutoProxyMonitor::taskFinish callback did not clean up!" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " callback did not clean up!" << std::endl; + print_stacktrace(); cleanUp = true; } } else if (t->async){ @@ -252,12 +261,13 @@ void rsAutoProxyMonitor::taskDone(taskTicket *t, autoProxyStatus::autoProxyStatu // we must take care of deleting cleanUp = true; if(t->data) - std::cerr << "(WW) rsAutoProxyMonitor::taskFinish async call with data attached but no callback set!" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " async call with data attached but no callback set!" << std::endl; } if (cleanUp) { if (t->data) { - std::cerr << "(WW) rsAutoProxyMonitor::taskFinish will try to delete void pointer!" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " will try to delete void pointer!" << std::endl; + print_stacktrace(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdelete-incomplete" delete t->data; @@ -290,7 +300,8 @@ void rsAutoProxyMonitor::taskFinished(taskTicket *&ticket) // clean up if (ticket->data) { - std::cerr << "rsAutoProxyMonitor::taskFinished data set. Will try to delete void pointer" << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " data set. Will try to delete void pointer" << std::endl; + print_stacktrace(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdelete-incomplete" delete ticket->data; @@ -308,7 +319,7 @@ autoProxyService *rsAutoProxyMonitor::lookUpService(autoProxyType::autoProxyType if ((itService = mProxies.find(t)) != mProxies.end()) { return itService->second; } - std::cerr << "sAutoProxyMonitor::lookUpService no service for type " << t << " found!" << std::endl; + RsDbg() << __PRETTY_FUNCTION__<< " no service for type " << t << " found!" << std::endl; return NULL; } From 9a5504bb4733384763a648222f39c636c73dd541 Mon Sep 17 00:00:00 2001 From: sehraf Date: Thu, 21 May 2020 10:17:13 +0200 Subject: [PATCH 57/72] autoproxy: make async really async --- libretroshare/src/services/autoproxy/rsautoproxymonitor.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc index 263663a4e..62b47dfea 100644 --- a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc +++ b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc @@ -174,7 +174,11 @@ void rsAutoProxyMonitor::task(taskTicket *ticket) *tt = *ticket; tt->types.clear(); tt->types.push_back(*it); - s->processTaskAsync(tt); + + // it's async! + RsThread::async([s, tt] { + s->processTaskAsync(tt); + }); } else { s->processTaskSync(ticket); } From a5a2b49d992df09249cff439a7474351643ae7f3 Mon Sep 17 00:00:00 2001 From: sehraf Date: Mon, 25 May 2020 21:51:06 +0200 Subject: [PATCH 58/72] i2p: bob: add option to disable bob CONFIG+=no_rs_bob --- libretroshare/src/rsserver/p3face.h | 2 ++ libretroshare/src/rsserver/rsinit.cc | 6 ++++++ retroshare-gui/src/gui/settings/ServerPage.cpp | 4 ++++ retroshare.pri | 9 +++++++++ 4 files changed, 21 insertions(+) diff --git a/libretroshare/src/rsserver/p3face.h b/libretroshare/src/rsserver/p3face.h index a2637ceaf..661cb244f 100644 --- a/libretroshare/src/rsserver/p3face.h +++ b/libretroshare/src/rsserver/p3face.h @@ -161,7 +161,9 @@ public: p3ChatService *chatSrv; p3StatusService *mStatusSrv; p3GxsTunnelService *mGxsTunnels; +#ifdef RS_USE_I2P_BOB p3I2pBob *mI2pBob; +#endif // This list contains all threaded services. It will be used to shut them down properly. diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 1c4e2a062..fa21b82e7 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -923,8 +923,10 @@ int RsServer::StartupRetroShare() mNetMgr->setManagers(mPeerMgr, mLinkMgr); rsAutoProxyMonitor *autoProxy = rsAutoProxyMonitor::instance(); +#ifdef RS_USE_I2P_BOB mI2pBob = new p3I2pBob(mPeerMgr); autoProxy->addProxy(autoProxyType::I2PBOB, mI2pBob); +#endif //load all the SSL certs as friends // std::list sslIds; @@ -1649,7 +1651,9 @@ int RsServer::StartupRetroShare() mConfigMgr->addConfiguration("wire.cfg", wire_ns); #endif #endif //RS_ENABLE_GXS +#ifdef RS_USE_I2P_BOB mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pBob); +#endif mPluginsManager->addConfigurations(mConfigMgr) ; @@ -1795,7 +1799,9 @@ int RsServer::StartupRetroShare() /**************************************************************************/ // auto proxy threads +#ifdef RS_USE_I2P_BOB startServiceThread(mI2pBob, "I2P-BOB"); +#endif #ifdef RS_ENABLE_GXS // Must Set the GXS pointers before starting threads. diff --git a/retroshare-gui/src/gui/settings/ServerPage.cpp b/retroshare-gui/src/gui/settings/ServerPage.cpp index f0f8566e0..37291be0f 100755 --- a/retroshare-gui/src/gui/settings/ServerPage.cpp +++ b/retroshare-gui/src/gui/settings/ServerPage.cpp @@ -83,6 +83,10 @@ ServerPage::ServerPage(QWidget * parent, Qt::WindowFlags flags) manager = NULL ; mOngoingConnectivityCheck = -1; +#ifndef RS_USE_I2P_BOB + ui.hiddenServiceTab->removeTab(TAB_HIDDEN_SERVICE_I2P_BOB); // warning: the order of operation here is very important. +#endif + if(RsAccounts::isHiddenNode()) { if(RsAccounts::isTorAuto()) diff --git a/retroshare.pri b/retroshare.pri index 6bccc1f5a..994ced0f2 100644 --- a/retroshare.pri +++ b/retroshare.pri @@ -140,6 +140,11 @@ rs_macos10.15:CONFIG -= rs_macos10.11 CONFIG *= no_rs_jsonapi rs_jsonapi:CONFIG -= no_rs_jsonapi +# Disable i2p BOB support for automatically setting up an i2p tunnel for RS +# "CONFIG+=no_rs_bob" +CONFIG *= rs_bob +no_rs_bob:CONFIG -= rs_bob + # To enable channel indexing append the following assignation to qmake command # line "CONFIG+=rs_deep_channel_index" CONFIG *= no_rs_deep_channel_index @@ -550,6 +555,10 @@ rs_webui { DEFINES *= RS_WEBUI } +rs_bob { + DEFINES *= RS_USE_I2P_BOB +} + rs_deep_channels_index:DEFINES *= RS_DEEP_CHANNEL_INDEX rs_deep_files_index:DEFINES *= RS_DEEP_FILES_INDEX From 950703c398f7f5bc920b04c60433f612c677020e Mon Sep 17 00:00:00 2001 From: sehraf Date: Mon, 25 May 2020 22:23:58 +0200 Subject: [PATCH 59/72] i2p: bob: handle CONFIG+=no_rs_bob when creating a new location --- retroshare-gui/src/gui/GenCertDialog.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/retroshare-gui/src/gui/GenCertDialog.cpp b/retroshare-gui/src/gui/GenCertDialog.cpp index 4fdb9f714..aae80b753 100644 --- a/retroshare-gui/src/gui/GenCertDialog.cpp +++ b/retroshare-gui/src/gui/GenCertDialog.cpp @@ -340,6 +340,10 @@ void GenCertDialog::setupState() ui.hiddenport_spinBox->setVisible(hidden_state && !tor_auto); ui.cbUseBob->setVisible(hidden_state && !tor_auto); +#ifndef RS_USE_I2P_BOB + ui.cbUseBob->setDisabled(true); + ui.cbUseBob->setToolTip(tr("BOB support is not available")); +#endif if(!mAllFieldsOk) { From d0c5dc4e2496204cf88e0b0c73091485d87be441 Mon Sep 17 00:00:00 2001 From: sehraf Date: Wed, 10 Jun 2020 21:22:45 +0200 Subject: [PATCH 60/72] i2p: bob: convert to RS_DBG --- .../src/services/autoproxy/p3i2pbob.cc | 129 +++++++++--------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index ef8e1ad48..c1b32fa76 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -26,7 +26,6 @@ #include "pqi/p3peermgr.h" #include "rsitems/rsconfigitems.h" -#include "util/rsdebug.h" #include "util/radix32.h" #include "util/radix64.h" #include "util/rsdebug.h" @@ -52,8 +51,6 @@ static const int sleepFactorDefault = 10; // 0.5s static const int sleepFactorFast = 1; // 0.05s static const int sleepFactorSlow = 20; // 1s -RS_SET_CONTEXT_DEBUG_LEVEL(0) - static const rstime_t selfCheckPeroid = 30; void doSleep(useconds_t timeToSleepMS) { @@ -83,7 +80,7 @@ bool p3I2pBob::isEnabled() bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) { - std::cout << "p3I2pBob::initialSetup" << std::endl; + RS_DBG("") << std::endl; // update config { @@ -96,7 +93,7 @@ bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) } } - std::cout << "p3I2pBob::initialSetup config updated" << std::endl; + RS_DBG("config updated") << std::endl; // request keys // p3I2pBob::stateMachineBOB expects mProcessing to be set therefore @@ -106,12 +103,12 @@ bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) fakeTicket->task = autoProxyTask::receiveKey; processTaskAsync(fakeTicket); - std::cout << "p3I2pBob::initialSetup fakeTicket requested" << std::endl; + RS_DBG("fakeTicket requested") << std::endl; // now start thread start("I2P-BOB gen key"); - std::cout << "p3I2pBob::initialSetup thread started" << std::endl; + RS_DBG("thread started") << std::endl; int counter = 0; // wait for keys @@ -125,24 +122,24 @@ bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) break; if (++counter > 30) { - std::cout << "p3I2pBob::initialSetup timeout!" << std::endl; + RS_DBG4("timeout!") << std::endl; return false; } } - std::cout << "p3I2pBob::initialSetup got keys" << std::endl; + RS_DBG("got keys") << std::endl; // stop thread fullstop(); - std::cout << "p3I2pBob::initialSetup thread stopped" << std::endl; + RS_DBG("thread stopped") << std::endl; { RS_STACK_MUTEX(mLock); addr = mSetting.address.base32; } - std::cout << "p3I2pBob::initialSetup addr '" << addr << "'" << std::endl; + RS_DBG4("addr '" + addr + "'") << std::endl; return true; } @@ -160,7 +157,7 @@ void p3I2pBob::processTaskAsync(taskTicket *ticket) } break; default: - RsDbg() << __PRETTY_FUNCTION__ << " unknown task" << std::endl; + RS_DBG("unknown task") << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -175,7 +172,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::status: // check if everything needed is set if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::status data is missing" << std::endl; + RS_DBG("autoProxyTask::status data is missing") << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -189,7 +186,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::getSettings: // check if everything needed is set if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::getSettings data is missing" << std::endl; + RS_DBG("autoProxyTask::getSettings data is missing") << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -203,7 +200,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::setSettings: // check if everything needed is set if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::setSettings data is missing" << std::endl; + RS_DBG("autoProxyTask::setSettings data is missing") << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -223,7 +220,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) break; case autoProxyTask::getErrorInfo: if (!data) { - RsDbg() << __PRETTY_FUNCTION__ << " autoProxyTask::getErrorInfo data is missing" << std::endl; + RS_DBG("autoProxyTask::getErrorInfo data is missing") << std::endl; rsAutoProxyMonitor::taskError(ticket); } else { RS_STACK_MUTEX(mLock); @@ -232,7 +229,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) } break; default: - RsDbg() << __PRETTY_FUNCTION__ << " unknown task" << std::endl; + RS_DBG("unknown task") << std::endl; rsAutoProxyMonitor::taskError(ticket); break; } @@ -253,7 +250,7 @@ void p3I2pBob::threadTick() RS_STACK_MUTEX(mLock); std::stringstream ss; ss << "data_tick mState: " << mState << " mTask: " << mTask << " mBOBState: " << mBOBState << " mPending: " << mPending.size(); - Dbg4() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; + RS_DBG4("" + ss.str()) << std::endl; } sleepTime += stateMachineController(); @@ -294,13 +291,13 @@ int p3I2pBob::stateMachineBOB() while (answer.find("OK Listing done") == std::string::npos) { std::stringstream ss; ss << "stateMachineBOB status check: read loop, counter: " << counter; - Dbg3() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; + RS_DBG3("" + ss.str()) << std::endl; answer += recv(); counter++; } if (answer.find(mTunnelName) == std::string::npos) { - RsDbg() << __PRETTY_FUNCTION__ << " status check: tunnel down!" << std::endl; + RS_DBG("status check: tunnel down!") << std::endl; // signal error *((bool *)mProcessing->data) = true; } @@ -340,8 +337,8 @@ int p3I2pBob::stateMachineBOB_locked_failure(const std::string &answer, const bo return sleepFactorDefault; } - RsDbg() << __PRETTY_FUNCTION__ << " FAILED to run command '" + currentState.command + "'" << std::endl; - RsDbg() << __PRETTY_FUNCTION__ << " '" + answer + "'" << std::endl; + RS_DBG("FAILED to run command '" + currentState.command + "'") << std::endl; + RS_DBG("'" + answer + "'") << std::endl; mErrorMsg.append("FAILED to run command '" + currentState.command + "'" + '\n'); mErrorMsg.append("reason '" + answer + "'" + '\n'); @@ -388,14 +385,14 @@ int p3I2pBob::stateMachineController() return stateMachineController_locked_idle(); case csDoConnect: if (!connectI2P()) { - RsDbg() << __PRETTY_FUNCTION__ << " doConnect: unable to connect" << std::endl; + RS_DBG("doConnect: unable to connect") << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "unable to connect to BOB port"; return sleepFactorSlow; } - Dbg4() << __PRETTY_FUNCTION__ << " doConnect: connected" << std::endl; + RS_DBG4("doConnect: connected") << std::endl; mState = csConnected; break; case csConnected: @@ -403,7 +400,7 @@ int p3I2pBob::stateMachineController() case csWaitForBob: // check connection problems if (mSocket == 0) { - RsDbg() << __PRETTY_FUNCTION__ << " waitForBob: conection lost" << std::endl; + RS_DBG("waitForBob: conection lost") << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "connection lost to BOB"; @@ -413,21 +410,21 @@ int p3I2pBob::stateMachineController() // check for finished BOB protocol if (mBOBState == bsCleared) { // done - Dbg4() << __PRETTY_FUNCTION__ << " waitForBob: mBOBState == bsCleared" << std::endl; + RS_DBG4("waitForBob: mBOBState == bsCleared") << std::endl; mState = csDoDisconnect; } break; case csDoDisconnect: if (!disconnectI2P() || mSocket != 0) { // just in case - RsDbg() << __PRETTY_FUNCTION__ << " doDisconnect: can't disconnect" << std::endl; + RS_DBG("doDisconnect: can't disconnect") << std::endl; mStateOld = mState; mState = csError; mErrorMsg = "unable to disconnect from BOB"; return sleepFactorDefault; } - Dbg4() << __PRETTY_FUNCTION__ << " doDisconnect: disconnected" << std::endl; + RS_DBG4("doDisconnect: disconnected") << std::endl; mState = csDisconnected; break; case csDisconnected: @@ -458,7 +455,7 @@ int p3I2pBob::stateMachineController_locked_idle() mProcessing->task == autoProxyTask::stop || mProcessing->task == autoProxyTask::proxyStatusCheck)) { // skip since we are not enabled - Dbg1() << __PRETTY_FUNCTION__ << " disabled -> skipping ticket" << std::endl; + RS_DBG1("disabled -> skipping ticket") << std::endl; rsAutoProxyMonitor::taskDone(mProcessing, autoProxyStatus::disabled); mProcessing = NULL; } else { @@ -480,7 +477,7 @@ int p3I2pBob::stateMachineController_locked_idle() mTask = ctRunCheck; break; default: - Dbg1() << __PRETTY_FUNCTION__ << " unknown async task" << std::endl; + RS_DBG1("unknown async task") << std::endl; rsAutoProxyMonitor::taskError(mProcessing); mProcessing = NULL; break; @@ -528,28 +525,28 @@ int p3I2pBob::stateMachineController_locked_connected() case ctRunSetUp: // when we have a key use it for server tunnel! if(mSetting.address.privateKey.empty()) { - Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = setnickC" << std::endl; + RS_DBG4("setting mBOBState = setnickC") << std::endl; mBOBState = bsSetnickC; } else { - Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = setnickS" << std::endl; + RS_DBG4("setting mBOBState = setnickS") << std::endl; mBOBState = bsSetnickS; } break; case ctRunShutDown: // shut down existing tunnel - Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = getnick" << std::endl; + RS_DBG4("setting mBOBState = getnick") << std::endl; mBOBState = bsGetnick; break; case ctRunCheck: - Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = list" << std::endl; + RS_DBG4("setting mBOBState = list") << std::endl; mBOBState = bsList; break; case ctRunGetKeys: - Dbg4() << __PRETTY_FUNCTION__ << " setting mBOBState = setnickN" << std::endl; + RS_DBG4("setting mBOBState = setnickN") << std::endl; mBOBState = bsSetnickN; break; case ctIdle: - RsDbg() << __PRETTY_FUNCTION__ << " task is idle. This should not happen!" << std::endl; + RS_DBG("task is idle. This should not happen!") << std::endl; break; } @@ -565,7 +562,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() if(errorHappened) { // reset old state mStateOld = csIdel; - RsDbg() << __PRETTY_FUNCTION__ << " error during process!" << std::endl; + RS_DBG("error during process!") << std::endl; } // answer ticket @@ -594,12 +591,12 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; if (!errorHappened) { - Dbg4() << __PRETTY_FUNCTION__ << " run check result: ok" << std::endl; + RS_DBG4("run check result: ok") << std::endl; break; } // switch to error newState = csError; - RsDbg() << __PRETTY_FUNCTION__ << " run check result: error" << std::endl; + RS_DBG("run check result: error") << std::endl; mErrorMsg = "Connection check failed. Will try to restart tunnel."; break; @@ -622,7 +619,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; break; case ctIdle: - RsDbg() << __PRETTY_FUNCTION__ << " task is idle. This should not happen!" << std::endl; + RS_DBG("task is idle. This should not happen!") << std::endl; rsAutoProxyMonitor::taskError(mProcessing); } mProcessing = NULL; @@ -638,14 +635,14 @@ int p3I2pBob::stateMachineController_locked_error() { // wait for bob protocoll if (mBOBState != bsCleared) { - Dbg4() << __PRETTY_FUNCTION__ << " waiting for BOB" << std::endl; + RS_DBG4("waiting for BOB") << std::endl; return sleepFactorFast; } #if 0 std::stringstream ss; ss << "stateMachineController_locked_error: mProcessing: " << (mProcessing ? "not null" : "null"); - Dbg4() << __PRETTY_FUNCTION__ << " " + ss.str() << std::endl; + RS_DBG4("" + ss.str()) << std::endl; #endif // try to finish ticket @@ -653,7 +650,7 @@ int p3I2pBob::stateMachineController_locked_error() switch (mTask) { case ctRunCheck: // connection check failed at some point - RsDbg() << __PRETTY_FUNCTION__ << " failed to check proxy status (it's likely dead)!" << std::endl; + RS_DBG("failed to check proxy status (it's likely dead)!") << std::endl; *((bool *)mProcessing->data) = true; mState = csDoDisconnect; mStateOld = csIdel; @@ -661,7 +658,7 @@ int p3I2pBob::stateMachineController_locked_error() break; case ctRunShutDown: // not a big deal though - RsDbg() << __PRETTY_FUNCTION__ << " failed to shut down tunnel (it's likely dead though)!" << std::endl; + RS_DBG("failed to shut down tunnel (it's likely dead though)!") << std::endl; mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); @@ -669,14 +666,14 @@ int p3I2pBob::stateMachineController_locked_error() case ctIdle: // should not happen but we need to deal with it // this will produce some error messages in the log and finish the task (marked as failed) - RsDbg() << __PRETTY_FUNCTION__ << " task is idle. This should not happen!" << std::endl; + RS_DBG("task is idle. This should not happen!") << std::endl; mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); break; case ctRunGetKeys: case ctRunSetUp: - RsDbg() << __PRETTY_FUNCTION__ << " failed to receive key / start up" << std::endl; + RS_DBG("failed to receive key / start up") << std::endl; mStateOld = csError; mState = csDoDisconnect; // keep the error message @@ -687,7 +684,7 @@ int p3I2pBob::stateMachineController_locked_error() // periodically retry if (mLastProxyCheck < time(NULL) - (selfCheckPeroid >> 1) && mTask == ctRunSetUp) { - RsDbg() << __PRETTY_FUNCTION__ << " retrying" << std::endl; + RS_DBG("retrying") << std::endl; mLastProxyCheck = time(NULL); mErrorMsg.clear(); @@ -700,7 +697,7 @@ int p3I2pBob::stateMachineController_locked_error() // check for new tickets if (!mPending.empty()) { - Dbg4() << __PRETTY_FUNCTION__ << " processing new ticket" << std::endl; + RS_DBG4("processing new ticket") << std::endl; // reset and try new task mTask = ctIdle; @@ -731,7 +728,7 @@ RsSerialiser *p3I2pBob::setupSerialiser() bool p3I2pBob::saveList(bool &cleanup, std::list &lst) { - Dbg4() << __PRETTY_FUNCTION__ << std::endl; + RS_DBG4("") << std::endl; cleanup = true; RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet; @@ -766,7 +763,7 @@ bool p3I2pBob::saveList(bool &cleanup, std::list &lst) bool p3I2pBob::loadList(std::list &load) { - Dbg4() << __PRETTY_FUNCTION__ << std::endl; + RS_DBG4("") << std::endl; for(std::list::const_iterator it = load.begin(); it!=load.end(); ++it) { RsConfigKeyValueSet *vitem = dynamic_cast(*it); @@ -786,7 +783,7 @@ bool p3I2pBob::loadList(std::list &load) getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity) getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance) else - RsDbg() << __PRETTY_FUNCTION__ << " unknown key: " + kit->key << std::endl; + RS_DBG("unknown key: " + kit->key) << std::endl; } } delete vitem; @@ -850,7 +847,7 @@ void p3I2pBob::getStates(bobStates *bs) std::string p3I2pBob::executeCommand(const std::string &command) { - Dbg4() << __PRETTY_FUNCTION__ << " running '" + command + "'" << std::endl; + RS_DBG4("running '" + command + "'") << std::endl; std::string copy = command; copy.push_back('\n'); @@ -862,7 +859,7 @@ std::string p3I2pBob::executeCommand(const std::string &command) // receive answer (trailing new line is already removed!) std::string ans = recv(); - Dbg4() << __PRETTY_FUNCTION__ << " answer '" + ans + "'" << std::endl; + RS_DBG4("answer '" + ans + "'") << std::endl; return ans; } @@ -872,7 +869,7 @@ bool p3I2pBob::connectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket != 0) { - RsDbg() << __PRETTY_FUNCTION__ << " mSocket != 0" << std::endl; + RS_DBG("mSocket != 0") << std::endl; return false; } @@ -880,21 +877,21 @@ bool p3I2pBob::connectI2P() mSocket = unix_socket(PF_INET, SOCK_STREAM, 0); if (mSocket < 0) { - RsDbg() << __PRETTY_FUNCTION__ << " Failed to open socket! Socket Error: " + socket_errorType(errno) << std::endl; + RS_DBG("Failed to open socket! Socket Error: " + socket_errorType(errno)) << std::endl; return false; } // connect int err = unix_connect(mSocket, mI2PProxyAddr); if (err != 0) { - RsDbg() << __PRETTY_FUNCTION__ << " Failed to connect to BOB! Socket Error: " + socket_errorType(errno) << std::endl; + RS_DBG("Failed to connect to BOB! Socket Error: " + socket_errorType(errno)) << std::endl; return false; } // receive hello msg recv(); - Dbg4() << __PRETTY_FUNCTION__ << " done" << std::endl; + RS_DBG4("done") << std::endl; return true; } @@ -903,17 +900,17 @@ bool p3I2pBob::disconnectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket == 0) { - RsDbg() << __PRETTY_FUNCTION__ << " mSocket == 0" << std::endl; + RS_DBG("mSocket == 0") << std::endl; return true; } int err = unix_close(mSocket); if (err != 0) { - RsDbg() << __PRETTY_FUNCTION__ << " Failed to close socket! Socket Error: " + socket_errorType(errno) << std::endl; + RS_DBG("Failed to close socket! Socket Error: " + socket_errorType(errno)) << std::endl; return false; } - Dbg4() << __PRETTY_FUNCTION__ << " done" << std::endl; + RS_DBG4("done") << std::endl; mSocket = 0; return true; } @@ -934,7 +931,7 @@ std::string toString(const std::string &a, const int8_t b) { void p3I2pBob::finalizeSettings_locked() { - Dbg4() << __PRETTY_FUNCTION__ << std::endl; + RS_DBG4("") << std::endl; sockaddr_storage_clear(mI2PProxyAddr); // get i2p proxy addr @@ -945,8 +942,8 @@ void p3I2pBob::finalizeSettings_locked() sockaddr_storage_setipv4(mI2PProxyAddr, (sockaddr_in*)&proxy); sockaddr_storage_setport(mI2PProxyAddr, 2827); - Dbg4() << __PRETTY_FUNCTION__ << " using " + sockaddr_storage_tostring(mI2PProxyAddr) << std::endl; - Dbg4() << __PRETTY_FUNCTION__ << " using " + mSetting.address.base32 << std::endl; + RS_DBG4("using " + sockaddr_storage_tostring(mI2PProxyAddr)) << std::endl; + RS_DBG4("using " + mSetting.address.base32) << std::endl; peerState ps; mPeerMgr->getOwnNetStatus(ps); @@ -961,7 +958,7 @@ void p3I2pBob::finalizeSettings_locked() std::vector tmp(len); RSRandom::random_bytes(tmp.data(), len); const std::string location = Radix32::encode(tmp.data(), len); - Dbg4() << __PRETTY_FUNCTION__ << " using suffix " + location << std::endl; + RS_DBG4("using suffix " + location) << std::endl; mTunnelName = "RetroShare-" + location; const std::string setnick = "setnick RetroShare-" + location; @@ -1029,7 +1026,7 @@ void p3I2pBob::finalizeSettings_locked() void p3I2pBob::updateSettings_locked() { - Dbg4() << __PRETTY_FUNCTION__ << std::endl; + RS_DBG4("") << std::endl; sockaddr_storage proxy; mPeerMgr->getProxyServerAddress(RS_HIDDEN_TYPE_I2P, proxy); @@ -1105,7 +1102,7 @@ std::string p3I2pBob::recv() // sanity check if (length != bufferSize) { // expectation: a full buffer was peeked) - Dbg1() << __PRETTY_FUNCTION__ << " peeked less than bufferSize but also didn't found a new line character" << std::endl; + RS_DBG1("peeked less than bufferSize but also didn't found a new line character") << std::endl; } // this should never happen assert(length <= bufferSize); From 008a4b87b30a04586fc8d5c4e2c12c9e9d10df4d Mon Sep 17 00:00:00 2001 From: sehraf Date: Wed, 10 Jun 2020 21:46:59 +0200 Subject: [PATCH 61/72] replace std::map with std::array --- libretroshare/src/util/i2pcommon.cpp | 8 +++---- libretroshare/src/util/i2pcommon.h | 36 +++++++++++++++------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp index e50fc9d83..46064ee83 100644 --- a/libretroshare/src/util/i2pcommon.cpp +++ b/libretroshare/src/util/i2pcommon.cpp @@ -133,20 +133,20 @@ std::string publicKeyFromPrivate(std::string const &priv) RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " has oversize" << std::endl; // calculate oversize - auto it = signingKeyLengths.find(static_cast(signingKeyType)); - if (it == signingKeyLengths.end()) { + if (signingKeyType >= signingKeyLengths.size()) { // just in case RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " cannot be found in size map!" << std::endl; return std::string(); } - if (it->second.first <= 128) { + auto values = signingKeyLengths[signingKeyType]; + if (values.first <= 128) { // just in case, it's supposed to be larger! RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " is oversize but size calculation would underflow!" << std::endl; return std::string(); } - publicKeyLen += it->second.first - 128; // 128 = default DSA key length = the space than can be used before the key must be splitted + publicKeyLen += values.first - 128; // 128 = default DSA key length = the space than can be used before the key must be splitted } // Crypto Public Key diff --git a/libretroshare/src/util/i2pcommon.h b/libretroshare/src/util/i2pcommon.h index 98815496b..63f1ba5af 100644 --- a/libretroshare/src/util/i2pcommon.h +++ b/libretroshare/src/util/i2pcommon.h @@ -161,25 +161,27 @@ enum class CryptoKeyType : uint16_t { X25519 = 4 }; -static const std::map> cryptoKeyLengths { - {CryptoKeyType::ElGamal, {256, 256}}, - {CryptoKeyType::P256, { 64, 32}}, - {CryptoKeyType::P384, { 96, 48}}, - {CryptoKeyType::P521, {132, 66}}, - {CryptoKeyType::X25519, { 32, 32}}, +static const std::array, 5> cryptoKeyLengths { + /*CryptoKeyType::ElGamal*/ std::make_pair(256, 256), + /*CryptoKeyType::P256, */ std::make_pair( 64, 32), + /*CryptoKeyType::P384, */ std::make_pair( 96, 48), + /*CryptoKeyType::P521, */ std::make_pair(132, 66), + /*CryptoKeyType::X25519,*/ std::make_pair( 32, 32), }; -static const std::map> signingKeyLengths { - {SigningKeyType::DSA_SHA1, {128, 128}}, - {SigningKeyType::ECDSA_SHA256_P256, { 64, 32}}, - {SigningKeyType::ECDSA_SHA384_P384, { 96, 48}}, - {SigningKeyType::ECDSA_SHA512_P521, {132, 66}}, - {SigningKeyType::RSA_SHA256_2048, {256, 512}}, - {SigningKeyType::RSA_SHA384_3072, {384, 768}}, - {SigningKeyType::RSA_SHA512_4096, {512,1024}}, - {SigningKeyType::EdDSA_SHA512_Ed25519, { 32, 32}}, - {SigningKeyType::EdDSA_SHA512_Ed25519ph,{ 32, 32}}, - {SigningKeyType::RedDSA_SHA512_Ed25519, { 32, 32}}, +static const std::array, 12> signingKeyLengths { + /*SigningKeyType::DSA_SHA1, */ std::make_pair(128, 128), + /*SigningKeyType::ECDSA_SHA256_P256, */ std::make_pair( 64, 32), + /*SigningKeyType::ECDSA_SHA384_P384, */ std::make_pair( 96, 48), + /*SigningKeyType::ECDSA_SHA512_P521, */ std::make_pair(132, 66), + /*SigningKeyType::RSA_SHA256_2048, */ std::make_pair(256, 512), + /*SigningKeyType::RSA_SHA384_3072, */ std::make_pair(384, 768), + /*SigningKeyType::RSA_SHA512_4096, */ std::make_pair(512,1024), + /*SigningKeyType::EdDSA_SHA512_Ed25519 */ std::make_pair( 32, 32), + /*SigningKeyType::EdDSA_SHA512_Ed25519ph */ std::make_pair( 32, 32), + /*reserved (GOST) */ std::make_pair( 64, 0), + /*reserved (GOST) */ std::make_pair(128, 0), + /*SigningKeyType::RedDSA_SHA512_Ed25519 */ std::make_pair( 32, 32), }; From 7207e6a2c15837ca22dc8e4ae8f34f72053f74f5 Mon Sep 17 00:00:00 2001 From: sehraf Date: Wed, 10 Jun 2020 21:59:45 +0200 Subject: [PATCH 62/72] use RS_DBG --- libretroshare/src/util/i2pcommon.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp index 46064ee83..86c83c367 100644 --- a/libretroshare/src/util/i2pcommon.cpp +++ b/libretroshare/src/util/i2pcommon.cpp @@ -73,7 +73,7 @@ std::string publicKeyFromPrivate(std::string const &priv) RsBase64::decode(priv_copy, dataPriv); auto p = dataPriv.cbegin(); - RsDbg() << __PRETTY_FUNCTION__ << " dataPriv.size " << dataPriv.size() << std::endl; + RS_DBG("dataPriv.size ") << dataPriv.size() << std::endl; size_t publicKeyLen = 256 + 128; // default length (bytes) uint8_t certType = 0; @@ -98,12 +98,12 @@ std::string publicKeyFromPrivate(std::string const &priv) * type null is followed by 0x00 0x00 * so has to be 0! */ - RsDbg() << __PRETTY_FUNCTION__ << " cert is CertType.Null" << std::endl; + RS_DBG("cert is CertType.Null"); publicKeyLen += 3; // add 0x00 0x00 0x00 if (len != 0) // weird - RsDbg() << __PRETTY_FUNCTION__ << " cert is CertType.Null but len != 0" << std::endl; + RS_DBG("cert is CertType.Null but len != 0"); break; } @@ -111,11 +111,11 @@ std::string publicKeyFromPrivate(std::string const &priv) // check for != 5 if (certType != static_cast::type>(CertType::Key)) { // unsupported - RsDbg() << __PRETTY_FUNCTION__ << " cert type " << certType << " is unsupported" << std::endl; + RS_DBG("cert type ") << certType << " is unsupported" << std::endl; return std::string(); } - RsDbg() << __PRETTY_FUNCTION__ << " cert is CertType.Key" << std::endl; + RS_DBG("cert is CertType.Key"); publicKeyLen += 7; // = 1 + 2 + 2 + 2 = 7 bytes /* @@ -128,21 +128,21 @@ std::string publicKeyFromPrivate(std::string const &priv) // likely 7 signingKeyType = readTwoBytesBE(p); - RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << std::endl; + RS_DBG("signing pubkey type ") << certType << std::endl; if (signingKeyType >= 3 && signingKeyType <= 6) { - RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " has oversize" << std::endl; + RS_DBG("signing pubkey type ") << certType << " has oversize" << std::endl; // calculate oversize if (signingKeyType >= signingKeyLengths.size()) { // just in case - RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " cannot be found in size map!" << std::endl; + RS_DBG("signing pubkey type ") << certType << " cannot be found in size map!" << std::endl; return std::string(); } auto values = signingKeyLengths[signingKeyType]; if (values.first <= 128) { // just in case, it's supposed to be larger! - RsDbg() << __PRETTY_FUNCTION__ << " signing pubkey type " << certType << " is oversize but size calculation would underflow!" << std::endl; + RS_DBG("signing pubkey type ") << certType << " is oversize but size calculation would underflow!" << std::endl; return std::string(); } @@ -152,12 +152,12 @@ std::string publicKeyFromPrivate(std::string const &priv) // Crypto Public Key // likely 0 cryptKey = readTwoBytesBE(p); - RsDbg() << __PRETTY_FUNCTION__ << " crypto pubkey type " << cryptKey << std::endl; + RS_DBG("crypto pubkey type ") << cryptKey << std::endl; // info: these are all smaller than the default 256 bytes, so no oversize calculation is needed break; } catch (const std::out_of_range &e) { - RsDbg() << __PRETTY_FUNCTION__ << " hit exception!" << e.what() << std::endl; + RS_DBG("hit exception! ") << e.what() << std::endl; return std::string(); } } while(false); From 86c30a01dd5fb20e1474ec6a859ee72d8306cc40 Mon Sep 17 00:00:00 2001 From: sehraf Date: Thu, 11 Jun 2020 10:45:33 +0200 Subject: [PATCH 63/72] next attampt to use RS_DBG correctly --- .../src/services/autoproxy/p3i2pbob.cc | 136 +++++++++--------- libretroshare/src/util/i2pcommon.cpp | 16 +-- 2 files changed, 74 insertions(+), 78 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index c1b32fa76..02b951397 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -33,6 +33,8 @@ #include "util/rsprint.h" #include "util/rsrandom.h" +#include "util/rsdebuglevel4.h" + static const std::string kConfigKeyBOBEnable = "BOB_ENABLE"; static const std::string kConfigKeyBOBKey = "BOB_KEY"; static const std::string kConfigKeyBOBAddr = "BOB_ADDR"; @@ -80,7 +82,7 @@ bool p3I2pBob::isEnabled() bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) { - RS_DBG("") << std::endl; + RS_DBG(""); // update config { @@ -93,7 +95,7 @@ bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) } } - RS_DBG("config updated") << std::endl; + RS_DBG("config updated"); // request keys // p3I2pBob::stateMachineBOB expects mProcessing to be set therefore @@ -103,12 +105,12 @@ bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) fakeTicket->task = autoProxyTask::receiveKey; processTaskAsync(fakeTicket); - RS_DBG("fakeTicket requested") << std::endl; + RS_DBG("fakeTicket requested"); // now start thread start("I2P-BOB gen key"); - RS_DBG("thread started") << std::endl; + RS_DBG("thread started"); int counter = 0; // wait for keys @@ -122,24 +124,24 @@ bool p3I2pBob::initialSetup(std::string &addr, uint16_t &/*port*/) break; if (++counter > 30) { - RS_DBG4("timeout!") << std::endl; + RS_DBG4("timeout!"); return false; } } - RS_DBG("got keys") << std::endl; + RS_DBG("got keys"); // stop thread fullstop(); - RS_DBG("thread stopped") << std::endl; + RS_DBG("thread stopped"); { RS_STACK_MUTEX(mLock); addr = mSetting.address.base32; } - RS_DBG4("addr '" + addr + "'") << std::endl; + RS_DBG4("addr ", addr); return true; } @@ -157,7 +159,7 @@ void p3I2pBob::processTaskAsync(taskTicket *ticket) } break; default: - RS_DBG("unknown task") << std::endl; + RS_DBG("unknown task"); rsAutoProxyMonitor::taskError(ticket); break; } @@ -172,7 +174,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::status: // check if everything needed is set if (!data) { - RS_DBG("autoProxyTask::status data is missing") << std::endl; + RS_DBG("autoProxyTask::status data is missing"); rsAutoProxyMonitor::taskError(ticket); break; } @@ -186,7 +188,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::getSettings: // check if everything needed is set if (!data) { - RS_DBG("autoProxyTask::getSettings data is missing") << std::endl; + RS_DBG("autoProxyTask::getSettings data is missing"); rsAutoProxyMonitor::taskError(ticket); break; } @@ -200,7 +202,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) case autoProxyTask::setSettings: // check if everything needed is set if (!data) { - RS_DBG("autoProxyTask::setSettings data is missing") << std::endl; + RS_DBG("autoProxyTask::setSettings data is missing"); rsAutoProxyMonitor::taskError(ticket); break; } @@ -220,7 +222,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) break; case autoProxyTask::getErrorInfo: if (!data) { - RS_DBG("autoProxyTask::getErrorInfo data is missing") << std::endl; + RS_DBG("autoProxyTask::getErrorInfo data is missing"); rsAutoProxyMonitor::taskError(ticket); } else { RS_STACK_MUTEX(mLock); @@ -229,7 +231,7 @@ void p3I2pBob::processTaskSync(taskTicket *ticket) } break; default: - RS_DBG("unknown task") << std::endl; + RS_DBG("unknown task"); rsAutoProxyMonitor::taskError(ticket); break; } @@ -247,10 +249,8 @@ void p3I2pBob::threadTick() { int sleepTime = 0; { - RS_STACK_MUTEX(mLock); - std::stringstream ss; - ss << "data_tick mState: " << mState << " mTask: " << mTask << " mBOBState: " << mBOBState << " mPending: " << mPending.size(); - RS_DBG4("" + ss.str()) << std::endl; + RS_STACK_MUTEX(mLock); + RS_DBG4("data_tick mState: ", mState, " mTask: ", mTask, " mBOBState: ", mBOBState, " mPending: ", mPending.size()); } sleepTime += stateMachineController(); @@ -289,15 +289,13 @@ int p3I2pBob::stateMachineBOB() if (mBOBState == bsList) { int counter = 0; while (answer.find("OK Listing done") == std::string::npos) { - std::stringstream ss; - ss << "stateMachineBOB status check: read loop, counter: " << counter; - RS_DBG3("" + ss.str()) << std::endl; + RS_DBG3("stateMachineBOB status check: read loop, counter: ", counter); answer += recv(); counter++; } if (answer.find(mTunnelName) == std::string::npos) { - RS_DBG("status check: tunnel down!") << std::endl; + RS_DBG("status check: tunnel down!"); // signal error *((bool *)mProcessing->data) = true; } @@ -337,8 +335,8 @@ int p3I2pBob::stateMachineBOB_locked_failure(const std::string &answer, const bo return sleepFactorDefault; } - RS_DBG("FAILED to run command '" + currentState.command + "'") << std::endl; - RS_DBG("'" + answer + "'") << std::endl; + RS_DBG("FAILED to run command: ", currentState.command); + RS_DBG("answer: ", answer); mErrorMsg.append("FAILED to run command '" + currentState.command + "'" + '\n'); mErrorMsg.append("reason '" + answer + "'" + '\n'); @@ -385,14 +383,14 @@ int p3I2pBob::stateMachineController() return stateMachineController_locked_idle(); case csDoConnect: if (!connectI2P()) { - RS_DBG("doConnect: unable to connect") << std::endl; + RS_DBG("doConnect: unable to connect"); mStateOld = mState; mState = csError; mErrorMsg = "unable to connect to BOB port"; return sleepFactorSlow; } - RS_DBG4("doConnect: connected") << std::endl; + RS_DBG4("doConnect: connected"); mState = csConnected; break; case csConnected: @@ -400,7 +398,7 @@ int p3I2pBob::stateMachineController() case csWaitForBob: // check connection problems if (mSocket == 0) { - RS_DBG("waitForBob: conection lost") << std::endl; + RS_DBG("waitForBob: conection lost"); mStateOld = mState; mState = csError; mErrorMsg = "connection lost to BOB"; @@ -410,21 +408,21 @@ int p3I2pBob::stateMachineController() // check for finished BOB protocol if (mBOBState == bsCleared) { // done - RS_DBG4("waitForBob: mBOBState == bsCleared") << std::endl; + RS_DBG4("waitForBob: mBOBState == bsCleared"); mState = csDoDisconnect; } break; case csDoDisconnect: if (!disconnectI2P() || mSocket != 0) { // just in case - RS_DBG("doDisconnect: can't disconnect") << std::endl; + RS_DBG("doDisconnect: can't disconnect"); mStateOld = mState; mState = csError; mErrorMsg = "unable to disconnect from BOB"; return sleepFactorDefault; } - RS_DBG4("doDisconnect: disconnected") << std::endl; + RS_DBG4("doDisconnect: disconnected"); mState = csDisconnected; break; case csDisconnected: @@ -455,7 +453,7 @@ int p3I2pBob::stateMachineController_locked_idle() mProcessing->task == autoProxyTask::stop || mProcessing->task == autoProxyTask::proxyStatusCheck)) { // skip since we are not enabled - RS_DBG1("disabled -> skipping ticket") << std::endl; + RS_DBG1("disabled -> skipping ticket"); rsAutoProxyMonitor::taskDone(mProcessing, autoProxyStatus::disabled); mProcessing = NULL; } else { @@ -477,7 +475,7 @@ int p3I2pBob::stateMachineController_locked_idle() mTask = ctRunCheck; break; default: - RS_DBG1("unknown async task") << std::endl; + RS_DBG1("unknown async task"); rsAutoProxyMonitor::taskError(mProcessing); mProcessing = NULL; break; @@ -525,28 +523,28 @@ int p3I2pBob::stateMachineController_locked_connected() case ctRunSetUp: // when we have a key use it for server tunnel! if(mSetting.address.privateKey.empty()) { - RS_DBG4("setting mBOBState = setnickC") << std::endl; + RS_DBG4("setting mBOBState = setnickC"); mBOBState = bsSetnickC; } else { - RS_DBG4("setting mBOBState = setnickS") << std::endl; + RS_DBG4("setting mBOBState = setnickS"); mBOBState = bsSetnickS; } break; case ctRunShutDown: // shut down existing tunnel - RS_DBG4("setting mBOBState = getnick") << std::endl; + RS_DBG4("setting mBOBState = getnick"); mBOBState = bsGetnick; break; case ctRunCheck: - RS_DBG4("setting mBOBState = list") << std::endl; + RS_DBG4("setting mBOBState = list"); mBOBState = bsList; break; case ctRunGetKeys: - RS_DBG4("setting mBOBState = setnickN") << std::endl; + RS_DBG4("setting mBOBState = setnickN"); mBOBState = bsSetnickN; break; case ctIdle: - RS_DBG("task is idle. This should not happen!") << std::endl; + RS_DBG("task is idle. This should not happen!"); break; } @@ -562,7 +560,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() if(errorHappened) { // reset old state mStateOld = csIdel; - RS_DBG("error during process!") << std::endl; + RS_DBG("error during process!"); } // answer ticket @@ -591,12 +589,12 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; if (!errorHappened) { - RS_DBG4("run check result: ok") << std::endl; + RS_DBG4("run check result: ok"); break; } // switch to error newState = csError; - RS_DBG("run check result: error") << std::endl; + RS_DBG("run check result: error"); mErrorMsg = "Connection check failed. Will try to restart tunnel."; break; @@ -619,7 +617,7 @@ int p3I2pBob::stateMachineController_locked_disconnected() mTask = mTaskOld; break; case ctIdle: - RS_DBG("task is idle. This should not happen!") << std::endl; + RS_DBG("task is idle. This should not happen!"); rsAutoProxyMonitor::taskError(mProcessing); } mProcessing = NULL; @@ -635,14 +633,12 @@ int p3I2pBob::stateMachineController_locked_error() { // wait for bob protocoll if (mBOBState != bsCleared) { - RS_DBG4("waiting for BOB") << std::endl; + RS_DBG4("waiting for BOB"); return sleepFactorFast; } #if 0 - std::stringstream ss; - ss << "stateMachineController_locked_error: mProcessing: " << (mProcessing ? "not null" : "null"); - RS_DBG4("" + ss.str()) << std::endl; + RS_DBG4("stateMachineController_locked_error: mProcessing: ", (mProcessing ? "not null" : "null")); #endif // try to finish ticket @@ -650,7 +646,7 @@ int p3I2pBob::stateMachineController_locked_error() switch (mTask) { case ctRunCheck: // connection check failed at some point - RS_DBG("failed to check proxy status (it's likely dead)!") << std::endl; + RS_DBG("failed to check proxy status (it's likely dead)!"); *((bool *)mProcessing->data) = true; mState = csDoDisconnect; mStateOld = csIdel; @@ -658,7 +654,7 @@ int p3I2pBob::stateMachineController_locked_error() break; case ctRunShutDown: // not a big deal though - RS_DBG("failed to shut down tunnel (it's likely dead though)!") << std::endl; + RS_DBG("failed to shut down tunnel (it's likely dead though)!"); mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); @@ -666,14 +662,14 @@ int p3I2pBob::stateMachineController_locked_error() case ctIdle: // should not happen but we need to deal with it // this will produce some error messages in the log and finish the task (marked as failed) - RS_DBG("task is idle. This should not happen!") << std::endl; + RS_DBG("task is idle. This should not happen!"); mState = csDoDisconnect; mStateOld = csIdel; mErrorMsg.clear(); break; case ctRunGetKeys: case ctRunSetUp: - RS_DBG("failed to receive key / start up") << std::endl; + RS_DBG("failed to receive key / start up"); mStateOld = csError; mState = csDoDisconnect; // keep the error message @@ -684,7 +680,7 @@ int p3I2pBob::stateMachineController_locked_error() // periodically retry if (mLastProxyCheck < time(NULL) - (selfCheckPeroid >> 1) && mTask == ctRunSetUp) { - RS_DBG("retrying") << std::endl; + RS_DBG("retrying"); mLastProxyCheck = time(NULL); mErrorMsg.clear(); @@ -697,7 +693,7 @@ int p3I2pBob::stateMachineController_locked_error() // check for new tickets if (!mPending.empty()) { - RS_DBG4("processing new ticket") << std::endl; + RS_DBG4("processing new ticket"); // reset and try new task mTask = ctIdle; @@ -728,7 +724,7 @@ RsSerialiser *p3I2pBob::setupSerialiser() bool p3I2pBob::saveList(bool &cleanup, std::list &lst) { - RS_DBG4("") << std::endl; + RS_DBG4(""); cleanup = true; RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet; @@ -763,7 +759,7 @@ bool p3I2pBob::saveList(bool &cleanup, std::list &lst) bool p3I2pBob::loadList(std::list &load) { - RS_DBG4("") << std::endl; + RS_DBG4(""); for(std::list::const_iterator it = load.begin(); it!=load.end(); ++it) { RsConfigKeyValueSet *vitem = dynamic_cast(*it); @@ -783,7 +779,7 @@ bool p3I2pBob::loadList(std::list &load) getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity) getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance) else - RS_DBG("unknown key: " + kit->key) << std::endl; + RS_DBG("unknown key: ", kit->key); } } delete vitem; @@ -847,7 +843,7 @@ void p3I2pBob::getStates(bobStates *bs) std::string p3I2pBob::executeCommand(const std::string &command) { - RS_DBG4("running '" + command + "'") << std::endl; + RS_DBG4("running: ", command); std::string copy = command; copy.push_back('\n'); @@ -859,7 +855,7 @@ std::string p3I2pBob::executeCommand(const std::string &command) // receive answer (trailing new line is already removed!) std::string ans = recv(); - RS_DBG4("answer '" + ans + "'") << std::endl; + RS_DBG4("answer: ", ans); return ans; } @@ -869,7 +865,7 @@ bool p3I2pBob::connectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket != 0) { - RS_DBG("mSocket != 0") << std::endl; + RS_DBG("mSocket != 0"); return false; } @@ -877,21 +873,21 @@ bool p3I2pBob::connectI2P() mSocket = unix_socket(PF_INET, SOCK_STREAM, 0); if (mSocket < 0) { - RS_DBG("Failed to open socket! Socket Error: " + socket_errorType(errno)) << std::endl; + RS_DBG("Failed to open socket! Socket Error: ", socket_errorType(errno)); return false; } // connect int err = unix_connect(mSocket, mI2PProxyAddr); if (err != 0) { - RS_DBG("Failed to connect to BOB! Socket Error: " + socket_errorType(errno)) << std::endl; + RS_DBG("Failed to connect to BOB! Socket Error: ", socket_errorType(errno)); return false; } // receive hello msg recv(); - RS_DBG4("done") << std::endl; + RS_DBG4("done"); return true; } @@ -900,17 +896,17 @@ bool p3I2pBob::disconnectI2P() // there is only one thread that touches mSocket - no need for a lock if (mSocket == 0) { - RS_DBG("mSocket == 0") << std::endl; + RS_DBG("mSocket == 0"); return true; } int err = unix_close(mSocket); if (err != 0) { - RS_DBG("Failed to close socket! Socket Error: " + socket_errorType(errno)) << std::endl; + RS_DBG("Failed to close socket! Socket Error: ", socket_errorType(errno)); return false; } - RS_DBG4("done") << std::endl; + RS_DBG4("done"); mSocket = 0; return true; } @@ -931,7 +927,7 @@ std::string toString(const std::string &a, const int8_t b) { void p3I2pBob::finalizeSettings_locked() { - RS_DBG4("") << std::endl; + RS_DBG4(""); sockaddr_storage_clear(mI2PProxyAddr); // get i2p proxy addr @@ -942,8 +938,8 @@ void p3I2pBob::finalizeSettings_locked() sockaddr_storage_setipv4(mI2PProxyAddr, (sockaddr_in*)&proxy); sockaddr_storage_setport(mI2PProxyAddr, 2827); - RS_DBG4("using " + sockaddr_storage_tostring(mI2PProxyAddr)) << std::endl; - RS_DBG4("using " + mSetting.address.base32) << std::endl; + RS_DBG4("using ", sockaddr_storage_tostring(mI2PProxyAddr)); + RS_DBG4("using ", mSetting.address.base32); peerState ps; mPeerMgr->getOwnNetStatus(ps); @@ -958,7 +954,7 @@ void p3I2pBob::finalizeSettings_locked() std::vector tmp(len); RSRandom::random_bytes(tmp.data(), len); const std::string location = Radix32::encode(tmp.data(), len); - RS_DBG4("using suffix " + location) << std::endl; + RS_DBG4("using suffix ", location); mTunnelName = "RetroShare-" + location; const std::string setnick = "setnick RetroShare-" + location; @@ -1026,7 +1022,7 @@ void p3I2pBob::finalizeSettings_locked() void p3I2pBob::updateSettings_locked() { - RS_DBG4("") << std::endl; + RS_DBG4(""); sockaddr_storage proxy; mPeerMgr->getProxyServerAddress(RS_HIDDEN_TYPE_I2P, proxy); @@ -1102,7 +1098,7 @@ std::string p3I2pBob::recv() // sanity check if (length != bufferSize) { // expectation: a full buffer was peeked) - RS_DBG1("peeked less than bufferSize but also didn't found a new line character") << std::endl; + RS_DBG1("peeked less than bufferSize but also didn't found a new line character"); } // this should never happen assert(length <= bufferSize); diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp index 86c83c367..eedb9596d 100644 --- a/libretroshare/src/util/i2pcommon.cpp +++ b/libretroshare/src/util/i2pcommon.cpp @@ -73,7 +73,7 @@ std::string publicKeyFromPrivate(std::string const &priv) RsBase64::decode(priv_copy, dataPriv); auto p = dataPriv.cbegin(); - RS_DBG("dataPriv.size ") << dataPriv.size() << std::endl; + RS_DBG("dataPriv.size ", dataPriv.size()); size_t publicKeyLen = 256 + 128; // default length (bytes) uint8_t certType = 0; @@ -111,7 +111,7 @@ std::string publicKeyFromPrivate(std::string const &priv) // check for != 5 if (certType != static_cast::type>(CertType::Key)) { // unsupported - RS_DBG("cert type ") << certType << " is unsupported" << std::endl; + RS_DBG("cert type ", certType, " is unsupported"); return std::string(); } @@ -128,21 +128,21 @@ std::string publicKeyFromPrivate(std::string const &priv) // likely 7 signingKeyType = readTwoBytesBE(p); - RS_DBG("signing pubkey type ") << certType << std::endl; + RS_DBG("signing pubkey type ", certType); if (signingKeyType >= 3 && signingKeyType <= 6) { - RS_DBG("signing pubkey type ") << certType << " has oversize" << std::endl; + RS_DBG("signing pubkey type ", certType, " has oversize"); // calculate oversize if (signingKeyType >= signingKeyLengths.size()) { // just in case - RS_DBG("signing pubkey type ") << certType << " cannot be found in size map!" << std::endl; + RS_DBG("signing pubkey type ", certType, " cannot be found in size data!"); return std::string(); } auto values = signingKeyLengths[signingKeyType]; if (values.first <= 128) { // just in case, it's supposed to be larger! - RS_DBG("signing pubkey type ") << certType << " is oversize but size calculation would underflow!" << std::endl; + RS_DBG("signing pubkey type ", certType, " is oversize but size calculation would underflow!"); return std::string(); } @@ -152,12 +152,12 @@ std::string publicKeyFromPrivate(std::string const &priv) // Crypto Public Key // likely 0 cryptKey = readTwoBytesBE(p); - RS_DBG("crypto pubkey type ") << cryptKey << std::endl; + RS_DBG("crypto pubkey type ", cryptKey); // info: these are all smaller than the default 256 bytes, so no oversize calculation is needed break; } catch (const std::out_of_range &e) { - RS_DBG("hit exception! ") << e.what() << std::endl; + RS_DBG("hit exception! ", e.what()); return std::string(); } } while(false); From 3d784e88714d5282f8f0fe01ee890ae0e0f3c152 Mon Sep 17 00:00:00 2001 From: sehraf Date: Thu, 11 Jun 2020 11:03:08 +0200 Subject: [PATCH 64/72] remove unnecessary std::replace --- libretroshare/src/services/autoproxy/p3i2pbob.cc | 2 -- libretroshare/src/util/i2pcommon.cpp | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index 02b951397..7fd7594b9 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -33,8 +33,6 @@ #include "util/rsprint.h" #include "util/rsrandom.h" -#include "util/rsdebuglevel4.h" - static const std::string kConfigKeyBOBEnable = "BOB_ENABLE"; static const std::string kConfigKeyBOBKey = "BOB_KEY"; static const std::string kConfigKeyBOBAddr = "BOB_ADDR"; diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp index eedb9596d..8c6f2238b 100644 --- a/libretroshare/src/util/i2pcommon.cpp +++ b/libretroshare/src/util/i2pcommon.cpp @@ -19,7 +19,8 @@ std::string keyToBase32Addr(const std::string &key) // replace I2P specific chars std::replace(copy.begin(), copy.end(), '~', '/'); - std::replace(copy.begin(), copy.end(), '-', '+'); + // replacing the - with a + is not necessary, as RsBase64 can handle base64url encoding, too + // std::replace(copy.begin(), copy.end(), '-', '+'); // decode std::vector bin; From 591d2ad8646758465df380bdc9e0c01de97b094f Mon Sep 17 00:00:00 2001 From: sehraf Date: Thu, 11 Jun 2020 11:15:09 +0200 Subject: [PATCH 65/72] autoproxy: use new RS_DBG --- .../services/autoproxy/rsautoproxymonitor.cc | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc index 62b47dfea..3c52cfdc5 100644 --- a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc +++ b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc @@ -44,7 +44,7 @@ void rsAutoProxyMonitor::addProxy(autoProxyType::autoProxyType_enum type, autoPr { RS_STACK_MUTEX(mLock); if (mProxies.find(type) != mProxies.end()) { - RsErr() << __PRETTY_FUNCTION__<< " type " << type << " already added - OVERWRITING" << std::endl; + RS_ERR("type ", type, " already added - OVERWRITING"); print_stacktrace(); } @@ -120,7 +120,7 @@ void rsAutoProxyMonitor::stopAllRSShutdown() do { rstime::rs_usleep(1000 * 1000); RS_STACK_MUTEX(mLock); - RsDbg() << __PRETTY_FUNCTION__<< " waiting for auto proxy service(s) to shut down " << t << "/" << timeout << " (remaining: " << mProxies.size() << ")" << std::endl; + RS_DBG("waiting for auto proxy service(s) to shut down ", t, "/", timeout, " (remaining: ", mProxies.size(), ")"); if (mProxies.empty()) break; t++; @@ -149,15 +149,15 @@ void rsAutoProxyMonitor::task(taskTicket *ticket) { // sanity checks if (!ticket->async && ticket->types.size() > 1) { - RsErr() << __PRETTY_FUNCTION__<< " synchronous call to multiple services. This can cause problems!" << std::endl; + RS_ERR("synchronous call to multiple services. This can cause problems!"); print_stacktrace(); } if (ticket->async && !ticket->cb && ticket->data) { - RsErr() << __PRETTY_FUNCTION__<< " asynchronous call with data but no callback. This will likely causes memory leak!" << std::endl; + RS_ERR("asynchronous call with data but no callback. This will likely causes memory leak!"); print_stacktrace(); } if (ticket->types.size() > 1 && ticket->data) { - RsErr() << __PRETTY_FUNCTION__<< " call with data to multiple services. This will likely causes memory leak!" << std::endl; + RS_ERR("call with data to multiple services. This will likely causes memory leak!"); print_stacktrace(); } @@ -197,7 +197,7 @@ void rsAutoProxyMonitor::taskAsync(std::vector if (isAsyncTask(task)) { // Usually the services will reject this ticket. // Just print a warning - maybe there is some special case where this is a good idea. - RsErr() << __PRETTY_FUNCTION__<< " called with an asynchronous task!" << std::endl; + RS_ERR("called with an asynchronous task!"); print_stacktrace(); } @@ -256,7 +256,7 @@ void rsAutoProxyMonitor::taskDone(taskTicket *t, autoProxyStatus::autoProxyStatu t->cb->taskFinished(t); if (t != NULL) { // callack did not clean up properly - RsErr() << __PRETTY_FUNCTION__<< " callback did not clean up!" << std::endl; + RS_ERR("callback did not clean up!"); print_stacktrace(); cleanUp = true; } @@ -265,12 +265,12 @@ void rsAutoProxyMonitor::taskDone(taskTicket *t, autoProxyStatus::autoProxyStatu // we must take care of deleting cleanUp = true; if(t->data) - RsErr() << __PRETTY_FUNCTION__<< " async call with data attached but no callback set!" << std::endl; + RS_ERR("sync call with data attached but no callback set!"); } if (cleanUp) { if (t->data) { - RsErr() << __PRETTY_FUNCTION__<< " will try to delete void pointer!" << std::endl; + RS_ERR("will try to delete void pointer!"); print_stacktrace(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdelete-incomplete" @@ -304,7 +304,7 @@ void rsAutoProxyMonitor::taskFinished(taskTicket *&ticket) // clean up if (ticket->data) { - RsErr() << __PRETTY_FUNCTION__<< " data set. Will try to delete void pointer" << std::endl; + RS_ERR(" data set. Will try to delete void pointer"); print_stacktrace(); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdelete-incomplete" @@ -323,7 +323,7 @@ autoProxyService *rsAutoProxyMonitor::lookUpService(autoProxyType::autoProxyType if ((itService = mProxies.find(t)) != mProxies.end()) { return itService->second; } - RsDbg() << __PRETTY_FUNCTION__<< " no service for type " << t << " found!" << std::endl; + RS_DBG("no service for type ", t, " found!"); return NULL; } From ebc5a116b22c28f59cf1f584c5a0afe020986550 Mon Sep 17 00:00:00 2001 From: sehraf Date: Thu, 11 Jun 2020 11:18:52 +0200 Subject: [PATCH 66/72] remove unnecessary std::replace --- libretroshare/src/util/i2pcommon.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp index 8c6f2238b..575b0fec0 100644 --- a/libretroshare/src/util/i2pcommon.cpp +++ b/libretroshare/src/util/i2pcommon.cpp @@ -67,7 +67,8 @@ std::string publicKeyFromPrivate(std::string const &priv) // creat a copy to work on, need to convert it to standard base64 auto priv_copy(priv); std::replace(priv_copy.begin(), priv_copy.end(), '~', '/'); - std::replace(priv_copy.begin(), priv_copy.end(), '-', '+'); + // replacing the - with a + is not necessary, as RsBase64 can handle base64url encoding, too + // std::replace(copy.begin(), copy.end(), '-', '+'); // get raw data std::vector dataPriv; From f88527dbd78fa8b37eb34d317f0f5f5d8c00e778 Mon Sep 17 00:00:00 2001 From: sehraf Date: Fri, 12 Jun 2020 17:28:08 +0200 Subject: [PATCH 67/72] i2pbob: remove sockaddr_storage_tostring --- libretroshare/src/services/autoproxy/p3i2pbob.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index 7fd7594b9..a170d08ca 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -936,7 +936,7 @@ void p3I2pBob::finalizeSettings_locked() sockaddr_storage_setipv4(mI2PProxyAddr, (sockaddr_in*)&proxy); sockaddr_storage_setport(mI2PProxyAddr, 2827); - RS_DBG4("using ", sockaddr_storage_tostring(mI2PProxyAddr)); + RS_DBG4("using ", mI2PProxyAddr); RS_DBG4("using ", mSetting.address.base32); peerState ps; From 5ff5a32df70c001caf930a3cdfabea3f8bf0c24d Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 12 Jun 2020 18:41:42 +0200 Subject: [PATCH 68/72] Proper naming for RsRandom string functions --- libretroshare/src/util/rsrandom.cc | 25 +++++++++++++++++++------ libretroshare/src/util/rsrandom.h | 12 +++++++++++- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/util/rsrandom.cc b/libretroshare/src/util/rsrandom.cc index 7ea8eb968..dc7d9ac67 100644 --- a/libretroshare/src/util/rsrandom.cc +++ b/libretroshare/src/util/rsrandom.cc @@ -121,13 +121,26 @@ double RsRandom::random_f64() return random_u64() / (double)(~(uint64_t)0) ; } -std::string RsRandom::random_alphaNumericString(uint32_t len) +/*static*/ std::string RsRandom::alphaNumeric(uint32_t length) { - std::string s = "" ; + std::string s; + while(s.size() < length) + { + uint8_t rChar; random_bytes(&rChar, 1); rChar = rChar % 123; + /* if(isalnum(val)) isalnum result may vary depend on locale!! */ + if( (rChar >= 48 && rChar <= 57) /* 0-9 */ || + (rChar >= 65 && rChar <= 90) /* A-Z */ || + (rChar >= 97 && rChar <= 122) /* a-z */ ) + s += static_cast(rChar); + } - for(uint32_t i=0;i(&ret[0]), length); + for(uint32_t i=0; i Date: Sat, 13 Jun 2020 10:57:08 +0200 Subject: [PATCH 69/72] fix typo --- libretroshare/src/services/autoproxy/rsautoproxymonitor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc index 3c52cfdc5..d58c871e3 100644 --- a/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc +++ b/libretroshare/src/services/autoproxy/rsautoproxymonitor.cc @@ -265,7 +265,7 @@ void rsAutoProxyMonitor::taskDone(taskTicket *t, autoProxyStatus::autoProxyStatu // we must take care of deleting cleanUp = true; if(t->data) - RS_ERR("sync call with data attached but no callback set!"); + RS_ERR("async call with data attached but no callback set!"); } if (cleanUp) { From 8323b94a6a13e574e95f909f3b6670568b85d945 Mon Sep 17 00:00:00 2001 From: sehraf Date: Sat, 13 Jun 2020 10:58:44 +0200 Subject: [PATCH 70/72] use modern c++ --- libretroshare/src/util/i2pcommon.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp index 575b0fec0..eef359576 100644 --- a/libretroshare/src/util/i2pcommon.cpp +++ b/libretroshare/src/util/i2pcommon.cpp @@ -39,10 +39,7 @@ std::string keyToBase32Addr(const std::string &key) } const std::string makeOption(const std::string &lhs, const int8_t &rhs) { - // convert number to int - std::ostringstream oss; - oss << (int)rhs; - return lhs + "=" + oss.str(); + return lhs + "=" + std::to_string(rhs); } uint16_t readTwoBytesBE(std::vector::const_iterator &p) From f18d49db483b0dc9d8f8371b538dd8f7d506a419 Mon Sep 17 00:00:00 2001 From: sehraf Date: Sat, 13 Jun 2020 11:07:07 +0200 Subject: [PATCH 71/72] use RsRandom::alphaNumeric --- libretroshare/src/services/autoproxy/p3i2pbob.cc | 10 +++------- libretroshare/src/util/i2pcommon.cpp | 8 -------- libretroshare/src/util/i2pcommon.h | 13 ------------- 3 files changed, 3 insertions(+), 28 deletions(-) diff --git a/libretroshare/src/services/autoproxy/p3i2pbob.cc b/libretroshare/src/services/autoproxy/p3i2pbob.cc index a170d08ca..f9eb3d9b3 100644 --- a/libretroshare/src/services/autoproxy/p3i2pbob.cc +++ b/libretroshare/src/services/autoproxy/p3i2pbob.cc @@ -945,13 +945,9 @@ void p3I2pBob::finalizeSettings_locked() // setup commands // new lines are appended later! - // generate random suffix for name - // RSRandom::random_alphaNumericString can return very weird looking strings like: ,,@z+M - // use base32 instead - size_t len = 5; // 5 characters = 8 base32 symbols - std::vector tmp(len); - RSRandom::random_bytes(tmp.data(), len); - const std::string location = Radix32::encode(tmp.data(), len); + // generate 8 characater long random suffix for name + constexpr size_t len = 8; + const std::string location = RsRandom::alphaNumeric(len); RS_DBG4("using suffix ", location); mTunnelName = "RetroShare-" + location; diff --git a/libretroshare/src/util/i2pcommon.cpp b/libretroshare/src/util/i2pcommon.cpp index eef359576..ec2ebfd6b 100644 --- a/libretroshare/src/util/i2pcommon.cpp +++ b/libretroshare/src/util/i2pcommon.cpp @@ -5,14 +5,6 @@ namespace i2p { -const std::string generateNameSuffix(const size_t len) { - std::vector tmp(len); - RsRandom::random_bytes(tmp.data(), len); - const std::string location = Radix32::encode(tmp.data(), len); - - return location; -} - std::string keyToBase32Addr(const std::string &key) { std::string copy(key); diff --git a/libretroshare/src/util/i2pcommon.h b/libretroshare/src/util/i2pcommon.h index 63f1ba5af..01d655bf9 100644 --- a/libretroshare/src/util/i2pcommon.h +++ b/libretroshare/src/util/i2pcommon.h @@ -184,19 +184,6 @@ static const std::array, 12> signingKeyLengths { /*SigningKeyType::RedDSA_SHA512_Ed25519 */ std::make_pair( 32, 32), }; - -/** - * @brief generateNameSuffix Generates a base32 name suffix for tunnel identification - * @param len lenght of random bytes, will be expanded by base32 encoding - * @return base32 string - * - * RSRandom::random_alphaNumericString can return very weird looking strings like: ,,@z+M - * -> so use base32 instead - * - * 5 characters = 8 base32 symbols - */ -const std::string generateNameSuffix(const size_t len = 5); - /** * @brief makeOption Creates the string "lhs=rhs" used by BOB and SAM. Converts rhs * @param lhs option to set From f13b429f1c09adff8bd99439e1c4a26766d305f8 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 13 Jun 2020 14:58:30 +0200 Subject: [PATCH 72/72] Document what chars are return RsRandom::alphaNumeric As requested by Cyril --- libretroshare/src/util/rsrandom.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/util/rsrandom.h b/libretroshare/src/util/rsrandom.h index 5e7e0d92c..94d619537 100644 --- a/libretroshare/src/util/rsrandom.h +++ b/libretroshare/src/util/rsrandom.h @@ -48,7 +48,7 @@ public: static void random_bytes(uint8_t* data, uint32_t length); - /** Return a random alphanumeric string of the given lenght */ + /// Return a random alphanumeric *[0-9,A-Z,a-z] string of the given lenght static std::string alphaNumeric(uint32_t length); /** Return a random printable string of the given lenght */