diff --git a/src/gxs/rsdataservice.cc b/src/gxs/rsdataservice.cc index c061e37e7..9be4dadc7 100644 --- a/src/gxs/rsdataservice.cc +++ b/src/gxs/rsdataservice.cc @@ -130,6 +130,8 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d mDb = new RetroDb(mDbPath, RetroDb::OPEN_READWRITE_CREATE, key); mUseCache = true; + mMsgMetaDataCache_ContainsAllDatabase = false; + mMsgMetaDataCache_ColdFullReads = 0; initialise(isNewDatabase); @@ -1290,6 +1292,54 @@ void RsDataService::locked_retrieveMessages(RetroCursor *c, std::vectorsqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, "", ""); + + if(!c) + { + RsErr() << __PRETTY_FUNCTION__ << ": failed to query message meta data" << std::endl; + return; + } + + bool valid = c->moveToFirst(); + + while(valid) + { + auto m = locked_getMsgMeta(*c, 0); + + if(m != nullptr) + mMsgMetaDataCache[m->mGroupId].updateMeta(m->mMsgId, m); + + valid = c->moveToNext(); + } + + delete c; + + // Every group of this database now holds all its metas, including the ones + // that have no message at all and would otherwise be re-queried forever. + for(auto& it: mMsgMetaDataCache) + it.second.setCacheUpToDate(true); + + mMsgMetaDataCache_ContainsAllDatabase = true; +} + int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaResult& msgMeta) { RsStackMutex stack(mDbMutex); @@ -1299,6 +1349,25 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes int resultCount = 0; #endif + // Whole-group requests that the cache cannot serve are the expensive ones: + // one row lookup per message, scattered over the whole file. Counting them + // across calls matters, because callers ask for one group at a time -- the + // GUI computes the statistics of each group with its own request. As soon as + // a second group needs such a read, sweeping the database is what is + // happening, and one sequential scan is cheaper than continuing group by + // group. See locked_loadAllMsgMetaInOneScan(). + if(mUseCache && !mMsgMetaDataCache_ContainsAllDatabase) + { + uint32_t cold_groups = 0; + + for(auto mit(reqIds.begin()); mit != reqIds.end(); ++mit) + if(mit->second.empty() && !mMsgMetaDataCache[mit->first].isCacheUpToDate()) + ++cold_groups; + + if(cold_groups + mMsgMetaDataCache_ColdFullReads > 1) + locked_loadAllMsgMetaInOneScan(); + } + for(auto mit(reqIds.begin()); mit != reqIds.end(); ++mit) { @@ -1316,6 +1385,8 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes cache->getFullMetaList(msgMeta[grpId]); else { + ++mMsgMetaDataCache_ColdFullReads; + RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgMetaColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", ""); if (c) diff --git a/src/gxs/rsdataservice.h b/src/gxs/rsdataservice.h index 0424e4e14..e5e33f836 100644 --- a/src/gxs/rsdataservice.h +++ b/src/gxs/rsdataservice.h @@ -308,6 +308,16 @@ private: */ void locked_retrieveMsgMetaList(RetroCursor* c, std::vector > &msgMeta); + /*! + * \brief Read the meta of every message of the database in one sequential + * scan and fill every per-group cache with it. + * + * Warming up the caches group by group costs one disk seek per message; a + * single scan reads the file in physical order and warms up all the groups + * at once. Called when more than one group still needs a cold full read. + */ + void locked_loadAllMsgMetaInOneScan(); + /*! * Retrieves all the grp meta results from a cursor * @param c cursor to result set @@ -468,6 +478,12 @@ private: t_MetaDataCache mGrpMetaDataCache; std::map > mMsgMetaDataCache; + /// True once locked_loadAllMsgMetaInOneScan() has run: no point scanning twice. + bool mMsgMetaDataCache_ContainsAllDatabase; + + /// Number of whole-group cold reads done so far, to decide when scanning wins. + uint32_t mMsgMetaDataCache_ColdFullReads; + bool mUseCache; };