From fa313718c07cdd8d0f7ba60d32afda00c48b61bd Mon Sep 17 00:00:00 2001 From: jolavillette Date: Mon, 27 Jul 2026 12:27:18 +0200 Subject: [PATCH] GXS channels: read only the post versions that will be displayed A channel keeps every version of every edited post. Only the latest of each chain is ever shown: sortPosts() read them all, then discarded the superseded ones keeping just their ids in mOlderVersions. Profiling a real channel shows how much that costs: 6409 messages read for 1931 displayed posts, 195MB of payload of which roughly two thirds belongs to versions thrown away immediately -- thumbnails decrypted, deserialised and freed for nothing. Resolve the version chains on the metas instead. They are small, come from the meta cache once warm, and sortPostMetas() already works on any type exposing a RsMsgMetaData. getChannelAllContent() now: - pulls the group's metas via getContentSummaries() - splits posts from comments and votes - runs sortPostMetas() to find the retained version of each chain - requests message data for those ids only, plus all comments and votes Since the request now carries an explicit id set, it goes through the batched IN(...) retrieval added earlier: a few queries instead of one, and the payload read drops by whatever the edit history weighs. Two behaviours of sortPosts() have to be reproduced, and applyPostVersions() does so from the resolved chains: - comments hang off whichever version was current when they were written, so they are remapped onto the retained post before being counted, which replaces the old "add up the counts of all older versions" pass; - sortPostMetas() normalises mOrigMsgId to the top of the chain, and callers match edited posts on that value (GUI updateSinglePost), so the normalised id is carried over to the post that is returned. The item conversion loop is factored out of getPostData() into convertMsgItems() so both paths share it; getPostData() itself, still used by getChannelContent() and the deprecated API, keeps calling sortPosts() unchanged. An empty id set means "every message of the group" to the data store, so an empty channel returns before any request is made rather than asking for everything. Co-Authored-By: Claude Opus 5 (1M context) --- src/services/p3gxschannels.cc | 166 +++++++++++++++++++++++++++++++--- src/services/p3gxschannels.h | 32 +++++++ 2 files changed, 184 insertions(+), 14 deletions(-) diff --git a/src/services/p3gxschannels.cc b/src/services/p3gxschannels.cc index 98b76a3f5..39001b211 100644 --- a/src/services/p3gxschannels.cc +++ b/src/services/p3gxschannels.cc @@ -609,14 +609,12 @@ bool p3GxsChannels::groupShareKeys( * at the moment - fix it up later */ -bool p3GxsChannels::getPostData( const uint32_t& token, std::vector& msgs, - std::vector& cmts, - std::vector& vots) +bool p3GxsChannels::convertMsgItems( const uint32_t& token, + std::vector& msgs, + std::vector& cmts, + std::vector& vots, + long& getmsgdata_ms, long& convert_ms ) { -#ifdef GXSCHANNELS_DEBUG - RsDbg() << __PRETTY_FUNCTION__ << std::endl; -#endif - RsGxsProfiler::Timer prof_timer; GxsMsgDataMap msgData; @@ -626,7 +624,7 @@ bool p3GxsChannels::getPostData( const uint32_t& token, std::vector& msgs, + std::vector& cmts, + std::vector& vots) +{ +#ifdef GXSCHANNELS_DEBUG + RsDbg() << __PRETTY_FUNCTION__ << std::endl; +#endif + + long prof_getmsgdata_ms = 0, prof_convert_ms = 0; + + if(!convertMsgItems(token, msgs, cmts, vots, prof_getmsgdata_ms, prof_convert_ms)) + return false; + + RsGxsProfiler::Timer prof_timer; sortPosts(msgs,cmts); // stores old versions in the right place. @@ -1535,28 +1551,150 @@ bool p3GxsChannels::getChannelAllContent( const RsGxsGroupId& channelId, std::vector& comments, std::vector& votes ) { + RsGxsProfiler::Timer prof_timer; + + // A channel keeps every version of every edited post. Only the latest + // version of each is ever displayed: sortPosts() used to read them all and + // throw the superseded ones away, after their whole payload -- thumbnail + // included -- had been read from the database and deserialised. On a real + // channel that is the bulk of the bytes read. + // + // Resolve the version chains from the metas instead, which are small and + // usually already cached, and only read the payload of the messages that + // will actually be used. + + std::vector metas; + + if(!getContentSummaries(channelId,metas)) + return false; + + std::vector post_metas; + std::set wanted_msgs; + + for(auto& m: metas) + if(m.mThreadId.isNull() && m.mParentId.isNull()) + post_metas.push_back(m); + else + wanted_msgs.insert(m.mMsgId); // comments and votes are all kept + + std::function< RsMsgMetaData& (RsMsgMetaData&) > get_meta = [](RsMsgMetaData& m)->RsMsgMetaData& { return m; }; + std::map > > original_versions; + + sortPostMetas(post_metas, get_meta, original_versions); + + // retained holds, for each post that will actually be read, its version set + // (what RsGxsChannelPost::mOlderVersions expects) and the mOrigMsgId that + // sortPostMetas() normalised to the top of the chain -- callers such as the + // GUI match edited posts on that value, so it must be carried over. + // version_to_latest maps any version id to the retained one, so that a + // comment written on a superseded version is still counted on the post. + + std::map retained; + std::map version_to_latest; + + for(const auto& ov_entry: original_versions) + { + const RsMsgMetaData& latest_meta(post_metas[ov_entry.second.first]); + + wanted_msgs.insert(latest_meta.mMsgId); + + auto& entry(retained[latest_meta.mMsgId]); + entry.mOrigMsgId = latest_meta.mOrigMsgId; + entry.mVersions = ov_entry.second.second; + + for(const auto& version_id: ov_entry.second.second) + version_to_latest[version_id] = latest_meta.mMsgId; + } + + const long prof_versions_ms = prof_timer.lap(); + + // An empty id set means "every message of the group" to the data store, so + // an empty channel must not be turned into a request at all. + if(wanted_msgs.empty()) + return true; + uint32_t token; RsTokReqOptions opts; opts.mReqType = GXS_REQUEST_TYPE_MSG_DATA; - RsGxsProfiler::Timer prof_timer; + GxsMsgReq msgIds; + msgIds[channelId] = wanted_msgs; - if( !requestMsgInfo(token, opts,std::list({channelId})) || waitToken(token,std::chrono::milliseconds(60000)) != RsTokenService::COMPLETE ) + if( !requestMsgInfo(token, opts, msgIds) || waitToken(token,std::chrono::milliseconds(60000)) != RsTokenService::COMPLETE ) return false; const long prof_wait_ms = prof_timer.lap(); - const bool res = getPostData(token, posts, comments,votes); + long prof_getmsgdata_ms = 0, prof_convert_ms = 0; + + if(!convertMsgItems(token, posts, comments, votes, prof_getmsgdata_ms, prof_convert_ms)) + return false; + + applyPostVersions(posts, comments, retained, version_to_latest); const long prof_read_ms = prof_timer.ms(); - const long prof_total_ms = prof_wait_ms + prof_read_ms; + const long prof_total_ms = prof_versions_ms + prof_wait_ms + prof_read_ms; RS_GXS_PROF( prof_total_ms, "getChannelAllContent grp=" << channelId + << " metas=" << metas.size() + << " read_msgs=" << wanted_msgs.size() + << " skipped_versions=" << (metas.size() - wanted_msgs.size()) + << " versions=" << prof_versions_ms << "ms" << " token_wait=" << prof_wait_ms << "ms" << " read=" << prof_read_ms << "ms" << " total=" << prof_total_ms << "ms" ); - return res; + return true; +} + +void p3GxsChannels::applyPostVersions( + std::vector& posts, + const std::vector& comments, + const std::map& retained, + const std::map& version_to_latest ) const +{ + // Same result as sortPosts(), but the version chains have already been + // resolved from the metas, so posts only holds the retained versions. + + std::map post_indices; + + for(uint32_t i=0;isecond.mVersions; + posts[i].mMeta.mOrigMsgId = it->second.mOrigMsgId; + } + } + + for(uint32_t i=0;isecond : thread_id); + + // Not finding the post is normal: because of sync periods we may hold + // comments for a post we never received. + + if(it == post_indices.end()) + continue; + + auto& p(posts[it->second]); + + ++p.mCommentCount; + + if(IS_MSG_NEW(comments[i].mMeta.mMsgStatus)) + ++p.mUnreadCommentCount; + } } bool p3GxsChannels::getChannelContent( const RsGxsGroupId& channelId, diff --git a/src/services/p3gxschannels.h b/src/services/p3gxschannels.h index 99b5f6a51..6f8aac4ca 100644 --- a/src/services/p3gxschannels.h +++ b/src/services/p3gxschannels.h @@ -242,6 +242,38 @@ protected: // made protected because it's all deprecated // helper function that moved old post versions in the mOldVersions of new posts. void sortPosts(std::vector& posts, const std::vector &comments) const; + /** Turn the message items of a completed request into posts, comments and + * votes. Shared by getPostData() and by getChannelAllContent(), which + * resolves post versions itself. The two long& outputs report timings to + * the caller so that it can emit a single profiling line. */ + bool convertMsgItems( const uint32_t& token, + std::vector& msgs, + std::vector& cmts, + std::vector& vots, + long& getmsgdata_ms, long& convert_ms ); + + /** What resolving a post's version chain on its metas yields, for the one + * version that is actually kept and read from the database. */ + struct RetainedPostVersions + { + /// Top level id of the chain, as normalised by sortPostMetas(). + RsGxsMessageId mOrigMsgId; + + /// Every version of the post, including the retained one. + std::set mVersions; + }; + + /** Counterpart of sortPosts() for the case where post versions have already + * been resolved from the metas, so that only the retained versions were + * read from the database. Fills mOlderVersions, the normalised mOrigMsgId + * and the comment counts. + * @param retained keyed by retained message id + * @param version_to_latest maps any version id onto the retained one */ + void applyPostVersions( std::vector& posts, + const std::vector& comments, + const std::map& retained, + const std::map& version_to_latest ) const; + //Not currently used //virtual bool getRelatedPosts(const uint32_t &token, std::vector &posts);