From 31b9bbf7965a350b506324d2b5820cc3f53cab27 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 5 Jan 2024 17:40:20 +0100 Subject: [PATCH 1/8] Support activity and placeholder in eviction script --- lib/storage/blob.js | 61 +++++++++++++++++++++++++++++++++++---- scripts/clean-activity.js | 21 ++++++++++++++ 2 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 scripts/clean-activity.js diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 97fc8b9e3..172af2e1a 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -464,7 +464,7 @@ var makeWalker = function (n, handleChild, done) { // do no more than 20 jobs at a time var tasks = Semaphore.create(n); - var recurse = function (path) { + var recurse = function (path, dir) { tasks.take(function (give) { var next = give(W()); @@ -477,7 +477,19 @@ var makeWalker = function (n, handleChild, done) { } if (!stats.isDirectory()) { w.abort(); - return void handleChild(void 0, path, next); + if (/\.activity$/.test(path)) { + // NOTE: some activity files were created for deleted blobs due to + // a bug. We're going to detect them here in order to be able to clean + // them. + if (!dir.includes(Path.basename(path.replace(/\.activity$/, '')))) { + return void handleChild(void 0, path, next, true); + } + // Ignore valid activity files + return next(); + } + // Ignore placeholder files + if (/\.placeholder$/.test(path)) { return next(); } + return void handleChild(void 0, path, next, false); } // fall through })); @@ -487,7 +499,7 @@ var makeWalker = function (n, handleChild, done) { if (err) { return next(); } // everything is fine and it's a directory... dir.forEach(function (d) { - recurse(Path.join(path, d)); + recurse(Path.join(path, d), dir); }); next(); }); @@ -502,7 +514,8 @@ var listProofs = function (root, handler, cb) { Fs.readdir(root, function (err, dir) { if (err) { return void cb(err); } - var walk = makeWalker(20, function (err, path, next) { + var walk = makeWalker(20, function (err, path, next, loneActivity) { + if (loneActivity) { return void next(); } // path is the path to a child node on the filesystem // next handles the next job in a queue @@ -537,12 +550,20 @@ var listProofs = function (root, handler, cb) { }); }; +var getActivityStat = function (path, base, cb) { + var suffix = base ? '' : '.activity'; + Fs.stat(path+suffix, function (err, stats) { + if (err && err.code === 'ENOENT' && !base) { return getActivityStat(path, true, cb); } + cb(err, stats); + }); +}; var listBlobs = function (root, handler, cb) { // iterate over files Fs.readdir(root, function (err, dir) { if (err) { return void cb(err); } - var walk = makeWalker(20, function (err, path, next) { - Fs.stat(path, function (err, stats) { + var walk = makeWalker(20, function (err, path, next, loneActivity) { + if (loneActivity) { return void next(); } + getActivityStat(path, false, function (err, stats) { if (err) { return void handler(err, void 0, next); } @@ -565,6 +586,30 @@ var listBlobs = function (root, handler, cb) { }); }; +var cleanLoneActivity = function (root, cb) { + // iterate over files + Fs.readdir(root, function (err, dir) { + if (err) { return void cb(err); } + var walk = makeWalker(20, function (err, path, next, loneActivity) { + if (!loneActivity) { return void next(); } + Fs.unlink(path, function (err) { + if (err) { + return console.error('ERROR', path, err); + } + console.log('DELETED', path); + next(); + }); + }, function () { + cb(); + }); + + dir.forEach(function (d) { + if (d.length !== 2) { return; } + walk(Path.join(root, d)); + }); + }); +}; + BlobStore.create = function (config, _cb) { var cb = Util.once(Util.mkAsync(_cb)); if (typeof(config.getSession) !== 'function') { @@ -651,6 +696,10 @@ BlobStore.create = function (config, _cb) { removeArchivedProof(Env, safeKey, blobId, cb); }, }, + loneActivity: function (_cb) { + var cb = Util.once(Util.mkAsync(_cb)); + cleanLoneActivity(Env.blobPath, cb); + } }, archive: { diff --git a/scripts/clean-activity.js b/scripts/clean-activity.js new file mode 100644 index 000000000..6cb3f75f6 --- /dev/null +++ b/scripts/clean-activity.js @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team and contributors +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +/** + * Some .activity file were created for deleted blob due to a bug. + * This script can be run once to remove these invalid activity file. +**/ +var nThen = require("nthen"); +var config = require("../lib/load-config"); +var BlobStore = require("../lib/storage/blob"); + +config.getSession = function () {}; +BlobStore.create(config, function (err, store) { + if (err) { return console.error('ERROR', err); } + console.log('Cleaning lone .activity files...'); + store.remove.loneActivity(function (err) { + if (err) { return console.error('ERROR', err); } + console.log('Done'); + }); +}); From 0e918643ce839ee86b6fdd94f2f7e6866c44fa91 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 5 Jan 2024 17:46:04 +0100 Subject: [PATCH 2/8] Check if a blob exists before creating the activity file --- lib/storage/blob.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 172af2e1a..b029b95c5 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -153,8 +153,12 @@ var clearActivity = function (Env, blobId, cb) { }; var updateActivity = function (Env, blobId, cb) { var path = makeActivityPath(Env, blobId); - var s_data = String(+new Date()); - Fs.writeFile(path, s_data, cb); + var blobPath = makeBlobPath(Env, blobId); + isFile(blobPath, (err, state) => { + if (err || !state) { return void cb(); } + var s_data = String(+new Date()); + Fs.writeFile(path, s_data, cb); + }); }; var archiveActivity = function (Env, blobId, cb) { From 70d84838dfa53369825637eb1ba7759e169f02e7 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 8 Jan 2024 14:08:26 +0100 Subject: [PATCH 3/8] Add DRY_RUN option to the eviction script --- lib/eviction.js | 39 +++++++++++++++++++++++++++++++++++++-- scripts/evict-inactive.js | 4 ++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/eviction.js b/lib/eviction.js index 8bd530f1e..42afbc2c5 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -200,6 +200,11 @@ var evictArchived = function (Env, cb) { // but if it's been stored for the configured time... // expire it + if (Env.DRY_RUN) { + if (item.channel.length === 32) { removed++; } + else if (item.channel.length === 44) { accounts++; } + return void Log.info("EVICT_ARCHIVED_CHANNEL_DRY_RUN", item.channel, next); + } store.removeArchivedChannel(item.channel, w(function (err) { if (err) { return Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', { @@ -246,6 +251,10 @@ var evictArchived = function (Env, cb) { return void next(); } if (item && item.mtime > retentionTime) { return void next(); } + if (Env.DRY_RUN) { + removed++; + return void Log.info("EVICT_ARCHIVED_BLOB_PROOF_DRY_RUN", item, next); + } blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) { if (err) { Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item); @@ -273,6 +282,10 @@ var evictArchived = function (Env, cb) { return void next(); } if (item && item.mtime > retentionTime) { return void next(); } + if (Env.DRY_RUN) { + removed++; + return void Log.info("EVICT_ARCHIVED_BLOB_DRY_RUN", item, next); + } blobs.remove.archived.blob(item.blobId, function (err) { if (err) { Log.error("EVICT_ARCHIVED_BLOB_ERROR", item); @@ -288,6 +301,7 @@ var evictArchived = function (Env, cb) { })); }; + if (Env.DRY_RUN) { Env.Log.info('DRY RUN'); } nThen(loadStorage) .nThen(migrateIncorrectBlobs) .nThen(removeArchivedChannels) @@ -544,6 +558,9 @@ module.exports = function (Env, cb) { } // remove the pin logs of inactive accounts if inactive account removal is configured + if (Env.DRY_RUN) { + return void Log.info("EVICT_INACTIVE_ACCOUNT_DRY_RUN", id, cb); + } pinStore.archiveChannel(id, undefined, function (err) { if (err) { return Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err, next); @@ -602,7 +619,12 @@ module.exports = function (Env, cb) { // unless we address this race condition with this last-minute double-check if (item.mtime > inactiveTime) { return void next(); } - removed++; + if (Env.DRY_RUN) { + removed++; + return void Log.info("EVICT_ARCHIVE_BLOB_DRY_RUN", { + item: item, + }, next); + } blobs.archive.blob(item.blobId, 'INACTIVE', function (err) { if (err) { return Log.error("EVICT_ARCHIVE_BLOB_ERROR", { @@ -610,6 +632,7 @@ module.exports = function (Env, cb) { item: item, }, next); } + removed++; Log.info("EVICT_ARCHIVE_BLOB", { item: item, }, next); @@ -658,6 +681,10 @@ module.exports = function (Env, cb) { } })); }).nThen(function () { + if (Env.DRY_RUN) { + removed++; + return void Log.info("EVICT_BLOB_PROOF_LONELY_DRY_RUN", item, next); + } blobs.remove.proof(item.safeKey, item.blobId, function (err) { if (err) { return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item, next); @@ -698,6 +725,9 @@ module.exports = function (Env, cb) { // check if the database has any ephemeral channels // if it does it's because of a bug, and they should be removed if (item.channel.length === 34) { + if (Env.DRY_RUN) { + return void Log.info("EVICT_EPHEMERAL_DRY_RUN", item.channel, cb); + } return void store.removeChannel(item.channel, w(function (err) { if (err) { return Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', { @@ -728,6 +758,10 @@ module.exports = function (Env, cb) { // else fall through to the archival })); }).nThen(function (w) { + if (Env.DRY_RUN) { + archived++; + return void Log.info("EVICT_CHANNEL_ARCHIVAL_DRY_RUN", item.channel, cb); + } return void store.archiveChannel(item.channel, 'INACTIVE', w(function (err) { if (err) { Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { @@ -736,8 +770,8 @@ module.exports = function (Env, cb) { }, w()); return; } - Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel, w()); archived++; + Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel, w()); })); }).nThen(cb); }; @@ -754,6 +788,7 @@ module.exports = function (Env, cb) { store.listChannels(handler, w(done), true); // using a hacky "fast mode" since we only need the channel id }; + if (Env.DRY_RUN) { Env.Log.info('DRY RUN'); } nThen(loadStorage) // iterate over all documents and add them to a bloom filter if they have been active diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index f53536ae3..521edf015 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -15,6 +15,10 @@ var config = require("../lib/load-config"); var Env = Environment.create(config); +// Set DRY_RUN to true to run the script without deleting anything. A log file +// will be created. +Env.DRY_RUN = false; + var loadPremiumAccounts = function (Env, cb) { nThen(function (w) { // load premium accounts From b22441446c829aec5613cbd1788d57e6fd952b40 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 8 Jan 2024 14:12:04 +0100 Subject: [PATCH 4/8] Fix eviction error and add DRY_RUN to the archive cleaning script --- lib/eviction.js | 2 +- scripts/evict-archived.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/eviction.js b/lib/eviction.js index 42afbc2c5..12aaeff0f 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -203,7 +203,7 @@ var evictArchived = function (Env, cb) { if (Env.DRY_RUN) { if (item.channel.length === 32) { removed++; } else if (item.channel.length === 44) { accounts++; } - return void Log.info("EVICT_ARCHIVED_CHANNEL_DRY_RUN", item.channel, next); + return void Log.info("EVICT_ARCHIVED_CHANNEL_DRY_RUN", item.channel, cb); } store.removeArchivedChannel(item.channel, w(function (err) { if (err) { diff --git a/scripts/evict-archived.js b/scripts/evict-archived.js index 124ebc30c..6914c90ad 100644 --- a/scripts/evict-archived.js +++ b/scripts/evict-archived.js @@ -15,6 +15,10 @@ var config = require("../lib/load-config"); var Env = Environment.create(config); +// Set DRY_RUN to true to run the script without deleting anything. A log file +// will be created. +Env.DRY_RUN = false; + var loadPremiumAccounts = function (Env, cb) { nThen(function (w) { // load premium accounts From 467e8ee7fe29df86072c757e5e867abb26afa710 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 8 Jan 2024 14:41:00 +0100 Subject: [PATCH 5/8] Use ctime to detect deletable archived blobs --- lib/eviction.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/eviction.js b/lib/eviction.js index 12aaeff0f..dd0d777ab 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -250,7 +250,7 @@ var evictArchived = function (Env, cb) { Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err); return void next(); } - if (item && item.mtime > retentionTime) { return void next(); } + if (item && item.ctime > retentionTime) { return void next(); } if (Env.DRY_RUN) { removed++; return void Log.info("EVICT_ARCHIVED_BLOB_PROOF_DRY_RUN", item, next); @@ -281,7 +281,7 @@ var evictArchived = function (Env, cb) { Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err); return void next(); } - if (item && item.mtime > retentionTime) { return void next(); } + if (item && item.ctime > retentionTime) { return void next(); } if (Env.DRY_RUN) { removed++; return void Log.info("EVICT_ARCHIVED_BLOB_DRY_RUN", item, next); From ff3a40aa3152d5e09a70d3afb02356e4ba516631 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 8 Jan 2024 19:11:20 +0100 Subject: [PATCH 6/8] Fix TypeError --- lib/eviction.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/eviction.js b/lib/eviction.js index dd0d777ab..4cac76e71 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -559,7 +559,7 @@ module.exports = function (Env, cb) { // remove the pin logs of inactive accounts if inactive account removal is configured if (Env.DRY_RUN) { - return void Log.info("EVICT_INACTIVE_ACCOUNT_DRY_RUN", id, cb); + return void Log.info("EVICT_INACTIVE_ACCOUNT_DRY_RUN", id, next); } pinStore.archiveChannel(id, undefined, function (err) { if (err) { From d92636bcf8920fb6c81a9d1f23f0d4143f6cc513 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 9 Jan 2024 15:09:41 +0100 Subject: [PATCH 7/8] Fix DRY_RUN issue --- lib/eviction.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/eviction.js b/lib/eviction.js index 4cac76e71..e57041ffb 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -760,6 +760,7 @@ module.exports = function (Env, cb) { }).nThen(function (w) { if (Env.DRY_RUN) { archived++; + w.abort(); return void Log.info("EVICT_CHANNEL_ARCHIVAL_DRY_RUN", item.channel, cb); } return void store.archiveChannel(item.channel, 'INACTIVE', w(function (err) { From 6e132a092f324884de958b53b54fb6e2a51fe1cf Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 16 Jan 2024 16:04:33 +0100 Subject: [PATCH 8/8] lint compliance --- scripts/clean-activity.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/clean-activity.js b/scripts/clean-activity.js index 6cb3f75f6..dbb58c8f3 100644 --- a/scripts/clean-activity.js +++ b/scripts/clean-activity.js @@ -6,7 +6,6 @@ * Some .activity file were created for deleted blob due to a bug. * This script can be run once to remove these invalid activity file. **/ -var nThen = require("nthen"); var config = require("../lib/load-config"); var BlobStore = require("../lib/storage/blob");