The_Great_Archive/cryptpad
1
0
Fork 0
mirror of https://github.com/cryptpad/cryptpad.git synced 2026-09-12 19:49:59 +05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1375 from cryptpad/5.6-eviction

Browse Source 
Improve eviction script
This commit is contained in:
yflory 2024-01-16 16:06:43 +01:00 committed by GitHub
commit 91a50338c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 129 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
lib/eviction.js
View File

@ -200,6 +200,11 @@ var evictArchived = function (Env, cb) {
// but if it's been stored for the configured time...
// expire it
if (Env.DRY_RUN) {
if (item.channel.length === 32) { removed++; }
else if (item.channel.length === 44) { accounts++; }
return void Log.info("EVICT_ARCHIVED_CHANNEL_DRY_RUN", item.channel, cb);
}
store.removeArchivedChannel(item.channel, w(function (err) {
if (err) {
return Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', {
@ -245,7 +250,11 @@ var evictArchived = function (Env, cb) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
return void next();
}
if (item && item.mtime > retentionTime) { return void next(); }
if (item && item.ctime > retentionTime) { return void next(); }
if (Env.DRY_RUN) {
removed++;
return void Log.info("EVICT_ARCHIVED_BLOB_PROOF_DRY_RUN", item, next);
}
blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) {
if (err) {
Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item);
@ -272,7 +281,11 @@ var evictArchived = function (Env, cb) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
return void next();
}
if (item && item.mtime > retentionTime) { return void next(); }
if (item && item.ctime > retentionTime) { return void next(); }
if (Env.DRY_RUN) {
removed++;
return void Log.info("EVICT_ARCHIVED_BLOB_DRY_RUN", item, next);
}
blobs.remove.archived.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_ARCHIVED_BLOB_ERROR", item);
@ -288,6 +301,7 @@ var evictArchived = function (Env, cb) {
}));
};
if (Env.DRY_RUN) { Env.Log.info('DRY RUN'); }
nThen(loadStorage)
.nThen(migrateIncorrectBlobs)
.nThen(removeArchivedChannels)
@ -544,6 +558,9 @@ module.exports = function (Env, cb) {
}
// remove the pin logs of inactive accounts if inactive account removal is configured
if (Env.DRY_RUN) {
return void Log.info("EVICT_INACTIVE_ACCOUNT_DRY_RUN", id, next);
}
pinStore.archiveChannel(id, undefined, function (err) {
if (err) {
return Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err, next);
@ -602,7 +619,12 @@ module.exports = function (Env, cb) {
// unless we address this race condition with this last-minute double-check
if (item.mtime > inactiveTime) { return void next(); }
removed++;
if (Env.DRY_RUN) {
removed++;
return void Log.info("EVICT_ARCHIVE_BLOB_DRY_RUN", {
item: item,
}, next);
}
blobs.archive.blob(item.blobId, 'INACTIVE', function (err) {
if (err) {
return Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
@ -610,6 +632,7 @@ module.exports = function (Env, cb) {
item: item,
}, next);
}
removed++;
Log.info("EVICT_ARCHIVE_BLOB", {
item: item,
}, next);
@ -658,6 +681,10 @@ module.exports = function (Env, cb) {
}
}));
}).nThen(function () {
if (Env.DRY_RUN) {
removed++;
return void Log.info("EVICT_BLOB_PROOF_LONELY_DRY_RUN", item, next);
}
blobs.remove.proof(item.safeKey, item.blobId, function (err) {
if (err) {
return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item, next);
@ -698,6 +725,9 @@ module.exports = function (Env, cb) {
// check if the database has any ephemeral channels
// if it does it's because of a bug, and they should be removed
if (item.channel.length === 34) {
if (Env.DRY_RUN) {
return void Log.info("EVICT_EPHEMERAL_DRY_RUN", item.channel, cb);
}
return void store.removeChannel(item.channel, w(function (err) {
if (err) {
return Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', {
@ -728,6 +758,11 @@ module.exports = function (Env, cb) {
// else fall through to the archival
}));
}).nThen(function (w) {
if (Env.DRY_RUN) {
archived++;
w.abort();
return void Log.info("EVICT_CHANNEL_ARCHIVAL_DRY_RUN", item.channel, cb);
}
return void store.archiveChannel(item.channel, 'INACTIVE', w(function (err) {
if (err) {
Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', {
@ -736,8 +771,8 @@ module.exports = function (Env, cb) {
}, w());
return;
}
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel, w());
archived++;
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel, w());
}));
}).nThen(cb);
};
@ -754,6 +789,7 @@ module.exports = function (Env, cb) {
store.listChannels(handler, w(done), true); // using a hacky "fast mode" since we only need the channel id
};
if (Env.DRY_RUN) { Env.Log.info('DRY RUN'); }
nThen(loadStorage)
// iterate over all documents and add them to a bloom filter if they have been active

69
lib/storage/blob.js
View File

@ -153,8 +153,12 @@ var clearActivity = function (Env, blobId, cb) {
};
var updateActivity = function (Env, blobId, cb) {
var path = makeActivityPath(Env, blobId);
var s_data = String(+new Date());
Fs.writeFile(path, s_data, cb);
var blobPath = makeBlobPath(Env, blobId);
isFile(blobPath, (err, state) => {
if (err || !state) { return void cb(); }
var s_data = String(+new Date());
Fs.writeFile(path, s_data, cb);
});
};
var archiveActivity = function (Env, blobId, cb) {
@ -464,7 +468,7 @@ var makeWalker = function (n, handleChild, done) {
// do no more than 20 jobs at a time
var tasks = Semaphore.create(n);
var recurse = function (path) {
var recurse = function (path, dir) {
tasks.take(function (give) {
var next = give(W());
@ -477,7 +481,19 @@ var makeWalker = function (n, handleChild, done) {
}
if (!stats.isDirectory()) {
w.abort();
return void handleChild(void 0, path, next);
if (/\.activity$/.test(path)) {
// NOTE: some activity files were created for deleted blobs due to
// a bug. We're going to detect them here in order to be able to clean
// them.
if (!dir.includes(Path.basename(path.replace(/\.activity$/, '')))) {
return void handleChild(void 0, path, next, true);
}
// Ignore valid activity files
return next();
}
// Ignore placeholder files
if (/\.placeholder$/.test(path)) { return next(); }
return void handleChild(void 0, path, next, false);
}
// fall through
}));
@ -487,7 +503,7 @@ var makeWalker = function (n, handleChild, done) {
if (err) { return next(); }
// everything is fine and it's a directory...
dir.forEach(function (d) {
recurse(Path.join(path, d));
recurse(Path.join(path, d), dir);
});
next();
});
@ -502,7 +518,8 @@ var listProofs = function (root, handler, cb) {
Fs.readdir(root, function (err, dir) {
if (err) { return void cb(err); }
var walk = makeWalker(20, function (err, path, next) {
var walk = makeWalker(20, function (err, path, next, loneActivity) {
if (loneActivity) { return void next(); }
// path is the path to a child node on the filesystem
// next handles the next job in a queue
@ -537,12 +554,20 @@ var listProofs = function (root, handler, cb) {
});
};
var getActivityStat = function (path, base, cb) {
var suffix = base ? '' : '.activity';
Fs.stat(path+suffix, function (err, stats) {
if (err && err.code === 'ENOENT' && !base) { return getActivityStat(path, true, cb); }
cb(err, stats);
});
};
var listBlobs = function (root, handler, cb) {
// iterate over files
Fs.readdir(root, function (err, dir) {
if (err) { return void cb(err); }
var walk = makeWalker(20, function (err, path, next) {
Fs.stat(path, function (err, stats) {
var walk = makeWalker(20, function (err, path, next, loneActivity) {
if (loneActivity) { return void next(); }
getActivityStat(path, false, function (err, stats) {
if (err) {
return void handler(err, void 0, next);
}
@ -565,6 +590,30 @@ var listBlobs = function (root, handler, cb) {
});
};
var cleanLoneActivity = function (root, cb) {
// iterate over files
Fs.readdir(root, function (err, dir) {
if (err) { return void cb(err); }
var walk = makeWalker(20, function (err, path, next, loneActivity) {
if (!loneActivity) { return void next(); }
Fs.unlink(path, function (err) {
if (err) {
return console.error('ERROR', path, err);
}
console.log('DELETED', path);
next();
});
}, function () {
cb();
});
dir.forEach(function (d) {
if (d.length !== 2) { return; }
walk(Path.join(root, d));
});
});
};
BlobStore.create = function (config, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (typeof(config.getSession) !== 'function') {
@ -651,6 +700,10 @@ BlobStore.create = function (config, _cb) {
removeArchivedProof(Env, safeKey, blobId, cb);
},
},
loneActivity: function (_cb) {
var cb = Util.once(Util.mkAsync(_cb));
cleanLoneActivity(Env.blobPath, cb);
}
},
archive: {

20
scripts/clean-activity.js Normal file
View File

@ -0,0 +1,20 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
/**
* Some .activity file were created for deleted blob due to a bug.
* This script can be run once to remove these invalid activity file.
**/
var config = require("../lib/load-config");
var BlobStore = require("../lib/storage/blob");
config.getSession = function () {};
BlobStore.create(config, function (err, store) {
if (err) { return console.error('ERROR', err); }
console.log('Cleaning lone .activity files...');
store.remove.loneActivity(function (err) {
if (err) { return console.error('ERROR', err); }
console.log('Done');
});
});

4
scripts/evict-archived.js
View File

@ -15,6 +15,10 @@ var config = require("../lib/load-config");
var Env = Environment.create(config);
// Set DRY_RUN to true to run the script without deleting anything. A log file
// will be created.
Env.DRY_RUN = false;
var loadPremiumAccounts = function (Env, cb) {
nThen(function (w) {
// load premium accounts

4
scripts/evict-inactive.js
View File

@ -15,6 +15,10 @@ var config = require("../lib/load-config");
var Env = Environment.create(config);
// Set DRY_RUN to true to run the script without deleting anything. A log file
// will be created.
Env.DRY_RUN = false;
var loadPremiumAccounts = function (Env, cb) {
nThen(function (w) {
// load premium accounts