diff --git a/lib/api.js b/lib/api.js index df3206b1e..d8313521c 100644 --- a/lib/api.js +++ b/lib/api.js @@ -27,6 +27,31 @@ nThen(function (w) { console.error(err); } })); +}).nThen(function (w) { + if (Env.proofsMigrated) { return; } + const { Worker } = require('node:worker_threads'); + const Admin = require("./commands/admin-rpc"); + + const worker = new Worker('./scripts/migrations/migrate-blob-proofs.js'); + + worker.on('message', message => { + if (message === 'READY') { + log.info('BLOB_PROOFS_MIGRATION'); + return void worker.postMessage({ + start: 1, + }); + } + if (message === 'MIGRATED') { + return void log.info('BLOB_PROOFS_DELETION'); + } + if (message === 'CLEANED') { + log.info('BLOB_PROOFS_MIGRATED'); + Admin.sendDecree(Env, null, function (err) { + if (err) { return void log.error('BLOB_PROOF', err); } + Env.flushCache(); + }, ['PROOFS_MIGRATED', ['PROOFS_MIGRATED', 1]], 'server'); + } + }); }).nThen(function (w) { let admins = Env.admins || []; diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 9030159f8..d029fc079 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -409,7 +409,7 @@ var getChannelMetadata = function (Env, Server, cb, data) { }; // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['RESTRICT_REGISTRATION', [true]]], console.log) -var adminDecree = function (Env, Server, cb, data, unsafeKey) { +var adminDecree = Admin.sendDecree = function (Env, Server, cb, data, unsafeKey) { var value = data[1]; if (!Array.isArray(value)) { return void cb('INVALID_DECREE'); } diff --git a/lib/decrees.js b/lib/decrees.js index 0124d9d90..c52476865 100644 --- a/lib/decrees.js +++ b/lib/decrees.js @@ -395,6 +395,14 @@ commands.ADD_ADMIN_KEY = function (Env, args) { return true; }; +commands.PROOFS_MIGRATED = function (Env, args) { + if (args !== 1) { + throw new Error("INVALID_ARGS"); + } + Env.proofsMigrated = true; + return true; +}; + commands.SET_BEARER_SECRET = function (Env, args) { if (!args_isString(args) || args.length !== 1 || !args[0]) { throw new Error("INVALID_ARGS"); diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 63932a2ce..f3d44c5d7 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -40,6 +40,7 @@ var makeBlobPath = function (Env, blobId) { return Path.join(Env.blobPath, blobId.slice(0, 2), blobId); }; + var makeActivityPath = function (Env, blobId) { return makeBlobPath(Env, blobId) + '.activity'; }; @@ -116,6 +117,18 @@ var isFile = function (filePath, cb) { }); }; +// PROOFS +// DEPRECATED, keep for compatibility +// /blob//// +var makeProofPath = function (Env, safeKey, blobId) { + return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId); +}; +// isOwnedBy(id, safeKey) +var isOwnedBy = function (Env, safeKey, blobId, cb) { + var proofPath = makeProofPath(Env, safeKey, blobId); + isFile(proofPath, cb); +}; + var makeFileStream = function (full, _cb) { var cb = Util.once(Util.mkAsync(_cb)); Fse.mkdirp(Path.dirname(full), function (e) { diff --git a/scripts/migrations/migrate-blob-proofs.js b/scripts/migrations/migrate-blob-proofs.js index 8c977693c..c8f87bb3a 100644 --- a/scripts/migrations/migrate-blob-proofs.js +++ b/scripts/migrations/migrate-blob-proofs.js @@ -2,24 +2,24 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later +const { parentPort } = require('node:worker_threads'); const Path = require('node:path'); +const Fs = require('node:fs'); const nThen = require("nthen"); const Semaphore = require("saferphore"); const Logger = require("../../lib/log"); -const config = require("../../lib/load-config"); const BlobStorage = require("../../lib/storage/blob"); -const Fs = require('node:fs'); +let config = require("../../lib/load-config"); const blobPath = config.blobPath || './blob'; let Log = {}; -// XXX NOTE: in cleaning mode, we DON'T migrate +// NOTE: in cleaning mode, we DON'T migrate // (we suppose data has already been migrated) -const DRY_RUN = true; -const CLEAN_OLD = false; +const start = (clean, dry, cb) => { + const DRY_RUN = dry; -const start = (clean) => { let dirList = []; let blobStore; nThen(w => { @@ -156,9 +156,43 @@ const start = (clean) => { }); n(() => { Log.info("DONE"); - process.exit(0); + cb(); }); }); }; -start(CLEAN_OLD); +if (parentPort) { + // Loaded as worker script + config = JSON.parse(JSON.stringify(config)); + config.logToStdout = false; + parentPort.on('message', (message) => { + let parsed = message; //JSON.parse(message); + if (!parsed?.start) { return; } + // Migrate + start(false, false, () => { + parentPort.postMessage('MIGRATED'); + // If success, clean + start(true, false, () => { + parentPort.postMessage('CLEANED'); + }); + }); + }); + parentPort.postMessage('READY'); +} else if (require.main === module) { + // Loaded from command-line + let dry = false; + let clean = false; + process.argv.forEach(key => { + if (key === '--dry') { + dry = true; + return; + } + if (key === '--clean') { + clean = true; + return; + } + }); + start(clean, dry, () => { + process.exit(0); + }); +}