Start blob proofs migration automatically

lib/api.js

@@ -27,6 +27,31 @@ nThen(function (w) {
             console.error(err);
         }
     }));
+}).nThen(function (w) {
+    if (Env.proofsMigrated) { return; }
+    const { Worker } = require('node:worker_threads');
+    const Admin = require("./commands/admin-rpc");
+
+    const worker = new Worker('./scripts/migrations/migrate-blob-proofs.js');
+
+    worker.on('message', message => {
+        if (message === 'READY') {
+            log.info('BLOB_PROOFS_MIGRATION');
+            return void worker.postMessage({
+                start: 1,
+            });
+        }
+        if (message === 'MIGRATED') {
+            return void log.info('BLOB_PROOFS_DELETION');
+        }
+        if (message === 'CLEANED') {
+            log.info('BLOB_PROOFS_MIGRATED');
+            Admin.sendDecree(Env, null, function (err) {
+                if (err) { return void log.error('BLOB_PROOF', err); }
+                Env.flushCache();
+            }, ['PROOFS_MIGRATED', ['PROOFS_MIGRATED', 1]], 'server');
+        }
+    });
 }).nThen(function (w) {
     let admins = Env.admins || [];
 

lib/commands/admin-rpc.js

@@ -409,7 +409,7 @@ var getChannelMetadata = function (Env, Server, cb, data) {
 };
 
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['RESTRICT_REGISTRATION', [true]]], console.log)
-var adminDecree = function (Env, Server, cb, data, unsafeKey) {
+var adminDecree = Admin.sendDecree = function (Env, Server, cb, data, unsafeKey) {
     var value = data[1];
     if (!Array.isArray(value)) { return void cb('INVALID_DECREE'); }
 

lib/decrees.js

@@ -395,6 +395,14 @@ commands.ADD_ADMIN_KEY = function (Env, args) {
     return true;
 };
 
+commands.PROOFS_MIGRATED = function (Env, args) {
+    if (args !== 1) {
+        throw new Error("INVALID_ARGS");
+    }
+    Env.proofsMigrated = true;
+    return true;
+};
+
 commands.SET_BEARER_SECRET = function (Env, args) {
     if (!args_isString(args) || args.length !== 1 || !args[0]) {
         throw new Error("INVALID_ARGS");

lib/storage/blob.js

@@ -40,6 +40,7 @@ var makeBlobPath = function (Env, blobId) {
     return Path.join(Env.blobPath, blobId.slice(0, 2), blobId);
 };
 
+
 var makeActivityPath = function (Env, blobId) {
     return makeBlobPath(Env, blobId) + '.activity';
 };
@@ -116,6 +117,18 @@ var isFile = function (filePath, cb) {
     });
 };
 
+// PROOFS
+// DEPRECATED, keep for compatibility
+// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
+var makeProofPath = function (Env, safeKey, blobId) {
+    return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId);
+};
+// isOwnedBy(id, safeKey)
+var isOwnedBy = function (Env, safeKey, blobId, cb) {
+    var proofPath = makeProofPath(Env, safeKey, blobId);
+    isFile(proofPath, cb);
+};
+
 var makeFileStream = function (full, _cb) {
     var cb = Util.once(Util.mkAsync(_cb));
     Fse.mkdirp(Path.dirname(full), function (e) {

scripts/migrations/migrate-blob-proofs.js

@@ -2,24 +2,24 @@
 //
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
+const { parentPort } = require('node:worker_threads');
 const Path = require('node:path');
+const Fs = require('node:fs');
 const nThen = require("nthen");
 const Semaphore = require("saferphore");
 const Logger = require("../../lib/log");
-const config = require("../../lib/load-config");
 const BlobStorage = require("../../lib/storage/blob");
-const Fs = require('node:fs');
+let config = require("../../lib/load-config");
 
 
 const blobPath = config.blobPath || './blob';
 let Log = {};
 
-// XXX NOTE: in cleaning mode, we DON'T migrate
+// NOTE: in cleaning mode, we DON'T migrate
 // (we suppose data has already been migrated)
-const DRY_RUN = true;
-const CLEAN_OLD = false;
+const start = (clean, dry, cb) => {
+    const DRY_RUN = dry;
 
-const start = (clean) => {
     let dirList = [];
     let blobStore;
     nThen(w => {
@@ -156,9 +156,43 @@ const start = (clean) => {
         });
         n(() => {
             Log.info("DONE");
-            process.exit(0);
+            cb();
         });
     });
 };
 
-start(CLEAN_OLD);
+if (parentPort) {
+    // Loaded as worker script
+    config = JSON.parse(JSON.stringify(config));
+    config.logToStdout = false;
+    parentPort.on('message', (message) => {
+        let parsed = message; //JSON.parse(message);
+        if (!parsed?.start) { return; }
+        // Migrate
+        start(false, false, () => {
+            parentPort.postMessage('MIGRATED');
+            // If success, clean
+            start(true, false, () => {
+                parentPort.postMessage('CLEANED');
+            });
+        });
+    });
+    parentPort.postMessage('READY');
+} else if (require.main === module) {
+    // Loaded from command-line
+    let dry = false;
+    let clean = false;
+    process.argv.forEach(key => {
+        if (key === '--dry') {
+            dry = true;
+            return;
+        }
+        if (key === '--clean') {
+            clean = true;
+            return;
+        }
+    });
+    start(clean, dry, () => {
+        process.exit(0);
+    });
+}