Merge branch 'blob-archive-paths' into soon

lib/eviction.js

@@ -3,6 +3,10 @@ var Bloom = require("@mcrowe/minibloom");
 var Util = require("../lib/common-util");
 var Pins = require("../lib/pins");
 var Keys = require("./keys");
+var Path = require('node:path');
+var config = require("./load-config");
+var Fs = require("node:fs");
+var Fse = require("fs-extra");
 
 var getNewestTime = function (stats) {
     return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
@@ -70,6 +74,103 @@ var evictArchived = function (Env, cb) {
         blobs = Env.blobStore;
     };
 
+    var migrateBlobRoot = function (from, to) {
+        // only migrate subpaths, leave everything else alone
+        if (!Path.dirname(from).startsWith(Path.dirname(to))) { return; }
+
+        // expects a directory
+        var recurse = function (relativePath) {
+            var src = Path.join(from, relativePath);
+            var children;
+            try {
+                children = Fs.readdirSync(src);
+            } catch (err) {
+                if (err.code === 'ENOENT') { return; }
+                // if you can't read a directory's contents
+                // then nothing else will work, so just abort
+                Log.verbose("EVICT_ARCHIVED_NOT_DIRECTORY", {
+                    error: err,
+                });
+                return;
+            }
+
+            var dest;
+            if (children.length === 0) {
+                try {
+                    Fse.removeSync(src);
+                } catch (err2) {
+                    Log.error('EVICT_ARCHIVED_EMPTY_DIR_REMOVAL', {
+                        error: err2,
+                    });
+                    // removal is non-essential, so we can continue
+                }
+            } else {
+                // make an equivalent path in the target directory
+                dest = Path.join(to, relativePath);
+
+                try {
+                    Fse.mkdirpSync(dest);
+                } catch (err3) {
+                    Log.error("EVICT_ARCHIVED_BLOB_MIGRATION", {
+                        error: err3,
+                    });
+
+                    // failure to create the host directory
+                    // will cause problems when we try to move
+                    // so bail out here
+                    return;
+                }
+            }
+
+            children.forEach(function (child) {
+                var childSrcPath = Path.join(src, child);
+                var stat = Fs.statSync(childSrcPath);
+                if (stat.isDirectory()) {
+                    return void recurse(Path.join(relativePath, child));
+                }
+
+                var childDestPath = Path.join(dest, child);
+
+                try {
+                    Log.verbose("EVICT_ARCHIVED_MOVE_FROM_DEPRECATED_PATH", {
+                        from: childSrcPath,
+                        to: childDestPath,
+                    });
+                    Fse.moveSync(childSrcPath, childDestPath, {
+                        overwrite: false,
+                    });
+                } catch (err4) {
+                    Log.error('EVICT_ARCHIVED_MOVE_FAILURE', {
+                        error: err4,
+                    });
+                }
+            });
+        };
+        recurse('');
+    };
+
+/*  In CryptPad 5.2.0 we merged a patch which converted
+    all of CryptPad's root filepaths to their absolute form,
+    rather than the relative paths we'd been using until then.
+    Unfortunately, we overlooked a case where two absolute
+    paths were concatenated together, resulting in blobs being
+    archived to an incorrect path.
+
+    This migration detects evidence of incorrect archivals
+    and moves such archived files to their intended location
+    before continuing with the normal eviction procedure.
+*/
+    var migrateIncorrectBlobs = function () {
+        var incorrectPaths = [
+            Path.join(Env.paths.archive, config.blobPath),
+            Path.join(Env.paths.archive, Path.resolve(config.blobPath))
+        ];
+        var correctPath = Path.join(Env.paths.archive, 'blob');
+        incorrectPaths.forEach(root => {
+            migrateBlobRoot(root, correctPath);
+        });
+    };
+
     var removeArchivedChannels = function (w) {
         // this block will iterate over archived channels and removes them
         // if they've been in cold storage for longer than your configured archive time
@@ -186,6 +287,7 @@ var evictArchived = function (Env, cb) {
     };
 
     nThen(loadStorage)
+    .nThen(migrateIncorrectBlobs)
     .nThen(removeArchivedChannels)
     .nThen(removeArchivedBlobProofs)
     .nThen(removeArchivedBlobs)

lib/storage/blob.js

@@ -19,7 +19,11 @@ var isValidId = function (id) {
 // helpers
 
 var prependArchive = function (Env, path) {
-    return Path.join(Env.archivePath, path);
+    // Env has an absolute path to the blob storage
+    // we want the path to the blob relative to that
+    var relativePathToBlob = Path.relative(Env.blobPath, path);
+    // the new path structure is the same, but relative to the blob archive root
+    return Path.join(Env.archivePath, 'blob', relativePathToBlob);
 };
 
 // /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
@@ -492,7 +496,7 @@ BlobStore.create = function (config, _cb) {
             if (e) { CB(e); }
         }));
 
-        Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) {
+        Fse.mkdirp(Path.join(Env.archivePath, './blob'), w(function (e) {
             if (e) { CB(e); }
         }));
     }).nThen(function (w) {