diff --git a/lib/eviction.js b/lib/eviction.js index 673c7313f..32e1bdd56 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -3,6 +3,10 @@ var Bloom = require("@mcrowe/minibloom"); var Util = require("../lib/common-util"); var Pins = require("../lib/pins"); var Keys = require("./keys"); +var Path = require('node:path'); +var config = require("./load-config"); +var Fs = require("node:fs"); +var Fse = require("fs-extra"); var getNewestTime = function (stats) { return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { @@ -70,6 +74,103 @@ var evictArchived = function (Env, cb) { blobs = Env.blobStore; }; + var migrateBlobRoot = function (from, to) { + // only migrate subpaths, leave everything else alone + if (!Path.dirname(from).startsWith(Path.dirname(to))) { return; } + + // expects a directory + var recurse = function (relativePath) { + var src = Path.join(from, relativePath); + var children; + try { + children = Fs.readdirSync(src); + } catch (err) { + if (err.code === 'ENOENT') { return; } + // if you can't read a directory's contents + // then nothing else will work, so just abort + Log.verbose("EVICT_ARCHIVED_NOT_DIRECTORY", { + error: err, + }); + return; + } + + var dest; + if (children.length === 0) { + try { + Fse.removeSync(src); + } catch (err2) { + Log.error('EVICT_ARCHIVED_EMPTY_DIR_REMOVAL', { + error: err2, + }); + // removal is non-essential, so we can continue + } + } else { + // make an equivalent path in the target directory + dest = Path.join(to, relativePath); + + try { + Fse.mkdirpSync(dest); + } catch (err3) { + Log.error("EVICT_ARCHIVED_BLOB_MIGRATION", { + error: err3, + }); + + // failure to create the host directory + // will cause problems when we try to move + // so bail out here + return; + } + } + + children.forEach(function (child) { + var childSrcPath = Path.join(src, child); + var stat = Fs.statSync(childSrcPath); + if (stat.isDirectory()) { + return void recurse(Path.join(relativePath, child)); + } + + var childDestPath = Path.join(dest, child); + + try { + Log.verbose("EVICT_ARCHIVED_MOVE_FROM_DEPRECATED_PATH", { + from: childSrcPath, + to: childDestPath, + }); + Fse.moveSync(childSrcPath, childDestPath, { + overwrite: false, + }); + } catch (err4) { + Log.error('EVICT_ARCHIVED_MOVE_FAILURE', { + error: err4, + }); + } + }); + }; + recurse(''); + }; + +/* In CryptPad 5.2.0 we merged a patch which converted + all of CryptPad's root filepaths to their absolute form, + rather than the relative paths we'd been using until then. + Unfortunately, we overlooked a case where two absolute + paths were concatenated together, resulting in blobs being + archived to an incorrect path. + + This migration detects evidence of incorrect archivals + and moves such archived files to their intended location + before continuing with the normal eviction procedure. +*/ + var migrateIncorrectBlobs = function () { + var incorrectPaths = [ + Path.join(Env.paths.archive, config.blobPath), + Path.join(Env.paths.archive, Path.resolve(config.blobPath)) + ]; + var correctPath = Path.join(Env.paths.archive, 'blob'); + incorrectPaths.forEach(root => { + migrateBlobRoot(root, correctPath); + }); + }; + var removeArchivedChannels = function (w) { // this block will iterate over archived channels and removes them // if they've been in cold storage for longer than your configured archive time @@ -186,6 +287,7 @@ var evictArchived = function (Env, cb) { }; nThen(loadStorage) + .nThen(migrateIncorrectBlobs) .nThen(removeArchivedChannels) .nThen(removeArchivedBlobProofs) .nThen(removeArchivedBlobs) diff --git a/lib/storage/blob.js b/lib/storage/blob.js index a2d9930f3..7d55676d7 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -19,7 +19,11 @@ var isValidId = function (id) { // helpers var prependArchive = function (Env, path) { - return Path.join(Env.archivePath, path); + // Env has an absolute path to the blob storage + // we want the path to the blob relative to that + var relativePathToBlob = Path.relative(Env.blobPath, path); + // the new path structure is the same, but relative to the blob archive root + return Path.join(Env.archivePath, 'blob', relativePathToBlob); }; // /blob//// @@ -492,7 +496,7 @@ BlobStore.create = function (config, _cb) { if (e) { CB(e); } })); - Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) { + Fse.mkdirp(Path.join(Env.archivePath, './blob'), w(function (e) { if (e) { CB(e); } })); }).nThen(function (w) {