The_Great_Archive/cryptpad
1
0
Fork 0
mirror of https://github.com/cryptpad/cryptpad.git synced 2026-09-12 19:49:59 +05:00
Code Issues Packages Projects Releases Wiki Activity

Blob metadata and migration

Browse Source
This commit is contained in:
yflory 2025-01-06 16:52:59 +01:00
parent 1e2f57a28f
commit ededf142d9
2 changed files with 171 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/storage/blob.js
View File

@ -226,7 +226,7 @@ var archiveMetadata = (Env, blobId, cb) => {
// if we fail to delete the metadata file, it can still be removed later by the eviction script
Fse.move(path, archivePath, { overwrite: true }, cb);
};
var restoreActivity = function (Env, blobId, cb) {
var restoreMetadata = function (Env, blobId, cb) {
var path = mkMetadataPath(Env, blobId);
var archivePath = prependArchive(Env, path);
Fse.move(archivePath, path, cb);
@ -732,6 +732,12 @@ BlobStore.create = function (config, _cb) {
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
writeMetadata(Env, blobId, data, cb);
},
hasMetadata: (blobId, _cb) => {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
var path = mkMetadataPath(Env, blobId);
isFile(path, cb);
},
remove: {
blob: function (blobId, _cb) {

164
scripts/migrations/migrate-blob-proofs.js Normal file
View File

@ -0,0 +1,164 @@
// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Path = require('node:path');
const nThen = require("nthen");
const Semaphore = require("saferphore");
const Logger = require("../../lib/log");
const config = require("../../lib/load-config");
const BlobStorage = require("../../lib/storage/blob");
const Fs = require('node:fs');
const blobPath = config.blobPath || './blob';
let Log = {};
// XXX NOTE: in cleaning mode, we DON'T migrate
// (we suppose data has already been migrated)
const DRY_RUN = true;
const CLEAN_OLD = false;
const start = (clean) => {
let dirList = [];
let blobStore;
nThen(w => {
Logger.create(config, w(function (_log) {
Log = _log;
}));
}).nThen(w => {
config.getSession = function () {};
BlobStorage.create(config, w(function (err, _store) {
if (err) {
w.abort();
return void Log.error("ERR_BLOB_STORE", err);
}
blobStore = _store;
}));
}).nThen(w => {
Fs.readdir(blobPath, w((err, list) => {
if (err) {
w.abort();
return void Log.error("ERR_READING_ROOT", err);
}
dirList = list;
}));
}).nThen(() => {
let n = nThen;
dirList.forEach(dir => {
if (dir.length !== 3) { return; }
// ./blob/abc
const nestedDirPath = Path.join(blobPath, dir);
if (clean) {
n = n(ww => {
Log.info("REMOVING_DIR", nestedDirPath);
if (DRY_RUN) { return; }
Fs.rm(nestedDirPath, {
recursive: true, force: true
}, ww(err => {
if (err) {
Log.error("ERR_REMOVE_DIR", {
path: nestedDirPath,
err
});
}
}));
}).nThen;
return;
}
n = n(w => {
// One user at a time
const sema = Semaphore.create(1);
let nestedDirList = [];
nThen(ww => {
Fs.readdir(nestedDirPath, ww((err, list) => {
if (err) {
w.abort();
ww.abort();
return Log.error("ERR_READING_DIR", {
path: nestedDirPath,
err
});
}
nestedDirList = list;
}));
}).nThen(ww => {
nestedDirList.forEach(key => {
// ./blob/abc/abcdefg...
const keyPath = Path.join(nestedDirPath, key);
sema.take(give => {
let edPublic = key.replace(/\-/g, '/');
let md = JSON.stringify({ owners: [edPublic] });
Log.info("START_USER", edPublic);
Fs.readdir(keyPath, ww((err, list) => {
if (err) {
w.abort();
ww.abort();
return Log.error("ERR_READING_DIR", {
path: keyPath,
err
});
}
let blobs = []
nThen(www => {
list.forEach(dir => {
// ./blob/abc/abcdefg.../01
const path = Path.join(keyPath, dir);
Fs.readdir(path, www((err, blobsList) => {
if (err) {
w.abort();
ww.abort();
www.abort();
return Log.error("ERR_READING_DIR", {
path, err
});
}
Array.prototype.push.apply(blobs, blobsList);
}));
});
}).nThen(www => {
// migrate 20 blobs at a time for a given user
const sema = Semaphore.create(20);
blobs.forEach(blobId => {
sema.take(ggive => {
blobStore.isBlobAvailable(blobId, www((err, blobExists) => {
blobStore.hasMetadata(blobId, www((err, exists) => {
// If blob is not available or metadata already
// exists, don't write md file
if (!blobExists || exists) { return void ggive(); }
Log.info('WRITE_METADATA', blobId);
if (DRY_RUN) { return void ggive(); }
blobStore.writeMetadata(blobId, md, www(e => {
if (e) {
w.abort();
ww.abort();
www.abort();
return Log.error("ERR_WRITING_MD", { blobId });
}
ggive();
}));
}));
}));
})
});
}).nThen(ww(give(() => {
Log.info("END_USER", edPublic);
})));
}));
});
});
}).nThen(w());
}).nThen;
});
n(() => {
Log.info("DONE");
process.exit(0);
});
});
};
start(CLEAN_OLD);