From ce572e813e4c97c1739b704d2fa777e478ae6b3e Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 23 Aug 2023 10:57:11 +0200 Subject: [PATCH 01/21] Update eviction script --- lib/eviction.js | 98 +++++++++++++++++++------------------------------ lib/log.js | 15 +++++--- 2 files changed, 48 insertions(+), 65 deletions(-) diff --git a/lib/eviction.js b/lib/eviction.js index 6a631de8c..9cf6a5558 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -186,8 +186,7 @@ var evictArchived = function (Env, cb) { var handler = function (err, item, cb) { if (err) { - Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err); - return void cb(); + return Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err, cb); } // don't mess with files that are freshly stored in cold storage // based on ctime because that's changed when the file is moved... @@ -199,13 +198,11 @@ var evictArchived = function (Env, cb) { // expire it store.removeArchivedChannel(item.channel, w(function (err) { if (err) { - Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', { + return Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', { error: err, channel: item.channel, - }); - return void cb(); + }, cb); } - Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel); if (item.channel.length === 32) { removed++; @@ -213,7 +210,7 @@ var evictArchived = function (Env, cb) { accounts++; } - cb(); + Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel, cb); })); }; @@ -399,8 +396,7 @@ module.exports = function (Env, cb) { } if (err) { - Log.error('EVICT_CHANNEL_CATEGORIZATION', err); - return void cb(); + return Log.error('EVICT_CHANNEL_CATEGORIZATION', err, cb); } // if the channel has been modified recently @@ -421,7 +417,7 @@ module.exports = function (Env, cb) { Log.info('EVICT_CHANNELS_CATEGORIZED', { active: active, channels: channels, - }); + }, w()); }; Log.info('EVICT_CHANNEL_ACTIVITY_START', 'Assessing channel activity'); @@ -443,12 +439,10 @@ module.exports = function (Env, cb) { } if (err) { - Log.error("EVICT_BLOB_CATEGORIZATION", err); - return void next(); + return Log.error("EVICT_BLOB_CATEGORIZATION", err, next); } if (!item) { - next(); - return void Log.error("EVICT_BLOB_CATEGORIZATION_INVALID", item); + return void Log.error("EVICT_BLOB_CATEGORIZATION_INVALID", item, next); } if (item.mtime > inactiveTime) { activeDocs.add(item.blobId); @@ -462,7 +456,7 @@ module.exports = function (Env, cb) { Log.info('EVICT_BLOBS_CATEGORIZED', { active: active, blobs: n_blobs, - }); + }, w()); })); }; @@ -530,31 +524,27 @@ module.exports = function (Env, cb) { // we plan to delete them, because it may be interesting information inactive++; if (PRESERVE_INACTIVE_ACCOUNTS) { - Log.info('EVICT_INACTIVE_ACCOUNT_PRESERVED', { + pinAll(pinList); + return Log.info('EVICT_INACTIVE_ACCOUNT_PRESERVED', { id: id, mtime: mtime, - }); - pinAll(pinList); - return void next(); + }, next); } if (isPremiumAccount(id)) { - Log.info("EVICT_INACTIVE_PREMIUM_ACCOUNT", { + pinAll(pinList); + return Log.info("EVICT_INACTIVE_PREMIUM_ACCOUNT", { id: id, mtime: mtime, - }); - pinAll(pinList); - return void next(); + }, next); } // remove the pin logs of inactive accounts if inactive account removal is configured pinStore.archiveChannel(id, function (err) { if (err) { - Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err); - return void next(); + return Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err, next); } - Log.info('EVICT_INACTIVE_ACCOUNT_LOG', id); - next(); + Log.info('EVICT_INACTIVE_ACCOUNT_LOG', id, next); }); }; @@ -588,12 +578,10 @@ module.exports = function (Env, cb) { blobs.list.blobs(function (err, item, next) { next = Util.mkAsync(next, THROTTLE_FACTOR); if (err) { - Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err); - return void next(); + return Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err, next); } if (!item) { - next(); - return void Log.error('EVICT_BLOB_LIST_BLOBS_NO_ITEM', item); + return void Log.error('EVICT_BLOB_LIST_BLOBS_NO_ITEM', item, next); } total++; if (total % PROGRESS_FACTOR === 0) { @@ -613,21 +601,19 @@ module.exports = function (Env, cb) { removed++; blobs.archive.blob(item.blobId, function (err) { if (err) { - Log.error("EVICT_ARCHIVE_BLOB_ERROR", { + return Log.error("EVICT_ARCHIVE_BLOB_ERROR", { error: err, item: item, - }); - return void next(); + }, next); } Log.info("EVICT_ARCHIVE_BLOB", { item: item, - }); - next(); + }, next); }); }, w(function () { report.totalBlobs = total; report.activeBlobs = total - removed; - Log.info('EVICT_BLOBS_REMOVED', removed); + Log.info('EVICT_BLOBS_REMOVED', removed, w()); })); }; @@ -641,12 +627,10 @@ module.exports = function (Env, cb) { blobs.list.proofs(function (err, item, next) { next = Util.mkAsync(next, THROTTLE_FACTOR); if (err) { - next(); - return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err); + return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next); } if (!item) { - next(); - return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item); + return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item, next); } total++; @@ -662,8 +646,7 @@ module.exports = function (Env, cb) { blobs.size(item.blobId, w(function (err, size) { if (err) { w.abort(); - next(); - return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err); + return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next); } if (size !== 0) { w.abort(); @@ -672,19 +655,18 @@ module.exports = function (Env, cb) { })); }).nThen(function () { blobs.remove.proof(item.safeKey, item.blobId, function (err) { - next(); if (err) { - return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item); + return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item, next); } removed++; - return Log.info("EVICT_BLOB_PROOF_LONELY", item); + return Log.info("EVICT_BLOB_PROOF_LONELY", item, next); }); }); }, w(function () { Log.info("EVICT_BLOB_PROOFS_REMOVED", { removed, total, - }); + }, w()); })); }; @@ -703,8 +685,7 @@ module.exports = function (Env, cb) { } if (err) { - Log.error('EVICT_CHANNEL_ITERATION', err); - return void cb(); + return Log.error('EVICT_CHANNEL_ITERATION', err, cb); } // ignore the special admin broadcast channel @@ -715,14 +696,12 @@ module.exports = function (Env, cb) { if (item.channel.length === 34) { return void store.removeChannel(item.channel, w(function (err) { if (err) { - Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', { + return Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', { error: err, channel: item.channel, - }); - return void cb(); + }, cb); } - Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel); - cb(); + Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel, cb); })); } @@ -744,20 +723,19 @@ module.exports = function (Env, cb) { } // else fall through to the archival })); - }).nThen(function () { + }).nThen(function (w) { return void store.archiveChannel(item.channel, w(function (err) { if (err) { Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { error: err, channel: item.channel, - }); - return void cb(); + }, w()); + return; } - Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); + Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel, w()); archived++; - cb(); })); - }); + }).nThen(cb); }; var done = function () { diff --git a/lib/log.js b/lib/log.js index abd8dee8e..d9aa85505 100644 --- a/lib/log.js +++ b/lib/log.js @@ -1,5 +1,6 @@ /*jshint esversion: 6 */ var Store = require("./storage/file"); +var Util = require("../common-util"); var Logger = module.exports; @@ -15,9 +16,13 @@ var messageTemplate = function (type, time, tag, info) { var noop = function () {}; -var write = function (ctx, content) { - if (!ctx.store) { return; } - ctx.store.log(ctx.channelName, content, noop); +var write = function (ctx, content, cb) { + if (typeof(cb) !== "function") { cb = noop; } + if (!ctx.store) { + cb = Util.mkAsync(cb); + return void cb(); + } + ctx.store.log(ctx.channelName, content, cb); }; // various degrees of logging @@ -37,7 +42,7 @@ var createLogType = function (ctx, type) { if (logLevels.indexOf(type) < logLevels.indexOf(ctx.logLevel)) { return noop; } - return function (tag, info) { + return function (tag, info, cb) { if (ctx.shutdown) { throw new Error("Logger has been shut down!"); } @@ -51,7 +56,7 @@ var createLogType = function (ctx, type) { if (ctx.logToStdout && typeof(handlers[type]) === 'function') { handlers[type](ctx, content); } - write(ctx, content); + write(ctx, content, cb); }; }; From 31a5cbafdb1e0676d854b3cf195321d60c40a8ce Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 23 Aug 2023 10:59:02 +0200 Subject: [PATCH 02/21] Fix typo --- lib/log.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/log.js b/lib/log.js index d9aa85505..c300fabab 100644 --- a/lib/log.js +++ b/lib/log.js @@ -1,6 +1,6 @@ /*jshint esversion: 6 */ var Store = require("./storage/file"); -var Util = require("../common-util"); +var Util = require("./common-util"); var Logger = module.exports; From 27b9c9bac372f500c387b519d6fff4e1720d3707 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 29 Aug 2023 17:50:39 +0200 Subject: [PATCH 03/21] Update blob activity when loaded --- lib/http-worker.js | 19 +++++++++++++++ lib/storage/blob.js | 51 ++++++++++++++++++++++++++++++++++++++++- www/common/media-tag.js | 39 +++++++++++++++++++------------ 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/lib/http-worker.js b/lib/http-worker.js index 8e3a10187..995bcdd7e 100644 --- a/lib/http-worker.js +++ b/lib/http-worker.js @@ -9,6 +9,7 @@ const Logger = require("./log"); const AuthCommands = require("./http-commands"); const MFA = require("./storage/mfa"); const Sessions = require("./storage/sessions"); +const BlobStore = require("./storage/blob"); const DEFAULT_QUERY_TIMEOUT = 5000; const PID = process.pid; @@ -206,6 +207,13 @@ app.use('/blob', function (req, res, next) { /* Head requests are used to check the size of a blob. Clients can configure a maximum size to download automatically, and can manually click to download blobs which exceed that limit. */ + const url = req.url; + if (typeof(url) === "string" && Env.blobStore) { + const s = url.split('/'); + if (s[1] && s[1].length === 2 && s[2] && s[2].length === Env.blobStore.BLOB_LENGTH) { + Env.blobStore.updateActivity(s[2], () => {}); + } + } if (req.method === 'HEAD') { Express.static(Path.resolve(Env.paths.blob), { setHeaders: function (res /*, path, stat */) { @@ -618,6 +626,17 @@ nThen(function (w) { // websocket traffic to the correct port (Env.websocketPort) wsProxy.upgrade(req, socket, head); }); + + var config = require("./load-config"); + BlobStore.create({ + blobPath: config.blobPath, + blobStagingPath: config.blobStagingPath, + archivePath: config.archivePath, + getSession: function () {}, + }, w(function (err, blob) { + if (err) { return; } + Env.blobStore = blob; + })); }).nThen(function () { // TODO inform the parent process that this worker is ready diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 7d55676d7..38d4a906f 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -8,12 +8,14 @@ var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../common-util"); +const BLOB_LENGTH = 48; + var isValidSafeKey = function (safeKey) { return typeof(safeKey) === 'string' && !/\//.test(safeKey) && safeKey.length === 44; }; var isValidId = function (id) { - return typeof(id) === 'string' && id.length === 48 && !/[^a-f0-9]/.test(id); + return typeof(id) === 'string' && id.length === BLOB_LENGTH && !/[^a-f0-9]/.test(id); }; // helpers @@ -31,6 +33,10 @@ var makeBlobPath = function (Env, blobId) { return Path.join(Env.blobPath, blobId.slice(0, 2), blobId); }; +var makeActivityPath = function (Env, blobId) { + return makeBlobPath(Env, blobId) + '.activity'; +}; + // /blobstate// var makeStagePath = function (Env, safeKey) { return Path.join(Env.blobStagingPath, safeKey.slice(0, 2), safeKey); @@ -103,6 +109,31 @@ var makeFileStream = function (full, _cb) { }); }; +var clearActivity = function (Env, blobId, cb) { + var path = makeActivityPath(Env, blobId); + // if we fail to delete the activity file, it can still be removed later by the eviction script + Fs.unlink(path, cb); +}; + +var updateActivity = function (Env, blobId, cb) { + var path = makeActivityPath(Env, blobId); + var s_data = String(+new Date()); + Fs.writeFile(path, s_data, cb); +}; + +var getActivity = function (Env, blobId, cb) { + var path = makeActivityPath(Env, blobId); + Fs.readFile(path, function (err, content) { + if (err) { return void cb(err); } + try { + var date = new Date(+content); + cb(void 0, date); + } catch (err2) { + cb(err2); + } + }); +}; + /********** METHODS **************/ var upload = function (Env, safeKey, content, cb) { @@ -506,6 +537,7 @@ BlobStore.create = function (config, _cb) { Fse.writeFile(fullPath, 'PLACEHOLDER\n', w()); }).nThen(function () { var methods = { + BLOB_LENGTH: BLOB_LENGTH, isFileId: isValidId, status: function (safeKey, _cb) { // TODO check if the final destination is a file @@ -623,6 +655,23 @@ BlobStore.create = function (config, _cb) { getUploadSize(Env, id, cb); }, + // ACTIVITY + clearActivity: function (id, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(id)) { return void cb("INVALID_ID"); } + clearActivity(Env, id, cb); + }, + updateActivity: function (id, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(id)) { return void cb("INVALID_ID"); } + updateActivity(Env, id, cb); + }, + getActivity: function (id, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(id)) { return void cb("INVALID_ID"); } + getActivity(Env, id, cb); + }, + list: { blobs: function (handler, _cb) { var cb = Util.once(Util.mkAsync(_cb)); diff --git a/www/common/media-tag.js b/www/common/media-tag.js index ffc024857..e96a612c2 100644 --- a/www/common/media-tag.js +++ b/www/common/media-tag.js @@ -234,6 +234,22 @@ var factory = function () { config.Cache.setBlobCache(id, u8, cb); }; + var headRequest = function (src, cb) { + var xhr = new XMLHttpRequest(); + xhr.open("HEAD", src); + if (sendCredentials) { xhr.withCredentials = true; } + xhr.onerror = function () { return void cb("XHR_ERROR"); }; + xhr.onreadystatechange = function() { + if (this.readyState === this.DONE) { + cb(null, Number(xhr.getResponseHeader("Content-Length"))); + } + }; + xhr.onload = function () { + if (/^4/.test('' + this.status)) { return void cb("XHR_ERROR " + this.status); } + }; + xhr.send(); + + }; var getFileSize = function (src, _cb) { var cb = function (e, res) { _cb(e, res); @@ -243,25 +259,14 @@ var factory = function () { var cacheKey = getCacheKey(src); var check = function () { - var xhr = new XMLHttpRequest(); - xhr.open("HEAD", src); - if (sendCredentials) { xhr.withCredentials = true; } - xhr.onerror = function () { return void cb("XHR_ERROR"); }; - xhr.onreadystatechange = function() { - if (this.readyState === this.DONE) { - cb(null, Number(xhr.getResponseHeader("Content-Length"))); - } - }; - xhr.onload = function () { - if (/^4/.test('' + this.status)) { return void cb("XHR_ERROR " + this.status); } - }; - xhr.send(); + headRequest(src, cb); }; if (!cacheKey) { return void check(); } getBlobCache(cacheKey, function (err, u8) { - if (err || !u8) { return void check(); } + check(); // send the HEAD request to update the blob activity + if (err || !u8) { return; } cb(null, 0); }); }; @@ -748,7 +753,11 @@ var factory = function () { }); }; - if (cfg.force) { dl(); return mediaObject; } + if (cfg.force) { + headRequest(src, function () {}); // Update activity + dl(); + return mediaObject; + } var maxSize = typeof(config.maxDownloadSize) === "number" ? config.maxDownloadSize : (5 * 1024 * 1024); From c09e191a16889eedf013e931468325a9bd38a786 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 30 Aug 2023 15:59:15 +0200 Subject: [PATCH 04/21] Blob activity archive --- lib/storage/blob.js | 30 +++++++++++++++++++++++------- package-lock.json | 2 +- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 38d4a906f..d2c0a7420 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -114,13 +114,29 @@ var clearActivity = function (Env, blobId, cb) { // if we fail to delete the activity file, it can still be removed later by the eviction script Fs.unlink(path, cb); }; - var updateActivity = function (Env, blobId, cb) { var path = makeActivityPath(Env, blobId); var s_data = String(+new Date()); Fs.writeFile(path, s_data, cb); }; +var archiveActivity = function (Env, blobId, cb) { + var path = makeActivityPath(Env, blobId); + var archivePath = prependArchive(Env, path); + // if we fail to delete the activity file, it can still be removed later by the eviction script + Fse.move(path, archivePath, { overwrite: true }, cb); +}; +var removeArchivedActivity = function (Env, blobId, cb) { + var path = makeActivityPath(Env, blobId); + var archivePath = prependArchive(Env, path); + Fs.unlink(archivePath, cb); +}; +var restoreActivity = function (Env, blobId, cb) { + var path = makeActivityPath(Env, blobId); + var archivePath = prependArchive(Env, path); + Fse.move(archivePath, path, cb); +}; + var getActivity = function (Env, blobId, cb) { var path = makeActivityPath(Env, blobId); Fs.readFile(path, function (err, content) { @@ -333,6 +349,7 @@ var owned_upload_complete = function (Env, safeKey, id, cb) { var remove = function (Env, blobId, cb) { var blobPath = makeBlobPath(Env, blobId); Fs.unlink(blobPath, cb); + clearActivity(Env, blobId, () => {}); }; // removeProof @@ -353,11 +370,13 @@ var archiveBlob = function (Env, blobId, cb) { var blobPath = makeBlobPath(Env, blobId); var archivePath = prependArchive(Env, blobPath); Fse.move(blobPath, archivePath, { overwrite: true }, cb); + archiveActivity(Env, blobId, () => {}); }; var removeArchivedBlob = function (Env, blobId, cb) { var archivePath = prependArchive(Env, makeBlobPath(Env, blobId)); Fs.unlink(archivePath, cb); + removeArchivedActivity(Env, blobId, () => {}); }; // restoreBlob @@ -365,6 +384,7 @@ var restoreBlob = function (Env, blobId, cb) { var blobPath = makeBlobPath(Env, blobId); var archivePath = prependArchive(Env, blobPath); Fse.move(archivePath, blobPath, cb); + restoreActivity(Env, blobId, () => {}); }; // archiveProof @@ -655,12 +675,7 @@ BlobStore.create = function (config, _cb) { getUploadSize(Env, id, cb); }, - // ACTIVITY - clearActivity: function (id, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidId(id)) { return void cb("INVALID_ID"); } - clearActivity(Env, id, cb); - }, + // ACTIVITY updateActivity: function (id, _cb) { var cb = Util.once(Util.mkAsync(_cb)); if (!isValidId(id)) { return void cb("INVALID_ID"); } @@ -690,6 +705,7 @@ BlobStore.create = function (config, _cb) { var cb = Util.once(Util.mkAsync(_cb)); listBlobs(prependArchive(Env, Env.blobPath), handler, cb); }, + // XXX activity } }, }; diff --git a/package-lock.json b/package-lock.json index 19931c216..6bdaea908 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "packages": { "": { "name": "cryptpad", - "version": "5.4.0", + "version": "5.4.1", "license": "AGPL-3.0+", "dependencies": { "@mcrowe/minibloom": "^0.2.0", From 708e36b3ee3f96af4c1440cf34d603460e9c81f1 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 5 Sep 2023 16:31:04 +0200 Subject: [PATCH 05/21] New admin command to archive an account --- lib/archive-account.js | 294 ++++++++++++++++++++++++++++++++++++++ lib/commands/admin-rpc.js | 62 ++++++++ 2 files changed, 356 insertions(+) create mode 100644 lib/archive-account.js diff --git a/lib/archive-account.js b/lib/archive-account.js new file mode 100644 index 000000000..da59c96f5 --- /dev/null +++ b/lib/archive-account.js @@ -0,0 +1,294 @@ +/* jshint esversion: 6, node: true */ +const nThen = require('nthen'); +const Pins = require('./pins'); +const Environment = require("./env"); +const Util = require("./common-util"); +const Store = require('./storage/file.js'); +const BlobStore = require("./storage/blob"); +const BlockStore = require("./storage/block"); +const Core = require("./commands/core"); +const Metadata = require("./commands/metadata"); +const Meta = require("./metadata"); +const Logger = require("./log"); + +const Path = require("path"); +const Fse = require("fs-extra"); + +const { parentPort } = require('node:worker_threads'); + +const config = require('./load-config'); +const Env = Environment.create(config); +const COMMANDS = {}; +let Log; + +Env.computeMetadata = function (channel, cb) { + const ref = {}; + const lineHandler = Meta.createLineHandler(ref, (err) => { console.log(err); }); + return void Env.store.readChannelMetadata(channel, lineHandler, function (err) { + if (err) { + // stream errors? + return void cb(err); + } + cb(void 0, ref.meta); + }); +}; + +const mkReportPath = function (Env, safeKey) { + return Path.join(Env.paths.archive, 'accounts', safeKey); +}; +const storeReport = (Env, report, cb) => { + let path = mkReportPath(Env, report.key); + let s_data; + try { + s_data = JSON.stringify(report); + Fse.outputFile(path, s_data, cb); + } catch (err) { + return void cb(err); + } +}; +const readReport = (Env, key, cb) => { + let path = mkReportPath(Env, key); + Fse.readJson(path, cb); +}; +const deleteReport = (Env, key, cb) => { + let path = mkReportPath(Env, key); + Fse.remove(path, cb); +}; + +const init = (cb) => { + nThen((waitFor) => { + Logger.create(config, waitFor(function (_) { + Log = Env.Log = _; + })); + Store.create(config, waitFor(function (err, _store) { + if (err) { + waitFor.abort(); + return void cb(err); + } + Env.store = _store; + })); + Store.create({ + filePath: config.pinPath, + archivePath: config.archivePath, + // archive pin logs to their own subpath + volumeId: 'pins', + }, waitFor(function (err, _) { + if (err) { + waitFor.abort(); + throw err; + } + Env.pinStore = _; + })); + BlobStore.create({ + blobPath: config.blobPath, + blobStagingPath: config.blobStagingPath, + archivePath: config.archivePath, + getSession: function () {}, + }, waitFor(function (err, blob) { + if (err) { + waitFor.abort(); + return void cb(err); + } + Env.blobStore = blob; + })); + }).nThen(cb); +}; + +COMMANDS.start = (edPublic, blockId) => { + const safeKey = Util.escapeKeyCharacters(edPublic); + let ref = {}; + let blobsToArchive = []; + let channelsToArchive = []; + let deletedChannels = []; + let deletedBlobs = []; + nThen((waitFor) => { + init(waitFor()); + }).nThen((waitFor) => { + let lineHandler = Pins.createLineHandler(ref, (err) => { console.log(err); }); + Env.pinStore.getMessages(safeKey, lineHandler, waitFor(() => {})); + }).nThen((waitFor) => { + Log.info('MODERATION_ACCOUNT_ARCHIVAL_START', edPublic, waitFor()); + var n = nThen; + Object.keys(ref.pins || {}).forEach((chanId) => { + n = n((w) => { + // Blobs + if (Env.blobStore.isFileId(chanId)) { + return void Env.blobStore.isOwnedBy(safeKey, chanId, w((err, owned) => { + if (err || !owned) { return; } + blobsToArchive.push(chanId); + })); + } + // Pads + Metadata.getMetadata(Env, chanId, w((err, metadata) => { + if (err) { return; } // Can't read metadata? Don't archive + if (!Core.hasOwners(metadata)) { return; } // No owner, don't archive + if (Core.isOwner(metadata, edPublic) && metadata.owners.length === 1) { + channelsToArchive.push(chanId); // Only owner: archive + } + })); + }).nThen; + }); + n(waitFor()); + }).nThen((waitFor) => { + Log.info('MODERATION_ACCOUNT_ARCHIVAL_LISTED', JSON.stringify({ + pads: channelsToArchive.length, + blobs: blobsToArchive.length + }), waitFor()); + + var n = nThen; + // Archive the pads + channelsToArchive.forEach((chanId) => { + n = n((w) => { + Env.store.archiveChannel(chanId, w(function (err) { + if (err) { + return Log.error('MODERATION_CHANNEL_ARCHIVAL_ERROR', { + error: err, + channel: chanId, + }, w()); + } + deletedChannels.push(chanId); + Log.info('MODERATION_CHANNEL_ARCHIVAL', chanId, w()); + })); + }).nThen; + }); + // Archive the blobs + blobsToArchive.forEach((blobId) => { + n = n((w) => { + Env.blobStore.archive.blob(blobId, w(function (err) { + if (err) { + return Log.error('MODERATION_BLOB_ARCHIVAL_ERROR', { + error: err, + item: blobId, + }, w()); + } + deletedBlobs.push(blobId); + Log.info('MODERATION_BLOB_ARCHIVAL', blobId, w()); + })); + }).nThen; + }); + n(waitFor(() => { + // Archive the pin log + Env.pinStore.archiveChannel(safeKey, waitFor(function (err) { + if (err) { + return Log.error('MODERATION_ACCOUNT_PIN_LOG', err, waitFor()); + } + Log.info('MODERATION_ACCOUNT_LOG', safeKey, waitFor()); + })); + if (!blockId) { return; } + BlockStore.archive(Env, blockId, waitFor(function (err) { + if (err) { + blockId = undefined; + return Log.error('MODERATION_ACCOUNT_BLOCK', err, waitFor()); + } + Log.info('MODERATION_ACCOUNT_BLOCK', safeKey, waitFor()); + })); + })); + }).nThen((waitFor) => { + var report = { + key: safeKey, + channels: deletedChannels, + blobs: deletedBlobs, + blockId: blockId + }; + storeReport(Env, report, waitFor((err) => { + if (err) { + return Log.error('MODERATION_ACCOUNT_REPORT', report, waitFor()); + } + })); + }).nThen(() => { + parentPort.postMessage(JSON.stringify(deletedChannels)); + process.exit(0); + }); +}; + + +COMMANDS.restore = (edPublic) => { + const safeKey = Util.escapeKeyCharacters(edPublic); + let pads, blobs; + let blockId; + let errors = []; + nThen((waitFor) => { + init(waitFor()); + }).nThen((waitFor) => { + Log.info('MODERATION_ACCOUNT_RESTORE_START', edPublic, waitFor()); + readReport(Env, safeKey, waitFor((err, report) => { + if (err) { throw new Error(err); } + pads = report.channels; + blobs = report.blobs; + blockId = report.blockId; + })); + }).nThen((waitFor) => { + Log.info('MODERATION_ACCOUNT_RESTORE_LISTED', JSON.stringify({ + pads: pads.length, + blobs: blobs.length + }), waitFor()); + var n = nThen; + pads.forEach((chanId) => { + n = n((w) => { + Env.store.restoreArchivedChannel(chanId, w(function (err) { + if (err) { + errors.push(chanId); + return Log.error('MODERATION_CHANNEL_RESTORE_ERROR', { + error: err, + channel: chanId, + }, w()); + } + Log.info('MODERATION_CHANNEL_RESTORE', chanId, w()); + })); + }).nThen; + }); + blobs.forEach((blobId) => { + n = n((w) => { + Env.blobStore.restore.blob(blobId, w(function (err) { + if (err) { + errors.push(blobId); + return Log.error('MODERATION_BLOB_RESTORE_ERROR', { + error: err, + item: blobId, + }, w()); + } + Log.info('MODERATION_BLOB_RESTORE', blobId, w()); + })); + }).nThen; + }); + n(waitFor(() => { + // remove the pin logs of inactive accounts if inactive account removal is configured + Env.pinStore.restoreArchivedChannel(safeKey, waitFor(function (err) { + if (err) { + return Log.error('MODERATION_ACCOUNT_PIN_LOG_RESTORE', err, waitFor()); + } + Log.info('MODERATION_ACCOUNT_LOG_RESTORE', safeKey, waitFor()); + })); + if (!blockId) { return; } + BlockStore.restore(Env, blockId, waitFor(function (err) { + if (err) { + blockId = undefined; + return Log.error('MODERATION_ACCOUNT_BLOCK_RESTORE', err, waitFor()); + } + Log.info('MODERATION_ACCOUNT_BLOCK_RESTORE', safeKey, waitFor()); + })); + })); + }).nThen((waitFor) => { + deleteReport(Env, safeKey, waitFor((err) => { + if (err) { + return Log.error('MODERATION_ACCOUNT_REPORT_DELETE', safeKey, waitFor()); + } + })); + }).nThen(() => { + parentPort.postMessage(JSON.stringify(errors)); + process.exit(0); + }); + +}; + +parentPort.on('message', (message) => { + let parsed = message; //JSON.parse(message); + let command = parsed.command; + let content = parsed.content; + let block = parsed.block; + COMMANDS[command](content, block); +}); + +parentPort.postMessage('READY'); + + diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 071ce31f0..0daf2c38d 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -10,6 +10,9 @@ const Core = require("./core"); const Channel = require("./channel"); const BlockStore = require("../storage/block"); const MFA = require("../storage/mfa"); +/* jshint ignore:start */ +const { Worker } = require('node:worker_threads'); +/* jshint ignore:end */ var Fs = require("fs"); @@ -279,6 +282,62 @@ var restoreArchivedDocument = function (Env, Server, cb, data) { } }; +// CryptPad_AsyncStore.rpc.send('ADMIN', ['ARCHIVE_ACCOUNT', edPublic], console.log) +var archiveAccount = function (Env, Server, _cb, data) { + const cb = Util.once(_cb); + const worker = new Worker('./lib/archive-account.js'); + worker.on('message', message => { + if (message === 'READY') { + return worker.postMessage({ + command: 'start', + content: data && data[1], + block: data && data[2] + }); + } + // DONE: disconnect all users from these channels + var deletedChannels = Util.tryParse(message); + if (Array.isArray(deletedChannels)) { + let n = nThen; + deletedChannels.forEach((chanId) => { + n = n((w) => { + setTimeout(w(() => { + Channel.disconnectChannelMembers(Env, Server, chanId, 'EDELETED', () => {}); + }), 10); + }).nThen; + }); + } + cb(void 0, { state: true }); + }); + worker.on('error', (err) => { + console.error(err); + cb(err); + }); + worker.on('exit', () => { worker.unref(); }); +}; +var restoreAccount = function (Env, Server, _cb, data) { + const cb = Util.once(_cb); + const worker = new Worker('./lib/archive-account.js'); + worker.on('message', message => { + if (message === 'READY') { + return worker.postMessage({ + command: 'restore', + content: data && data[1] + }); + } + // Response + cb(void 0, { + state: true, + errors: Util.tryParse(message) + }); + }); + worker.on('error', (err) => { + console.error(err); + cb(err); + }); + worker.on('exit', () => { worker.unref(); }); + +}; + // CryptPad_AsyncStore.rpc.send('ADMIN', ['CLEAR_CACHED_CHANNEL_INDEX', documentID], console.log) var clearChannelIndex = function (Env, Server, cb, data) { var id = Array.isArray(data) && data[1]; @@ -780,6 +839,9 @@ var commands = { ARCHIVE_DOCUMENT: archiveDocument, RESTORE_ARCHIVED_DOCUMENT: restoreArchivedDocument, + ARCHIVE_ACCOUNT: archiveAccount, + RESTORE_ACCOUNT: restoreAccount, + CLEAR_CACHED_CHANNEL_INDEX: clearChannelIndex, GET_CACHED_CHANNEL_INDEX: getChannelIndex, // TODO implement admin historyTrim From dda4b8777c72d291bb0a82532296dff3e377e0ec Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 6 Sep 2023 17:15:00 +0200 Subject: [PATCH 06/21] Add drive channel metadata to the pin log --- lib/archive-account.js | 7 ++++-- lib/commands/channel.js | 10 +++----- lib/pins.js | 43 ++++++++++++++++++++++++++++++--- lib/workers/db-worker.js | 18 ++++++++------ www/common/cryptpad-common.js | 12 ++++++++- www/common/outer/async-store.js | 7 +++++- www/common/outer/team.js | 2 +- 7 files changed, 78 insertions(+), 21 deletions(-) diff --git a/lib/archive-account.js b/lib/archive-account.js index da59c96f5..085b4878c 100644 --- a/lib/archive-account.js +++ b/lib/archive-account.js @@ -105,7 +105,10 @@ COMMANDS.start = (edPublic, blockId) => { init(waitFor()); }).nThen((waitFor) => { let lineHandler = Pins.createLineHandler(ref, (err) => { console.log(err); }); - Env.pinStore.getMessages(safeKey, lineHandler, waitFor(() => {})); + Env.pinStore.readMessagesBin(safeKey, 0, (msgObj, readMore) => { + lineHandler(msgObj.buff.toString('utf8')); + readMore(); + }, waitFor()); }).nThen((waitFor) => { Log.info('MODERATION_ACCOUNT_ARCHIVAL_START', edPublic, waitFor()); var n = nThen; @@ -174,6 +177,7 @@ COMMANDS.start = (edPublic, blockId) => { } Log.info('MODERATION_ACCOUNT_LOG', safeKey, waitFor()); })); + blockId = blockId || ref.block; if (!blockId) { return; } BlockStore.archive(Env, blockId, waitFor(function (err) { if (err) { @@ -291,4 +295,3 @@ parentPort.on('message', (message) => { parentPort.postMessage('READY'); - diff --git a/lib/commands/channel.js b/lib/commands/channel.js index eb5a8821c..55ef88995 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -253,20 +253,18 @@ Channel.isNewChannel = function (Env, channel, cb) { if (channel.length !== HK.STANDARD_CHANNEL_LENGTH && channel.length !== HK.ADMIN_CHANNEL_LENGTH) { return void cb('INVALID_CHAN'); } - // TODO replace with readMessagesBin - var done = false; - Env.msgStore.getMessages(channel, function (msg) { - if (done) { return; } + Env.msgStore.readMessagesBin(channel, 0, function (msgObj, readMore, abort) { try { + var msg = msgObj.buff.toString('utf8'); if (typeof(msg) === 'string' && ARRAY_LINE.test(msg)) { - done = true; + abort(); return void cb(void 0, false); } } catch (e) { Env.WARN('invalid message read from store', e); } + readMore(); }, function () { - if (done) { return; } // no more messages... cb(void 0, true); }); diff --git a/lib/pins.js b/lib/pins.js index b0d524944..03225e840 100644 --- a/lib/pins.js +++ b/lib/pins.js @@ -33,6 +33,29 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { ref.index = 0; ref.latest = 0; // the latest message (timestamp in ms) ref.surplus = 0; // how many lines exist behind a reset + + + // Extract metadata from the channel list (#block, #drive) + let sanitize = (id, isPin) => { + if (typeof(id) !== "string") { return; } + let idx = id.indexOf('#'); + if (idx < 0) { return id; } + + let type = id.slice(idx+1); + let sanitized = id.slice(0, idx); + if (!isPin) { return sanitized; } + + if (type === 'block') { // Note: teams don't have a block + ref.block = sanitized; + return; + } + if (type === 'drive') { + ref.drive = sanitized; + return sanitized; + } + return sanitized; + }; + return function (line) { ref.index++; if (!Boolean(line)) { return; } @@ -55,17 +78,31 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { switch (l[0]) { case 'RESET': { pins = ref.pins = {}; - if (l[1] && l[1].length) { l[1].forEach((x) => { ref.pins[x] = 1; }); } + if (l[1] && l[1].length) { + l[1].forEach((x) => { + x = sanitize(x, true); + if (!x) { return; } + ref.pins[x] = 1; + }); + } ref.surplus = ref.index; //jshint -W086 // fallthrough } case 'PIN': { - l[1].forEach((x) => { pins[x] = 1; }); + l[1].forEach((x) => { + x = sanitize(x, true); + if (!x) { return; } + pins[x] = 1; + }); break; } case 'UNPIN': { - l[1].forEach((x) => { delete pins[x]; }); + l[1].forEach((x) => { + x = sanitize(x, false); + if (!x) { return; } + delete pins[x]; + }); break; } default: diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index d0b460b11..d12e62365 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -387,8 +387,10 @@ const getPinState = function (data, cb) { var lineHandler = Pins.createLineHandler(ref, Env.Log.error); // if channels aren't in memory. load them from disk - // TODO replace with readMessagesBin - pinStore.getMessages(safeKey, lineHandler, function () { + pinStore.readMessagesBin(safeKey, 0, (msgObj, readMore) => { + lineHandler(msgObj.buff.toString('utf8')); + readMore(); + }, function () { cb(void 0, ref.pins); // FIXME no error handling? }); }; @@ -599,16 +601,18 @@ const getPinActivity = function (data, cb) { var safeKey = Util.escapeKeyCharacters(data.key); var first; var latest; - pinStore.getMessages(safeKey, line => { - if (!line || !line.trim()) { return; } + pinStore.readMessagesBin(safeKey, 0, (msgObj, readMore) => { + var line = msgObj.buff.toString('utf8'); + if (!line || !line.trim()) { return readMore(); } try { var parsed = JSON.parse(line); var temp = parsed[parsed.length - 1]; - if (!temp || typeof(temp) !== 'number') { return; } + if (!temp || typeof(temp) !== 'number') { return readMore(); } latest = temp; - if (first) { return; } + if (first) { return readMore(); } first = latest; - } catch (err) { } + readMore(); + } catch (err) { readMore(); } }, function (err) { if (err) { return void cb(err); } cb(void 0, { diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 2d13c42f5..aaf722c8a 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -2496,6 +2496,15 @@ define([ })); } }).nThen(function (waitFor) { + var blockHash = LocalStore.getBlockHash(); + var blockId = ''; + try { + var blockPath = (new URL(blockHash)).pathname; + var blockSplit = blockPath.split('/'); + if (blockSplit[1] === 'block') { + blockId = blockSplit[3]; + } + } catch (e) { } var cfg = { init: true, userHash: userHash || LocalStore.getUserHash(), @@ -2508,7 +2517,8 @@ define([ neverDrive: rdyCfg.neverDrive, disableCache: localStorage['CRYPTPAD_STORE|disableCache'], driveEvents: !rdyCfg.noDrive, //rdyCfg.driveEvents // Boolean - lastVisit: Number(localStorage.lastVisit) || undefined + lastVisit: Number(localStorage.lastVisit) || undefined, + blockId: blockId }; common.userHash = userHash; diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 48b332039..85e2186ab 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -203,7 +203,7 @@ define([ }; var getUserChannelList = function () { - var userChannel = store.driveChannel; + var userChannel = `${store.driveChannel}#drive`; if (!userChannel) { return null; } // Get the list of pads' channel ID in your drive @@ -245,6 +245,11 @@ define([ } list.push(userChannel); + + if (store.data && store.data.blockId) { + //list.push(`${store.data.blockId}#block`); // XXX 5.5.0? + } + list.sort(); return list; diff --git a/www/common/outer/team.js b/www/common/outer/team.js index f07be289b..c7a78170f 100644 --- a/www/common/outer/team.js +++ b/www/common/outer/team.js @@ -149,7 +149,7 @@ define([ var list = store.manager.getChannelsList('pin'); var team = ctx.store.proxy.teams[id]; - list.push(team.channel); + list.push(`${team.channel}#drive`); var chatChannel = Util.find(team, ['keys', 'chat', 'channel']); var membersChannel = Util.find(team, ['keys', 'roster', 'channel']); var mailboxChannel = Util.find(team, ['keys', 'mailbox', 'channel']); From 91af47994be9adfef8f2149b57e9cccac5307656 Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 7 Sep 2023 17:03:20 +0200 Subject: [PATCH 07/21] Placeholder on file deletion --- lib/archive-account.js | 13 +++++--- lib/commands/admin-rpc.js | 25 +++++++++----- lib/commands/channel.js | 29 ++++++++++------ lib/commands/pin-rpc.js | 2 +- lib/eviction.js | 6 ++-- lib/hk-util.js | 24 ++++++++++---- lib/storage/blob.js | 44 ++++++++++++++++++++++--- lib/storage/file.js | 45 ++++++++++++++++++++++--- lib/workers/db-worker.js | 13 ++++++-- lib/workers/index.js | 3 +- www/common/common-ui-elements.js | 15 +++++++++ www/common/cryptpad-common.js | 36 ++++++++++++-------- www/common/drive-ui.js | 55 +++++++++++++++++++------------ www/common/outer/async-store.js | 12 +++---- www/common/outer/onlyoffice.js | 4 +-- www/common/outer/sharedfolder.js | 4 +-- www/common/outer/team.js | 4 +-- www/common/outer/userObject.js | 5 +-- www/common/pinpad.js | 7 ++-- www/common/proxy-manager.js | 30 +++++++++++------ www/common/sframe-common-outer.js | 29 ++++++++++++++-- www/common/sframe-common.js | 7 ++++ www/file/main.js | 1 + 23 files changed, 305 insertions(+), 108 deletions(-) diff --git a/lib/archive-account.js b/lib/archive-account.js index 085b4878c..f3ce033e8 100644 --- a/lib/archive-account.js +++ b/lib/archive-account.js @@ -94,8 +94,13 @@ const init = (cb) => { }).nThen(cb); }; -COMMANDS.start = (edPublic, blockId) => { +COMMANDS.start = (edPublic, blockId, reason) => { const safeKey = Util.escapeKeyCharacters(edPublic); + const archiveReason = { + code: 'MODERATION_ACCOUNT', + txt: reason + }; + let ref = {}; let blobsToArchive = []; let channelsToArchive = []; @@ -142,7 +147,7 @@ COMMANDS.start = (edPublic, blockId) => { // Archive the pads channelsToArchive.forEach((chanId) => { n = n((w) => { - Env.store.archiveChannel(chanId, w(function (err) { + Env.store.archiveChannel(chanId, archiveReason, w(function (err) { if (err) { return Log.error('MODERATION_CHANNEL_ARCHIVAL_ERROR', { error: err, @@ -157,7 +162,7 @@ COMMANDS.start = (edPublic, blockId) => { // Archive the blobs blobsToArchive.forEach((blobId) => { n = n((w) => { - Env.blobStore.archive.blob(blobId, w(function (err) { + Env.blobStore.archive.blob(blobId, archiveReason, w(function (err) { if (err) { return Log.error('MODERATION_BLOB_ARCHIVAL_ERROR', { error: err, @@ -171,7 +176,7 @@ COMMANDS.start = (edPublic, blockId) => { }); n(waitFor(() => { // Archive the pin log - Env.pinStore.archiveChannel(safeKey, waitFor(function (err) { + Env.pinStore.archiveChannel(safeKey, undefined, waitFor(function (err) { if (err) { return Log.error('MODERATION_ACCOUNT_PIN_LOG', err, waitFor()); } diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 0daf2c38d..765257975 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -175,20 +175,26 @@ var archiveDocument = function (Env, Server, cb, data) { if (typeof(id) !== 'string' || id.length < 32) { return void cb("EINVAL"); } + const archiveReason = { + code: 'MODERATION_PAD', + txt: reason + }; + const reasonStr = `MODERATION_PAD:${reason}`; + switch (id.length) { case 32: - return void Env.msgStore.archiveChannel(id, Util.both(cb, function (err) { + return void Env.msgStore.archiveChannel(id, archiveReason, Util.both(cb, function (err) { Env.Log.info("ARCHIVAL_CHANNEL_BY_ADMIN_RPC", { channelId: id, reason: reason, status: err? String(err): "SUCCESS", }); - Channel.disconnectChannelMembers(Env, Server, id, 'EDELETED', err => { + Channel.disconnectChannelMembers(Env, Server, id, 'EDELETED', reasonStr, err => { if (err) { } // TODO }); })); case 48: - return void Env.blobStore.archive.blob(id, Util.both(cb, function (err) { + return void Env.blobStore.archive.blob(id, archiveReason, Util.both(cb, function (err) { Env.Log.info("ARCHIVAL_BLOB_BY_ADMIN_RPC", { id: id, reason: reason, @@ -213,7 +219,7 @@ var removeDocument = function (Env, Server, cb, data) { id = args; } else if (args && typeof(args) === 'object') { id = args.id; - reason = args.reason; + reason = `MODERATION_DESTROY:${args.reason}`; } if (typeof(id) !== 'string' || id.length < 32) { return void cb("EINVAL"); } @@ -226,7 +232,7 @@ var removeDocument = function (Env, Server, cb, data) { reason: reason, status: err? String(err): "SUCCESS", }); - Channel.disconnectChannelMembers(Env, Server, id, 'EDELETED', err => { + Channel.disconnectChannelMembers(Env, Server, id, 'EDELETED', reason, err => { if (err) { } // TODO }); })); @@ -291,17 +297,20 @@ var archiveAccount = function (Env, Server, _cb, data) { return worker.postMessage({ command: 'start', content: data && data[1], - block: data && data[2] + block: data && data[2], + reason: data && data[3] }); } + // DONE: disconnect all users from these channels + const reason = `MODERATION_ACCOUNT:${data && data[3]}`; var deletedChannels = Util.tryParse(message); if (Array.isArray(deletedChannels)) { let n = nThen; deletedChannels.forEach((chanId) => { n = n((w) => { setTimeout(w(() => { - Channel.disconnectChannelMembers(Env, Server, chanId, 'EDELETED', () => {}); + Channel.disconnectChannelMembers(Env, Server, chanId, 'EDELETED', reason, () => {}); }), 10); }).nThen; }); @@ -645,7 +654,7 @@ var archivePinLog = function (Env, Server, cb, data) { if (!isValidKey(key)) { return void cb("EINVAL"); } var safeKey = Util.escapeKeyCharacters(key); - Env.pinStore.archiveChannel(safeKey, function (err) { + Env.pinStore.archiveChannel(safeKey, undefined, function (err) { Core.expireSession(Env.Sessions, safeKey); if (err) { Env.Log.error('ARCHIVE_PIN_LOG_BY_ADMIN', { diff --git a/lib/commands/channel.js b/lib/commands/channel.js index 55ef88995..c3d354881 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -8,7 +8,7 @@ const Metadata = require("./metadata"); const HK = require("../hk-util"); const Nacl = require("tweetnacl/nacl-fast"); -Channel.disconnectChannelMembers = function (Env, Server, channelId, code, cb) { +Channel.disconnectChannelMembers = function (Env, Server, channelId, code, reason, cb) { var done = Util.once(Util.mkAsync(cb)); if (!Core.isValidId(channelId)) { return done('INVALID_ID'); } @@ -39,6 +39,7 @@ Channel.disconnectChannelMembers = function (Env, Server, channelId, code, cb) { userId, JSON.stringify({ error: code, //'EDELETED', + message: reason, channel: channelId, }) ], w()); @@ -52,6 +53,7 @@ Channel.disconnectChannelMembers = function (Env, Server, channelId, code, cb) { Env.Log.warn('DISCONNECT_CHANNEL_MEMBERS_TIMEOUT', { channelId, code, + reason }); clear(); done(); @@ -105,9 +107,10 @@ Channel.clearOwnedChannel = function (Env, safeKey, channelId, cb, Server) { }); }; -var archiveOwnedChannel = function (Env, safeKey, channelId, __cb, Server) { +var archiveOwnedChannel = function (Env, safeKey, channelId, reason, __cb, Server) { var _cb = Util.once(Util.mkAsync(__cb)); var unsafeKey = Util.unescapeKeyCharacters(safeKey); + reason = reason || 'ARCHIVE_OWNED'; nThen(function (w) { // confirm that the channel exists before worrying about whether // we have permission to delete it. @@ -130,7 +133,7 @@ var archiveOwnedChannel = function (Env, safeKey, channelId, __cb, Server) { }).nThen(function () { var cb = _cb; // temporarily archive the file - return void Env.msgStore.archiveChannel(channelId, function (e) { + return void Env.msgStore.archiveChannel(channelId, reason, function (e) { Env.Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { unsafeKey: unsafeKey, channelId: channelId, @@ -141,16 +144,19 @@ var archiveOwnedChannel = function (Env, safeKey, channelId, __cb, Server) { } cb(void 0, 'OK'); - Channel.disconnectChannelMembers(Env, Server, channelId, 'EDELETED', err => { + Channel.disconnectChannelMembers(Env, Server, channelId, 'EDELETED', reason, err => { if (err) { } // TODO }); }); }); }; -Channel.removeOwnedChannel = function (Env, safeKey, channelId, __cb, Server) { +Channel.removeOwnedChannel = function (Env, safeKey, obj, __cb, Server) { var _cb = Util.once(Util.mkAsync(__cb)); + var channelId = obj.channel; + var reason = obj.reason; + if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { return _cb('INVALID_ARGUMENTS'); } @@ -160,9 +166,9 @@ Channel.removeOwnedChannel = function (Env, safeKey, channelId, __cb, Server) { Env.queueDeletes(safeKey, function (next) { var cb = Util.both(_cb, next); if (Env.blobStore.isFileId(channelId)) { - return void Env.removeOwnedBlob(channelId, safeKey, cb); + return void Env.removeOwnedBlob(channelId, safeKey, reason, cb); } - archiveOwnedChannel(Env, safeKey, channelId, cb, Server); + archiveOwnedChannel(Env, safeKey, channelId, reason, cb, Server); }); }; @@ -258,15 +264,18 @@ Channel.isNewChannel = function (Env, channel, cb) { var msg = msgObj.buff.toString('utf8'); if (typeof(msg) === 'string' && ARRAY_LINE.test(msg)) { abort(); - return void cb(void 0, false); + return void cb(void 0, {isNew: false}); } } catch (e) { Env.WARN('invalid message read from store', e); } readMore(); - }, function () { + }, function (err, reason) { // no more messages... - cb(void 0, true); + cb(void 0, { + isNew: true, + reason: reason + }); }); }; diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index cad040e4c..1540c9843 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -115,7 +115,7 @@ Pinning.getTotalSize = function (Env, safeKey, cb) { */ Pinning.removePins = function (Env, safeKey, cb) { // FIXME respect the queue - Env.pinStore.archiveChannel(safeKey, function (err) { + Env.pinStore.archiveChannel(safeKey, undefined, function (err) { Core.expireSession(Env.Sessions, safeKey); Env.Log.info('ARCHIVAL_PIN_BY_OWNER_RPC', { safeKey: safeKey, diff --git a/lib/eviction.js b/lib/eviction.js index 9cf6a5558..16b456078 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -540,7 +540,7 @@ module.exports = function (Env, cb) { } // remove the pin logs of inactive accounts if inactive account removal is configured - pinStore.archiveChannel(id, function (err) { + pinStore.archiveChannel(id, undefined, function (err) { if (err) { return Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err, next); } @@ -599,7 +599,7 @@ module.exports = function (Env, cb) { if (item.mtime > inactiveTime) { return void next(); } removed++; - blobs.archive.blob(item.blobId, function (err) { + blobs.archive.blob(item.blobId, 'INACTIVE', function (err) { if (err) { return Log.error("EVICT_ARCHIVE_BLOB_ERROR", { error: err, @@ -724,7 +724,7 @@ module.exports = function (Env, cb) { // else fall through to the archival })); }).nThen(function (w) { - return void store.archiveChannel(item.channel, w(function (err) { + return void store.archiveChannel(item.channel, 'INACTIVE', w(function (err) { if (err) { Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { error: err, diff --git a/lib/hk-util.js b/lib/hk-util.js index 455964ccb..81fc4c16e 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -124,7 +124,7 @@ var CHECKPOINT_PATTERN = /^cp\|(([A-Za-z0-9+\/=]+)\|)?/; but for some reason are still present */ const expireChannel = HK.expireChannel = function (Env, channel) { - return void Env.store.archiveChannel(channel, function (err) { + return void Env.store.archiveChannel(channel, 'EXPIRED', function (err) { Env.Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { channelId: channel, status: err? String(err): "SUCCESS", @@ -526,7 +526,12 @@ const getHistoryAsync = (Env, channelName, lastKnownHash, beforeHash, handler, c getHistoryOffset(Env, channelName, lastKnownHash, waitFor((err, os) => { if (err) { waitFor.abort(); - return void cb(err); + var reason; + if (err && err.reason) { + reason = err.reason; + err = err.error; + } + return void cb(err, reason); } offset = os; })); @@ -540,8 +545,8 @@ const getHistoryAsync = (Env, channelName, lastKnownHash, beforeHash, handler, c const parsed = tryParse(Env, msgObj.buff.toString('utf8')); if (!parsed) { return void readMore(); } handler(parsed, readMore); - }, waitFor(function (err) { - return void cb(err); + }, waitFor(function (err, reason) { + return void cb(err, reason); })); }); }; @@ -682,7 +687,7 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } if (txid) { msg[0] = txid; } Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(msg)], readMore); - }, (err) => { + }, (err, reason) => { // Any error but ENOENT: abort // ENOENT is allowed in case we want to create a new pad if (err && err.code !== 'ENOENT') { @@ -700,10 +705,15 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { stack: err && err.stack, }); } // FIXME err.message isn't useful for users - const parsedMsg = {error:err.message, channel: channelName, txid: txid}; + const parsedMsg = {error:err.message, test:'a', channel: channelName, txid: txid}; Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg)]); return; } + if (err && err.code === 'ENOENT' && reason) { + const parsedMsg2 = {error:'EDELETED', message: reason, channel: channelName, txid: txid}; + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg2)]); + return; + } // If we're asking for a specific version (lastKnownHash) but we receive an // ENOENT, this is not a pad creation so we need to abort. @@ -716,7 +726,7 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { is no longer on the server so they don't abuse the data and so that they don't unintentionally continue to edit it in a broken state. */ - const parsedMsg2 = {error:'EDELETED', channel: channelName, txid: txid}; + const parsedMsg2 = {error:'EDELETED', test:'b',channel: channelName, txid: txid}; Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg2)]); return; } diff --git a/lib/storage/blob.js b/lib/storage/blob.js index d2c0a7420..98c6a9266 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -47,6 +47,10 @@ var makeProofPath = function (Env, safeKey, blobId) { return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId); }; +var mkPlaceholderPath = function (Env, blobId) { + return makeBlobPath(Env, blobId) + '.placeholder'; +}; + var parseProofPath = function (path) { var parts = path.split('/'); return { @@ -55,6 +59,26 @@ var parseProofPath = function (path) { }; }; +// Placeholder for deleted files +var addPlaceholder = function (Env, blobId, reason, cb) { + if (!reason) { return cb(); } + var path = mkPlaceholderPath(Env, blobId); + var s_data = typeof(reason) === "string" ? reason : `${reason.code}:${reason.txt}`; + Fs.writeFile(path, s_data, cb); +}; +var clearPlaceholder = function (Env, blobId, cb) { + var path = mkPlaceholderPath(Env, blobId); + Fs.unlink(path, cb); +}; +var readPlaceholder = function (Env, blobId, cb) { + var path = mkPlaceholderPath(Env, blobId); + Fs.readFile(path, function (err, content) { + if (err) { return void cb(); } + cb(content.toString('utf8')); + }); +}; + + // getUploadSize: used by // getFileSize var getUploadSize = function (Env, blobId, cb) { @@ -63,7 +87,16 @@ var getUploadSize = function (Env, blobId, cb) { Fs.stat(path, function (err, stats) { if (err) { // if a file was deleted, its size is 0 bytes - if (err.code === 'ENOENT') { return cb(void 0, 0); } + if (err.code === 'ENOENT') { + return readPlaceholder(Env, blobId, (content) => { + if (!content) { return cb(void 0, 0); } + cb({ + code: err.code, + reason: content + }); + + }); + } return void cb(err.code); } cb(void 0, stats.size); @@ -345,6 +378,7 @@ var owned_upload_complete = function (Env, safeKey, id, cb) { }); }; + // removeBlob var remove = function (Env, blobId, cb) { var blobPath = makeBlobPath(Env, blobId); @@ -366,11 +400,12 @@ var isOwnedBy = function (Env, safeKey, blobId, cb) { // archiveBlob -var archiveBlob = function (Env, blobId, cb) { +var archiveBlob = function (Env, blobId, reason, cb) { var blobPath = makeBlobPath(Env, blobId); var archivePath = prependArchive(Env, blobPath); Fse.move(blobPath, archivePath, { overwrite: true }, cb); archiveActivity(Env, blobId, () => {}); + addPlaceholder(Env, blobId, reason, () => {}); }; var removeArchivedBlob = function (Env, blobId, cb) { @@ -385,6 +420,7 @@ var restoreBlob = function (Env, blobId, cb) { var archivePath = prependArchive(Env, blobPath); Fse.move(archivePath, blobPath, cb); restoreActivity(Env, blobId, () => {}); + clearPlaceholder(Env, blobId, () => {}); }; // archiveProof @@ -614,10 +650,10 @@ BlobStore.create = function (config, _cb) { }, archive: { - blob: function (blobId, _cb) { + blob: function (blobId, reason, _cb) { var cb = Util.once(Util.mkAsync(_cb)); if (!isValidId(blobId)) { return void cb("INVALID_ID"); } - archiveBlob(Env, blobId, cb); + archiveBlob(Env, blobId, reason, cb); }, proof: function (safeKey, blobId, _cb) { var cb = Util.once(Util.mkAsync(_cb)); diff --git a/lib/storage/file.js b/lib/storage/file.js index 5f915f5c8..60ff9dfc1 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -70,6 +70,10 @@ var mkOffsetPath = function (env, channelId) { return mkPath(env, channelId) + '.offset'; }; +var mkPlaceholderPath = function (env, channelId) { + return mkPath(env, channelId) + '.placeholder'; +}; + // pass in the path so we can reuse the same function for archived files var channelExists = function (filepath, cb) { Fs.stat(filepath, function (err, stat) { @@ -141,6 +145,25 @@ var isChannelArchived = function (env, channelName, cb) { }); }; +var addPlaceholder = function (env, channelId, reason, cb) { + if (!reason) { return cb(); } + var path = mkPlaceholderPath(env, channelId); + var s_data = typeof(reason) === "string" ? reason : `${reason.code}:${reason.txt}`; + Fs.writeFile(path, s_data, cb); +}; +var clearPlaceholder = function (env, channelId, cb) { + var path = mkPlaceholderPath(env, channelId); + Fs.unlink(path, cb); +}; +var readPlaceholder = function (env, channelId, cb) { + var path = mkPlaceholderPath(env, channelId); + Fs.readFile(path, function (err, content) { + if (err) { return void cb(); } + cb(content.toString('utf8')); + }); +}; + + const destroyStream = function (stream) { if (!stream) { return; } try { @@ -190,7 +213,16 @@ const readMessagesBin = (env, id, start, msgHandler, cb) => { const stream = Fs.createReadStream(mkPath(env, id), { start: start }); const collector = createIdleStreamCollector(stream); const handleMessageAndKeepStreamAlive = Util.both(msgHandler, collector.keepAlive); - const done = Util.both(cb, collector); + const done = Util.both((err) => { + if (err && err.code === 'ENOENT') { + // If the channel doesn't exists, look for a placeholder. + // If a placeholder exists, call back with its content in addition to the original error + return readPlaceholder(env, id, (content) => { + cb(err, content); + }); + } + cb(err); + }, collector); return void readFileBin(stream, handleMessageAndKeepStreamAlive, done, { offset: start, }); @@ -670,7 +702,7 @@ var listChannels = function (root, handler, cb, fast) { // move a channel's log file from its current location // to an equivalent location in the cold storage directory -var archiveChannel = function (env, channelName, cb) { +var archiveChannel = function (env, channelName, reason, cb) { // TODO close channels before archiving them? // ctime is the most reliable indicator of when a file was archived @@ -700,6 +732,9 @@ var archiveChannel = function (env, channelName, cb) { })); }).nThen(function (w) { clearOffset(env, channelName, w()); + }).nThen(function (w) { + if (!reason) { return; } + addPlaceholder(env, channelName, reason, w()); }).nThen(function (w) { // archive the dedicated metadata channel var metadataPath = mkMetadataPath(env, channelName); @@ -775,6 +810,8 @@ var unarchiveChannel = function (env, channelName, cb) { return void CB(err); } })); + }).nThen(function (w) { + clearPlaceholder(env, channelName, w()); }).nThen(function (w) { var archiveMetadataPath = mkArchiveMetadataPath(env, channelName); // TODO validate that it's ok to move metadata non-atomically @@ -1290,12 +1327,12 @@ module.exports.create = function (conf, _cb) { isChannelArchived(env, channelName, cb); }, // move a channel from the database to the archive, along with its metadata - archiveChannel: function (channelName, cb) { + archiveChannel: function (channelName, reason, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // again, the semantics around archiving and appending are really muddy. // so I'm calling this 'unordered' again schedule.unordered(channelName, function (next) { - archiveChannel(env, channelName, Util.both(cb, next)); + archiveChannel(env, channelName, reason, Util.both(cb, next)); }); }, // restore a channel from the archive to the database, along with its metadata diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index d12e62365..76f03421d 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -499,8 +499,13 @@ const getHashOffset = function (data, cb) { } offset = msgObj.offset; abort(); - }, function (err) { - if (err) { return void cb(err); } + }, function (err, reason) { + if (err) { + return void cb({ + error: err, + reason: reason + }); + } cb(void 0, offset); }); }; @@ -510,6 +515,8 @@ const removeOwnedBlob = function (data, cb) { const blobId = data.blobId; const safeKey = Util.escapeKeyCharacters(data.safeKey); + const reason = data.reason || 'ARCHIVE_OWNED'; + nThen(function (w) { // check if you have permissions blobStore.isOwnedBy(safeKey, blobId, w(function (err, owned) { @@ -520,7 +527,7 @@ const removeOwnedBlob = function (data, cb) { })); }).nThen(function (w) { // remove the blob - blobStore.archive.blob(blobId, w(function (err) { + blobStore.archive.blob(blobId, reason, w(function (err) { Env.Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { safeKey: safeKey, blobId: blobId, diff --git a/lib/workers/index.js b/lib/workers/index.js index 7b11e7d62..deddd8afb 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -382,11 +382,12 @@ Workers.initialize = function (Env, config, _cb) { }); }; - Env.removeOwnedBlob = function (blobId, safeKey, cb) { + Env.removeOwnedBlob = function (blobId, safeKey, reason, cb) { sendCommand({ command: 'REMOVE_OWNED_BLOB', blobId: blobId, safeKey: safeKey, + reason: reason }, cb); }; diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index a8992250c..9971ccab5 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -2876,6 +2876,15 @@ define([ delete autoStoreModal[priv.channel]; } + if (err.message && err.message !== "PASSWORD_CHANGE") { + UI.errorLoadingScreen(h('div', [ + h('p', Messages.deletedError), + h('p', err.message) // XXX custom message for each reason + // XXX reasons: ARCHIVE_OWNED, INACTIVE, MODERATION_ACCOUNT:{reason}, MODERATION_PAD:{reason} + ]), Boolean(err.loaded), Boolean(err.loaded)); + return; + } + if (err.ownDeletion) { if (toolbar && typeof toolbar.deleted === "function") { toolbar.deleted(); } (cb || function () {})(); @@ -2921,6 +2930,12 @@ define([ var error; if (isError) { error = setHTML(h('p.cp-password-error'), Messages.password_error); } +// XXX +if (cfg.legacy) { + // XXX Legacy mode: we don't know if pad destroyed or password changed +} + + var info = h('p.cp-password-info', Messages.password_info); var info_loaded = setHTML(h('p.cp-password-info'), Messages.errorCopy); diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index aaf722c8a..5e9c9fab3 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -649,15 +649,23 @@ define([ }).nThen(function (waitFor) { // If it's not in the cache or it's not a blob, try to get the value from the server - postMessage("GET_FILE_SIZE", {channel:channel}, waitFor(function (obj) { - if (obj && obj.error) { - // If disconnected, try to get the value from the channel cache (next nThen) - error = obj.error; - return; - } - waitFor.abort(); - cb(undefined, obj.size); - })); + var getSize = () => { + postMessage("GET_FILE_SIZE", {channel:channel}, waitFor(function (obj) { + if (obj && obj.error === "ANON_RPC_NOT_READY") { return void setTimeout(waitFor(getSize), 100); } + + if (obj && obj.error && obj.error.code === 'ENOENT' && obj.error.reason) { + waitFor.abort(); + cb(obj.error.reason); + } else if (obj && obj.error) { + // If disconnected, try to get the value from the channel cache (next nThen) + error = obj.error; + return; + } + waitFor.abort(); + cb(undefined, obj.size); + })); + }; + getSize(); }).nThen(function () { Cache.getChannelCache(channel, function(err, data) { if (err) { return void cb(error); } @@ -681,7 +689,7 @@ define([ var error = obj && obj.error; if (error) { return void cb(error); } if (!obj) { return void cb('ERROR'); } - cb (null, obj.isNew); + cb (null, obj.isNew, obj.reason); }, {timeout: -1}); }; // This function is used when we want to open a pad. We first need @@ -711,7 +719,7 @@ define([ } else if (error) { return void cb(error); } - cb(undefined, obj.isNew); + cb(undefined, obj.isNew, obj.reason); }, {timeout: -1}); }; isNew(); @@ -1484,7 +1492,8 @@ define([ // delete the old pad common.removeOwnedChannel({ channel: oldChannel, - teamId: teamId + teamId: teamId, + reason: 'PASSWORD_CHANGE', }, waitFor(function (obj) { if (obj && obj.error) { waitFor.abort(); @@ -1620,7 +1629,8 @@ define([ // delete the old pad common.removeOwnedChannel({ channel: oldChannel, - teamId: teamId + teamId: teamId, + reason: 'PASSWORD_CHANGE' }, waitFor(function (obj) { if (obj && obj.error) { waitFor.abort(); diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 11908cf38..b574345ce 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -1152,10 +1152,36 @@ define([ var FILTER_BY = "filterBy"; + var refreshDeprecated = function () { + if (!APP.passwordModal) { return; } + var deprecated = files.sharedFoldersTemp; + if (JSONSortify(deprecated) === APP.deprecatedSF) { return; } + APP.deprecatedSF = JSONSortify(deprecated); + if (typeof (deprecated) === "object" && Object.keys(deprecated).length) { + var nt = nThen; + Object.keys(deprecated).forEach(function (fId) { + var data = deprecated[fId]; + var sfId = manager.user.userObject.getSFIdFromHref(data.href); + if (folders[fId] || sfId) { // This shared folder is already stored in the drive... + return void manager.delete([['sharedFoldersTemp', fId]], function () { }); + } + nt = nt(function (waitFor) { + UI.openCustomModal(APP.passwordModal(fId, data, waitFor())); + }).nThen; + }); + nt(function () { + APP.refresh(); + }); + } + + }; var refresh = APP.refresh = function (cb) { var type = APP.store[FILTER_BY]; var path = type ? [FILTER, type, currentPath] : currentPath; - APP.displayDirectory(path, undefined, cb); + APP.displayDirectory(path, undefined, () => { + refreshDeprecated(); + if (typeof(cb) === "function") { cb(); } + }); }; // `app`: true (force open wiht the app), false (force open in preview), @@ -5370,9 +5396,13 @@ define([ }); } */ - var nt = nThen; - var passwordModal = function (fId, data, cb) { + APP.passwordModal = function (fId, data, cb) { var content = []; + + var legacy = data.legacy; // XXX in legacy mode, it's either deletion or password change + legacy = legacy; + // XXX when legacy is false, we know it's a password change + var folderName = ''+ (data.lastTitle || Messages.fm_newFolder) +''; content.push(UI.setHTML(h('p'), Messages._getKey('drive_sfPassword', [folderName]))); var newPassword = UI.passwordInput({ @@ -5425,24 +5455,7 @@ define([ onClose: cb }); }; - onConnectEvt.reg(function () { - var deprecated = files.sharedFoldersTemp; - if (typeof (deprecated) === "object" && Object.keys(deprecated).length) { - Object.keys(deprecated).forEach(function (fId) { - var data = deprecated[fId]; - var sfId = manager.user.userObject.getSFIdFromHref(data.href); - if (folders[fId] || sfId) { // This shared folder is already stored in the drive... - return void manager.delete([['sharedFoldersTemp', fId]], function () { }); - } - nt = nt(function (waitFor) { - UI.openCustomModal(passwordModal(fId, data, waitFor())); - }).nThen; - }); - nt(function () { - refresh(); - }); - } - }); + onConnectEvt.reg(refreshDeprecated); return { refresh: refresh, diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 85e2186ab..e81738c04 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -365,10 +365,12 @@ define([ var channel = data; var force = false; var teamId; + var reason; if (data && typeof(data) === "object") { channel = data.channel; force = data.force; teamId = data.teamId; + reason = data.reason; } if (channel === store.driveChannel && !force) { @@ -385,7 +387,7 @@ define([ s.rpc.removeOwnedChannel(channel, function (err) { if (err) { delete myDeletions[channel]; } cb({error:err}); - }); + }, reason); }; var arePinsSynced = function (cb) { @@ -512,11 +514,9 @@ define([ var channelId = data.channel || Hash.hrefToHexChannelId(data.href, data.password); store.anon_rpc.send("IS_NEW_CHANNEL", channelId, function (e, response) { if (e) { return void cb({error: e}); } - if (response && response.length && typeof(response[0]) === 'boolean') { - if (response[0]) { Cache.clearChannel(channelId); } - return void cb({ - isNew: response[0] - }); + if (response && response.length && typeof(response[0]) === 'object') { + if (response[0].isNew) { Cache.clearChannel(channelId); } + return void cb(response[0]); } else { cb({error: 'INVALID_RESPONSE'}); } diff --git a/www/common/outer/onlyoffice.js b/www/common/outer/onlyoffice.js index dbde5e199..60a3e412a 100644 --- a/www/common/outer/onlyoffice.js +++ b/www/common/outer/onlyoffice.js @@ -251,8 +251,8 @@ define([ ctx.store.anon_rpc.send("IS_NEW_CHANNEL", channel, function (e, response) { if (e) { return void cb({error: e}); } var isNew; - if (response && response.length && typeof(response[0]) === 'boolean') { - isNew = response[0]; + if (response && response.length && typeof(response[0]) === 'object') { + isNew = response[0].isNew; } else { cb({error: 'INVALID_RESPONSE'}); } diff --git a/www/common/outer/sharedfolder.js b/www/common/outer/sharedfolder.js index e86682746..c121b5c24 100644 --- a/www/common/outer/sharedfolder.js +++ b/www/common/outer/sharedfolder.js @@ -120,7 +120,7 @@ define([ }).nThen(function (waitFor) { isNewChannel(null, { channel: secret.channel }, waitFor(function (obj) { if (obj.isNew && !isNew) { - store.manager.deprecateProxy(id, secret.channel); + store.manager.deprecateProxy(id, secret.channel, obj.reason); waitFor.abort(); return void cb(null); } @@ -245,7 +245,7 @@ define([ // Deprecate the shared folder from each team // We can only hide it sf.teams.forEach(function (obj) { - obj.store.manager.deprecateProxy(obj.id, secret.channel); + obj.store.manager.deprecateProxy(obj.id, secret.channel, info.message); if (obj.store.handleSharedFolder) { obj.store.handleSharedFolder(obj.id, null); } diff --git a/www/common/outer/team.js b/www/common/outer/team.js index c7a78170f..641ec59ff 100644 --- a/www/common/outer/team.js +++ b/www/common/outer/team.js @@ -423,7 +423,7 @@ define([ }; if (channel) { ctx.store.anon_rpc.send("IS_NEW_CHANNEL", channel, waitFor(function (e, res) { - if (res && res.length && typeof(res[0]) === 'boolean' && res[0]) { + if (res && res.length && typeof(res[0]) === 'object' && res[0].isNew) { // Channel is empty: remove this team close(); } @@ -431,7 +431,7 @@ define([ } if (roster) { ctx.store.anon_rpc.send("IS_NEW_CHANNEL", roster, waitFor(function (e, res) { - if (res && res.length && typeof(res[0]) === 'boolean' && res[0]) { + if (res && res.length && typeof(res[0]) === 'object' && res[0].isNew) { // Channel is empty: remove this team close(); } diff --git a/www/common/outer/userObject.js b/www/common/outer/userObject.js index 84d64401e..66163d06d 100644 --- a/www/common/outer/userObject.js +++ b/www/common/outer/userObject.js @@ -118,13 +118,14 @@ define([ cb(null, id); }; - exp.deprecateSharedFolder = function (id) { + exp.deprecateSharedFolder = function (id, reason) { if (readOnly) { return; } var data = files[SHARED_FOLDERS][id]; if (!data) { return; } var ro = !data.href || exp.cryptor.decrypt(data.href).indexOf('#') === -1; if (!ro) { - files[SHARED_FOLDERS_TEMP][id] = JSON.parse(JSON.stringify(data)); + var obj = files[SHARED_FOLDERS_TEMP][id] = JSON.parse(JSON.stringify(data)); + obj.legacy = reason !== "PASSWORD_CHANGE"; } var paths = exp.findFile(Number(id)); exp.delete(paths, null, true); diff --git a/www/common/pinpad.js b/www/common/pinpad.js index ab1a8c8c6..084b2d482 100644 --- a/www/common/pinpad.js +++ b/www/common/pinpad.js @@ -131,12 +131,15 @@ var factory = function (Util, Rpc) { }); }; - exp.removeOwnedChannel = function (channel, cb) { + exp.removeOwnedChannel = function (channel, cb, reason) { if (typeof(channel) !== 'string' || [32,48].indexOf(channel.length) === -1) { console.error('invalid channel to remove', channel); return void cb('INVALID_ARGUMENTS'); } - rpc.send('REMOVE_OWNED_CHANNEL', channel, function (e, response) { + rpc.send('REMOVE_OWNED_CHANNEL', { + channel: channel, + reason: reason + }, function (e, response) { if (e) { return void cb(e); } if (response && response.length && response[0] === "OK") { cb(); diff --git a/www/common/proxy-manager.js b/www/common/proxy-manager.js index 79f19bef9..71097864d 100644 --- a/www/common/proxy-manager.js +++ b/www/common/proxy-manager.js @@ -71,7 +71,7 @@ define([ }; // Password may have changed - var deprecateProxy = function (Env, id, channel) { + var deprecateProxy = function (Env, id, channel, reason) { if (Env.folders[id] && Env.folders[id].deleting) { // Folder is being deleted by its owner, don't deprecate it return; @@ -85,11 +85,19 @@ define([ return void Env.Store.refreshDriveUI(); } if (channel) { Env.unpinPads([channel], function () {}); } - Env.user.userObject.deprecateSharedFolder(id); - removeProxy(Env, id); - if (Env.Store && Env.Store.refreshDriveUI) { - Env.Store.refreshDriveUI(); + + // If it's explicitely a deletion, no need to deprecate, just delete + if (reason && reason !== "PASSWORD_CHANGE") { + var temp = Util.find(Env, ['user', 'proxy', UserObject.SHARED_FOLDERS]); + delete temp[id]; + if (Env.Store && Env.Store.refreshDriveUI) { Env.Store.refreshDriveUI(); } + return; } + + // It's explicitely a password change, better message in drive: provide the "reason" to the UI + Env.user.userObject.deprecateSharedFolder(id, reason); + removeProxy(Env, id); + if (Env.Store && Env.Store.refreshDriveUI) { Env.Store.refreshDriveUI(); } }; var restrictedProxy = function (Env, id) { @@ -632,17 +640,19 @@ define([ if (isNew) { return void cb({ error: 'ENOTFOUND' }); } + var newData = Util.clone(data); var parsed = Hash.parsePadUrl(href); var secret = Hash.getSecrets(parsed.type, parsed.hash, newPassword); - data.password = newPassword; - data.channel = secret.channel; + newData.password = newPassword; + newData.channel = secret.channel; if (secret.keys.editKeyStr) { - data.href = '/drive/#'+Hash.getEditHashFromKeys(secret); + newData.href = '/drive/#'+Hash.getEditHashFromKeys(secret); } - data.roHref = '/drive/#'+Hash.getViewHashFromKeys(secret); + newData.roHref = '/drive/#'+Hash.getViewHashFromKeys(secret); + delete newData.legacy; _addSharedFolder(Env, { path: ['root'], - folderData: data, + folderData: newData, }, function () { delete temp[fId]; Env.onSync(cb); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 8fb767788..921e447ed 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -469,12 +469,20 @@ define([ // `hasChannelHistory` doesn't work for files (not a channel) // `getFileSize` is not adapted to channels because of metadata Cryptpad.getFileSize(currentPad.href, password, function (e, size) { + if (e && e !== "PASSWORD_CHANGE") { + return sframeChan.event("EV_DELETED_ERROR", e); + } next(e, size === 0); }); return; } // Not a file, so we can use `hasChannelHistory` - Cryptpad.hasChannelHistory(currentPad.href, password, next); + Cryptpad.hasChannelHistory(currentPad.href, password, (e, isNew, reason) => { + if (isNew && reason && reason !== "PASSWORD_CHANGE") { + return sframeChan.event("EV_DELETED_ERROR", reason); + } + next(); + }); }); sframeChan.event("EV_PAD_PASSWORD", cfg); }; @@ -564,20 +572,32 @@ define([ // `hasChannelHistory` doesn't work for files (not a channel) // `getFileSize` is not adapted to channels because of metadata Cryptpad.getFileSize(currentPad.href, password, w(function (e, size) { - if (size !== 0) { return void todo(); } + if (e && e !== "PASSWORD_CHANGE") { + sframeChan.event("EV_DELETED_ERROR", e); + waitFor.abort(); + return; + } + if (!e && size !== 0) { return void todo(); } // Wrong password or deleted file? + passwordCfg.legacy = !e; // Legacy means we don't know if it's a deletion or pw change askPassword(true, passwordCfg); })); return; } // Not a file, so we can use `hasChannelHistory` - Cryptpad.hasChannelHistory(currentPad.href, password, w(function(e, isNew) { + Cryptpad.hasChannelHistory(currentPad.href, password, w(function(e, isNew, reason) { if (isNew && expire && expire < (+new Date())) { sframeChan.event("EV_EXPIRED_ERROR"); waitFor.abort(); return; } if (!e && !isNew) { return void todo(); } + // NOTE: Legacy mode ==> no reason may indicate a password change + if (isNew && reason && reason !== "PASSWORD_CHANGE") { + sframeChan.event("EV_DELETED_ERROR", reason); + waitFor.abort(); + return; + } if (parsed.hashData.mode === 'view' && (password || !parsed.hashData.password)) { // Error, wrong password stored, the view seed has changed with the password // password will never work @@ -586,6 +606,7 @@ define([ return; } // Wrong password or deleted file? + passwordCfg.legacy = !reason; // Legacy means we don't know if it's a deletion or pw change askPassword(true, passwordCfg); })); }).nThen(done); @@ -1915,6 +1936,8 @@ define([ // redirect window.location.href = url; document.location.reload(); + // XXX XXX XXX TEST TEST TEST + // XXX XXX XXX TEST TEST TEST }); return; diff --git a/www/common/sframe-common.js b/www/common/sframe-common.js index 35f51ef03..8a698d5f9 100644 --- a/www/common/sframe-common.js +++ b/www/common/sframe-common.js @@ -844,6 +844,13 @@ define([ UI.errorLoadingScreen(Messages.restrictedError); }); + ctx.sframeChan.on("EV_DELETED_ERROR", function (reason) { + funcs.onServerError({ + type: 'EDELETED', + message: reason + }); + }); + ctx.sframeChan.on("EV_PAD_PASSWORD_ERROR", function () { UI.errorLoadingScreen(Messages.password_error_seed); }); diff --git a/www/file/main.js b/www/file/main.js index ba2497a93..0329b65a6 100644 --- a/www/file/main.js +++ b/www/file/main.js @@ -20,6 +20,7 @@ define([ }; SFCommonO.start({ cache: true, + noDrive: true, hash: hash, href: href, noRealtime: true, From 975a177cbbbc060281156a8314e0de7c83092e75 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 8 Sep 2023 18:10:02 +0200 Subject: [PATCH 08/21] Add UI/UX when reading a placeholder --- customize.dist/login.js | 16 ++++++++++- lib/archive-account.js | 5 ++-- lib/challenge-commands/base.js | 4 +-- lib/challenge-commands/totp.js | 4 +-- lib/commands/admin-rpc.js | 6 ++++- lib/commands/block.js | 4 +-- lib/http-worker.js | 18 +++++++++++++ lib/storage/block.js | 37 +++++++++++++++++++++---- www/common/common-interface.js | 45 +++++++++++++++++++++++++++++++ www/common/common-ui-elements.js | 25 ++++++++--------- www/common/common-util.js | 6 ++--- www/common/cryptpad-common.js | 13 ++++++++- www/common/drive-ui.js | 11 ++++---- www/common/outer/async-store.js | 11 ++++++++ www/common/outer/login-block.js | 5 ++-- www/common/sframe-common-outer.js | 13 ++++++++- www/common/sframe-common.js | 13 +++++++-- 17 files changed, 194 insertions(+), 42 deletions(-) diff --git a/customize.dist/login.js b/customize.dist/login.js index cc3ae191a..82826366d 100644 --- a/customize.dist/login.js +++ b/customize.dist/login.js @@ -10,6 +10,7 @@ define([ '/common/common-constants.js', '/common/common-interface.js', '/common/common-feedback.js', + '/common/hyperscript.js', '/common/outer/local-store.js', '/customize/messages.js', '/components/nthen/index.js', @@ -20,7 +21,7 @@ define([ '/components/tweetnacl/nacl-fast.min.js', '/components/scrypt-async/scrypt-async.min.js', // better load speed ], function ($, Listmap, Crypto, Util, NetConfig, Cred, ChainPad, Realtime, Constants, UI, - Feedback, LocalStore, Messages, nThen, Block, Hash, ServerCommand) { + Feedback, h, LocalStore, Messages, nThen, Block, Hash, ServerCommand) { var Exports = { Cred: Cred, Block: Block, @@ -210,6 +211,13 @@ define([ return void console.log("Block requires 2FA"); } + if (err === 404 && response && response.reason + && response.reason !== 'PASSWORD_CHANGE') { + waitFor.abort(); + w.abort(); + return void cb('DELETED_USER', response); + } + // Some other error? if (err) { console.error(err); @@ -595,6 +603,12 @@ define([ }); }); break; + case 'DELETED_USER': + if (result.reason === 'PASSWORD_CHANGE') { + // XXX PLACEHOLDER: account password change login page + } + UI.errorLoadingScreen(UI.getDestroyedPlaceholder(result.reason, true)); + break; case 'INVAL_PASS': UI.removeLoadingScreen(function () { UI.alert(Messages.login_invalPass, function () { diff --git a/lib/archive-account.js b/lib/archive-account.js index f3ce033e8..059d96680 100644 --- a/lib/archive-account.js +++ b/lib/archive-account.js @@ -184,7 +184,7 @@ COMMANDS.start = (edPublic, blockId, reason) => { })); blockId = blockId || ref.block; if (!blockId) { return; } - BlockStore.archive(Env, blockId, waitFor(function (err) { + BlockStore.archive(Env, blockId, archiveReason, waitFor(function (err) { if (err) { blockId = undefined; return Log.error('MODERATION_ACCOUNT_BLOCK', err, waitFor()); @@ -295,7 +295,8 @@ parentPort.on('message', (message) => { let command = parsed.command; let content = parsed.content; let block = parsed.block; - COMMANDS[command](content, block); + let reason = parsed.reason; + COMMANDS[command](content, block, reason); }); parentPort.postMessage('READY'); diff --git a/lib/challenge-commands/base.js b/lib/challenge-commands/base.js index 9efbc7d3b..a1c55ff3f 100644 --- a/lib/challenge-commands/base.js +++ b/lib/challenge-commands/base.js @@ -69,8 +69,8 @@ const removeBlock = Commands.REMOVE_BLOCK = function (Env, body, cb) { }; removeBlock.complete = function (Env, body, cb) { - const { publicKey } = body; - Block.removeLoginBlock(Env, publicKey, cb); + const { publicKey, reason } = body; + Block.removeLoginBlock(Env, publicKey, reason, cb); }; diff --git a/lib/challenge-commands/totp.js b/lib/challenge-commands/totp.js index 1ae863b91..a61af68ee 100644 --- a/lib/challenge-commands/totp.js +++ b/lib/challenge-commands/totp.js @@ -483,10 +483,10 @@ const removeBlock = Commands.TOTP_REMOVE_BLOCK = function (Env, body, cb) { }; removeBlock.complete = function (Env, body, cb) { - const { publicKey } = body; + const { publicKey, reason } = body; nThen(function (w) { // Remove the block - Block.removeLoginBlock(Env, publicKey, w((err) => { + Block.removeLoginBlock(Env, publicKey, reason, w((err) => { if (err) { w.abort(); return void cb(err); diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 765257975..3d2e50fee 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -678,7 +678,11 @@ var archiveBlock = function (Env, Server, cb, data) { var key = args.key; var reason = args.reason; if (!isValidKey(key)) { return void cb("EINVAL"); } - BlockStore.archive(Env, key, err => { + const archiveReason = { + code: 'MODERATION_BLOCK', + txt: reason + }; + BlockStore.archive(Env, key, archiveReason, err => { Env.Log.info("ARCHIVE_BLOCK_BY_ADMIN", { error: err, key: key, diff --git a/lib/commands/block.js b/lib/commands/block.js index bb7150b17..ca4a77e1e 100644 --- a/lib/commands/block.js +++ b/lib/commands/block.js @@ -172,10 +172,10 @@ Block.writeLoginBlock = function (Env, msg, _cb) { information, we can just sign some constant and use that as proof. */ -Block.removeLoginBlock = function (Env, publicKey, _cb) { +Block.removeLoginBlock = function (Env, publicKey, reason, _cb) { var cb = Util.once(Util.mkAsync(_cb)); - BlockStore.archive(Env, publicKey, function (err) { + BlockStore.archive(Env, publicKey, reason, function (err) { Env.Log.info('ARCHIVAL_BLOCK_BY_OWNER_RPC', { publicKey: publicKey, status: err? String(err): 'SUCCESS', diff --git a/lib/http-worker.js b/lib/http-worker.js index 995bcdd7e..177509689 100644 --- a/lib/http-worker.js +++ b/lib/http-worker.js @@ -10,6 +10,7 @@ const AuthCommands = require("./http-commands"); const MFA = require("./storage/mfa"); const Sessions = require("./storage/sessions"); const BlobStore = require("./storage/blob"); +const BlockStore = require("./storage/block"); const DEFAULT_QUERY_TIMEOUT = 5000; const PID = process.pid; @@ -420,6 +421,23 @@ app.use('/block/', function (req, res, next) { app.use("/block", Express.static(Path.resolve(Env.paths.block), { maxAge: "0d", })); +// In case of a 404 for the block, check if a placeholder exists +// and provide the result if that's the case +app.use("/block", (req, res, next) => { + const url = req.url; + if (typeof(url) === "string") { + const s = url.split('/'); + if (s[1] && s[1].length === 2 && BlockStore.isValidKey(s[2])) { + return BlockStore.readPlaceholder(Env, s[2], (content) => { + res.status(404).json({ + reason: content, + code: 404 + }); + }); + } + } + next(); +}); app.use("/customize", Express.static('customize')); app.use("/customize", Express.static('customize.dist')); diff --git a/lib/storage/block.js b/lib/storage/block.js index d0bd29cbf..a4e9238ef 100644 --- a/lib/storage/block.js +++ b/lib/storage/block.js @@ -32,7 +32,28 @@ Block.mkArchivePath = function (Env, publicKey) { return Path.join(Env.paths.archive, 'block', safeKey.slice(0, 2), safeKey); }; -Block.archive = function (Env, publicKey, _cb) { +var mkPlaceholderPath = function (Env, publicKey) { + return Block.mkPath(Env, publicKey) + '.placeholder'; +}; +var addPlaceholder = function (Env, publicKey, reason, cb) { + if (!reason) { return cb(); } + var path = mkPlaceholderPath(Env, publicKey); + var s_data = typeof(reason) === "string" ? reason : `${reason.code}:${reason.txt}`; + Fs.writeFile(path, s_data, cb); +}; +var clearPlaceholder = function (Env, publicKey, cb) { + var path = mkPlaceholderPath(Env, publicKey); + Fs.unlink(path, cb); +}; +Block.readPlaceholder = function (Env, publicKey, cb) { + var path = mkPlaceholderPath(Env, publicKey); + Fs.readFile(path, function (err, content) { + if (err) { return void cb(); } + cb(content.toString('utf8')); + }); +}; + +Block.archive = function (Env, publicKey, reason, _cb) { var cb = Util.once(Util.mkAsync(_cb)); // derive the filepath @@ -52,7 +73,10 @@ Block.archive = function (Env, publicKey, _cb) { // TODO Env.incrementBytesWritten Fse.move(currentPath, archivePath, { overwrite: true, - }, cb); + }, (err) => { + cb(err); + if (!err && reason) { addPlaceholder(Env, publicKey, reason, () => {}); } + }); }; Block.restore = function (Env, publicKey, _cb) { @@ -75,10 +99,13 @@ Block.restore = function (Env, publicKey, _cb) { // TODO Env.incrementBytesWritten Fse.move(archivePath, livePath, { //overwrite: true, - }, cb); + }, (err) => { + cb(err); + if (!err) { clearPlaceholder(Env, publicKey, () => {}); } + }); }; -var isValidKey = function (publicKey) { +var isValidKey = Block.isValidKey = function (publicKey) { return typeof(publicKey) === 'string' && publicKey.length === 44; }; @@ -131,7 +158,7 @@ Block.write = function (Env, publicKey, buffer, _cb) { cb(err); })); }).nThen(function (w) { - Block.archive(Env, publicKey, w(function (/* err */) { + Block.archive(Env, publicKey, 'PASSWORD_CHANGE', w(function (/* err */) { /* we proceed even if there are errors. it might be ENOENT (there is no file to archive) diff --git a/www/common/common-interface.js b/www/common/common-interface.js index cd3d1b406..99f51ba43 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -1544,5 +1544,50 @@ define([ }; + // XXX + Messages.dph_reason = "Reason: {0}"; + Messages.dph_account_destroyed = "This account has been deleted by its owner"; + Messages.dph_account_inactive = "This account has been deleted for inactivity"; + Messages.dph_account_moderated = "This account has been suspended by the moderation team"; + Messages.dph_account_pw = "This account's password has been changed"; + Messages.dph_pad_destroyed = "This document has been destroyed by its owner"; + Messages.dph_pad_inactive = "This document has been deleted for inactivity"; + Messages.dph_pad_moderated = "This document has been deleted by the moderation team"; + Messages.dph_pad_moderated_account = "This document has deleted with its owner's account"; // Keep this key ??? + Messages.dph_pad_pw = "The document you are trying to open is protected with a new password. Enter the correct password to access the content."; + + + UI.getDestroyedPlaceholderMessage = (code, isAccount) => { + var account = { + ARCHIVE_OWNED: Messages.dph_account_destroyed, + INACTIVE: Messages.dph_account_inactive, + MODERATION_ACCOUNT: Messages.dph_account_moderated, + MODERATION_BLOCK: Messages.dph_account_moderated, + PASSWORD_CHANGE: Messages.dph_account_pw, + }; + var pad = { + ARCHIVE_OWNED: Messages.dph_pad_destroyed, + INACTIVE: Messages.dph_pad_inactive, + MODERATION_PAD: Messages.dph_pad_moderated, + MODERATION_DESTROY: Messages.dph_pad_moderated, + MODERATION_ACCOUNT: Messages.dph_pad_moderated_account, + PASSWORD_CHANGE: Messages.dph_pad_pw + }; + return isAccount ? account[code] : pad[code]; + }; + UI.getDestroyedPlaceholder = function (reason, isAccount) { + if (typeof(reason) !== "string") { return; } + var split = reason.split(':'); + var code = split[0]; // Generated code + var input = split[1]; // User/admin manual input + var text = UI.getDestroyedPlaceholderMessage(code, isAccount); + if (!text) { return; } // XXX + var reasonBlock = input ? h('p', Messages._getKey('dph_reason', [input])) : undefined; + return h('div', [ + h('p', text), + reasonBlock + ]); + }; + return UI; }); diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index 9971ccab5..d7037dc7d 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -2860,6 +2860,7 @@ define([ var priv = common.getMetadataMgr().getPrivateData(); var sframeChan = common.getSframeChannel(); var msg = err.type; + var exitable = Boolean(err.loaded); if (err.type === 'EEXPIRED') { msg = Messages.expiredError; if (err.loaded) { @@ -2876,12 +2877,13 @@ define([ delete autoStoreModal[priv.channel]; } + if (err.message && err.drive) { + UI.errorLoadingScreen(UI.getDestroyedPlaceholder(err.message, true)); + return; + } if (err.message && err.message !== "PASSWORD_CHANGE") { - UI.errorLoadingScreen(h('div', [ - h('p', Messages.deletedError), - h('p', err.message) // XXX custom message for each reason - // XXX reasons: ARCHIVE_OWNED, INACTIVE, MODERATION_ACCOUNT:{reason}, MODERATION_PAD:{reason} - ]), Boolean(err.loaded), Boolean(err.loaded)); + UI.errorLoadingScreen(UI.getDestroyedPlaceholder(err.message, false), + exitable, exitable); return; } @@ -2930,13 +2932,12 @@ define([ var error; if (isError) { error = setHTML(h('p.cp-password-error'), Messages.password_error); } -// XXX -if (cfg.legacy) { - // XXX Legacy mode: we don't know if pad destroyed or password changed -} - - - var info = h('p.cp-password-info', Messages.password_info); + var pwMsg = UI.getDestroyedPlaceholderMessage('PASSWORD_CHANGE', false); + if (cfg.legacy) { + // Legacy mode: we don't know if the pad has been destroyed or its password has changed + pwMsg = Messages.password_info; + } + var info = h('p.cp-password-info', pwMsg); var info_loaded = setHTML(h('p.cp-password-info'), Messages.errorCopy); var password = UI.passwordInput({placeholder: Messages.password_placeholder}); diff --git a/www/common/common-util.js b/www/common/common-util.js index d734f6b2f..5ae15a9fa 100644 --- a/www/common/common-util.js +++ b/www/common/common-util.js @@ -334,11 +334,11 @@ // this is resulting in some code duplication return void CB(void 0, response); } - if (response.status === 401) { + if (response.status === 401 || response.status === 404) { response.json().then((data) => { - CB(401, data); + CB(response.status, data); }).catch(() => { - CB(401); + CB(response.status); }); return; diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 5e9c9fab3..d98b4d6ac 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -476,6 +476,7 @@ define([ common.drive.onLog = Util.mkEvent(); common.drive.onChange = Util.mkEvent(); common.drive.onRemove = Util.mkEvent(); + common.drive.onDeleted = Util.mkEvent(); // Profile common.getProfileEditUrl = function (cb) { postMessage("GET", { key: ['profile', 'edit'] }, function (obj) { @@ -2064,6 +2065,7 @@ define([ if (!blockHash) { return; } console.log('removing old login block'); Block.removeLoginBlock({ + reason: 'PASSWORD_CHANGE', auth: auth, blockKeys: oldBlockKeys, }, waitFor(function (err) { @@ -2075,7 +2077,8 @@ define([ common.removeOwnedChannel({ channel: secret.channel, teamId: null, - force: true + force: true, + reason: 'PASSWORD_CHANGE' }, waitFor(function (obj) { if (obj && obj.error) { // Deal with it as if it was not owned @@ -2352,6 +2355,7 @@ define([ DRIVE_LOG: common.drive.onLog.fire, DRIVE_CHANGE: common.drive.onChange.fire, DRIVE_REMOVE: common.drive.onRemove.fire, + DRIVE_DELETED: common.drive.onDeleted.fire, // Account deletion DELETE_ACCOUNT: common.startAccountDeletion, // Loading @@ -2469,6 +2473,13 @@ define([ }); } + if (err === 404) { + // Not found: account deleted + waitFor.abort(); + f(response || err); + return; + } + if (err) { // TODO // it seems wrong that errors here aren't reported or handled diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index b574345ce..85c11dbdd 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -5396,15 +5396,14 @@ define([ }); } */ + Messages.dph_sf_pw = "Your shared folder {0} is no longer available, it is now protected with a new password. You can remove this folder from your CryptDrive or recover access using the new password."; // XXX PLACEHOLDER APP.passwordModal = function (fId, data, cb) { var content = []; - var legacy = data.legacy; // XXX in legacy mode, it's either deletion or password change - legacy = legacy; - // XXX when legacy is false, we know it's a password change - - var folderName = ''+ (data.lastTitle || Messages.fm_newFolder) +''; - content.push(UI.setHTML(h('p'), Messages._getKey('drive_sfPassword', [folderName]))); + var legacy = data.legacy; // Legacy mode: we don't know if the sf has been destroyed or its password has changed + var folderName = ''+ (Util.fixHTML(data.lastTitle) || Messages.fm_newFolder) +''; + var pwMsg = legacy ? Messages._getKey('drive_sfPassword', [folderName]) : Messages._getKey('dph_sf_pw', [folderName]); + content.push(UI.setHTML(h('p'), pwMsg)); var newPassword = UI.passwordInput({ id: 'cp-app-prop-change-password', placeholder: Messages.settings_changePasswordNew, diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index e81738c04..55b6d903c 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -887,6 +887,7 @@ define([ }).nThen(function (waitFor) { if (!blockKeys) { return; } Block.removeLoginBlock({ + reason: 'ARCHIVE_OWNED', auth: auth, blockKeys: blockKeys, }, waitFor(function (err) { @@ -3049,6 +3050,16 @@ define([ rt.realtime.abort(); sendDriveEvent('NETWORK_DISCONNECT'); } + }) + .on('error', function (info) { + if (info.error && info.error === 'EDELETED') { + if (info.message === 'PASSWORD_CHANGE') { + // XXX PLACEHOLDER ?? currently redirected to login page + } + // XXX PLACEHOLDER - logout instead of clear cache? + broadcast([], "DRIVE_DELETED", info.message); + Cache.clear(); + } }); // Proxy handlers (reconnect only called when the proxy is ready) diff --git a/www/common/outer/login-block.js b/www/common/outer/login-block.js index 9f3bbb817..39dea3884 100644 --- a/www/common/outer/login-block.js +++ b/www/common/outer/login-block.js @@ -192,7 +192,7 @@ define([ }, cb); }; Block.removeLoginBlock = function (data, cb) { - const { blockKeys, auth } = data; + const { reason, blockKeys, auth } = data; var command = 'REMOVE_BLOCK'; if (auth && auth.type === 'TOTP') { @@ -201,7 +201,8 @@ define([ ServerCommand(blockKeys.sign, { command: command, - auth: auth && auth.data + auth: auth && auth.data, + reason: reason }, cb); }; diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 921e447ed..f3dd34287 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -268,7 +268,15 @@ define([ // NOTE: Driveless mode should only work for existing pads, but we can't check that // before creating the worker because we need the anon RPC to do so. // We're only going to check if a hash exists in the URL or not. - Cryptpad.ready(waitFor(), { + Cryptpad.ready(waitFor((err) => { + if (err) { + waitFor.abort(); + if (err.code === 404) { + return void sframeChan.event("EV_DRIVE_DELETED", err.reason); + } + sframeChan.event('EV_LOADING_ERROR', 'ACCOUNT'); + } + }), { noDrive: cfg.noDrive && AppConfig.allowDrivelessMode && currentPad.hash, neverDrive: cfg.integration, driveEvents: cfg.driveEvents, @@ -799,6 +807,9 @@ define([ //Test.registerOuter(sframeChan); + Cryptpad.drive.onDeleted.reg(function (message) { + sframeChan.event("EV_DRIVE_DELETED", message); + }); Cryptpad.onNewVersionReconnect.reg(function () { sframeChan.event("EV_NEW_VERSION"); }); diff --git a/www/common/sframe-common.js b/www/common/sframe-common.js index 8a698d5f9..a90bca9b3 100644 --- a/www/common/sframe-common.js +++ b/www/common/sframe-common.js @@ -851,6 +851,14 @@ define([ }); }); + ctx.sframeChan.on("EV_DRIVE_DELETED", function (reason) { + funcs.onServerError({ + type: 'EDELETED', + drive: true, + message: reason + }); + }); + ctx.sframeChan.on("EV_PAD_PASSWORD_ERROR", function () { UI.errorLoadingScreen(Messages.password_error_seed); }); @@ -890,9 +898,10 @@ define([ // XXX You can still use the current version in read-only mode by pressing Esc. // what if they don't have a keyboard (ie. mobile) msg = Messages.deletedError + '
' + Messages.errorRedirectToHome; - } - if (err === "INVALID_HASH") { + } else if (err === "INVALID_HASH") { msg = Messages.invalidHashError; + } else if (err === 'ACCOUNT') { // block 404 but no placeholder + msg = Messages.login_unhandledError; } UI.errorLoadingScreen(msg, false, function () { funcs.gotoURL('/drive/'); From 2135a95531fc6978292eab2777571b6a1406bf13 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 11 Sep 2023 11:53:59 +0200 Subject: [PATCH 09/21] Fix form responses deletion #1239 --- lib/http-worker.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/http-worker.js b/lib/http-worker.js index 995bcdd7e..9088b04c2 100644 --- a/lib/http-worker.js +++ b/lib/http-worker.js @@ -505,6 +505,7 @@ var serveBroadcast = makeRouteCache(function () { return [ 'define(function(){', 'return ' + JSON.stringify({ + curvePublic: Env.curvePublic, lastBroadcastHash: Env.lastBroadcastHash, surveyURL: Env.surveyURL, maintenance: maintenance From 63bd707cea3ac9a04f2f1d9da077a8846cae1d40 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 12 Sep 2023 17:10:18 +0200 Subject: [PATCH 10/21] Fix account password change issue --- www/common/cryptpad-common.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index d98b4d6ac..8cf8e69db 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -1952,8 +1952,9 @@ define([ console.log("checking if old drive is owned"); common.anonRpcMsg('GET_METADATA', secret.channel, waitFor(function (err, obj) { if (err || obj.error) { return; } - if (obj.owners && Array.isArray(obj.owners) && - obj.owners.indexOf(edPublic) !== -1) { + var md = obj[0]; + if (md && md.owners && Array.isArray(md.owners) && + md.owners.indexOf(edPublic) !== -1) { oldIsOwned = true; } })); From 6acf8aa150bde75be2a68d57700058497cec0991 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 12 Sep 2023 17:24:45 +0200 Subject: [PATCH 11/21] Improve error screen on password change and account deletion --- www/common/common-interface.js | 3 +++ www/common/common-ui-elements.js | 10 ++++++++-- www/common/cryptpad-common.js | 26 ++++++++++++++++++-------- www/common/outer/async-store.js | 11 ++++------- www/common/sframe-common-outer.js | 5 +++++ www/common/sframe-common.js | 2 +- 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/www/common/common-interface.js b/www/common/common-interface.js index 267184198..b9812c2c0 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -1042,9 +1042,12 @@ define([ window.parent.location = href; }); if (exitable) { + // XXX if true or function, ALSO add a button to leave $(window).focus(); $(window).keydown(function (e) { // XXX what if they don't have a keyboard? if (e.which === 27) { + e.preventDefault(); + e.stopPropagation(); $loading.hide(); $('html').toggleClass('cp-loading-noscroll', false); if (typeof(exitable) === "function") { exitable(); } diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index d7037dc7d..b9349f3da 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -2878,8 +2878,14 @@ define([ } if (err.message && err.drive) { - UI.errorLoadingScreen(UI.getDestroyedPlaceholder(err.message, true)); - return; + let msg = UI.getDestroyedPlaceholder(err.message, true); + return UI.errorLoadingScreen(msg, false, () => { + // When closing error screen + if (err.message === 'PASSWORD_CHANGE') { + return common.setLoginRedirect('login'); + } + return common.setLoginRedirect(''); + }); } if (err.message && err.message !== "PASSWORD_CHANGE") { UI.errorLoadingScreen(UI.getDestroyedPlaceholder(err.message, false), diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 8cf8e69db..6fd93f48b 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -2071,6 +2071,7 @@ define([ blockKeys: oldBlockKeys, }, waitFor(function (err) { if (err) { return void console.error(err); } + common.passwordUpdated = true; })); }).nThen(function (waitFor) { if (!oldIsOwned) { return; } @@ -2086,9 +2087,7 @@ define([ oldIsOwned = false; return; } - common.logoutFromAll(waitFor(function () { - common.stopWorker(); - })); + common.stopWorker(); })); }).nThen(function (waitFor) { if (oldIsOwned) { return; } @@ -2101,9 +2100,7 @@ define([ if (obj && obj.error) { console.error(obj.error); } - common.logoutFromAll(waitFor(function () { - common.stopWorker(); - })); + common.stopWorker(); })); }).nThen(function () { // We have the new drive, with the new login block @@ -2280,6 +2277,14 @@ define([ cb(); }; + common.storeLogout = function (data) { + if (common.passwordUpdated) { return; } + LocalStore.logout(function () { + common.stopWorker(); + common.drive.onDeleted.fire(data.reason); + }); + }; + var lastPing = +new Date(); var onPing = function (data, cb) { lastPing = +new Date(); @@ -2359,6 +2364,7 @@ define([ DRIVE_DELETED: common.drive.onDeleted.fire, // Account deletion DELETE_ACCOUNT: common.startAccountDeletion, + LOGOUT: common.storeLogout, // Loading LOADING_DRIVE: common.loading.onDriveEvent.fire, // AutoStore @@ -2477,8 +2483,9 @@ define([ if (err === 404) { // Not found: account deleted waitFor.abort(); - f(response || err); - return; + return LocalStore.logout(function () { + f(response || err); + }); } if (err) { @@ -2818,6 +2825,9 @@ define([ LocalStore.loginReload(); } else if (o && !n) { LocalStore.logout(); + } else if (o && n && o !== n) { + common.passwordUpdated = true; + window.location.reload(); } }); LocalStore.onLogout(function () { diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 55b6d903c..7c258bf9b 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -247,7 +247,7 @@ define([ list.push(userChannel); if (store.data && store.data.blockId) { - //list.push(`${store.data.blockId}#block`); // XXX 5.5.0? + list.push(`${store.data.blockId}#block`); // XXX 5.5.0? } list.sort(); @@ -3053,12 +3053,9 @@ define([ }) .on('error', function (info) { if (info.error && info.error === 'EDELETED') { - if (info.message === 'PASSWORD_CHANGE') { - // XXX PLACEHOLDER ?? currently redirected to login page - } - // XXX PLACEHOLDER - logout instead of clear cache? - broadcast([], "DRIVE_DELETED", info.message); - Cache.clear(); + broadcast([], "LOGOUT", { + reason: info.message + }); } }); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index f3dd34287..1fe44e859 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -272,6 +272,11 @@ define([ if (err) { waitFor.abort(); if (err.code === 404) { + sframeChan.on('EV_SET_LOGIN_REDIRECT', function (page) { + var href = Utils.Hash.hashToHref('', page); + var url = Utils.Hash.getNewPadURL(href, { href: currentPad.href }); + window.location.href = url; + }); return void sframeChan.event("EV_DRIVE_DELETED", err.reason); } sframeChan.event('EV_LOADING_ERROR', 'ACCOUNT'); diff --git a/www/common/sframe-common.js b/www/common/sframe-common.js index a90bca9b3..ef39e02bf 100644 --- a/www/common/sframe-common.js +++ b/www/common/sframe-common.js @@ -433,7 +433,7 @@ define([ }; funcs.setLoginRedirect = function (page) { - ctx.sframeChan.query('EV_SET_LOGIN_REDIRECT', page); + ctx.sframeChan.event('EV_SET_LOGIN_REDIRECT', page); }; funcs.isPresentUrl = function (cb) { From d9741d53dc472ebd0e157039e25de3b004449ea5 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 12 Sep 2023 18:36:57 +0200 Subject: [PATCH 12/21] Add notification when a pinned shared folder has been destroyed --- www/common/notifications.js | 22 ++++++++++++++++++++ www/common/outer/async-store.js | 2 ++ www/common/outer/mailbox-handlers.js | 23 +++++++++++++++++++++ www/common/outer/team.js | 4 +++- www/common/proxy-manager.js | 31 +++++++++++++++++++++++++++- 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/www/common/notifications.js b/www/common/notifications.js index 6d921929d..b272e97cc 100644 --- a/www/common/notifications.js +++ b/www/common/notifications.js @@ -407,6 +407,28 @@ define([ } }; + handlers['SF_DELETED'] = function(common, data) { + var content = data.content; + var msg = content.msg; + + // Display the notification + var title = Util.fixHTML(msg.content.title); + var teamName = Util.fixHTML(msg.content.teamName); + + Messages.dph_sf_destroyed = "Your shared folder {0} has been destroyed by its owner."; // XXX + Messages.dph_sf_destroyed_team= "The shared folder {0} from your team {1} has been destroyed by its owner."; // XXX + + content.getFormatText = function() { + if (teamName) { + return Messages._getKey('dph_sf_destroyed_team', [title, teamName]); + } + return Messages._getKey('dph_sf_destroyed', [title]); + }; + if (!content.archived) { + content.dismissHandler = defaultDismiss(common, data); + } + }; + handlers['MOVE_TODO'] = function(common, data) { var content = data.content; var msg = content.msg; diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 7c258bf9b..89821b842 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -639,6 +639,7 @@ define([ } }; cb(JSON.parse(JSON.stringify(metadata))); + return metadata; }; Store.onMaintenanceUpdate = function (uid) { @@ -2726,6 +2727,7 @@ define([ loadSharedFolder: loadSharedFolder, settings: proxy.settings, removeOwnedChannel: function (channel, cb) { Store.removeOwnedChannel('', channel, cb); }, + store: store, Store: Store }, { outer: true, diff --git a/www/common/outer/mailbox-handlers.js b/www/common/outer/mailbox-handlers.js index 74ea68d7f..7ae77ce34 100644 --- a/www/common/outer/mailbox-handlers.js +++ b/www/common/outer/mailbox-handlers.js @@ -794,6 +794,29 @@ define([ cb(true); }; + var sfDeleted = {}; + handlers['SF_DELETED'] = function (ctx, box, data, cb) { + var msg = data.msg; + var content = msg.content; + var teamId = content.team; + var sfId = content.sfId; + + if (sfDeleted[sfId]) { return void cb(true); } + sfDeleted[sfId] = 1; + + // If it's a team SF, add the team name here + + if (!teamId) { return void cb(false); } + + var team = ctx.store.proxy.teams[teamId]; + content.teamName = team.metadata && team.metadata.name; + cb(false); + }; + removeHandlers['SF_DELETED'] = function (ctx, box, data) { + var id = data.content.sfId; + delete sfDeleted[id]; + }; + return { add: function (ctx, box, data, cb) { /** diff --git a/www/common/outer/team.js b/www/common/outer/team.js index 641ec59ff..935d6eb6a 100644 --- a/www/common/outer/team.js +++ b/www/common/outer/team.js @@ -319,8 +319,10 @@ define([ } ctx.Store.removeOwnedChannel('', data, cb); }, - Store: ctx.Store + Store: ctx.Store, + store: ctx.store }, { + teamId: team.id, outer: true, edPublic: keys.drive.edPublic, loggedIn: true, diff --git a/www/common/proxy-manager.js b/www/common/proxy-manager.js index 71097864d..0aa58c5f8 100644 --- a/www/common/proxy-manager.js +++ b/www/common/proxy-manager.js @@ -70,6 +70,30 @@ define([ delete Env.folders[id]; }; + var sendNotification = (Env, sfId, title) => { + var mailbox = Env.store.mailbox; + if (!mailbox) { return; } + var team = Env.cfg.teamId; + var box; + if (team) { + let teams = Env.store.modules['team'].getTeamsData(); + box = teams[team]; + } else { + let md = Env.Store.getMetadata(null, null, () => {}); + box = md.user; + } + mailbox.sendTo('SF_DELETED', { + sfId: sfId, + team: team, + title: title + }, { + curvePublic: box.curvePublic, + channel: box.notifications + }, (err) => { + console.error(err); + }); + }; + // Password may have changed var deprecateProxy = function (Env, id, channel, reason) { if (Env.folders[id] && Env.folders[id].deleting) { @@ -88,8 +112,12 @@ define([ // If it's explicitely a deletion, no need to deprecate, just delete if (reason && reason !== "PASSWORD_CHANGE") { - var temp = Util.find(Env, ['user', 'proxy', UserObject.SHARED_FOLDERS]); + let temp = Util.find(Env, ['user', 'proxy', UserObject.SHARED_FOLDERS]); + let title = temp[id] && temp[id].lastTitle; + if (title) { sendNotification(Env, id, title); } + delete temp[id]; + if (Env.Store && Env.Store.refreshDriveUI) { Env.Store.refreshDriveUI(); } return; } @@ -1317,6 +1345,7 @@ define([ unpinPads: data.unpin, onSync: data.onSync, Store: data.Store, + store: data.store, removeOwnedChannel: data.removeOwnedChannel, loadSharedFolder: data.loadSharedFolder, cfg: uoConfig, From f69dcbdda985e36c250552504f3a2e9cdb39840d Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 13 Sep 2023 14:48:16 +0200 Subject: [PATCH 13/21] Fix 'Unhandled RPC' log with isNewChannel --- lib/commands/channel.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/commands/channel.js b/lib/commands/channel.js index c3d354881..72b9d6bcd 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -254,7 +254,8 @@ var ARRAY_LINE = /^\[/; call back with true if the channel log has no content other than metadata otherwise false */ -Channel.isNewChannel = function (Env, channel, cb) { +Channel.isNewChannel = function (Env, channel, _cb) { + var cb = Util.once(_cb); if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } if (channel.length !== HK.STANDARD_CHANNEL_LENGTH && channel.length !== HK.ADMIN_CHANNEL_LENGTH) { return void cb('INVALID_CHAN'); } From 91b1af15fd5bc1a39ef53ed468fa31063e1397f9 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 13 Sep 2023 14:56:55 +0200 Subject: [PATCH 14/21] Remove unnecessary error log from deleted media-tag --- www/common/common-thumbnail.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/common-thumbnail.js b/www/common/common-thumbnail.js index 355e349b6..850dc52f0 100644 --- a/www/common/common-thumbnail.js +++ b/www/common/common-thumbnail.js @@ -316,7 +316,7 @@ define([ var key = secret.keys && secret.keys.cryptKey; MediaTag.fetchDecryptedMetadata(src, key, function (e, metadata) { if (e) { - if (e === 'XHR_ERROR') { return; } + if (/^XHR_ERROR/.test(e)) { return; } return console.error(e); } if (!metadata) { return console.error("NO_METADATA"); } From 4d643e51a146351d574553687bebb10ccb3ee243 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 13 Sep 2023 15:43:20 +0200 Subject: [PATCH 15/21] Add new error message with deleted template --- customize.dist/login.js | 3 --- www/common/common-interface.js | 34 +++++++++++++++++++++++++++---- www/common/cryptpad-common.js | 8 ++++---- www/common/sframe-common-outer.js | 8 ++++---- www/common/sframe-common.js | 9 ++++++-- www/drive/inner.js | 2 +- 6 files changed, 46 insertions(+), 18 deletions(-) diff --git a/customize.dist/login.js b/customize.dist/login.js index 82826366d..3517af787 100644 --- a/customize.dist/login.js +++ b/customize.dist/login.js @@ -604,9 +604,6 @@ define([ }); break; case 'DELETED_USER': - if (result.reason === 'PASSWORD_CHANGE') { - // XXX PLACEHOLDER: account password change login page - } UI.errorLoadingScreen(UI.getDestroyedPlaceholder(result.reason, true)); break; case 'INVAL_PASS': diff --git a/www/common/common-interface.js b/www/common/common-interface.js index b9812c2c0..e8698e124 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -1048,9 +1048,11 @@ define([ if (e.which === 27) { e.preventDefault(); e.stopPropagation(); + // Function: call the function (should be a redirect) + if (typeof(exitable) === "function") { return void exitable(); } + // Otherwise remove the loading screen $loading.hide(); $('html').toggleClass('cp-loading-noscroll', false); - if (typeof(exitable) === "function") { exitable(); } } }); } @@ -1549,18 +1551,29 @@ define([ // XXX Messages.dph_reason = "Reason: {0}"; + + Messages.dph_default = "This content is no longer available"; // default key when custom message not found + Messages.dph_account_destroyed = "This account has been deleted by its owner"; Messages.dph_account_inactive = "This account has been deleted for inactivity"; Messages.dph_account_moderated = "This account has been suspended by the moderation team"; Messages.dph_account_pw = "This account's password has been changed"; + Messages.dph_pad_destroyed = "This document has been destroyed by its owner"; Messages.dph_pad_inactive = "This document has been deleted for inactivity"; Messages.dph_pad_moderated = "This document has been deleted by the moderation team"; - Messages.dph_pad_moderated_account = "This document has deleted with its owner's account"; // Keep this key ??? + Messages.dph_pad_moderated_account = "This document has been deleted with its owner's account"; // Keep this key ??? Messages.dph_pad_pw = "The document you are trying to open is protected with a new password. Enter the correct password to access the content."; + Messages.dph_tmp_destroyed = "This template has been destroyed by its owner"; + Messages.dph_tmp_moderated = "This template has been deleted by the moderation team"; + Messages.dph_tmp_moderated_account = "This template has been deleted with its owner's account"; // Keep this key ??? + Messages.dph_tmp_pw = "This template is protected with a new password. Open it from your drive to enter its new password."; - UI.getDestroyedPlaceholderMessage = (code, isAccount) => { + + + + UI.getDestroyedPlaceholderMessage = (code, isAccount, isTemplate) => { var account = { ARCHIVE_OWNED: Messages.dph_account_destroyed, INACTIVE: Messages.dph_account_inactive, @@ -1568,6 +1581,12 @@ define([ MODERATION_BLOCK: Messages.dph_account_moderated, PASSWORD_CHANGE: Messages.dph_account_pw, }; + var template = { + ARCHIVE_OWNED: Messages.dph_tmp_destroyed, + MODERATION_PAD: Messages.dph_tmp_moderated, + MODERATION_ACCOUNT: Messages.dph_tmp_moderated_account, + PASSWORD_CHANGE: Messages.dph_tmp_pw + }; var pad = { ARCHIVE_OWNED: Messages.dph_pad_destroyed, INACTIVE: Messages.dph_pad_inactive, @@ -1576,7 +1595,14 @@ define([ MODERATION_ACCOUNT: Messages.dph_pad_moderated_account, PASSWORD_CHANGE: Messages.dph_pad_pw }; - return isAccount ? account[code] : pad[code]; + var msg = pad[code]; + if (isAccount) { + msg = account[code]; + } else if (isTemplate) { + msg = template[code]; + } + if (!msg) { msg = Messages.dph_default; } + return msg; }; UI.getDestroyedPlaceholder = function (reason, isAccount) { if (typeof(reason) !== "string") { return; } diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 6fd93f48b..ed20b69cc 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -961,9 +961,9 @@ define([ optsPut.accessKeys = keys; })); }).nThen(function () { - Crypt.get(parsed.hash, function (err, val) { + Crypt.get(parsed.hash, function (err, val, errData) { if (err) { - return void cb(err); + return void cb(err, errData); } if (!val) { return void cb('ENOENT'); @@ -1005,10 +1005,10 @@ define([ optsGet.accessKeys = keys; })); }).nThen(function () { - Crypt.get(parsed.hash, function (err, _val) { + Crypt.get(parsed.hash, function (err, _val, errData) { if (err) { _waitFor.abort(); - return void cb(err); + return void cb(err, errData); } try { val = JSON.parse(_val); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 1fe44e859..b8db20e59 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -2264,13 +2264,13 @@ define([ // server Cryptpad.useTemplate({ href: data.template - }, Cryptget, function (err) { + }, Cryptget, function (err, errData) { if (err) { // TODO: better messages in case of expired, deleted, etc.? if (err === 'ERESTRICTED') { sframeChan.event('EV_RESTRICTED_ERROR'); } else { - sframeChan.query("EV_LOADING_ERROR", "DELETED"); + sframeChan.query("EV_LOADING_ERROR", errData || 'DELETED'); } return; } @@ -2281,13 +2281,13 @@ define([ } // if we open a new code from a file if (Cryptpad.fromFileData && !isOO) { - Cryptpad.useFile(Cryptget, function (err) { + Cryptpad.useFile(Cryptget, function (err, errData) { if (err) { // TODO: better messages in case of expired, deleted, etc.? if (err === 'ERESTRICTED') { sframeChan.event('EV_RESTRICTED_ERROR'); } else { - sframeChan.query("EV_LOADING_ERROR", "DELETED"); + sframeChan.query("EV_LOADING_ERROR", errData || 'DELETED'); } return; } diff --git a/www/common/sframe-common.js b/www/common/sframe-common.js index ef39e02bf..a71d0babc 100644 --- a/www/common/sframe-common.js +++ b/www/common/sframe-common.js @@ -894,10 +894,15 @@ define([ ctx.sframeChan.on('EV_LOADING_ERROR', function (err) { var msg = err; - if (err === 'DELETED') { + if (err === 'DELETED' || (err && err.type === 'EDELETED')) { // XXX You can still use the current version in read-only mode by pressing Esc. // what if they don't have a keyboard (ie. mobile) - msg = Messages.deletedError + '
' + Messages.errorRedirectToHome; + if (err.type && err.message) { + msg = UI.getDestroyedPlaceholderMessage(err.message, false, true); + } else { + msg = Messages.deletedError; + } + msg += '
' + Messages.errorRedirectToHome; } else if (err === "INVALID_HASH") { msg = Messages.invalidHashError; } else if (err === 'ACCOUNT') { // block 404 but no placeholder diff --git a/www/drive/inner.js b/www/drive/inner.js index 91ee3c3c1..28dc8ef38 100644 --- a/www/drive/inner.js +++ b/www/drive/inner.js @@ -57,7 +57,7 @@ define([ if (!newObj || !Object.keys(newObj).length) { // Empty anon drive: deleted var msg = Messages.deletedError + '
' + Messages.errorRedirectToHome; - setTimeout(function () { UI.errorLoadingScreen(msg, false, function () {}); }); + setTimeout(function () { UI.errorLoadingScreen(msg, false, true); }); APP.newSharedFolder = null; } } From 235d5594f7eb556a308630c38c8519d6aafbcc52 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 13 Sep 2023 16:54:05 +0200 Subject: [PATCH 16/21] Detect placeholder when reading metadata --- lib/storage/file.js | 9 ++++++++- www/common/cryptget.js | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/storage/file.js b/lib/storage/file.js index 60ff9dfc1..a60944771 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -392,7 +392,14 @@ var getDedicatedMetadata = function (env, channelId, handler, _cb) { readMore(); }, function (err) { // ENOENT => there is no metadata log - if (!err || err.code === 'ENOENT') { return void cb(); } + if (!err || err.code === 'ENOENT') { + if (err && err.code === 'ENOENT') { + return readPlaceholder(env, channelId, (content) => { + cb(content); + }); + } + return void cb(); + } // otherwise stream errors? cb(err); }); diff --git a/www/common/cryptget.js b/www/common/cryptget.js index afc93b53d..233fa9cd4 100644 --- a/www/common/cryptget.js +++ b/www/common/cryptget.js @@ -193,6 +193,11 @@ define([ finish(Session, void 0, doc); }); }; + + config.onChannelError = function (info) { + finish(Session, info); + }; + overwrite(config, opt); start(Session, config); From 91adf31ff5fd45f468b308c6302d382da8eb59ac Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 13 Sep 2023 17:28:24 +0200 Subject: [PATCH 17/21] Fix page reload after pad password change --- www/common/inner/access.js | 6 +++ www/common/sframe-common-outer.js | 86 +++++++++++++++---------------- 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/www/common/inner/access.js b/www/common/inner/access.js index 1870d469e..55c07dd16 100644 --- a/www/common/inner/access.js +++ b/www/common/inner/access.js @@ -982,11 +982,17 @@ define([ if (data.warning) { return void UI.alert(Messages.properties_passwordWarning, function () { + if (isNotStored) { + return sframeChan.query('Q_PASSWORD_CHECK', newPass, () => { common.gotoURL(_href); }); + } common.gotoURL(_href); }, {force: true}); } return void UI.alert(UIElements.fixInlineBRs(Messages.properties_passwordSuccess), function () { if (!isSharedFolder) { + if (isNotStored) { + return sframeChan.query('Q_PASSWORD_CHECK', newPass, () => { common.gotoURL(_href); }); + } common.gotoURL(_href); } }); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index b8db20e59..15fe77b59 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -146,6 +146,7 @@ define([ '/common/common-constants.js', '/common/common-feedback.js', '/common/outer/local-store.js', + '/common/outer/login-block.js', '/common/outer/cache-store.js', '/customize/application_config.js', //'/common/test.js', @@ -153,7 +154,7 @@ define([ 'optional!/api/instance' ], waitFor(function (_CpNfOuter, _Cryptpad, _Crypto, _Cryptget, _SFrameChannel, _SecureIframe, _UnsafeIframe, _OOIframe, _Messaging, _Notifier, _Hash, _Util, _Realtime, _Notify, - _Constants, _Feedback, _LocalStore, _Cache, _AppConfig, /* _Test,*/ _UserObject, + _Constants, _Feedback, _LocalStore, _Block, _Cache, _AppConfig, /* _Test,*/ _UserObject, _Instance) { CpNfOuter = _CpNfOuter; Cryptpad = _Cryptpad; @@ -176,6 +177,7 @@ define([ Utils.Notify = _Notify; Utils.currentPad = currentPad; Utils.Instance = _Instance; + Utils.Block = _Block; AppConfig = _AppConfig; //Test = _Test; @@ -386,9 +388,9 @@ define([ Cryptpad.initialPath = newPad.p; if (newPad.pw) { try { - var uHash = Utils.LocalStore.getUserHash(); - var uSecret = Utils.Hash.getSecrets('drive', uHash); - var uKey = uSecret.keys.cryptKey; + var uHash = Utils.LocalStore.getBlockHash(); + var uSecret = Utils.Block.parseBlockHash(uHash); + var uKey = uSecret.keys.symmetric; newPadPassword = Crypto.decrypt(newPad.pw, uKey); } catch (e) { console.error(e); } } @@ -1263,6 +1265,43 @@ define([ }); }); }); + + sframeChan.on('Q_PASSWORD_CHECK', function (pw, cb) { + Cryptpad.isNewChannel(currentPad.href, pw, function (e, isNew) { + if (isNew === false) { + nThen(function (w) { + // If the pad is stored, update its data + var _secret = Utils.Hash.getSecrets(parsed.type, parsed.hash, pw); + var chan = _secret.channel; + var editH = Utils.Hash.getEditHashFromKeys(_secret); + var viewH = Utils.Hash.getViewHashFromKeys(_secret); + var href = Utils.Hash.hashToHref(editH, parsed.type); + var roHref = Utils.Hash.hashToHref(viewH, parsed.type); + Cryptpad.setPadAttribute('password', password, w(), parsed.getUrl()); + Cryptpad.setPadAttribute('channel', chan, w(), parsed.getUrl()); + Cryptpad.setPadAttribute('href', href, w(), parsed.getUrl()); + Cryptpad.setPadAttribute('roHref', roHref, w(), parsed.getUrl()); + }).nThen(function () { + // Get redirect URL + var uHash = Utils.LocalStore.getBlockHash(); + var uSecret = Utils.Block.parseBlockHash(uHash); + var uKey = uSecret.keys.symmetric; + var url = Utils.Hash.getNewPadURL(currentPad.href, { + pw: Crypto.encrypt(pw, uKey), + f: 1 + }); + // redirect + window.location.href = url; + document.location.reload(); + }); + + return; + } + cb({ + error: e + }); + }); + }); }; addCommonRpc(sframeChan, isSafe); @@ -1925,45 +1964,6 @@ define([ }); }); - sframeChan.on('Q_PASSWORD_CHECK', function (pw, cb) { - Cryptpad.isNewChannel(currentPad.href, pw, function (e, isNew) { - if (isNew === false) { - nThen(function (w) { - // If the pad is stored, update its data - var _secret = Utils.Hash.getSecrets(parsed.type, parsed.hash, pw); - var chan = _secret.channel; - var editH = Utils.Hash.getEditHashFromKeys(_secret); - var viewH = Utils.Hash.getViewHashFromKeys(_secret); - var href = Utils.Hash.hashToHref(editH, parsed.type); - var roHref = Utils.Hash.hashToHref(viewH, parsed.type); - Cryptpad.setPadAttribute('password', password, w(), parsed.getUrl()); - Cryptpad.setPadAttribute('channel', chan, w(), parsed.getUrl()); - Cryptpad.setPadAttribute('href', href, w(), parsed.getUrl()); - Cryptpad.setPadAttribute('roHref', roHref, w(), parsed.getUrl()); - }).nThen(function () { - // Get redirect URL - var uHash = Utils.LocalStore.getUserHash(); - var uSecret = Utils.Hash.getSecrets('drive', uHash); - var uKey = uSecret.keys.cryptKey; - var url = Utils.Hash.getNewPadURL(currentPad.href, { - pw: Crypto.encrypt(pw, uKey), - f: 1 - }); - // redirect - window.location.href = url; - document.location.reload(); - // XXX XXX XXX TEST TEST TEST - // XXX XXX XXX TEST TEST TEST - }); - - return; - } - cb({ - error: e - }); - }); - }); - sframeChan.on('Q_COPY_VIEW_URL', function (data, cb) { require(['/common/clipboard.js'], function (Clipboard) { var url = window.location.origin + From 399d50e9410b8366b4175f0b9f173d96fbcfed2a Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 13 Sep 2023 17:46:00 +0200 Subject: [PATCH 18/21] Clean unnecessary error log --- lib/hk-util.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/hk-util.js b/lib/hk-util.js index 81fc4c16e..09839e12c 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -199,10 +199,7 @@ const getMetadata = HK.getMetadata = function (Env, channelName, _cb) { } MetaRPC.getMetadataRaw(Env, channelName, function (err, metadata) { - if (err) { - console.error(err); - return void cb(err); - } + if (err) { return void cb(err); } if (!(metadata && typeof(metadata.channel) === 'string' && metadata.channel.length === STANDARD_CHANNEL_LENGTH)) { return cb(); } @@ -705,7 +702,7 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { stack: err && err.stack, }); } // FIXME err.message isn't useful for users - const parsedMsg = {error:err.message, test:'a', channel: channelName, txid: txid}; + const parsedMsg = {error:err.message, channel: channelName, txid: txid}; Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg)]); return; } @@ -726,7 +723,7 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { is no longer on the server so they don't abuse the data and so that they don't unintentionally continue to edit it in a broken state. */ - const parsedMsg2 = {error:'EDELETED', test:'b',channel: channelName, txid: txid}; + const parsedMsg2 = {error:'EDELETED', channel: channelName, txid: txid}; Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg2)]); return; } From f282db912149a64da56a582cd87bb781d4bc3db9 Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 14 Sep 2023 17:49:16 +0200 Subject: [PATCH 19/21] Add new placeholder data and account script to admin panel --- customize.dist/login.js | 22 ++++++- lib/archive-account.js | 79 ++++++++++++++++--------- lib/commands/admin-rpc.js | 54 ++++++++++++++--- lib/storage/blob.js | 3 + lib/storage/file.js | 3 + www/admin/inner.js | 101 +++++++++++++++++++++++++++----- www/common/cryptpad-common.js | 57 +++++++++++------- www/common/outer/async-store.js | 2 +- 8 files changed, 244 insertions(+), 77 deletions(-) diff --git a/customize.dist/login.js b/customize.dist/login.js index 3517af787..4e465a3dd 100644 --- a/customize.dist/login.js +++ b/customize.dist/login.js @@ -122,6 +122,9 @@ define([ .on('ready', function () { setTimeout(function () { cb(void 0, rt); }); }) + .on('error', function (info) { + cb(info.type, {reason: info.message}); + }) .on('disconnect', function (info) { cb('E_DISCONNECT', info); }); @@ -211,10 +214,13 @@ define([ return void console.log("Block requires 2FA"); } - if (err === 404 && response && response.reason - && response.reason !== 'PASSWORD_CHANGE') { + if (err === 404 && response && response.reason) { waitFor.abort(); w.abort(); + /* + // the following block prevent users from re-using an old password + if (isRegister) { return void cb('HAS_PLACEHOLDER'); } + */ return void cb('DELETED_USER', response); } @@ -299,6 +305,7 @@ define([ loadUserObject(opt, waitFor(function (err, rt) { if (err) { waitFor.abort(); + if (err === 'EDELETED') { return void cb('DELETED_USER', rt); } return void cb(err); } @@ -396,6 +403,7 @@ define([ loadUserObject(opt, waitFor(function (err, rt) { if (err) { waitFor.abort(); + if (err === 'EDELETED') { return void cb('DELETED_USER', rt); } return void cb('MODERN_REGISTRATION_INIT'); } @@ -603,8 +611,16 @@ define([ }); }); break; +/* + case 'HAS_PLACEHOLDER': + UI.errorLoadingScreen('UNAVAILABLE', true, true); + break; +*/ case 'DELETED_USER': - UI.errorLoadingScreen(UI.getDestroyedPlaceholder(result.reason, true)); + UI.errorLoadingScreen( + UI.getDestroyedPlaceholder(result.reason, true), true, () => { + window.location.reload(); + }); break; case 'INVAL_PASS': UI.removeLoadingScreen(function () { diff --git a/lib/archive-account.js b/lib/archive-account.js index 059d96680..b714bffed 100644 --- a/lib/archive-account.js +++ b/lib/archive-account.js @@ -1,7 +1,6 @@ /* jshint esversion: 6, node: true */ const nThen = require('nthen'); const Pins = require('./pins'); -const Environment = require("./env"); const Util = require("./common-util"); const Store = require('./storage/file.js'); const BlobStore = require("./storage/blob"); @@ -16,23 +15,9 @@ const Fse = require("fs-extra"); const { parentPort } = require('node:worker_threads'); -const config = require('./load-config'); -const Env = Environment.create(config); const COMMANDS = {}; let Log; -Env.computeMetadata = function (channel, cb) { - const ref = {}; - const lineHandler = Meta.createLineHandler(ref, (err) => { console.log(err); }); - return void Env.store.readChannelMetadata(channel, lineHandler, function (err) { - if (err) { - // stream errors? - return void cb(err); - } - cb(void 0, ref.meta); - }); -}; - const mkReportPath = function (Env, safeKey) { return Path.join(Env.paths.archive, 'accounts', safeKey); }; @@ -56,6 +41,21 @@ const deleteReport = (Env, key, cb) => { }; const init = (cb) => { + const Environment = require("./env"); + const config = require('./load-config'); + const Env = Environment.create(config); + Env.computeMetadata = function (channel, cb) { + const ref = {}; + const lineHandler = Meta.createLineHandler(ref, (err) => { console.log(err); }); + return void Env.store.readChannelMetadata(channel, lineHandler, function (err) { + if (err) { + // stream errors? + return void cb(err); + } + cb(void 0, ref.meta); + }); + }; + nThen((waitFor) => { Logger.create(config, waitFor(function (_) { Log = Env.Log = _; @@ -91,7 +91,9 @@ const init = (cb) => { } Env.blobStore = blob; })); - }).nThen(cb); + }).nThen(() => { + cb(Env); + }); }; COMMANDS.start = (edPublic, blockId, reason) => { @@ -106,8 +108,11 @@ COMMANDS.start = (edPublic, blockId, reason) => { let channelsToArchive = []; let deletedChannels = []; let deletedBlobs = []; + let Env; nThen((waitFor) => { - init(waitFor()); + init(waitFor((_Env) => { + Env = _Env; + })); }).nThen((waitFor) => { let lineHandler = Pins.createLineHandler(ref, (err) => { console.log(err); }); Env.pinStore.readMessagesBin(safeKey, 0, (msgObj, readMore) => { @@ -197,7 +202,8 @@ COMMANDS.start = (edPublic, blockId, reason) => { key: safeKey, channels: deletedChannels, blobs: deletedBlobs, - blockId: blockId + blockId: blockId, + reason: reason }; storeReport(Env, report, waitFor((err) => { if (err) { @@ -216,8 +222,11 @@ COMMANDS.restore = (edPublic) => { let pads, blobs; let blockId; let errors = []; + let Env; nThen((waitFor) => { - init(waitFor()); + init(waitFor((_Env) => { + Env = _Env; + })); }).nThen((waitFor) => { Log.info('MODERATION_ACCOUNT_RESTORE_START', edPublic, waitFor()); readReport(Env, safeKey, waitFor((err, report) => { @@ -290,14 +299,28 @@ COMMANDS.restore = (edPublic) => { }; -parentPort.on('message', (message) => { - let parsed = message; //JSON.parse(message); - let command = parsed.command; - let content = parsed.content; - let block = parsed.block; - let reason = parsed.reason; - COMMANDS[command](content, block, reason); -}); +const getStatus = (Env, edPublic, cb) => { + const safeKey = Util.escapeKeyCharacters(edPublic); + readReport(Env, safeKey, (err, report) => { + if (err) { return void cb(err); } + cb(void 0, report); + }); +}; -parentPort.postMessage('READY'); +if (parentPort) { + parentPort.on('message', (message) => { + let parsed = message; //JSON.parse(message); + let command = parsed.command; + let content = parsed.content; + let block = parsed.block; + let reason = parsed.reason; + COMMANDS[command](content, block, reason); + }); + parentPort.postMessage('READY'); +} else { + + module.exports = { + getStatus: getStatus + }; +} diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 3d2e50fee..512fd001b 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -10,6 +10,7 @@ const Core = require("./core"); const Channel = require("./channel"); const BlockStore = require("../storage/block"); const MFA = require("../storage/mfa"); +const ArchiveAccount = require('../archive-account'); /* jshint ignore:start */ const { Worker } = require('node:worker_threads'); /* jshint ignore:end */ @@ -288,22 +289,28 @@ var restoreArchivedDocument = function (Env, Server, cb, data) { } }; -// CryptPad_AsyncStore.rpc.send('ADMIN', ['ARCHIVE_ACCOUNT', edPublic], console.log) +// CryptPad_AsyncStore.rpc.send('ADMIN', ['ARCHIVE_ACCOUNT', {key, block, reason}], console.log) var archiveAccount = function (Env, Server, _cb, data) { const cb = Util.once(_cb); const worker = new Worker('./lib/archive-account.js'); + const args = Array.isArray(data) && data[1]; + if (!args || typeof(args) !== 'object') { return void cb("EINVAL"); } worker.on('message', message => { if (message === 'READY') { return worker.postMessage({ command: 'start', - content: data && data[1], - block: data && data[2], - reason: data && data[3] + content: args.key, + block: args.block, // optional, may be including in pin log + reason: args.reason }); } // DONE: disconnect all users from these channels - const reason = `MODERATION_ACCOUNT:${data && data[3]}`; + Env.Log.info('ARCHIVE_ACCOUNT_BY_ADMIN', { + safeKey: args.key, + reason: args.reason, + }); + const reason = `MODERATION_ACCOUNT:${args.reason}`; var deletedChannels = Util.tryParse(message); if (Array.isArray(deletedChannels)) { let n = nThen; @@ -326,14 +333,20 @@ var archiveAccount = function (Env, Server, _cb, data) { var restoreAccount = function (Env, Server, _cb, data) { const cb = Util.once(_cb); const worker = new Worker('./lib/archive-account.js'); + const args = Array.isArray(data) && data[1]; + if (!args || typeof(args) !== 'object') { return void cb("EINVAL"); } worker.on('message', message => { if (message === 'READY') { return worker.postMessage({ command: 'restore', - content: data && data[1] + content: args.key }); } // Response + Env.Log.info('RESTORE_ACCOUNT_BY_ADMIN', { + safeKey: args.key, + reason: args.reason, + }); cb(void 0, { state: true, errors: Util.tryParse(message) @@ -344,7 +357,11 @@ var restoreAccount = function (Env, Server, _cb, data) { cb(err); }); worker.on('exit', () => { worker.unref(); }); - +}; +var getAccountArchiveStatus = function (Env, Server, cb, data) { + const args = Array.isArray(data) && data[1]; + if (!args || typeof(args) !== 'object') { return void cb("EINVAL"); } + ArchiveAccount.getStatus(Env, args.key, cb); }; // CryptPad_AsyncStore.rpc.send('ADMIN', ['CLEAR_CACHED_CHANNEL_INDEX', documentID], console.log) @@ -567,6 +584,10 @@ var getDocumentStatus = function (Env, Server, cb, data) { } response.archived = result; })); + BlockStore.readPlaceholder(Env, id, w((result) => { + if (!result) { return; } + response.placeholder = result; + })); MFA.read(Env, id, w(function (err, v) { if (err === 'ENOENT') { response.totp = 'DISABLED'; @@ -598,6 +619,10 @@ var getDocumentStatus = function (Env, Server, cb, data) { } response.archived = result; })); + Env.blobStore.getPlaceholder(id, w((result) => { + if (!result) { return; } + response.placeholder = result; + })); }).nThen(function () { cb(void 0, response); }); @@ -616,6 +641,10 @@ var getDocumentStatus = function (Env, Server, cb, data) { } response.archived = result; })); + Env.store.getPlaceholder(id, w((result) => { + if (!result) { return; } + response.placeholder = result; + })); }).nThen(function () { cb(void 0, response); }); @@ -646,6 +675,8 @@ var getPinHistory = function (Env, Server, cb, data) { cb("NOT_IMPLEMENTED"); }; +/* +// NOTE: Deprecated, archive whole account now var archivePinLog = function (Env, Server, cb, data) { var args = Array.isArray(data) && data[1]; if (!args || typeof(args) !== 'object') { return void cb("EINVAL"); } @@ -671,6 +702,7 @@ var archivePinLog = function (Env, Server, cb, data) { cb(err); }); }; +*/ var archiveBlock = function (Env, Server, cb, data) { var args = Array.isArray(data) && data[1]; @@ -708,6 +740,8 @@ var restoreArchivedBlock = function (Env, Server, cb, data) { }); }; +/* +// NOTE: Deprecated, archive whole account now var restoreArchivedPinLog = function (Env, Server, cb, data) { var args = Array.isArray(data) && data[1]; if (!args || typeof(args) !== 'object') { return void cb("EINVAL"); } @@ -732,6 +766,7 @@ var restoreArchivedPinLog = function (Env, Server, cb, data) { cb(err); }); }; +*/ var archiveOwnedDocuments = function (Env, Server, cb, data) { Env.Log.debug('ARCHIVE_OWNED_DOCUMENTS', data); @@ -842,9 +877,9 @@ var commands = { GET_PIN_LIST: getPinList, GET_PIN_HISTORY: getPinHistory, - ARCHIVE_PIN_LOG: archivePinLog, + //ARCHIVE_PIN_LOG: archivePinLog, ARCHIVE_OWNED_DOCUMENTS: archiveOwnedDocuments, - RESTORE_ARCHIVED_PIN_LOG: restoreArchivedPinLog, + //RESTORE_ARCHIVED_PIN_LOG: restoreArchivedPinLog, ARCHIVE_BLOCK: archiveBlock, RESTORE_ARCHIVED_BLOCK: restoreArchivedBlock, @@ -854,6 +889,7 @@ var commands = { ARCHIVE_ACCOUNT: archiveAccount, RESTORE_ACCOUNT: restoreAccount, + GET_ACCOUNT_ARCHIVE_STATUS: getAccountArchiveStatus, CLEAR_CACHED_CHANNEL_INDEX: clearChannelIndex, GET_CACHED_CHANNEL_INDEX: getChannelIndex, diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 98c6a9266..b0b79dbce 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -689,6 +689,9 @@ BlobStore.create = function (config, _cb) { var path = prependArchive(Env, makeBlobPath(Env, blobId)); isFile(path, cb); }, + getPlaceholder: function (blobId, cb) { + readPlaceholder(Env, blobId, cb); + }, closeBlobstage: function (safeKey) { closeBlobstage(Env, safeKey); diff --git a/lib/storage/file.js b/lib/storage/file.js index a60944771..66cc8ba7e 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -1429,6 +1429,9 @@ module.exports.create = function (conf, _cb) { channelBytes(env, channelName, Util.both(cb, next)); }); }, + getPlaceholder: function (channelName, cb) { + readPlaceholder(env, channelName, cb); + }, // OTHER DATABASE FUNCTIONALITY // remove a particular channel from the cache closeChannel: function (channelName, cb) { diff --git a/www/admin/inner.js b/www/admin/inner.js index abe82d839..da5c0a896 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -53,6 +53,20 @@ define([ var common; var sFrameChan; +// XXX +// TO DELETE +// Messages.admin_archivePinLog +// Messages.admin_restoreArchivedPins +// TO ADD +Messages.admin_archiveAccount = "Archive this account"; +Messages.admin_archiveAccountInfo = "Including its owned documents"; +Messages.admin_archiveAccountConfirm = "Please specify the reason for archival, this will be shown to the user."; +Messages.admin_restoreAccount = "Restore this account"; +Messages.admin_accountSuspended = "Account archived by admin"; +Messages.admin_accountReport = "Account archive report"; +Messages.admin_accountReportFull = "Get detailed report"; +Messages.admin_channelPlaceholder = "Destroyed document placeholder"; + var categories = { 'general': [ // Msg.admin_cat_general 'cp-admin-flush-cache', @@ -226,6 +240,7 @@ define([ data.currentlyOnline = response[0]; })); }).nThen(function (w) { + if (!data.first) { return; } sframeCommand('GET_USER_QUOTA', key, w((err, response) => { if (err || !response) { return void console.error('quota', err, response); @@ -236,6 +251,7 @@ define([ } })); }).nThen(function (w) { + if (!data.first) { return; } // storage used sframeCommand('GET_USER_TOTAL_SIZE', key, w((err, response) => { if (err || !Array.isArray(response)) { @@ -246,6 +262,7 @@ define([ } })); }).nThen(function (w) { + if (!data.first) { return; } // channels pinned // files pinned sframeCommand('GET_USER_STORAGE_STATS', key, w((err, response) => { @@ -268,6 +285,17 @@ define([ data.archived = response[0].archived; } })); + }).nThen(function (w) { + if (data.first) { return; } + // Account is probably deleted + sframeCommand('GET_ACCOUNT_ARCHIVE_STATUS', {key}, w((err, response) => { + if (err || !Array.isArray(response) || !response[0]) { + console.error('account status', err, response); + } else { + console.info('account status', response); + data.archiveReport = response[0]; + } + })); }).nThen(function () { //console.log(data); try { @@ -281,6 +309,12 @@ define([ if (typeof(val) !== 'number') { return; } data[`${k}_formatted`] = getPrettySize(val); }); + if (data.archiveReport) { + let formatted = Util.clone(data.archiveReport); + formatted.channels = data.archiveReport.channels.length; + formatted.blobs = data.archiveReport.blobs.length; + data['archiveReport_formatted'] = JSON.stringify(formatted, 0, 2); + } } catch (err) { console.error(err); } @@ -391,11 +425,13 @@ define([ ])); } - // First pin activity time - row(Messages.admin_firstPinTime, maybeDate(data.first)); + if (data.first || data.latest) { + // First pin activity time + row(Messages.admin_firstPinTime, maybeDate(data.first)); - // last pin activity time - row(Messages.admin_lastPinTime, maybeDate(data.latest)); + // last pin activity time + row(Messages.admin_lastPinTime, maybeDate(data.latest)); + } // currently online row(Messages.admin_currentlyOnline, data.currentlyOnline); @@ -407,27 +443,46 @@ define([ row(Messages.admin_note, data.note || Messages.ui_none); // storage limit - row(Messages.admin_planlimit, getPrettySize(data.limit)); + if (data.limit) { row(Messages.admin_planlimit, getPrettySize(data.limit)); } // data stored - row(Messages.admin_storageUsage, getPrettySize(data.usage)); + if (data.usage) { row(Messages.admin_storageUsage, getPrettySize(data.usage)); } // number of channels - row(Messages.admin_channelCount, data.channels); + if (typeof(data.channel) === "number") { + row(Messages.admin_channelCount, data.channels); + } // number of files pinned - row(Messages.admin_fileCount, data.files); + if (typeof(data.channel) === "number") { + row(Messages.admin_fileCount, data.files); + } row(Messages.admin_pinLogAvailable, localizeState(data.live)); // pin log archived row(Messages.admin_pinLogArchived, localizeState(data.archived)); + if (data.archiveReport) { + row(Messages.admin_accountSuspended, localizeState(Boolean(data.archiveReport))); + } + if (data.archiveReport_formatted) { + let button, pre; + row(Messages.admin_accountReport, h('div', [ + pre = h('pre', data.archiveReport_formatted), + button = primary(Messages.admin_accountReportFull, () => { + $(button).remove(); + $(pre).html(JSON.stringify(data.archiveReport, 0, 2)); + }) + ])); + } + + // actions - if (data.archived && data.live === false) { - row(Messages.admin_restoreArchivedPins, primary(Messages.ui_restore, function () { + if (data.archived && data.live === false && data.archiveReport) { + row(Messages.admin_restoreAccount, primary(Messages.ui_restore, function () { justifyRestorationDialog('', reason => { - sframeCommand('RESTORE_ARCHIVED_PIN_LOG', { + sframeCommand('RESTORE_ACCOUNT', { key: data.key, reason: reason, }, function (err) { @@ -497,9 +552,10 @@ define([ // archive pin log var archiveHandler = () => { - justifyArchivalDialog(Messages.admin_archivePinLogConfirm, reason => { - sframeCommand('ARCHIVE_PIN_LOG', { + justifyArchivalDialog(Messages.admin_archiveAccountConfirm, reason => { + sframeCommand('ARCHIVE_ACCOUNT', { key: data.key, + block: data.blockId, reason: reason, }, (err /*, response */) => { console.error(err); @@ -512,7 +568,12 @@ define([ }); }; - row(Messages.admin_archivePinLog, danger(Messages.admin_archiveButton, archiveHandler)); + var archiveAccountLabel = h('span', [ + Messages.admin_archiveAccount, + h('br'), + h('small', Messages.archiveAccountInfo) + ]); + row(archiveAccountLabel, danger(Messages.admin_archiveButton, archiveHandler)); // archive owned documents /* // TODO not implemented @@ -678,6 +739,7 @@ define([ } data.live = res[0].live; data.archived = res[0].archived; + data.placeholder = res[0].placeholder; //console.error("get channel status", err, res); })); }).nThen(function () { @@ -696,7 +758,6 @@ define([ /* FIXME Messages.admin_getFullPinHistory = 'Pin history'; - Messages.admin_archivePinLogConfirm = "All content in this user's drive will be un-listed, meaning it may be deleted if it is not in any other drive."; Messages.admin_archiveOwnedAccountDocuments = "Archive this account's owned documents (not implemented)"; Messages.admin_archiveOwnedDocumentsConfirm = "All content owned exclusively by this user will be archived. This means their documents, drive, and accounts will be made inaccessible. This action cannot be undone. Please save the full pin list before proceeding to ensure individual documents can be restored."; */ @@ -791,6 +852,11 @@ define([ } + if (data.placeholder) { + console.warn('Placeholder code', data.placeholder); + row(Messages.admin_channelPlaceholder, UI.getDestroyedPlaceholderMessage(data.placeholder)); + } + if (data.live && data.archived) { let disableButtons; let restoreButton = danger(Messages.admin_unarchiveButton, function () { @@ -1075,6 +1141,7 @@ define([ data.live = res[0].live; data.archived = res[0].archived; data.totp = res[0].totp; + data.placeholder = res[0].placeholder; })); }).nThen(function () { try { @@ -1120,6 +1187,10 @@ define([ }); row(Messages.admin_archiveBlock, archiveButton); } + if (data.placeholder) { + console.warn('Placeholder code', data.placeholder); + row(Messages.admin_channelPlaceholder, UI.getDestroyedPlaceholderMessage(data.placeholder, true)); + } if (data.archived && !data.live) { var restoreButton = danger(Messages.ui_restore, function () { justifyRestorationDialog('', reason => { diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index ed20b69cc..aceb62526 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -1969,6 +1969,42 @@ define([ return void cb({ error: 'INVALID_CODE' }); } })); + }).nThen(function (waitFor) { + var blockUrl = Block.getBlockUrl(blockKeys); + // Check whether there is a block at that new location + Util.getBlock(blockUrl, {}, waitFor(function (err, response) { + // If there is no block or the block is invalid, continue. + // error 401 means protected block + + /* + // the following block prevent users from re-using an old password + if (err === 404 && response && response.reason) { + waitFor.abort(); + return void cb({ + error: 'EDELELED', + reason: response.reason + }); + } + */ + + if (err && err !== 401) { + console.log("no block found"); + return; + } + + response.arrayBuffer().then(waitFor(arraybuffer => { + var block = new Uint8Array(arraybuffer); + var decryptedBlock = Block.decrypt(block, blockKeys); + if (!decryptedBlock) { + console.error("Found a login block but failed to decrypt"); + return; + } + + // If there is already a valid block, abort! We risk overriding another user's data + waitFor.abort(); + cb({ error: 'EEXISTS' }); + })); + })); }).nThen(function (waitFor) { // Create a new user hash // Get the current content, store it in the new user file @@ -1997,27 +2033,6 @@ define([ } }), optsPut); })); - }).nThen(function (waitFor) { - var blockUrl = Block.getBlockUrl(blockKeys); - // Check whether there is a block at that new location - Util.fetch(blockUrl, waitFor(function (err, block) { - // If there is no block or the block is invalid, continue. - // error 401 means protected block - if (err && err !== 401) { - console.log("no block found"); - return; - } - - var decryptedBlock = Block.decrypt(block, blockKeys); - if (!decryptedBlock) { - console.error("Found a login block but failed to decrypt"); - return; - } - - // If there is already a valid block, abort! We risk overriding another user's data - waitFor.abort(); - cb({ error: 'EEXISTS' }); - })); }).nThen(function (waitFor) { // Write the new login block var content = { diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 89821b842..954ca4ba7 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -247,7 +247,7 @@ define([ list.push(userChannel); if (store.data && store.data.blockId) { - list.push(`${store.data.blockId}#block`); // XXX 5.5.0? + //list.push(`${store.data.blockId}#block`); // XXX 5.5.0? } list.sort(); From 1c7e5164bb9986409ce12eb5d6981fb246754b7e Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 22 Sep 2023 15:28:59 +0200 Subject: [PATCH 20/21] Fix account passord change for pre-block accounts --- www/common/cryptpad-common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index aceb62526..67d268973 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -2564,7 +2564,7 @@ define([ lastVisit: Number(localStorage.lastVisit) || undefined, blockId: blockId }; - common.userHash = userHash; + common.userHash = userHash || LocalStore.getUserHash(); // FIXME Backward compatibility if (sessionStorage.newPadFileData) { From 30743c295a113eca017c971cfcf30fcc37457d1d Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 28 Sep 2023 14:57:07 +0200 Subject: [PATCH 21/21] Bypass placeholder when removing a pad password --- lib/hk-util.js | 3 ++- www/common/cryptpad-common.js | 2 ++ www/common/inner/access.js | 9 +++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/hk-util.js b/lib/hk-util.js index 09839e12c..1060bc9fc 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -580,6 +580,7 @@ const handleFirstMessage = function (Env, channelName, metadata) { // Set the selfdestruct flag to history keeper ID to handle server crash. metadata.selfdestruct = Env.id; } + delete metadata.forcePlaceholder; Env.store.writeMetadata(channelName, JSON.stringify(metadata), function (err) { if (err) { // FIXME tell the user that there was a channel error? @@ -706,7 +707,7 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg)]); return; } - if (err && err.code === 'ENOENT' && reason) { + if (err && err.code === 'ENOENT' && reason && !metadata.forcePlaceholder) { const parsedMsg2 = {error:'EDELETED', message: reason, channel: channelName, txid: txid}; Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg2)]); return; diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 67d268973..b5a5fc770 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -1452,8 +1452,10 @@ define([ }).nThen(function (waitFor) { optsPut.metadata.restricted = oldMetadata.restricted; optsPut.metadata.allowed = oldMetadata.allowed; + if (!newPassword) { optsPut.metadata.forcePlaceholder = true; } Crypt.put(newHash, cryptgetVal, waitFor(function (err) { if (err) { + if (err === "EDELETED") { err = "PASSWORD_ALREADY_USED"; } waitFor.abort(); return void cb({ error: err }); } diff --git a/www/common/inner/access.js b/www/common/inner/access.js index 55c07dd16..40240540c 100644 --- a/www/common/inner/access.js +++ b/www/common/inner/access.js @@ -929,6 +929,7 @@ define([ }); } + Messages.access_passwordUsed = "This password has already been used for this pad. It can't be used again."; // XXX NEW var href = data.href; var isNotStored = Boolean(data.isNotStored); sframeChan.query(q, { @@ -939,8 +940,12 @@ define([ }, function (err, data) { $(passwordOk).text(Messages.properties_changePasswordButton); pLocked = false; - if (err || data.error) { - console.error(err || data.error); + err = err || data.error; + if (err) { + if (err === "PASSWORD_ALREADY_USED") { + return void UI.alert(Messages.access_passwordUsed); + } + console.error(err); return void UI.alert(Messages.properties_passwordError); } UI.findOKButton().click();