The_Great_Archive/cryptpad
1
0
Fork 0
mirror of https://github.com/cryptpad/cryptpad.git synced 2026-09-12 19:49:59 +05:00
Code Issues Packages Projects Releases Wiki Activity

feat!: remove old cryptpad server

Browse Source 
BREAKING CHANGE: remove old cryptpad server from the codebase
This commit is contained in:
Fabrice Mouhartem 2026-08-14 15:44:07 +02:00
parent 2860bb68d8
commit f7bfba9146
No known key found for this signature in database
56 changed files with 0 additions and 16807 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
lib/api.js
View File

@ -1,234 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const WebSocketServer = require('ws').Server;
const NetfluxSrv = require('chainpad-server');
const Decrees = require("./decrees");
const nThen = require("nthen");
const Fs = require("fs");
const Fse = require("fs-extra");
const Path = require("path");
const Nacl = require("tweetnacl/nacl-fast");
const Hash = require('./common-hash');
const Util = require('./common-util');
module.exports.create = function (Env) {
var log = Env.Log;
nThen(function (w) {
Decrees.load(Env, w(function (err) {
Env.flushCache();
if (err) {
log.error('DECREES_LOADING', {
error: err.code || err,
message: err.message,
});
console.error(err);
}
}));
}).nThen(function () {
if (Env.proofsMigrated) { return; }
const { Worker } = require('node:worker_threads');
const Admin = require("./commands/admin-rpc");
const worker = new Worker('./scripts/migrations/migrate-blob-proofs.js');
worker.on('message', message => {
if (message === 'READY') {
log.info('BLOB_PROOFS_MIGRATION');
return void worker.postMessage({
start: 1,
});
}
if (message === 'MIGRATED') {
return void log.info('BLOB_PROOFS_DELETION');
}
if (message === 'CLEANED') {
log.info('BLOB_PROOFS_MIGRATED');
Admin.sendDecree(Env, null, function (err) {
if (err) { return void log.error('BLOB_PROOF', err); }
Env.flushCache();
}, ['PROOFS_MIGRATED', ['PROOFS_MIGRATED', 1]], 'server');
}
});
}).nThen(function (w) {
let admins = Env.admins || [];
// If we don't have any admin on this instance, print an onboarding link
if (Array.isArray(admins) && admins.length) { return; }
let token = Env.installToken;
let printLink = () => {
let url = `${Env.httpUnsafeOrigin}/install/#${token}`;
console.log('=============================');
console.log('Create your first admin account and customize your instance by visiting');
console.log(url);
console.log('=============================');
};
// If we already have a token, print it
if (token) { return void printLink(); }
// Otherwise create a new token
let decreeName = Path.join(Env.paths.decree, 'decree.ndjson');
token = Hash.createChannelId() + Hash.createChannelId();
let decree = ["ADD_INSTALL_TOKEN",[token],"",+new Date()];
Fs.appendFile(decreeName, JSON.stringify(decree) + '\n', w(function (err) {
if (err) { console.log(err); return; }
Env.installToken = token;
Env.envUpdated.fire();
printLink();
}));
}).nThen(function () {
if (!Env.admins.length) {
Env.Log.info('NO_ADMIN_CONFIGURED', {
message: `Your instance is not correctly configured for production usage. Review its checkup page for more information.`,
details: new URL('/checkup/', Env.httpUnsafeOrigin).href,
});
}
}).nThen(function (w) {
// we assume the server has generated a secret used to validate JWT tokens
if (typeof(Env.bearerSecret) === 'string') { return; }
// if one does not exist, then create one and remember it
// 256 bits
var bearerSecret = Util.encodeBase64(Nacl.randomBytes(32));
Env.bearerSecret = bearerSecret;
Env.Log.info("GENERATING_BEARER_SECRET", {});
Decrees.write(Env, [
'SET_BEARER_SECRET',
[bearerSecret],
'INTERNAL',
+new Date()
], w(function (err) {
if (err) { throw err; }
}));
}).nThen(function (w) {
Fse.mkdirp(Env.paths.block, w(function (err) {
if (err) {
log.error("BLOCK_FOLDER_CREATE_FAILED", err);
}
}));
}).nThen(function (w) {
var fullPath = Path.join(Env.paths.block, 'placeholder.txt');
Fs.writeFile(fullPath, 'PLACEHOLDER\n', w(function (err) {
if (err) {
log.error('BLOCK_PLACEHOLDER_CREATE_FAILED', err);
}
}));
}).nThen(function () {
// asynchronously create a historyKeeper and RPC together
require('./historyKeeper.js').create(Env, function (err, historyKeeper) {
if (err) { throw err; }
var noop = function () {};
var special_errors = {};
['EPIPE', 'ECONNRESET'].forEach(function (k) { special_errors[k] = noop; });
special_errors.NF_ENOENT = function (error, label, info) {
delete info.stack;
log.error(label, {
info: info,
});
};
// spawn ws server and attach netflux event handlers
let Server = Env.Server = NetfluxSrv.create(new WebSocketServer({ server: Env.httpServer}))
.on('channelClose', historyKeeper.channelClose)
.on('channelMessage', historyKeeper.channelMessage)
.on('channelOpen', historyKeeper.channelOpen)
.on('sessionClose', historyKeeper.sessionClose)
.on('sessionOpen', historyKeeper.sessionOpen)
.on('error', function (error, label, info) {
if (!error) { return; }
var code = error && (error.code || error.message);
if (code) {
/* EPIPE,ECONNERESET, NF_ENOENT */
if (typeof(special_errors[code]) === 'function') {
return void special_errors[code](error, label, info);
}
}
/* labels:
SEND_MESSAGE_FAIL, SEND_MESSAGE_FAIL_2, FAIL_TO_DISCONNECT,
FAIL_TO_TERMINATE, HANDLE_CHANNEL_LEAVE, NETFLUX_BAD_MESSAGE,
NETFLUX_WEBSOCKET_ERROR
*/
log.error(label, {
code: error.code,
message: error.message,
stack: error.stack,
info: info,
});
})
.register(historyKeeper.id, historyKeeper.directMessage);
// Store max active WS during the last day (reset when sending ping if enabled)
setInterval(() => {
try {
// Concurrent usage data
let oldWs = Env.maxConcurrentWs || 0;
let oldUniqueWs = Env.maxConcurrentUniqueWs || 0;
let oldChans = Env.maxActiveChannels || 0;
let oldUsers = Env.maxConcurrentRegUsers || 0;
let stats = Server.getSessionStats();
let chans = Server.getActiveChannelCount();
const map = Env.netfluxUsers;
// Extract public key from each ws connection
let regKeys = Object.keys(map).map(id => {
return Object.keys(map[id] || {})[0];
}).filter(Boolean);
// Convert to set to get only unique values
let regSet = new Set(regKeys);
const reg = regSet.size;
Env.maxConcurrentWs = Math.max(oldWs, stats.total);
Env.maxConcurrentUniqueWs = Math.max(oldUniqueWs, stats.unique);
Env.maxConcurrentRegUsers = Math.max(oldUsers, reg);
Env.maxActiveChannels = Math.max(oldChans, chans);
} catch (e) {}
}, 10000);
// Clean up active registered users and channels (possible memory leak)
setInterval(() => {
try {
let users = Env.netfluxUsers || {};
let online = Server.getOnlineUsers() || [];
let onlineSet = new Set(online);
let removed = 0;
Object.keys(users).forEach(id => {
if (!onlineSet.has(id)) {
delete users[id];
removed++;
}
});
if (removed) {
Env.Log.info("CLEANED_ACTIVE_USERS_MAP", {removed});
}
} catch (e) {}
try {
let HK = require('./hk-utils');
let chans = Env.channel_cache || {};
let active = Server.getActiveChannels() || [];
let activeSet = new Set(active);
let removed = 0;
Object.keys(chans).forEach(id => {
if (!activeSet.has(id)) {
HK.dropChannel(Env, id);
removed++;
}
});
if (Env.store) {
Env.store.closeInactiveChannels(activeSet);
}
if (removed) {
Env.Log.info("CLEANED_ACTIVE_CHANNELS_MAP", {removed});
}
} catch (e) {}
}, 30000);
});
});
};

356
lib/archive-account.js
View File

@ -1,356 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const nThen = require('nthen');
const Pins = require('./pins');
const Util = require("./common-util");
const Store = require('./storage/file.js');
const BlobStore = require("./storage/blob");
const BlockStore = require("./storage/block");
const Core = require("./commands/core");
const Metadata = require("./commands/metadata");
const Meta = require("./metadata");
const Logger = require("./log");
const plugins = require("./plugin-manager");
const HK = require('./hk-util');
const MFA = require("./storage/mfa");
let SSOUtils = plugins.SSO && plugins.SSO.utils;
const Path = require("path");
const Fse = require("fs-extra");
const { parentPort } = require('node:worker_threads');
const COMMANDS = {};
let Log;
const mkReportPath = function (Env, safeKey) {
return Path.join(Env.paths.archive, 'accounts', safeKey);
};
const storeReport = (Env, report, cb) => {
let path = mkReportPath(Env, report.key);
let s_data;
try {
s_data = JSON.stringify(report);
Fse.outputFile(path, s_data, cb);
} catch (err) {
return void cb(err);
}
};
const readReport = (Env, key, cb) => {
let path = mkReportPath(Env, key);
Fse.readJson(path, cb);
};
const deleteReport = (Env, key, cb) => {
let path = mkReportPath(Env, key);
Fse.remove(path, cb);
};
const init = (cb) => {
const Environment = require("./env");
const config = require('./load-config');
const Env = Environment.create(config);
Env.computeMetadata = function (channel, cb) {
const ref = {};
const lineHandler = Meta.createLineHandler(ref, (err) => { console.log(err); });
let f = Env.store.readChannelMetadata;
if (channel.length === HK.BLOB_ID_LENGTH) {
f = Env.blobStore.readMetadata;
}
return void f(channel, lineHandler, function (err) {
if (err) {
// stream errors?
return void cb(err);
}
cb(void 0, ref.meta);
});
};
nThen((waitFor) => {
Logger.create(config, waitFor(function (_) {
Log = Env.Log = _;
}));
Store.create(config, waitFor(function (err, _store) {
if (err) {
waitFor.abort();
return void cb(err);
}
Env.store = _store;
}));
Store.create({
filePath: config.pinPath,
archivePath: config.archivePath,
// archive pin logs to their own subpath
volumeId: 'pins',
}, waitFor(function (err, _) {
if (err) {
waitFor.abort();
throw err;
}
Env.pinStore = _;
}));
BlobStore.create({
blobPath: config.blobPath,
blobStagingPath: config.blobStagingPath,
archivePath: config.archivePath,
getSession: function () {},
}, waitFor(function (err, blob) {
if (err) {
waitFor.abort();
return void cb(err);
}
Env.blobStore = blob;
}));
}).nThen(() => {
cb(Env);
});
};
COMMANDS.start = (edPublic, blockId, reason) => {
const safeKey = Util.escapeKeyCharacters(edPublic);
const archiveReason = {
code: 'MODERATION_ACCOUNT',
txt: reason
};
let ref = {};
let blobsToArchive = [];
let channelsToArchive = [];
let deletedChannels = [];
let deletedBlobs = [];
let Env;
nThen((waitFor) => {
init(waitFor((_Env) => {
Env = _Env;
}));
}).nThen((waitFor) => {
let lineHandler = Pins.createLineHandler(ref, (err) => { console.log(err); });
Env.pinStore.readMessagesBin(safeKey, 0, (msgObj, readMore) => {
lineHandler(msgObj.buff.toString('utf8'));
readMore();
}, waitFor());
}).nThen((waitFor) => {
Log.info('MODERATION_ACCOUNT_ARCHIVAL_START', edPublic, waitFor());
var n = nThen;
Object.keys(ref.pins || {}).forEach((chanId) => {
n = n((w) => {
// Blobs
if (Env.blobStore.isFileId(chanId)) {
return Env.computeMetadata(chanId, w((e, md) => {
if (e || !md) { return; }
if (md && md.owners
&& md.owners.includes(edPublic)) {
blobsToArchive.push(chanId);
}
}));
}
// Pads
Metadata.getMetadata(Env, chanId, w((err, metadata) => {
if (err) { return; } // Can't read metadata? Don't archive
if (!Core.hasOwners(metadata)) { return; } // No owner, don't archive
if (Core.isOwner(metadata, edPublic) && metadata.owners.length === 1) {
channelsToArchive.push(chanId); // Only owner: archive
}
}));
}).nThen;
});
n(waitFor());
}).nThen((waitFor) => {
Log.info('MODERATION_ACCOUNT_ARCHIVAL_LISTED', JSON.stringify({
pads: channelsToArchive.length,
blobs: blobsToArchive.length
}), waitFor());
var n = nThen;
// Archive the pads
channelsToArchive.forEach((chanId) => {
n = n((w) => {
Env.store.archiveChannel(chanId, archiveReason, w(function (err) {
if (err) {
return Log.error('MODERATION_CHANNEL_ARCHIVAL_ERROR', {
error: err,
channel: chanId,
}, w());
}
deletedChannels.push(chanId);
Log.info('MODERATION_CHANNEL_ARCHIVAL', chanId, w());
}));
}).nThen;
});
// Archive the blobs
blobsToArchive.forEach((blobId) => {
n = n((w) => {
Env.blobStore.archive.blob(blobId, archiveReason, w(function (err) {
if (err) {
return Log.error('MODERATION_BLOB_ARCHIVAL_ERROR', {
error: err,
item: blobId,
}, w());
}
deletedBlobs.push(blobId);
Log.info('MODERATION_BLOB_ARCHIVAL', blobId, w());
}));
}).nThen;
});
n(waitFor(() => {
// Archive the pin log
Env.pinStore.archiveChannel(safeKey, undefined, waitFor(function (err) {
if (err) {
return Log.error('MODERATION_ACCOUNT_PIN_LOG', err, waitFor());
}
Log.info('MODERATION_ACCOUNT_LOG', safeKey, waitFor());
}));
blockId = blockId || ref.block;
if (!blockId) { return; }
BlockStore.archive(Env, blockId, archiveReason, waitFor(function (err) {
if (err) {
blockId = undefined;
return Log.error('MODERATION_ACCOUNT_BLOCK', err, waitFor());
}
Log.info('MODERATION_ACCOUNT_BLOCK', safeKey, waitFor());
}));
MFA.delete(Env, blockId, waitFor());
if (!SSOUtils) { return; }
SSOUtils.deleteAccount(Env, blockId, waitFor((err) => {
if (err) {
return Log.error('MODERATION_ACCOUNT_BLOCK_SSO', err, waitFor());
}
}));
}));
}).nThen((waitFor) => {
var report = {
key: safeKey,
channels: deletedChannels,
blobs: deletedBlobs,
blockId: blockId,
reason: reason
};
storeReport(Env, report, waitFor((err) => {
if (err) {
return Log.error('MODERATION_ACCOUNT_REPORT', report, waitFor());
}
}));
}).nThen(() => {
parentPort.postMessage(JSON.stringify(deletedChannels));
process.exit(0);
});
};
COMMANDS.restore = (edPublic) => {
const safeKey = Util.escapeKeyCharacters(edPublic);
let pads, blobs;
let blockId;
let errors = [];
let Env;
nThen((waitFor) => {
init(waitFor((_Env) => {
Env = _Env;
}));
}).nThen((waitFor) => {
Log.info('MODERATION_ACCOUNT_RESTORE_START', edPublic, waitFor());
readReport(Env, safeKey, waitFor((err, report) => {
if (err) { throw new Error(err); }
pads = report.channels;
blobs = report.blobs;
blockId = report.blockId;
}));
}).nThen((waitFor) => {
Log.info('MODERATION_ACCOUNT_RESTORE_LISTED', JSON.stringify({
pads: pads.length,
blobs: blobs.length
}), waitFor());
var n = nThen;
pads.forEach((chanId) => {
n = n((w) => {
Env.store.restoreArchivedChannel(chanId, w(function (err) {
if (err) {
errors.push(chanId);
return Log.error('MODERATION_CHANNEL_RESTORE_ERROR', {
error: err,
channel: chanId,
}, w());
}
Log.info('MODERATION_CHANNEL_RESTORE', chanId, w());
}));
}).nThen;
});
blobs.forEach((blobId) => {
n = n((w) => {
Env.blobStore.restore.blob(blobId, w(function (err) {
if (err) {
errors.push(blobId);
return Log.error('MODERATION_BLOB_RESTORE_ERROR', {
error: err,
item: blobId,
}, w());
}
Log.info('MODERATION_BLOB_RESTORE', blobId, w());
}));
}).nThen;
});
n(waitFor(() => {
// remove the pin logs of inactive accounts if inactive account removal is configured
Env.pinStore.restoreArchivedChannel(safeKey, waitFor(function (err) {
if (err) {
return Log.error('MODERATION_ACCOUNT_PIN_LOG_RESTORE', err, waitFor());
}
Log.info('MODERATION_ACCOUNT_LOG_RESTORE', safeKey, waitFor());
}));
if (!blockId) { return; }
BlockStore.restore(Env, blockId, waitFor(function (err) {
if (err) {
blockId = undefined;
return Log.error('MODERATION_ACCOUNT_BLOCK_RESTORE', err, waitFor());
}
Log.info('MODERATION_ACCOUNT_BLOCK_RESTORE', safeKey, waitFor());
}));
if (!SSOUtils) { return; }
SSOUtils.restoreAccount(Env, blockId, waitFor(function (err) {
if (err) {
return Log.error('MODERATION_ACCOUNT_BLOCK_RESTORE_SSO', err, waitFor());
}
}));
}));
}).nThen((waitFor) => {
deleteReport(Env, safeKey, waitFor((err) => {
if (err) {
return Log.error('MODERATION_ACCOUNT_REPORT_DELETE', safeKey, waitFor());
}
}));
}).nThen(() => {
parentPort.postMessage(JSON.stringify(errors));
process.exit(0);
});
};
const getStatus = (Env, edPublic, cb) => {
const safeKey = Util.escapeKeyCharacters(edPublic);
readReport(Env, safeKey, (err, report) => {
if (err) { return void cb(err); }
cb(void 0, report);
});
};
if (parentPort) {
parentPort.on('message', (message) => {
let parsed = message; //JSON.parse(message);
let command = parsed.command;
let content = parsed.content;
let block = parsed.block;
let reason = parsed.reason;
COMMANDS[command](content, block, reason);
});
parentPort.postMessage('READY');
} else {
module.exports = {
getStatus: getStatus
};
}

68
lib/batch-read.js
View File

@ -1,68 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
/*
## Purpose
To avoid running expensive IO or computation concurrently.
If the result of IO or computation is requested while an identical request
is already in progress, wait until the first one completes and provide its
result to every routine that requested it.
Asynchrony is guaranteed.
## Usage
Provide:
1. a named key for the computation or resource,
2. a callback to handle the result
3. an implementation which calls back with the result
```
var batch = Batch();
var read = function (path, cb) {
batch(path, cb, function (done) {
console.log("reading %s", path);
fs.readFile(path, 'utf8', done);
});
};
read('./pewpew.txt', function (err, data) {
if (err) { return void console.error(err); }
console.log(data);
});
read('./pewpew.txt', function (err, data) {
if (err) { return void console.error(err); }
console.log(data);
});
```
*/
module.exports = function (/* task */) {
var map = {};
return function (id, cb, impl) {
if (typeof(cb) !== 'function' || typeof(impl) !== 'function') {
throw new Error("expected callback and implementation");
}
if (map[id]) { return void map[id].push(cb); }
map[id] = [cb];
impl(function () {
var args = Array.prototype.slice.call(arguments);
//if (map[id] && map[id].length > 1) { console.log("BATCH-READ DID ITS JOB for [%s][%s]", task, id); }
setTimeout(function () {
map[id].forEach(function (h) {
h.apply(null, args);
});
delete map[id];
});
});
};
};

113
lib/challenge-commands/base.js
View File

@ -1,113 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Block = require("../commands/block");
const MFA = require("../storage/mfa");
const Util = require("../common-util");
const Sessions = require("../storage/sessions");
const Commands = module.exports;
var isValidBlockId = Block.isValidBlockId;
// Read the MFA settings for the given public key
const checkMFA = (Env, publicKey, cb) => {
// Success if we can't get the MFA settings
MFA.read(Env, publicKey, function (err, content) {
if (err) {
if (err.code !== "ENOENT") {
Env.Log.error('TOTP_VALIDATE_MFA_READ', {
error: err,
publicKey: publicKey,
});
}
return void cb();
}
var parsed = Util.tryParse(content);
if (!parsed) { return void cb(); }
cb("NOT_ALLOWED");
});
};
// Make sure the block is not protected by MFA but don't do anything else
const check = Commands.MFA_CHECK = function (Env, body, cb) {
var { publicKey } = body;
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
checkMFA(Env, publicKey, cb);
};
check.complete = function (Env, body, cb) { cb(); };
// Write a login block IFF
// 1. You can sign for the block's public key
// 2. the block is not protected by MFA
// Note: the internal WRITE_LOGIN_BLOCK will check is you're allowed to create this block
const writeBlock = Commands.WRITE_BLOCK = function (Env, body, cb) {
const { publicKey, content } = body;
// they must provide a valid block public key
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
if (publicKey !== content.publicKey) { return void cb("INVALID_KEY"); }
// check MFA
checkMFA(Env, publicKey, cb);
};
writeBlock.complete = function (Env, body, cb) {
const { publicKey, content, session } = body;
Block.writeLoginBlock(Env, content, (err) => {
if (err) { return void cb(err); }
if (!session) { return void cb(); }
const proof = Util.tryParse(content.registrationProof);
const oldKey = proof && proof[0];
Sessions.update(Env, publicKey, oldKey, session, "", cb);
});
};
// Remove a login block IFF
// 1. You can sign for the block's public key
// 2. the block is not protected by MFA
const removeBlock = Commands.REMOVE_BLOCK = function (Env, body, cb) {
const { publicKey } = body;
// they must provide a valid block public key
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
// check MFA
checkMFA(Env, publicKey, cb);
};
removeBlock.complete = function (Env, body, cb) {
const { publicKey, edPublic, reason } = body;
Block.removeLoginBlock(Env, publicKey, reason, edPublic, cb);
};
// Get an upload cookie
// Get a cookie allowing you to upload to the blobstage of your user
const uploadCookie = Commands.UPLOAD_COOKIE = function (Env, body, cb) {
const { publicKey } = body;
// they must provide a valid public key
if (publicKey && typeof(publicKey) === "string"
&& publicKey.length === 44) {
return cb();
}
cb("INVALID_KEY");
};
uploadCookie.complete = function (Env, body, cb) {
const { publicKey } = body;
const safeKey = Util.escapeKeyCharacters(publicKey);
Env.blobStore.uploadCookie(safeKey, (err, cookie) => {
if (err) { return void cb(err); }
cb(void 0, {cookie});
});
};

529
lib/challenge-commands/totp.js
View File

@ -1,529 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const B32 = require("thirty-two");
const OTP = require("notp");
const nThen = require("nthen");
const Util = require("../common-util");
const MFA = require("../storage/mfa");
const Sessions = require("../storage/sessions");
const BlockStore = require("../storage/block");
const Block = require("../commands/block");
const config = require("../load-config");
const Commands = module.exports;
var isString = s => typeof(s) === 'string';
// basic definition of what we'll accept as an OTP code
// exactly six numerical digits
var isValidOTP = otp => {
return isString(otp) &&
// in the future this could be updated to support 8 digits
otp.length === 6 &&
// \D is non-digit characters, so this tests that it is exclusively numeric
!/\D/.test(otp);
};
// basic definition of what we'll accept as a recovery key
// 24 bytes encoded as b64 ==> 32 characters
var isValidRecoveryKey = otp => {
return isString(otp) &&
// in the future this could be updated to support 8 digits
otp.length === 32 &&
// \D is non-digit characters, so this tests that it is exclusively numeric
/[A-Za-z0-9+\/]{32}/.test(otp);
};
// we'll only allow users to set up multi-factor auth
// for keypairs they control which already have blocks
// this check doesn't confirm that their id is valid base64
// any attempt relying on this should fail when we can't decode
// the id they provided.
var isValidBlockId = Block.isValidBlockId;
// the base32 library can throw when decoding under various conditions.
// we have some basic requirements for the length of base32 as well,
// so we just do all the validation here. It either returns a buffer
// of length 20 or undefined, so the caller can just check whether it's
// falsey and otherwise assume it was well-formed
// Length === 20 comes from the recommendation of 160 bits of entropy
// in RFC4226 (https://www.rfc-editor.org/rfc/rfc4226#section-4)
var decode32 = S => {
let decoded;
try {
decoded = B32.decode(S);
} catch (err) { return; }
if (!(decoded instanceof Buffer) || decoded.length !== 20) { return; }
return decoded;
};
// Allow user settings?
var EXPIRATION = (config.otpSessionExpiration || 7 * 24) * 3600 * 1000;
// Create a session with a token for the given public key
const makeSession = (Env, publicKey, oldKey, ssoSession, cb) => {
const sessionId = ssoSession || Sessions.randomId();
let SSOUtils = Env.plugins && Env.plugins.SSO && Env.plugins.SSO.utils;
// For password change, we need to get the sso session associated to the old block key
// In other cases (login and totp_setup), the sso session is associated to the current block
oldKey = oldKey || publicKey; // use the current block if no old key
let isUpdate = false;
nThen(function (w) {
if (!ssoSession || !SSOUtils) { return; }
// If we have an session token, confirm this is an sso account
SSOUtils.readBlock(Env, oldKey, w((err) => {
if (err === 'ENOENT') { return; } // No sso block, no need to update the session
if (err) {
w.abort();
return void cb('TOTP_VALIDATE_READ_SSO');
}
// We have an existing session for an SSO account: update the existing session
isUpdate = true;
}));
}).nThen(function (w) {
// store the token
let sessionData = {
mfa: {
type: 'otp',
exp: (+new Date()) + EXPIRATION
}
};
var then = w(function (err) {
if (err) {
Env.Log.error("TOTP_VALIDATE_SESSION_WRITE", {
error: Util.serializeError(err),
publicKey: publicKey,
sessionId: sessionId,
});
w.abort();
return void cb("SESSION_WRITE_ERROR");
}
// else continue
});
if (isUpdate) {
Sessions.update(Env, publicKey, oldKey, sessionId, JSON.stringify(sessionData), then);
} else {
Sessions.write(Env, publicKey, sessionId, JSON.stringify(sessionData), then);
}
}).nThen(function () {
cb(void 0, {
bearer: sessionId,
});
});
};
// Read the MFA settings for the given public key
const readMFA = (Env, publicKey, cb) => {
// check that there is an MFA configuration for the given account
MFA.read(Env, publicKey, function (err, content) {
if (err) {
Env.Log.error('TOTP_VALIDATE_MFA_READ', {
error: err,
publicKey: publicKey,
});
return void cb('NO_MFA_CONFIGURED');
}
var parsed = Util.tryParse(content);
if (!parsed) { return void cb("INVALID_CONFIGURATION"); }
cb(undefined, parsed);
});
};
// Check if an OTP code is valid against the provided secret
const checkCode = (Env, secret, code, publicKey, _cb) => {
const cb = Util.mkAsync(_cb);
let decoded = decode32(secret);
if (!decoded) {
Env.Log.error("TOTP_VALIDATE_INVALID_SECRET", {
publicKey, // log the public key so the admin can investigate further
// don't log the problematic secret directly as
// logs are likely to be pasted in random places
});
return void cb("E_INVALID_SECRET");
}
// validate the code
var validated = OTP.totp.verify(code, decoded, {
window: 1,
});
if (!validated) {
// I won't worry about logging these OTPs as they shouldn't leak any useful information
Env.Log.error("TOTP_VALIDATE_BAD_OTP", {
code,
});
return void cb("INVALID_OTP");
}
// call back to indicate that their request was well-formed and valid
cb();
};
// This command allows clients to configure TOTP as a second factor protecting
// their login block IFF they:
// 1. provide a sufficiently strong TOTP secret
// 2. are able to produce a valid OTP code for that secret (indicating that their clock is sufficiently close to ours)
// 3. such a login block actually exists
// 4. are able to sign an arbitrary message for the login block's public key
// 5. have not already configured TOTP protection for this account
// (changing to a new secret can be done by disabling and re-enabling TOTP 2FA)
const TOTP_SETUP = Commands.TOTP_SETUP = function (Env, body, cb) {
const { publicKey, secret, code, contact } = body;
// the client MUST provide an OTP code of the expected format
// this doesn't check if it matches the secret and time, just that it's well-formed
if (!isValidOTP(code)) { return void cb("E_INVALID"); }
// if they provide an (optional) point of contact as a recovery mechanism then it should be a string.
// the intent is to allow to specify some side channel for those who inevitably lock themselves out
// we should be able to use that to validate their identity.
// I don't want to assume email, but limiting its length to 254 (the maximum email length) seems fair.
if (contact && (!isString(contact) || contact.length > 254)) { return void cb("INVALID_CONTACT"); }
// Check that the provided public key is the expected format for a block
if (!isValidBlockId(publicKey)) {
return void cb("INVALID_KEY");
}
// decode32 checks whether the secret decodes to a sufficiently long buffer
var decoded = decode32(secret);
if (!decoded) { return void cb('INVALID_SECRET'); }
// Reject attempts to setup TOTP if a record of their preferences already exists
MFA.read(Env, publicKey, function (err) {
// There **should be** an error here, because anything else
// means that a record already exists
// This may need to be adjusted as other methods of MFA are added
if (!err) { return void cb("EEXISTS"); }
// if no MFA settings exist then we expect ENOENT
// anything else indicates a problem and should result in rejection
if (err.code !== 'ENOENT') { return void cb(err); }
try {
// allow for 30s of clock drift in either direction
// returns an object ({ delta: 0 }) indicating the amount of clock drift
// if successful, otherwise `null`
var validated = OTP.totp.verify(code, decoded, {
window: 1,
});
if (!validated) { return void cb("INVALID_OTP"); }
cb();
} catch (err2) {
Env.Log.error('TOTP_SETUP_VERIFICATION_ERROR', {
error: err2,
});
return void cb("INTERNAL_ERROR");
}
});
};
// The 'complete' step for TOTP_SETUP will only be called if the client
// passed earlier validation and successfully signed the server's challenge.
// There's still a little bit more to do and it could still fail.
TOTP_SETUP.complete = function (Env, body, cb) {
// the OTP code should have already been validated
var { publicKey, secret, contact, session } = body;
// the device from which they configure MFA settings
// is assumed to be safe, so we'll respond with a JWT token
// the remainder of the setup is successfully completed.
// Otherwise they would have to reauthenticate.
// The session id is used as a reference to this particular session.
nThen(function (w) {
// confirm that the block exists
BlockStore.check(Env, publicKey, w(function (err) {
if (err) {
Env.Log.error("TOTP_SETUP_NO_BLOCK", {
publicKey,
});
w.abort();
return void cb("NO_BLOCK");
}
// otherwise the block exists, continue
}));
}).nThen(function (w) {
// store the data you'll need in the future
var data = {
method: 'TOTP', // specify this so it's easier to add other methods later?
secret: secret, // the 160 bit, base32-encoded secret that is used for OTP validation
creation: new Date(), // the moment at which the MFA was configured
};
if (isString(contact)) {
// 'contact' is an arbitary (and optional) string for manual recovery from 2FA auth fails
// it should already be validated
data.contact = contact;
}
// We attempt to store a record of the above preferences
// if it fails then we abort and inform the client of an error.
MFA.write(Env, publicKey, JSON.stringify(data), w(function (err) {
if (err) {
w.abort();
Env.Log.error("TOTP_SETUP_STORAGE_FAILURE", {
publicKey: publicKey,
error: err,
});
return void cb('STORAGE_FAILURE');
}
// otherwise continue
}));
}).nThen(function () {
// we have already stored the MFA data, which will cause access to the resource to be restricted to the provided TOTP secret.
// we attempt to create a session as a matter of convenience - so if it fails
// that just means they'll be forced to authenticate
makeSession(Env, publicKey, null, session, cb);
});
};
// This command is somewhat simpler than TOTP_SETUP
// Issue a client a JWT which will allow them to access a login block IFF:
// 1. That login block exists
// 2. That login block is protected by TOTP 2FA
// 3. They can produce a valid OTP for that block's TOTP secret
// 4. They can sign for the block's public key
const validate = Commands.TOTP_VALIDATE = function (Env, body, cb) {
var { publicKey, code } = body;
// they must provide a valid OTP code
if (!isValidOTP(code)) { return void cb('E_INVALID'); }
// they must provide a valid block public key
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
var secret;
nThen(function (w) {
// check that there is an MFA configuration for the given account
readMFA(Env, publicKey, w(function (err, content) {
if (err) {
w.abort();
return void cb(err);
}
secret = content.secret;
}));
}).nThen(function () {
checkCode(Env, secret, code, publicKey, cb);
});
};
validate.complete = function (Env, body, cb) {
/*
if they are here then they:
1. have a valid block configured with TOTP-based 2FA
2. were able to provide a valid TOTP for that block's secret
3. were able to sign their messages for the block's public key
So, we should:
1. instanciate a session for them by generating and storing a token for their public key
2. send them the token
*/
var { publicKey, session } = body;
makeSession(Env, publicKey, null, session, cb);
};
// Same as TOTP_VALIDATE but without making a session at the end
const check = Commands.TOTP_MFA_CHECK = function (Env, body, cb) {
var { publicKey, auth } = body;
const code = auth;
if (!isValidOTP(code)) { return void cb('E_INVALID'); }
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
var secret;
nThen(function (w) {
readMFA(Env, publicKey, w(function (err, content) {
if (err) {
w.abort();
return void cb(err);
}
secret = content.secret;
}));
}).nThen(function () {
checkCode(Env, secret, code, publicKey, cb);
});
};
check.complete = function (Env, body, cb) { cb(); };
// Revoke a client TOTP secret which will allow them to disable TOTP for a login block IFF:
// 1. That login block exists
// 2. That login block is protected by TOTP 2FA
// 3. They can produce a valid OTP for that block's TOTP secret
// 4. They can sign for the block's public key
const revoke = Commands.TOTP_REVOKE = function (Env, body, cb) {
var { publicKey, code, recoveryKey } = body;
// they must provide a valid OTP code
if (!isValidOTP(code) && !isValidRecoveryKey(recoveryKey)) { return void cb('E_INVALID'); }
// they must provide a valid block public key
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
var secret, recoveryStored;
nThen(function (w) {
// check that there is an MFA configuration for the given account
readMFA(Env, publicKey, w(function (err, content) {
if (err) {
w.abort();
return void cb(err);
}
secret = content.secret;
recoveryStored = content.contact;
}));
}).nThen(function (w) {
if (!recoveryKey) { return; }
w.abort();
if (!/^secret:/.test(recoveryStored)) {
return void cb("E_NO_RECOVERY_KEY");
}
recoveryStored = recoveryStored.slice(7);
if (recoveryKey !== recoveryStored) {
return void cb("E_WRONG_RECOVERY_KEY");
}
cb();
}).nThen(function () {
checkCode(Env, secret, code, publicKey, cb);
});
};
revoke.complete = function (Env, body, cb) {
/*
if they are here then they:
1. have a valid block configured with TOTP-based 2FA
2. were able to provide a valid TOTP for that block's secret
3. were able to sign their messages for the block's public key
So, we should:
1. Revoke the TOTP authentication for their block
2. Remove all existing sessions
*/
var { publicKey } = body;
MFA.revoke(Env, publicKey, cb);
};
// Write a login block using an existing OTP block IFF
// 1. You can sign for the block's public key
// 2. You have a proof for the old block
// 3. The old block is OTP protected
// 4. The OTP code is valid
// Note: this is used when users change their password
const writeBlock = Commands.TOTP_WRITE_BLOCK = function (Env, body, cb) {
const { publicKey, content } = body;
const code = content.auth;
const registrationProof = content.registrationProof;
// they must provide a valid block public key
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
if (publicKey !== content.publicKey) { return void cb("INVALID_KEY"); }
if (!isValidOTP(code)) { return void cb('E_INVALID'); }
if (!registrationProof) { return void cb('MISSING_ANCESTOR'); }
let secret;
let oldKey;
nThen(function (w) {
Block.validateAncestorProof(Env, registrationProof, w((err, provenKey) => {
if (err || !provenKey) {
w.abort();
return void cb('INVALID_ANCESTOR');
}
oldKey = provenKey;
}));
}).nThen(function (w) {
// check that there is an MFA configuration for the ancestor account
readMFA(Env, oldKey, w(function (err, content) {
if (err) {
w.abort();
return void cb(err);
}
secret = content.secret;
}));
}).nThen(function () {
// check that the OTP code is valid
checkCode(Env, secret, code, oldKey, cb);
});
};
writeBlock.complete = function (Env, body, cb) {
const { publicKey, content, session } = body;
let oldKey;
nThen(function (w) {
// Write new block
Block.writeLoginBlock(Env, content, w((err) => {
if (err) {
w.abort();
return void cb("BLOCK_WRITE_ERROR");
}
}));
}).nThen(function (w) {
// Copy MFA settings
const proof = Util.tryParse(content.registrationProof);
oldKey = proof && proof[0];
if (!oldKey) {
w.abort();
return void cb('INVALID_ANCESTOR');
}
MFA.copy(Env, oldKey, publicKey, w());
}).nThen(function () {
// Create a session for the current user
makeSession(Env, publicKey, oldKey, session, cb);
});
};
// Remove a login block IFF
// 1. You can sign for the block's public key
const removeBlock = Commands.TOTP_REMOVE_BLOCK = function (Env, body, cb) {
const { publicKey, auth } = body;
const code = auth;
// they must provide a valid block public key
if (!isValidBlockId(publicKey)) { return void cb("INVALID_KEY"); }
if (!isValidOTP(code)) { return void cb('E_INVALID'); }
let secret;
nThen(function (w) {
// check that there is an MFA configuration for this block
readMFA(Env, publicKey, w(function (err, content) {
if (err) {
w.abort();
return void cb(err);
}
secret = content.secret;
}));
}).nThen(function () {
// check that the OTP code is valid
checkCode(Env, secret, code, publicKey, cb);
});
};
removeBlock.complete = function (Env, body, cb) {
const { publicKey, edPublic, reason } = body;
nThen(function (w) {
// Remove the block
Block.removeLoginBlock(Env, publicKey, reason, edPublic, w((err) => {
if (err) {
w.abort();
return void cb(err);
}
}));
}).nThen(() => {
// Delete the MFA settings and sessions
MFA.revoke(Env, publicKey, cb);
});
};

102
lib/client/index.js
View File

@ -1,102 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Netflux = require("netflux-websocket");
var WebSocket = require("ws");
var nThen = require("nthen");
var Util = require("../../www/common/common-util");
var Nacl = require("tweetnacl/nacl-fast");
var Client = module.exports;
var createNetwork = Client.createNetwork = function (url, cb) {
var CB = Util.once(cb);
var info = {};
Netflux.connect(url, function (url) {
// this websocket seems to never close properly if the error is
// ECONNREFUSED
info.websocket = new WebSocket(url)
.on('error', function (err) {
CB(err);
})
.on('close', function (/* err */) {
delete info.websocket;
});
return info.websocket;
}).then(function (network) {
info.network = network;
CB(void 0, info);
}, function (err) {
CB(err);
});
};
var die = function (client) {
var disconnect = Util.find(client, ['config', 'network', 'disconnect']);
if (typeof(disconnect) === 'function') {
disconnect();
} else {
console.error("disconnect was not a function");
}
var close = Util.find(client, ['config', 'websocket', 'close']);
if (typeof(close) === 'function') {
client.config.websocket.close();
} else {
console.error("close was not a function");
}
};
Client.create = function (config, cb) {
if (typeof(config) === 'function') {
cb = config;
config = {};
}
var client = {
config: config,
};
var CB = Util.once(function (err, arg) {
if (err) { die(client); }
cb(err, arg);
});
client.shutdown = function () {
die(client);
};
nThen(function (w) {
if (config.network) { return; }
// connect to the network...
createNetwork('ws://localhost:3000/cryptpad_websocket', w(function (err, info) {
if (err) {
w.abort();
return void CB(err);
}
config.network = info.network;
config.websocket = info.websocket;
}));
}).nThen(function (w) {
// make sure the network has a historyKeeper id on it
// we're responsible for adding it
if (config.network.historyKeeper) { return; }
var channel = Util.uint8ArrayToHex(Nacl.randomBytes(16));
config.network.join(channel).then(w(function (wc) {
wc.members.some(function (member) {
if (member.length !== 16) { return; }
config.network.historyKeeper = member;
return true;
});
wc.leave();
}), function (err) {
w.abort();
CB(err);
});
}).nThen(function () {
CB(void 0, client);
});
};

1221
lib/commands/admin-rpc.js
View File

File diff suppressed because it is too large Load Diff

258
lib/commands/block.js
View File

@ -1,258 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Block = module.exports;
const Nacl = require("tweetnacl/nacl-fast");
const nThen = require("nthen");
const Util = require("../common-util");
const BlockStore = require("../storage/block");
const Invitation = require("./invitation");
const Users = require("./users");
var isString = s => typeof(s) === 'string';
Block.isValidBlockId = id => {
return id && isString(id) && id.length === 44;
};
/*
We assume that the server is secured against MitM attacks
via HTTPS, and that malicious actors do not have code execution
capabilities. If they do, we have much more serious problems.
The capability to replay a block write or remove results in either
a denial of service for the user whose block was removed, or in the
case of a write, a rollback to an earlier password.
Since block modification is destructive, this can result in loss
of access to the user's drive.
So long as the detached signature is never observed by a malicious
party, and the server discards it after proof of knowledge, replays
are not possible. However, this precludes verification of the signature
at a later time.
Despite this, an integrity check is still possible by the original
author of the block, since we assume that the block will have been
encrypted with xsalsa20-poly1305 which is authenticated.
*/
Block.validateLoginBlock = function (Env, publicKey, signature, block, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
// convert the public key to a Uint8Array and validate it
if (typeof(publicKey) !== 'string') { return void cb('E_INVALID_KEY'); }
var u8_public_key;
try {
u8_public_key = Util.decodeBase64(publicKey);
} catch (e) {
return void cb('E_INVALID_KEY');
}
var u8_signature;
try {
u8_signature = Util.decodeBase64(signature);
} catch (e) {
Env.Log.error('INVALID_BLOCK_SIGNATURE', e);
return void cb('E_INVALID_SIGNATURE');
}
// convert the block to a Uint8Array
var u8_block;
try {
u8_block = Util.decodeBase64(block);
} catch (e) {
return void cb('E_INVALID_BLOCK');
}
// take its hash
var hash = Nacl.hash(u8_block);
// validate the signature against the hash of the content
var verified = Nacl.sign.detached.verify(hash, u8_signature, u8_public_key);
// existing authentication ensures that users cannot replay old blocks
// call back with (err) if unsuccessful
if (!verified) { return void cb("E_COULD_NOT_VERIFY"); }
return void cb(null, block);
};
Block.validateAncestorProof = function (Env, proof, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
/* prove that you own an existing block by signing for its publicKey */
try {
var parsed = JSON.parse(proof);
var pub = parsed[0];
var u8_pub = Util.decodeBase64(pub);
var sig = parsed[1];
var u8_sig = Util.decodeBase64(sig);
var valid = false;
nThen(function (w) {
valid = Nacl.sign.detached.verify(u8_pub, u8_sig, u8_pub);
if (!valid) {
w.abort();
return void cb('E_INVALID_ANCESTOR_PROOF');
}
// else fall through to next step
}).nThen(function () {
BlockStore.check(Env, pub, function (err) {
if (err) { return void cb('E_MISSING_ANCESTOR'); }
cb(void 0, pub);
});
});
} catch (err) {
return void cb(err);
}
};
Block.writeLoginBlock = function (Env, msg, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
const { publicKey, signature, ciphertext, registrationProof, userData, inviteToken, isSSO } = msg;
var previousKey;
var validatedBlock, path;
var validatedInvite;
nThen(function (w) {
if (!inviteToken) { return; }
Invitation.check(Env, inviteToken, w((err, state) => {
if (err || !state) { return; } // Invalid token, don't abort, check registration proof
validatedInvite = true;
}));
}).nThen(function (w) {
if (!Env.restrictRegistration) { return; }
var ssoAllowed = isSSO && !Env.restrictSsoRegistration;
if (!(registrationProof || validatedInvite || ssoAllowed)) {
// we allow users with existing blocks to create new ones
// call back with error if registration is restricted and no proof of an existing block was provided
w.abort();
Env.Log.info("BLOCK_REJECTED_REGISTRATION", {
publicKey: publicKey,
});
return cb("E_RESTRICTED");
}
if (!registrationProof) { return; }
Block.validateAncestorProof(Env, registrationProof, w(function (err, provenKey) {
if (err || !provenKey) { // double check that a key was validated
w.abort();
Env.Log.warn('BLOCK_REJECTED_INVALID_ANCESTOR', {
error: err,
});
return void cb("E_RESTRICTED");
}
previousKey = provenKey;
}));
}).nThen(function (w) {
Block.validateLoginBlock(Env, publicKey, signature, ciphertext, w(function (e, _validatedBlock) {
if (e) {
w.abort();
return void cb(e);
}
if (typeof(_validatedBlock) !== 'string') {
w.abort();
return void cb('E_INVALID_BLOCK_RETURNED');
}
validatedBlock = _validatedBlock;
}));
}).nThen(function () {
var buffer;
try {
buffer = Buffer.from(Util.decodeBase64(validatedBlock));
} catch (err) {
return void cb('E_BLOCK_DESERIALIZATION');
}
BlockStore.write(Env, publicKey, buffer, function (err) {
Env.Log.info('BLOCK_WRITE_BY_OWNER', {
blockId: publicKey,
isChange: Boolean(registrationProof),
previousKey: previousKey,
path: path,
});
cb(err);
if (!err && registrationProof) {
Users.checkUpdate(Env, userData, publicKey, (err) => {
if (!err) { return; }
Env.Log.error('UPDATE_KNOWN_USER', {
userData,
publicKey
});
});
}
});
if (validatedInvite) {
Invitation.use(Env, inviteToken, publicKey, userData, (err) => {
if (!err) { return; }
Env.Log.error('USE_INVITATION_LINK', {
inviteToken,
userData,
publicKey
});
});
} else if (isSSO && !Env.dontStoreSSOUsers && !registrationProof) {
let edPublic = Array.isArray(userData) && userData[1];
let name = Array.isArray(userData) && userData[0];
if (!edPublic) { return; }
let data = {
block: publicKey,
name,
edPublic,
type: 'sso',
alias: name
};
Users.add(Env, edPublic, data, null, (err) => {
if (err) {
Env.Log.error('INVITATION_ADD_USER', {
error: err,
data: data
});
}
});
}
});
};
/*
When users write a block, they upload the block, and provide
a signature proving that they deserve to be able to write to
the location determined by the public key.
When removing a block, there is nothing to upload, but we need
to sign something. Since the signature is considered sensitive
information, we can just sign some constant and use that as proof.
*/
Block.removeLoginBlock = function (Env, publicKey, reason, edPublic, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
BlockStore.archive(Env, publicKey, reason, function (err) {
Env.Log.info('ARCHIVAL_BLOCK_BY_OWNER_RPC', {
publicKey: publicKey,
status: err? String(err): 'SUCCESS',
});
cb(err);
});
if (edPublic && reason !== 'PASSWORD_CHANGE') {
Users.delete(Env, edPublic, (err) => {
if (err) { Env.Log.error('KNOWN_USER_DELETION_ERROR', { error: err, key: edPublic }); }
});
}
// We should also try to remove the SSO data. Errors will be logged
// but they don't have to be shown to the user. The account data
// is already deleted anyway.
// If this is NOT a password change, also delete sso user.
let SSOUtils = Env.plugins && Env.plugins.SSO && Env.plugins.SSO.utils;
if (!SSOUtils) { return; }
if (reason !== 'PASSWORD_CHANGE') {
SSOUtils.deleteAccount(Env, publicKey, () => {});
} else {
SSOUtils.deleteBlock(Env, publicKey, () => {});
}
};

389
lib/commands/channel.js
View File

@ -1,389 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Channel = module.exports;
const Util = require("../common-util");
const nThen = require("nthen");
const Core = require("./core");
const Metadata = require("./metadata");
const Linked = require("./linked");
const HK = require("../hk-util");
const Nacl = require("tweetnacl/nacl-fast");
Channel.disconnectChannelMembers = function (Env, Server, channelId, code, reason, cb) {
var done = Util.once(Util.mkAsync(cb));
if (!Core.isValidId(channelId)) { return done('INVALID_ID'); }
const channel_cache = Env.channel_cache;
const metadata_cache = Env.metadata_cache;
const clear = function () {
delete channel_cache[channelId];
Server.clearChannel(channelId);
delete metadata_cache[channelId];
};
// an owner of a channel deleted it
nThen(function (w) {
// close the channel in the store
Env.msgStore.closeChannel(channelId, w());
}).nThen(function (w) {
// Server.channelBroadcast would be better
// but we can't trust it to track even one callback,
// let alone many in parallel.
// so we simulate it on this side to avoid race conditions
Server.getChannelUserList(channelId).forEach(function (userId) {
Server.send(userId, [
0,
Env.historyKeeper.id,
"MSG",
userId,
JSON.stringify({
error: code, //'EDELETED',
message: reason,
channel: channelId,
})
], w());
});
}).nThen(function () {
// clear the channel's data from memory
// once you've sent everyone a notice that the channel has been deleted
clear();
done();
}).orTimeout(function () {
Env.Log.warn('DISCONNECT_CHANNEL_MEMBERS_TIMEOUT', {
channelId,
code,
reason
});
clear();
done();
}, 30000);
};
Channel.clearOwnedChannel = function (Env, safeKey, channelId, cb, Server) {
if (typeof(channelId) !== 'string' || channelId.length !== 32) {
return cb('INVALID_ARGUMENTS');
}
var unsafeKey = Util.unescapeKeyCharacters(safeKey);
Metadata.getMetadata(Env, channelId, function (err, metadata) {
if (err) { return void cb(err); }
if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); }
// Confirm that the channel is owned by the user in question
if (!Core.isOwner(metadata, unsafeKey)) {
return void cb('INSUFFICIENT_PERMISSIONS');
}
return void Env.msgStore.clearChannel(channelId, function (e) {
if (e) { return void cb(e); }
cb();
const channel_cache = Env.channel_cache;
const clear = function () {
// delete the channel cache because it will have been invalidated
delete channel_cache[channelId];
};
nThen(function (w) {
Server.getChannelUserList(channelId).forEach(function (userId) {
Server.send(userId, [
0,
Env.historyKeeper.id,
'MSG',
userId,
JSON.stringify({
error: 'ECLEARED',
channel: channelId
})
], w());
});
}).nThen(function () {
clear();
}).orTimeout(function () {
Env.Log.warn("ON_CHANNEL_CLEARED_TIMEOUT", channelId);
clear();
}, 30000);
});
});
};
var archiveOwnedChannel = function (Env, safeKey, channelId, reason, __cb, Server) {
var _cb = Util.once(Util.mkAsync(__cb));
var unsafeKey = Util.unescapeKeyCharacters(safeKey);
reason = reason || 'ARCHIVE_OWNED';
nThen(function (w) {
// confirm that the channel exists before worrying about whether
// we have permission to delete it.
var cb = _cb;
Env.msgStore.getChannelSize(channelId, w(function (err, bytes) {
if (!bytes) {
w.abort();
return cb(err || "ENOENT");
}
}));
}).nThen(function (w) {
var cb = Util.both(w.abort, _cb);
Metadata.getMetadata(Env, channelId, w(function (err, metadata) {
if (err) { return void cb(err); }
if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); }
if (!Core.isOwner(metadata, unsafeKey)) {
return void cb('INSUFFICIENT_PERMISSIONS');
}
}));
}).nThen(function () {
var cb = _cb;
// temporarily archive the file
return void Env.msgStore.archiveChannel(channelId, reason, function (e) {
Env.Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', {
unsafeKey: unsafeKey,
channelId: channelId,
status: e? String(e): 'SUCCESS',
});
if (e) {
return void cb(e);
}
cb(void 0, 'OK');
Channel.disconnectChannelMembers(Env, Server, channelId, 'EDELETED', reason, err => {
if (err) { } // TODO
});
});
});
};
Channel.removeOwnedChannel = function (Env, safeKey, obj, __cb, Server) {
var _cb = Util.once(Util.mkAsync(__cb));
var channelId = obj.channel;
var reason = obj.reason;
if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) {
return _cb('INVALID_ARGUMENTS');
}
// archiving large channels or files can be expensive, so do it one at a time
// for any given user to ensure that nobody can use too much of the server's resources
Env.queueDeletes(safeKey, function (next) {
var cb = Util.both(_cb, next);
if (Env.blobStore.isFileId(channelId)) {
return void Env.removeOwnedBlob(channelId, safeKey, reason, cb);
}
Linked.listLinkedDocuments(Env, channelId, (err, channels) => {
archiveOwnedChannel(Env, safeKey, channelId, reason, (err, data) => {
if (!channels) { return void cb(err, data); }
if (err) { return void cb(err); }
Linked.archiveLinkedData(Env, channelId, reason, channels, () => {
cb(void 0, data);
});
}, Server);
});
});
};
Channel.trimHistory = function (Env, safeKey, data, cb) {
if (!(data && typeof(data.channel) === 'string' && typeof(data.hash) === 'string' && data.hash.length === 64)) {
return void cb('INVALID_ARGS');
}
var channelId = data.channel;
var unsafeKey = Util.unescapeKeyCharacters(safeKey);
var hash = data.hash;
nThen(function (w) {
Metadata.getMetadataRaw(Env, channelId, w(function (err, metadata) {
if (err) {
w.abort();
return void cb(err);
}
if (!Core.hasOwners(metadata)) {
w.abort();
return void cb('E_NO_OWNERS');
}
if (!Core.isOwner(metadata, unsafeKey)) {
w.abort();
return void cb("INSUFFICIENT_PERMISSIONS");
}
// else fall through to the next block
}));
}).nThen(function (w) {
// Archive old checkpoints
Linked.trimHistory(Env, { channel: channelId }, w());
}).nThen(function () {
// Trim chainpad doc:
Env.msgStore.trimChannel(channelId, hash, function (err) {
Env.Log.info('HK_TRIM_HISTORY', {
unsafeKey: unsafeKey,
channelId: channelId,
status: err? String(err): 'SUCCESS',
});
if (err) { return void cb(err); }
// clear historyKeeper's cache for this channel
Env.historyKeeper.channelClose(channelId);
cb(void 0, 'OK');
delete Env.channel_cache[channelId];
delete Env.metadata_cache[channelId];
});
});
};
// Delete a signed mailbox message. This is used when users want
// to delete their form reponses.
Channel.deleteMailboxMessage = function (Env, data, cb) {
const channelId = data.channel;
const hash = data.hash;
const proof = data.proof;
let nonce, proofBytes;
try {
nonce = Util.decodeBase64(proof.split('|')[0]);
proofBytes = Util.decodeBase64(proof.split('|')[1]);
} catch (e) {
return void cb('EINVAL');
}
Env.msgStore.deleteChannelLine(channelId, hash, function (msg) {
// Check if you're allowed to delete this hash
try {
const mySecret = Env.curvePrivate;
const msgBytes = Util.decodeBase64(msg).subarray(64); // Remove signature
const theirPublic = msgBytes.subarray(24,56); // 0-24 = nonce; 24-56=publickey (32 bytes)
const hashBytes = Nacl.box.open(proofBytes, nonce, theirPublic, mySecret);
return Util.encodeUTF8(hashBytes) === hash;
} catch (e) {
return false;
}
}, function (err) {
if (err) { return void cb(err); }
// clear historyKeeper's cache for this channel
Env.historyKeeper.channelClose(channelId);
cb();
delete Env.channel_cache[channelId];
delete Env.metadata_cache[channelId];
});
};
var ARRAY_LINE = /^\[/;
/* Files can contain metadata but not content
call back with true if the channel log has no content other than metadata
otherwise false
*/
Channel.isNewChannel = function (Env, channel, _cb) {
var cb = Util.once(_cb);
if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); }
if (channel.length !== HK.STANDARD_CHANNEL_LENGTH &&
channel.length !== HK.ADMIN_CHANNEL_LENGTH) { return void cb('INVALID_CHAN'); }
Env.msgStore.readMessagesBin(channel, 0, function (msgObj, readMore, abort) {
try {
var msg = msgObj.buff.toString('utf8');
if (typeof(msg) === 'string' && ARRAY_LINE.test(msg)) {
abort();
return void cb(void 0, {isNew: false});
}
} catch (e) {
Env.WARN('invalid message read from store', e);
}
readMore();
}, function (err, reason) {
// no more messages...
cb(void 0, {
isNew: true,
reason: reason
});
});
};
/* writePrivateMessage
allows users to anonymously send a message to the channel
prevents their netflux-id from being stored in history
and from being broadcast to anyone that might currently be in the channel
Otherwise behaves the same as sending to a channel
*/
Channel.writePrivateMessage = function (Env, args, _cb, Server, netfluxId) {
var cb = Util.once(Util.mkAsync(_cb));
var channelId = args[0];
var msg = args[1];
// don't bother handling empty messages
if (!msg) { return void cb("INVALID_MESSAGE"); }
// don't support anything except regular channels
if (!Core.isValidId(channelId) || (channelId.length !== HK.STANDARD_CHANNEL_LENGTH
&& channelId.length !== HK.ADMIN_CHANNEL_LENGTH)) {
return void cb("INVALID_CHAN");
}
// We expect a modern netflux-websocket-server instance
// if this API isn't here everything will fall apart anyway
if (!(Server && typeof(Server.send) === 'function')) {
return void cb("NOT_IMPLEMENTED");
}
nThen(function (w) {
Metadata.getMetadataRaw(Env, channelId, w(function (err, metadata) {
if (err) {
w.abort();
Env.Log.error('HK_WRITE_PRIVATE_MESSAGE', err);
return void cb('METADATA_ERR');
}
// treat the broadcast channel as write-protected
if (channelId.length === HK.ADMIN_CHANNEL_LENGTH) {
metadata.restricted = true;
}
if (!metadata || !metadata.restricted) {
return;
}
var session = HK.getNetfluxSession(Env, netfluxId);
var allowed = HK.listAllowedUsers(metadata);
if (HK.isUserSessionAllowed(allowed, session)) { return; }
w.abort();
cb('INSUFFICIENT_PERMISSIONS');
}));
}).nThen(function () {
// historyKeeper expects something with an 'id' attribute
// it will fail unless you provide it, but it doesn't need anything else
var channelStruct = {
id: channelId,
};
// construct a message to store and broadcast
var fullMessage = [
0, // idk
null, // normally the netflux id, null isn't rejected, and it distinguishes messages written in this way
"MSG", // indicate that this is a MSG
channelId, // channel id
msg // the actual message content. Generally a string
];
// historyKeeper already knows how to handle metadata and message validation, so we just pass it off here
// if the message isn't valid it won't be stored.
Env.historyKeeper.channelMessage(Server, channelStruct, fullMessage, function (err, time) {
if (err) {
// Message not stored...
return void cb(err);
}
// Broadcast the message
Server.getChannelUserList(channelId).forEach(function (userId) {
Server.send(userId, fullMessage);
});
cb(void 0, time);
});
});
};

155
lib/commands/core.js
View File

@ -1,155 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Core = module.exports;
const Util = require("../common-util");
const escapeKeyCharacters = Util.escapeKeyCharacters;
//const { fork } = require('child_process');
Core.DEFAULT_LIMIT = 50 * 1024 * 1024;
Core.SESSION_EXPIRATION_TIME = 60 * 1000;
Core.isValidId = function (chan) {
return chan && chan.length && /^[a-zA-Z0-9=+-]*$/.test(chan) &&
[32, 33, 48].indexOf(chan.length) > -1;
};
Core.isValidPublicKey = function (owner) {
return typeof(owner) === 'string' && owner.length === 44;
};
var makeToken = Core.makeToken = function () {
return Number(Math.floor(Math.random() * Number.MAX_SAFE_INTEGER))
.toString(16);
};
Core.makeCookie = function (token) {
var time = (+new Date());
time -= time % 5000;
return [
time,
process.pid,
token
];
};
var parseCookie = function (cookie) {
if (!(cookie && cookie.split)) { return null; }
var parts = cookie.split('|');
if (parts.length !== 3) { return null; }
var c = {};
c.time = new Date(parts[0]);
c.pid = Number(parts[1]);
c.seq = parts[2];
return c;
};
Core.getSession = function (Sessions, key) {
var safeKey = escapeKeyCharacters(key);
if (Sessions[safeKey]) {
Sessions[safeKey].atime = +new Date();
return Sessions[safeKey];
}
var user = Sessions[safeKey] = {};
user.atime = +new Date();
user.tokens = [
makeToken()
];
return user;
};
Core.expireSession = function (Sessions, safeKey) {
var session = Sessions[safeKey];
if (!session) { return; }
if (session.blobstage) {
session.blobstage.close();
}
delete Sessions[safeKey];
};
Core.expireSessionAsync = function (Env, safeKey, cb) {
setTimeout(function () {
Core.expireSession(Env.Sessions, safeKey);
cb(void 0, 'OK');
});
};
var isTooOld = function (time, now) {
return (now - time) > 300000;
};
Core.expireSessions = function (Sessions) {
var now = +new Date();
Object.keys(Sessions).forEach(function (safeKey) {
var session = Sessions[safeKey];
if (session && isTooOld(session.atime, now)) {
Core.expireSession(Sessions, safeKey);
}
});
};
var addTokenForKey = function (Sessions, publicKey, token) {
if (!Sessions[publicKey]) { throw new Error('undefined user'); }
var user = Core.getSession(Sessions, publicKey);
user.tokens.push(token);
user.atime = +new Date();
if (user.tokens.length > 2) { user.tokens.shift(); }
};
Core.isValidCookie = function (Sessions, publicKey, cookie) {
var parsed = parseCookie(cookie);
if (!parsed) { return false; }
var now = +new Date();
if (!parsed.time) { return false; }
if (isTooOld(parsed.time, now)) {
return false;
}
// different process. try harder
if (process.pid !== parsed.pid) {
return false;
}
var user = Core.getSession(Sessions, publicKey);
if (!user) { return false; }
var idx = user.tokens.indexOf(parsed.seq);
if (idx === -1) { return false; }
if (idx > 0) {
// make a new token
addTokenForKey(Sessions, publicKey, Core.makeToken());
}
return true;
};
// E_NO_OWNERS
Core.hasOwners = function (metadata) {
return Boolean(metadata && Array.isArray(metadata.owners));
};
Core.hasPendingOwners = function (metadata) {
return Boolean(metadata && Array.isArray(metadata.pending_owners));
};
// INSUFFICIENT_PERMISSIONS
Core.isOwner = function (metadata, unsafeKey) {
return metadata.owners.indexOf(unsafeKey) !== -1;
};
Core.isPendingOwner = function (metadata, unsafeKey) {
return metadata.pending_owners.indexOf(unsafeKey) !== -1;
};
Core.haveACookie = function (Env, safeKey, cb) {
cb();
};

89
lib/commands/invitation.js
View File

@ -1,89 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Invitation = module.exports;
const Invite = require('../storage/invite');
const Util = require("../common-util");
const Users = require("./users");
const Crypto = require('node:crypto');
const getUid = () => {
return Crypto.randomBytes(18).toString('hex');
};
Invitation.getAll = (Env, cb) => {
Invite.getAll(Env, (err, data) => {
if (err) { return void cb(err); }
cb(null, data);
});
};
Invitation.create = (Env, alias, email, _cb, unsafeKey) => {
const cb = Util.once(Util.mkAsync(_cb));
const id = getUid();
const invitation = {
alias,
email,
createdBy: unsafeKey,
time: +new Date()
};
Invite.write(Env, id, invitation, (err) => {
if (err) { return void cb(err); }
cb(null, id);
});
};
Invitation.delete = (Env, id, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
Invite.delete(Env, id, (err) => {
if (err && err !== 'ENOENT') { return void cb(err); }
cb(void 0, true);
});
};
Invitation.check = (Env, id, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
Invite.read(Env, id, (err) => {
if (err) { return void cb(err); }
cb(void 0, true);
});
};
Invitation.use = (Env, id, blockId, userData, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
Invite.read(Env, id, (err, _data) => {
if (err) { return void cb(err); }
let data = Util.clone(_data);
if (!Array.isArray(userData)) { userData = []; }
let name = userData[0];
let edPublic = userData[1];
data.block = blockId;
data.name = name;
data.edPublic = edPublic;
data.type = 'invite:' + id;
let adminKey = data.createdBy;
if (!Env.dontStoreInvitedUsers) {
Users.add(Env, edPublic, data, adminKey, (err) => {
if (err) {
Env.Log.error('INVITATION_ADD_USER', {
error: err,
data: data
});
}
});
}
Invite.delete(Env, id, (err) => {
if (err) {
Env.Log.error('INVITATION_DELETE_USE', {
error: err,
id: id
});
}
});
});
};

445
lib/commands/linked.js
View File

@ -1,445 +0,0 @@
// SPDX-FileCopyrightText: 2026 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Linked = module.exports;
const nThen = require("nthen");
//const Core = require("./core");
//const CPCrypto = require('../crypto');
const Util = require("../common-util");
const MetaRPC = require("./metadata");
const HK = require("../hk-util");
const getMetadata = (Env, channel, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
const metadata = Env.metadata_cache[channel];
if (metadata && typeof(metadata) === 'object') {
return void cb(undefined, metadata);
}
MetaRPC.getMetadataRaw(Env, channel, (err, metadata) => {
if (err) { return void cb(err); }
if (metadata?.channel !== channel && channel.length !== HK.BLOB_ID_LENGTH) {
return cb();
}
// cache it
if (channel.length !== HK.BLOB_ID_LENGTH) {
Env.metadata_cache[channel] = metadata;
}
cb(undefined, metadata);
});
};
const allowedTypes = ['checkpoints', 'media', 'channels'];
// XXX add logs
Linked.getLinkedDocuments = (Env, data, cb) => {
Env.store.getLinkedDocuments(data.channel, (err, json) => {
if (err && err !== 'ENOENT') { return void cb(err?.message); }
cb(void 0, json || {});
});
};
Linked.listLinkedDocuments = (Env, channel, _cb) => {
const cb = Util.mkAsync(_cb);
if (channel.length !== HK.STANDARD_CHANNEL_LENGTH) {
return void cb(void 0, []);
}
const list = new Set();
Linked.getLinkedDocuments(Env, { channel }, (err, json) => {
if (err) { return void cb(err); }
// For each type, add the channels and/or blobs
allowedTypes.forEach(type => {
const data = json[type];
if (!Array.isArray(data)) { return; }
// Media or channel:
if (type !== 'checkpoints') {
data.forEach(id => { list.add(id); });
return;
}
// Checkpoint:
data.forEach(obj => {
if (obj?.rtChannel) { list.add(obj.rtChannel); }
if (obj?.blob) { list.add(obj.blob); }
});
});
cb(void 0, Array.from(list));
});
};
Linked.listOldCheckpoints = (Env, channel, cb) => {
const list = new Set();
Linked.getLinkedDocuments(Env, { channel }, (err, json) => {
if (err) { return void cb(err); }
const cps = json.checkpoints || [];
cps.pop(); // preserve last cp
cps.forEach(obj => {
if (obj?.rtChannel) { list.add(obj.rtChannel); }
if (obj?.blob) { list.add(obj.blob); }
});
cb(void 0, Array.from(list));
});
};
const checkContent = (content, user) => {
const { type, data } = content;
if (type === 'checkpoints' && data) {
const { rtChannel, blob } = data;
if (rtChannel?.length !== 32 || (blob && blob?.length !== 48)) {
return false;
}
return {
rtChannel, blob, user,
time: Date.now()
};
}
if (type === 'media') {
return data?.length === 48 ? data : false;
}
if (type === 'channels') {
return data?.length === 32 ? data : false;
}
return false;
};
Linked.addLinkedDocument = (Env, data, cb, _S, userId) => {
// data.user
// data.channel
// data.content
// type, data (channelId or blobId or checkpoint {blob, rtChannel}})
// data.proof
// (sign "{ user, channel, content }" with pad signing key
const { user, channel, content, netfluxId, proof } = data;
if (userId !== netfluxId) { return void cb('EFORBIDDEN'); }
const msg = Util.clone(data);
delete msg.proof;
const signedMsg = JSON.stringify(msg);
const type = content?.type;
if (!allowedTypes.includes(type)) {
return void cb('INVALID_TYPE');
}
const value = checkContent(content, user);
if (!value) { return void cb('INVALID_CONTENT'); }
let validateKey;
nThen(waitFor => {
getMetadata(Env, channel, waitFor((err, metadata) => {
if (!metadata?.validateKey) {
waitFor.abort();
return void cb(err || 'METADATA_ERROR');
}
validateKey = metadata.validateKey;
}));
}).nThen(waitFor => {
Env.checkSignature(signedMsg, proof, validateKey, waitFor((err)=> {
if (err) {
waitFor.abort();
return void cb('INVALID_PROOF');
}
}));
}).nThen(() => {
Env.store.addLinkedDocument(channel, type, value, cb);
});
};
Linked.resetLinkedDocuments = (Env, data, cb, _S, userId) => {
// data.user
// data.channel
// data.content
// data.proof
// (sign "{ user, channel, content }" with pad signing key
const { user, channel, content, netfluxId, proof } = data;
if (userId !== netfluxId) { return void cb('EFORBIDDEN'); }
const msg = Util.clone(data);
delete msg.proof;
const signedMsg = JSON.stringify(msg);
let validateKey;
const newContent = {};
allowedTypes.forEach(type => { newContent[type] = []; });
nThen(waitFor => {
getMetadata(Env, channel, waitFor((err, metadata) => {
if (!metadata?.validateKey) {
waitFor.abort();
return void cb(err || 'METADATA_ERROR');
}
validateKey = metadata.validateKey;
}));
}).nThen(waitFor => {
Env.checkSignature(signedMsg, proof, validateKey, waitFor((err)=> {
if (err) {
waitFor.abort();
return void cb('INVALID_PROOF');
}
}));
}).nThen(waitFor => {
Linked.getLinkedDocuments(Env, { channel }, waitFor((err, json = {}) => {
// checkpoints
if (Array.isArray(content?.checkpoints)) {
const old = json?.checkpoints || [];
// add last 10 valid checkpoints
let i = 0;
content.checkpoints.reverse().some(data => {
// If cp already exists, recover user and time
// Otherwise, check integrity of new value and add them now
const oldValue = old.find(obj => {
return obj.blob === data.blob &&
obj.rtChannel === data.rtChannel;
});
const toAdd = oldValue || checkContent({
type: 'checkpoints',
data
}, user);
if (!toAdd) { return false; }
newContent.checkpoints.unshift(toAdd);
// Abort after 10 cps
if (++i >= 10) { return true; }
});
}
// channels and media
['channels', 'media'].forEach(type => {
if (!Array.isArray(content?.[type])) { return; }
content[type].forEach(data => {
const toAdd = checkContent({type, data}, user);
if (!toAdd) { return false; }
newContent[type].push(toAdd);
});
});
}));
}).nThen(() => {
Env.store.resetLinkedDocuments(channel, newContent, (err, data) => {
const { oldContent } = data;
Env.Log.info('RESET_LINKED_DOCUMENTS', {user, channel, oldContent, content});
cb();
});
});
};
Linked.removeLinkedDocument = (Env, allData, cb, _S, userId) => {
// data.user
// data.channel
// data.content
// type, channelId or blobId
// data.proof
// (sign "{ user, channel, content }" with pad signing key
const { channel, content, netfluxId, proof } = allData;
if (userId !== netfluxId) { return void cb('EFORBIDDEN'); }
const { type, data } = content;
const msg = Util.clone(data);
delete msg.proof;
const signedMsg = JSON.stringify(msg);
if (!allowedTypes.includes(type)) {
return void cb('INVALID_TYPE');
}
if (typeof(data) !== "string" || ![32,48].includes(data.length)) {
return void cb('INVALID_CONTENT');
}
let validateKey;
nThen(waitFor => {
getMetadata(Env, channel, waitFor((err, metadata) => {
if (!metadata?.validateKey) {
waitFor.abort();
return void cb(err || 'METADATA_ERROR');
}
validateKey = metadata.validateKey;
}));
}).nThen(waitFor => {
Env.checkSignature(signedMsg, proof, validateKey, waitFor((err)=> {
if (err) {
waitFor.abort();
return void cb('INVALID_PROOF');
}
}));
}).nThen(() => {
Env.store.removeLinkedDocument(channel, type, data, cb);
});
};
Linked.getFileSize = (Env, data, _cb) => {
const cb = Util.once(_cb);
const channel = data.channel;
let linked;
nThen(waitFor => {
Linked.listLinkedDocuments(Env, channel, waitFor((err, channels) => {
if (err) {
waitFor.abort();
return void cb(err);
}
linked = channels || [];
}));
}).nThen(() => {
linked.push(channel);
Env.getTotalSize(linked, cb);
});
};
Linked.getHistorySize = (Env, data, _cb) => {
const cb = Util.once(_cb);
const channel = data.channel;
let linked;
let channelTotalSize = 0;
let size = 0;
let start = 0;
let hash;
nThen(waitFor => {
Linked.getLinkedDocuments(Env, data, waitFor((err, json) => {
if (err) {
waitFor.abort();
return void cb(err);
}
linked = Util.clone(json);
}));
}).nThen(waitFor => {
// Get main channel size (chainpad)
Env.getFileSize(channel, waitFor((err, _size) => {
if (err) {
waitFor.abort();
return void cb(err);
}
channelTotalSize = _size;
}), true);
}).nThen(waitFor => {
// Get history offset to compute non-history size
HK.getHistoryOffset(Env, channel, null, waitFor((err, offset) => {
if (err) {
waitFor.abort();
return void cb(err);
}
start = offset;
const chanSize = channelTotalSize - offset;
size += chanSize;
}));
}).nThen(waitFor => {
// Get oldest hash of non-history data
Env.store.readMessagesBin(channel, start, (msgObj, readMore, abort) => {
const parsed = Util.tryParse(msgObj.buff.toString('utf8'));
if (!parsed) { return void readMore(); }
hash = HK.getHash(parsed[4]);
abort();
}, waitFor());
}).nThen(waitFor => {
// Get last checkpoint size (blob + rtChannel)
// Note: blob may be falsy if no checkpoint
const lastCp = (linked?.checkpoints || []).pop();
if (!lastCp) { return; }
const { blob, rtChannel } = lastCp;
if (blob) {
Env.getFileSize(blob, waitFor((err, _size) => {
if (err) {
waitFor.abort();
return void cb(err);
}
size += _size;
}), true);
}
Env.getFileSize(rtChannel, waitFor((err, _size) => {
if (err) {
waitFor.abort();
return void cb(err);
}
size += _size;
}), true);
}).nThen(() => {
cb(void 0, {
size, hash
});
});
};
Linked.trimHistory = (Env, data, cb) => {
const channel = data.channel;
let linked;
// if we reach this step, it means this user is an owner of "channel"
// so we can also delete any document linked to "channel" (from metadata)
nThen(waitFor => {
// List all but the current checkpoints
Linked.listOldCheckpoints(Env, channel, waitFor((err, channels) => {
if (err) {
waitFor.abort();
return void cb(err);
}
linked = channels || [];
}));
}).nThen(() => {
let n = nThen;
linked.forEach(chan => {
n = n(w => {
// If channel is "linked", we can archive all but last cp
getMetadata(Env, chan, w((err, md) => {
if (md?.linked !== channel) { return; }
// This is an old checkpoint linked to our document,
// we can archive it
const reason = "TRIM_HISTORY";
if (chan.length === HK.BLOB_ID_LENGTH) {
return Env.blobStore.archive.blob(chan, reason, w());
}
Env.store.archiveChannel(chan, reason, w());
}));
}).nThen;
});
n(() => {
cb();
});
});
};
// Archive all linked documents that inherit metadata from their
// parent. We consider ownership has already been checked when
// this function is called.
Linked.archiveLinkedData = (Env, channel, reason, channels, _cb) => {
const cb = Util.once(_cb);
let n = nThen;
channels.forEach(chan => {
n = n(w => {
// For each linked document, check if they inherit properties
getMetadata(Env, chan, w((err, md) => {
if (md?.linked !== channel) { return; }
// If they do, archive the document
if (chan.length === HK.BLOB_ID_LENGTH) {
return Env.blobStore.archive.blob(chan, reason, w());
}
Env.store.archiveChannel(chan, reason, w());
}));
}).nThen;
});
n(() => {
cb();
});
};

228
lib/commands/metadata.js
View File

@ -1,228 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Data = module.exports;
const Meta = require("../metadata");
const Core = require("./core");
const Util = require("../common-util");
const HK = require("../hk-util");
Data.getMetadataRaw = function (Env, channel, _cb, resolveLinked) {
const cb = Util.once(Util.mkAsync(_cb));
if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); }
if (channel.length !== HK.STANDARD_CHANNEL_LENGTH &&
channel.length !== HK.ADMIN_CHANNEL_LENGTH &&
channel.length !== HK.BLOB_ID_LENGTH) { return cb("INVALID_CHAN_LENGTH"); }
// return synthetic metadata for admin broadcast channels as a safety net
// in case anybody manages to write metadata
if (channel.length === HK.ADMIN_CHANNEL_LENGTH) {
return void cb(void 0, {
channel: channel,
creation: +new Date(),
owners: Env.admins,
});
}
var cached = Env.metadata_cache[channel];
if (HK.isMetadataMessage(cached)) {
Env.checkCache(channel);
return void cb(void 0, cached);
}
Env.batchMetadata(channel, cb, function (done) {
Env.computeMetadata(channel, function (err, meta) {
if (!err && HK.isMetadataMessage(meta)) {
Env.metadata_cache[channel] = meta;
// clear metadata after a delay if nobody has joined the channel within 30s
Env.checkCache(channel);
}
if (resolveLinked && meta?.linked?.length === HK.STANDARD_CHANNEL_LENGTH
&& meta?.linked !== channel) {
Data.getMetadataRaw(Env, meta.linked, (err, _meta) => {
meta.owners = _meta.owners;
meta.restricted = _meta.restricted;
meta.allowed = _meta.allowed;
done(err, meta);
});
return;
}
done(err, meta);
});
});
};
Data.getMetadata = function (Env, channel, cb, Server, netfluxId) {
Data.getMetadataRaw(Env, channel, function (err, metadata) {
if (err) { return void cb(err); }
if (!(metadata && metadata.restricted)) {
// if it's not restricted then just call back
return void cb(void 0, metadata);
}
const session = HK.getNetfluxSession(Env, netfluxId);
const allowed = HK.listAllowedUsers(metadata);
if (!HK.isUserSessionAllowed(allowed, session)) {
return void cb(void 0, {
restricted: metadata.restricted,
allowed: allowed,
rejected: true,
});
}
cb(void 0, metadata);
});
};
/* setMetadata
- write a new line to the metadata log if a valid command is provided
- data is an object: {
channel: channelId,
command: metadataCommand (string),
value: value
}
*/
Data.setMetadata = function (Env, safeKey, data, cb, Server) {
var unsafeKey = Util.unescapeKeyCharacters(safeKey);
var channel = data.channel;
var command = data.command;
if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); }
if (!command || typeof (command) !== 'string') { return void cb('INVALID_COMMAND'); }
if (Meta.commands.indexOf(command) === -1) { return void cb('UNSUPPORTED_COMMAND'); }
Env.queueMetadata(channel, function (next) {
Data.getMetadataRaw(Env, channel, function (err, metadata) {
if (err) {
cb(err);
return void next();
}
if (!Core.hasOwners(metadata)) {
cb('E_NO_OWNERS');
return void next();
}
// if you are a pending owner and not an owner
// you can either ADD_OWNERS, or RM_PENDING_OWNERS
// and you should only be able to add yourself as an owner
// everything else should be rejected
// else if you are not an owner
// you should be rejected
// else write the command
// Confirm that the channel is owned by the user in question
// or the user is accepting a pending ownership offer
if (Core.hasPendingOwners(metadata) &&
Core.isPendingOwner(metadata, unsafeKey) &&
!Core.isOwner(metadata, unsafeKey)) {
// If you are a pending owner, make sure you can only add yourelf as an owner
if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS')
|| !Array.isArray(data.value)
|| data.value.length !== 1
|| data.value[0] !== unsafeKey) {
cb('INSUFFICIENT_PERMISSIONS');
return void next();
}
// FIXME wacky fallthrough is hard to read
// we could pass this off to a writeMetadataCommand function
// and make the flow easier to follow
} else if (!Core.isOwner(metadata, unsafeKey)) {
cb('INSUFFICIENT_PERMISSIONS');
return void next();
}
// Add the new metadata line
var line = [command, data.value, +new Date()];
var changed = false;
try {
changed = Meta.handleCommand(metadata, line);
} catch (e) {
cb(e);
return void next();
}
// if your command is valid but it didn't result in any change to the metadata,
// call back now and don't write any "useless" line to the log
if (!changed) {
cb(void 0, metadata);
return void next();
}
let store = Env.msgStore;
if (channel.length === HK.BLOB_ID_LENGTH) {
store = Env.blobStore;
}
store.writeMetadata(channel, JSON.stringify(line), function (e) {
if (e) {
cb(e);
return void next();
}
// send the message back to the person who changed it
// since we know they're allowed to see it
cb(void 0, metadata);
next();
const metadata_cache = Env.metadata_cache;
// update the cached metadata
metadata_cache[channel] = metadata;
Env.checkCache(channel);
// it's easy to check if the channel is restricted
const isRestricted = metadata.restricted;
// and these values will be used in any case
const s_metadata = JSON.stringify(metadata);
const hk_id = Env.historyKeeper.id;
if (!isRestricted) {
// pre-allow-list behaviour
// if it's not restricted, broadcast the new metadata to everyone
return void Server.channelBroadcast(channel, s_metadata, hk_id);
}
// otherwise derive the list of users (unsafeKeys) that are allowed to stay
const allowed = HK.listAllowedUsers(metadata);
// anyone who is not allowed will get the same error message
const s_error = JSON.stringify({
error: 'ERESTRICTED',
channel: channel,
});
// iterate over the channel's userlist
const toRemove = [];
Server.getChannelUserList(channel).forEach(function (userId) {
const session = HK.getNetfluxSession(Env, userId);
// if the user is allowed to remain, send them the metadata
if (HK.isUserSessionAllowed(allowed, session)) {
return void Server.send(userId, [
0,
hk_id,
'MSG',
userId,
s_metadata
], function () {});
}
// otherwise they are not in the list.
// send them an error and kick them out!
toRemove.push(userId);
Server.send(userId, [
0,
hk_id,
'MSG',
userId,
s_error
], function () {});
});
Server.removeFromChannel(channel, toRemove);
});
}, true);
});
};

54
lib/commands/moderators.js
View File

@ -1,54 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
/*jshint esversion: 6 */
const Moderators = module.exports;
const Moderator = require('../storage/moderator');
const Util = require("../common-util");
Moderators.getAll = (Env, cb) => {
Moderator.getAll(Env, (err, data) => {
if (err) { return void cb(err); }
let res = {};
Object.keys(data).forEach(safeKey => {
const unsafeKey = Util.unescapeKeyCharacters(safeKey);
res[unsafeKey] = data[safeKey];
});
cb(null, res);
});
};
Moderators.getKeysSync = (Env) => {
return Moderator.getAllKeys(Env);
};
Moderators.add = (Env, edPublic, data, adminKey, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
data.createdBy = adminKey;
data.time = +new Date();
const safeKey = Util.escapeKeyCharacters(edPublic);
Moderator.write(Env, safeKey, data, (err) => {
if (err) { return void cb(err); }
if (!Env.moderators.includes(edPublic)) {
Env.moderators.push(edPublic);
}
Env.envUpdated.fire();
cb();
});
};
Moderators.delete = (Env, id, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
const safeKey = Util.escapeKeyCharacters(id);
Moderator.delete(Env, safeKey, (err) => {
if (err && err !== 'ENOENT') { return void cb(err); }
let idx = Env.moderators.indexOf(id);
if (idx !== -1) {
Env.moderators.splice(idx, 1);
Env.envUpdated.fire();
}
cb(void 0, true);
});
};

339
lib/commands/pin-rpc.js
View File

@ -1,339 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Core = require("./core");
const Pinning = module.exports;
const Util = require("../common-util");
const nThen = require("nthen");
const Linked = require('./linked');
const escapeKeyCharacters = Util.escapeKeyCharacters;
const unescapeKeyCharacters = Util.unescapeKeyCharacters;
var sumChannelSizes = function (sizes) { // FIXME this synchronous code could be done by a worker
return Object.keys(sizes).map(function (id) { return sizes[id]; })
.filter(function (x) {
// only allow positive numbers
return !(typeof(x) !== 'number' || x <= 0);
})
.reduce(function (a, b) { return a + b; }, 0);
};
// FIXME it's possible for this to respond before the server has had a chance
// to fetch the limits. Maybe we should respond with an error...
// or wait until we actually know the limits before responding
var getLimit = Pinning.getLimit = function (Env, safeKey, cb) {
var unsafeKey = unescapeKeyCharacters(safeKey);
var limit = Env.limits[unsafeKey];
var defaultLimit = typeof(Env.defaultStorageLimit) === 'number'?
Env.defaultStorageLimit: Core.DEFAULT_LIMIT;
var toSend = limit && typeof(limit.limit) === "number"?
[limit.limit, limit.plan, limit.note] : [defaultLimit, '', ''];
cb(void 0, toSend);
};
var getMultipleFileSize = function (Env, channels, cb) {
Env.getMultipleFileSize(channels, cb);
};
var loadUserPins = function (Env, safeKey, cb) {
var session = Core.getSession(Env.Sessions, safeKey);
if (session.channels) {
return cb(session.channels);
}
Env.batchUserPins(safeKey, cb, function (done) {
Env.getPinState(safeKey, function (err, value) {
if (!err) {
// only put this into the cache if it completes
session.channels = value;
}
done(value);
});
});
};
var truthyKeys = function (O) {
try {
return Object.keys(O).filter(function (k) {
return O[k];
});
} catch (err) {
return [];
}
};
var getChannelList = Pinning.getChannelList = function (Env, safeKey, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
loadUserPins(Env, safeKey, function (pins) {
cb(truthyKeys(pins));
});
};
var addLinkedDocuments = (Env, channels, cb) => {
var arr = Array.from(channels);
var n = nThen;
arr.forEach(chan => {
// For each channel, add their linked documents
n = n(w => {
Linked.listLinkedDocuments(Env, chan, w((err, linked) => {
if (err || !linked) { return; }
linked.forEach(id => {
channels.add(id);
});
}));
}).nThen;
});
n(() => {
cb();
});
};
Pinning.getTotalSize = function (Env, safeKey, cb) {
var unsafeKey = unescapeKeyCharacters(safeKey);
var limit = Env.limits[unsafeKey];
// Get a common key if multiple users share the same quota, otherwise take the public key
var batchKey = (limit && Array.isArray(limit.users)) ? limit.users.join('') : safeKey;
Env.batchTotalSize(batchKey, cb, function (done) {
let channels = new Set();
nThen(function (waitFor) {
// Get the channels list for our user account
getChannelList(Env, safeKey, waitFor(function (_channels) {
if (!_channels) {
waitFor.abort();
return done('INVALID_PIN_LIST');
}
for (let channel of _channels) {
channels.add(channel);
}
}));
// Get the channels list for users sharing our quota
if (limit && Array.isArray(limit.users) && limit.users.length > 1) {
limit.users.forEach(function (key) {
if (key === unsafeKey) { return; } // Don't count ourselves twice
getChannelList(Env, key, waitFor(function (_channels) {
if (!_channels) { return; } // Broken user, don't count their quota
for (let channel of _channels) {
channels.add(channel);
}
}));
});
}
}).nThen(function (waitFor) {
addLinkedDocuments(Env, channels, waitFor());
}).nThen(function () {
Env.getTotalSize(Array.from(channels), done);
});
});
};
/* Users should be able to clear their own pin log with an authenticated RPC
*/
Pinning.removePins = function (Env, safeKey, cb) {
// FIXME respect the queue
Env.pinStore.archiveChannel(safeKey, undefined, function (err) {
Core.expireSession(Env.Sessions, safeKey);
Env.Log.info('ARCHIVAL_PIN_BY_OWNER_RPC', {
safeKey: safeKey,
status: err? String(err): 'SUCCESS',
});
if (err) { return void cb(err); }
cb(void 0, 'OK');
});
};
Pinning.trimPins = function (Env, safeKey, cb) {
cb("NOT_IMPLEMENTED");
};
var getFreeSpace = Pinning.getFreeSpace = function (Env, safeKey, cb) {
getLimit(Env, safeKey, function (e, limit) {
if (e) { return void cb(e); }
Pinning.getTotalSize(Env, safeKey, function (e, size) {
if (typeof(size) === 'undefined') { return void cb(e); }
var rem = limit[0] - size;
if (typeof(rem) !== 'number') {
return void cb('invalid_response');
}
cb(void 0, rem);
});
});
};
Pinning.getHash = function (Env, safeKey, cb) {
getChannelList(Env, safeKey, function (channels) {
Env.hashChannelList(channels, cb);
});
};
Pinning.pinChannel = function (Env, safeKey, channels, cb) {
if (!channels && channels.filter) {
return void cb('INVALID_PIN_LIST');
}
// get channel list ensures your session has a cached channel list
getChannelList(Env, safeKey, function (pinned) {
var session = Core.getSession(Env.Sessions, safeKey);
// only pin channels which are not already pinned
var toStore = channels.filter(function (channel) {
return channel && pinned.indexOf(channel) === -1;
});
if (toStore.length === 0) {
return void cb();
}
let pin = function () {
Env.pinStore.message(safeKey, JSON.stringify(['PIN', toStore, +new Date()]),
function (e) {
if (e) { return void cb(e); }
if (!session || !session.channels) { return void cb(); }
toStore.forEach(function (channel) {
session.channels[channel] = true;
});
cb();
});
};
// Support tickets are always pinned, no need to check the limit
if (safeKey === escapeKeyCharacters(Env.supportPinKey)) {
return void pin();
}
getMultipleFileSize(Env, toStore, function (e, sizes) {
if (typeof(sizes) === 'undefined') { return void cb(e); }
var pinSize = sumChannelSizes(sizes); // FIXME don't do this in the main thread...
getFreeSpace(Env, safeKey, function (e, free) {
if (typeof(free) === 'undefined') {
Env.WARN('getFreeSpace', e);
return void cb(e);
}
if (pinSize > free) { return void cb('E_OVER_LIMIT'); }
pin();
});
});
});
};
Pinning.unpinChannel = function (Env, safeKey, channels, cb) {
if (!channels && channels.filter) {
// expected array
return void cb('INVALID_PIN_LIST');
}
getChannelList(Env, safeKey, function (pinned) {
var session = Core.getSession(Env.Sessions, safeKey);
// only unpin channels which are pinned
var toStore = channels.filter(function (channel) {
return channel && pinned.indexOf(channel) !== -1;
});
if (toStore.length === 0) {
return void cb();
}
Env.pinStore.message(safeKey, JSON.stringify(['UNPIN', toStore, +new Date()]),
function (e) {
if (e) { return void cb(e); }
toStore.forEach(function (channel) {
delete session.channels[channel];
});
cb();
});
});
};
Pinning.resetUserPins = function (Env, safeKey, channelList, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!Array.isArray(channelList)) { return void cb('INVALID_PIN_LIST'); }
var session = Core.getSession(Env.Sessions, safeKey);
if (!channelList.length) {
return void cb();
}
let reset = function () {
var pins = {};
Env.pinStore.message(safeKey, JSON.stringify(['RESET', channelList, +new Date()]),
function (e) {
if (e) { return void cb(e); }
channelList.forEach(function (channel) {
pins[channel] = true;
});
// update in-memory cache IFF the reset was allowed.
if (session) { session.channels = pins; }
cb();
});
};
// Support tickets are always pinned, no need to check the limit
if (safeKey === escapeKeyCharacters(Env.supportPinKey)) {
return void reset();
}
getMultipleFileSize(Env, channelList, function (e, sizes) {
if (typeof(sizes) === 'undefined') { return void cb(e); }
var pinSize = sumChannelSizes(sizes);
getLimit(Env, safeKey, function (e, limit) {
if (e) {
Env.WARN('[RESET_ERR]', e);
return void cb(e);
}
/* we want to let people pin, even if they are over their limit,
but they should only be able to do this once.
This prevents data loss in the case that someone registers, but
does not have enough free space to pin their migrated data.
They will not be able to pin additional pads until they upgrade
or delete enough files to go back under their limit. */
if (pinSize > limit[0] && session.hasPinned) { return void(cb('E_OVER_LIMIT')); }
reset();
});
});
};
Pinning.getFileSize = function (Env, channel, cb) {
Env.getFileSize(channel, cb);
};
/* accepts a list, and returns a sublist of channel or file ids which seem
to have been deleted from the server (file size 0)
we might consider that we should only say a file is gone if fs.stat returns
ENOENT, but for now it's simplest to just rely on getFileSize...
*/
Pinning.getDeletedPads = function (Env, channels, cb) {
Env.getDeletedPads(channels, cb);
};
// FIXME this will be removed from the client
Pinning.isChannelPinned = function (Env, channel, cb) {
return void cb(void 0, true);
};
Pinning.isPremium = function (Env, userKey, cb) {
const limit = Env.limits[userKey];
return void cb(void 0, !!limit?.plan);
//return void cb(void 0, (limit?.plan && limit.plan !== "custom"));
};

322
lib/commands/quota.js
View File

@ -1,322 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Quota = module.exports;
//const Util = require("../common-util");
const Keys = require("../keys");
const Https = require("https");
const Http = require("http");
const Util = require("../common-util");
const Stats = require("../stats");
const Admin = require("./admin-rpc.js");
const nThen = require('nthen');
var validLimitFields = ['limit', 'plan', 'note', 'users', 'origin'];
Quota.isValidLimit = function (o) {
var valid = o && typeof(o) === 'object' &&
typeof(o.limit) === 'number' &&
typeof(o.plan) === 'string' &&
typeof(o.note) === 'string' &&
// optionally contains a 'users' array
(Array.isArray(o.users) || typeof(o.users) === 'undefined') &&
// check that the object contains only the expected fields
!Object.keys(o).some(function (k) {
return validLimitFields.indexOf(k) === -1;
});
return valid;
};
Quota.applyCustomLimits = function (Env) {
// DecreedLimits > customLimits > serverLimits;
// FIXME perform an integrity check on shared limits
// especially relevant because we use Env.limits
// when considering whether to archive inactive accounts
// read custom limits from the Environment (taken from config)
var customLimits = (function (custom) {
var limits = {};
Object.keys(custom).forEach(function (k) {
var unsafeKey = Keys.canonicalize(k);
if (!unsafeKey) { return; }
limits[unsafeKey] = custom[k];
});
return limits;
}(Env.customLimits || {}));
Env.limits = Env.limits || {};
Object.keys(customLimits).forEach(function (k) {
if (!Quota.isValidLimit(customLimits[k])) { return; }
Env.limits[k] = customLimits[k];
});
// console.log(Env.limits);
};
var isRemoteVersionNewer = function (local, remote) {
try {
local = local.split('.').map(Number);
remote = remote.split('.').map(Number);
for (var i = 0; i < 3; i++) {
if (remote[i] < local[i]) { return false; }
if (remote[i] > local[i]) { return true; }
}
} catch (err) {
// if anything goes wrong just fall through and return false
// false negatives are better than false positives
}
return false;
};
/*
var Assert = require("assert");
[
// remote versions
['4.5.0', '4.5.0', false], // equal semver should not prompt
['4.5.0', '4.5.1', true], // patch versions should prompt
['4.5.0', '4.6.0', true], // minor versions should prompt
['4.5.0', '5.0.0', true], // major versions should prompt
// local
['5.3.1', '4.9.0', false], // newer major should not prompt
['4.7.0', '4.6.0', false], // newer minor should not prompt
['4.7.0', '4.6.1', false], // newer patch should not prompt if other values are greater
].forEach(function (x) {
var result = isRemoteVersionNewer(x[0], x[1]);
Assert.equal(result, x[2]);
});
*/
// check if the remote endpoint reported an available server version
// which is newer than your current version (Env.version)
// if so, set Env.updateAvailable to the URL of its release notes
var checkUpdateAvailability = function (Env, json) {
if (!(json && typeof(json.updateAvailable) === 'string' && typeof(json.version) === 'string')) { return; }
// expects {updateAvailable: 'https://github.com/cryptpad/cryptpad/releases/4.7.0', version: '4.7.0'}
// the version string is provided explicitly even though it could be parsed from GitHub's URL
// this will allow old instances to understand responses of arbitrary URLs
// as long as we keep using semver for 'version'
if (!isRemoteVersionNewer(Env.version, json.version)) {
Env.updateAvailable = undefined;
return;
}
Env.updateAvailable = json.updateAvailable;
Env.Log.info('AN_UPDATE_IS_AVAILABLE', {
version: json.version,
updateAvailable: json.updateAvaiable,
});
};
var queryAccountServer = function (Env, cb) {
var done = Util.once(Util.mkAsync(cb));
var rawBody = Stats.instanceData(Env);
let send = () => {
Env.Log.info("SERVER_TELEMETRY", rawBody);
var body = JSON.stringify(rawBody);
var options = {
host: 'accounts.cryptpad.fr',
path: '/api/getauthorized',
method: 'POST',
headers: {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(body)
}
};
var req = Https.request(options, function (response) {
if (!('' + response.statusCode).match(/^2\d\d$/)) {
return void cb('SERVER ERROR ' + response.statusCode);
}
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
try {
var json = JSON.parse(str);
checkUpdateAvailability(Env, json);
done(void 0, json);
} catch (e) {
done(e);
}
});
});
req.on('error', function () {
done();
});
req.end(body);
};
if (Env.provideAggregateStatistics) {
let stats = {};
nThen(waitFor => {
Admin.getRegisteredUsers(Env, null, waitFor((err, data) => {
if (err) { return; }
stats.registered = data.users;
if (Env.lastPingRegisteredUsers) {
stats.usersDiff = stats.registered - Env.lastPingRegisteredUsers;
}
Env.lastPingRegisteredUsers = stats.registered;
}));
}).nThen(() => {
if (Env.maxConcurrentWs) {
stats.maxConcurrentWs = Env.maxConcurrentWs;
Env.maxConcurrentWs = 0;
}
if (Env.maxConcurrentUniqueWs) {
stats.maxConcurrentUniqueIPs = Env.maxConcurrentUniqueWs;
Env.maxConcurrentUniqueWs = 0;
}
if (Env.maxConcurrentRegUsers) {
stats.maxConcurrentRegUsers = Env.maxConcurrentRegUsers;
Env.maxConcurrentRegUsers = 0;
}
if (Env.maxActiveChannels) {
stats.maxConcurrentChannels = Env.maxActiveChannels;
Env.maxActiveChannels = 0;
}
rawBody.statistics = stats;
send();
});
return;
}
send();
};
Quota.shouldContactServer = function (Env) {
return !(Env.blockDailyCheck === true ||
(
typeof(Env.blockDailyCheck) === 'undefined' &&
Env.adminEmail === false
)
);
};
Quota.pingAccountsDaily = function (Env, _cb) {
var cb = Util.mkAsync(_cb);
if (!Quota.shouldContactServer(Env)) { return void cb(); }
queryAccountServer(Env, function (err) {
cb(err);
});
};
var queryQuotaServer = function (Env, cb) {
var done = Util.once(Util.mkAsync(cb));
var rawBody = Stats.instanceData(Env);
Env.Log.info("QUOTA_UPDATE", rawBody);
var body = JSON.stringify(rawBody);
var options = {
host: undefined,
path: '/api/getquota',
method: 'POST',
headers: {
"Content-Type": "application/json",
"Content-Length": Buffer.byteLength(body)
}
};
const accountsOrigin = Env.accounts_api;
var H = Https;
if (typeof(accountsOrigin) === 'string') {
try {
let url = new URL(accountsOrigin);
if (!['https:', 'http:'].includes(url.protocol)) { throw new Error("INVALID_PROTOCOL"); }
if (url.protocol === 'http:') { H = Http; }
let port = Number(url.port);
if (port && typeof(port) === 'number') { options.port = port; }
options.host = url.hostname;
Env.Log.info("USING_CUSTOM_ACCOUNTS_API", {
value: accountsOrigin,
});
} catch (err) {
Env.Log.error("INVALID_CUSTOM_QUOTA_API", {
error: err.message,
value: accountsOrigin,
});
}
}
var req = H.request(options, function (response) {
if (!('' + response.statusCode).match(/^2\d\d$/)) {
return void cb('SERVER ERROR ' + response.statusCode);
}
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
try {
var json = JSON.parse(str);
// don't overwrite the limits with junk data
if (json && json.message === 'EINVAL') { return void done(); }
done(void 0, json);
} catch (e) {
done(e);
}
});
});
req.on('error', function (e) {
Quota.applyCustomLimits(Env);
done(e);
});
req.end(body);
};
Quota.queryQuotaServer = function (Env, cb) {
Env.batchAccountQuery('', cb, function (done) {
queryQuotaServer(Env, done);
});
};
Quota.updateCachedLimits = function (Env, _cb) {
var cb = Util.mkAsync(_cb);
if (!Env.accounts_api) {
Quota.applyCustomLimits(Env);
return void cb();
}
Quota.queryQuotaServer(Env, function (err, json) {
if (err) { return void cb(err); }
if (!json) { return void cb(); }
for (var k in json) {
if (k.length === 44 && json[k]) {
json[k].origin = 'remote';
}
}
Env.limits = json;
Quota.applyCustomLimits(Env);
cb();
});
};
// The limits object contains storage limits for all the publicKey that have paid
// To each key is associated an object containing the 'limit' value and a 'note' explaining that limit
Quota.getUpdatedLimit = function (Env, safeKey, cb) {
Quota.updateCachedLimits(Env, function (err) {
if (err) { return void cb(err); }
var limit = Env.limits[safeKey];
if (limit && typeof(limit.limit) === 'number') {
return void cb(void 0, [limit.limit, limit.plan, limit.note]);
}
return void cb(void 0, [Env.defaultStorageLimit, '', '']);
});
};

98
lib/commands/upload.js
View File

@ -1,98 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Upload = module.exports;
const Util = require("../common-util");
const Pinning = require("./pin-rpc");
const nThen = require("nthen");
const Core = require("./core");
Upload.status = function (Env, safeKey, data, _cb) { // FIXME FILES
const filesize = data.size;
var cb = Util.once(Util.mkAsync(_cb));
// validate that the provided size is actually a positive number
if (typeof(filesize) !== 'number' &&
filesize >= 0) { return void cb('E_INVALID_SIZE'); }
nThen(function (w) {
// if the proposed upload size is within the regular limit
// jump ahead to the next block
if (filesize <= Env.maxUploadSize) { return; }
// if larger uploads aren't explicitly enabled then reject them
if (typeof(Env.premiumUploadSize) !== 'number') {
w.abort();
return void cb('TOO_LARGE');
}
// otherwise go and retrieve info about the user's quota
Pinning.getLimit(Env, safeKey, w(function (err, limit) {
if (err) {
w.abort();
return void cb("E_BAD_LIMIT");
}
var plan = limit[1];
// see if they have a special plan, reject them if not
if (plan === '') {
w.abort();
return void cb('TOO_LARGE');
}
// and that they're not over the greater limit
if (filesize >= Env.premiumUploadSize) {
w.abort();
return void cb("TOO_LARGE");
}
// fallthrough will proceed to the next block
}));
}).nThen(function (w) {
var abortAndCB = Util.both(w.abort, cb);
Env.blobStore.status(safeKey, w(function (err, inProgress) {
// if there's an error something is weird
if (err) { return void abortAndCB(err); }
// we cannot upload two things at once
if (inProgress) { return void abortAndCB(void 0, true); }
}));
}).nThen(function () {
// if yuo're here then there are no pending uploads
// check if you have space in your quota to upload something of this size
Pinning.getFreeSpace(Env, safeKey, function (e, free) {
if (e) { return void cb(e); }
if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); }
var user = Core.getSession(Env.Sessions, safeKey);
user.pendingUploadSize = filesize;
user.currentUploadSize = 0;
user.linked = data.linked;
cb(void 0, false);
});
});
};
Upload.upload = function (Env, safeKey, data, cb) {
Env.blobStore.uploadWs(safeKey, data?.chunk, cb);
};
Upload.cancel = function (Env, safeKey, arg, cb) {
Env.blobStore.cancel(safeKey, arg?.size, cb);
};
var completeUpload = function (owned) {
return function (Env, safeKey, arg, cb) {
Env.blobStore.closeBlobstage(safeKey);
var user = Core.getSession(Env.Sessions, safeKey);
var size = user.pendingUploadSize;
var linked = user.linked;
Env.completeUpload(safeKey, arg, Boolean(owned), size, linked, cb);
};
};
Upload.complete = completeUpload(false);
Upload.complete_owned = completeUpload(true);

79
lib/commands/users.js
View File

@ -1,79 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Users = module.exports;
const User = require('../storage/user');
const Util = require("../common-util");
Users.getAll = (Env, cb) => {
User.getAll(Env, (err, data) => {
if (err) { return void cb(err); }
cb(null, data);
});
};
Users.add = (Env, edPublic, data, adminKey, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
data.createdBy = adminKey;
data.time = +new Date();
const safeKey = Util.escapeKeyCharacters(edPublic);
User.write(Env, safeKey, data, (err) => {
if (err) { return void cb(err); }
cb();
});
};
Users.delete = (Env, id, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
User.delete(Env, id, (err) => {
if (err && err !== 'ENOENT') { return void cb(err); }
cb(void 0, true);
});
};
Users.read = (Env, edPublic, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
User.read(Env, edPublic, (err, data) => {
if (err) { return void cb(err); }
cb(void 0, data);
});
};
Users.update = (Env, edPublic, changes, _cb) => {
const cb = Util.once(Util.mkAsync(_cb));
Users.read(Env, edPublic, (err, data) => {
if (err === 'ENOENT') { return void cb(); }
if (err) { return void cb(err); }
if (typeof(changes) !== "object") { return void cb('EINVAL'); }
// User exists, update their data
var aborted = Object.keys(changes || {}).some((key) => {
if (changes[key] === false) {
delete data[key];
return;
}
if (String(changes[key]).length > 300) {
cb('E_TOO_LONG');
return true;
}
data[key] = changes[key];
});
if (aborted) { return; }
User.update(Env, edPublic, data, cb);
});
};
// On password change, update the block
Users.checkUpdate = (Env, userData, newBlock, cb) => {
if (!Array.isArray(userData)) { userData = []; }
let edPublic = userData[1];
if (!edPublic) { return void cb('INVALID_PUBLIC_KEY'); }
Users.read(Env, edPublic, (err, data) => {
if (err === 'ENOENT') { return void cb(); }
if (err) { return void cb(err); }
// User exists, update their block
data.block = newBlock;
User.update(Env, edPublic, data, cb);
});
};

6
lib/common-hash.js
View File

@ -1,6 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
module.exports = require("../src/common/common-hash");

5
lib/common-util.js
View File

@ -1,5 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
module.exports = require("../src/common/common-util");

28
lib/crypto.js
View File

@ -1,28 +0,0 @@
// SPDX-FileCopyrightText: 2024 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Nacl = require('tweetnacl/nacl-fast');
const CPCrypto = module.exports;
const plugins = require('./plugin-manager');
CPCrypto.init = (cb) => {
const crypto = {};
crypto.open = (signedMsg, validateKey) => {
return Nacl.sign.open(signedMsg, validateKey);
};
crypto.detachedVerify = (signedBuffer, signatureBuffer, validateKey) => {
return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, validateKey);
};
if (plugins.SODIUM && plugins.SODIUM.crypto) {
let c = plugins.SODIUM.crypto;
if (c.open) { crypto.open = c.open; }
if (c.detachedVerify) { crypto.detachedVerify = c.detachedVerify; }
}
// Make async because we might need it later with libsodium's promise
// libsodium.ready.then(() => {});
setTimeout(() => {
cb(void 0, crypto);
});
};

140
lib/decrees-core.js
View File

@ -1,140 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Decrees = module.exports;
var Util = require("./common-util");
var Fs = require("fs");
var Path = require("path");
var readFileBin = require("./stream-file").readFileBin;
var Schedule = require("./schedule");
var Fse = require("fs-extra");
var nThen = require("nthen");
const Utils = Decrees.Utils = {};
var isString = (str) => {
return typeof(str) === "string";
};
var isInteger = function (n) {
return !(typeof(n) !== 'number' || isNaN(n) || (n % 1) !== 0);
};
Utils.args_isBoolean = function (args) {
return !(!Array.isArray(args) || typeof(args[0]) !== 'boolean');
};
Utils.args_isString = function (args) {
return !(!Array.isArray(args) || !isString(args[0]));
};
Utils.args_isInteger = function (args) {
return !(!Array.isArray(args) || !isInteger(args[0]));
};
Utils.args_isPositiveInteger = function (args) {
return Array.isArray(args) && isInteger(args[0]) && args[0] > 0;
};
Decrees.create = (name, commands) => {
// [<command>, <args>, <author>, <time>]
const handleCommand = function (Env, line) {
var command = line[0];
var args = line[1];
if (typeof(commands[command]) !== 'function') {
throw new Error("DECREE_UNSUPPORTED_COMMAND");
}
var outcome = commands[command](Env, args);
if (outcome) {
// trigger Env change event...
Env.envUpdated.fire();
}
return outcome;
};
const createLineHandler = function (Env) {
var Log = Env.Log;
var index = -1;
return function (err, line) {
index++;
if (err) {
// Log the error and bail out
return void Log.error("DECREE_LINE_ERR", {
error: err.message,
index: index,
line: line,
});
}
if (Array.isArray(line)) {
try {
return void handleCommand(Env, line);
} catch (err2) {
return void Log.error("DECREE_COMMAND_ERR", {
error: err2.message,
index: index,
line: line,
});
}
}
Log.error("DECREE_HANDLER_WEIRD_LINE", {
line: line,
index: index,
});
};
};
const load = function (Env, _cb) {
Env.scheduleDecree = Env.scheduleDecree || Schedule();
var cb = Util.once(Util.mkAsync(function (err) {
if (err && err.code !== 'ENOENT') {
return void _cb(err);
}
_cb();
}));
Env.scheduleDecree.blocking('', function (unblock) {
var done = Util.once(Util.both(cb, unblock));
nThen(function (w) {
// ensure that the path to the decree log exists
Fse.mkdirp(Env.paths.decree, w(function (err) {
if (!err) { return; }
w.abort();
done(err);
}));
}).nThen(function () {
var decreeName = Path.join(Env.paths.decree, name);
var stream = Fs.createReadStream(decreeName, {start: 0});
var handler = createLineHandler(Env);
readFileBin(stream, function (msgObj, next) {
var text = msgObj.buff.toString('utf8');
try {
handler(void 0, JSON.parse(text));
} catch (err) {
handler(err, text);
}
next();
}, function (err) {
done(err);
});
});
});
};
const write = function (Env, decree, _cb) {
var path = Path.join(Env.paths.decree, name);
Env.scheduleDecree.ordered('', function (next) {
var cb = Util.both(Util.mkAsync(_cb), next);
Fs.appendFile(path, JSON.stringify(decree) + '\n', cb);
});
};
return {
handleCommand,
load,
write
};
};

433
lib/decrees.js
View File

@ -1,433 +0,0 @@
// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Core = require("./commands/core");
const DecreesCore = require("./decrees-core");
const config = require('./load-config');
const Quota = require("./commands/quota");
const Keys = require("./keys");
const DECREE_NAME = 'decree.ndjson';
const {
args_isBoolean,
args_isString,
args_isInteger,
args_isPositiveInteger
} = DecreesCore.Utils;
// Toggles a simple boolean
const makeBooleanSetter = function (attr) {
return function (Env, args) {
if (!args_isBoolean(args)) {
throw new Error('INVALID_ARGS');
}
var bool = args[0];
if (bool === Env[attr]) { return false; }
Env[attr] = bool;
return true;
};
};
const default_validator = function () { return true; };
const makeGenericSetter = function (attr, validator) {
validator = validator || default_validator;
return function (Env, args) {
if (!validator(args)) {
throw new Error("INVALID_ARGS");
}
var value = args[0];
if (value === Env[attr]) { return false; }
Env[attr] = value;
return true;
};
};
const makeIntegerSetter = function (attr) {
return makeGenericSetter(attr, args_isInteger);
};
const makeTranslation = function (attr) {
return function (Env, args) {
if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); }
var value = args[0];
var state = Env[attr];
if (typeof(value) === 'string') {
if (state.default === value) { return false; }
state.default = value;
return true;
}
if (value && typeof(value) === 'object') {
var changed = false;
Object.keys(value).forEach(function (lang) {
if (state[lang] === value[lang]) { return; }
state[lang] = value[lang];
changed = true;
});
return changed;
}
return false;
};
};
/* commands have a simple API:
* they receive the global Env and the arguments to be applied
* if the arguments are invalid the operation will not be applied
* the command throws
* if the arguments are valid but do not result in a change, the operation is redundant.
* return false
* if the arguments are valid and will result in a change, the operation should be applied
* apply it
* return true to indicate that it was applied
*/
const commands = {};
/* Admin decrees which modify global server state
IMPLEMENTED:
RESTRICT_REGISTRATION(<boolean>)
RESTRICT_SSO_REGISTRATION(<boolean>)
UPDATE_DEFAULT_STORAGE(<number>)
// QUOTA MANAGEMENT
SET_QUOTA(<string:signkey>, limit)
RM_QUOTA(<string:signkey>)
// INACTIVITY
SET_INACTIVE_TIME
SET_ACCOUNT_RETENTION_TIME
SET_ARCHIVE_RETENTION_TIME
// UPLOADS
SET_MAX_UPLOAD_SIZE
SET_PREMIUM_UPLOAD_SIZE
// BACKGROUND PROCESSES
DISABLE_INTEGRATED_TASKS
DISABLE_INTEGRATED_EVICTION
ENABLE_PROFILING
SET_PROFILING_WINDOW
// BROADCAST
SET_LAST_BROADCAST_HASH
SET_SURVEY_URL
SET_MAINTENANCE
// EASIER CONFIG
SET_ADMIN_EMAIL
SET_SUPPORT_MAILBOX
SET_SUPPORT_KEYS
// COMMUNITY PARTICIPATION AND GOVERNANCE
CONSENT_TO_CONTACT
LIST_MY_INSTANCE
PROVIDE_AGGREGATE_STATISTICS
REMOVE_DONATE_BUTTON
BLOCK_DAILY_CHECK
// Customized instance info
SET_INSTANCE_JURISDICTION
SET_INSTANCE_DESCRIPTION
SET_INSTANCE_NAME
SET_INSTANCE_NOTICE
// bearer secret
SET_BEARER_SECRET
NOT IMPLEMENTED:
// RESTRICTED REGISTRATION
ADD_INVITE
REVOKE_INVITE
REDEEM_INVITE
DISABLE_EMBEDDING
// 2.0
Env.DEV_MODE || Env.FRESH_MODE,
ADD_ADMIN_KEY
RM_ADMIN_KEY
*/
// Maintenance: Empty string or an object with a start and end time
const isNumber = function (value) {
return typeof(value) === "number" && !isNaN(value);
};
const args_isMaintenance = function (args) {
return Array.isArray(args) && args[0] &&
(args[0] === "" || (isNumber(args[0].end) && isNumber(args[0].start)));
};
// we anticipate that we'll add language-specific surveys in the future
// whenever that happens we can relax validation a bit to support more formats
const makeBroadcastSetter = function (attr, validation) {
return function (Env, args) {
if ((validation && !validation(args)) && !args_isString(args)) {
throw new Error('INVALID_ARGS');
}
var str = args[0];
if (str === Env[attr]) { return false; }
Env[attr] = str;
Env.broadcastCache = {};
return true;
};
};
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['DISABLE_EMBEDDING', [true]]], console.log)
commands.ENABLE_EMBEDDING = makeBooleanSetter('enableEmbedding');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['ENFORCE_MFA', [true]]], console.log)
commands.ENFORCE_MFA = makeBooleanSetter('enforceMFA');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['RESTRICT_REGISTRATION', [true]]], console.log)
commands.RESTRICT_REGISTRATION = makeBooleanSetter('restrictRegistration');
commands.RESTRICT_SSO_REGISTRATION = makeBooleanSetter('restrictSsoRegistration');
commands.DISABLE_STORE_INVITED_USERS = makeBooleanSetter('dontStoreInvitedUsers');
commands.DISABLE_STORE_SSO_USERS = makeBooleanSetter('dontStoreSSOUsers');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['DISABLE_INTEGRATED_EVICTION', [true]]], console.log)
commands.DISABLE_INTEGRATED_EVICTION = makeBooleanSetter('disableIntegratedEviction');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['DISABLE_INTEGRATED_TASKS', [true]]], console.log)
commands.DISABLE_INTEGRATED_TASKS = makeBooleanSetter('disableIntegratedTasks');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['CONSENT_TO_CONTACT', [true]]], console.log)
commands.CONSENT_TO_CONTACT = makeBooleanSetter('consentToContact');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['LIST_MY_INSTANCE', [true]]], console.log)
commands.LIST_MY_INSTANCE = makeBooleanSetter('listMyInstance');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['PROVIDE_AGGREGATE_STATISTICS', [true]]], console.log)
commands.PROVIDE_AGGREGATE_STATISTICS = makeBooleanSetter('provideAggregateStatistics');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['REMOVE_DONATE_BUTTON', [true]]], console.log)
commands.REMOVE_DONATE_BUTTON = makeBooleanSetter('removeDonateButton');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['BLOCK_DAILY_CHECK', [true]]], console.log)
commands.BLOCK_DAILY_CHECK = makeBooleanSetter('blockDailyCheck');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_LOGO_MIME', ['image/png']]], console.log)
commands.SET_LOGO_MIME = makeGenericSetter('logoMimeType', args_isString);
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_ACCENT_COLOR', ['#ff0073']]], console.log)
commands.SET_ACCENT_COLOR = makeGenericSetter('accentColor', args_isString);
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['ENABLE_PROFILING', [true]]], console.log)
commands.ENABLE_PROFILING = makeBooleanSetter('enableProfiling');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_PROFILING_WINDOW', [10000]]], console.log)
commands.SET_PROFILING_WINDOW = makeGenericSetter('profilingWindow', args_isPositiveInteger);
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAX_UPLOAD_SIZE', [50 * 1024 * 1024]]], console.log)
commands.SET_MAX_UPLOAD_SIZE = makeIntegerSetter('maxUploadSize');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_PREMIUM_UPLOAD_SIZE', [150 * 1024 * 1024]]], console.log)
commands.SET_PREMIUM_UPLOAD_SIZE = makeIntegerSetter('premiumUploadSize');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['UPDATE_DEFAULT_STORAGE', [100 * 1024 * 1024]]], console.log)
commands.UPDATE_DEFAULT_STORAGE = makeIntegerSetter('defaultStorageLimit');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INACTIVE_TIME', [90]]], console.log)
commands.SET_INACTIVE_TIME = makeIntegerSetter('inactiveTime');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_ARCHIVE_RETENTION_TIME', [30]]], console.log)
commands.SET_ARCHIVE_RETENTION_TIME = makeIntegerSetter('archiveRetentionTime');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_ACCOUNT_RETENTION_TIME', [365]]], console.log)
commands.SET_ACCOUNT_RETENTION_TIME = makeIntegerSetter('accountRetentionTime');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_ADMIN_EMAIL', ['admin@website.tld']]], console.log)
commands.SET_ADMIN_EMAIL = makeGenericSetter('adminEmail', args_isString);
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_SUPPORT_MAILBOX', ["Tdz6+fE9N9XXBY93rW5qeNa/k27yd40c0vq7EJyt7jA="]]], console.log)
commands.SET_SUPPORT_MAILBOX = makeGenericSetter('supportMailbox', function (args) {
return args_isString(args) && Core.isValidPublicKey(args[0]);
});
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_SUPPORT_KEYS', ["Tdz6+fE9N9XXBY93rW5qeNa/k27yd40c0vq7EJyt7jA=", "Tdz6+fE9N9XXBY93rW5qeNa/k27yd40c0vq7EJyt7jA="]]], console.log)
commands.DISABLE_APPS = function (Env, args) {
if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); }
if (JSON.stringify(args) === JSON.stringify(Env.appsToDisable)) { return false; }
Env.appsToDisable = args;
return true;
};
commands.SET_SUPPORT_KEYS = function (Env, args) {
const curvePublic = args[0]; // Support mailbox key
const edPublic = args[1]; // Support pin log
let validated = typeof(curvePublic) === "string" &&
(Core.isValidPublicKey(curvePublic) || !curvePublic) &&
typeof(edPublic) === "string" &&
(Core.isValidPublicKey(edPublic) || !edPublic);
if (!validated) { throw new Error('INVALID_ARGS'); }
if (Env.supportMailboxKey === curvePublic && Env.supportPinKey === edPublic) { return false; }
Env.supportMailboxKey = curvePublic;
Env.supportPinKey = edPublic;
return true;
};
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_PURPOSE', ["development"]]], console.log)
commands.SET_INSTANCE_PURPOSE = makeGenericSetter('instancePurpose', args_isString);
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_JURISDICTION', ['France']]], console.log)
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_JURISDICTION', [{default:'France',de:'Frankreich'}]]], console.log)
commands.SET_INSTANCE_JURISDICTION = makeTranslation('instanceJurisdiction');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_NAME', ['My Personal CryptPad']]], console.log)
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_NAME', [{default:'My Personal CryptPad', fr: "Mon CryptPad personnel"}]]], console.log)
commands.SET_INSTANCE_NAME = makeTranslation('instanceName');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_DESCRIPTION', ['A personal instance, hosted for me and nobody else']]], console.log)
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_DESCRIPTION', [{default:'A personal server, not intended for public usage', fr: 'Un serveur personnel, non destiné à un usage public'}]]], console.log)
commands.SET_INSTANCE_DESCRIPTION = makeTranslation('instanceDescription');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_NOTICE', ['Our hosting costs have increased during the pandemic. Please consider donating!']]], console.log)
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_NOTICE', [{default:'Our hosting costs have increased during the pandemic. Please consider donating!',fr:'Nos coûts d'hébergement ont augmenté pendant la pandémie. Veuillez envisager de faire un don !']]], console.log)
commands.SET_INSTANCE_NOTICE = makeTranslation('instanceNotice');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_LAST_BROADCAST_HASH', [hash]]], console.log)
commands.SET_LAST_BROADCAST_HASH = makeBroadcastSetter('lastBroadcastHash');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_SURVEY_URL', [url]]], console.log)
commands.SET_SURVEY_URL = makeBroadcastSetter('surveyURL');
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAINTENANCE', [{start: +Date, end: +Date}]]], console.log)
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAINTENANCE', [""]]], console.log)
commands.SET_MAINTENANCE = makeBroadcastSetter('maintenance', args_isMaintenance);
// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_QUOTA', ['[user@box:3000/VzeS4vP1DF+tXGuq1i50DKYuBL+09Yqy8kGxoUKRzhA=]', { limit: 2 * 1024 * 1024 * 1024, plan: 'buddy', note: "you're welcome" } ] ] ], console.log)
commands.SET_QUOTA = function (Env, args) {
if (!Array.isArray(args) || args.length !== 2) {
throw new Error("INVALID_ARGS");
}
var unsafeKey = Keys.canonicalize(args[0]);
if (!unsafeKey) {
throw new Error("INVALID_ARGS");
}
// make sure you're not overwriting an existing limit
//if (Env.customLimits[unsafeKey]) { throw new Error("EEXISTS"); }
var limit = args[1];
if (!Quota.isValidLimit(limit)) { // do we really want this?
throw new Error("INVALID_ARGS");
}
limit.origin = 'decree';
// map the new limit to the user's unsafeKey
Env.customLimits[unsafeKey] = limit;
Env.limits[unsafeKey] = limit;
return true;
};
commands.RM_QUOTA = function (Env, args) {
if (!Array.isArray(args) || args.length !== 1) {
throw new Error("INVALID_ARGS");
}
var unsafeKey = Keys.canonicalize(args[0]);
if (!unsafeKey) {
throw new Error("INVALID_ARGS");
}
if (!Env.customLimits[unsafeKey]) {
throw new Error("ENOENT");
}
delete Env.customLimits[unsafeKey];
delete Env.limits[unsafeKey];
return true;
};
commands.ADD_INSTALL_TOKEN = function (Env, args) {
if (!Array.isArray(args) || args.length !== 1 || !args[0]) {
throw new Error("INVALID_ARGS");
}
var token = args[0];
Env.installToken = token;
return true;
};
commands.ADD_ADMIN_KEY = function (Env, args) {
if (!Array.isArray(args) || args.length !== 1 || !args[0]) {
throw new Error("INVALID_ARGS");
}
Env.admins = Env.admins || [];
var key = Keys.canonicalize(args[0]);
if (!key) { throw new Error("INVALID_KEY"); }
if (Env.admins.includes(key)) { // Nothing to change
return false;
}
Env.admins.push(key);
Env.adminsData.push(args[0]);
return true;
};
commands.RM_ADMIN_KEY = function (Env, args) {
if (!Array.isArray(args) || args.length !== 1 || !args[0]) {
throw new Error("INVALID_ARGS");
}
const key = Keys.canonicalize(args[0]);
if (!key) { throw new Error("INVALID_KEY"); }
Env.admins = Env.admins || [];
if (!Env.admins.includes(key)) { // Nothing to change
return false;
}
// NOTE prevent removing config.js hardcoded admin keys
if (Array.isArray(config?.adminKeys) && config.adminKeys.includes(key)) {
throw new Error("CANT_REMOVE_CONFIG");
}
let idx = Env.admins.indexOf(key);
if (idx < 0) { return false; } // should never happen
if (Env.admins.length === 1) { throw new Error("CANT_REMOVE_LAST_ADMIN"); }
Env.admins.splice(idx, 1);
Env.adminsData = Env.adminsData.filter(str => {
const ed = Keys.canonicalize(str);
if (!ed) { return true; }
return ed !== key;
});
return true;
};
commands.PROOFS_MIGRATED = function (Env, args) {
if (args !== 1) {
throw new Error("INVALID_ARGS");
}
Env.proofsMigrated = true;
return true;
};
commands.SET_BEARER_SECRET = function (Env, args) {
if (!args_isString(args) || args.length !== 1 || !args[0]) {
throw new Error("INVALID_ARGS");
}
var secret = args[0];
if (secret === Env.bearerSecret) { return false; }
Env.bearerSecret = secret;
return true;
};
module.exports = DecreesCore.create(DECREE_NAME, commands);

98
lib/defaults.js
View File

@ -1,98 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Default = module.exports;
Default.commonCSP = function (Env) {
var domain = Env.httpUnsafeOrigin;
var sandbox = Env.httpSafeOrigin;
sandbox = (sandbox && sandbox !== domain ? sandbox : '');
// Content-Security-Policy
var accounts_api = Env.accounts_api || '';
var wsURL = domain.replace('https://', 'wss://').replace('http://', 'ws://');
return [
`default-src 'none'`,
`style-src 'unsafe-inline' 'self' ${domain}`,
`font-src 'self' data: ${domain}`,
/* child-src is used to restrict iframes to a set of allowed domains.
* connect-src is used to restrict what domains can connect to the websocket.
*
* it is recommended that you configure these fields to match the
* domain which will serve your CryptPad instance.
*/
`child-src ${domain}`,
// IE/Edge
`frame-src 'self' blob: ${sandbox}`,
/* this allows connections over secure or insecure websockets
if you are deploying to production, you'll probably want to remove
the ws://* directive
*/
`connect-src 'self' blob: ${domain} ${sandbox} ${accounts_api} ${wsURL}`,
// data: is used by codemirror
`img-src 'self' data: blob: ${domain}`,
`media-src blob:`,
// for accounts.cryptpad.fr authentication and cross-domain iframe sandbox
Env.enableEmbedding? `frame-ancestors 'self' ${Env.protocol} vector:`: `frame-ancestors 'self' ${domain}`,
`worker-src 'self'`,
""
];
};
Default.contentSecurity = function (Env) {
return (Default.commonCSP(Env).join('; ') + "script-src 'self' resource: " + Env.httpUnsafeOrigin).replace(/\s+/g, ' ');
};
Default.padContentSecurity = function (Env) {
return (Default.commonCSP(Env).join('; ') + "script-src 'self' 'unsafe-eval' 'unsafe-inline' resource: " + Env.httpUnsafeOrigin).replace(/\s+/g, ' ');
};
Default.httpHeaders = function (Env) {
return {
"X-XSS-Protection": "1; mode=block",
"X-Content-Type-Options": "nosniff",
"Access-Control-Allow-Origin": Env.enableEmbedding? '*': Env.permittedEmbedders,
"Referrer-Policy": "same-origin",
"Permissions-policy":"interest-cohort=()"
};
};
Default.mainPages = function () {
return [
'index',
'contact',
'features',
'maintenance'
];
};
/* The recommmended minimum Node.js version
* ideally managed using NVM and not your system's
* package manager, which usually provides a very outdated version
*/
Default.recommendedVersion = [16,14,2];
/* By default the CryptPad server will run scheduled tasks every five minutes
* If you want to run scheduled tasks in a separate process (like a crontab)
* you can disable this behaviour by setting the following value to true
*/
//disableIntegratedTasks: false,
/* CryptPad's file storage adaptor closes unused files after a configurable
* number of milliseconds (default 30000 (30 seconds))
*/
// channelExpirationMs: 30000,
/* CryptPad's file storage adaptor is limited by the number of open files.
* When the adaptor reaches openFileLimit, it will clean up older files
*/
//openFileLimit: 2048,

449
lib/env.js
View File

@ -1,449 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const { existsSync, readdirSync } = require('node:fs');
const Crypto = require('crypto');
const WriteQueue = require("./write-queue");
const BatchRead = require("./batch-read");
const Keys = require("./keys");
const Core = require("./commands/core");
const Quota = require("./commands/quota");
const Util = require("./common-util");
const Package = require("../package.json");
const Default = require("./defaults");
const Path = require("path");
const plugins = require('./plugin-manager');
const Nacl = require("tweetnacl/nacl-fast");
var canonicalizeOrigin = function (s) {
if (typeof(s) === 'undefined') { return; }
return (s || '').trim().replace(/\/+$/, '');
};
var isValidPort = function (p) {
return typeof(p) === 'number' && p < 65535;
};
var deriveSandboxOrigin = function (unsafe, port) {
var url = new URL(unsafe);
url.port = port;
return url.origin;
};
var isRecentVersion = function () {
var R = Default.recommendedVersion;
var V = process.version;
if (typeof(V) !== 'string') { return false; }
var parts = V.replace(/^v/, '').split('.').map(Number);
if (parts.length < 3) { return false; }
if (!parts.every(n => typeof(n) === 'number' && !isNaN(n))) {
return false;
}
if (parts[0] < R[0]) { return false; }
if (parts[0] > R[0]) { return true; }
// v16
if (parts[1] < R[1]) { return false; }
if (parts[1] > R[1]) { return true; }
if (parts[2] >= R[2]) { return true; }
return false;
};
const getInstalledOOVersions = function() {
if (!existsSync('www/common/onlyoffice/dist')) {
return [];
}
return readdirSync('www/common/onlyoffice/dist');
};
module.exports.create = function (config) {
var httpUnsafeOrigin = canonicalizeOrigin(config.httpUnsafeOrigin);
var httpSafeOrigin;
var NO_SANDBOX = false;
var httpSafePort;
var httpPort = isValidPort(config.httpPort)? config.httpPort: 3000;
if (typeof(config.httpSafeOrigin) !== 'string') {
NO_SANDBOX = true;
if (typeof(config.httpSafePort) !== 'number') { httpSafePort = httpPort + 1; }
httpSafeOrigin = deriveSandboxOrigin(httpUnsafeOrigin, httpSafePort);
} else {
httpSafeOrigin = canonicalizeOrigin(config.httpSafeOrigin);
}
if (typeof(config.websocketPort) !== 'number') {
config.websocketPort = 3003;
}
var permittedEmbedders = config.permittedEmbedders;
if (typeof(permittedEmbedders) === 'string') {
permittedEmbedders = permittedEmbedders.trim();
}
const curve = Nacl.box.keyPair();
const Env = {
plugins: plugins,
logFeedback: Boolean(config.logFeedback),
mainPages: config.mainPages || Default.mainPages(),
protocol: new URL(httpUnsafeOrigin).protocol,
fileHost: config.fileHost? new URL(config.fileHost).origin : undefined,
NO_SANDBOX: NO_SANDBOX,
httpSafePort: httpSafePort,
websocketPort: config.websocketPort,
accounts_api: config.accounts_api,
shouldUpdateNode: !isRecentVersion(),
version: Package.version,
installMethod: config.installMethod || undefined,
httpUnsafeOrigin: httpUnsafeOrigin,
httpSafeOrigin: httpSafeOrigin,
permittedEmbedders: typeof(permittedEmbedders) === 'string' && permittedEmbedders? permittedEmbedders: httpSafeOrigin,
removeDonateButton: config.removeDonateButton,
httpPort: isValidPort(config.httpPort)? config.httpPort: 3000,
httpAddress: typeof(config.httpAddress) === 'string'? config.httpAddress: 'localhost',
websocketPath: config.externalWebsocketURL,
logIP: config.logIP,
OFFLINE_MODE: false,
FRESH_KEY: '',
FRESH_MODE: true,
DEV_MODE: false,
configCache: {},
broadcastCache: {},
officeHeadersCache: undefined,
standardHeadersCache: undefined,
apiHeadersCache: undefined,
flushCache: function () {
Env.FRESH_KEY = +new Date();
if (!(Env.DEV_MODE || Env.FRESH_MODE)) { Env.FRESH_MODE = true; }
Env.cacheFlushed.fire();
if (!Env.Log) { return; }
Env.Log.info("UPDATING_FRESH_KEY", Env.FRESH_KEY);
},
Log: undefined,
// store
id: Crypto.randomBytes(8).toString('hex'),
launchTime: +new Date(),
enableProfiling: false,
profilingWindow: 10000,
bytesWritten: 0,
inactiveTime: config.inactiveTime,
archiveRetentionTime: config.archiveRetentionTime,
accountRetentionTime: config.accountRetentionTime,
adminEmail: config.adminEmail,
supportMailbox: config.supportMailboxPublicKey,
supportMailboxKey: undefined,
metadata_cache: {},
channel_cache: {},
cache_checks: {},
queueStorage: WriteQueue(),
queueDeletes: WriteQueue(),
queueValidation: WriteQueue(),
queueMetadata: WriteQueue(),
batchIndexReads: BatchRead("HK_GET_INDEX"),
batchMetadata: BatchRead('GET_METADATA'),
batchRegisteredUsers: BatchRead("GET_REGISTERED_USERS"),
batchDiskUsage: BatchRead('GET_DISK_USAGE'),
batchUserPins: BatchRead('LOAD_USER_PINS'),
batchTotalSize: BatchRead('GET_TOTAL_SIZE'),
batchAccountQuery: BatchRead("QUERY_ACCOUNT_SERVER"),
intervals: {},
maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024),
premiumUploadSize: false, // overridden below...
Sessions: {},
paths: {},
//msgStore: config.store,
// /api/broadcast
lastBroadcastHash: '',
surveyURL: undefined,
maintenance: undefined,
netfluxUsers: {},
pinStore: undefined,
limits: {},
admins: [],
installToken: undefined,
WARN: function (e, output) { // TODO deprecate this
if (!Env.Log) { return; }
if (e && output) {
Env.Log.warn(e, {
output: output,
message: String(e),
stack: new Error(e).stack,
});
}
},
// as of 4.14.0 you need to opt-in to remote embedding.
enableEmbedding: false,
/* FIXME restrictRegistration is initialized as false and then overridden by admin decree
There is a narrow window in which someone could register before the server updates this value.
See also the cached 'restrictRegistration' value in server.js#serveConfig
*/
restrictRegistration: false,
blockDailyCheck: config.blockDailyCheck === true,
consentToContact: false,
listMyInstance: false,
provideAggregateStatistics: false,
updateAvailable: undefined,
instanceName: {},
instanceDescription: {},
instanceJurisdiction: {},
instanceNotice: {},
customLimits: {},
// FIXME this attribute isn't in the default conf
// but it is referenced in Quota
domain: config.domain,
maxWorkers: undefined,
disableIntegratedTasks: config.disableIntegratedTasks || false,
disableIntegratedEviction: typeof(config.disableIntegratedEviction) === 'undefined'? true: config.disableIntegratedEviction,
lastEviction: +new Date(),
evictionReport: {},
commandTimers: {},
sso: plugins?.SSO?.config || {},
enforceMFA: config.enforceMFA,
...(getInstalledOOVersions().length > 0
? {
onlyOffice: {
availableVersions: getInstalledOOVersions(),
},
}
: {}),
// initialized as undefined
bearerSecret: void 0,
curvePrivate: curve.secretKey,
curvePublic: Util.encodeBase64(curve.publicKey),
selfDestructTo: {},
};
Object.keys(plugins || {}).forEach(name => {
let plugin = plugins[name];
if (!plugin.customizeEnv) { return; }
try { plugin.customizeEnv(Env); }
catch (e) {}
});
(function () {
var max = config.maxWorkers;
// if the supplied value is not a positive number, leave maxWorkers undefined
// one worker will be created for each CPU core
if (typeof(max) !== 'number' || isNaN(max) || max < 1) { return; }
Env.maxWorkers = max;
}());
(function () {
// mode can be FRESH (default), DEV, or PACKAGE
if (process.env.PACKAGE) {
// `PACKAGE=1 node server` uses the version string from package.json as the cache string
//console.log("PACKAGE MODE ENABLED");
Env.FRESH_MODE = false;
Env.DEV_MODE = false;
} else if (process.env.DEV) {
// `DEV=1 node server` will use a random cache string on every page reload
//console.log("DEV MODE ENABLED");
Env.FRESH_MODE = false;
Env.DEV_MODE = true;
} else {
// `FRESH=1 node server` will set a random cache string when the server is launched
// and use it for the process lifetime or until it is reset from the admin panel
//console.log("FRESH MODE ENABLED");
Env.FRESH_KEY = +new Date();
}
// Offline mode is mostly for development. It lets us test clientside cache and offline support
if (process.env.OFFLINE) { Env.OFFLINE_MODE = true; }
}());
Env.checkCache = function (channel) {
var f = Env.cache_checks[channel] || Util.throttle(function () {
delete Env.cache_checks[channel];
if (Env.channel_cache[channel]) { return; }
delete Env.metadata_cache[channel];
}, 30000);
f();
};
(function () {
var custom = config.customLimits;
if (!custom) { return; }
var stored = Env.customLimits;
Object.keys(custom).forEach(function (k) {
var unsafeKey = Keys.canonicalize(k);
if (!unsafeKey) {
console.log("INVALID_CUSTOM_LIMIT_ID", {
message: "A custom quota upgrade was provided via your config with an invalid identifier. It will be ignored.",
key: k,
value: custom[k],
});
return;
}
if (stored[unsafeKey]) {
console.log("INVALID_CUSTOM_LIMIT_DUPLICATED", {
message: "A duplicated custom quota upgrade was provided via your config which would have overridden an existing value. It will be ignored.",
key: k,
value: custom[k],
});
return;
}
if (!Quota.isValidLimit(custom[k])) {
console.log("INVALID_CUSTOM_LIMIT_VALUE", {
message: "A custom quota upgrade was provided via your config with an invalid value. It will be ignored.",
key: k,
value: custom[k],
});
return;
}
var limit = stored[unsafeKey] = Util.clone(custom[k]);
limit.origin = 'config';
});
}());
(function () {
var pes = config.premiumUploadSize;
if (!isNaN(pes) && pes >= Env.maxUploadSize) {
Env.premiumUploadSize = pes;
}
}());
var paths = Env.paths;
var keyOrDefaultString = function (key, def) {
return Path.resolve(typeof(config[key]) === 'string'? config[key]: def);
};
Env.incrementBytesWritten = function (n) {
if (!Env.enableProfiling) { return; }
if (!n || typeof(n) !== 'number' || n < 0) { return; }
Env.bytesWritten += n;
setTimeout(function () {
Env.bytesWritten -= n;
}, Env.profilingWindow);
};
paths.pin = keyOrDefaultString('pinPath', './pins');
paths.block = keyOrDefaultString('blockPath', './block');
paths.data = keyOrDefaultString('filePath', './datastore');
paths.staging = keyOrDefaultString('blobStagingPath', './blobstage');
paths.blob = keyOrDefaultString('blobPath', './blob');
paths.decree = keyOrDefaultString('decreePath', './data/');
paths.base = keyOrDefaultString('base', './data');
paths.archive = keyOrDefaultString('archivePath', './data/archive');
paths.task = keyOrDefaultString('taskPath', './tasks');
Env.defaultStorageLimit = typeof(config.defaultStorageLimit) === 'number' && config.defaultStorageLimit >= 0?
config.defaultStorageLimit:
Core.DEFAULT_LIMIT;
try {
Env.adminsData = (config.adminKeys || []).slice();
Env.admins = (config.adminKeys || []).map(function (k) {
try {
return Keys.canonicalize(k);
} catch (err) {
return;
}
}).filter(Boolean);
} catch (e) {
console.error("Can't parse admin keys. Please update or fix your config.js file!");
}
try {
let moderators = require('./commands/moderators').getKeysSync(Env);
Env.moderators = moderators.map(safeKey => {
return Util.unescapeKeyCharacters(safeKey);
}) || [];
} catch (e) {
console.log(e);
console.error("Can't parse support keys.");
}
Env.envUpdated = Util.mkEvent();
Env.cacheFlushed = Util.mkEvent();
return Env;
};
// don't serialize these things
const BAD = [
'Log',
'envUpdated',
'cacheFlushed',
'evictionReports',
'commandTimers',
'metadata_cache',
'channel_cache',
'cache_checks',
'intervals',
'Sessions',
'netfluxUsers',
'limits',
'customLimits',
'scheduleDecree',
'plugins',
'httpServer',
'pinStore',
'msgStore',
'store',
'blobStore',
];
module.exports.serialize = function (Env) {
return JSON.stringify(Env, function (key, value) {
if (value === Env) { return value; }
if (BAD.includes(key)) { return; }
if (typeof(value) === 'function') { return; }
//console.log('serializing', { key, value, });
if (Util.isCircular(value)) { return; }
return value;
});
};

671
lib/eviction.js
View File

@ -1,671 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var nThen = require("nthen");
var Bloom = require("@mcrowe/minibloom");
var Util = require("../lib/common-util");
var Linked = require("../lib/commands/linked");
var Pins = require("../lib/pins");
var Keys = require("./keys");
var Path = require('node:path');
var Fs = require("node:fs");
var getNewestTime = function (stats) {
return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
return stats[b] > stats[a]? b: a;
})];
};
/*
Env = {
limits: {
<unsafeKey>: <limit>,
},
archiveRetentionTime: <number of days>,
accountRetentionTime: <number of days>,
inactiveTime: <number of days>,
paths: {
pin: <path to pin storage>
},
store,
pinStore,
Log,
blobStore,
};
*/
// the number of ms artificially introduced between CPU-intensive operations
var THROTTLE_FACTOR = 10;
var PROGRESS_FACTOR = 1000;
var evictArchived = function (Env, cb) {
var Log;
var store;
var blobs;
var retentionTime = +new Date() - (Env.archiveRetentionTime * 24 * 3600 * 1000);
var report = {
// archivedChannelsRemoved,
// archivedAccountsRemoved,
// archivedBlobsRemoved,
// totalChannels,
// activeChannels,
// totalBlobs,
// activeBlobs,
// totalAccounts,
// activeAccounts,
// channelsArchived,
launchTime: +new Date(),
// runningTime,
};
var loadStorage = function () {
store = Env.store;
Log = Env.Log;
blobs = Env.blobStore;
};
var removeArchivedChannels = function (w) {
// this block will iterate over archived channels and removes them
// if they've been in cold storage for longer than your configured archive time
// if the admin has not set an 'archiveRetentionTime', this block makes no sense
// so just skip it
if (typeof(Env.archiveRetentionTime) !== "number") { return; }
// count the number of files which have been removed in this run
var removed = 0;
var accounts = 0;
var handler = function (err, item, cb) {
if (err) {
return Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err, cb);
}
// don't mess with files that are freshly stored in cold storage
// based on ctime because that's changed when the file is moved...
if (+new Date(item.ctime) > retentionTime) {
return void cb();
}
// but if it's been stored for the configured time...
// expire it
if (Env.DRY_RUN) {
if (item.channel.length === 32) { removed++; }
else if (item.channel.length === 44) { accounts++; }
return void Log.info("EVICT_ARCHIVED_CHANNEL_DRY_RUN", item.channel, cb);
}
store.removeArchivedChannel(item.channel, w(function (err) {
if (err) {
return Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', {
error: err,
channel: item.channel,
}, cb);
}
if (item.channel.length === 32) {
removed++;
} else if (item.channel.length === 44) {
accounts++;
}
Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel, cb);
}));
};
// if you hit an error, log it
// otherwise, when there are no more channels to process
// log some stats about how many were removed
var done = function (err) {
if (err) {
return Log.error('EVICT_ARCHIVED_FINAL_ERROR', err);
}
report.archivedChannelsRemoved = removed;
report.archivedAccountsRemoved = accounts;
Log.info('EVICT_ARCHIVED_CHANNELS_REMOVED', removed);
Log.info('EVICT_ARCHIVED_ACCOUNTS_REMOVED', accounts);
};
store.listArchivedChannels(handler, w(done));
};
// Blob proofs are no longer supported and can't be restored
// so we can delete them all
var removeArchivedBlobProofs = function (w) {
var archivePath = Path.join(Env.paths.archive, 'blob');
const cb = Util.once(w());
let i = 0;
nThen(w => {
Fs.readdir(archivePath, w((err, list) => {
if (err) { return; }
list.forEach(dir => {
// Look for 3 characters long folders
if (dir.length !== 3) { return; }
let path = Path.join(archivePath, dir);
Fs.rm(path, { recursive: true, force: true }, w(err => {
if (err) { return; }
i++;
}));
});
}));
}).nThen(() => {
Log.info('EVICT_ARCHIVED_BLOB_PROOFS', i);
cb();
});
};
var removeArchivedBlobs = function (w) {
if (typeof(Env.archiveRetentionTime) !== "number") { return; }
// Iterate over archived blobs and remove them
// if they are older than the specified retention time
var removed = 0;
blobs.list.archived.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
return void next();
}
if (item && item.ctime > retentionTime) { return void next(); }
if (Env.DRY_RUN) {
removed++;
return void Log.info("EVICT_ARCHIVED_BLOB_DRY_RUN", item, next);
}
blobs.remove.archived.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_ARCHIVED_BLOB_ERROR", item);
return void next();
}
Log.info("EVICT_ARCHIVED_BLOB", item);
removed++;
next();
});
}, w(function () {
report.archivedBlobsRemoved = removed;
Log.info('EVICT_ARCHIVED_BLOBS_REMOVED', removed);
}));
};
if (Env.DRY_RUN) { Env.Log.info('DRY RUN'); }
nThen(loadStorage)
.nThen(removeArchivedChannels)
.nThen(removeArchivedBlobProofs)
.nThen(removeArchivedBlobs)
.nThen(function () {
cb(void 0, report);
});
};
module.exports = function (Env, cb) {
var complete = Util.once(Util.mkAsync(cb));
var report = {
// archivedChannelsRemoved,
// archivedAccountsRemoved,
// archivedBlobsRemoved,
// totalChannels,
// activeChannels,
// totalBlobs,
// activeBlobs,
// totalAccounts,
// activeAccounts,
// channelsArchived,
launchTime: +new Date(),
// runningTime,
};
// the administrator should have set an 'inactiveTime' in their config
// if they didn't, just exit.
if (!Env.inactiveTime || typeof(Env.inactiveTime) !== "number") {
return void complete("NO_INACTIVE_TIME");
}
// get a list of premium accounts on this instance
// pre-converted to the 'safeKey' format so we can easily compare
// them against ids we see on the filesystem
var premiumSafeKeys = Object.keys(Env.limits || {})
.map(function (id) {
return Keys.canonicalize(id);
})
.filter(Boolean)
.map(Util.escapeKeyCharacters);
// files which have not been changed since before this date can be considered inactive
var inactiveTime = +new Date() - (Env.inactiveTime * 24 * 3600 * 1000);
// files which were archived before this date can be considered safe to remove
var retentionTime = +new Date() - (Env.archiveRetentionTime * 24 * 3600 * 1000);
var store;
var pinStore;
var Log;
var blobs;
/* It's fairly easy to know if a channel or blob is active
but knowing whether it is pinned requires that we
keep the set of pinned documents in memory.
Some users will share the same set of documents in their pin lists,
so the representation of pinned documents should scale sub-linearly
with the number of users and pinned documents.
That said, sub-linear isn't great...
A Bloom filter is "a space-efficient probabilistic data structure"
which lets us check whether an item is _probably_ or _definitely not_
in a set. This is good enough for our purposes since we just want to
know whether something can safely be removed and false negatives
(not safe to remove when it actually is) are acceptable.
We set our capacity to some large number, and the error rate to whatever
we think is acceptable.
TODO make this configurable ?
*/
var BLOOM_CAPACITY = (1 << 24) - 1; // over two million items
var BLOOM_ERROR = 1 / 10000; // an error rate of one in ten thousand
// we'll use one filter for the set of active documents
var activeDocs = Bloom.optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR);
// and another one for the set of pinned documents
var pinnedDocs = Bloom. optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR);
var startTime = +new Date();
var msSinceStart = function () {
return (+new Date()) - startTime;
};
var loadStorage = function () {
store = Env.store;
pinStore = Env.pinStore;
Log = Env.Log;
blobs = Env.blobStore;
};
var categorizeChannelsByActivity = function (w) {
var channels = 0;
var active = 0;
var handler = function (err, item, cb) {
channels++;
if (channels % PROGRESS_FACTOR === 0) {
Log.info('EVICT_CHANNEL_CATEGORIZATION_PROGRESS', {
channels: channels,
});
}
if (err) {
return Log.error('EVICT_CHANNEL_CATEGORIZATION', err, cb);
}
// if the channel has been modified recently
// we don't use mtime because we don't want to count access to the file, just modifications
if (+new Date(item.mtime) > inactiveTime) {
// add it to the set of activeDocs
activeDocs.add(item.channel);
active++;
return void cb();
}
return void cb();
};
var done = function () {
report.activeChannels = active;
report.totalChannels = channels;
Log.info('EVICT_CHANNELS_CATEGORIZED', {
active: active,
channels: channels,
}, w());
};
Log.info('EVICT_CHANNEL_ACTIVITY_START', 'Assessing channel activity');
store.listChannels(handler, w(done));
};
var categorizeBlobsByActivity = function (w) {
var n_blobs = 0;
var active = 0;
Log.info('EVICT_BLOBS_ACTIVITY_START', 'Assessing blob activity');
blobs.list.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
n_blobs++;
if (n_blobs % PROGRESS_FACTOR === 0) {
Log.info('EVICT_BLOB_CATEGORIZATION_PROGRESS', {
blobs: n_blobs,
});
}
if (err) {
return Log.error("EVICT_BLOB_CATEGORIZATION", err, next);
}
if (!item) {
return void Log.error("EVICT_BLOB_CATEGORIZATION_INVALID", item, next);
}
if (item.mtime > inactiveTime) {
activeDocs.add(item.blobId);
active++;
return void next();
}
next();
}, w(function () {
report.totalBlobs = n_blobs;
report.activeBlobs = active;
Log.info('EVICT_BLOBS_CATEGORIZED', {
active: active,
blobs: n_blobs,
}, w());
}));
};
var categorizeAccountsByActivity = function (w) {
// iterate over all accounts
var accounts = 0;
var inactive = 0;
var accountRetentionTime;
if (typeof(Env.accountRetentionTime) === 'number' && Env.accountRetentionTime > 0) {
accountRetentionTime = +new Date() - (24 * 3600 * 1000 * Env.accountRetentionTime);
} else {
accountRetentionTime = -1;
}
var pinAll = function (pinList, cb) {
let n = nThen;
pinList.forEach(function (docId) {
pinnedDocs.add(docId);
// Add linked documents
if (docId.length !== 32) { return; }
n = n(w => {
Linked.listLinkedDocuments(Env, docId, w((err, channels) => {
if (!Array.isArray(channels)) { return; }
channels.forEach(chan => { pinnedDocs.add(chan); });
}));
}).nThen;
});
n(() => { cb(); });
};
var docIsActive = function (docId) {
return activeDocs.test(docId);
};
var accountIsActive = function (mtime, pinList) {
// console.log("id [%s] in premiumSafeKeys", id, premiumSafeKeys.indexOf(id) !== -1);
// if their pin log has changed recently then consider them active
if (mtime && mtime > accountRetentionTime) {
return true;
}
// iterate over their pinned documents until you find one that has been active
return pinList.some(docIsActive);
};
var isPremiumAccount = function (id) {
return premiumSafeKeys.indexOf(id) !== -1;
};
var PRESERVE_INACTIVE_ACCOUNTS = accountRetentionTime <= 0;
// otherwise, we'll only retain data from active accounts
// so we need more heuristics
var handler = function (content, id, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
accounts++;
if (accounts % PROGRESS_FACTOR === 0) {
Log.info('EVICT_ACCOUNT_CATEGORIZATION_PROGRESS', {
accounts: accounts,
});
}
var mtime = content.latest;
var pinList = Object.keys(content.pins);
if (accountIsActive(mtime, pinList)) {
// add active accounts' pinned documents to a second bloom filter
return pinAll(pinList, next);
}
// Otherwise they are inactive.
// We keep track of how many accounts are inactive whether or not
// we plan to delete them, because it may be interesting information
inactive++;
if (PRESERVE_INACTIVE_ACCOUNTS) {
return pinAll(pinList, () => {
Log.info('EVICT_INACTIVE_ACCOUNT_PRESERVED', {
id: id,
mtime: mtime,
}, next);
});
}
if (isPremiumAccount(id)) {
return pinAll(pinList, () => {
Log.info("EVICT_INACTIVE_PREMIUM_ACCOUNT", {
id: id,
mtime: mtime,
}, next);
});
}
// remove the pin logs of inactive accounts if inactive account removal is configured
if (Env.DRY_RUN) {
return void Log.info("EVICT_INACTIVE_ACCOUNT_DRY_RUN", id, next);
}
pinStore.archiveChannel(id, undefined, function (err) {
if (err) {
return Log.error('EVICT_INACTIVE_ACCOUNT_PIN_LOG', err, next);
}
Log.info('EVICT_INACTIVE_ACCOUNT_LOG', id, next);
});
};
var done = function () {
var label = PRESERVE_INACTIVE_ACCOUNTS?
"EVICT_COUNT_ACCOUNTS":
"EVICT_INACTIVE_ACCOUNTS";
report.totalAccounts = accounts;
report.activeAccounts = accounts - inactive;
Log.info(label, {
accounts: accounts,
inactive: inactive,
});
};
Log.info('EVICT_ACCOUNTS_ACTIVITY_START', 'Assessing account activity');
Pins.load(w(done), {
pinPath: Env.paths.pin,
handler: handler,
});
};
var archiveInactiveBlobs = function (w) {
// iterate over blobs and remove them
// if they have not been accessed within the specified retention time
var removed = 0;
var total = 0;
Log.info('EVICT_BLOB_START', {});
blobs.list.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) {
return Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err, next);
}
if (!item) {
return void Log.error('EVICT_BLOB_LIST_BLOBS_NO_ITEM', item, next);
}
total++;
if (total % PROGRESS_FACTOR === 0) {
Log.info("EVICT_BLOB_PROGRESS", {
blobs: total,
});
}
if (pinnedDocs.test(item.blobId)) { return void next(); }
if (activeDocs.test(item.blobId)) { return void next(); }
// NOTE: fast mode allows us to skip getStats for
// the pinned and active channels
nThen(function (w) {
// double check that the channel really is inactive before archiving it
// because it might have been created after the initial activity scan
blobs.getStats(item.blobId, w(function (err, newerItem) {
if (err) { return; }
if (newerItem && getNewestTime(newerItem) > retentionTime) {
// it's actually active, so don't archive it.
w.abort();
next();
}
// else fall through to the archival
}));
}).nThen(function () {
if (Env.DRY_RUN) {
removed++;
return void Log.info("EVICT_ARCHIVE_BLOB_DRY_RUN", {
item: item,
}, next);
}
blobs.archive.blob(item.blobId, 'INACTIVE', function (err) {
if (err) {
return Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
error: err,
item: item,
}, next);
}
removed++;
Log.info("EVICT_ARCHIVE_BLOB", {
item: item,
}, next);
});
});
}, w(function () {
report.totalBlobs = total;
report.activeBlobs = total - removed;
Log.info('EVICT_BLOBS_REMOVED', removed, w());
}), true);
};
var archiveInactiveChannels = function (w) {
var channels = 0;
var archived = 0;
var handler = function (err, item, cb) {
cb = Util.mkAsync(cb, THROTTLE_FACTOR);
channels++;
if (channels % PROGRESS_FACTOR === 0) {
Log.info('EVICT_INACTIVE_CHANNELS_PROGRESS', {
channels,
archived,
});
}
if (err) {
return Log.error('EVICT_CHANNEL_ITERATION', err, cb);
}
// ignore the special admin broadcast channel
if (item.channel.length === 33) { return void cb(); }
// check if the database has any ephemeral channels
// if it does it's because of a bug, and they should be removed
if (item.channel.length === 34) {
if (Env.DRY_RUN) {
return void Log.info("EVICT_EPHEMERAL_DRY_RUN", item.channel, cb);
}
return void store.removeChannel(item.channel, w(function (err) {
if (err) {
return Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', {
error: err,
channel: item.channel,
}, cb);
}
Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel, cb);
}));
}
// bail out if the channel is in the set of activeDocs
if (activeDocs.test(item.channel)) { return void cb(); }
// ignore the channel if it's pinned
if (pinnedDocs.test(item.channel)) { return void cb(); }
nThen(function (w) {
// double check that the channel really is inactive before archiving it
// because it might have been created after the initial activity scan
store.getChannelStats(item.channel, w(function (err, newerItem) {
if (err) { return; }
if (newerItem && getNewestTime(newerItem) > retentionTime) {
// it's actually active, so don't archive it.
w.abort();
cb();
}
// else fall through to the archival
}));
}).nThen(function (w) {
if (Env.DRY_RUN) {
archived++;
w.abort();
return void Log.info("EVICT_CHANNEL_ARCHIVAL_DRY_RUN", item.channel, cb);
}
return void store.archiveChannel(item.channel, 'INACTIVE', w(function (err) {
if (err) {
Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', {
error: err,
channel: item.channel,
}, w());
return;
}
archived++;
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel, w());
}));
}).nThen(cb);
};
var done = function () {
report.channelsArchived = archived;
return void Log.info('EVICT_CHANNELS_ARCHIVED', {
channels,
archived,
});
};
Log.info('EVICT_INACTIVE_CHANNELS_START', {});
store.listChannels(handler, w(done), true); // using a hacky "fast mode" since we only need the channel id
};
if (Env.DRY_RUN) { Env.Log.info('DRY RUN'); }
nThen(loadStorage)
// iterate over all documents and add them to a bloom filter if they have been active
.nThen(categorizeChannelsByActivity)
.nThen(categorizeBlobsByActivity)
// iterate over all accounts and add them to a bloom filter if they are active
.nThen(categorizeAccountsByActivity)
// iterate again and archive inactive unpinned documents
// (documents which are not in either bloom filter)
.nThen(archiveInactiveBlobs)
.nThen(archiveInactiveChannels)
.nThen(function () {
var runningTime = report.runningTime = msSinceStart();
Log.info("EVICT_TIME_TO_RUN_SCRIPT", runningTime);
}).nThen(function () {
complete(void 0, report);
});
};
module.exports.archived = evictArchived;

249
lib/historyKeeper.js
View File

@ -1,249 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const nThen = require('nthen');
const RPC = require("./rpc");
const HK = require("./hk-util.js");
const Store = require("./storage/file");
const BlobStore = require("./storage/blob");
const Workers = require("./workers/index");
const Core = require("./commands/core");
module.exports.create = function (Env, cb) {
const Log = Env.Log;
Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE');
Env.historyKeeper = {
metadata_cache: Env.metadata_cache,
channel_cache: Env.channel_cache,
id: Env.id,
channelMessage: function (Server, channel, msgStruct, cb) {
// netflux-server emits 'channelMessage' events whenever someone broadcasts to a channel
// historyKeeper stores these messages if the channel id indicates that they are
// a channel type with permanent history
HK.onChannelMessage(Env, Server, channel, msgStruct, cb);
},
channelClose: function (channelName) {
// netflux-server emits 'channelClose' events whenever everyone leaves a channel
// we drop cached metadata and indexes at the same time
HK.dropChannel(Env, channelName);
},
channelOpen: function (Server, channelName, userId, wait) {
Env.channel_cache[channelName] = Env.channel_cache[channelName] || {};
var sendHKJoinMessage = function () {
Server.send(userId, [
0,
Env.id,
'JOIN',
channelName
]);
};
// a little backwards compatibility in case you don't have the latest server
// allow lists won't work unless you update, though
if (typeof(wait) !== 'function') { return void sendHKJoinMessage(); }
var next = wait();
var cb = function (err, info) {
next(err, info, sendHKJoinMessage);
};
// only conventional channels can be restricted
if ((channelName || "").length !== HK.STANDARD_CHANNEL_LENGTH) {
return void cb();
}
// gets and caches the metadata...
HK.getMetadata(Env, channelName, function (err, metadata) {
if (err) {
Log.error('HK_METADATA_ERR', {
channel: channelName,
error: err,
});
}
if (metadata && metadata.selfdestruct && metadata.selfdestruct !== Env.id) {
HK.removeChannel(Env, channelName);
return void cb('ESELFDESTRUCT');
}
if (Env.selfDestructTo && Env.selfDestructTo[channelName]) {
clearTimeout(Env.selfDestructTo[channelName]);
}
if (!metadata || (metadata && !metadata.restricted)) {
// the channel doesn't have metadata, or it does and it's not restricted
// either way, let them join.
return void cb();
}
// this channel is restricted. verify that the user in question is in the allow list
// construct a definitive list (owners + allowed)
var allowed = HK.listAllowedUsers(metadata);
// and get the list of keys for which this user has already authenticated
var session = HK.getNetfluxSession(Env, userId);
if (HK.isUserSessionAllowed(allowed, session)) {
return void cb();
}
// If the channel is restricted, send the history keeper ID so that they
// can try to authenticate
allowed.unshift(Env.id);
// otherwise they're not allowed.
// respond with a special error that includes the list of keys
// which would be allowed...
// FIXME RESTRICT bonus points if you hash the keys to limit data exposure
cb("ERESTRICTED", allowed);
});
},
sessionClose: function (userId, reason) {
HK.closeNetfluxSession(Env, userId);
if (Env.logIP && !['SOCKET_CLOSED', 'INACTIVITY'].includes(reason)) {
return void Log.info('USER_DISCONNECTED_ERROR', {
userId: userId,
reason: reason
});
}
if (['BAD_MESSAGE', 'SEND_MESSAGE_FAIL_2'].indexOf(reason) !== -1) {
if (reason && reason.code === 'ECONNRESET') { return; }
return void Log.error('SESSION_CLOSE_WITH_ERROR', {
userId: userId,
reason: reason,
});
}
if (['SOCKET_CLOSED', 'SOCKET_ERROR'].includes(reason)) { return; }
Log.verbose('SESSION_CLOSE_ROUTINE', {
userId: userId,
reason: reason,
});
},
sessionOpen: function (userId, ip) {
if (!Env.logIP) { return; }
Log.info('USER_CONNECTION', {
userId: userId,
ip: ip,
});
},
directMessage: function (Server, seq, userId, json) {
// netflux-server allows you to register an id with a handler
// this handler is invoked every time someone sends a message to that id
HK.onDirectMessage(Env, Server, seq, userId, json);
},
};
Log.verbose('HK_ID', 'History keeper ID: ' + Env.id);
var pinPath = Env.paths.pin;
nThen(function (w) {
// create a pin store
Store.create({
filePath: pinPath,
archivePath: Env.paths.archive,
// indicate that archives should be put in a 'pins' archvie folder
volumeId: 'pins',
}, w(function (err, s) {
if (err) { throw err; }
Env.pinStore = s;
}));
// create a channel store
Store.create({
filePath: Env.paths.data,
archivePath: Env.paths.archive,
}, w(function (err, _store) {
if (err) { throw err; }
Env.msgStore = _store; // API used by rpc
Env.store = _store; // API used by historyKeeper
}));
// create a blob store
BlobStore.create({
blobPath: Env.paths.blob,
blobStagingPath: Env.paths.staging,
archivePath: Env.paths.archive,
getSession: function (safeKey) {
return Core.getSession(Env.Sessions, safeKey);
},
}, w(function (err, blob) {
if (err) { throw new Error(err); }
Env.blobStore = blob;
}));
}).nThen(function (w) {
Workers.initialize(Env, {
blobPath: Env.paths.blob,
blobStagingPath: Env.paths.staging,
taskPath: Env.paths.task,
pinPath: Env.paths.pin,
filePath: Env.paths.data,
archivePath: Env.paths.archive,
blockPath: Env.paths.block,
inactiveTime: Env.inactiveTime,
archiveRetentionTime: Env.archiveRetentionTime,
accountRetentionTime: Env.accountRetentionTime,
maxWorkers: Env.maxWorkers,
}, w(function (err) {
if (err) {
throw new Error(err);
}
}));
}).nThen(function () {
var tasks_running;
Env.intervals.taskExpiration = setInterval(function () {
if (Env.disableIntegratedTasks) { return; }
if (tasks_running) { return; }
tasks_running = true;
Env.runTasks(function (err) {
if (err) {
Log.error('TASK_RUNNER_ERR', err);
}
tasks_running = false;
});
}, 1000 * 60 * 5); // run every five minutes
}).nThen(function () {
const ONE_DAY = 24 * 1000 * 60 * 60;
// setting the time of the last eviction to "now"
// effectively makes it so that we'll start evicting after the server
// has been up for at least one day
var active = false;
Env.intervals.eviction = setInterval(function () {
if (Env.disableIntegratedEviction) { return; }
if (active) { return; }
var now = +new Date();
// evict inactive data once per day
if ((now - ONE_DAY) < Env.lastEviction) { return; }
active = true;
Env.evictInactive(function (err, report) {
if (err) {
// NO_INACTIVE_TIME
Log.error('EVICT_INACTIVE_MAIN_ERROR', err);
}
active = false;
Env.lastEviction = now;
if (report) {
Log.info('EVICT_INACTIVE_REPORT', report);
}
Env.evictionReport = report || {};
});
}, 60 * 1000);
}).nThen(function () {
RPC.create(Env, function (err, _rpc) {
if (err) { throw err; }
Env.rpc = _rpc;
cb(void 0, Env.historyKeeper);
});
});
};

1118
lib/hk-util.js
View File

File diff suppressed because it is too large Load Diff

349
lib/http-commands.js
View File

@ -1,349 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Nacl = require("tweetnacl/nacl-fast");
var Util = require('./common-util.js');
const plugins = require("./plugin-manager");
var Challenge = require("./storage/challenge.js");
// C.read(Env, id, cb)
// C.write(Env,id, data, cb)
// C.delete(Env, id, cb)
/*
The API for command definition consists of two stages:
Clients first send a command and its associated parameters.
The server validates that the command is supported, and that
the provided parameters are valid. If it fails validation for any reason,
the server responds with an error and the protocol is aborted.
COMMANDS[COMMAND_NAME] = function (Env, body, cb) {
// inspect parameters in the request body
if (!body.essential_parameter) {
return void cb('NO');
}
cb();
};
Commands whose parameters are successfully validated
have those parameters stored on the disk (or a relational DB in the future).
The server then requests that the client sign their well-formulated
command along with a server-generated transaction id ('txid': randomized to prevent replays)
and a date (so that it can ensure that the client responds within a reasonable window.
Clients then respond with a txid and a cryptographic signature
which matches the parameters of the command. The server loads the command
with the corresponding txid, checks that it was signed within a reasonable time window,
validates the signature, and attempts to complete the command's execution:
COMMAND[COMMAND_NAME].complete = function (Env, body, cb) {
doAThing(function (err, values) {
if (err) {
// Log the error and respond that the command was not successful
return void cb("SORRY_BUT_IM_NOT_OK");
}
cb(void 0, {
arbitrary: values,
});
});
};
In this second stage the protocol can be aborted if the client has done something wrong:
(ie. if it did not produce a valid signature for the command)
or it can can fail because the server was not able to complete the requested task
(ie. because of an I/O error or because an error was thrown and caught).
It is intended that the server will respond with an appropriate error if
the request cannot be completed, and it will respond OK if everything completed successfully.
*/
var COMMANDS = {};
// Methods allowing clients to configure Time-based One-Time Passwords for their login-block,
// and to authenticate new sessions once a TOTP secret has been associated with their account,
const NOAUTH = require("./challenge-commands/base.js");
COMMANDS.MFA_CHECK = NOAUTH.MFA_CHECK;
COMMANDS.WRITE_BLOCK = NOAUTH.WRITE_BLOCK; // Account creation + password change
COMMANDS.REMOVE_BLOCK = NOAUTH.REMOVE_BLOCK;
COMMANDS.UPLOAD_COOKIE = NOAUTH.UPLOAD_COOKIE;
const TOTP = require("./challenge-commands/totp.js");
COMMANDS.TOTP_SETUP = TOTP.TOTP_SETUP;
COMMANDS.TOTP_VALIDATE = TOTP.TOTP_VALIDATE;
COMMANDS.TOTP_MFA_CHECK = TOTP.TOTP_MFA_CHECK;
COMMANDS.TOTP_REVOKE = TOTP.TOTP_REVOKE;
COMMANDS.TOTP_WRITE_BLOCK = TOTP.TOTP_WRITE_BLOCK; // Password change only for now (v5.5.0)
COMMANDS.TOTP_REMOVE_BLOCK = TOTP.TOTP_REMOVE_BLOCK;
// Load challenges added by plugins
Object.keys(plugins || {}).forEach(id => {
try {
let plugin = plugins[id];
if (!plugin.challenge) { return; }
let commands = plugin.challenge;
Object.keys(commands).forEach(cmd => {
if (COMMANDS[cmd]) { return; } // Don't overwrite
COMMANDS[cmd] = commands[cmd];
});
} catch (e) {}
});
/*
const SSO = plugins.SSO && plugins.SSO.challenge;
COMMANDS.SSO_AUTH = SSO.SSO_AUTH;
COMMANDS.SSO_AUTH_CB = SSO.SSO_AUTH_CB;
COMMANDS.SSO_WRITE_BLOCK = SSO.SSO_WRITE_BLOCK; // Account creation only
COMMANDS.SSO_UPDATE_BLOCK = SSO.SSO_UPDATE_BLOCK; // Password change
COMMANDS.SSO_VALIDATE = SSO.SSO_VALIDATE;
*/
var randomToken = () => Util.encodeBase64(Nacl.randomBytes(24)).replace(/\//g, '-');
// this function handles the first stage of the protocol
// (the server's validation of the client's request and the generation of its challenge)
var handleCommand = function (Env, req, res) {
var body = req.body;
var command = body.command;
// reject if the command does not have a corresponding function
if (typeof(COMMANDS[command]) !== 'function') {
Env.Log.error('CHALLENGE_UNSUPPORTED_COMMAND', command);
return void res.status(500).json({
error: 'invalid command',
});
}
var publicKey = body.publicKey;
// reject if they did not provide a valid public key
if (!publicKey || typeof(publicKey) !== 'string' || publicKey.length !== 44) {
Env.Log.error('CHALLENGE_INVALID_KEY', publicKey);
return void res.status(500).json({
error: 'Invalid key',
});
}
try {
COMMANDS[command](Env, body, function (err) {
if (err) {
Env.Log.error('CHALLENGE_COMMAND_EXECUTION_ERROR', {
command,
error: Util.serializeError(err),
});
// errors returned from commands are passed back to the client
// as a weak precaution, we try to only send an error's message
// if one exists. This makes it less likely that we'll respond with any
// sensitive information in a stack trace. Ideally functions should
// only return error messages or codes in the form of a string or number,
// but mistakes happen.
return void res.status(500).json({
error: (err && err.message) || err,
});
}
var txid = randomToken();
var date = new Date().toISOString();
var copy = Util.clone(body);
copy.txid = txid;
copy.date = date;
// Write the command and challenge to disk, because the challenge protocol
// is interactive and the subsequent response might be handled by a different http worker
// this makes it so we can avoid holding state in memory
Challenge.write(Env, txid, JSON.stringify(copy), function (err) {
if (err) {
Env.Log.error('CHALLENGE_WRITE_ERROR', Util.serializeError(err));
return void res.status(500).json({
// arbitrary error message, only intended for debugging
error: 'Internal server error 6250',
});
}
// respond with challenge parameters
return void res.status(200).json({
txid: txid,
date: date,
});
});
}, req);
} catch (err) {
Env.Log.error("CHALLENGE_COMMAND_THROWN_ERROR", {
error: Util.serializeError(err),
});
return void res.status(500).json({
// arbitrary error message, only intended for debugging
error: 'Internal server error 7692',
});
}
};
// this function handles the second stage of the protocol
// (the client's response to the server's challenge)
var handleResponse = function (Env, req, res) {
var body = req.body;
if (Object.keys(body).some(k => !/(sig|txid)/.test(k))) {
Env.Log.error("CHALLENGE_RESPONSE_DEBUGGING", body);
// we expect the response to only have two keys
// if any more are present then the response is malformed
return void res.status(500).json({
error: 'extraneous parameters',
});
}
// transaction ids are issued to the client by the server
// they allow it to recall the full details of the challenge
// to which the client is responding
var txid = body.txid;
// if no txid is present, then the server can't look up the corresponding challenge
// the response is definitely malformed, so reject it.
// Additionally, we expect txids to be 32 characters long (24 Uint8s as base64)
// reject txids of any other length
if (!txid || typeof(txid) !== 'string' || txid.length !== 32) {
Env.Log.error('CHALLENGE_RESPONSE_BAD_TXID', body);
return void res.status(500).json({
error: "Invalid txid",
});
}
var sig = body.sig;
if (!sig || typeof(sig) !== 'string' || sig.length !== 88) {
Env.Log.error("CHALLENGE_RESPONSE_BAD_SIG", body);
return void res.status(500).json({
error: "Missing signature",
});
}
Challenge.read(Env, txid, function (err, text) {
if (err) {
Env.Log.error("CHALLENGE_READ_ERROR", {
txid: txid,
error: Util.serializeError(err),
});
return void res.status(500).json({
error: "Unexpected response",
});
}
var json = Util.tryParse(text);
if (!json) {
Env.Log.error("CHALLENGE_PARSE_ERROR", {
txid: txid,
});
return void res.status(500).json({
error: "Internal server error 129",
});
}
var publicKey = json.publicKey;
if (!publicKey || typeof(publicKey) !== 'string') {
// This shouldn't happen, as we expect that the server
// will have validated the key to an extent before storing the challenge
Env.Log.error('CHALLENGE_INVALID_PUBLICKEY', {
publicKey: publicKey,
});
return res.status(500).json({
error: "Invalid public key",
});
}
var action;
try {
action = COMMANDS[json.command].complete;
} catch (err2) {}
if (typeof(action) !== 'function') {
Env.Log.error("CHALLENGE_RESPONSE_ACTION_NOT_IMPLEMENTED", json.command);
return res.status(501).json({
error: 'Not implemented',
});
}
var u8_toVerify,
u8_sig,
u8_publicKey;
try {
u8_toVerify = Util.decodeUTF8(text);
u8_sig = Util.decodeBase64(sig);
u8_publicKey = Util.decodeBase64(publicKey);
} catch (err3) {
Env.Log.error('CHALLENGE_RESPONSE_DECODING_ERROR', {
command: json.command,
publicKey: publicKey,
error: Util.serializeError(err3),
});
return res.status(500).json({
error: "decoding error"
});
}
// validate the response
var success = Nacl.sign.detached.verify(u8_toVerify, u8_sig, u8_publicKey);
if (success !== true) {
Env.Log.error("CHALLENGE_RESPONSE_SIGNATURE_FAILURE", {
publicKey,
});
return void res.status(500).json({
error: 'Failed signature validation',
});
}
// garbage collection can clean this up later
Challenge.delete(Env, txid, function (err) {
if (err) {
Env.Log.error("CHALLENGE_DELETION_ERROR", {
txid: txid,
error: Util.serializeError(err),
});
}
});
// execute the command
action(Env, json, function (err, content) {
if (err) {
Env.Log.error("CHALLENGE_RESPONSE_ACTION_ERROR", {
error: Util.serializeError(err),
});
return res.status(500).json({
error: 'Execution error',
errorCode: Util.serializeError(err)
});
}
res.status(200).json(content);
}, req, res);
});
};
module.exports.handle = function (Env, req, res /*, next */) {
var body = req.body;
// we expect that the client has posted some JSON data
if (!body) {
return void res.status(500).json({
error: 'invalid request',
});
}
// we only expect responses to challenges to have a 'txid' attribute
// further validation is performed in handleResponse
if (body.txid) {
return void handleResponse(Env, req, res);
}
// we only expect initial requests to have a 'command' attribute
// further validation is performed in handleCommand
if (body.command) {
return void handleCommand(Env, req, res);
}
// if a request is neither a command nor a response, then reject it with an error
res.status(500).json({
error: 'invalid request',
});
};

906
lib/http-worker.js
View File

@ -1,906 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const process = require("node:process");
const Http = require("node:http");
const Default = require("./defaults");
const Path = require("node:path");
const Fs = require("node:fs");
const nThen = require("nthen");
const Util = require("./common-util");
const Logger = require("./log");
const AuthCommands = require("./http-commands");
const MFA = require("./storage/mfa");
const Sessions = require("./storage/sessions");
const cookieParser = require("cookie-parser");
const bodyParser = require('body-parser');
const BlobStore = require("./storage/blob");
const BlockStore = require("./storage/block");
const plugins = require("./plugin-manager");
const gzipStatic = require('connect-gzip-static');
const CPCrypto = require('./crypto');
const DEFAULT_QUERY_TIMEOUT = 5000;
const PID = process.pid;
let SSOUtils = plugins.SSO && plugins.SSO.utils;
var Env = JSON.parse(process.env.Env);
let blobStore;
let cpcrypto;
Env.plugins = plugins;
const response = Util.response(function (errLabel, info) {
if (!Env.Log) { return; }
Env.Log.error(errLabel, info);
});
const guid = () => {
return Util.guid(response._pending);
};
const sendMessage = Env.sendMessage = (msg, cb, opt) => {
var txid = guid();
var timeout = (opt && opt.timeout) || DEFAULT_QUERY_TIMEOUT;
var obj = {
pid: PID,
txid: txid,
content: msg,
};
response.expect(txid, cb, timeout);
process.send(obj);
};
const Log = {};
Logger.levels.forEach(level => {
Log[level] = function (tag, info) {
sendMessage({
command: 'LOG',
level: level,
tag: tag,
info: info,
}, (err) => {
if (err) {
return void console.error(new Error(err));
}
});
};
});
Env.Log = Log;
Env.incrementBytesWritten = function () {};
const EVENTS = {};
EVENTS.ENV_UPDATE = function (data /*, cb */) {
try {
Env = JSON.parse(data);
Env.blobStore = blobStore;
Env.Log = Log;
Env.plugins = plugins;
Env.sendMessage = sendMessage;
Env.incrementBytesWritten = function () {};
} catch (err) {
Log.error('HTTP_WORKER_ENV_UPDATE', Util.serializeError(err));
}
};
EVENTS.FLUSH_CACHE = function (data) {
if (typeof(data) !== 'number') {
return Log.error('INVALID_FRESH_KEY', data);
}
Env.FRESH_KEY = data;
[ 'configCache', 'broadcastCache', ].forEach(key => {
Env[key] = {};
});
[ 'officeHeadersCache', 'standardHeadersCache', 'apiHeadersCache', ].forEach(key => {
Env[key] = undefined;
});
};
Object.keys(plugins || {}).forEach(name => {
let plugin = plugins[name];
if (!plugin.addHttpEvents) { return; }
try {
let events = plugin.addHttpEvents(Env);
Object.keys(events || {}).forEach(cmd => {
// Uppercase event name?
if (cmd !== cmd.toUpperCase()) { return; }
// Event is a function?
if (typeof(events[cmd]) !== "function") { return; }
// Event doesn't already exists?
if (EVENTS[cmd]) { return; }
EVENTS[cmd] = events[cmd];
});
} catch (e) {}
});
process.on('message', msg => {
if (!(msg && msg.txid)) { return; }
if (msg.type === 'REPLY') {
var txid = msg.txid;
return void response.handle(txid, [msg.error, msg.value]);
} else if (msg.type === 'EVENT') {
// response to event...
// ie. Update Env, flush cache, etc.
var ev = EVENTS[msg.command];
if (typeof(ev) === 'function') {
return void ev(msg.data, () => {});
}
}
//console.error("UNHANDLED_MESSAGE", msg);
});
var applyHeaderMap = function (res, map) {
for (let header in map) {
if (typeof(map[header]) === 'string') { res.setHeader(header, map[header]); }
}
};
var EXEMPT = [
/^\/common\/onlyoffice\/.*\.html.*/,
/^\/common\/onlyoffice\/dist\/.*\/sdkjs\/common\/spell\/spell\/spell.js.*/, // OnlyOffice loads spell.wasm in a way that needs unsave-eval
/^\/(sheet|presentation|doc)\/inner\.html.*/,
/^\/unsafeiframe\/inner\.html.*$/,
];
var cacheHeaders = function (Env, key, headers) {
if (Env.DEV_MODE) { return; }
Env[key] = headers;
};
var getHeaders = function (Env, type) {
var key = type + 'HeadersCache';
if (Env[key]) { return Util.clone(Env[key]); }
var headers = Default.httpHeaders(Env);
var csp;
if (type === 'office') {
csp = Default.padContentSecurity(Env);
} else {
csp = Default.contentSecurity(Env);
}
headers['Content-Security-Policy'] = csp;
headers["Cross-Origin-Resource-Policy"] = 'cross-origin';
headers["Cross-Origin-Embedder-Policy"] = 'require-corp';
cacheHeaders(Env, key, headers);
// Don't set CSP headers on /api/ endpoints
// because they aren't necessary and they cause problems
// when duplicated by NGINX in production environments
if (type === 'api') { delete headers['Content-Security-Policy']; }
return Util.clone(headers);
};
var setHeaders = function (req, res) {
var type;
if (EXEMPT.some(regex => regex.test(req.url))) {
type = 'office';
} else if (/^\/api\/(broadcast|config)/.test(req.url)) {
type = 'api';
} else {
type = 'standard';
}
var h = getHeaders(Env, type);
// Allow main domain to load resources from the sandbox URL
if (!Env.enableEmbedding && req.get('origin') === Env.httpUnsafeOrigin &&
/^\/common\/onlyoffice\/dist\/.*\/fonts\/.*/.test(req.url)) {
h['Access-Control-Allow-Origin'] = Env.httpUnsafeOrigin;
}
applyHeaderMap(res, h);
};
const Express = require("express");
Express.static.mime.define({'application/wasm': ['wasm']});
var app = Express();
app.use(bodyParser.urlencoded({
extended: true
}));
app.use(cookieParser());
(function () {
if (!Env.logFeedback) { return; }
const logFeedback = function (url) {
url.replace(/\?(.*?)=/, function (all, fb) {
Log.feedback(fb, '');
});
};
app.head(/^\/common\/feedback\.html/, function (req, res, next) {
logFeedback(req.url);
next();
});
}());
const { createProxyMiddleware } = require("http-proxy-middleware");
var httpAddress = Env.httpAddress === '::' ? 'localhost' : Env.httpAddress;
var proxyTarget = new URL('', `ws:${httpAddress}`);
proxyTarget.port = Env.websocketPort;
const wsProxy = createProxyMiddleware({
target: proxyTarget.href,
ws: true,
logLevel: 'error',
onProxyReqWs: function (proxyReq, req) {
proxyReq.setHeader('X-Real-Ip', req.socket.remoteAddress);
},
logProvider: (p) => {
p.error = (data) => {
if (/ECONNRESET/.test(data)) { return; }
Env.Log.error('HTTP_PROXY_MIDDLEWARE', data);
};
return p;
}
});
app.use('/cryptpad_websocket', wsProxy);
app.use('/ssoauth', (req, res, next) => {
if (SSOUtils && req && req.body && req.body.SAMLResponse) {
req.method = 'GET';
let token = Util.uid();
let smres = req.body.SAMLResponse;
return SSOUtils.writeRequest(Env, {
id: token,
type: 'saml',
content: smres
}, (err) => {
if (err) {
Log.error('E_SSO_WRITE_REQ', err);
return res.sendStatus(500);
}
let value = `samltoken="${token}"; SameSite=Strict; HttpOnly; Path=/; Secure`;
res.setHeader('Set-Cookie', value);
next();
});
}
next();
});
app.use('/blob', function (req, res, next) {
/* Head requests are used to check the size of a blob.
Clients can configure a maximum size to download automatically,
and can manually click to download blobs which exceed that limit. */
const url = req.url;
if (typeof(url) === "string" && Env.blobStore) {
const s = url.split('/');
if (s[1] && s[1].length === 2 && s[2] && s[2].length === Env.blobStore.BLOB_LENGTH) {
Env.blobStore.updateActivity(s[2], () => {});
}
}
if (req.method === 'HEAD') {
Express.static(Path.resolve(Env.paths.blob), {
setHeaders: function (res /*, path, stat */) {
res.set('Access-Control-Allow-Origin', Env.enableEmbedding? '*': Env.permittedEmbedders);
res.set('Access-Control-Allow-Headers', 'Content-Length');
res.set('Access-Control-Expose-Headers', 'Content-Length');
}
})(req, res, next);
return;
}
/* Some GET requests concern the whole file,
others only target ranges, either:
1. a two octet prefix which encodes the length of the metadata in octets
2. the metadata itself, excluding the two preceding octets
*/
/*
// Example code to demonstrate the types of requests which are handled
if (req.method === 'GET') {
if (!req.headers.range) {
// metadata
} else {
// full request
}
}
*/
next();
});
app.use(function (req, res, next) {
/* These are pre-flight requests, through which the client
confirms with the server that it is permitted to make the
actual requests which will follow */
if (req.method === 'OPTIONS' && /\/blob\//.test(req.url)) {
res.setHeader('Access-Control-Allow-Origin', Env.enableEmbedding? '*': Env.permittedEmbedders);
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range,Access-Control-Allow-Origin');
res.setHeader('Access-Control-Max-Age', 1728000);
res.setHeader('Content-Type', 'application/octet-stream; charset=utf-8');
res.setHeader('Content-Length', 0);
res.statusCode = 204;
return void res.end();
}
setHeaders(req, res);
if (/[\?\&]ver=[^\/]+$/.test(req.url)) { res.setHeader("Cache-Control", "max-age=31536000"); }
else { res.setHeader("Cache-Control", "no-cache"); }
next();
});
Object.keys(plugins || {}).forEach(name => {
let plugin = plugins[name];
if (!plugin.addHttpEndpoints) { return; }
plugin.addHttpEndpoints(Env, app);
});
// serve custom app content from the customize directory
// useful for testing pages customized with opengraph data
app.use(Express.static(Path.resolve('./customize/www')));
app.use(gzipStatic(Path.resolve('./www')));
app.use("/common", Express.static('./src/common'));
var mainPages = Env.mainPages || Default.mainPages();
var mainPagePattern = new RegExp('^\/(' + mainPages.join('|') + ').html$');
app.get(mainPagePattern, Express.static('./customize'));
app.get(mainPagePattern, Express.static('./customize.dist'));
app.use("/blob", Express.static(Path.resolve(Env.paths.blob), {
maxAge: Env.DEV_MODE? "0d": "365d"
}));
app.use("/datastore",
(req, res, next) => {
if (req.method === 'HEAD') {
next();
} else {
res.status(403).end();
}
},
Express.static(Env.paths.data, {
maxAge: "0d"
}
));
app.use('/block/', function (req, res, next) {
var parsed = Path.parse(req.url);
var name = parsed.name;
// block access control only applies to files
// identified by base64-encoded public keys
// skip everything else, ie. /block/placeholder.txt
if (/placeholder\.txt(\?.+)?/.test(parsed.base)) {
return void next();
}
if (typeof(name) !== 'string' || name.length !== 44) {
return void res.status(404).json({
error: "INVALID_ID",
});
}
var authorization = req.headers.authorization;
var mfa_params, sso_params;
nThen(function (w) {
// First, check whether the block id in question has any MFA settings stored
MFA.read(Env, name, w(function (err, content) {
// ENOENT means there are no settings configured
// it could be a 404 or an existing block without MFA protection
// in either case you can abort and fall through
// allowing the static webserver to handle either case
if (err && err.code === 'ENOENT') {
return;
}
// we're not expecting other errors. the sensible thing is to fail
// closed - meaning assume some protection is in place but that
// the settings couldn't be loaded for some reason. block access
// to the resource, logging for the admin and responding to the client
// with a vague error code
if (err) {
Log.error('GET_BLOCK_METADATA', err);
return void res.status(500).json({
code: 500,
error: "UNEXPECTED_ERROR",
});
}
// Otherwise, some settings were loaded correctly.
// We're expecting stringified JSON, so try to parse it.
// Log and respond with an error again if this fails.
// If it parses successfully then fall through to the next block.
try {
mfa_params = JSON.parse(content);
} catch (err2) {
w.abort();
Log.error("INVALID_BLOCK_METADATA", err2);
return res.status(500).json({
code: 500,
error: "UNEXPECTED_ERROR",
});
}
}));
// Same for SSO settings
if (!SSOUtils) { return; }
SSOUtils.readBlock(Env, name, w(function (err, content) {
if (err && (err.code === 'ENOENT' || err === 'ENOENT')) {
return;
}
if (err) {
Log.error('GET_BLOCK_METADATA', err);
return void res.status(500).json({
code: 500,
error: "UNEXPECTED_ERROR",
});
}
sso_params = content;
}));
}).nThen(function (w) {
if (!mfa_params && !sso_params) {
w.abort();
next();
}
}).nThen(function (w) {
// Block is protected with 2FA or SSO, make sure it still exists
const url = req.url;
if (typeof(url) !== "string") { return; }
const s = url.split('/');
const id = s[2];
if (!(s[1]?.length === 2 && BlockStore.isValidKey(id))) { return; }
BlockStore.isAvailable(Env, id, w((err, val) => {
if (err) { return; }
if (val !== false) { return; }
// Block doesn't exist, send the placeholder
w.abort();
return BlockStore.readPlaceholder(Env, id, reason => {
res.status(404).json({
reason,
code: 404
});
});
}));
}).nThen(function (w) {
// We should only be able to reach this logic
// if we successfully loaded and parsed some JSON
// representing the user's MFA and/or SSO settings.
// Failures at this point relate to insufficient or incorrect authorization.
// This function standardizes how we reject such requests.
// So far the only additional factor which is supported is TOTP.
// We specify what the method is to allow for future alternatives
// and inform the client so they can determine how to respond
// "401" means "Unauthorized"
var no = function () {
w.abort();
res.status(401).json({
sso: Boolean(sso_params),
method: mfa_params && mfa_params.method,
code: 401
});
};
// if you are here it is because this block is protected by MFA or SSO.
// they will need to provide a JSON Web Token, so we can reject them outright
// if one is not present in their authorization header
if (!authorization) { return void no(); }
// The authorization header should be of the form
// "Authorization: Bearer <SessionId>"
// We can reject the request if it is malformed.
let token = authorization.replace(/^Bearer\s+/, '').trim();
if (!token) { return void no(); }
Sessions.read(Env, name, token, function (err, contentStr) {
if (err) {
Log.error('SESSION_READ_ERROR', err);
return res.status(401).json({
sso: Boolean(sso_params),
method: mfa_params && mfa_params.method,
code: 401,
});
}
let content = Util.tryParse(contentStr);
if (mfa_params && !content.mfa) { return void no(); }
if (sso_params && !content.sso) { return void no(); }
if (content.mfa && content.mfa.exp && (+new Date()) > content.mfa.exp) {
Log.error("OTP_SESSION_EXPIRED", content.mfa);
Sessions.delete(Env, name, token, function (err) {
if (err) {
Log.error('SESSION_DELETE_EXPIRED_ERROR', err);
return;
}
Log.info('SESSION_DELETE_EXPIRED', err);
});
return void no();
}
if (content.sso && content.sso.exp && (+new Date()) > content.sso.exp) {
Log.error("SSO_SESSION_EXPIRED", content.sso);
Sessions.delete(Env, name, token, function (err) {
if (err) {
Log.error('SSO_SESSION_DELETE_EXPIRED_ERROR', err);
return;
}
Log.info('SSO_SESSION_DELETE_EXPIRED', err);
});
return void no();
}
// Interpret the existence of a file in that location as the continued
// validity of the session. Fall through and let the built-in webserver
// handle the 404 or serving the file.
next();
});
});
});
// TODO this would be a good place to update a block's atime
// in a manner independent of the filesystem. ie. for detecting and archiving
// inactive accounts in a way that will not be invalidated by other forms of access
// like filesystem backups.
app.use("/block", Express.static(Path.resolve(Env.paths.block), {
maxAge: "0d",
}));
// In case of a 404 for the block, check if a placeholder exists
// and provide the result if that's the case
app.use("/block", (req, res, next) => {
const url = req.url;
if (typeof(url) === "string") {
const s = url.split('/');
if (s[1] && s[1].length === 2 && BlockStore.isValidKey(s[2])) {
return BlockStore.readPlaceholder(Env, s[2], (content) => {
res.status(404).json({
reason: content,
code: 404
});
});
}
}
next();
});
app.use("/customize", Express.static('customize'));
app.use("/customize", Express.static('customize.dist'));
app.use("/customize.dist", Express.static('customize.dist'));
app.use(/^\/[^\/]*$/, Express.static('customize'));
app.use(/^\/[^\/]*$/, Express.static('customize.dist'));
// if dev mode: never cache
var cacheString = function () {
return (Env.FRESH_KEY? '-' + Env.FRESH_KEY: '') + (Env.DEV_MODE? '-' + (+new Date()): '');
};
var makeRouteCache = function (template, cacheName) {
var cleanUp = {};
return function (req, res) {
var cache = Env[cacheName] = Env[cacheName] || {};
var host = req.headers.host.replace(/\:[0-9]+/, '');
res.setHeader('Content-Type', 'text/javascript');
// don't cache anything if you're in dev mode
if (Env.DEV_MODE) {
return void res.send(template(host));
}
// generate a lookup key for the cache
var cacheKey = host + ':' + cacheString();
// FIXME mutable
// we must be able to clear the cache when updating any mutable key
// if there's nothing cached for that key...
if (!cache[cacheKey]) {
// generate the response and cache it in memory
cache[cacheKey] = template(host);
// and create a function to conditionally evict cache entries
// which have not been accessed in the last 20 seconds
cleanUp[cacheKey] = Util.throttle(function () {
delete cleanUp[cacheKey];
delete cache[cacheKey];
}, 20000);
}
// successive calls to this function
if (typeof (cleanUp[cacheKey]) === "function") {
cleanUp[cacheKey]();
}
return void res.send(cache[cacheKey]);
};
};
var serveConfig = makeRouteCache(function () {
// NOTE: we may extract JSON from this config using slice(27, -5)
const ssoList = Env.sso && Array.isArray(Env.sso.list) &&
Env.sso.list.map(function (obj) { return obj.name; }) || [];
let forceSso = Env?.sso?.enforced ?? true;
let allowPw = Env?.sso?.cpPassword ?? true;
let forcePw = Env?.sso?.forceCpPassword ?? true;
const ssoCfg = (Env?.sso?.enabled) ? {
enabled: Env?.sso?.enabled,
force: forceSso ? 1 : 0,
password: allowPw && (forcePw ? 2 : 1) || 0,
list: ssoList
} : false;
return [
'define(function(){',
'return ' + JSON.stringify({
requireConf: {
waitSeconds: 600,
urlArgs: 'ver=' + Env.version + cacheString(),
},
removeDonateButton: (Env.removeDonateButton === true),
accounts_api: Env.accounts_api,
websocketPath: Env.websocketPath,
httpUnsafeOrigin: Env.httpUnsafeOrigin,
adminEmail: Env.adminEmail,
adminKeys: Env.admins,
moderatorKeys: Env.moderators,
inactiveTime: Env.inactiveTime,
supportMailbox: Env.supportMailbox,
supportMailboxKey: Env.supportMailboxKey,
defaultStorageLimit: Env.defaultStorageLimit,
maxUploadSize: Env.maxUploadSize,
premiumUploadSize: Env.premiumUploadSize,
restrictRegistration: Env.restrictRegistration,
appsToDisable: Env.appsToDisable,
restrictSsoRegistration: Env.restrictSsoRegistration,
httpSafeOrigin: Env.httpSafeOrigin,
enableEmbedding: Env.enableEmbedding,
fileHost: Env.fileHost,
shouldUpdateNode: Env.shouldUpdateNode || undefined,
listMyInstance: Env.listMyInstance,
sso: ssoCfg,
enforceMFA: Env.enforceMFA,
onlyOffice: Env.onlyOffice
}, null, '\t'),
'});'
].join(';\n');
}, 'configCache');
var serveBroadcast = makeRouteCache(function () {
var maintenance = Env.maintenance;
if (maintenance && maintenance.end && maintenance.end < (+new Date())) {
maintenance = undefined;
}
return [
'define(function(){',
'return ' + JSON.stringify({
curvePublic: Env.curvePublic,
lastBroadcastHash: Env.lastBroadcastHash,
surveyURL: Env.surveyURL,
maintenance: maintenance
}, null, '\t'),
'});'
].join(';\n');
}, 'broadcastCache');
app.get('/api/config', serveConfig);
app.get('/api/broadcast', serveBroadcast);
(function () {
let extensions = plugins._extensions;
let styles = plugins._styles;
let str = JSON.stringify(extensions);
let str2 = JSON.stringify(styles);
let js = `let extensions = ${str};
let styles = ${str2};
let lang = window.cryptpadLanguage;
let paths = [];
extensions.forEach(name => {
paths.push(\`optional!/\${name}/extensions.js\`);
paths.push(\`optional!json!/\${name}/translations/messages.json\`);
const l = lang === "en" ? '' : \`\${lang}.\`;
paths.push(\`optional!json!/\${name}/translations/messages.\${l}json\`);
});
styles.forEach(name => {
paths.push(\`optional!less!/\${name}/style.less\`);
});
define(paths, function () {
let args = Array.prototype.slice.apply(arguments);
return args;
}, function () {
// ignore missing files
});`;
app.get('/extensions.js', (req, res) => {
res.setHeader('Content-Type', 'text/javascript');
res.send(js);
});
})();
var Define = function (obj) {
return `define(function (){
return ${JSON.stringify(obj, null, '\t')};
});`;
};
app.get('/api/instance', function (req, res) {
res.setHeader('Content-Type', 'text/javascript');
res.send(Define({
color: Env.accentColor,
name: Env.instanceName,
description: Env.instanceDescription,
location: Env.instanceJurisdiction,
notice: Env.instanceNotice,
}));
});
var four04_path = Path.resolve('./customize.dist/404.html');
var fivehundred_path = Path.resolve('./customize.dist/500.html');
var custom_four04_path = Path.resolve('./customize/404.html');
var custom_fivehundred_path = Path.resolve('./customize/500.html');
var send404 = function (res, path) {
if (!path && path !== four04_path) { path = four04_path; }
Fs.exists(path, function (exists) {
res.setHeader('Content-Type', 'text/html; charset=utf-8');
if (exists) { return Fs.createReadStream(path).pipe(res); }
send404(res);
});
};
var send500 = function (res, path) {
if (!path && path !== fivehundred_path) { path = fivehundred_path; }
Fs.exists(path, function (exists) {
res.setHeader('Content-Type', 'text/html; charset=utf-8');
if (exists) { return Fs.createReadStream(path).pipe(res); }
send500(res);
});
};
app.get('/api/profiling', function (req, res) {
if (!Env.enableProfiling) { return void send404(res); }
sendMessage({
command: 'GET_PROFILING_DATA',
}, (err, value) => {
if (err) {
res.status(500);
return void send500(res);
}
res.setHeader('Content-Type', 'text/javascript');
res.send(JSON.stringify({
bytesWritten: value,
}));
});
});
app.get('/api/logo', function (req, res) {
let path = Path.resolve('./customize/CryptPad_logo_hero.svg');
let base = Path.resolve('./customize.dist/CryptPad_logo_hero.svg');
Fs.exists(path, function (exists) {
res.setHeader('Content-Disposition', 'inline');
if (exists) {
let mime = Env.logoMimeType || 'image/svg+xml';
res.setHeader('Content-Type', mime);
return res.sendFile(path);
}
res.sendFile(base);
});
});
app.use('/upload-blob', Express.json({limit:"500kb"}), (req, res) => {
if (req.method !== "POST") {
return res.status(403).send();
}
const { chunk, sig, edPublic } = req.body;
if (!cpcrypto) {
return void res.status(500).send({error: 'NOCRYPTO'});
}
const forbidden = reason => {
return void res.status(403).send({error: reason});
};
try {
// Check signature
const sigu8 = Util.decodeBase64(sig);
const vkey = Util.decodeBase64(edPublic);
const ok = cpcrypto.open(sigu8, vkey);
if (!ok) { return forbidden('INVALID_KEY'); }
const cookie = Util.encodeUTF8(sigu8.subarray(64));
// Check cookie
const safeKey = Util.escapeKeyCharacters(edPublic);
Env.blobStore.checkUploadCookie(safeKey, value => {
if (value !== cookie) {
return forbidden('INVALID_COOKIE');
}
// Upload chunk
Env.blobStore.upload(safeKey, chunk, (err) => {
if (err) {
return res.status(500).send({error: err});
}
// Get new cookie
Env.blobStore.uploadCookie(safeKey, (err, _c) => {
if (err) {
return res.status(500).send({error: err});
}
res.status(200).send({
cookie: _c
});
});
});
});
} catch (e) {
return void res.status(500).send({error: e.message});
}
});
// This endpoint handles authenticated RPCs over HTTP
// via an interactive challenge-response protocol
app.use(Express.json());
app.post('/api/auth', function (req, res, next) {
AuthCommands.handle(Env, req, res, next);
});
app.use(function (req, res /*, next */) {
if (/^(\/favicon\.ico\/|.*\.js\.map|.*\/translations\/.*\.json)/.test(req.url)) {
// ignore common 404s
} else {
Log.info('HTTP_404', req.url);
}
res.status(404);
send404(res, custom_four04_path);
});
// default message for thrown errors in ExpressJS routes
app.use(function (err, req, res /*, next*/) {
Log.error('EXPRESSJS_ROUTING', {
error: err.stack || err,
});
res.status(500);
send500(res, custom_fivehundred_path);
});
var server = Http.createServer(app);
nThen(function (w) {
server.listen(Env.httpPort, Env.httpAddress, w());
if (Env.httpSafePort) {
let safeServer = Http.createServer(app);
safeServer.listen(Env.httpSafePort, Env.httpAddress, w());
}
server.on('upgrade', function (req, socket, head) {
// TODO warn admins that websockets should only be proxied in this way in a dev environment
// in production it's more efficient to have your reverse proxy (NGINX) directly forward
// websocket traffic to the correct port (Env.websocketPort)
wsProxy.upgrade(req, socket, head);
});
var config = require("./load-config");
BlobStore.create({
blobPath: config.blobPath,
blobStagingPath: config.blobStagingPath,
archivePath: config.archivePath,
getSession: function () {},
}, w(function (err, blob) {
if (err) { return; }
Env.blobStore = blobStore = blob;
}));
CPCrypto.init(w(function (err, crypto) {
cpcrypto = crypto;
}));
}).nThen(function () {
// TODO inform the parent process that this worker is ready
Object.keys(Env.plugins || {}).forEach(name => {
let plugin = plugins[name];
if (!plugin.initialize) { return; }
try { plugin.initialize(Env, "http-worker"); }
catch (e) {}
});
});
process.on('uncaughtException', function (err) {
console.error('[%s] UNCAUGHT EXCEPTION IN HTTP WORKER', new Date());
console.error(err);
console.error("TERMINATING");
process.exit(1);
});

5
lib/keys.js
View File

@ -1,5 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
module.exports = require("../src/common/common-signing-keys");

112
lib/log.js
View File

@ -1,112 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Store = require("./storage/file");
var Util = require("./common-util");
var Logger = module.exports;
/* Every line in the log should contain:
* timestamp
* public key of initiator
* the action
* the event's tag
*/
var messageTemplate = function (type, time, tag, info) {
return JSON.stringify([type.toUpperCase(), time, tag, info]);
};
var noop = function () {};
var write = function (ctx, content, cb) {
if (typeof(cb) !== "function") { cb = noop; }
if (!ctx.store) {
cb = Util.mkAsync(cb);
return void cb();
}
ctx.store.log(ctx.channelName, content, cb);
};
// various degrees of logging
const logLevels = Logger.levels = ['silly', 'verbose', 'debug', 'feedback', 'info', 'warn', 'error'];
var handlers = {};
['silly', 'debug', 'verbose', 'feedback', 'info'].forEach(function (level) {
handlers[level] = function (ctx, content) { console.log(content); };
});
['warn', 'error'].forEach(function (level) {
handlers[level] = function (ctx, content) { console.error(content); };
});
var createLogType = function (ctx, type) {
if (logLevels.indexOf(type) < logLevels.indexOf(ctx.logLevel)) {
return noop;
}
return function (tag, info, cb) {
if (ctx.shutdown) {
throw new Error("Logger has been shut down!");
}
var time = new Date().toISOString();
var content;
try {
content = messageTemplate(type, time, tag, info);
} catch (e) {
return;
}
if (ctx.logToStdout && typeof(handlers[type]) === 'function') {
handlers[type](ctx, content);
}
write(ctx, content, cb);
};
};
var createMethods = function (ctx) {
var log = {};
logLevels.forEach(function (type) {
log[type] = createLogType(ctx, type);
});
return log;
};
Logger.create = function (config, cb) {
if (typeof(config.logLevel) !== 'string') {
config.logLevel = 'info';
}
var date = new Date();
var launchTime = ('' + date.getUTCFullYear()).slice(-2) + date.toISOString();
var ctx = {
channelName: launchTime,
logFeedback: Boolean(config.logFeedback),
logLevel: config.logLevel,
logToStdout: config.logToStdout,
};
if (!config.logPath) {
console.log("No logPath configured. Logging to file disabled");
var logger = createMethods(ctx);
logger.shutdown = noop;
return void cb(Object.freeze(logger));
}
Store.create({
filePath: config.logPath,
archivePath: config.archivePath,
}, function (err, store) {
if (err) {
throw err;
}
ctx.store = store;
var logger = createMethods(ctx);
logger.shutdown = function () {
delete ctx.store;
ctx.shutdown = true;
store.shutdown();
};
cb(Object.freeze(logger));
});
};

485
lib/metadata.js
View File

@ -1,485 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Meta = module.exports;
var Core = require("./commands/core");
var deduplicate = require("./common-util").deduplicateString;
/* Metadata fields and the commands that can modify them
we assume that these commands can only be performed
by owners or in some cases pending owners. Thus
the owners field is guaranteed to exist.
* channel <STRING>
* validateKey <STRING>
* owners <ARRAY>
* ADD_OWNERS
* RM_OWNERS
* RESET_OWNERS
* pending_owners <ARRAY>
* ADD_PENDING_OWNERS
* RM_PENDING_OWNERS
* expire <NUMBER>
* UPDATE_EXPIRATION (NOT_IMPLEMENTED)
* restricted <BOOLEAN>
* RESTRICT_ACCESS
* allowed <ARRAY>
* ADD_ALLOWED
* RM_ALLOWED
* RESET_ALLOWED
* ADD_OWNERS
* RESET_OWNERS
* mailbox <STRING|MAP>
* ADD_MAILBOX
* RM_MAILBOX
* deleteLines <BOOLEAN>
* ALLOW_LINE_DELETION
*/
var commands = {};
var isValidPublicKey = Core.isValidPublicKey;
// isValidPublicKey is a better indication of what the above function does
// I'm preserving this function name in case we ever want to expand its
// criteria at a later time...
var isValidOwner = isValidPublicKey;
// ["RESTRICT_ACCESS", [true], 1561623438989]
// ["RESTRICT_ACCESS", [false], 1561623438989]
commands.RESTRICT_ACCESS = function (meta, args) {
if (!Array.isArray(args) || typeof(args[0]) !== 'boolean') {
throw new Error('INVALID_STATE');
}
var bool = args[0];
// reject the proposed command if there is no change in state
if (meta.restricted === bool) { return false; }
// apply the new state
meta.restricted = args[0];
// if you're disabling access restrictions then you can assume
// then there is nothing more to do. Leave the existing list as-is
if (!bool) { return true; }
// you're all set if an allow list already exists
if (Array.isArray(meta.allowed)) { return true; }
// otherwise define it
meta.allowed = [];
return true;
};
// ["ALLOW_LINE_DELETION", [true], 1561623438989]
// ["ALLOW_LINE_DELETION", [false], 1561623438989]
commands.ALLOW_LINE_DELETION = function (meta, args) {
if (!Array.isArray(args) || typeof(args[0]) !== 'boolean') {
throw new Error('INVALID_STATE');
}
var bool = args[0];
// reject the proposed command if there is no change in state
if (meta.deleteLines === bool) { return false; }
// apply the new state
meta.deleteLines = args[0];
return true;
};
// ["ADD_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989]
commands.ADD_ALLOWED = function (meta, args) {
if (!Array.isArray(args)) {
throw new Error("INVALID_ARGS");
}
var allowed = meta.allowed || [];
var changed = false;
args.forEach(function (arg) {
// don't add invalid public keys
if (!isValidPublicKey(arg)) { return; }
// don't add owners to the allow list
if (meta.owners.indexOf(arg) >= 0) { return; }
// don't duplicate entries in the allow list
if (allowed.indexOf(arg) >= 0) { return; }
allowed.push(arg);
changed = true;
});
if (changed) {
meta.allowed = meta.allowed || allowed;
}
return changed;
};
// ["RM_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989]
commands.RM_ALLOWED = function (meta, args) {
if (!Array.isArray(args)) {
throw new Error("INVALID_ARGS");
}
// there may not be anything to remove
if (!meta.allowed) { return false; }
var changed = false;
args.forEach(function (arg) {
var index = meta.allowed.indexOf(arg);
if (index < 0) { return; }
meta.allowed.splice(index, 1);
changed = true;
});
return changed;
};
var arrayHasChanged = function (A, B) {
var changed;
A.some(function (a) {
if (B.indexOf(a) < 0) { return (changed = true); }
});
if (changed) { return true; }
B.some(function (b) {
if (A.indexOf(b) < 0) { return (changed = true); }
});
return changed;
};
var filterInPlace = function (A, f) {
for (var i = A.length - 1; i >= 0; i--) {
if (f(A[i], i, A)) { A.splice(i, 1); }
}
};
// ["RESET_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989]
commands.RESET_ALLOWED = function (meta, args) {
if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); }
var updated = args.filter(function (arg) {
// don't allow invalid public keys
if (!isValidPublicKey(arg)) { return false; }
// don't ever add owners to the allow list
if (meta.owners.indexOf(arg)) { return false; }
return true;
});
// this is strictly an optimization...
// a change in length is a clear indicator of a functional change
if (meta.allowed && meta.allowed.length !== updated.length) {
meta.allowed = updated;
return true;
}
// otherwise we must check that the arrays contain distinct elements
// if there is no functional change, then return false
if (!arrayHasChanged(meta.allowed, updated)) { return false; }
// otherwise overwrite the in-memory data and indicate that there was a change
meta.allowed = updated;
return true;
};
// ["ADD_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623438989]
commands.ADD_OWNERS = function (meta, args) {
// bail out if args isn't an array
if (!Array.isArray(args)) {
throw new Error('METADATA_INVALID_OWNERS');
}
// you shouldn't be able to get here if there are no owners
// because only an owner should be able to change the owners
if (!Array.isArray(meta.owners)) {
throw new Error("METADATA_NONSENSE_OWNERS");
}
var changed = false;
args.forEach(function (owner) {
if (!isValidOwner(owner)) { return; }
if (meta.owners.indexOf(owner) >= 0) { return; }
meta.owners.push(owner);
changed = true;
});
if (changed && Array.isArray(meta.allowed)) {
// make sure owners are not included in the allow list
filterInPlace(meta.allowed, function (member) {
return meta.owners.indexOf(member) !== -1;
});
}
return changed;
};
// ["RM_OWNERS", ["CrufexqXcY-z+eKJlEbNELVy5Sb7E-EAAEFI8GnEtZ0="], 1561623439989]
commands.RM_OWNERS = function (meta, args) {
// what are you doing if you don't have owners to remove?
if (!Array.isArray(args)) {
throw new Error('METADATA_INVALID_OWNERS');
}
// if there aren't any owners to start, this is also pointless
if (!Array.isArray(meta.owners)) {
throw new Error("METADATA_NONSENSE_OWNERS");
}
var changed = false;
// remove owners one by one
// we assume there are no duplicates
args.forEach(function (owner) {
var index = meta.owners.indexOf(owner);
if (index < 0) { return; }
if (meta.mailbox) {
if (typeof(meta.mailbox) === "string") {
delete meta.mailbox;
} else {
delete meta.mailbox[owner];
}
}
meta.owners.splice(index, 1);
changed = true;
});
if (meta.owners.length === 0 && meta.restricted) {
meta.restricted = false;
}
return changed;
};
// ["ADD_PENDING_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623438989]
commands.ADD_PENDING_OWNERS = function (meta, args) {
// bail out if args isn't an array
if (!Array.isArray(args)) {
throw new Error('METADATA_INVALID_PENDING_OWNERS');
}
// you shouldn't be able to get here if there are no owners
// because only an owner should be able to change the owners
if (meta.pending_owners && !Array.isArray(meta.pending_owners)) {
throw new Error("METADATA_NONSENSE_PENDING_OWNERS");
}
var changed = false;
// Add pending_owners array if it doesn't exist
if (!meta.pending_owners) {
meta.pending_owners = deduplicate(args);
return true;
}
// or fill it
args.forEach(function (owner) {
if (!isValidOwner(owner)) { return; }
if (meta.pending_owners.indexOf(owner) >= 0) { return; }
meta.pending_owners.push(owner);
changed = true;
});
return changed;
};
// ["RM_PENDING_OWNERS", ["CrufexqXcY-z+eKJlEbNELVy5Sb7E-EAAEFI8GnEtZ0="], 1561623439989]
commands.RM_PENDING_OWNERS = function (meta, args) {
// what are you doing if you don't have owners to remove?
if (!Array.isArray(args)) {
throw new Error('METADATA_INVALID_PENDING_OWNERS');
}
// if there aren't any owners to start, this is also pointless
if (!Array.isArray(meta.pending_owners)) {
throw new Error("METADATA_NONSENSE_PENDING_OWNERS");
}
var changed = false;
// remove owners one by one
// we assume there are no duplicates
args.forEach(function (owner) {
var index = meta.pending_owners.indexOf(owner);
if (index < 0) { return; }
meta.pending_owners.splice(index, 1);
changed = true;
});
return changed;
};
// ["RESET_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623439989]
commands.RESET_OWNERS = function (meta, args) {
// expect a new array, even if it's empty
if (!Array.isArray(args)) {
throw new Error('METADATA_INVALID_OWNERS');
}
// assume there are owners to start
if (!Array.isArray(meta.owners)) {
throw new Error("METADATA_NONSENSE_OWNERS");
}
// overwrite the existing owners with the new one
meta.owners = deduplicate(args.filter(isValidOwner));
if (Array.isArray(meta.allowed)) {
// make sure owners are not included in the allow list
filterInPlace(meta.allowed, function (member) {
return meta.owners.indexOf(member) !== -1;
});
}
if (meta.owners.length === 0 && meta.restricted) {
meta.restricted = false;
}
return true;
};
// ["SET_LINKED", "9fb7485b8e824d07489ed5586d848232", 1561623439989]
commands.SET_LINKED = function (meta, linked) {
if (typeof(linked) !== "string") {
throw new Error('METADATA_INVALID_LINKED');
}
// reject the proposed command if there is no change in state
if (meta.linked === linked) { return false; }
// apply the new state
meta.linked = linked;
return true;
};
// ["ADD_MAILBOX", {"7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=": mailbox, ...}, 1561623439989]
commands.ADD_MAILBOX = function (meta, args) {
// expect a new array, even if it's empty
if (!args || typeof(args) !== "object") {
throw new Error('METADATA_INVALID_MAILBOX');
}
// assume there are owners to start
if (!Array.isArray(meta.owners)) {
throw new Error("METADATA_NONSENSE_OWNERS");
}
var changed = false;
// For each mailbox we try to add, check if the associated edPublic is an owner
// If they are, add or replace the mailbox
Object.keys(args).forEach(function (edPublic) {
if (meta.owners.indexOf(edPublic) === -1) { return; }
if (typeof(meta.mailbox) === "string") {
var str = meta.mailbox;
meta.mailbox = {};
meta.mailbox[meta.owners[0]] = str;
}
// Make sure mailbox is defined
if (!meta.mailbox) { meta.mailbox = {}; }
meta.mailbox[edPublic] = args[edPublic];
changed = true;
});
return changed;
};
commands.RM_MAILBOX = function (meta, args) {
if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); }
if (!meta.mailbox || typeof(meta.mailbox) === 'undefined') {
return false;
}
if (typeof(meta.mailbox) === 'string' && args.length === 0) {
delete meta.mailbox;
return true;
}
var changed = false;
args.forEach(function (arg) {
if (meta.mailbox[arg] === 'undefined') { return; }
delete meta.mailbox[arg];
changed = true;
});
return changed;
};
commands.UPDATE_EXPIRATION = function () {
throw new Error("E_NOT_IMPLEMENTED");
};
var handleCommand = Meta.handleCommand = function (meta, line) {
var command = line[0];
var args = line[1];
//var time = line[2];
if (typeof(commands[command]) !== 'function') {
throw new Error("METADATA_UNSUPPORTED_COMMAND");
}
return commands[command](meta, args);
};
Meta.commands = Object.keys(commands);
Meta.createLineHandler = function (ref, errorHandler) {
ref.meta = {};
ref.index = 0;
ref.logged = {};
var overwritten = false;
return function (err, line) {
if (err) {
// it's not abnormal that metadata exists without a corresponding log
// so ENOENT is fine
if (ref.index === 0 && err.code === 'ENOENT') { return; }
// any other errors are abnormal
return void errorHandler('METADATA_HANDLER_LINE_ERR', {
error: err,
index: ref.index,
line: JSON.stringify(line),
});
}
// the case above is special, everything else should increment the index
var index = ref.index++;
if (typeof(line) === 'undefined') { return; }
if (Array.isArray(line)) {
try {
handleCommand(ref.meta, line);
} catch (err2) {
var code = err2.message;
if (ref.logged[code]) { return; }
ref.logged[code] = true;
errorHandler("METADATA_COMMAND_ERR", {
error: err2.stack,
line: line,
});
}
return;
}
// the first line of a channel is processed before the dedicated metadata log.
// it can contain a map, in which case it should be used as the initial state.
// it's possible that a trim-history command was interrupted, in which case
// this first message might exist in parallel with the more recent metadata log
// which will contain the computed state of the previous metadata log
// which has since been archived.
// Thus, accept both the first and second lines you process as valid initial state
// preferring the second if it exists
if (index < 2 && line && typeof(line) === 'object') {
if (overwritten) { return; } // hack to avoid overwriting metadata a second time
overwritten = true;
// special case!
ref.meta = line;
return;
}
errorHandler("METADATA_HANDLER_WEIRDLINE", {
line: line,
index: index,
});
};
};

333
lib/pins.js
View File

@ -1,333 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Pins = module.exports;
const Fs = require("fs");
const Path = require("path");
const Util = require("./common-util");
const Plan = require("./plan");
const Store = require('./storage/file');
const Semaphore = require('saferphore');
const nThen = require('nthen');
/* Accepts a reference to an object, and...
either a string describing which log is being processed (backwards compatibility),
or a function which will log the error with all relevant data
*/
var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
var fileName;
if (typeof(errorHandler) === 'string') {
fileName = errorHandler;
errorHandler = function (label, data) {
console.error(label, {
log: fileName,
data: data,
});
};
}
// passing the reference to an object allows us to overwrite accumulated pins
// make sure to get ref.pins as the result
// it's a weird API but it's faster than unpinning manually
var pins = ref.pins = {};
ref.index = 0;
ref.first = 0;
ref.latest = 0; // the latest message (timestamp in ms)
ref.surplus = 0; // how many lines exist behind a reset
// Extract metadata from the channel list (#block, #drive)
let sanitize = (id, isPin) => {
if (typeof(id) !== "string") { return; }
let idx = id.indexOf('#');
if (idx < 0) { return id; }
let type = id.slice(idx+1);
let sanitized = id.slice(0, idx);
if (!isPin) { return sanitized; }
if (type === 'block') { // Note: teams don't have a block
ref.block = sanitized;
return;
}
if (type === 'drive') {
ref.drive = sanitized;
return sanitized;
}
return sanitized;
};
return function (line, i) {
ref.index++;
if (!Boolean(line)) { return; }
var l;
try {
l = JSON.parse(line);
} catch (e) {
return void errorHandler('PIN_LINE_PARSE_ERROR', line);
}
if (!Array.isArray(l)) {
return void errorHandler('PIN_LINE_NOT_FORMAT_ERROR', l);
}
if (typeof(l[2]) === 'number') {
if (!ref.first) { ref.first = l[2]; }
ref.latest = l[2]; // date
}
switch (l[0]) {
case 'RESET': {
pins = ref.pins = {};
if (l[1] && l[1].length) {
l[1].forEach((x) => {
x = sanitize(x, true);
if (!x) { return; }
ref.pins[x] = 1;
});
}
ref.surplus = ref.index;
// fallthrough
}
case 'PIN': {
l[1].forEach((x) => {
x = sanitize(x, true);
if (!x) { return; }
pins[x] = 1;
});
break;
}
case 'UNPIN': {
l[1].forEach((x) => {
x = sanitize(x, false);
if (!x) { return; }
delete pins[x];
});
break;
}
default:
errorHandler("PIN_LINE_UNSUPPORTED_COMMAND", l);
}
if (i === 0) { // First line when using Pins.load
if (l[0] === 'PIN' || ref.block) { ref.user = true; } // teams always start with RESET
}
};
};
var processPinFile = function (pinFile, fileName) {
var ref = {};
var handler = createLineHandler(ref, fileName);
pinFile.split('\n').forEach(handler);
return ref;
};
/*
takes contents of a pinFile (UTF8 string)
and the pin file's name
returns an array of of channel ids which are pinned
throw errors on pin logs with invalid pin data
*/
Pins.calculateFromLog = function (pinFile, fileName) {
var ref = processPinFile(pinFile, fileName);
return Object.keys(ref.pins);
};
/*
pins/
pins/A+/
pins/A+/A+hyhrQLrgYixOomZYxpuEhwfiVzKk1bBp+arH-zbgo=.ndjson
*/
const getSafeKeyFromPath = function (path) {
return path.replace(/^.*\//, '').replace(/\.ndjson/, '');
};
const addUserPinToState = Pins.addUserPinToState = function (state, safeKey, itemId) {
(state[itemId] = state[itemId] || {})[safeKey] = 1;
};
Pins.list = function (_done, config) {
// allow for a configurable pin store location
const pinPath = config.pinPath || './data/pins';
// allow for a configurable amount of parallelism
const plan = Plan(config.workers || 5);
// run a supplied handler whenever you finish reading a log
// or noop if not supplied.
const handler = config.handler || function () {};
// use and mutate a supplied object for state if it's passed
const pinned = config.pinned || {};
var isDone = false;
// ensure that 'done' is only called once
// that it calls back asynchronously
// and that it sets 'isDone' to true, so that pending processes
// know to abort
const done = Util.once(Util.both(Util.mkAsync(_done), function () {
isDone = true;
}));
const errorHandler = function (label, info) {
console.log(label, info);
};
// TODO replace this with lib-readline?
const streamFile = function (path, cb) {
const id = getSafeKeyFromPath(path);
return void Fs.readFile(path, 'utf8', function (err, body) {
if (err) { return void cb(err); }
const ref = {};
const pinHandler = createLineHandler(ref, errorHandler);
var lines = body.split('\n');
lines.forEach(pinHandler);
handler(ref, id, pinned);
cb(void 0, ref);
});
};
const scanDirectory = function (path, cb) {
Fs.readdir(path, function (err, list) {
if (err) {
return void cb(err);
}
cb(void 0, list.map(function (item) {
return {
path: Path.join(path, item),
id: item.replace(/\.ndjson$/, ''),
};
}));
});
};
scanDirectory(pinPath, function (err, dirs) {
if (err) {
if (err.code === 'ENOENT') { return void done(void 0, {}); }
return void done(err);
}
dirs.forEach(function (dir) {
plan.job(1, function (next) {
if (isDone) { return void next(); }
scanDirectory(dir.path, function (nested_err, logs) {
if (nested_err) {
return void done(err);
}
logs.forEach(function (log) {
if (!/\.ndjson$/.test(log.path)) { return; }
plan.job(0, function (next) {
if (isDone) { return void next(); }
streamFile(log.path, function (err, ref) {
if (err) { return void done(err); }
var set = ref.pins;
for (var item in set) {
addUserPinToState(pinned, log.id, item);
}
next();
});
});
});
next();
});
});
});
plan.done(function () {
// err ?
done(void 0, pinned);
}).start();
});
};
Pins.load = function (cb, config) {
const sema = Semaphore.create(config.workers || 5);
let dirList;
const fileList = [];
const pinned = {};
var pinPath = config.pinPath || './pins';
var done = Util.once(cb);
var handler = config.handler;
let store;
nThen((waitFor) => {
Store.create({
filePath: config.pinPath,
volumeId: 'pins'
}, waitFor((err, _) => {
if (err) {
waitFor.abort();
return void done(err);
}
store = _;
}));
}).nThen((waitFor) => {
// recurse over the configured pinPath, or the default
Fs.readdir(pinPath, waitFor((err, list) => {
if (err) {
if (err.code === 'ENOENT') {
dirList = [];
return; // this ends up calling back with an empty object
}
waitFor.abort();
return void done(err);
}
dirList = list;
}));
}).nThen((waitFor) => {
dirList.forEach((f) => {
sema.take((returnAfter) => {
// iterate over all the subdirectories in the pin store
Fs.readdir(Path.join(pinPath, f), waitFor(returnAfter((err, list2) => {
if (err) {
waitFor.abort();
return void done(err);
}
list2.forEach((ff) => {
if (config && config.exclude && config.exclude.indexOf(ff) > -1) { return; }
fileList.push(ff.replace(/(\.ndjson)$/, ''));
});
})));
});
});
}).nThen((waitFor) => {
fileList.forEach((id) => {
sema.take((returnAfter) => {
var next = waitFor(returnAfter());
var ref = {};
var h = createLineHandler(ref, id);
store.readMessagesBin(id, 0, (msgObj, next) => {
h(msgObj.buff.toString('utf8'));
next();
}, (err) => {
if (err) {
waitFor.abort();
return void done(err);
}
if (handler) {
return void handler(ref, id, next);
}
const hashes = Object.keys(ref.pins);
hashes.forEach((x) => {
(pinned[x] = pinned[x] || {})[id] = 1;
});
next();
});
});
});
}).nThen(() => {
done(void 0, pinned);
});
};

239
lib/plan.js
View File

@ -1,239 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
/*
There are many situations where we want to do lots of little jobs
in parallel and with few constraints as to their ordering.
One example is recursing over a bunch of directories and reading files.
The naive way to do this is to recurse over all the subdirectories
relative to a root while adding files to a list. Then to iterate over
the files in that list. Unfortunately, this means holding the complete
list of file paths in memory, which can't possible scale as our database grows.
A better way to do this is to recurse into one directory and
iterate over its contents until there are no more, then to backtrack
to the next directory and repeat until no more directories exist.
This kind of thing is easy enough when you perform one task at a time
and use synchronous code, but with multiple asynchronous tasks it's
easy to introduce subtle bugs.
This module is designed for these situations. It allows you to easily
and efficiently schedule a large number of tasks with an associated
degree of priority from 0 (highest priority) to Number.MAX_SAFE_INTEGER.
Initialize your scheduler with a degree of parallelism, and start planning
some initial jobs. Set it to run and it will keep going until all jobs are
complete, at which point it will optionally execute a 'done' callback.
Getting back to the original example:
List the contents of the root directory, then plan subsequent jobs
with a priority of 1 to recurse into subdirectories. The callback
of each of these recursions can then plan higher priority tasks
to actually process the contained files with a priority of 0.
As long as there are more files scheduled it will continue to process
them first. When there are no more files the scheduler will read
the next directory and repopulate the list of files to process.
This will repeat until everything is done.
// load the module
const Plan = require("./plan");
// instantiate a scheduler with a parallelism of 5
var plan = Plan(5)
// plan the first job which schedules more jobs...
.job(1, function (next) {
listRootDirectory(function (files) {
files.forEach(function (file) {
// highest priority, run as soon as there is a free worker
plan.job(0, function (next) {
processFile(file, function (result) {
console.log(result);
// don't forget to call next
next();
});
});
});
next(); // call 'next' to free up one worker
});
})
// chain commands together if you want
.done(function () {
console.log("DONE");
})
// it won't run unless you launch it
.start();
*/
module.exports = function (max) {
var plan = {};
max = max || 5;
// finds an id that isn't in use in a particular map
// accepts an id in case you have one already chosen
// otherwise generates random new ids if one is not passed
// or if there is a collision
var uid = function (map, id) {
if (typeof(id) === 'undefined') {
id = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
}
if (id && typeof(map[id]) === 'undefined') {
return id;
}
return uid(map);
};
// the queue of jobs is an array, which will be populated
// with maps for each level of priority
var jobs = [];
// the count of currently running jobs
var count = 0;
// a list of callbacks to be executed once everything is done
var completeHandlers = [];
// the recommended usage is to create a new scheduler for every job
// use it for internals in a scope, and let the garbage collector
// clean up when everything stops. This means you shouldn't
// go passing 'plan' around in a long-lived process!
var FINISHED = false;
var done = function () {
// 'done' gets called when there are no more jobs in the queue
// but other jobs might still be running...
// the count of running processes should never be less than zero
// because we guard against multiple callbacks
if (count < 0) { throw new Error("should never happen"); }
// greater than zero is definitely possible, it just means you aren't done yet
if (count !== 0) { return; }
// you will finish twice if you call 'start' a second time
// this behaviour isn't supported yet.
if (FINISHED) { throw new Error('finished twice'); }
FINISHED = true;
// execute all your 'done' callbacks
completeHandlers.forEach(function (f) { f(); });
};
var run;
// this 'next' is internal only.
// it iterates over all known jobs, running them until
// the scheduler achieves the desired amount of parallelism.
// If there are no more jobs it will call 'done'
// which will shortcircuit if there are still pending tasks.
// Whenever any tasks finishes it will return its lock and
// run as many new jobs as are allowed.
var next = function () {
// array.some skips over bare indexes in sparse arrays
var pending = jobs.some(function (bag /*, priority*/) {
if (!bag || typeof(bag) !== 'object') { return; }
// a bag is a map of jobs for any particular degree of priority
// iterate over jobs in the bag until you're out of 'workers'
for (var id in bag) {
// bail out if you hit max parallelism
if (count >= max) { return true; }
run(bag, id, next);
}
});
// check whether you're done if you hit the end of the array
if (!pending) { done(); }
};
// and here's the part that actually handles jobs...
run = function (bag, id) {
// this is just a sanity check.
// there should only ever be jobs in each bag.
if (typeof(bag[id]) !== 'function') {
throw new Error("expected function");
}
// keep a local reference to the function
var f = bag[id];
// remove it from the bag.
delete bag[id];
// increment the count of running jobs
count++;
// guard against it being called twice.
var called = false;
f(function () {
// watch out! it'll bite you.
// maybe this should just return?
// support that option for 'production' ?
if (called) { throw new Error("called twice"); }
// the code below is safe because we can't call back a second time
called = true;
// decrement the count of running jobs...
count--;
// and finally call next to replace this worker with more job(s)
next();
});
};
// this is exposed as API
plan.job = function (priority, cb) {
// you have to pass both the priority (a non-negative number) and an actual job
if (typeof(priority) !== 'number' || priority < 0) { throw new Error('expected a non-negative number'); }
// a job is an asynchronous function that takes a single parameter:
// a 'next' callback which will keep the whole thing going.
// forgetting to call 'next' means you'll never complete.
if (typeof(cb) !== 'function') { throw new Error('expected function'); }
// initialize the specified priority level if it doesn't already exist
var bag = jobs[priority] = jobs[priority] || {};
// choose a random id that isn't already in use for this priority level
var id = uid(bag);
// add the job to this priority level's bag
// most (all?) javascript engines will append this job to the bottom
// of the map. Meaning when we iterate it will be run later than
// other jobs that were scheduled first, effectively making a FIFO queue.
// However, this is undefined behaviour and you shouldn't ever rely on it.
bag[id] = function (next) {
cb(next);
};
// returning 'plan' lets us chain methods together.
return plan;
};
var started = false;
plan.start = function () {
// don't allow multiple starts
// even though it should work, it's simpler not to.
if (started) { return plan; }
// this seems to imply a 'stop' method
// but I don't need it, so I'm not implementing it now --ansuz
started = true;
// start asynchronously, otherwise jobs will start running
// before you've had a chance to return 'plan', and weird things
// happen.
setTimeout(function () {
next();
});
return plan;
};
// you can pass any number of functions to be executed
// when all pending jobs are complete.
// We don't pass any arguments, so you need to handle return values
// yourself if you want them.
plan.done = function (f) {
if (typeof(f) !== 'function') { throw new Error('expected function'); }
completeHandlers.push(f);
return plan;
};
// That's all! I hope you had fun reading this!
return plan;
};

37
lib/plugin-manager.js
View File

@ -1,37 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const fs = require('node:fs');
const plugins = {};
const extensions = plugins._extensions = [];
const styles = plugins._styles = [];
try {
let pluginsDir = fs.readdirSync(__dirname + '/plugins');
pluginsDir.forEach((name) => {
if (name=== "README.md") { return; }
try {
let plugin = require(`./plugins/${name}/index`);
plugins[plugin.name] = plugin.modules;
try {
let hasExt = fs.existsSync(`lib/plugins/${name}/client/extensions.js`);
if (hasExt) {
extensions.push(plugin.name.toLowerCase());
}
} catch (e) {}
try {
let hasStyle = fs.existsSync(`lib/plugins/${name}/client/style.less`);
if (hasStyle) {
styles.push(plugin.name.toLowerCase());
}
} catch (e) {}
} catch (err) {
console.error(err);
}
});
} catch (err) {
if (err.code !== 'ENOENT') { console.error(err); }
}
module.exports = plugins;

7
lib/plugins/README.md
View File

@ -1,7 +0,0 @@
<!--
SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
-->
# CryptPad's plugins directory

253
lib/rpc.js
View File

@ -1,253 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Util = require("./common-util");
const Core = require("./commands/core");
const Admin = require("./commands/admin-rpc");
const Pinning = require("./commands/pin-rpc");
const Quota = require("./commands/quota");
const Metadata = require("./commands/metadata");
const Channel = require("./commands/channel");
const Upload = require("./commands/upload");
const Linked = require("./commands/linked");
const HK = require("./hk-util");
var RPC = module.exports;
const UNAUTHENTICATED_CALLS = {
GET_FILE_SIZE: Pinning.getFileSize,
GET_MULTIPLE_FILE_SIZE: Pinning.getMultipleFileSize,
GET_DELETED_PADS: Pinning.getDeletedPads,
IS_CHANNEL_PINNED: Pinning.isChannelPinned, // FIXME drop this RPC
IS_NEW_CHANNEL: Channel.isNewChannel,
WRITE_PRIVATE_MESSAGE: Channel.writePrivateMessage,
DELETE_MAILBOX_MESSAGE: Channel.deleteMailboxMessage,
GET_METADATA: Metadata.getMetadata,
IS_PREMIUM: Pinning.isPremium,
ADD_FIRST_ADMIN: Admin.addFirstAdmin,
GET_LINKED_DOCUMENTS: Linked.getLinkedDocuments,
ADD_LINKED_DOCUMENT: Linked.addLinkedDocument,
RESET_LINKED_DOCUMENTS: Linked.resetLinkedDocuments,
GET_HISTORY_SIZE: Linked.getHistorySize
};
var isUnauthenticateMessage = function (msg) {
return msg && msg.length === 2 && typeof(UNAUTHENTICATED_CALLS[msg[0]]) === 'function';
};
var handleUnauthenticatedMessage = function (Env, msg, respond, Server, netfluxId) {
Env.Log.silly('LOG_RPC', msg[0]);
Env.plugins?.MONITORING?.increment(`rpc_${msg[0]}`);
var method = UNAUTHENTICATED_CALLS[msg[0]];
method(Env, msg[1], function (err, value) {
if (err) {
Env.WARN(err, msg[1]);
return void respond(err);
}
respond(err, [null, value, null]);
}, Server, netfluxId);
};
const AUTHENTICATED_USER_TARGETED = {
RESET: Pinning.resetUserPins,
PIN: Pinning.pinChannel,
UNPIN: Pinning.unpinChannel,
CLEAR_OWNED_CHANNEL: Channel.clearOwnedChannel,
REMOVE_OWNED_CHANNEL: Channel.removeOwnedChannel,
TRIM_HISTORY: Channel.trimHistory,
UPLOAD_STATUS: Upload.status,
UPLOAD: Upload.upload,
UPLOAD_COMPLETE: Upload.complete,
UPLOAD_CANCEL: Upload.cancel,
OWNED_UPLOAD_COMPLETE: Upload.complete_owned,
ADMIN: Admin.command,
SET_METADATA: Metadata.setMetadata,
};
const AUTHENTICATED_USER_SCOPED = {
GET_HASH: Pinning.getHash,
GET_TOTAL_SIZE: Pinning.getTotalSize,
UPDATE_LIMITS: Quota.getUpdatedLimit,
GET_LIMIT: Pinning.getLimit,
EXPIRE_SESSION: Core.expireSessionAsync,
REMOVE_PINS: Pinning.removePins,
TRIM_PINS: Pinning.trimPins,
COOKIE: Core.haveACookie,
DESTROY: () => {}
};
var isAuthenticatedCall = function (call) {
if (call === 'UPLOAD') { return false; }
return typeof(AUTHENTICATED_USER_TARGETED[call] || AUTHENTICATED_USER_SCOPED[call]) === 'function';
};
var handleAuthenticatedMessage = function (Env, unsafeKey, msg, respond, Server) {
/* If you have gotten this far, you have signed the message with the
public key which you provided.
*/
var safeKey = Util.escapeKeyCharacters(unsafeKey);
var Respond = function (e, value) {
var session = Env.Sessions[safeKey];
var token = session? session.tokens.slice(-1)[0]: '';
var cookie = Core.makeCookie(token).join('|');
respond(e ? String(e): e, [cookie].concat(typeof(value) !== 'undefined' ?value: []));
};
msg.shift();
// discard validated cookie from message
if (!msg.length) {
return void Respond('INVALID_MSG');
}
var TYPE = msg[0];
Env.Log.silly('LOG_RPC', TYPE);
if (typeof(AUTHENTICATED_USER_TARGETED[TYPE]) === 'function') {
return void AUTHENTICATED_USER_TARGETED[TYPE](Env, safeKey, msg[1], function (e, value) {
Env.WARN(e, value);
return void Respond(e, value);
}, Server);
}
if (typeof(AUTHENTICATED_USER_SCOPED[TYPE]) === 'function') {
return void AUTHENTICATED_USER_SCOPED[TYPE](Env, safeKey, function (e, value) {
if (e) {
Env.WARN(e, safeKey);
return void Respond(e);
}
Respond(e, value);
});
}
return void Respond('UNSUPPORTED_RPC_CALL', msg);
};
var rpc = function (Env, Server, userId, data, respond) {
if (!Array.isArray(data)) {
Env.Log.debug('INVALID_ARG_FORMET', data);
return void respond('INVALID_ARG_FORMAT');
}
if (!data.length) {
return void respond("INSUFFICIENT_ARGS");
} else if (data.length !== 1) {
Env.Log.debug('UNEXPECTED_ARGUMENTS_LENGTH', data);
}
var msg = data[0].slice(0);
if (!Array.isArray(msg)) {
return void respond('INVALID_ARG_FORMAT');
}
if (isUnauthenticateMessage(msg)) {
return handleUnauthenticatedMessage(Env, msg, respond, Server, userId);
}
var signature = msg.shift();
var publicKey = msg.shift();
var safeKey = Util.escapeKeyCharacters(publicKey);
var hadSession = Boolean(Env.Sessions[safeKey]);
// make sure a user object is initialized in the cookie jar
if (publicKey) {
Core.getSession(Env.Sessions, publicKey);
} else {
Env.Log.debug("NO_PUBLIC_KEY_PROVIDED", publicKey);
}
var cookie = msg[0];
if (!Core.isValidCookie(Env.Sessions, publicKey, cookie)) {
// no cookie is fine if the RPC is to get a cookie
if (msg[1] !== 'COOKIE') {
return void respond('NO_COOKIE');
}
}
var serialized = JSON.stringify(msg);
if (!(serialized && typeof(publicKey) === 'string')) {
return void respond('INVALID_MESSAGE_OR_PUBLIC_KEY');
}
var command = msg[1];
Env.plugins?.MONITORING?.increment(`rpc_${command}`);
if (command === 'UPLOAD') {
// UPLOAD is a special case that skips signature validation
// intentional fallthrough behaviour
return void handleAuthenticatedMessage(Env, publicKey, msg, respond, Server);
}
if (isAuthenticatedCall(command)) {
// check the signature on the message
// refuse the command if it doesn't validate
return void Env.checkSignature(serialized, signature, publicKey, function (err) {
if (err) {
return void respond("INVALID_SIGNATURE_OR_PUBLIC_KEY");
}
if (command === 'COOKIE' && !hadSession && Env.logIP) {
Env.Log.info('NEW_RPC_SESSION', {userId: userId, publicKey: publicKey});
}
if (command === "DESTROY") {
HK.unauthenticateNetfluxSession(Env, userId, publicKey);
return; // No need to respond, user will close the session
}
HK.authenticateNetfluxSession(Env, userId, publicKey);
return void handleAuthenticatedMessage(Env, publicKey, msg, respond, Server);
});
}
Env.Log.warn('INVALID_RPC_CALL', command);
return void respond("INVALID_RPC_CALL");
};
RPC.create = function (Env, cb) {
var Sessions = Env.Sessions;
var pingAccountsDaily = function () {
Quota.pingAccountsDaily(Env, function (e) {
if (e) {
Env.WARN('dailyPing', e);
}
});
};
pingAccountsDaily();
Env.intervals.dailyPing = setInterval(pingAccountsDaily, 24*3600*1000);
var updateLimitInterval = function () {
Quota.updateCachedLimits(Env, function (e) {
// failure is expected if they have not specified a quota API endpoint
if (!Env.accounts_api) { return; }
if (e) {
Env.WARN('LIMIT_UPDATE', e);
}
});
};
Quota.applyCustomLimits(Env);
updateLimitInterval();
if (Env.accounts_api) {
Env.intervals.quotaUpdate = setInterval(updateLimitInterval, 3600*1000);
}
// expire old sessions once per minute
Env.intervals.sessionExpirationInterval = setInterval(function () {
Core.expireSessions(Sessions);
}, Core.SESSION_EXPIRATION_TIME);
cb(void 0, function (Server, userId, data, respond) {
try {
return rpc(Env, Server, userId, data, respond);
} catch (e) {
console.log("Error from RPC with data " + JSON.stringify(data));
console.log(e.stack);
}
});
};

176
lib/schedule.js
View File

@ -1,176 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var WriteQueue = require("./write-queue");
var Util = require("./common-util");
/* This module provides implements a FIFO scheduler
which assumes the existence of three types of async tasks:
1. ordered tasks which must be executed sequentially
2. unordered tasks which can be executed in parallel
3. blocking tasks which must block the execution of all other tasks
The scheduler assumes there will be many resources identified by strings,
and that the constraints described above will only apply in the context
of identical string ids.
Many blocking tasks may be executed in parallel so long as they
concern resources identified by different ids.
USAGE:
const schedule = require("./schedule")();
// schedule two sequential tasks using the resource 'pewpew'
schedule.ordered('pewpew', function (next) {
appendToFile('beep\n', next);
});
schedule.ordered('pewpew', function (next) {
appendToFile('boop\n', next);
});
// schedule a task that can happen whenever
schedule.unordered('pewpew', function (next) {
displayFileSize(next);
});
// schedule a blocking task which will wait
// until the all unordered tasks have completed before commencing
schedule.blocking('pewpew', function (next) {
deleteFile(next);
});
// this will be queued for after the blocking task
schedule.ordered('pewpew', function (next) {
appendFile('boom', next);
});
*/
// return a uid which is not already in a map
var unusedUid = function (set) {
var uid = Util.uid();
if (set[uid]) { return unusedUid(); }
return uid;
};
// return an existing session, creating one if it does not already exist
var lookup = function (map, id) {
return (map[id] = map[id] || {
//blocking: [],
active: {},
blocked: {},
});
};
var isEmpty = function (map) {
for (var key in map) {
if (map.hasOwnProperty(key)) { return false; }
}
return true;
};
module.exports = function () {
// every scheduler instance has its own queue
var queue = WriteQueue();
// ordered tasks don't require any extra logic
var Ordered = function (id, task) {
queue(id, task);
};
// unordered and blocking tasks need a little extra state
var map = {};
// regular garbage collection keeps memory consumption low
var collectGarbage = function (id) {
// avoid using 'lookup' since it creates a session implicitly
var local = map[id];
// bail out if no session
if (!local) { return; }
// bail out if there are blocking or active tasks
if (local.lock) { return; }
if (!isEmpty(local.active)) { return; }
// if there are no pending actions then delete the session
delete map[id];
};
// unordered tasks run immediately if there are no blocking tasks scheduled
// or immediately after blocking tasks finish
var runImmediately = function (local, task) {
// set a flag in the map of active unordered tasks
// to prevent blocking tasks from running until you finish
var uid = unusedUid(local.active);
local.active[uid] = true;
task(function () {
// remove the flag you set to indicate that your task completed
delete local.active[uid];
// don't do anything if other unordered tasks are still running
if (!isEmpty(local.active)) { return; }
// bail out if there are no blocking tasks scheduled or ready
if (typeof(local.waiting) !== 'function') {
return void collectGarbage();
}
setTimeout(local.waiting);
});
};
var runOnceUnblocked = function (local, task) {
var uid = unusedUid(local.blocked);
local.blocked[uid] = function () {
runImmediately(local, task);
};
};
// 'unordered' tasks are scheduled to run in after the most recently received blocking task
// or immediately and in parallel if there are no blocking tasks scheduled.
var Unordered = function (id, task) {
var local = lookup(map, id);
if (local.lock) { return runOnceUnblocked(local, task); }
runImmediately(local, task);
};
var runBlocked = function (local) {
for (var task in local.blocked) {
runImmediately(local, local.blocked[task]);
}
};
// 'blocking' tasks must be run alone.
// They are queued alongside ordered tasks,
// and wait until any running 'unordered' tasks complete before commencing.
var Blocking = function (id, task) {
var local = lookup(map, id);
queue(id, function (next) {
// start right away if there are no running unordered tasks
if (isEmpty(local.active)) {
local.lock = true;
return void task(function () {
delete local.lock;
runBlocked(local);
next();
});
}
// otherwise wait until the running tasks have completed
local.waiting = function () {
local.lock = true;
task(function () {
delete local.lock;
delete local.waiting;
runBlocked(local);
next();
});
};
});
};
return {
ordered: Ordered,
unordered: Unordered,
blocking: Blocking,
};
};

88
lib/stats.js
View File

@ -1,88 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Stats = module.exports;
var truthyStringOrNothing = function (s) {
if (typeof(s) !== 'string' || !s) { return undefined; }
return s.trim() || undefined;
};
Stats.instanceData = function (Env) {
var data = {
version: Env.version,
installMethod: Env.installMethod,
domain: Env.accounts_domain,
subdomain: Env.accounts_subdomain,
httpUnsafeOrigin: Env.httpUnsafeOrigin,
httpSafeOrigin: Env.httpSafeOrigin,
adminEmail: Env.consentToContact? Env.adminEmail: undefined,
consentToContact: Boolean(Env.consentToContact),
instancePurpose: Env.instancePurpose === 'noanswer'? undefined: Env.instancePurpose,
};
/* We reserve the right to choose not to include instances
in our public directory at our discretion.
The following details will be included in your telemetry
as factors that may contribute to that decision.
These values are publicly available via /api/config
posting them to our server just makes it easier for us.
*/
if (Env.listMyInstance) {
// clearly indicate that you want to be listed
data.listMyInstance = Env.listMyInstance;
// you should have enabled your admin panel
data.adminKeys = Env.admins.length > 0;
// we expect that you enable your support mailbox
data.supportMailbox = Boolean(Env.supportMailbox);
data.supportMailboxKey = Boolean(Env.supportMailboxKey);
// do you allow registration?
data.restrictRegistration = Boolean(Env.restrictRegistration);
// have you removed the donate button?
data.removeDonateButton = Boolean(Env.removeDonateButton);
// after how long do you consider a document to be inactive?
data.inactiveTime = Env.inactiveTime;
// how much storage do you offer to registered users?
data.defaultStorageLimit = Env.defaultStorageLimit;
// what size file upload do you permit
data.maxUploadSize = Env.maxUploadSize;
// how long do you retain inactive accounts?
data.accountRetentionTime = Env.accountRetentionTime;
// does this instance have a name???
data.instanceName = truthyStringOrNothing(Env.instanceName.default);
// does this instance have a jurisdiction ???
data.instanceJurisdiction = truthyStringOrNothing(Env.instanceJurisdiction.default);
// does this instance have a description???
data.instanceDescription = truthyStringOrNothing(Env.instanceDescription.default);
// how long do you retain archived data?
//data.archiveRetentionTime = Env.archiveRetentionTime,
}
// Admins can opt-in to providing more detailed information about the extent of the instance's usage
if (Env.provideAggregateStatistics) {
// check how many instances provide stats before we put more work into it
data.providesAggregateStatistics = true;
data.statistics = {}; // Filled in lib/commands/quota.js because of async calls
}
return data;
};

95
lib/storage/basic.js
View File

@ -1,95 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
/* Mulfi-factor auth requires some rudimentary storage methods
for a number of data types:
* "challenges" (described in challenge.js)
* account settings for MFA (described in mfa.js)
* session tokens (described in sessions.js)
Each data type requires the same three simple methods:
* read
* write
* delete
These could be implemented as tables in a relational database, but committing to a relational DB
is a big decision, so these methods are instead implemented using the filesystem, with each
file's path and naming convention implemented outside of this module.
Feel free to migrate all of these to a relational DB at some point in the future if you like.
*/
const Basic = module.exports;
const Fs = require("node:fs");
const Fse = require("fs-extra");
const Path = require("node:path");
var pathError = (cb) => {
setTimeout(function () {
cb(new Error("INVALID_PATH"));
});
};
Basic.read = function (Env, path, cb) {
if (!path) { return void pathError(cb); }
Fs.readFile(path, 'utf8', (err, content) => {
if (err) { return void cb(err); }
cb(void 0, content);
});
};
Basic.readDir = function (Env, path, cb) {
if (!path) { return void pathError(cb); }
Fs.readdir(path, cb);
};
Basic.readDirSync = function (Env, path) {
if (!path) { return []; }
return Fs.readdirSync(path);
};
Basic.write = function (Env, path, data, cb) {
if (!path) { return void pathError(cb); }
var dirpath = Path.dirname(path);
Fs.mkdir(dirpath, { recursive: true }, function (err) {
if (err) { return void cb(err); }
// the 'wx' flag causes writes to fail with EEXIST if a file is already present at the given path
// this could be overridden with options in the future if necessary, but it seems like a sensible default
Fs.writeFile(path, data, { flag: 'wx', }, cb);
});
};
// TODO I didn't bother implementing the usual "archive/restore/delete-from-archives" methods
// because they didn't seem particularly important for the data implemented with this module.
// They're still worth considering, though, so don't let my ommission stop you.
// Login blocks could probably be implemented with this module if these methods were supported.
// --Aaron
Basic.delete = function (Env, path, cb) {
if (!path) { return void pathError(cb); }
Fs.rm(path, cb);
};
Basic.deleteDir = function (Env, path, cb) {
if (!path) { return void pathError(cb); }
Fs.rm(path, { recursive: true, force: true }, cb);
};
Basic.archive = function (Env, path, archivePath, cb) {
Fse.move(path, archivePath, {
overwrite: true,
}, (err) => {
cb(err);
});
};
Basic.restore = function (Env, archivePath, path, cb) {
Fse.move(archivePath, path, {
//overwrite: true,
}, (err) => {
cb(err);
});
};
Basic.isValidId = id => {
return id && typeof(id) === "string" && /^[a-zA-Z0-9-+_=]+$/.test(id);
};

946
lib/storage/blob.js
View File

@ -1,946 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Fs = require("fs");
var Fse = require("fs-extra");
var Path = require("path");
var BlobStore = module.exports;
var nThen = require("nthen");
var Semaphore = require("saferphore");
var Util = require("../common-util");
const Crypto = require('crypto');
const PERMISSIVE = 511;
const readFileBin = require("../stream-file").readFileBin;
const BLOB_LENGTH = 48;
var isValidSafeKey = function (safeKey) {
return typeof(safeKey) === 'string' && !/\//.test(safeKey) && safeKey.length === 44;
};
var isValidId = function (id) {
return typeof(id) === 'string' && id.length === BLOB_LENGTH && !/[^a-f0-9]/.test(id);
};
// helpers
var prependArchive = function (Env, path) {
// Env has an absolute path to the blob storage
// we want the path to the blob relative to that
var relativePathToBlob = Path.relative(Env.blobPath, path);
// the new path structure is the same, but relative to the blob archive root
return Path.join(Env.archivePath, 'blob', relativePathToBlob);
};
// /blob/<blobPrefix>/<blobId>
var makeBlobPath = function (Env, blobId) {
return Path.join(Env.blobPath, blobId.slice(0, 2), blobId);
};
var makeActivityPath = function (Env, blobId) {
return makeBlobPath(Env, blobId) + '.activity';
};
// /blob/<blobPrefix>/<blobId>.metadata.ndjson
var mkMetadataPath = function (Env, blobId) {
return Path.join(Env.blobPath, blobId.slice(0, 2), blobId) + '.metadata.ndjson';
};
// /blobstate/<safeKeyPrefix>/<safeKey>
var makeStagePath = function (Env, safeKey) {
return Path.join(Env.blobStagingPath, safeKey.slice(0, 2), safeKey);
};
var mkPlaceholderPath = function (Env, blobId) {
return makeBlobPath(Env, blobId) + '.placeholder';
};
// Placeholder for deleted files
var addPlaceholder = function (Env, blobId, reason, cb) {
if (!reason) { return cb(); }
var path = mkPlaceholderPath(Env, blobId);
var s_data = typeof(reason) === "string" ? reason : `${reason.code}:${reason.txt}`;
Fs.writeFile(path, s_data, cb);
};
var clearPlaceholder = function (Env, blobId, cb) {
var path = mkPlaceholderPath(Env, blobId);
Fs.unlink(path, cb);
};
var readPlaceholder = function (Env, blobId, cb) {
var path = mkPlaceholderPath(Env, blobId);
Fs.readFile(path, function (err, content) {
if (err) { return void cb(); }
cb(content.toString('utf8'));
});
};
// getUploadSize: used by
// getFileSize
var getUploadSize = function (Env, blobId, cb) {
var path = makeBlobPath(Env, blobId);
if (!path) { return cb('INVALID_UPLOAD_ID'); }
Fs.stat(path, function (err, stats) {
if (err) {
// if a file was deleted, its size is 0 bytes
if (err.code === 'ENOENT') {
return readPlaceholder(Env, blobId, (content) => {
if (!content) { return cb(void 0, 0); }
cb({
code: err.code,
reason: content
});
});
}
return void cb(err.code);
}
cb(void 0, stats.size);
});
};
// isFile: used by
// removeOwnedBlob
// uploadComplete
// uploadStatus
var isFile = function (filePath, cb) {
Fs.stat(filePath, function (e, stats) {
if (e) {
if (e.code === 'ENOENT') { return void cb(void 0, false); }
return void cb(e.message);
}
return void cb(void 0, stats.isFile());
});
};
// PROOFS
// DEPRECATED, keep for compatibility
// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
var makeProofPath = function (Env, safeKey, blobId) {
return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId);
};
// isOwnedBy(id, safeKey)
var isOwnedBy = function (Env, safeKey, blobId, cb) {
var proofPath = makeProofPath(Env, safeKey, blobId);
isFile(proofPath, cb);
};
var makeFileStream = function (full, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
Fse.mkdirp(Path.dirname(full), function (e) {
if (e || !full) { // !full for pleasing flow, it's already checked
return void cb(e ? e.message : 'INTERNAL_ERROR');
}
try {
var stream = Fs.createWriteStream(full, {
flags: 'a',
encoding: 'binary',
highWaterMark: Math.pow(2, 16),
});
stream.on('open', function () {
cb(void 0, stream);
});
stream.on('error', function (err) {
cb(err);
});
} catch (err) {
cb('BAD_STREAM');
}
});
};
var clearActivity = function (Env, blobId, cb) {
var path = makeActivityPath(Env, blobId);
// if we fail to delete the activity file, it can still be removed later by the eviction script
Fs.unlink(path, cb);
};
var updateActivity = function (Env, blobId, cb) {
var path = makeActivityPath(Env, blobId);
var blobPath = makeBlobPath(Env, blobId);
isFile(blobPath, (err, state) => {
if (err || !state) { return void cb(); }
var s_data = String(+new Date());
Fs.writeFile(path, s_data, cb);
});
};
var archiveActivity = function (Env, blobId, cb) {
var path = makeActivityPath(Env, blobId);
var archivePath = prependArchive(Env, path);
// if we fail to delete the activity file, it can still be removed later by the eviction script
Fse.move(path, archivePath, { overwrite: true }, cb);
};
var removeArchivedActivity = function (Env, blobId, cb) {
var path = makeActivityPath(Env, blobId);
var archivePath = prependArchive(Env, path);
Fs.unlink(archivePath, cb);
};
var restoreActivity = function (Env, blobId, cb) {
var path = makeActivityPath(Env, blobId);
var archivePath = prependArchive(Env, path);
Fse.move(archivePath, path, cb);
};
var getActivity = function (Env, blobId, cb) {
var path = makeActivityPath(Env, blobId);
Fs.readFile(path, function (err, content) {
if (err) { return void cb(err); }
try {
var date = new Date(+content);
cb(void 0, date);
} catch (err2) {
cb(err2);
}
});
};
// destroyStream && createIdleStreamCollector
// copied from lib/storage/file.js
// see comments there
const STREAM_CLOSE_TIMEOUT = 120000;
const STREAM_DESTROY_TIMEOUT = 30000;
const destroyStream = function (stream) {
if (!stream) { return; }
try {
stream.close();
if (stream.closed && stream.fd === null) { return; }
} catch (err) {
console.error(err);
}
setTimeout(function () {
try { stream.destroy(); } catch (err) { console.error(err); }
}, STREAM_DESTROY_TIMEOUT);
};
const createIdleStreamCollector = function (stream) {
var collector = Util.once(Util.mkAsync(Util.bake(destroyStream, [stream])));
collector.keepAlive = Util.throttle(collector, STREAM_CLOSE_TIMEOUT);
collector.keepAlive();
return collector;
};
// writeMetadata appends to the dedicated log of metadata amendments
var writeMetadata = function (env, channelId, data, cb) {
var path = mkMetadataPath(env, channelId);
Fse.mkdirp(Path.dirname(path), PERMISSIVE, function (err) {
if (err && err.code !== 'EEXIST') { return void cb(err); }
Fs.appendFile(path, data + '\n', cb);
});
};
var archiveMetadata = (Env, blobId, cb) => {
var path = mkMetadataPath(Env, blobId);
var archivePath = prependArchive(Env, path);
// TODO eviction clean lone md files
// if we fail to delete the metadata file, it can still be removed later by the eviction script
Fse.move(path, archivePath, { overwrite: true }, cb);
};
var restoreMetadata = function (Env, blobId, cb) {
var path = mkMetadataPath(Env, blobId);
var archivePath = prependArchive(Env, path);
Fse.move(archivePath, path, cb);
};
var readBlobMetadata = function (env, blobId, handler, _cb) {
var metadataPath = mkMetadataPath(env, blobId);
var stream = Fs.createReadStream(metadataPath, {start: 0});
const collector = createIdleStreamCollector(stream);
var cb = Util.both(_cb, collector);
readFileBin(stream, function (msgObj, readMore) {
collector.keepAlive();
var line = msgObj.buff.toString('utf8');
try {
var parsed = JSON.parse(line);
handler(null, parsed);
} catch (err) {
handler(err, line);
}
readMore();
}, function (err) {
// ENOENT => there is no metadata log
if (!err || err.code === 'ENOENT') { return void cb(); }
// otherwise stream errors?
cb(err);
});
};
/********** METHODS **************/
var uploadWs = function (Env, safeKey, content, cb) {
var dec;
try { dec = Buffer.from(content, 'base64'); }
catch (e) { return void cb('DECODE_BUFFER'); }
var len = dec.length;
var session = Env.getSession(safeKey);
/*
if (typeof(session.currentUploadSize) !== 'number' ||
typeof(session.pendingUploadSize) !== 'number') {
// improperly initialized... maybe they didn't check before uploading?
// reject it, just in case
return cb('NOT_READY');
}
if (session.currentUploadSize > session.pendingUploadSize) {
return cb('E_OVER_LIMIT');
}
*/
var stagePath = makeStagePath(Env, safeKey);
if (!session.blobstage) {
makeFileStream(stagePath, function (e, stream) {
if (!stream) { return void cb(e); }
var blobstage = session.blobstage = stream;
blobstage.write(dec);
session.currentUploadSize += len;
cb(void 0, dec.length);
});
} else {
session.blobstage.write(dec);
session.currentUploadSize += len;
cb(void 0, dec.length);
}
};
var upload = function (Env, safeKey, content, cb) {
var dec;
try { dec = Buffer.from(content, 'base64'); }
catch (e) { return void cb('DECODE_BUFFER'); }
var path = makeStagePath(Env, safeKey);
Fs.appendFile(path, dec, cb);
};
const getRandomCookie = function () {
return Crypto.randomBytes(16).toString('hex');
};
var uploadCookie = function (Env, safeKey, cb) {
var stagePath = makeStagePath(Env, safeKey);
var cookiePath = stagePath + '.cookie';
const cookie = getRandomCookie();
Fse.mkdirp(Path.dirname(cookiePath), PERMISSIVE, function (err) {
if (err && err.code !== 'EEXIST') { return void cb(err); }
Fs.writeFile(cookiePath, cookie, err => {
cb(err, cookie);
});
});
};
var checkUploadCookie = function (Env, safeKey, cb) {
var stagePath = makeStagePath(Env, safeKey);
var cookiePath = stagePath + '.cookie';
Fs.readFile(cookiePath, function (err, content) {
if (err) { return void cb(); }
let expireTime = +new Date() - (5*60*1000);
Fs.stat(cookiePath, function (err, stats) {
if (stats.mtime < expireTime) { return void cb(); }
cb(content.toString('utf8'));
});
});
};
var closeBlobstage = function (Env, safeKey) {
var session = Env.getSession(safeKey);
if (!(session && session.blobstage && typeof(session.blobstage.close) === 'function')) {
return;
}
session.blobstage.close();
delete session.blobstage;
};
// upload_cancel
var upload_cancel = function (Env, safeKey, fileSize, cb) {
var session = Env.getSession(safeKey);
session.pendingUploadSize = fileSize;
session.currentUploadSize = 0;
if (session.blobstage) {
session.blobstage.close();
delete session.blobstage;
}
var path = makeStagePath(Env, safeKey);
Fs.unlink(path, function (e) {
if (e) { return void cb('E_UNLINK'); }
cb(void 0);
});
};
// upload_complete
var upload_complete = function (Env, safeKey, id, cb, linked) {
closeBlobstage(Env, safeKey);
var oldPath = makeStagePath(Env, safeKey);
var newPath = makeBlobPath(Env, id);
nThen(function (w) {
// make sure the path to your final location exists
Fse.mkdirp(Path.dirname(newPath), w(function (e) {
if (e) {
w.abort();
return void cb('RENAME_ERR');
}
}));
}).nThen(function (w) {
// make sure there's not already something in that exact location
isFile(newPath, w(function (e, yes) {
if (e) {
w.abort();
return void cb(e);
}
if (yes) {
w.abort();
return void cb('RENAME_ERR');
}
cb(void 0, id);
}));
}).nThen(function (w) {
if (!linked) { return; }
// Write the metadata
let meta = { linked };
let md = JSON.stringify(meta);
writeMetadata(Env, id, md, w());
}).nThen(function () {
// finally, move the old file to the new path
// FIXME we could just move and handle the EEXISTS instead of the above block
Fse.move(oldPath, newPath, function (e) {
if (e) { return void cb('RENAME_ERR'); }
// clear upload cookie
Fs.unlink(oldPath+'.cookie', function () {});
cb(void 0, id);
});
});
};
var tryId = function (path, cb) {
Fs.access(path, Fs.constants.R_OK | Fs.constants.W_OK, function (e) {
if (!e) {
// generate a new id (with the same prefix) and recurse
return void cb('EEXISTS');
} else if (e.code === 'ENOENT') {
// no entry, so it's safe for us to proceed
return void cb();
} else {
// it failed in an unexpected way. log it
return void cb(e.code);
}
});
};
// owned_upload_complete
let unescapeKeyCharacters = function (key) {
return key.replace(/\-/g, '/');
};
var owned_upload_complete = function (Env, safeKey, id, cb, linked) {
closeBlobstage(Env, safeKey);
if (!isValidId(id)) {
return void cb('EINVAL_ID');
}
var oldPath = makeStagePath(Env, safeKey);
if (typeof(oldPath) !== 'string') {
return void cb('EINVAL_CONFIG');
}
var finalPath = makeBlobPath(Env, id);
let unsafeKey = unescapeKeyCharacters(safeKey);
// the user wants to move it into blob and create a metadata log with an owner
nThen(function (w) {
// make the requisite directory structure using Mkdirp
Fse.mkdirp(Path.dirname(finalPath), w(function (e /*, path */) {
if (e) { // does not throw error if the directory already existed
w.abort();
return void cb(e.code);
}
}));
}).nThen(function (w) {
// make sure the id does not collide with another
tryId(finalPath, w(function (e) {
if (e) {
w.abort();
return void cb(e);
}
}));
}).nThen(function (w) {
// Write the metadata
let meta = { owners: [unsafeKey] };
if (linked) { meta.linked = linked; }
let md = JSON.stringify(meta);
writeMetadata(Env, id, md, w((e) => {
if (e) {
w.abort();
return void cb(e.code);
}
// otherwise it worked...
}));
}).nThen(function (w) {
// move the existing file to its new path
Fse.move(oldPath, finalPath, w(function (e) {
if (e) {
w.abort();
return void cb(e.code);
}
// otherwise it worked...
}));
}).nThen(function () {
// clear upload cookie
Fs.unlink(oldPath+'.cookie', function () {});
// clean up their session when you're done
// call back with the blob id...
cb(void 0, id);
});
};
// removeBlob
var remove = function (Env, blobId, cb) {
var blobPath = makeBlobPath(Env, blobId);
Fs.unlink(blobPath, cb);
clearActivity(Env, blobId, () => {});
};
// archiveBlob
var archiveBlob = function (Env, blobId, reason, cb) {
var blobPath = makeBlobPath(Env, blobId);
var archivePath = prependArchive(Env, blobPath);
Fse.move(blobPath, archivePath, { overwrite: true }, cb);
archiveMetadata(Env, blobId, () => {});
archiveActivity(Env, blobId, () => {});
addPlaceholder(Env, blobId, reason, () => {});
};
var removeArchivedBlob = function (Env, blobId, cb) {
var CB = Util.once(cb);
var archivePath = prependArchive(Env, makeBlobPath(Env, blobId));
var metadataPath = prependArchive(Env, mkMetadataPath(Env, blobId));
Fs.unlink(archivePath, cb);
nThen(function (w) {
Fs.unlink(archivePath, w(function (err) {
if (err) {
if (err.code === "ENOENT") { return; }
w.abort();
CB("E_ARCHIVED_BLOB_REMOVAL_"+ err.code);
}
}));
Fs.unlink(metadataPath, w(function (err) {
if (err) {
if (err.code === "ENOENT") { return; }
w.abort();
CB("E_ARCHIVED_BLOBMD_REMOVAL_"+ err.code);
}
}));
removeArchivedActivity(Env, blobId, () => {});
}).nThen(function () {
CB();
});
};
// restoreBlob
var restoreBlob = function (Env, blobId, cb) {
var blobPath = makeBlobPath(Env, blobId);
var archivePath = prependArchive(Env, blobPath);
Fse.move(archivePath, blobPath, cb);
restoreMetadata(Env, blobId, () => {});
restoreActivity(Env, blobId, () => {});
clearPlaceholder(Env, blobId, () => {});
};
var makeWalker = function (n, handleChild, done) {
if (!n || typeof(n) !== 'number' || n < 2) { n = 2; }
var W;
nThen(function (w) {
// this asynchronous bit defers the completion of this block until
// synchronous execution has completed. This means you must create
// the walker and start using it synchronously or else it will call back
// prematurely
setTimeout(w());
W = w;
}).nThen(function () {
done();
});
// do no more than 20 jobs at a time
var tasks = Semaphore.create(n);
var recurse = function (path, dir) {
tasks.take(function (give) {
var next = give(W());
nThen(function (w) {
// check if the path is a directory...
Fs.stat(path, w(function (err, stats) {
if (err) {
w.abort();
return next();
}
if (!stats.isDirectory()) {
w.abort();
if (/\.activity$/.test(path)) {
// NOTE: some activity files were created for deleted blobs due to
// a bug. We're going to detect them here in order to be able to clean
// them.
if (!dir.includes(Path.basename(path.replace(/\.activity$/, '')))) {
return void handleChild(void 0, path, next, true);
}
// Ignore valid activity files
return next();
}
// Ignore placeholder files
if (/\.placeholder$/.test(path)) { return next(); }
return void handleChild(void 0, path, next, false);
}
// fall through
}));
}).nThen(function () {
// handle directories
Fs.readdir(path, function (err, dir) {
if (err) { return next(); }
// everything is fine and it's a directory...
dir.forEach(function (d) {
recurse(Path.join(path, d), dir);
});
next();
});
});
});
};
return recurse;
};
var getActivityStat = function (path, base, cb) {
var suffix = base ? '' : '.activity';
Fs.stat(path+suffix, function (err, stats) {
if (err && err.code === 'ENOENT' && !base) { return getActivityStat(path, true, cb); }
cb(err, stats);
});
};
var getStats = function (Env, blobId, cb) {
var path = makeBlobPath(Env, blobId);
getActivityStat(path, false, cb);
};
let blobRegex = /^[0-9a-fA-F]{48}(\.metadata)*(\.ndjson)*$/;
var listBlobs = function (root, handler, fast, cb) {
var dirList = [];
nThen(function (w) {
// the root of your datastore contains nested directories...
Fs.readdir(root, w(function (err, list) {
if (err) {
w.abort();
// TODO check if we normally return strings or errors
return void cb(err);
}
dirList = list;
}));
}).nThen(function (waitFor) {
// search inside the nested directories
// stream it so you don't put unnecessary data in memory
var n = nThen;
dirList.forEach(function (dir) {
if (dir.length !== 2) { return; }
// Handle one directory at a time to save some memory
n = n(function (w) {
// do twenty things at a time
var sema = Semaphore.create(20);
var nestedDirPath = Path.join(root, dir);
Fs.readdir(nestedDirPath, w(function (err, list) {
if (err) { return void handler(err); } // Is this correct?
const s = new Set(list);
list.forEach(function (item) {
// ignore hidden files
if (/^\./.test(item)) { return; }
// ignore anything that isn't channel or metadata
if (!blobRegex.test(item)) { return; }
var isLonelyMetadata = false;
var blobName;
// if the current file is not the channel data, then it must be metadata
if (!/^[0-9a-fA-F]{48}$/.test(item)) {
blobName = item.replace(/\.metadata\.ndjson/, '');
// check if blob already exists
if (s.has(blobName)) { return; }
// otherwise set a flag indicating that we should
// handle the metadata on its own
isLonelyMetadata = true;
} else {
blobName = item;
}
if (blobName.length !== 48) { return; }
sema.take(function (give) {
var next = w(give());
if (fast) {
return void handler(void 0, {
blobId: blobName
}, next);
}
var filePath = Path.join(nestedDirPath, blobName);
if (isLonelyMetadata) {
// Set time to 0 to delete this
// lonely metadata file
return void handler(void 0, {
blobId: blobName,
mtime: 0,
atime: 0,
ctime: 0
}, next);
}
return void getActivityStat(filePath, false, (err, data) => {
data.blobId = blobName;
handler(err, data, next);
});
});
});
}));
}).nThen;
});
n(waitFor());
}).nThen(function () {
cb();
});
};
var cleanLoneActivity = function (root, cb) {
// iterate over files
Fs.readdir(root, function (err, dir) {
if (err) { return void cb(err); }
var walk = makeWalker(20, function (err, path, next, loneActivity) {
if (!loneActivity) { return void next(); }
Fs.unlink(path, function (err) {
if (err) {
return console.error('ERROR', path, err);
}
console.log('DELETED', path);
next();
});
}, function () {
cb();
});
dir.forEach(function (d) {
if (d.length !== 2) { return; }
walk(Path.join(root, d));
});
});
};
BlobStore.create = function (config, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (typeof(config.getSession) !== 'function') {
return void cb("getSession method required");
}
var Env = {
blobPath: config.blobPath || './blob',
blobStagingPath: config.blobStagingPath || './blobstage',
archivePath: config.archivePath || './data/archive',
getSession: config.getSession,
};
nThen(function (w) {
var CB = Util.both(w.abort, cb);
Fse.mkdirp(Env.blobPath, w(function (e) {
if (e) { CB(e); }
}));
Fse.mkdirp(Env.blobStagingPath, w(function (e) {
if (e) { CB(e); }
}));
Fse.mkdirp(Path.join(Env.archivePath, './blob'), w(function (e) {
if (e) { CB(e); }
}));
}).nThen(function (w) {
// make a placeholder file in the root of the blob path
// so that the checkup page always has a resource it can check
var fullPath = Path.join(Env.blobPath, 'placeholder.txt');
Fse.writeFile(fullPath, 'PLACEHOLDER\n', w());
}).nThen(function () {
var methods = {
BLOB_LENGTH: BLOB_LENGTH,
isFileId: isValidId,
status: function (safeKey, _cb) {
// TODO check if the final destination is a file
// because otherwise two people can try to upload to the same location
// and one will fail, invalidating their hard work
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
isFile(makeStagePath(Env, safeKey), cb);
},
uploadWs: function (safeKey, content, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
uploadWs(Env, safeKey, content, Util.once(Util.mkAsync(cb)));
},
upload: function (safeKey, content, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
upload(Env, safeKey, content, Util.once(Util.mkAsync(cb)));
},
uploadCookie: function (safeKey, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
uploadCookie(Env, safeKey, Util.once(Util.mkAsync(cb)));
},
checkUploadCookie: function (safeKey, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
checkUploadCookie(Env, safeKey, Util.once(Util.mkAsync(cb)));
},
cancel: function (safeKey, fileSize, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (typeof(fileSize) !== 'number' || isNaN(fileSize) || fileSize <= 0) { return void cb("INVALID_FILESIZE"); }
upload_cancel(Env, safeKey, fileSize, cb);
},
isOwnedBy: function (safeKey, blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
isOwnedBy(Env, safeKey, blobId, cb);
},
readMetadata: (blobId, handler, cb) => {
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
readBlobMetadata(Env, blobId, handler, cb);
},
writeMetadata: (blobId, data, cb) => {
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
writeMetadata(Env, blobId, data, cb);
},
hasMetadata: (blobId, _cb) => {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
var path = mkMetadataPath(Env, blobId);
isFile(path, cb);
},
remove: {
blob: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
remove(Env, blobId, cb);
},
archived: {
blob: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
removeArchivedBlob(Env, blobId, cb);
},
},
loneActivity: function (_cb) {
var cb = Util.once(Util.mkAsync(_cb));
cleanLoneActivity(Env.blobPath, cb);
}
},
archive: {
blob: function (blobId, reason, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
archiveBlob(Env, blobId, reason, cb);
},
},
restore: {
blob: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
restoreBlob(Env, blobId, cb);
},
},
isBlobAvailable: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
var path = makeBlobPath(Env, blobId);
isFile(path, cb);
},
isBlobArchived: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
var path = prependArchive(Env, makeBlobPath(Env, blobId));
isFile(path, cb);
},
getPlaceholder: function (blobId, cb) {
readPlaceholder(Env, blobId, cb);
},
closeBlobstage: function (safeKey) {
closeBlobstage(Env, safeKey);
},
complete: function (safeKey, id, _cb, linked) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(id)) { return void cb("INVALID_ID"); }
upload_complete(Env, safeKey, id, cb, linked);
},
completeOwned: function (safeKey, id, _cb, linked) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(id)) { return void cb("INVALID_ID"); }
owned_upload_complete(Env, safeKey, id, cb, linked);
},
size: function (id, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(id)) { return void cb("INVALID_ID"); }
getUploadSize(Env, id, cb);
},
// ACTIVITY
updateActivity: function (id, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(id)) { return void cb("INVALID_ID"); }
updateActivity(Env, id, cb);
},
getActivity: function (id, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(id)) { return void cb("INVALID_ID"); }
getActivity(Env, id, cb);
},
getStats: function (id, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(id)) { return void cb("INVALID_ID"); }
getStats(Env, id, cb);
},
list: {
blobs: function (handler, _cb, fast) {
var cb = Util.once(Util.mkAsync(_cb));
listBlobs(Env.blobPath, handler, fast, cb);
},
archived: {
blobs: function (handler, _cb, fast) {
var cb = Util.once(Util.mkAsync(_cb));
listBlobs(prependArchive(Env, Env.blobPath), handler, fast, cb);
},
}
},
};
cb(void 0, methods);
});
};

177
lib/storage/block.js
View File

@ -1,177 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Block = module.exports;
const Util = require("../common-util");
const Path = require("path");
const Fs = require("fs");
const Fse = require("fs-extra");
const nThen = require("nthen");
Block.mkPath = function (Env, publicKey) {
// prepare publicKey to be used as a file name
var safeKey = Util.escapeKeyCharacters(publicKey);
// validate safeKey
if (typeof(safeKey) !== 'string') { return; }
// derive the full path
// /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd
return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey);
};
Block.mkArchivePath = function (Env, publicKey) {
// prepare publicKey to be used as a file name
var safeKey = Util.escapeKeyCharacters(publicKey);
// validate safeKey
if (typeof(safeKey) !== 'string') {
return;
}
// derive the full path
// /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd
return Path.join(Env.paths.archive, 'block', safeKey.slice(0, 2), safeKey);
};
var mkPlaceholderPath = function (Env, publicKey) {
return Block.mkPath(Env, publicKey) + '.placeholder';
};
var addPlaceholder = function (Env, publicKey, reason, cb) {
if (!reason) { return cb(); }
var path = mkPlaceholderPath(Env, publicKey);
var s_data = typeof(reason) === "string" ? reason : `${reason.code}:${reason.txt}`;
Fs.writeFile(path, s_data, cb);
};
var clearPlaceholder = function (Env, publicKey, cb) {
var path = mkPlaceholderPath(Env, publicKey);
Fs.unlink(path, cb);
};
Block.readPlaceholder = function (Env, publicKey, cb) {
var path = mkPlaceholderPath(Env, publicKey);
Fs.readFile(path, function (err, content) {
if (err) { return void cb(); }
cb(content.toString('utf8'));
});
};
Block.archive = function (Env, publicKey, reason, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
// derive the filepath
var currentPath = Block.mkPath(Env, publicKey);
// make sure the path is valid
if (typeof(currentPath) !== 'string') {
return void cb('E_INVALID_BLOCK_PATH');
}
var archivePath = Block.mkArchivePath(Env, publicKey);
// make sure the path is valid
if (typeof(archivePath) !== 'string') {
return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH');
}
// TODO Env.incrementBytesWritten
Fse.move(currentPath, archivePath, {
overwrite: true,
}, (err) => {
cb(err);
if (!err && reason) { addPlaceholder(Env, publicKey, reason, () => {}); }
});
};
Block.restore = function (Env, publicKey, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
// derive the filepath
var livePath = Block.mkPath(Env, publicKey);
// make sure the path is valid
if (typeof(livePath) !== 'string') {
return void cb('E_INVALID_BLOCK_PATH');
}
var archivePath = Block.mkArchivePath(Env, publicKey);
// make sure the path is valid
if (typeof(archivePath) !== 'string') {
return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH');
}
// TODO Env.incrementBytesWritten
Fse.move(archivePath, livePath, {
//overwrite: true,
}, (err) => {
cb(err);
if (!err) { clearPlaceholder(Env, publicKey, () => {}); }
});
};
var isValidKey = Block.isValidKey = function (publicKey) {
return typeof(publicKey) === 'string' && publicKey.length === 44;
};
var exists = function (path, cb) {
Fs.stat(path, function (err, stat) {
if (err) {
if (err.code === 'ENOENT') {
return void cb(void 0, false);
}
return void cb(err);
}
if (!stat.isFile()) { return void cb('E_NOT_FILE'); }
return void cb(void 0, true);
});
};
var checkPath = function (Env, publicKey, pathFunction, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidKey(publicKey)) { return void cb("INVALID_ARGS"); }
var path = pathFunction(Env, publicKey);
exists(path, cb);
};
Block.isAvailable = function (Env, publicKey, _cb) {
checkPath(Env, publicKey, Block.mkPath, _cb);
};
Block.isArchived = function (Env, publicKey, _cb) {
checkPath(Env, publicKey, Block.mkArchivePath, _cb);
};
Block.check = function (Env, publicKey, _cb) { // 'check' because 'exists' implies boolean
var cb = Util.once(Util.mkAsync(_cb));
var path = Block.mkPath(Env, publicKey);
Fs.access(path, Fs.constants.F_OK, cb);
};
Block.MAX_SIZE = 256;
Block.write = function (Env, publicKey, buffer, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
var path = Block.mkPath(Env, publicKey);
if (typeof(path) !== 'string') { return void cb('INVALID_PATH'); }
var parsed = Path.parse(path);
nThen(function (w) {
Fse.mkdirp(parsed.dir, w(function (err) {
if (!err) { return; }
w.abort();
cb(err);
}));
}).nThen(function (w) {
Block.archive(Env, publicKey, 'PASSWORD_CHANGE', w(function (/* err */) {
/*
we proceed even if there are errors.
it might be ENOENT (there is no file to archive)
or EACCES (bad filesystem permissions for the existing archived block?)
or lots of other things, none of which justify preventing the write
*/
}));
}).nThen(function () {
Fs.writeFile(path, buffer, { encoding: 'binary' }, cb);
Env.incrementBytesWritten(buffer && buffer.length);
});
};

45
lib/storage/challenge.js
View File

@ -1,45 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Basic = require("./basic.js");
const Path = require("node:path");
const Challenge = module.exports;
/* This module manages storage used to implement a public-key authenticated
challenge-response protocol.
Each 'challenge' is only intended to be valid for a short period of time.
1. A client makes a request of the server
2. The server stores their request with a nonce and challenges them to sign for this request
3. If the client successfully signs for the request within a short window then the request is executed
4. Whether the signature is valid or not, the challenge is removed
Thus, we only expect challenges to remain in storage if the request was aborted or interrupted
for some unexpected reason.
Some form of garbage collection should be implemented in the future.
*/
const pathFromId = function (Env, id) {
if (!Basic.isValidId(id)) {
return void console.error('CHALLENGE_BAD_ID', id);
}
return Path.join(Env.paths.base, "challenges", id.slice(0, 2), id);
};
Challenge.read = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.read(Env, path, cb);
};
Challenge.write = function (Env, id, data, cb) {
var path = pathFromId(Env, id);
Basic.write(Env, path, data, cb);
};
Challenge.delete = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.delete(Env, path, cb);
};

1649
lib/storage/file.js
View File

File diff suppressed because it is too large Load Diff

84
lib/storage/invite.js
View File

@ -1,84 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Basic = require("./basic.js");
const Path = require("node:path");
const nThen = require('nthen');
const Util = require('../common-util');
const Invite = module.exports;
/* This module manages storage used to implement instance invitations when registration
is closed. This "database" will store individual invitation and their state.
An invitation is created with a random uid and an alias (username, email, etc.)
Once it is used by the user, their newly created blockId is added which will mark
it as completed.
*/
const pathFromId = function (Env, id) {
if (!Basic.isValidId(id)) {
return void console.error('INVITE_BAD_ID', id);
}
return Path.join(Env.paths.base, "invitations", id.slice(0, 2), id);
};
Invite.read = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.read(Env, path, (err, data) => {
if (err) { return void cb(err.code); }
cb(void 0, Util.tryParse(data));
});
};
Invite.getAll = function (Env, cb) {
let invitations = {};
nThen((waitFor) => {
let dirPath = Path.join(Env.paths.base, "invitations");
Basic.readDir(Env, dirPath, waitFor((err, prefixes) => {
if (err && err.code === 'ENOENT') { return void cb(void 0, {}); }
if (err) { waitFor.abort(); return void cb(err.code); }
prefixes.forEach((prefix) => {
var dirPath2 = Path.join(Env.paths.base, "invitations", prefix);
Basic.readDir(Env, dirPath2, waitFor((err, files) => {
if (err) { waitFor.abort(); return void cb(err.code); }
files.forEach((id) => {
Invite.read(Env, id, waitFor((err, data) => {
invitations[id] = data || { error: err };
}));
});
}));
});
}));
}).nThen(() => {
cb(null, invitations);
});
};
Invite.write = function (Env, id, data, cb) {
var path = pathFromId(Env, id);
Basic.write(Env, path, JSON.stringify(data), (err) => {
if (err) { return void cb(err.code); }
cb();
});
};
Invite.delete = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.delete(Env, path, (err) => {
if (err) { return void cb(err.code); }
cb();
});
};
Invite.update = function (Env, id, data, cb) {
Invite.delete(Env, id, (err) => {
if (err) { return void cb(err); }
Invite.write(Env, id, data, cb);
});
};

93
lib/storage/mfa.js
View File

@ -1,93 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Basic = require("./basic");
const Path = require("node:path");
const Util = require("../common-util");
const Sessions = require("./sessions");
const nThen = require("nthen");
const MFA = module.exports;
/*
This module manages storage related to accounts' multi-factor authentication settings.
These settings are checked every time a block is accessed, so we do as little as possible
so that it can be accessed quickly.
*/
/* The path for a given account's settings is based on the public signing key
which identifies its "login block". We expect that any action to create or access
a block will be authenticated with a challenge-response protocol, so we
don't bother checking the validity of an identifier in here aside from
ensuring that it won't throw when using string methods.
*/
var pathFromId = function (Env, id) {
if (!id || typeof(id) !== 'string') { return; }
id = Util.escapeKeyCharacters(id);
if (!Basic.isValidId(id)) { return; }
return Path.join(Env.paths.base, "mfa", id.slice(0, 2), `${id}.json`);
};
MFA.read = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.read(Env, path, cb);
};
// data should be a string
MFA.write = function (Env, id, data, cb) {
var path = pathFromId(Env, id);
Basic.write(Env, path, data, cb);
};
MFA.delete = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.delete(Env, path, cb);
};
MFA.revoke = function (Env, publicKey, cb) {
nThen(function (w) {
MFA.delete(Env, publicKey, w(function (err) {
if (!err) { return; }
w.abort();
Env.Log.error('TOTP_REVOKE_MFA_DELETE', {
error: err,
publicKey: publicKey,
});
cb('MFA_ERROR');
}));
}).nThen(function () {
Sessions.deleteUser(Env, publicKey, function (err) {
if (!err) { return; }
// If we can't delete the sessions, don't send an error, just log to the server.
// The MFA will still be correctly disabled as long as the first step is done.
Env.Log.error('TOTP_REVOKE_SESSIONS__DELETE', {
error: err,
publicKey: publicKey,
});
});
}).nThen(function () {
cb(void 0, {
success: true
});
});
};
MFA.copy = function (Env, oldKey, newKey, cb) {
let content;
nThen(function (w) {
MFA.read(Env, oldKey, w(function (err, c) {
if (err) {
// No MFA configured, nothing to copy
w.abort();
return void cb();
}
content = c;
}));
}).nThen(function () {
MFA.write(Env, newKey, content, cb);
});
};

91
lib/storage/moderator.js
View File

@ -1,91 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Basic = require("./basic.js");
const Path = require("node:path");
const nThen = require('nthen');
const Util = require('../common-util');
const Moderator = module.exports;
/* This module manages storage used to implement user management. "Known users" can
be added here in order to store their public key, their block ID and an alias
used to recognize them.
*/
const pathFromId = function (Env, id) {
id = Util.escapeKeyCharacters(id || '');
if (!Basic.isValidId(id)) {
return void console.error('MODERATOR_BAD_ID', id);
}
return Path.join(Env.paths.base, "support", id.slice(0, 2), id);
};
Moderator.read = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.read(Env, path, (err, data) => {
if (err) { return void cb(err.code); }
cb(void 0, Util.tryParse(data));
});
};
Moderator.getAllKeys = function (Env) {
let keys = [];
let dirPath = Path.join(Env.paths.base, "support");
try {
let prefixes = Basic.readDirSync(Env, dirPath);
prefixes.forEach((prefix) => {
let dirPath2 = Path.join(Env.paths.base, "support", prefix);
try {
let newKeys = Basic.readDirSync(Env, dirPath2);
keys.push(...newKeys);
} catch (e) {}
});
return keys;
} catch (e) {
// ENOENT, return empty array
return [];
}
};
Moderator.getAll = function (Env, cb) {
let users = {};
nThen((waitFor) => {
let dirPath = Path.join(Env.paths.base, "support");
Basic.readDir(Env, dirPath, waitFor((err, prefixes) => {
if (err && err.code === 'ENOENT') { return void cb(void 0, {}); }
if (err) { waitFor.abort(); return void cb(err.code); }
prefixes.forEach((prefix) => {
var dirPath2 = Path.join(Env.paths.base, "support", prefix);
Basic.readDir(Env, dirPath2, waitFor((err, files) => {
if (err) { waitFor.abort(); return void cb(err.code); }
files.forEach((id) => {
Moderator.read(Env, id, waitFor((err, data) => {
users[id] = data || { error: err };
}));
});
}));
});
}));
}).nThen(() => {
cb(null, users);
});
};
Moderator.write = function (Env, id, data, cb) {
var path = pathFromId(Env, id);
Basic.write(Env, path, JSON.stringify(data), (err) => {
if (err) { return void cb(err.code); }
cb();
});
};
Moderator.delete = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.delete(Env, path, (err) => {
if (err) { return void cb(err.code); }
cb();
});
};

76
lib/storage/sessions.js
View File

@ -1,76 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Basic = require("./basic");
const Path = require("node:path");
const Nacl = require("tweetnacl/nacl-fast");
const Util = require("../common-util");
const Sessions = module.exports;
/* This module manages storage for per-acccount session tokens - currently assumed to be
JSON Web Tokens (JWTs).
Decisions about what goes into each of those JWTs happens upstream, so the storage
itself is relatively unopinionated.
The key things to understand are:
* valid sessions allow the holder of a given JWT to access a given "login block"
* JWTs are signed with a key held in the server's memory. If that key leaks then it should be rotated (with the SET_BEARER_SECRET decree) to invalidate all existing JWTs. Under these conditions then all tokens signed with the old key can be removed. Garbage collection of these older tokens is not implemented.
* it is expected that any given login-block can have multiple active sessions (for different devices, or if their browser clears its cache automatically). All sessions for a given block are stored in a per-user directory which is intended to make listing or iterating over them simple.
* It could be desirable to expose the list of sessions to the relevant user and allow them to revoke sessions individually or en-masse, though this is not currently implemented.
*/
var pathFromId = function (Env, id, ref) {
if (!id || typeof(id) !== 'string') { return; }
if (!Basic.isValidId(ref)) { return; }
id = Util.escapeKeyCharacters(id);
return Path.join(Env.paths.base, "sessions", id.slice(0, 2), id, ref);
};
Sessions.randomId = () => Util.encodeBase64(Nacl.randomBytes(24)).replace(/\//g, '-');
Sessions.read = function (Env, id, ref, cb) {
var path = pathFromId(Env, id, ref);
Basic.read(Env, path, cb);
};
Sessions.write = function (Env, id, ref, data, cb) {
var path = pathFromId(Env, id, ref);
Basic.write(Env, path, data, cb);
};
Sessions.delete = function (Env, id, ref, cb) {
var path = pathFromId(Env, id, ref);
Basic.delete(Env, path, cb);
};
Sessions.update = function (Env, id, oldId, ref, dataStr, cb) {
var data = Util.tryParse(dataStr);
Sessions.read(Env, oldId, ref, (err, oldData) => {
let content = Util.tryParse(oldData) || {};
Object.keys(data || {}).forEach((type) => {
content[type] = data[type];
});
Sessions.delete(Env, oldId, ref, () => {
Sessions.write(Env, id, ref, JSON.stringify(content), cb);
});
});
};
Sessions.deleteUser = function (Env, id, cb) {
if (!id || typeof(id) !== 'string') { return; }
id = Util.escapeKeyCharacters(id);
var dirPath = Path.join(Env.paths.base, "sessions", id.slice(0, 2), id);
Basic.readDir(Env, dirPath, (err, files) => {
var checkContent = !files || (Array.isArray(files) && files.every((file) => {
return file && file.length === 32;
}));
if (!checkContent) { return void cb('INVALID_SESSIONS_DIR'); }
Basic.deleteDir(Env, dirPath, cb);
});
};

111
lib/storage/sso.js
View File

@ -1,111 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Basic = require("./basic");
const Path = require("node:path");
const Util = require("../common-util");
const SSO = module.exports;
/* This module manages storage related to Single Sign-On (SSO) settings.
A first part (sso-requests) contains temporary files for sso authentication with a remote service
A second part (sso-users) is a database of accounts registered via SSO (SSO id ==> block seed)
A third part (sso-blocks) is a database of blocks that are sso-protected (block id ==> SSO id...)
The path for requests is based on the "authentication request" token depending on the type of SSO.
The path for the user database is based on their persistent identifier (id) from the SSO.
*/
var pathFromId = function (Env, id, subPath) {
if (!id || typeof(id) !== 'string') { return; }
id = Util.escapeKeyCharacters(id);
if (!Basic.isValidId(id)) { return; }
return Path.join(Env.paths.base, subPath, id.slice(0, 2), `${id}.json`);
};
var reqPathFromId = function (Env, id) {
return pathFromId(Env, id, 'sso_request');
};
var blockPathFromId = function (Env, id) {
return pathFromId(Env, id, 'sso_block');
};
var userPathFromId = function (Env, id, provider) {
if (!id || typeof(id) !== 'string') { return; }
if (!provider || typeof(provider) !== 'string') { return; }
id = Util.escapeKeyCharacters(id);
return Path.join(Env.paths.base, 'sso_user', provider, id.slice(0, 2), `${id}.json`);
};
var blockArchivePath = function (Env, id) {
return Path.join(Env.paths.archive, 'sso_block', id.slice(0, 2), `${id}.json`);
};
var userArchivePath = function (Env, id, provider) {
return Path.join(Env.paths.archive, 'sso_user', provider, id.slice(0, 2), `${id}.json`);
};
const Req = SSO.request = {};
Req.read = function (Env, id, cb) {
var path = reqPathFromId(Env, id);
Basic.read(Env, path, cb);
};
Req.write = function (Env, id, data, cb) {
var path = reqPathFromId(Env, id);
Basic.write(Env, path, data, cb);
};
Req.delete = function (Env, id, cb) {
var path = reqPathFromId(Env, id);
Basic.delete(Env, path, cb);
};
const User = SSO.user = {};
User.read = function (Env, provider, id, cb) {
var path = userPathFromId(Env, id, provider);
Basic.read(Env, path, cb);
};
User.write = function (Env, provider, id, data, cb) {
var path = userPathFromId(Env, id, provider);
Basic.write(Env, path, data, cb);
};
User.delete = function (Env, provider, id, cb) {
var path = userPathFromId(Env, id, provider);
Basic.delete(Env, path, cb);
};
User.archive = function (Env, provider, id, cb) {
var path = userPathFromId(Env, id, provider);
var archivePath = userArchivePath(Env, id, provider);
Basic.archive(Env, path, archivePath, cb);
};
User.restore = function (Env, provider, id, cb) {
var path = userPathFromId(Env, id, provider);
var archivePath = userArchivePath(Env, id, provider);
Basic.restore(Env, archivePath, path, cb);
};
const Block = SSO.block = {};
Block.read = function (Env, id, cb) {
var path = blockPathFromId(Env, id);
Basic.read(Env, path, cb);
};
Block.write = function (Env, id, data, cb) {
var path = blockPathFromId(Env, id);
Basic.write(Env, path, data, cb);
};
Block.delete = function (Env, id, cb) {
var path = blockPathFromId(Env, id);
Basic.delete(Env, path, cb);
};
Block.archive = function (Env, id, cb) {
var path = blockPathFromId(Env, id);
var archivePath = blockArchivePath(Env, id);
Basic.archive(Env, path, archivePath, cb);
};
Block.restore = function (Env, id, cb) {
var path = blockPathFromId(Env, id);
var archivePath = blockArchivePath(Env, id);
Basic.restore(Env, archivePath, path, cb);
};

403
lib/storage/tasks.js
View File

@ -1,403 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
var Fs = require("fs");
var Fse = require("fs-extra");
var Path = require("path");
var nacl = require("tweetnacl/nacl-fast");
var nThen = require("nthen");
var Util = require('../common-util');
var Tasks = module.exports;
var tryParse = function (s) {
try { return JSON.parse(s); }
catch (e) { return null; }
};
var encode = function (time, command, args) {
if (typeof(time) !== 'number') { return null; }
if (typeof(command) !== 'string') { return null; }
if (!Array.isArray(args)) { return [time, command]; }
return [time, command].concat(args);
};
/*
var randomId = function () {
var bytes = Array.prototype.slice.call(nacl.randomBytes(16));
return bytes.map(function (b) {
var n = Number(b & 0xff).toString(16);
return n.length === 1? '0' + n: n;
}).join('');
};
var mkPath = function (env, id) {
return Path.join(env.root, id.slice(0, 2), id) + '.ndjson';
};
*/
// make a new folder every MODULUS ms
var MODULUS = 1000 * 60 * 60 * 24; // one day
var moduloTime = function (d) {
return d - (d % MODULUS);
};
var makeDirectoryId = function (d) {
return '' + moduloTime(d);
};
var write = function (env, task, cb) {
var str = JSON.stringify(task) + '\n';
var id = Util.encodeBase64(nacl.hash(Util.decodeUTF8(str))).replace(/\//g, '-');
var dir = makeDirectoryId(task[0]);
var path = Path.join(env.root, dir);
nThen(function (w) {
// create the parent directory if it does not exist
Fse.mkdirp(path, 0x1ff, w(function (err) {
if (err) {
w.abort();
return void cb(err);
}
}));
}).nThen(function () {
// write the file to the path
var fullPath = Path.join(path, id + '.ndjson');
// the file ids are based on the hash of the file contents to be written
// as such, writing an exact task a second time will overwrite the first with the same contents
// this shouldn't be a problem
Fs.writeFile(fullPath, str, function (e) {
if (e) {
env.log.error("TASK_WRITE_FAILURE", {
error: e,
path: fullPath,
});
return void cb(e);
}
env.log.info("SUCCESSFUL_WRITE", {
path: fullPath,
});
cb();
});
});
};
var remove = function (env, path, cb) {
// FIXME COLDSTORAGE?
Fs.unlink(path, cb);
};
var removeDirectory = function (env, path, cb) {
Fs.rmdir(path, cb);
};
var list = Tasks.list = function (env, cb, migration) {
var rootDirs;
nThen(function (w) {
// read the root directory
Fs.readdir(env.root, w(function (e, list) {
if (e) {
env.log.error("TASK_ROOT_DIR", {
root: env.root,
error: e,
});
w.abort();
return void cb(e);
}
if (list.length === 0) {
w.abort();
return void cb(void 0, []);
}
rootDirs = list;
}));
}).nThen(function () {
// schedule the nested directories for exploration
// return a list of paths to tasks
var queue = nThen(function () {});
var allPaths = [];
var currentWindow = moduloTime(+new Date() + MODULUS);
// We prioritize a small footprint over speed, so we
// iterate over directories in serial rather than parallel
rootDirs.forEach(function (dir) {
// if a directory is two characters, it's the old format
// otherwise, it indicates when the file is set to expire
// so we can ignore directories which are clearly in the future
var dirTime;
if (migration) {
// this block handles migrations. ignore new formats
if (dir.length !== 2) {
return;
}
} else {
// not in migration mode, check if it's a new format
if (dir.length >= 2) {
// might be the new format.
// check its time to see if it should be skipped
dirTime = parseInt(dir);
if (!isNaN(dirTime) && dirTime >= currentWindow) {
return;
}
}
}
queue.nThen(function (w) {
var subPath = Path.join(env.root, dir);
Fs.readdir(subPath, w(function (e, paths) {
if (e) {
env.log.error("TASKS_INVALID_SUBDIR", {
path: subPath,
error: e,
});
return;
}
if (paths.length === 0) {
removeDirectory(env, subPath, function (err) {
if (err) {
env.log.error('TASKS_REMOVE_EMPTY_DIRECTORY', {
error: err,
path: subPath,
});
}
});
}
// concat in place
Array.prototype.push.apply(allPaths, paths.map(function (p) {
return Path.join(subPath, p);
}));
}));
});
});
queue.nThen(function () {
cb(void 0, allPaths);
});
});
};
var read = function (env, filePath, cb) {
Fs.readFile(filePath, 'utf8', function (e, str) {
if (e) { return void cb(e); }
var task = tryParse(str);
if (!Array.isArray(task) || task.length < 2) {
env.log("INVALID_TASK", {
path: filePath,
task: task,
});
return cb(new Error('INVALID_TASK'));
}
cb(void 0, task);
});
};
var expire = function (env, task, cb) {
// TODO magic numbers, maybe turn task parsing into a function
// and also maybe just encode tasks in a better format to start...
var Log = env.log;
var args = task.slice(2);
Log.info('ARCHIVAL_SCHEDULED_EXPIRATION', {
task: task,
});
env.store.archiveChannel(args[0], 'EXPIRED', function (err) {
if (err) {
Log.error('ARCHIVE_SCHEDULED_EXPIRATION_ERROR', {
task: task,
error: err,
});
}
cb();
});
};
var run = Tasks.run = function (env, path, cb) {
var CURRENT = +new Date();
var Log = env.log;
var task, time, command;
//var args;
nThen(function (w) {
read(env, path, w(function (err, _task) {
if (err) {
w.abort();
// there was a file but it wasn't valid?
return void cb(err);
}
task = _task;
time = task[0];
if (time > CURRENT) {
w.abort();
return cb();
}
command = task[1];
//args = task.slice(2);
}));
}).nThen(function (w) {
switch (command) {
case 'EXPIRE':
return void expire(env, task, w());
default:
Log.warn("TASKS_UNKNOWN_COMMAND", task);
}
}).nThen(function () {
// remove the task file...
remove(env, path, function (err) {
if (err) {
Log.error('TASKS_RECORD_REMOVAL', {
path: path,
err: err,
});
}
cb();
});
});
};
var runAll = function (env, cb) {
// check if already running and bail out if so
if (env.running) {
return void cb("TASK_CONCURRENCY");
}
// if not, set a flag to block concurrency and proceed
env.running = true;
var paths;
nThen(function (w) {
list(env, w(function (err, _paths) {
if (err) {
w.abort();
env.running = false;
return void cb(err);
}
paths = _paths;
}));
}).nThen(function (w) {
var done = w();
var nt = nThen(function () {});
paths.forEach(function (path) {
nt = nt.nThen(function (w) {
run(env, path, w(function (err) {
if (err) {
// Any errors are already logged in 'run'
// the admin will need to review the logs and clean up
}
}));
});
});
nt = nt.nThen(function () {
done();
});
}).nThen(function (/*w*/) {
env.running = false;
cb();
});
};
var migrate = function (env, cb) {
// list every task
list(env, function (err, paths) {
if (err) {
return void cb(err);
}
var nt = nThen(function () {});
paths.forEach(function (path) {
var bypass;
var task;
nt = nt.nThen(function (w) {
// read
read(env, path, w(function (err, _task) {
if (err) {
bypass = true;
env.log.error("TASK_MIGRATION_READ", {
error: err,
path: path,
});
return;
}
task = _task;
}));
}).nThen(function (w) {
if (bypass) { return; }
// rewrite in new format
write(env, task, w(function (err) {
if (err) {
bypass = true;
env.log.error("TASK_MIGRATION_WRITE", {
error: err,
task: task,
});
}
}));
}).nThen(function (w) {
if (bypass) { return; }
// remove
remove(env, path, w(function (err) {
if (err) {
env.log.error("TASK_MIGRATION_REMOVE", {
error: err,
path: path,
});
}
}));
});
});
nt = nt.nThen(function () {
cb();
});
}, true);
};
Tasks.create = function (config, cb) {
if (!config.store) { throw new Error("E_STORE_REQUIRED"); }
if (!config.log) { throw new Error("E_LOG_REQUIRED"); }
var env = {
root: config.taskPath || './tasks',
log: config.log,
store: config.store,
};
// make sure the path exists...
Fse.mkdirp(env.root, 0x1ff, function (err) {
if (err) { return void cb(err); }
cb(void 0, {
write: function (time, command, args, cb) {
var task = encode(time, command, args);
write(env, task, cb);
},
list: function (olderThan, cb) {
list(env, olderThan, cb);
},
remove: function (id, cb) {
remove(env, id, cb);
},
run: function (id, cb) {
run(env, id, cb);
},
runAll: function (cb) {
runAll(env, cb);
},
migrate: function (cb) {
migrate(env, cb);
},
});
});
};

83
lib/storage/user.js
View File

@ -1,83 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Basic = require("./basic.js");
const Path = require("node:path");
const nThen = require('nthen');
const Util = require('../common-util');
const User = module.exports;
/* This module manages storage used to implement user management. "Known users" can
be added here in order to store their public key, their block ID and an alias
used to recognize them.
*/
const pathFromId = function (Env, id) {
if (!id || typeof(id) !== 'string') { return; }
id = Util.escapeKeyCharacters(id);
if (!Basic.isValidId(id)) {
return void console.error('USER_BAD_ID', id);
}
return Path.join(Env.paths.base, "users", id.slice(0, 2), id);
};
User.read = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.read(Env, path, (err, data) => {
if (err) { return void cb(err.code); }
cb(void 0, Util.tryParse(data));
});
};
User.getAll = function (Env, cb) {
let users = {};
nThen((waitFor) => {
let dirPath = Path.join(Env.paths.base, "users");
Basic.readDir(Env, dirPath, waitFor((err, prefixes) => {
if (err && err.code === 'ENOENT') { return void cb(void 0, {}); }
if (err) { waitFor.abort(); return void cb(err.code); }
prefixes.forEach((prefix) => {
var dirPath2 = Path.join(Env.paths.base, "users", prefix);
Basic.readDir(Env, dirPath2, waitFor((err, files) => {
if (err) { waitFor.abort(); return void cb(err.code); }
files.forEach((id) => {
User.read(Env, id, waitFor((err, data) => {
users[id] = data || { error: err };
}));
});
}));
});
}));
}).nThen(() => {
cb(null, users);
});
};
User.write = function (Env, id, data, cb) {
var path = pathFromId(Env, id);
Basic.write(Env, path, JSON.stringify(data), (err) => {
if (err) { return void cb(err.code); }
cb();
});
};
User.delete = function (Env, id, cb) {
var path = pathFromId(Env, id);
Basic.delete(Env, path, (err) => {
if (err) { return void cb(err.code); }
cb();
});
};
User.update = function (Env, id, data, cb) {
User.delete(Env, id, (err) => {
if (err) { return void cb(err); }
User.write(Env, id, data, cb);
});
};

86
lib/stream-file.js
View File

@ -1,86 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const ToPull = require('stream-to-pull-stream');
const Pull = require('pull-stream');
const Stream = module.exports;
// transform a stream of arbitrarily divided data
// into a stream of buffers divided by newlines in the source stream
// TODO see if we could improve performance by using libnewline
const NEWLINE_CHR = ('\n').charCodeAt(0);
const mkBufferSplit = () => {
let remainder = null;
return Pull((read) => {
return (abort, cb) => {
read(abort, function (end, data) {
if (end) {
if (data) { console.log("mkBufferSplit() Data at the end"); }
cb(end, remainder ? [remainder, data] : [data]);
remainder = null;
return;
}
const queue = [];
for (;;) {
const offset = data.indexOf(NEWLINE_CHR);
if (offset < 0) {
remainder = remainder ? Buffer.concat([remainder, data]) : data;
break;
}
let subArray = data.slice(0, offset);
if (remainder) {
subArray = Buffer.concat([remainder, subArray]);
remainder = null;
}
queue.push(subArray);
data = data.slice(offset + 1);
}
cb(end, queue);
});
};
}, Pull.flatten());
};
// return a streaming function which transforms buffers into objects
// containing the buffer and the offset from the start of the stream
const mkOffsetCounter = (offset) => {
offset = offset || 0;
return Pull.map((buff) => {
const out = { offset: offset, buff: buff };
// +1 for the eaten newline
offset += buff.length + 1;
return out;
});
};
// readMessagesBin asynchronously iterates over the messages in a channel log
// the handler for each message must call back to read more, which should mean
// that this function has a lower memory profile than our classic method
// of reading logs line by line.
// it also allows the handler to abort reading at any time
Stream.readFileBin = (stream, msgHandler, cb, opt) => {
opt = opt || {};
//const stream = Fs.createReadStream(path, { start: start });
let keepReading = true;
Pull(
ToPull.read(stream),
mkBufferSplit(),
mkOffsetCounter(opt.offset),
Pull.asyncMap((data, moreCb) => {
msgHandler(data, moreCb, () => {
try {
stream.close();
} catch (err) {
console.error("READ_FILE_BIN_ERR", err);
}
keepReading = false;
moreCb();
});
}),
Pull.drain(() => (keepReading), (err) => {
cb((keepReading) ? err : undefined);
})
);
};

928
lib/workers/db-worker.js
View File

@ -1,928 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const HK = require("../hk-util");
const Store = require("../storage/file");
const BlobStore = require("../storage/blob");
const Block = require("../commands/block");
const Util = require("../common-util");
const nThen = require("nthen");
const Meta = require("../metadata");
const Pins = require("../pins");
const Core = require("../commands/core");
const Saferphore = require("saferphore");
const Logger = require("../log");
const Tasks = require("../storage/tasks");
const Nacl = require('tweetnacl/nacl-fast');
const Eviction = require("../eviction");
const CPCrypto = require('../crypto');
const plugins = require("../plugin-manager");
const Env = {
Log: {},
};
const Monitoring = plugins && plugins.MONITORING;
const monitoringIncrement = key => {
if (!Monitoring || !Monitoring.increment) { return; }
Monitoring.increment(key);
};
// support the usual log API but pass it to the main process
Logger.levels.forEach(function (level) {
Env.Log[level] = function (label, info) {
process.send({
log: level,
label: label,
info: info,
});
};
});
//const HISTORY_SIZE_LIMIT = 1024 * 1024 * 1024; // 1GB
var DETAIL = 1000;
var round = function (n) {
return Math.floor(n * DETAIL) / DETAIL;
};
var ready = false;
var store;
var pinStore;
var blobStore;
const init = function (config, _cb) {
const cb = Util.once(Util.mkAsync(_cb));
if (!config) {
return void cb('E_INVALID_CONFIG');
}
Env.paths = {
pin: config.pinPath,
block: config.blockPath,
};
Env.inactiveTime = config.inactiveTime;
Env.archiveRetentionTime = config.archiveRetentionTime;
Env.accountRetentionTime = config.accountRetentionTime;
Env.sendMessage = data => {
process.send(data);
};
Object.keys(plugins || {}).forEach(name => {
let plugin = plugins[name];
if (!plugin.initialize) { return; }
try { plugin.initialize(Env, "db-worker"); }
catch (e) {}
});
nThen(function (w) {
Store.create(config, w(function (err, _store) {
if (err) {
w.abort();
return void cb(err);
}
Env.store = store = _store;
}));
Store.create({
filePath: config.pinPath,
archivePath: config.archivePath,
// important to initialize the pinstore with its own volume id
// otherwise archived pin logs will get mixed in with channels
volumeId: 'pins',
}, w(function (err, _pinStore) {
if (err) {
w.abort();
return void cb(err);
}
Env.pinStore = pinStore = _pinStore;
}));
BlobStore.create({
blobPath: config.blobPath,
blobStagingPath: config.blobStagingPath,
archivePath: config.archivePath,
getSession: function () {},
}, w(function (err, blob) {
if (err) {
w.abort();
return void cb(err);
}
Env.blobStore = blobStore = blob;
}));
}).nThen(function (w) {
Tasks.create({
log: Env.Log,
taskPath: config.taskPath,
store: store,
}, w(function (err, tasks) {
if (err) {
w.abort();
return void cb(err);
}
Env.tasks = tasks;
}));
}).nThen(function (w) {
CPCrypto.init(w(function (err, crypto) {
Env.crypto = crypto;
}));
}).nThen(function () {
cb();
});
};
/* computeIndex
can call back with an error or a computed index which includes:
* cpIndex:
* array including any checkpoints pushed within the last 100 messages
* processed by 'sliceCpIndex(cpIndex, line)'
* offsetByHash:
* a map containing message offsets by their hash
* this is for every message in history, so it could be very large...
* except we remove offsets from the map if they occur before the oldest relevant checkpoint
* size: in bytes
* metadata:
* validationKey
* expiration time
* owners
* ??? (anything else we might add in the future)
* line
* the number of messages in history
* including the initial metadata line, if it exists
*/
const OPEN_CURLY_BRACE = Buffer.from('{');
const CHECKPOINT_PREFIX = Buffer.from('cp|');
const isValidOffsetNumber = function (n) {
return typeof(n) === 'number' && n >= 0;
};
const updateEnv = data => {
const {value} = data;
let env = Util.tryParse(value) || {};
Env.proofsMigrated = env?.proofsMigrated;
};
const computeIndexFromOffset = function (channelName, offset, cb) {
let cpIndex = [];
let messageBuf = [];
let i = 0;
const CB = Util.once(cb);
const offsetByHash = {};
let offsetCount = 0;
let size = offset || 0;
var start = offset || 0;
let unconventional = false;
nThen(function (w) {
// iterate over all messages in the channel log
// old channels can contain metadata as the first message of the log
// skip over metadata as that is handled elsewhere
// otherwise index important messages in the log
store.readMessagesBin(channelName, start, (msgObj, readMore, abort) => {
let msg;
// keep an eye out for the metadata line if you haven't already seen it
// but only check for metadata on the first line
if (i) {
// fall through intentionally because the following blocks are invalid
// for all but the first message
} else if (msgObj.buff.includes(OPEN_CURLY_BRACE)) {
msg = HK.tryParse(Env, msgObj.buff.toString('utf8'));
if (typeof msg === "undefined") {
i++; // always increment the message counter
return readMore();
}
// validate that the current line really is metadata before storing it as such
// skip this, as you already have metadata...
if (HK.isMetadataMessage(msg)) {
i++; // always increment the message counter
return readMore();
}
} else if (!(msg = HK.tryParse(Env, msgObj.buff.toString('utf8')))) {
w.abort();
abort();
return CB("OFFSET_ERROR");
}
i++;
if (msgObj.buff.includes(CHECKPOINT_PREFIX)) {
msg = msg || HK.tryParse(Env, msgObj.buff.toString('utf8'));
if (typeof msg === "undefined") { return readMore(); }
// cache the offsets of checkpoints if they can be parsed
if (msg[2] === 'MSG' && msg[4].indexOf('cp|') === 0) {
cpIndex.push({
offset: msgObj.offset,
line: i
});
// we only want to store messages since the latest checkpoint
// so clear the buffer every time you see a new one
messageBuf = [];
}
} else if (messageBuf.length > 100 && cpIndex.length === 0) {
// take the last 50 messages
unconventional = true;
messageBuf = messageBuf.slice(-50);
}
// if it's not metadata or a checkpoint then it should be a regular message
// store it in the buffer
messageBuf.push(msgObj);
return readMore();
}, w((err) => {
if (err && err.code !== 'ENOENT') {
w.abort();
return void CB(err);
}
// once indexing is complete you should have a buffer of messages since the latest checkpoint
// or the 50-100 latest messages if the channel is of a type without checkpoints.
// map the 'hash' of each message to its byte offset in the log, to be used for reconnecting clients
messageBuf.forEach((msgObj) => {
const msg = HK.tryParse(Env, msgObj.buff.toString('utf8'));
if (typeof msg === "undefined") { return; }
if (msg[0] === 0 && msg[2] === 'MSG' && typeof(msg[4]) === 'string') {
// msgObj.offset is API guaranteed by our storage module
// it should always be a valid positive integer
offsetByHash[HK.getHash(msg[4])] = msgObj.offset;
offsetCount++;
}
// There is a trailing \n at the end of the file
size = msgObj.offset + msgObj.buff.length + 1;
});
}));
}).nThen(function (w) {
cpIndex = HK.sliceCpIndex(cpIndex, i);
var new_start;
if (cpIndex.length) {
new_start = cpIndex[0].offset;
} else if (unconventional && messageBuf.length && isValidOffsetNumber(messageBuf[0].offset)) {
new_start = messageBuf[0].offset;
}
if (new_start === start) { return; }
if (!isValidOffsetNumber(new_start)) { return; }
// store the offset of the earliest relevant line so that you can start from there next time...
store.writeOffset(channelName, {
start: new_start,
created: +new Date(),
}, w(function () {
var diff = new_start - start;
Env.Log.info('WORKER_OFFSET_UPDATE', {
channel: channelName,
start: start,
startMB: round(start / 1024 / 1024),
update: new_start,
updateMB: round(new_start / 1024 / 1024),
diff: diff,
diffMB: round(diff / 1024 / 1024),
});
}));
}).nThen(function () {
// return the computed index
CB(null, {
// Only keep the checkpoints included in the last 100 messages
cpIndex: cpIndex,
offsetByHash: offsetByHash,
offsets: offsetCount,
size: size,
//metadata: metadata,
line: i
});
});
};
const computeIndex = function (data, cb) {
if (!data || !data.channel) {
return void cb('E_NO_CHANNEL');
}
const channelName = data.channel;
const CB = Util.once(cb);
monitoringIncrement('computeIndex');
var start = 0;
nThen(function (w) {
store.getOffset(channelName, w(function (err, obj) {
if (err) { return; }
if (obj && typeof(obj.start) === 'number' && obj.start > 0) {
start = obj.start;
Env.Log.verbose('WORKER_OFFSET_RECOVERY', {
channel: channelName,
start: start,
startMB: round(start / 1024 / 1024),
});
}
}));
}).nThen(function (w) {
computeIndexFromOffset(channelName, start, w(function (err, index) {
if (err === 'OFFSET_ERROR') {
return Env.Log.error("WORKER_OFFSET_ERROR", {
channel: channelName,
});
}
w.abort();
monitoringIncrement('computeIndexFromOffset');
CB(err, index);
}));
}).nThen(function (w) {
// if you're here there was an OFFSET_ERROR..
// first remove the offset that caused the problem to begin with
store.clearOffset(channelName, w());
}).nThen(function () {
// now get the history as though it were the first time
monitoringIncrement('computeIndexFromStart');
computeIndexFromOffset(channelName, 0, CB);
});
};
const computeMetadata = function (data, cb) {
const ref = {};
const lineHandler = Meta.createLineHandler(ref, Env.Log.error);
monitoringIncrement('computeMetadata');
let f = store.readChannelMetadata;
if (data.channel.length === HK.BLOB_ID_LENGTH) {
f = blobStore.readMetadata;
}
return void f(data.channel, lineHandler, function (err) {
if (err) {
// stream errors?
return void cb(err);
}
cb(void 0, ref.meta);
});
};
const _getFileSize = function (channel, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); }
if (channel.length === HK.STANDARD_CHANNEL_LENGTH ||
channel.length === HK.ADMIN_CHANNEL_LENGTH) {
return void store.getChannelSize(channel, function (e, size) {
if (e) {
if (e.code === 'ENOENT') { return void cb(void 0, 0); }
return void cb(e.code);
}
cb(void 0, size);
});
}
// 'channel' refers to a file, so you need another API
blobStore.size(channel, function (e, size) {
if (typeof(size) === 'undefined') { return void cb(e); }
cb(void 0, size);
});
};
const getFileSize = function (data, cb) {
_getFileSize(data.channel, cb);
};
/* getOlderHistory
* allows clients to query for all messages until a known hash is read
* stores all messages in history as they are read
* can therefore be very expensive for memory
* should probably be converted to a streaming interface
Used by:
* GET_HISTORY_RANGE
*/
const getOlderHistory = function (data, cb) {
const oldestKnownHash = data.hash;
const channelName = data.channel;
const desiredMessages = data.desiredMessages;
const desiredCheckpoint = data.desiredCheckpoint;
let messages = [];
store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {
const parsed = HK.tryParse(Env, msgObj.buff.toString('utf8'));
if (!parsed) { return void readMore(); }
if (HK.isMetadataMessage(parsed)) { return void readMore(); }
const content = parsed[4];
if (typeof(content) !== 'string') { return void readMore(); }
const hash = HK.getHash(content);
messages.push(parsed);
// "X" messages before oldestKnownHash
if (typeof (desiredMessages) === "number") {
messages = messages.slice(-desiredMessages);
if (hash === oldestKnownHash) { return void abort(); }
return void readMore();
}
// "X" checkpoints before oldestKnownHash
if (hash === oldestKnownHash) { return void abort(); }
if (/^cp\|/.test(content)) { // clean whenever we push a cp
let foundCp = 0;
const idx = messages.findLastIndex(parsed => {
let isCp = /^cp\|/.test(parsed[4]);
if (!isCp) { return; }
foundCp++;
return foundCp >= desiredCheckpoint;
});
if (idx > 0) {
messages = messages.slice(idx);
}
}
readMore();
}, function (err, reason) {
if (err) { return void cb(err, reason); }
cb(void 0, messages);
});
/*
const untilHash = data.toHash;
var next = () => {
var messages = [];
var found = false;
store.getMessages(channelName, function (msgStr) {
if (found) { return; }
let parsed = HK.tryParse(Env, msgStr);
if (typeof parsed === "undefined") { return; }
// identify classic metadata messages by their inclusion of a channel.
// and don't send metadata, since:
// 1. the user won't be interested in it
// 2. this metadata is potentially incomplete/incorrect
if (HK.isMetadataMessage(parsed)) { return; }
var content = parsed[4];
if (typeof(content) !== 'string') { return; }
var hash = HK.getHash(content);
if (hash === oldestKnownHash) {
found = true;
}
messages.push(parsed);
}, function (err) {
var toSend = [];
if (typeof (desiredMessages) === "number") {
toSend = messages.slice(-desiredMessages);
} else if (untilHash) {
for (var j = messages.length - 1; j >= 0; j--) {
toSend.unshift(messages[j]);
if (Array.isArray(messages[j]) && HK.getHash(messages[j][4]) === untilHash) {
break;
}
}
} else {
let cpCount = 0;
for (var i = messages.length - 1; i >= 0; i--) {
if (/^cp\|/.test(messages[i][4]) && i !== (messages.length - 1)) {
cpCount++;
}
toSend.unshift(messages[i]);
if (cpCount >= desiredCheckpoint) { break; }
}
}
cb(err, toSend);
});
};
_getFileSize(channelName, function (err, size) {
if (err) { return void cb(err); }
if (size > HISTORY_SIZE_LIMIT) {
return void cb('HISTORY_TOO_LARGE');
}
next();
});
*/
};
const getPinState = function (data, cb) {
if (typeof(data.key) !== 'string') { return void cb('INVALID_KEY'); }
const safeKey = Util.escapeKeyCharacters(data.key);
var ref = {};
var lineHandler = Pins.createLineHandler(ref, Env.Log.error);
// if channels aren't in memory. load them from disk
monitoringIncrement('getPin');
pinStore.readMessagesBin(safeKey, 0, (msgObj, readMore) => {
lineHandler(msgObj.buff.toString('utf8'));
readMore();
}, function () {
cb(void 0, ref.pins); // FIXME no error handling?
});
};
const _iterateFiles = function (channels, handler, cb) {
if (!Array.isArray(channels)) { return cb('INVALID_LIST'); }
var L = channels.length;
var sem = Saferphore.create(10);
// (channel, next) => { ??? }
var job = function (channel, wait) {
return function (give) {
handler(channel, wait(give()));
};
};
nThen(function (w) {
for (var i = 0; i < L; i++) {
sem.take(job(channels[i], w));
}
}).nThen(function () {
cb();
});
};
const getTotalSize = function (data, cb) {
var bytes = 0;
monitoringIncrement('getTotalSize');
_iterateFiles(data.channels, function (channel, next) {
_getFileSize(channel, function (err, size) {
if (!err) { bytes += size; }
next();
});
}, function (err) {
if (err) { return cb(err); }
cb(void 0, bytes);
});
};
const getDeletedPads = function (data, cb) {
var absentees = [];
_iterateFiles(data.channels, function (channel, next) {
_getFileSize(channel, function (err, size) {
if (err) { return next(); }
if (size === 0) { absentees.push(channel); }
next();
});
}, function (err) {
if (err) { return void cb(err); }
cb(void 0, absentees);
});
};
const getMultipleFileSize = function (data, cb) {
const counts = {};
_iterateFiles(data.channels, function (channel, next) {
_getFileSize(channel, function (err, size) {
counts[channel] = err? 0: size;
next();
});
}, function (err) {
if (err) {
return void cb(err);
}
cb(void 0, counts);
});
};
const getHashOffset = function (data, cb) {
const channelName = data.channel;
const lastKnownHash = data.hash;
if (typeof(lastKnownHash) !== 'string') { return void cb("INVALID_HASH"); }
monitoringIncrement('getHashOffset');
var offset = -1;
store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {
// tryParse return a parsed message or undefined
const msg = HK.tryParse(Env, msgObj.buff.toString('utf8'));
// if it was undefined then go onto the next message
if (typeof msg === "undefined") { return readMore(); }
if (typeof(msg[4]) !== 'string' || lastKnownHash !== HK.getHash(msg[4])) {
return void readMore();
}
offset = msgObj.offset;
abort();
}, function (err, reason) {
if (err) {
return void cb({
error: err,
reason: reason
});
}
cb(void 0, offset);
});
};
const removeOwnedBlob = function (data, cb) {
if (typeof(data.safeKey) !== 'string') { return void cb("INVALID_KEY"); }
const blobId = data.blobId;
const safeKey = Util.escapeKeyCharacters(data.safeKey);
const unsafeKey = Util.unescapeKeyCharacters(data.safeKey);
const reason = data.reason || 'ARCHIVE_OWNED';
nThen(function (w) {
// check if you have permissions
computeMetadata({channel: blobId}, w((err, meta) => {
if (err || !meta) {
w.abort();
return void cb("INSUFFICIENT_PERMISSIONS");
}
let owners = meta.owners;
if (!owners && !Env.proofsMigrated) {
// Check old proofs during migration
blobStore.isOwnedBy(safeKey, blobId, w((e, owned) => {
if (e || !owned) {
w.abort();
return void cb("INSUFFICIENT_PERMISSIONS");
}
}));
return;
}
if (!owners || !owners.includes(unsafeKey)) {
w.abort();
return void cb("INSUFFICIENT_PERMISSIONS");
}
// Owned, continue
}));
}).nThen(function (w) {
// remove the blob
blobStore.archive.blob(blobId, reason, w(function (err) {
Env.Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', {
safeKey: safeKey,
blobId: blobId,
status: err? String(err): 'SUCCESS',
});
if (err) {
w.abort();
return void cb(err);
}
cb(void 0, 'OK');
}));
});
};
const runTasks = function (data, cb) {
Env.tasks.runAll(cb);
};
const writeTask = function (data, cb) {
Env.tasks.write(data.time, data.task_command, data.args, cb);
};
const evictInactive = function (data, cb) {
Eviction(Env, cb);
};
var reportStatus = function (Env, label, safeKey, err, id, size) {
var data = {
safeKey: safeKey,
err: err && err.message || err,
id: id,
size: size,
sizeMB: round((size || 0) / 1024 / 1024),
};
var method = err? 'error': 'info';
Env.Log[method](label, data);
};
const completeUpload = function (data, cb) {
if (!data) { return void cb('INVALID_ARGS'); }
if (typeof(data.safeKey) !== 'string') { return void cb("INVALID_KEY"); }
var owned = data.owned;
var safeKey = Util.escapeKeyCharacters(data.safeKey);
var arg = data.arg;
var size = data.size;
monitoringIncrement('uploadedBlob');
var method;
var label;
if (owned) {
method = 'completeOwned';
label = 'UPLOAD_COMPLETE_OWNED';
} else {
method = 'complete';
label = 'UPLOAD_COMPLETE';
}
Env.blobStore[method](safeKey, arg, function (err, id) {
reportStatus(Env, label, safeKey, err, id, size);
cb(err, id);
}, data.linked);
};
const getPinActivity = function (data, cb) {
if (!data) { return void cb("INVALID_ARGS"); }
if (typeof(data.key) !== 'string') { return void cb("INVALID_KEY"); }
var safeKey = Util.escapeKeyCharacters(data.key);
var first;
var latest;
pinStore.readMessagesBin(safeKey, 0, (msgObj, readMore) => {
var line = msgObj.buff.toString('utf8');
if (!line || !line.trim()) { return readMore(); }
try {
var parsed = JSON.parse(line);
var temp = parsed[parsed.length - 1];
if (!temp || typeof(temp) !== 'number') { return readMore(); }
latest = temp;
if (first) { return readMore(); }
first = latest;
readMore();
} catch (err) { readMore(); }
}, function (err) {
if (err) { return void cb(err); }
cb(void 0, {
first: first,
latest: latest,
});
});
};
const getLastChannelTime = function (data, cb) {
if (!data) { return void cb("INVALID_ARGS"); }
var latest;
store.getMessages(data.channel, function (line) {
try {
var parsed = JSON.parse(line);
var temp = parsed[parsed.length - 1];
if (!temp || typeof(temp) !== 'number') { return; }
latest = temp;
} catch (err) { }
}, function (err) {
if (err) { return void cb(err); }
cb(void 0, latest);
});
};
const COMMANDS = {
ENV_UPDATE: updateEnv,
COMPUTE_INDEX: computeIndex,
COMPUTE_METADATA: computeMetadata,
GET_OLDER_HISTORY: getOlderHistory,
GET_PIN_STATE: getPinState,
GET_FILE_SIZE: getFileSize,
GET_TOTAL_SIZE: getTotalSize,
GET_DELETED_PADS: getDeletedPads,
GET_MULTIPLE_FILE_SIZE: getMultipleFileSize,
GET_HASH_OFFSET: getHashOffset,
REMOVE_OWNED_BLOB: removeOwnedBlob,
RUN_TASKS: runTasks,
WRITE_TASK: writeTask,
EVICT_INACTIVE: evictInactive,
COMPLETE_UPLOAD: completeUpload,
GET_PIN_ACTIVITY: getPinActivity,
GET_LAST_CHANNEL_TIME: getLastChannelTime,
};
COMMANDS.INLINE = function (data, cb) {
monitoringIncrement('inlineValidation');
var signedMsg;
try {
signedMsg = Util.decodeBase64(data.msg);
} catch (e) {
return void cb('E_BAD_MESSAGE');
}
var validateKey;
try {
validateKey = Util.decodeBase64(data.key);
} catch (e) {
return void cb("E_BADKEY");
}
// validate the message
//const validated = Nacl.sign.open(signedMsg, validateKey);
const validated = Env.crypto.open(signedMsg, validateKey);
if (!validated) {
return void cb("FAILED");
}
cb();
};
const checkDetachedSignature = function (signedMsg, signature, publicKey) {
if (!(signedMsg && publicKey)) { return false; }
monitoringIncrement('detachedValidation');
var signedBuffer;
var pubBuffer;
var signatureBuffer;
try {
signedBuffer = Util.decodeUTF8(signedMsg);
} catch (e) {
throw new Error("INVALID_SIGNED_BUFFER");
}
try {
pubBuffer = Util.decodeBase64(publicKey);
} catch (e) {
throw new Error("INVALID_PUBLIC_KEY");
}
try {
signatureBuffer = Util.decodeBase64(signature);
} catch (e) {
throw new Error("INVALID_SIGNATURE");
}
if (pubBuffer.length !== 32) {
throw new Error("INVALID_PUBLIC_KEY_LENGTH");
}
if (signatureBuffer.length !== 64) {
throw new Error("INVALID_SIGNATURE_LENGTH");
}
//if (Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer) !== true) {
if (Env.crypto.detachedVerify(signedBuffer, signatureBuffer, pubBuffer) !== true) {
throw new Error("FAILED");
}
};
COMMANDS.DETACHED = function (data, cb) {
try {
checkDetachedSignature(data.msg, data.sig, data.key);
} catch (err) {
return void cb(err && err.message);
}
cb();
};
COMMANDS.HASH_CHANNEL_LIST = function (data, cb) {
var channels = data.channels;
if (!Array.isArray(channels)) { return void cb('INVALID_CHANNEL_LIST'); }
var uniques = [];
channels.forEach(function (a) {
if (uniques.indexOf(a) === -1) { uniques.push(a); }
});
uniques.sort();
var hash = Util.encodeBase64(Nacl.hash(Util.decodeUTF8(JSON.stringify(uniques))));
cb(void 0, hash);
};
COMMANDS.VALIDATE_ANCESTOR_PROOF = function (data, cb) {
monitoringIncrement('validateAncestorProof');
Block.validateAncestorProof(Env, data && data.proof, cb);
};
COMMANDS.VALIDATE_LOGIN_BLOCK = function (data, cb) {
monitoringIncrement('validateLoginBlock');
Block.validateLoginBlock(Env, data.publicKey, data.signature, data.block, cb);
};
Object.keys(plugins || {}).forEach(name => {
let plugin = plugins[name];
if (!plugin.addWorkerCommands) { return; }
try {
let events = plugin.addWorkerCommands(Env);
Object.keys(events || {}).forEach(cmd => {
if (typeof(events[cmd]) !== "function") { return; }
if (COMMANDS[cmd]) { return; }
COMMANDS[cmd] = events[cmd];
});
} catch (e) {}
});
process.on('message', function (data) {
if (!data || !data.txid || !data.pid) {
return void process.send({
error:'E_INVAL',
data: data,
});
}
const command = COMMANDS[data.command];
// Command broadcasted to all workers: no callback expected
if (data.type === 'broadcast') {
return void command(data);
}
const cb = function (err, value) {
process.send({
error: Util.serializeError(err),
txid: data.txid,
pid: data.pid,
value: value,
});
};
if (!ready) {
if (data.env) {
updateEnv({value:data.env});
}
return void init(data.config, function (err) {
if (err) { return void cb(Util.serializeError(err)); }
ready = true;
cb();
});
}
if (typeof(command) !== 'function') {
return void cb("E_BAD_COMMAND");
}
command(data, cb);
});
process.on('uncaughtException', function (err) {
console.error('[%s] UNCAUGHT EXCEPTION IN DB WORKER', new Date());
console.error(err);
console.error("TERMINATING");
process.exit(1);
});

628
lib/workers/index.js
View File

@ -1,628 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
const Util = require("../common-util");
const nThen = require('nthen');
const OS = require("os");
const { fork } = require('child_process');
const Workers = module.exports;
const PID = process.pid;
const Block = require("../storage/block");
const Environment = require('../env');
const Linked = require("../commands/linked");
const DB_PATH = 'lib/workers/db-worker';
const MAX_JOBS = 16;
const DEFAULT_QUERY_TIMEOUT = 60000 * 15; // increased from three to fifteen minutes because queries for very large files were taking as long as seven minutes
Workers.initialize = function (Env, config, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
var incrementTime = function (command, start) {
if (!command) { return; }
var end = +new Date();
var T = Env.commandTimers;
var diff = (end - start);
T[command] = (T[command] || 0) + (diff / 1000);
};
const workers = [];
const response = Util.response(function (errLabel, info) {
Env.Log.error('HK_DB_WORKER__' + errLabel, info);
});
const Log = Env.Log;
const handleLog = function (level, label, info) {
if (typeof(Log[level]) !== 'function') { return; }
Log[level](label, info);
};
var isWorker = function (value) {
return value && value.worker && typeof(value.worker.send) === 'function';
};
// pick ids that aren't already in use...
const guid = function () {
var id = Util.uid();
return response.expected(id)? guid(): id;
};
const countWorkerTasks = function (/* index */) {
return 0; // FIXME this disables all queueing until it can be proven correct
//return Object.keys(workers[index].tasks || {}).length;
};
const WORKER_TASK_LIMIT = 250000; // XXX
var workerOffset = -1;
var queue = [];
var getAvailableWorkerIndex = function (isQueue) {
// If there is already a backlog of tasks you can avoid some work
// by going to the end of the line (unless we're trying to
// empty the queue)
if (queue.length && !isQueue) { return -1; }
var L = workers.length;
if (L === 0) {
Log.warn('NO_WORKERS_AVAILABLE', {
queue: queue.length,
});
return -1;
}
// cycle through the workers once
// start from a different offset each time
// return -1 if none are available
workerOffset = (workerOffset + 1) % L;
var temp;
for (let i = 0; i < L; i++) {
temp = (workerOffset + i) % L;
/* I'd like for this condition to be more efficient
(`Object.keys` is sub-optimal) but I found some bugs in my initial
implementation stemming from a task counter variable going out-of-sync
with reality when a worker crashed and its tasks were re-assigned to
its substitute. I'm sure it can be done correctly and efficiently,
but this is a relatively easy way to make sure it's always up to date.
We'll see how it performs in practice before optimizing.
*/
if (workers[temp] && countWorkerTasks(temp) <= MAX_JOBS) {
return temp;
}
}
return -1;
};
var drained = true;
var sendCommand = function (msg, _cb, opt, isQueue) {
if (!_cb) {
return void Log.error('WORKER_COMMAND_MISSING_CB', {
msg: msg,
opt: opt,
});
}
opt = opt || {};
var index = getAvailableWorkerIndex(isQueue);
var state = workers[index];
// if there is no worker available:
if (!isWorker(state)) {
// queue the message for when one becomes available
queue.push({
msg: msg,
cb: _cb,
});
if (drained) {
drained = false;
Log.warn('WORKER_QUEUE_BACKLOG', {
workers: workers.length,
});
}
return;
}
const txid = guid();
var start = +new Date();
var cb = Util.once(Util.mkAsync(Util.both(_cb, function (err /*, value */) {
incrementTime(msg && msg.command, start);
if (err !== 'TIMEOUT') { return; }
Log.warn("WORKER_TIMEOUT_CAUSE", msg);
// in the event of a timeout the user will receive an error
// but the state used to resend a query in the event of a worker crash
// won't be cleared. This also leaks a slot that could be used to keep
// an upper bound on the amount of parallelism for any given worker.
// if you run out of slots then the worker locks up.
delete state.tasks[txid];
state.checkTasks();
})));
if (!msg) {
return void cb('ESERVERERR');
}
msg.txid = txid;
msg.pid = PID;
// include the relevant worker process id in messages so that it will be logged
// in the event that the message times out or fails in other ways.
msg.worker = state.pid;
// track which worker is doing which jobs
state.tasks[txid] = msg;
// default to timing out affter 180s if no explicit timeout is passed
var timeout = typeof(opt.timeout) !== 'undefined'? opt.timeout: DEFAULT_QUERY_TIMEOUT;
response.expect(txid, cb, timeout);
delete msg._cb;
delete msg._opt;
state.worker.send(msg);
// Add original callback to message data in case we need
// to resend the command. setTimeout to avoid interfering
// with worker.send
setTimeout(function () {
msg._cb = _cb;
msg._opt = opt;
});
state.count++;
if (state.count > WORKER_TASK_LIMIT) {
// Remove from list and spawn new one
if (state.replaceWorker) { state.replaceWorker(); }
}
};
const pluginsResponses = {};
Object.keys(Env.plugins || {}).forEach(name => {
let plugin = Env.plugins[name];
if (!plugin.addWorkerResponses) { return; }
try {
let res = plugin.addWorkerResponses(Env);
Object.keys(res || {}).forEach(key => {
if (typeof(res[key]) !== "function") { return; }
if (pluginsResponses[key]) { return; }
pluginsResponses[key] = res[key];
});
} catch (e) {}
});
var handleResponse = function (state, res) {
if (!res) { return; }
// handle log messages before checking if it was addressed to your PID
// it might still be useful to know what happened inside an orphaned worker
if (res.log) {
return void handleLog(res.log, res.label, res.info);
}
// handle plugins
if (res.plugin) {
Object.keys(pluginsResponses).some(key => {
if (res.type !== key) { return; }
pluginsResponses[key](res.data);
return true;
});
return;
}
// but don't bother handling things addressed to other processes
// since it's basically guaranteed not to work
if (res.pid !== PID) {
return void Log.error("WRONG_PID", res);
}
if (!res.txid) { return; }
response.handle(res.txid, [res.error, res.value]);
delete state.tasks[res.txid];
state.checkTasks();
if (!queue.length) {
if (!drained) {
drained = true;
Log.debug('WORKER_QUEUE_DRAINED', {
workers: workers.length,
});
}
return;
}
var nextMsg = queue.shift();
if (!nextMsg || !nextMsg.msg) {
return void Log.error('WORKER_QUEUE_EMPTY_MESSAGE', {
item: nextMsg,
});
}
/* `nextMsg` was at the top of the queue.
We know that a job just finished and all of this code
is synchronous, so calling `sendCommand` should take the worker
which was just freed up. This is somewhat fragile though, so
be careful if you want to modify this block. The risk is that
we take something that was at the top of the queue and push it
to the back because the following msg took its place. OR, in an
even worse scenario, we cycle through the queue but don't run anything.
*/
sendCommand(nextMsg.msg, nextMsg.cb, {}, true);
};
const initWorker = function (worker, cb) {
const txid = guid();
const state = {
worker: worker,
tasks: {},
count: Math.floor(Math.random()*(WORKER_TASK_LIMIT/10)),
pid: worker.pid, // store the child process's id in an easily accessible location
};
let pid = worker.pid;
const onWorkerClosed = () => {
Object.keys(Env.plugins || {}).forEach(name => {
let plugin = Env.plugins[name];
if (!plugin.onWorkerClosed) { return; }
try { plugin.onWorkerClosed("db-worker", pid); }
catch (e) {}
});
};
state.replaceWorker = () => {
let index = workers.indexOf(state);
if (index === -1) { return; }
// Remove old
workers.splice(index, 1);
// Create new
state.complete = true;
const w = fork(DB_PATH);
Log.info('WORKER_REPLACE_START', {
from: state.worker.pid,
to: w.pid
});
initWorker(w, function (err) {
if (err) {
throw new Error(err);
}
});
};
// If we've reached the limit, kill the worker once
// all the tasks are complete or timed out
state.checkTasks = () => {
// Check limit
if (!state.complete || !state.worker) { return; }
// Check remaining tasks
if (Object.keys(state.tasks).length) { return; }
// Kill
Log.info('WORKER_KILL', {
worker: state.worker.pid,
count: state.count
});
onWorkerClosed();
delete state.worker;
worker.kill();
};
response.expect(txid, function (err) {
if (err) { return void cb(err); }
workers.push(state);
cb(void 0, state);
// We just pushed a new worker, available to receive
// a task, so we can empty the queue if necessary
if (queue.length) {
const nextMsg = queue.shift();
if (!nextMsg || !nextMsg.msg) {
return Log.error('WORKER_QUEUE_EMPTY_MESSAGE', {
item: nextMsg,
});
}
sendCommand(nextMsg.msg, nextMsg.cb, {}, true);
}
}, 15000);
worker.send({
pid: PID,
txid: txid,
config: config,
env: Environment.serialize(Env)
});
worker.on('message', function (res) {
handleResponse(state, res);
});
var substituteWorker = Util.once(function () {
onWorkerClosed();
Env.Log.info("SUBSTITUTE_DB_WORKER", '');
var idx = workers.indexOf(state);
if (idx !== -1) {
workers.splice(idx, 1);
}
Object.keys(state.tasks).forEach(function (txid) {
const cb = response.expectation(txid);
if (typeof(cb) !== 'function') { return; }
const task = state.tasks[txid];
if (!task) { return; }
response.clear(txid);
Log.info('DB_WORKER_RESEND', task);
sendCommand(task, task._cb || cb, task._opt);
});
var w = fork(DB_PATH);
initWorker(w, function (err) {
if (err) {
throw new Error(err);
}
});
});
worker.on('exit', function () {
if (!state.worker) { return; } // Manually killed
substituteWorker();
Env.Log.error("DB_WORKER_EXIT", {
pid: state.pid,
});
});
worker.on('close', function () {
if (!state.worker) { return; } // Manually killed
substituteWorker();
Env.Log.error("DB_WORKER_CLOSE", {
pid: state.pid,
});
});
worker.on('error', function (err) {
if (!state.worker) { return; } // Manually killed
substituteWorker();
Env.Log.error("DB_WORKER_ERROR", {
pid: state.pid,
error: err,
});
});
};
nThen(function (w) {
var limit = Env.maxWorkers;
var logged;
OS.cpus().forEach(function (cpu, index) {
if (limit && index >= limit) {
if (!logged) {
logged = true;
Log.info('WORKER_LIMIT', "(Opting not to use available CPUs beyond " + index + ')');
}
return;
}
initWorker(fork(DB_PATH), w(function (err) {
if (!err) { return; }
w.abort();
return void cb(err);
}));
});
}).nThen(function () {
Env.broadcastWorkerCommand = (data) => {
workers.forEach(state => {
state.worker.send({
type: 'broadcast',
pid: PID,
command: data.command,
txid: data.txid,
value: data.value
});
});
return workers;
};
Env.computeIndex = function (Env, channel, cb) {
Env.store.getWeakLock(channel, function (next) {
sendCommand({
channel: channel,
command: 'COMPUTE_INDEX',
}, function (err, index) {
next();
cb(err, index);
});
});
};
Env.computeMetadata = function (channel, cb) {
Env.store.getWeakLock(channel, function (next) {
sendCommand({
channel: channel,
command: 'COMPUTE_METADATA',
}, function (err, metadata) {
next();
cb(err, metadata);
});
});
};
Env.getOlderHistory = function (channel, oldestKnownHash, toHash, desiredMessages, desiredCheckpoint, cb) {
Env.store.getWeakLock(channel, function (next) {
sendCommand({
channel: channel,
command: "GET_OLDER_HISTORY",
hash: oldestKnownHash,
toHash: toHash,
desiredMessages: desiredMessages,
desiredCheckpoint: desiredCheckpoint,
}, Util.both(next, cb));
});
};
Env.getPinState = function (safeKey, cb) {
Env.pinStore.getWeakLock(safeKey, function (next) {
sendCommand({
key: safeKey,
command: 'GET_PIN_STATE',
}, Util.both(next, cb));
});
};
Env.getPinActivity = function (safeKey, cb) {
Env.pinStore.getWeakLock(safeKey, function (next) {
sendCommand({
key: safeKey,
command: 'GET_PIN_ACTIVITY',
}, Util.both(next, cb));
});
};
Env.getLastChannelTime = function (channel, cb) {
sendCommand({
command: 'GET_LAST_CHANNEL_TIME',
channel: channel,
}, cb);
};
Env.getFileSize = function (channel, cb, singleFile) {
if (!singleFile) {
return Linked.getFileSize(Env, { channel }, cb);
}
sendCommand({
command: 'GET_FILE_SIZE',
singleFile: singleFile, // ignore linked documents
channel: channel,
}, cb);
};
Env.getDeletedPads = function (channels, cb) {
sendCommand({
command: "GET_DELETED_PADS",
channels: channels,
}, cb);
};
Env.getTotalSize = function (channels, cb) {
// we could take out locks for all of these channels,
// but it's OK if the size is slightly off
sendCommand({
command: 'GET_TOTAL_SIZE',
channels: channels,
}, cb);
};
Env.getMultipleFileSize = function (channels, cb) {
sendCommand({
command: "GET_MULTIPLE_FILE_SIZE",
channels: channels,
}, cb);
};
Env.getHashOffset = function (channel, hash, cb) {
Env.store.getWeakLock(channel, function (next) {
sendCommand({
command: 'GET_HASH_OFFSET',
channel: channel,
hash: hash,
}, Util.both(next, cb));
});
};
Env.removeOwnedBlob = function (blobId, safeKey, reason, cb) {
sendCommand({
command: 'REMOVE_OWNED_BLOB',
blobId: blobId,
safeKey: safeKey,
reason: reason
}, cb);
};
Env.evictInactive = function (cb) {
sendCommand({
command: 'EVICT_INACTIVE',
}, cb, {
timeout: 1000 * 60 * 300, // time out after 300 minutes (5 hours)
});
};
Env.runTasks = function (cb) {
sendCommand({
command: 'RUN_TASKS',
}, cb, {
timeout: 1000 * 60 * 10, // time out after 10 minutes
});
};
Env.writeTask = function (time, command, args, cb) {
sendCommand({
command: 'WRITE_TASK',
time: time,
task_command: command,
args: args,
}, cb);
};
// Synchronous crypto functions
Env.validateMessage = function (signedMsg, key, cb) {
sendCommand({
msg: signedMsg,
key: key,
command: 'INLINE',
}, cb);
};
Env.checkSignature = function (signedMsg, signature, publicKey, cb) {
sendCommand({
command: 'DETACHED',
sig: signature,
msg: signedMsg,
key: publicKey,
}, cb);
};
Env.hashChannelList = function (channels, cb) {
sendCommand({
command: 'HASH_CHANNEL_LIST',
channels: channels,
}, cb);
};
Env.completeUpload = function (safeKey, arg, owned, size, linked, cb) {
sendCommand({
command: "COMPLETE_UPLOAD",
linked,
owned: owned, // Boolean
safeKey: safeKey, // String (public key)
arg: arg, // String (file id)
size: size, // Number || undefined
}, cb);
};
Env.validateAncestorProof = function (proof, cb) {
sendCommand({
command: 'VALIDATE_ANCESTOR_PROOF',
proof: proof,
}, cb);
};
Env.validateLoginBlock = function (publicKey, signature, block, cb) {
if (!block || !block.length || block.length > Block.MAX_SIZE) {
return void setTimeout(function () {
Env.Log.error('E_INVALID_BLOCK_SIZE', {
size: block.length,
});
cb('E_INVALID_BLOCK_SIZE');
});
}
sendCommand({
command: 'VALIDATE_LOGIN_BLOCK',
publicKey: publicKey,
signature: signature,
block: block,
}, cb);
};
cb(void 0);
});
};

46
lib/write-queue.js
View File

@ -1,46 +0,0 @@
// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
//
// SPDX-License-Identifier: AGPL-3.0-or-later
/*
var q = Queue();
q(id, function (next) {
// whatever you need to do....
// when you're done
next(); // guaranteed to be asynchronous :D
});
*/
var fix1 = function (f, x) {
return function () { f(x); };
};
module.exports = function () {
var map = {};
var next = function (id) {
setTimeout(function () {
if (!map[id] || map[id].length === 0) { return void delete map[id]; }
var task = map[id].shift();
task(fix1(next, id));
});
};
return function (id, task) {
// support initialization with just a function
if (typeof(id) === 'function' && typeof(task) === 'undefined') {
task = id;
id = '';
}
// ...but you really need to pass a function
if (typeof(task) !== 'function') { throw new Error("Expected function"); }
// if the intended queue already has tasks in progress, add this one to the end of the queue
if (map[id]) { return void map[id].push(task); }
// otherwise create a queue containing the given task
map[id] = [task];
next(id);
};
};