From 9be21c21c46bb65bf92c279f33014f1cd7b80afa Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 29 Aug 2023 15:19:34 +0200 Subject: [PATCH 01/41] WIP blob metadata --- lib/commands/metadata.js | 6 ++++- lib/hk-util.js | 2 ++ lib/storage/blob.js | 50 +++++++++++++++++++++++++++++++++++++++- lib/workers/db-worker.js | 1 + 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js index 9b6a23e02..70a811664 100644 --- a/lib/commands/metadata.js +++ b/lib/commands/metadata.js @@ -10,7 +10,8 @@ Data.getMetadataRaw = function (Env, channel /* channelName */, _cb) { const cb = Util.once(Util.mkAsync(_cb)); if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } if (channel.length !== HK.STANDARD_CHANNEL_LENGTH && - channel.length !== HK.ADMIN_CHANNEL_LENGTH) { return cb("INVALID_CHAN_LENGTH"); } + channel.length !== HK.ADMIN_CHANNEL_LENGTH && + channel.length !== HK.BLOB_ID_LENGTH) { return cb("INVALID_CHAN_LENGTH"); } // return synthetic metadata for admin broadcast channels as a safety net // in case anybody manages to write metadata @@ -76,6 +77,7 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) { var channel = data.channel; var command = data.command; + // XXX BLOBMD allow blobs if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } if (!command || typeof (command) !== 'string') { return void cb('INVALID_COMMAND'); } if (Meta.commands.indexOf(command) === -1) { return void cb('UNSUPPORTED_COMMAND'); } @@ -137,6 +139,7 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) { cb(void 0, metadata); return void next(); } + // XXX BLOBMD use correct store for blobs Env.msgStore.writeMetadata(channel, JSON.stringify(line), function (e) { if (e) { cb(e); @@ -152,6 +155,7 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) { // update the cached metadata metadata_cache[channel] = metadata; + Env.checkCache(channel); // XXX ??? // it's easy to check if the channel is restricted const isRestricted = metadata.restricted; diff --git a/lib/hk-util.js b/lib/hk-util.js index 455964ccb..472e9efb1 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -40,6 +40,8 @@ const ADMIN_CHANNEL_LENGTH = HK.ADMIN_CHANNEL_LENGTH = 33; // with a 34 character id const EPHEMERAL_CHANNEL_LENGTH = HK.EPHEMERAL_CHANNEL_LENGTH = 34; +const BLOB_ID_LENGTH = HK.BLOB_ID_LENGTH = 48; + // Temporary channels are archived X ms after everyone has left them const TEMPORARY_CHANNEL_LIFETIME = 30 * 1000; diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 7d55676d7..9126c761c 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -7,6 +7,11 @@ var BlobStore = module.exports; var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../common-util"); +var Meta = require("../metadata"); + +const BatchRead = require("../batch-read"); +const readFileBin = require("../stream-file").readFileBin; +const Schedule = require("../schedule"); var isValidSafeKey = function (safeKey) { return typeof(safeKey) === 'string' && !/\//.test(safeKey) && safeKey.length === 44; @@ -26,11 +31,16 @@ var prependArchive = function (Env, path) { return Path.join(Env.archivePath, 'blob', relativePathToBlob); }; -// /blob//// +// /blob// var makeBlobPath = function (Env, blobId) { return Path.join(Env.blobPath, blobId.slice(0, 2), blobId); }; +// /blob//.metadata.ndjson +var mkMetadataPath = function (env, channelId) { + return Path.join(env.root, channelId.slice(0, 2), channelId) + '.metadata.ndjson'; +}; + // /blobstate// var makeStagePath = function (Env, safeKey) { return Path.join(Env.blobStagingPath, safeKey.slice(0, 2), safeKey); @@ -355,6 +365,43 @@ var restoreProof = function (Env, safeKey, blobId, cb) { Fse.move(archivePath, proofPath, cb); }; +var getDedicatedMetadata = function (env, blobId, handler, _cb) { + var metadataPath = mkMetadataPath(env, blobId); + var stream = Fs.createReadStream(metadataPath, {start: 0}); + + const collector = createIdleStreamCollector(stream); + var cb = Util.both(_cb, collector); + + readFileBin(stream, function (msgObj, readMore) { + collector.keepAlive(); + var line = msgObj.buff.toString('utf8'); + try { + var parsed = JSON.parse(line); + handler(null, parsed); + } catch (err) { + handler(err, line); + } + readMore(); + }, function (err) { + // ENOENT => there is no metadata log + if (!err || err.code === 'ENOENT') { return void cb(); } + // otherwise stream errors? + cb(err); + }); +}; +/* readMetadata + Load the log of metadata amendments. +*/ +var readMetadata = function (Env, blobId, handler, cb) { + getDedicatedMetadata(env, channelId, handler, function (err) { + if (err) { + // stream errors? + return void cb(err); + } + cb(); + }); +}; + var makeWalker = function (n, handleChild, done) { if (!n || typeof(n) !== 'number' || n < 2) { n = 2; } @@ -486,6 +533,7 @@ BlobStore.create = function (config, _cb) { archivePath: config.archivePath || './data/archive', getSession: config.getSession, }; + var schedule = Env.schedule = Schedule(); nThen(function (w) { var CB = Util.both(w.abort, cb); diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index d0b460b11..adf21456a 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -307,6 +307,7 @@ const computeIndex = function (data, cb) { const computeMetadata = function (data, cb) { const ref = {}; const lineHandler = Meta.createLineHandler(ref, Env.Log.error); + // XXX BLOBMD use correct store return void store.readChannelMetadata(data.channel, lineHandler, function (err) { if (err) { // stream errors? From 0a0b018df0fa25c71eb912539d796afcab5b2757 Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Fri, 8 Nov 2024 11:35:12 +0100 Subject: [PATCH 02/41] Form answers download in Drive zip - WIP --- www/common/make-backup.js | 4 + www/common/sframe-common-outer.js | 158 ++++++++++++++++++++++++++++++ www/form/main.js | 6 ++ 3 files changed, 168 insertions(+) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index 4e1d2b205..405427020 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -239,6 +239,10 @@ define([ transform(ctx, parsed.type, val, function (res) { if (ctx.stop) { return; } if (!res.data) { return void error('EEMPTY'); } + ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", JSON.parse(val)["answers"], function (err, obj) { + var answers = obj && obj.results; + console.log("ANSWERS", answers) + }); var fileName = getUnique(sanitize(rawName), res.ext, existingNames); existingNames.push(fileName.toLowerCase()); zip.file(fileName, res.data, opts); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index b4031d155..3f6d9aeae 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -9,7 +9,10 @@ define([ '/common/requireconfig.js', '/customize/messages.js', 'jquery', + '/components/tweetnacl/nacl-fast.min.js', + ], function (nThen, ApiConfig, RequireConfig, Messages, $) { + var Nacl = window.nacl; var common = {}; var embeddableApps = [ @@ -2429,6 +2432,161 @@ define([ Utils.Feedback.send("BURN_AFTER_READING", Boolean(cfg.noDrive)); }); + sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { + var cb = Utils.Util.once(_cb); + var myKeys = {}; + var myFormKeys; + var accessKeys; + var CPNetflux, Pinpad; + var network; + var noDriveAnswered = false; + nThen(function (w) { + require([ + 'chainpad-netflux', + '/common/pinpad.js', + ], w(function (_CPNetflux, _Pinpad) { + CPNetflux = _CPNetflux; + Pinpad = _Pinpad; + })); + var personalDrive = !Cryptpad.initialTeam || Cryptpad.initialTeam === -1; + Cryptpad.getAccessKeys(w(function (_keys) { + if (!Array.isArray(_keys)) { return; } + accessKeys = _keys; + + _keys.some(function (_k) { + if ((personalDrive && !_k.id) || Cryptpad.initialTeam === Number(_k.id)) { + myKeys = _k; + return true; + } + }); + })); + Cryptpad.getFormKeys(w(function (keys) { + if (!keys.curvePublic && !keys.formSeed) { + // No drive mode + var answered = JSON.parse(localStorage.CP_formAnswered || "[]"); + noDriveAnswered = answered.indexOf(data.channel) !== -1; + } + myFormKeys = keys; + })); + Cryptpad.makeNetwork(w(function (err, nw) { + network = nw; + })); + Cryptpad.getPadMetadata({channel: data.channel}, w(function (md) { + if (md && md.deleteLines) { deleteLines = true; } + })); + }).nThen(function () { + if (!network) { return void cb({error: "E_CONNECT"}); } + var getAnonymousKeys = function (formSeed, channel) { + var array = Nacl.util.decodeBase64(formSeed + channel); + var hash = Nacl.hash(array); + var secretKey = Nacl.util.encodeBase64(hash.subarray(32)); + var publicKey = Utils.Hash.getCurvePublicFromPrivate(secretKey); + return { + curvePrivate: secretKey, + curvePublic: publicKey, + }; + }; + if (myFormKeys.formSeed) { + myFormKeys = getAnonymousKeys(myFormKeys.formSeed, data.channel); + } + + var keys = Utils.secret && Utils.secret.keys; + + var formData = Utils.Hash.getFormData(Utils.secret); + console.log("SECRET", Utils.secret) + console.log("FORMDATA", formData) + privateKey = formData.form_private; + publicKey = formData.form_public; + + var curvePrivate = privateKey || data.privateKey; + if (!curvePrivate) { return void cb({error: 'EFORBIDDEN'}); } + var crypto = Utils.Crypto.Mailbox.createEncryptor({ + curvePrivate: curvePrivate, + curvePublic: publicKey || data.publicKey, + validateKey: data.validateKey + }); + + console.log("PRIVATE", curvePrivate) + console.log("PUBLIC", publicKey, data.publicKey) + console.log("VALIDATE", data.validateKey) + + var config = { + network: network, + channel: data.channel, + noChainPad: true, + validateKey: keys.secondaryValidateKey, + owners: [myKeys.edPublic], + crypto: crypto, + metadata: { + deleteLines: true + } + //Cache: Utils.Cache // TODO enable cache for form responses when the cache stops evicting old answers + }; + var results = {}; + config.onError = function (info) { + cb({ error: info.type }); + }; + config.onRejected = function (data, cb) { + if (!Array.isArray(data) || !data.length || data[0].length !== 16) { + return void cb(true); + } + if (!Array.isArray(accessKeys)) { return void cb(true); } + network.historyKeeper = data[0]; + nThen(function (waitFor) { + accessKeys.forEach(function (obj) { + Pinpad.create(network, obj, waitFor(function (e) { + if (e) { console.error(e); } + })); + }); + }).nThen(function () { + cb(); + }); + }; + config.onReady = function () { + var myKey; + // If we have submitted an anonymous answer, retrieve it + if (myFormKeys.curvePublic && results[myFormKeys.curvePublic]) { + myKey = myFormKeys.curvePublic; + } + cb({ + noDriveAnswered: noDriveAnswered, + myKey: myKey, + results: results + }); + network.disconnect(); + }; + config.onMessage = function (msg, peer, vKey, isCp, hash, senderCurve, cfg) { + var parsed = Utils.Util.tryParse(msg); + if (!parsed) { return; } + var uid = parsed._uid || '000'; + + // If we have a "non-anonymous" answer, it may be the edition of a + // previous anonymous answer. Check if a previous anonymous answer exists + // with the same uid and delete it. + if (parsed._proof) { + var check = checkAnonProof(parsed._proof, data.channel, curvePrivate); + var theirAnonKey = parsed._proof.key; + if (check && results[theirAnonKey] && results[theirAnonKey][uid]) { + delete results[theirAnonKey][uid]; + } + } + + parsed._time = cfg && cfg.time; + if (deleteLines) { parsed._hash = hash; } + + if (data.cantEdit && results[senderCurve] + && results[senderCurve][uid]) { return; } + results[senderCurve] = results[senderCurve] || {}; + results[senderCurve][uid] = { + msg: parsed, + hash: hash, + time: cfg && cfg.time + }; + }; + CPNetflux.start(config); + }); + }); + sframeChan.ready(); Utils.Feedback.reportAppUsage(); diff --git a/www/form/main.js b/www/form/main.js index 05a4536dd..5efc81aab 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -36,6 +36,8 @@ define([ } var formData = Utils.Hash.getFormData(Utils.secret); + console.log("SECRET", Utils.secret) + console.log("FORMDATA", formData) if (!formData) { return; } var validateKey = keys.secondaryValidateKey; @@ -187,6 +189,10 @@ define([ curvePublic: publicKey || data.publicKey, validateKey: data.validateKey }); + + console.log("PRIVATE", curvePrivate) + console.log("PUBLIC", publicKey, data.publicKey) + console.log("VALIDATE", data.validateKey) var config = { network: network, channel: data.channel, From af9d7646f78488fee935a7d57ddb85d6edda311b Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Mon, 11 Nov 2024 23:10:51 +0100 Subject: [PATCH 03/41] Moved FORM_FETCH_ANSWERS to sframe-common-outer.js --- www/common/make-backup.js | 21 +++++++++++--- www/common/sframe-common-outer.js | 47 +++++++++++++++++++++++-------- www/form/inner.js | 5 +++- www/form/main.js | 3 ++ 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index 405427020..fe9ac110f 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -239,10 +239,23 @@ define([ transform(ctx, parsed.type, val, function (res) { if (ctx.stop) { return; } if (!res.data) { return void error('EEMPTY'); } - ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", JSON.parse(val)["answers"], function (err, obj) { - var answers = obj && obj.results; - console.log("ANSWERS", answers) - }); + var data = JSON.parse(val) + var _answers = data["answers"] + if (data.form) { + _answers['href'] = parsed.hash + _answers['password'] = fData.password + _answers['drive'] = true + var answers + ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { + answers = obj && obj.results; + console.log("ANSWERS", answers) + }); + // var opts = { + // binary: true, + // }; + // zip.file(fileName, answers, opts); + } + var fileName = getUnique(sanitize(rawName), res.ext, existingNames); existingNames.push(fileName.toLowerCase()); zip.file(fileName, res.data, opts); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 3f6d9aeae..54ad6b92c 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -2433,6 +2433,7 @@ define([ }); sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { + var formHref = data.href var cb = Utils.Util.once(_cb); var myKeys = {}; var myFormKeys; @@ -2489,14 +2490,18 @@ define([ if (myFormKeys.formSeed) { myFormKeys = getAnonymousKeys(myFormKeys.formSeed, data.channel); } - - var keys = Utils.secret && Utils.secret.keys; - - var formData = Utils.Hash.getFormData(Utils.secret); - console.log("SECRET", Utils.secret) - console.log("FORMDATA", formData) - privateKey = formData.form_private; - publicKey = formData.form_public; + var keys; + var privateKey, publicKey; + if (data.drive) { + var secret = Utils.Hash.getSecrets('form', formHref, data.password); + keys = secret && secret.keys; + var formData = Utils.Hash.getFormData(secret); + privateKey = formData.form_private; + publicKey = formData.form_public; + + } else { + keys = Utils.secret && Utils.secret.keys; + } var curvePrivate = privateKey || data.privateKey; if (!curvePrivate) { return void cb({error: 'EFORBIDDEN'}); } @@ -2506,10 +2511,6 @@ define([ validateKey: data.validateKey }); - console.log("PRIVATE", curvePrivate) - console.log("PUBLIC", publicKey, data.publicKey) - console.log("VALIDATE", data.validateKey) - var config = { network: network, channel: data.channel, @@ -2563,6 +2564,28 @@ define([ // If we have a "non-anonymous" answer, it may be the edition of a // previous anonymous answer. Check if a previous anonymous answer exists // with the same uid and delete it. + var u8_slice = function (A, start, end) { + return new Uint8Array(Array.prototype.slice.call(A, start, end)); + }; + var checkAnonProof = function (proofObj, channel, curvePrivate) { + var pub = proofObj.key; + var proofTxt = proofObj.proof; + try { + var u8_bundle = Nacl.util.decodeBase64(proofTxt); + var u8_nonce = u8_slice(u8_bundle, 0, Nacl.box.nonceLength); + var u8_cipher = u8_slice(u8_bundle, Nacl.box.nonceLength); + var u8_plain = Nacl.box.open( + u8_cipher, + u8_nonce, + Nacl.util.decodeBase64(pub), + Nacl.util.decodeBase64(curvePrivate) + ); + return channel === Nacl.util.encodeUTF8(u8_plain); + } catch (e) { + console.error(e); + return false; + } + }; if (parsed._proof) { var check = checkAnonProof(parsed._proof, data.channel, curvePrivate); var theirAnonKey = parsed._proof.key; diff --git a/www/form/inner.js b/www/form/inner.js index 3428d98b6..b14c932a3 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -50,6 +50,7 @@ define([ 'css!/lib/datepicker/flatpickr.min.css', 'css!/components/components-font-awesome/css/font-awesome.min.css', 'less!/form/app-form.less', + '/common/sframe-common-outer.js' ], function ( $, Sortify, @@ -78,7 +79,8 @@ define([ DatePicker, Share, Access, Properties, Flatpickr, - Sortable + Sortable, + SFCommonO ) { @@ -5375,6 +5377,7 @@ define([ } var getMyAnswers = APP.getMyAnswers = function () { + var sframeChan = framework._.sfCommon.getSframeChannel(); sframeChan.query("Q_FETCH_MY_ANSWERS", { channel: content.answers.channel, validateKey: content.answers.validateKey, diff --git a/www/form/main.js b/www/form/main.js index 5efc81aab..6d8a02bf7 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -129,6 +129,9 @@ define([ return false; } }; + var beep = function () { + + } var deleteLines = false; // "false" to support old forms sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { From cd832156a02ec6fdb3b9e34f687ef0f20c564c8b Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Tue, 12 Nov 2024 10:25:06 +0100 Subject: [PATCH 04/41] Refactoring download function --- www/common/make-backup.js | 16 ++++++++++------ www/form/main.js | 3 --- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index fe9ac110f..41894479e 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -240,20 +240,24 @@ define([ if (ctx.stop) { return; } if (!res.data) { return void error('EEMPTY'); } var data = JSON.parse(val) - var _answers = data["answers"] + if (data.form) { + var _answers = data["answers"] _answers['href'] = parsed.hash _answers['password'] = fData.password _answers['drive'] = true - var answers + var answers; ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { answers = obj && obj.results; console.log("ANSWERS", answers) }); - // var opts = { - // binary: true, - // }; - // zip.file(fileName, answers, opts); + var opts = { + binary: true, + }; + var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); + existingNames.push(fileName.toLowerCase()); + var content = new Blob([answers], { type : "application/json" }); + zip.file(fileName, content, opts); } var fileName = getUnique(sanitize(rawName), res.ext, existingNames); diff --git a/www/form/main.js b/www/form/main.js index 6d8a02bf7..5efc81aab 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -129,9 +129,6 @@ define([ return false; } }; - var beep = function () { - - } var deleteLines = false; // "false" to support old forms sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { From 96e6e8dc64750b61fe3d8e44526fbda11c1f397b Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Tue, 12 Nov 2024 14:33:53 +0100 Subject: [PATCH 05/41] Made command compatible with both Drive + Form apps, cleanup --- www/common/make-backup.js | 37 +++++-- www/common/sframe-common-outer.js | 13 +-- www/form/main.js | 164 ------------------------------ 3 files changed, 35 insertions(+), 179 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index 41894479e..f0ae62da0 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -15,8 +15,9 @@ define([ '/components/nthen/index.js', '/components/saferphore/index.js', '/components/jszip/dist/jszip.min.js', + '/form/export.js', ], function ($, FileCrypto, Hash, Util, UI, h, Feedback, - Cache, Messages, nThen, Saferphore, JsZip) { + Cache, Messages, nThen, Saferphore, JsZip, Exporter) { var saveAs = window.saveAs; var sanitize = function (str) { @@ -249,15 +250,33 @@ define([ var answers; ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { answers = obj && obj.results; - console.log("ANSWERS", answers) + var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); + existingNames.push(fileName.toLowerCase()); + var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}} + var getFullOrder = function (content) { + var order = content.order.slice(); + var getSections = function (content) { + var uids = Object.keys(content.form).filter(function (uid) { + return content.form[uid].type === 'section'; + }); + return uids; + }; + getSections(content).forEach(function (uid) { + var block = content.form[uid]; + if (!block.opts || !Array.isArray(block.opts.questions)) { return; } + var idx = order.indexOf(uid); + if (idx === -1) { return; } + idx++; + block.opts.questions.forEach(function (el, i) { + order.splice(idx+i, 0, el); + }); + }); + return order; + }; + var arr = Exporter.results(data, answers, types, getFullOrder(data), "json"); + var content = new Blob([arr], { type : "application/json" }); + zip.file(fileName, content, opts); }); - var opts = { - binary: true, - }; - var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); - existingNames.push(fileName.toLowerCase()); - var content = new Blob([answers], { type : "application/json" }); - zip.file(fileName, content, opts); } var fileName = getUnique(sanitize(rawName), res.ext, existingNames); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 54ad6b92c..e7c6ab93e 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -2432,6 +2432,7 @@ define([ Utils.Feedback.send("BURN_AFTER_READING", Boolean(cfg.noDrive)); }); + var deleteLines = false; // "false" to support old forms sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { var formHref = data.href var cb = Utils.Util.once(_cb); @@ -2477,7 +2478,7 @@ define([ })); }).nThen(function () { if (!network) { return void cb({error: "E_CONNECT"}); } - var getAnonymousKeys = function (formSeed, channel) { + var getAnonymousKeys = function (formSeed, channel) { var array = Nacl.util.decodeBase64(formSeed + channel); var hash = Nacl.hash(array); var secretKey = Nacl.util.encodeBase64(hash.subarray(32)); @@ -2492,17 +2493,17 @@ define([ } var keys; var privateKey, publicKey; + var formData; if (data.drive) { var secret = Utils.Hash.getSecrets('form', formHref, data.password); keys = secret && secret.keys; - var formData = Utils.Hash.getFormData(secret); - privateKey = formData.form_private; - publicKey = formData.form_public; - + formData = Utils.Hash.getFormData(secret); } else { + formData = Utils.Hash.getFormData(Utils.secret); keys = Utils.secret && Utils.secret.keys; } - + privateKey = formData.form_private; + publicKey = formData.form_public; var curvePrivate = privateKey || data.privateKey; if (!curvePrivate) { return void cb({error: 'EFORBIDDEN'}); } var crypto = Utils.Crypto.Mailbox.createEncryptor({ diff --git a/www/form/main.js b/www/form/main.js index 5efc81aab..aa6f613a0 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -36,8 +36,6 @@ define([ } var formData = Utils.Hash.getFormData(Utils.secret); - console.log("SECRET", Utils.secret) - console.log("FORMDATA", formData) if (!formData) { return; } var validateKey = keys.secondaryValidateKey; @@ -77,9 +75,6 @@ define([ curvePublic: publicKey, }; }; - var u8_slice = function (A, start, end) { - return new Uint8Array(Array.prototype.slice.call(A, start, end)); - }; var u8_concat = function (A) { var length = 0; A.forEach(function (a) { length += a.length; }); @@ -110,165 +105,6 @@ define([ proof: Nacl.util.encodeBase64(u8_bundle) }; }; - var checkAnonProof = function (proofObj, channel, curvePrivate) { - var pub = proofObj.key; - var proofTxt = proofObj.proof; - try { - var u8_bundle = Nacl.util.decodeBase64(proofTxt); - var u8_nonce = u8_slice(u8_bundle, 0, Nacl.box.nonceLength); - var u8_cipher = u8_slice(u8_bundle, Nacl.box.nonceLength); - var u8_plain = Nacl.box.open( - u8_cipher, - u8_nonce, - Nacl.util.decodeBase64(pub), - Nacl.util.decodeBase64(curvePrivate) - ); - return channel === Nacl.util.encodeUTF8(u8_plain); - } catch (e) { - console.error(e); - return false; - } - }; - - var deleteLines = false; // "false" to support old forms - sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { - var cb = Utils.Util.once(_cb); - var myKeys = {}; - var myFormKeys; - var accessKeys; - var CPNetflux, Pinpad; - var network; - var noDriveAnswered = false; - nThen(function (w) { - require([ - 'chainpad-netflux', - '/common/pinpad.js', - ], w(function (_CPNetflux, _Pinpad) { - CPNetflux = _CPNetflux; - Pinpad = _Pinpad; - })); - var personalDrive = !Cryptpad.initialTeam || Cryptpad.initialTeam === -1; - Cryptpad.getAccessKeys(w(function (_keys) { - if (!Array.isArray(_keys)) { return; } - accessKeys = _keys; - - _keys.some(function (_k) { - if ((personalDrive && !_k.id) || Cryptpad.initialTeam === Number(_k.id)) { - myKeys = _k; - return true; - } - }); - })); - Cryptpad.getFormKeys(w(function (keys) { - if (!keys.curvePublic && !keys.formSeed) { - // No drive mode - var answered = JSON.parse(localStorage.CP_formAnswered || "[]"); - noDriveAnswered = answered.indexOf(data.channel) !== -1; - } - myFormKeys = keys; - })); - Cryptpad.makeNetwork(w(function (err, nw) { - network = nw; - })); - Cryptpad.getPadMetadata({channel: data.channel}, w(function (md) { - if (md && md.deleteLines) { deleteLines = true; } - })); - }).nThen(function () { - if (!network) { return void cb({error: "E_CONNECT"}); } - - if (myFormKeys.formSeed) { - myFormKeys = getAnonymousKeys(myFormKeys.formSeed, data.channel); - } - - var keys = Utils.secret && Utils.secret.keys; - - var curvePrivate = privateKey || data.privateKey; - if (!curvePrivate) { return void cb({error: 'EFORBIDDEN'}); } - var crypto = Utils.Crypto.Mailbox.createEncryptor({ - curvePrivate: curvePrivate, - curvePublic: publicKey || data.publicKey, - validateKey: data.validateKey - }); - - console.log("PRIVATE", curvePrivate) - console.log("PUBLIC", publicKey, data.publicKey) - console.log("VALIDATE", data.validateKey) - var config = { - network: network, - channel: data.channel, - noChainPad: true, - validateKey: keys.secondaryValidateKey, - owners: [myKeys.edPublic], - crypto: crypto, - metadata: { - deleteLines: true - } - //Cache: Utils.Cache // TODO enable cache for form responses when the cache stops evicting old answers - }; - var results = {}; - config.onError = function (info) { - cb({ error: info.type }); - }; - config.onRejected = function (data, cb) { - if (!Array.isArray(data) || !data.length || data[0].length !== 16) { - return void cb(true); - } - if (!Array.isArray(accessKeys)) { return void cb(true); } - network.historyKeeper = data[0]; - nThen(function (waitFor) { - accessKeys.forEach(function (obj) { - Pinpad.create(network, obj, waitFor(function (e) { - if (e) { console.error(e); } - })); - }); - }).nThen(function () { - cb(); - }); - }; - config.onReady = function () { - var myKey; - // If we have submitted an anonymous answer, retrieve it - if (myFormKeys.curvePublic && results[myFormKeys.curvePublic]) { - myKey = myFormKeys.curvePublic; - } - cb({ - noDriveAnswered: noDriveAnswered, - myKey: myKey, - results: results - }); - network.disconnect(); - }; - config.onMessage = function (msg, peer, vKey, isCp, hash, senderCurve, cfg) { - var parsed = Utils.Util.tryParse(msg); - if (!parsed) { return; } - var uid = parsed._uid || '000'; - - // If we have a "non-anonymous" answer, it may be the edition of a - // previous anonymous answer. Check if a previous anonymous answer exists - // with the same uid and delete it. - if (parsed._proof) { - var check = checkAnonProof(parsed._proof, data.channel, curvePrivate); - var theirAnonKey = parsed._proof.key; - if (check && results[theirAnonKey] && results[theirAnonKey][uid]) { - delete results[theirAnonKey][uid]; - } - } - - parsed._time = cfg && cfg.time; - if (deleteLines) { parsed._hash = hash; } - - if (data.cantEdit && results[senderCurve] - && results[senderCurve][uid]) { return; } - results[senderCurve] = results[senderCurve] || {}; - results[senderCurve][uid] = { - msg: parsed, - hash: hash, - time: cfg && cfg.time - }; - }; - CPNetflux.start(config); - }); - }); sframeChan.on("Q_FETCH_MY_ANSWERS", function (data, cb) { var answers = []; var myKeys; From 6c1536b7bb05c08861b2f84066b19c92aae9a715 Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Tue, 12 Nov 2024 14:56:18 +0100 Subject: [PATCH 06/41] Cleanup --- www/form/inner.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/www/form/inner.js b/www/form/inner.js index b14c932a3..3428d98b6 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -50,7 +50,6 @@ define([ 'css!/lib/datepicker/flatpickr.min.css', 'css!/components/components-font-awesome/css/font-awesome.min.css', 'less!/form/app-form.less', - '/common/sframe-common-outer.js' ], function ( $, Sortify, @@ -79,8 +78,7 @@ define([ DatePicker, Share, Access, Properties, Flatpickr, - Sortable, - SFCommonO + Sortable ) { @@ -5377,7 +5375,6 @@ define([ } var getMyAnswers = APP.getMyAnswers = function () { - var sframeChan = framework._.sfCommon.getSframeChannel(); sframeChan.query("Q_FETCH_MY_ANSWERS", { channel: content.answers.channel, validateKey: content.answers.validateKey, From fdd4987004c1fa7107c24ce525030a7cef8224df Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Tue, 12 Nov 2024 15:29:00 +0100 Subject: [PATCH 07/41] Linting --- www/common/make-backup.js | 13 ++++++------- www/common/sframe-common-outer.js | 2 +- www/form/main.js | 5 +---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index f0ae62da0..db11bd481 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -240,19 +240,18 @@ define([ transform(ctx, parsed.type, val, function (res) { if (ctx.stop) { return; } if (!res.data) { return void error('EEMPTY'); } - var data = JSON.parse(val) - + var data = JSON.parse(val); if (data.form) { - var _answers = data["answers"] - _answers['href'] = parsed.hash - _answers['password'] = fData.password - _answers['drive'] = true + var _answers = data["answers"]; + _answers['href'] = parsed.hash; + _answers['password'] = fData.password; + _answers['drive'] = true; var answers; ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { answers = obj && obj.results; var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); existingNames.push(fileName.toLowerCase()); - var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}} + var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}}; var getFullOrder = function (content) { var order = content.order.slice(); var getSections = function (content) { diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index e7c6ab93e..39eb639da 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -2434,7 +2434,7 @@ define([ var deleteLines = false; // "false" to support old forms sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { - var formHref = data.href + var formHref = data.href; var cb = Utils.Util.once(_cb); var myKeys = {}; var myFormKeys; diff --git a/www/form/main.js b/www/form/main.js index aa6f613a0..41800fc63 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -21,7 +21,6 @@ define([ href = obj.href; hash = obj.hash; }).nThen(function (/*waitFor*/) { - var privateKey, publicKey; var channels = {}; var getPropChannels = function () { return channels; @@ -40,9 +39,6 @@ define([ var validateKey = keys.secondaryValidateKey; meta.form_answerValidateKey = validateKey; - - publicKey = meta.form_public = formData.form_public; - privateKey = meta.form_private = formData.form_private; meta.form_auditorHash = formData.form_auditorHash; }; var addRpc = function (sframeChan, Cryptpad, Utils) { @@ -105,6 +101,7 @@ define([ proof: Nacl.util.encodeBase64(u8_bundle) }; }; + var deleteLines = false; // "false" to support old forms sframeChan.on("Q_FETCH_MY_ANSWERS", function (data, cb) { var answers = []; var myKeys; From 28c400e049524a4b3fd6ed92231c5e2bda1e33cd Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Tue, 12 Nov 2024 15:37:55 +0100 Subject: [PATCH 08/41] Revert "Linting" This reverts commit fdd4987004c1fa7107c24ce525030a7cef8224df. --- www/common/make-backup.js | 13 +++++++------ www/common/sframe-common-outer.js | 2 +- www/form/main.js | 5 ++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index db11bd481..f0ae62da0 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -240,18 +240,19 @@ define([ transform(ctx, parsed.type, val, function (res) { if (ctx.stop) { return; } if (!res.data) { return void error('EEMPTY'); } - var data = JSON.parse(val); + var data = JSON.parse(val) + if (data.form) { - var _answers = data["answers"]; - _answers['href'] = parsed.hash; - _answers['password'] = fData.password; - _answers['drive'] = true; + var _answers = data["answers"] + _answers['href'] = parsed.hash + _answers['password'] = fData.password + _answers['drive'] = true var answers; ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { answers = obj && obj.results; var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); existingNames.push(fileName.toLowerCase()); - var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}}; + var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}} var getFullOrder = function (content) { var order = content.order.slice(); var getSections = function (content) { diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 39eb639da..e7c6ab93e 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -2434,7 +2434,7 @@ define([ var deleteLines = false; // "false" to support old forms sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { - var formHref = data.href; + var formHref = data.href var cb = Utils.Util.once(_cb); var myKeys = {}; var myFormKeys; diff --git a/www/form/main.js b/www/form/main.js index 41800fc63..aa6f613a0 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -21,6 +21,7 @@ define([ href = obj.href; hash = obj.hash; }).nThen(function (/*waitFor*/) { + var privateKey, publicKey; var channels = {}; var getPropChannels = function () { return channels; @@ -39,6 +40,9 @@ define([ var validateKey = keys.secondaryValidateKey; meta.form_answerValidateKey = validateKey; + + publicKey = meta.form_public = formData.form_public; + privateKey = meta.form_private = formData.form_private; meta.form_auditorHash = formData.form_auditorHash; }; var addRpc = function (sframeChan, Cryptpad, Utils) { @@ -101,7 +105,6 @@ define([ proof: Nacl.util.encodeBase64(u8_bundle) }; }; - var deleteLines = false; // "false" to support old forms sframeChan.on("Q_FETCH_MY_ANSWERS", function (data, cb) { var answers = []; var myKeys; From 0c26b1d2731f7155f812425f6bf6b24a09d21271 Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Tue, 12 Nov 2024 15:51:42 +0100 Subject: [PATCH 09/41] Linting #2 --- www/common/make-backup.js | 13 ++++++------- www/common/sframe-common-outer.js | 2 +- www/form/main.js | 6 +++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index f0ae62da0..db11bd481 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -240,19 +240,18 @@ define([ transform(ctx, parsed.type, val, function (res) { if (ctx.stop) { return; } if (!res.data) { return void error('EEMPTY'); } - var data = JSON.parse(val) - + var data = JSON.parse(val); if (data.form) { - var _answers = data["answers"] - _answers['href'] = parsed.hash - _answers['password'] = fData.password - _answers['drive'] = true + var _answers = data["answers"]; + _answers['href'] = parsed.hash; + _answers['password'] = fData.password; + _answers['drive'] = true; var answers; ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { answers = obj && obj.results; var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); existingNames.push(fileName.toLowerCase()); - var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}} + var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}}; var getFullOrder = function (content) { var order = content.order.slice(); var getSections = function (content) { diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index e7c6ab93e..39eb639da 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -2434,7 +2434,7 @@ define([ var deleteLines = false; // "false" to support old forms sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { - var formHref = data.href + var formHref = data.href; var cb = Utils.Util.once(_cb); var myKeys = {}; var myFormKeys; diff --git a/www/form/main.js b/www/form/main.js index aa6f613a0..dab04bb8f 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -21,7 +21,6 @@ define([ href = obj.href; hash = obj.hash; }).nThen(function (/*waitFor*/) { - var privateKey, publicKey; var channels = {}; var getPropChannels = function () { return channels; @@ -41,8 +40,8 @@ define([ var validateKey = keys.secondaryValidateKey; meta.form_answerValidateKey = validateKey; - publicKey = meta.form_public = formData.form_public; - privateKey = meta.form_private = formData.form_private; + meta.form_public = formData.form_public; + meta.form_private = formData.form_private; meta.form_auditorHash = formData.form_auditorHash; }; var addRpc = function (sframeChan, Cryptpad, Utils) { @@ -105,6 +104,7 @@ define([ proof: Nacl.util.encodeBase64(u8_bundle) }; }; + var deleteLines = false; // "false" to support old forms sframeChan.on("Q_FETCH_MY_ANSWERS", function (data, cb) { var answers = []; var myKeys; From 5ba6de2354201bab6705918f20603066f298edba Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Thu, 14 Nov 2024 16:22:01 +0100 Subject: [PATCH 10/41] Refactor --- www/common/make-backup.js | 53 +---- www/common/sframe-common-outer.js | 369 +++++++++++++++--------------- www/form/export.js | 58 ++++- 3 files changed, 245 insertions(+), 235 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index db11bd481..bf73d6c3a 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -15,9 +15,8 @@ define([ '/components/nthen/index.js', '/components/saferphore/index.js', '/components/jszip/dist/jszip.min.js', - '/form/export.js', ], function ($, FileCrypto, Hash, Util, UI, h, Feedback, - Cache, Messages, nThen, Saferphore, JsZip, Exporter) { + Cache, Messages, nThen, Saferphore, JsZip) { var saveAs = window.saveAs; var sanitize = function (str) { @@ -33,7 +32,7 @@ define([ return n; }; - var transform = function (ctx, type, sjson, cb, padData) { + var transform = function (ctx, parsed, sjson, cb, padData, zip, existingNames) { var result = { data: sjson, ext: '.json', @@ -44,13 +43,13 @@ define([ } catch (e) { return void cb(result); } - var path = '/' + type + '/export.js'; + var path = '/' + parsed.type + '/export.js'; require([path], function (Exporter) { Exporter.main(json, function (data, _ext) { result.ext = _ext || Exporter.ext || ''; result.data = data; cb(result); - }, null, ctx.sframeChan, padData); + }, null, ctx.sframeChan, padData, zip, sanitize, getUnique, existingNames); }, function () { cb(result); }); @@ -141,7 +140,7 @@ define([ if (cancelled) { return; } if (err) { return; } if (!val) { return; } - transform(ctx, parsed.type, val, function (res) { + transform(ctx, parsed, val, function (res) { if (cancelled) { return; } if (!res.data) { return; } var dl = function () { @@ -237,47 +236,9 @@ define([ var opts = { binary: true, }; - transform(ctx, parsed.type, val, function (res) { + transform(ctx, parsed, val, function (res) { if (ctx.stop) { return; } if (!res.data) { return void error('EEMPTY'); } - var data = JSON.parse(val); - if (data.form) { - var _answers = data["answers"]; - _answers['href'] = parsed.hash; - _answers['password'] = fData.password; - _answers['drive'] = true; - var answers; - ctx.sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { - answers = obj && obj.results; - var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); - existingNames.push(fileName.toLowerCase()); - var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}}; - var getFullOrder = function (content) { - var order = content.order.slice(); - var getSections = function (content) { - var uids = Object.keys(content.form).filter(function (uid) { - return content.form[uid].type === 'section'; - }); - return uids; - }; - getSections(content).forEach(function (uid) { - var block = content.form[uid]; - if (!block.opts || !Array.isArray(block.opts.questions)) { return; } - var idx = order.indexOf(uid); - if (idx === -1) { return; } - idx++; - block.opts.questions.forEach(function (el, i) { - order.splice(idx+i, 0, el); - }); - }); - return order; - }; - var arr = Exporter.results(data, answers, types, getFullOrder(data), "json"); - var content = new Blob([arr], { type : "application/json" }); - zip.file(fileName, content, opts); - }); - } - var fileName = getUnique(sanitize(rawName), res.ext, existingNames); existingNames.push(fileName.toLowerCase()); zip.file(fileName, res.data, opts); @@ -286,7 +247,7 @@ define([ }, { hash: parsed.hash, password: fData.password - }); + }, zip, existingNames); }); }; diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 39eb639da..703b22c75 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -9,10 +9,7 @@ define([ '/common/requireconfig.js', '/customize/messages.js', 'jquery', - '/components/tweetnacl/nacl-fast.min.js', - ], function (nThen, ApiConfig, RequireConfig, Messages, $) { - var Nacl = window.nacl; var common = {}; var embeddableApps = [ @@ -160,6 +157,7 @@ define([ '/common/userObject.js', 'optional!/api/instance', '/common/pad-types.js', + '/components/tweetnacl/nacl-fast.min.js', ], waitFor(function (_CpNfOuter, _Cryptpad, _Crypto, _Cryptget, _SFrameChannel, _SecureIframe, _UnsafeIframe, _OOIframe, _Notifier, _Hash, _Util, _Realtime, _Notify, _Constants, _Feedback, _LocalStore, _Block, _Cache, _AppConfig, /* _Test,*/ _UserObject, @@ -187,6 +185,7 @@ define([ Utils.Block = _Block; Utils.PadTypes = _PadTypes; AppConfig = _AppConfig; + var Nacl = window.nacl; //Test = _Test; if (localStorage.CRYPTPAD_URLARGS !== ApiConfig.requireConf.urlArgs) { @@ -235,7 +234,11 @@ define([ } }; + + var addFirstHandlers = () => { + + sframeChan.on('Q_SETTINGS_CHECK_PASSWORD', function (data, cb) { var blockHash = Utils.LocalStore.getBlockHash(); var userHash = Utils.LocalStore.getUserHash(); @@ -287,6 +290,184 @@ define([ console.error(err || obj); }); }); + var checkAnonProof = function (proofObj, channel, curvePrivate) { + var pub = proofObj.key; + var proofTxt = proofObj.proof; + try { + var u8_bundle = Nacl.util.decodeBase64(proofTxt); + var u8_nonce = u8_slice(u8_bundle, 0, Nacl.box.nonceLength); + var u8_cipher = u8_slice(u8_bundle, Nacl.box.nonceLength); + var u8_plain = Nacl.box.open( + u8_cipher, + u8_nonce, + Nacl.util.decodeBase64(pub), + Nacl.util.decodeBase64(curvePrivate) + ); + return channel === Nacl.util.encodeUTF8(u8_plain); + } catch (e) { + console.error(e); + return false; + } + }; + var u8_slice = function (A, start, end) { + return new Uint8Array(Array.prototype.slice.call(A, start, end)); + }; + var deleteLines = false; // "false" to support old forms + sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { + var formHref = data.href; + var cb = Utils.Util.once(_cb); + var myKeys = {}; + var myFormKeys; + var accessKeys; + var CPNetflux, Pinpad; + var network; + var noDriveAnswered = false; + nThen(function (w) { + require([ + 'chainpad-netflux', + '/common/pinpad.js', + ], w(function (_CPNetflux, _Pinpad) { + CPNetflux = _CPNetflux; + Pinpad = _Pinpad; + })); + var personalDrive = !Cryptpad.initialTeam || Cryptpad.initialTeam === -1; + Cryptpad.getAccessKeys(w(function (_keys) { + if (!Array.isArray(_keys)) { return; } + accessKeys = _keys; + + _keys.some(function (_k) { + if ((personalDrive && !_k.id) || Cryptpad.initialTeam === Number(_k.id)) { + myKeys = _k; + return true; + } + }); + })); + Cryptpad.getFormKeys(w(function (keys) { + if (!keys.curvePublic && !keys.formSeed) { + // No drive mode + var answered = JSON.parse(localStorage.CP_formAnswered || "[]"); + noDriveAnswered = answered.indexOf(data.channel) !== -1; + } + myFormKeys = keys; + })); + Cryptpad.makeNetwork(w(function (err, nw) { + network = nw; + })); + Cryptpad.getPadMetadata({channel: data.channel}, w(function (md) { + if (md && md.deleteLines) { deleteLines = true; } + })); + }).nThen(function () { + if (!network) { return void cb({error: "E_CONNECT"}); } + var getAnonymousKeys = function (formSeed, channel) { + var array = Nacl.util.decodeBase64(formSeed + channel); + var hash = Nacl.hash(array); + var secretKey = Nacl.util.encodeBase64(hash.subarray(32)); + var publicKey = Utils.Hash.getCurvePublicFromPrivate(secretKey); + return { + curvePrivate: secretKey, + curvePublic: publicKey, + }; + }; + if (myFormKeys.formSeed) { + myFormKeys = getAnonymousKeys(myFormKeys.formSeed, data.channel); + } + var keys; + var privateKey, publicKey; + var formData; + if (data.drive) { + var secret = Utils.Hash.getSecrets('form', formHref, data.password); + keys = secret && secret.keys; + formData = Utils.Hash.getFormData(secret); + } else { + formData = Utils.Hash.getFormData(Utils.secret); + keys = Utils.secret && Utils.secret.keys; + } + privateKey = formData.form_private; + publicKey = formData.form_public; + var curvePrivate = privateKey || data.privateKey; + if (!curvePrivate) { return void cb({error: 'EFORBIDDEN'}); } + var crypto = Utils.Crypto.Mailbox.createEncryptor({ + curvePrivate: curvePrivate, + curvePublic: publicKey || data.publicKey, + validateKey: data.validateKey + }); + + var config = { + network: network, + channel: data.channel, + noChainPad: true, + validateKey: keys.secondaryValidateKey, + owners: [myKeys.edPublic], + crypto: crypto, + metadata: { + deleteLines: true + } + //Cache: Utils.Cache // TODO enable cache for form responses when the cache stops evicting old answers + }; + var results = {}; + config.onError = function (info) { + cb({ error: info.type }); + }; + config.onRejected = function (data, cb) { + if (!Array.isArray(data) || !data.length || data[0].length !== 16) { + return void cb(true); + } + if (!Array.isArray(accessKeys)) { return void cb(true); } + network.historyKeeper = data[0]; + nThen(function (waitFor) { + accessKeys.forEach(function (obj) { + Pinpad.create(network, obj, waitFor(function (e) { + if (e) { console.error(e); } + })); + }); + }).nThen(function () { + cb(); + }); + }; + config.onReady = function () { + var myKey; + // If we have submitted an anonymous answer, retrieve it + if (myFormKeys.curvePublic && results[myFormKeys.curvePublic]) { + myKey = myFormKeys.curvePublic; + } + cb({ + noDriveAnswered: noDriveAnswered, + myKey: myKey, + results: results + }); + network.disconnect(); + }; + config.onMessage = function (msg, peer, vKey, isCp, hash, senderCurve, cfg) { + var parsed = Utils.Util.tryParse(msg); + if (!parsed) { return; } + var uid = parsed._uid || '000'; + + // If we have a "non-anonymous" answer, it may be the edition of a + // previous anonymous answer. Check if a previous anonymous answer exists + // with the same uid and delete it. + if (parsed._proof) { + var check = checkAnonProof(parsed._proof, data.channel, curvePrivate); + var theirAnonKey = parsed._proof.key; + if (check && results[theirAnonKey] && results[theirAnonKey][uid]) { + delete results[theirAnonKey][uid]; + } + } + + parsed._time = cfg && cfg.time; + if (deleteLines) { parsed._hash = hash; } + + if (data.cantEdit && results[senderCurve] + && results[senderCurve][uid]) { return; } + results[senderCurve] = results[senderCurve] || {}; + results[senderCurve][uid] = { + msg: parsed, + hash: hash, + time: cfg && cfg.time + }; + }; + CPNetflux.start(config); + }); + }); }; var whenReady = waitFor(function (msg) { @@ -897,6 +1078,7 @@ define([ sframeChan.event("EV_NEW_VERSION"); }); + // Put in the following function the RPC queries that should also work in filepicker @@ -1387,6 +1569,8 @@ define([ }); }); }); + + }; addCommonRpc(sframeChan, isSafe); @@ -2432,185 +2616,6 @@ define([ Utils.Feedback.send("BURN_AFTER_READING", Boolean(cfg.noDrive)); }); - var deleteLines = false; // "false" to support old forms - sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { - var formHref = data.href; - var cb = Utils.Util.once(_cb); - var myKeys = {}; - var myFormKeys; - var accessKeys; - var CPNetflux, Pinpad; - var network; - var noDriveAnswered = false; - nThen(function (w) { - require([ - 'chainpad-netflux', - '/common/pinpad.js', - ], w(function (_CPNetflux, _Pinpad) { - CPNetflux = _CPNetflux; - Pinpad = _Pinpad; - })); - var personalDrive = !Cryptpad.initialTeam || Cryptpad.initialTeam === -1; - Cryptpad.getAccessKeys(w(function (_keys) { - if (!Array.isArray(_keys)) { return; } - accessKeys = _keys; - - _keys.some(function (_k) { - if ((personalDrive && !_k.id) || Cryptpad.initialTeam === Number(_k.id)) { - myKeys = _k; - return true; - } - }); - })); - Cryptpad.getFormKeys(w(function (keys) { - if (!keys.curvePublic && !keys.formSeed) { - // No drive mode - var answered = JSON.parse(localStorage.CP_formAnswered || "[]"); - noDriveAnswered = answered.indexOf(data.channel) !== -1; - } - myFormKeys = keys; - })); - Cryptpad.makeNetwork(w(function (err, nw) { - network = nw; - })); - Cryptpad.getPadMetadata({channel: data.channel}, w(function (md) { - if (md && md.deleteLines) { deleteLines = true; } - })); - }).nThen(function () { - if (!network) { return void cb({error: "E_CONNECT"}); } - var getAnonymousKeys = function (formSeed, channel) { - var array = Nacl.util.decodeBase64(formSeed + channel); - var hash = Nacl.hash(array); - var secretKey = Nacl.util.encodeBase64(hash.subarray(32)); - var publicKey = Utils.Hash.getCurvePublicFromPrivate(secretKey); - return { - curvePrivate: secretKey, - curvePublic: publicKey, - }; - }; - if (myFormKeys.formSeed) { - myFormKeys = getAnonymousKeys(myFormKeys.formSeed, data.channel); - } - var keys; - var privateKey, publicKey; - var formData; - if (data.drive) { - var secret = Utils.Hash.getSecrets('form', formHref, data.password); - keys = secret && secret.keys; - formData = Utils.Hash.getFormData(secret); - } else { - formData = Utils.Hash.getFormData(Utils.secret); - keys = Utils.secret && Utils.secret.keys; - } - privateKey = formData.form_private; - publicKey = formData.form_public; - var curvePrivate = privateKey || data.privateKey; - if (!curvePrivate) { return void cb({error: 'EFORBIDDEN'}); } - var crypto = Utils.Crypto.Mailbox.createEncryptor({ - curvePrivate: curvePrivate, - curvePublic: publicKey || data.publicKey, - validateKey: data.validateKey - }); - - var config = { - network: network, - channel: data.channel, - noChainPad: true, - validateKey: keys.secondaryValidateKey, - owners: [myKeys.edPublic], - crypto: crypto, - metadata: { - deleteLines: true - } - //Cache: Utils.Cache // TODO enable cache for form responses when the cache stops evicting old answers - }; - var results = {}; - config.onError = function (info) { - cb({ error: info.type }); - }; - config.onRejected = function (data, cb) { - if (!Array.isArray(data) || !data.length || data[0].length !== 16) { - return void cb(true); - } - if (!Array.isArray(accessKeys)) { return void cb(true); } - network.historyKeeper = data[0]; - nThen(function (waitFor) { - accessKeys.forEach(function (obj) { - Pinpad.create(network, obj, waitFor(function (e) { - if (e) { console.error(e); } - })); - }); - }).nThen(function () { - cb(); - }); - }; - config.onReady = function () { - var myKey; - // If we have submitted an anonymous answer, retrieve it - if (myFormKeys.curvePublic && results[myFormKeys.curvePublic]) { - myKey = myFormKeys.curvePublic; - } - cb({ - noDriveAnswered: noDriveAnswered, - myKey: myKey, - results: results - }); - network.disconnect(); - }; - config.onMessage = function (msg, peer, vKey, isCp, hash, senderCurve, cfg) { - var parsed = Utils.Util.tryParse(msg); - if (!parsed) { return; } - var uid = parsed._uid || '000'; - - // If we have a "non-anonymous" answer, it may be the edition of a - // previous anonymous answer. Check if a previous anonymous answer exists - // with the same uid and delete it. - var u8_slice = function (A, start, end) { - return new Uint8Array(Array.prototype.slice.call(A, start, end)); - }; - var checkAnonProof = function (proofObj, channel, curvePrivate) { - var pub = proofObj.key; - var proofTxt = proofObj.proof; - try { - var u8_bundle = Nacl.util.decodeBase64(proofTxt); - var u8_nonce = u8_slice(u8_bundle, 0, Nacl.box.nonceLength); - var u8_cipher = u8_slice(u8_bundle, Nacl.box.nonceLength); - var u8_plain = Nacl.box.open( - u8_cipher, - u8_nonce, - Nacl.util.decodeBase64(pub), - Nacl.util.decodeBase64(curvePrivate) - ); - return channel === Nacl.util.encodeUTF8(u8_plain); - } catch (e) { - console.error(e); - return false; - } - }; - if (parsed._proof) { - var check = checkAnonProof(parsed._proof, data.channel, curvePrivate); - var theirAnonKey = parsed._proof.key; - if (check && results[theirAnonKey] && results[theirAnonKey][uid]) { - delete results[theirAnonKey][uid]; - } - } - - parsed._time = cfg && cfg.time; - if (deleteLines) { parsed._hash = hash; } - - if (data.cantEdit && results[senderCurve] - && results[senderCurve][uid]) { return; } - results[senderCurve] = results[senderCurve] || {}; - results[senderCurve][uid] = { - msg: parsed, - hash: hash, - time: cfg && cfg.time - }; - }; - CPNetflux.start(config); - }); - }); - sframeChan.ready(); Utils.Feedback.reportAppUsage(); diff --git a/www/form/export.js b/www/form/export.js index 9a4ac0f7a..af8e626f1 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -4,7 +4,8 @@ define([ '/common/common-util.js', - '/customize/messages.js' + '/customize/messages.js', + '' ], function (Util, Messages) { var Export = { ext: '.json' @@ -162,13 +163,56 @@ define([ return csv; }; - Export.main = function (content, cb) { - var json = Util.clone(content || {}); - delete json.answers; - cb(new Blob([JSON.stringify(json, 0, 2)], { - type: 'application/json;charset=utf-8' - })); + var getFullOrder = function (content) { + var order = content.order.slice(); + var getSections = function (content) { + var uids = Object.keys(content.form).filter(function (uid) { + return content.form[uid].type === 'section'; + }); + return uids; + }; + getSections(content).forEach(function (uid) { + var block = content.form[uid]; + if (!block.opts || !Array.isArray(block.opts.questions)) { return; } + var idx = order.indexOf(uid); + if (idx === -1) { return; } + idx++; + block.opts.questions.forEach(function (el, i) { + order.splice(idx+i, 0, el); + }); + }); + return order; }; + Export.main = function (content, cb, ext, sframeChan, parsed, zip, sanitize, getUnique, existingNames) { + if (sframeChan && content.form) { + var _answers = content["answers"]; + _answers['href'] = parsed.hash; + _answers['password'] = parsed.password; + _answers['drive'] = true; + var answers; + sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { + answers = obj && obj.results; + var rawName = content.metadata.title || 'File'; + var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); + existingNames.push(fileName.toLowerCase()); + var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}}; + + var arr = Export.results(content, answers, types, getFullOrder(content), "json"); + var results = new Blob([arr], { type : "application/json" }); + var opts = { + binary: true, + }; + zip.file(fileName, results, opts); + }); + } + + var json = Util.clone(content || {}); + delete json.answers; + cb(new Blob([JSON.stringify(json, 0, 2)], { + type: 'application/json;charset=utf-8' + })); + }; + return Export; }); From 915156523ab86ad8d20e8f8ef04b50eb39ae8398 Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Thu, 14 Nov 2024 16:27:44 +0100 Subject: [PATCH 11/41] Cleaning --- www/common/sframe-common-outer.js | 7 ------- www/form/export.js | 1 - 2 files changed, 8 deletions(-) diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 703b22c75..d7675d50a 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -234,11 +234,7 @@ define([ } }; - - var addFirstHandlers = () => { - - sframeChan.on('Q_SETTINGS_CHECK_PASSWORD', function (data, cb) { var blockHash = Utils.LocalStore.getBlockHash(); var userHash = Utils.LocalStore.getUserHash(); @@ -1078,7 +1074,6 @@ define([ sframeChan.event("EV_NEW_VERSION"); }); - // Put in the following function the RPC queries that should also work in filepicker @@ -1569,8 +1564,6 @@ define([ }); }); }); - - }; addCommonRpc(sframeChan, isSafe); diff --git a/www/form/export.js b/www/form/export.js index af8e626f1..4ae7fe8d4 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -5,7 +5,6 @@ define([ '/common/common-util.js', '/customize/messages.js', - '' ], function (Util, Messages) { var Export = { ext: '.json' From f253645b674da434638364fb05d15222183352ef Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Fri, 15 Nov 2024 12:40:46 +0100 Subject: [PATCH 12/41] Fixed date format --- www/form/export.js | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/www/form/export.js b/www/form/export.js index 4ae7fe8d4..9ced9c83b 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -59,18 +59,7 @@ define([ var type = obj.type; if (!TYPES[type]) { return; } // Ignore static types var id = `q${i++}`; - if (TYPES[type] && TYPES[type].exportCSV) { - var _obj = Util.clone(obj); - _obj.q = "tmp"; - q[id] = { - question: obj.q, - items: TYPES[type].exportCSV(false, _obj).map(function (str) { - return str.slice(6); // Remove "tmp | " - }) - }; - } else { - q[id] = obj.q || Messages.form_default; - } + q[id] = obj.q || Messages.form_default; }); sortedKeys.forEach(function (k) { @@ -84,14 +73,14 @@ define([ }; var i = 1; + order.forEach(function (key) { if (!form[key]) { return; } var type = form[key].type; if (!TYPES[type]) { return; } // Ignore static types var id = `q${i++}`; - if (TYPES[type].exportCSV) { - data[id] = TYPES[type].exportCSV(msg[key], form[key]); - return; + if (type === 'date') { + msg[key] = new Date(msg[key]).toISOString(); } data[id] = msg[key]; }); @@ -115,7 +104,6 @@ define([ var form = content.form; var questions = [Messages.form_poll_time, Messages.share_formView]; - order.forEach(function (key) { var obj = form[key]; if (!obj) { return; } @@ -196,8 +184,7 @@ define([ var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); existingNames.push(fileName.toLowerCase()); var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}}; - - var arr = Export.results(content, answers, types, getFullOrder(content), "json"); + var arr = Export.results(content, answers, types, getFullOrder(content), "json"); var results = new Blob([arr], { type : "application/json" }); var opts = { binary: true, From ccfe1472e66ed158cbf01699b81daa88dc97e32c Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 4 Dec 2024 13:50:26 +0100 Subject: [PATCH 13/41] lint compliance --- www/common/sframe-common-outer.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 61fc87bb7..53bda5d9e 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -285,6 +285,9 @@ define([ console.error(err || obj); }); }); + var u8_slice = function (A, start, end) { + return new Uint8Array(Array.prototype.slice.call(A, start, end)); + }; var checkAnonProof = function (proofObj, channel, curvePrivate) { var pub = proofObj.key; var proofTxt = proofObj.proof; @@ -304,9 +307,6 @@ define([ return false; } }; - var u8_slice = function (A, start, end) { - return new Uint8Array(Array.prototype.slice.call(A, start, end)); - }; var deleteLines = false; // "false" to support old forms sframeChan.on('Q_FORM_FETCH_ANSWERS', function (data, _cb) { var formHref = data.href; From 9c87556332838473584f105bec92df7ea1b63cd7 Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Thu, 5 Dec 2024 14:00:35 +0100 Subject: [PATCH 14/41] Refactoring - WIP --- www/common/cryptpad-common.js | 11 +++++++++++ www/common/sframe-common-outer.js | 12 +----------- www/form/export.js | 12 ++++++------ www/form/main.js | 16 +++------------- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index ada56e9be..bf7ea514d 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -2449,6 +2449,17 @@ define([ window.RTCPeerConnection); }; + common.getAnonymousKeys = function (formSeed, channel, Utils) { + var array = window.nacl.util.decodeBase64(formSeed + channel); + var hash = window.nacl.hash(array); + var secretKey = window.nacl.util.encodeBase64(hash.subarray(32)); + var publicKey = Utils.Hash.getCurvePublicFromPrivate(secretKey); + return { + curvePrivate: secretKey, + curvePublic: publicKey, + }; + }; + common.ready = (function () { var env = {}; var initialized = false; diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 53bda5d9e..036e7edfd 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -353,18 +353,8 @@ define([ })); }).nThen(function () { if (!network) { return void cb({error: "E_CONNECT"}); } - var getAnonymousKeys = function (formSeed, channel) { - var array = Nacl.util.decodeBase64(formSeed + channel); - var hash = Nacl.hash(array); - var secretKey = Nacl.util.encodeBase64(hash.subarray(32)); - var publicKey = Utils.Hash.getCurvePublicFromPrivate(secretKey); - return { - curvePrivate: secretKey, - curvePublic: publicKey, - }; - }; if (myFormKeys.formSeed) { - myFormKeys = getAnonymousKeys(myFormKeys.formSeed, data.channel); + myFormKeys = _Cryptpad.getAnonymousKeys(myFormKeys.formSeed, data.channel, Utils); } var keys; var privateKey, publicKey; diff --git a/www/form/export.js b/www/form/export.js index 9ced9c83b..a3b3a1c3f 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -4,7 +4,7 @@ define([ '/common/common-util.js', - '/customize/messages.js', + '/customize/messages.js' ], function (Util, Messages) { var Export = { ext: '.json' @@ -194,11 +194,11 @@ define([ } var json = Util.clone(content || {}); - delete json.answers; - cb(new Blob([JSON.stringify(json, 0, 2)], { - type: 'application/json;charset=utf-8' - })); - }; + delete json.answers; + cb(new Blob([JSON.stringify(json, 0, 2)], { + type: 'application/json;charset=utf-8' + })); + }; return Export; }); diff --git a/www/form/main.js b/www/form/main.js index dab04bb8f..aa47abf73 100644 --- a/www/form/main.js +++ b/www/form/main.js @@ -64,16 +64,6 @@ define([ if (!a) { sframeChan.event('EV_POPUP_BLOCKED'); } delete sessionStorage.CP_formExportSheet; }); - var getAnonymousKeys = function (formSeed, channel) { - var array = Nacl.util.decodeBase64(formSeed + channel); - var hash = Nacl.hash(array); - var secretKey = Nacl.util.encodeBase64(hash.subarray(32)); - var publicKey = Utils.Hash.getCurvePublicFromPrivate(secretKey); - return { - curvePrivate: secretKey, - curvePublic: publicKey, - }; - }; var u8_concat = function (A) { var length = 0; A.forEach(function (a) { length += a.length; }); @@ -150,7 +140,7 @@ define([ console.error('ANONYMOUS_ERROR', answer); return; } - finalKeys = getAnonymousKeys(myKeys.formSeed, data.channel); + finalKeys = Cryptpad.getAnonymousKeys(myKeys.formSeed, data.channel, Utils); } Cryptpad.getHistoryRange({ channel: data.channel, @@ -209,9 +199,9 @@ define([ var myAnonymousKeys; if (data.anonymous) { if (!myKeys.formSeed) { return void cb({ error: "ANONYMOUS_ERROR" }); } - myKeys = getAnonymousKeys(myKeys.formSeed, box.channel); + myKeys = Cryptpad.getAnonymousKeys(myKeys.formSeed, box.channel, Utils); } else { - myAnonymousKeys = getAnonymousKeys(myKeys.formSeed, box.channel); + myAnonymousKeys = Cryptpad.getAnonymousKeys(myKeys.formSeed, box.channel, Utils); } var keys = Utils.secret && Utils.secret.keys; myKeys.signingKey = keys.secondarySignKey; From 4f14e6db4fec3e137f629fff61748e421543993e Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Tue, 17 Dec 2024 16:01:23 +0100 Subject: [PATCH 15/41] Reverting deleted code --- www/form/export.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/www/form/export.js b/www/form/export.js index a3b3a1c3f..0f48b22ef 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -59,7 +59,18 @@ define([ var type = obj.type; if (!TYPES[type]) { return; } // Ignore static types var id = `q${i++}`; - q[id] = obj.q || Messages.form_default; + if (TYPES[type] && TYPES[type].exportCSV) { + var _obj = Util.clone(obj); + _obj.q = "tmp"; + q[id] = { + question: obj.q, + items: TYPES[type].exportCSV(false, _obj).map(function (str) { + return str.slice(6); // Remove "tmp | " + }) + }; + } else { + q[id] = obj.q || Messages.form_default; + } }); sortedKeys.forEach(function (k) { @@ -73,14 +84,14 @@ define([ }; var i = 1; - order.forEach(function (key) { if (!form[key]) { return; } var type = form[key].type; if (!TYPES[type]) { return; } // Ignore static types var id = `q${i++}`; - if (type === 'date') { - msg[key] = new Date(msg[key]).toISOString(); + if (TYPES[type].exportCSV) { + data[id] = TYPES[type].exportCSV(msg[key], form[key]); + return; } data[id] = msg[key]; }); From 90c95beefee96de3ea61e112dfd5833e1ecc7c2b Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 17 Dec 2024 16:05:28 +0100 Subject: [PATCH 16/41] Fix issues --- lib/archive-account.js | 18 ++- lib/eviction.js | 59 ---------- lib/storage/blob.js | 245 +++++++++++++++------------------------ lib/workers/db-worker.js | 6 - 4 files changed, 107 insertions(+), 221 deletions(-) diff --git a/lib/archive-account.js b/lib/archive-account.js index 43ff1357e..44928f6be 100644 --- a/lib/archive-account.js +++ b/lib/archive-account.js @@ -13,6 +13,7 @@ const Metadata = require("./commands/metadata"); const Meta = require("./metadata"); const Logger = require("./log"); const plugins = require("./plugin-manager"); +const HK = require('./hk-util'); let SSOUtils = plugins.SSO && plugins.SSO.utils; @@ -53,7 +54,13 @@ const init = (cb) => { Env.computeMetadata = function (channel, cb) { const ref = {}; const lineHandler = Meta.createLineHandler(ref, (err) => { console.log(err); }); - return void Env.store.readChannelMetadata(channel, lineHandler, function (err) { + + let f = Env.store.readChannelMetadata; + if (channel.length === HK.BLOB_ID_LENGTH) { + f = Env.blobStore.readMetadata; + } + + return void f(channel, lineHandler, function (err) { if (err) { // stream errors? return void cb(err); @@ -132,9 +139,12 @@ COMMANDS.start = (edPublic, blockId, reason) => { n = n((w) => { // Blobs if (Env.blobStore.isFileId(chanId)) { - return void Env.blobStore.isOwnedBy(safeKey, chanId, w((err, owned) => { - if (err || !owned) { return; } - blobsToArchive.push(chanId); + return Env.computeMetadata(chanId, w((e, md) => { + if (e || !md) { return; } + if (md && md.owners + && md.owners.includes(edPublic)) { + blobsToArchive.push(chanId); + } })); } // Pads diff --git a/lib/eviction.js b/lib/eviction.js index 95f430cdf..8975e3fdd 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -642,64 +642,6 @@ module.exports = function (Env, cb) { })); }; - var archiveInactiveBlobProofs = function (w) { - // iterate over blob proofs and remove them - // if they don't correspond to a pinned or active file - var removed = 0; - var total = 0; - - Log.info("EVICT_ARCHIVE_INACTIVE_BLOB_PROOFS_START", {}); - blobs.list.proofs(function (err, item, next) { - next = Util.mkAsync(next, THROTTLE_FACTOR); - if (err) { - return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next); - } - if (!item) { - return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item, next); - } - total++; - - if (total % PROGRESS_FACTOR === 0) { - Log.info('EVICT_BLOB_PROOF_PROGRESS', { - proofs: total, - }); - } - - if (pinnedDocs.test(item.blobId)) { return void next(); } - if (item.mtime > inactiveTime) { return void next(); } - nThen(function (w) { - blobs.size(item.blobId, w(function (err, size) { - if (err && err === 'ENOENT') { return; } // XXX delete the proof - if (err) { - w.abort(); - return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next); - } - if (size !== 0) { - w.abort(); - next(); - } - })); - }).nThen(function () { - if (Env.DRY_RUN) { - removed++; - return void Log.info("EVICT_BLOB_PROOF_LONELY_DRY_RUN", item, next); - } - blobs.remove.proof(item.safeKey, item.blobId, function (err) { - if (err) { - return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item, next); - } - removed++; - return Log.info("EVICT_BLOB_PROOF_LONELY", item, next); - }); - }); - }, w(function () { - Log.info("EVICT_BLOB_PROOFS_REMOVED", { - removed, - total, - }, w()); - })); - }; - var archiveInactiveChannels = function (w) { var channels = 0; var archived = 0; @@ -802,7 +744,6 @@ module.exports = function (Env, cb) { // (documents which are not in either bloom filter) .nThen(archiveInactiveBlobs) - .nThen(archiveInactiveBlobProofs) .nThen(archiveInactiveChannels) .nThen(function () { var runningTime = report.runningTime = msSinceStart(); diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 8c6b9b2be..629dfd9d3 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -54,23 +54,10 @@ var makeStagePath = function (Env, safeKey) { return Path.join(Env.blobStagingPath, safeKey.slice(0, 2), safeKey); }; -// /blob//// -var makeProofPath = function (Env, safeKey, blobId) { - return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId); -}; - var mkPlaceholderPath = function (Env, blobId) { return makeBlobPath(Env, blobId) + '.placeholder'; }; -var parseProofPath = function (path) { - var parts = path.split('/'); - return { - blobId: parts[parts.length -1], - safeKey: parts[parts.length - 3], - }; -}; - // Placeholder for deleted files var addPlaceholder = function (Env, blobId, reason, cb) { if (!reason) { return cb(); } @@ -239,6 +226,11 @@ var archiveMetadata = (Env, blobId, cb) => { // if we fail to delete the metadata file, it can still be removed later by the eviction script Fse.move(path, archivePath, { overwrite: true }, cb); }; +var restoreActivity = function (Env, blobId, cb) { + var path = mkMetadataPath(Env, blobId); + var archivePath = prependArchive(Env, path); + Fse.move(archivePath, path, cb); +}; var readBlobMetadata = function (env, blobId, handler, _cb) { var metadataPath = mkMetadataPath(env, blobId); var stream = Fs.createReadStream(metadataPath, {start: 0}); @@ -403,7 +395,6 @@ var owned_upload_complete = function (Env, safeKey, id, cb) { var finalPath = makeBlobPath(Env, id); let unsafeKey = unescapeKeyCharacters(safeKey); - //var finalOwnPath = makeProofPath(Env, safeKey, id); // the user wants to move it into blob and create a metadata log with an owner @@ -465,19 +456,6 @@ var remove = function (Env, blobId, cb) { clearActivity(Env, blobId, () => {}); }; -// removeProof -var removeProof = function (Env, safeKey, blobId, cb) { - var proofPath = makeProofPath(Env, safeKey, blobId); - Fs.unlink(proofPath, cb); -}; - -// isOwnedBy(id, safeKey) -var isOwnedBy = function (Env, safeKey, blobId, cb) { - var proofPath = makeProofPath(Env, safeKey, blobId); - isFile(proofPath, cb); -}; - - // archiveBlob var archiveBlob = function (Env, blobId, reason, cb) { var blobPath = makeBlobPath(Env, blobId); @@ -499,29 +477,11 @@ var restoreBlob = function (Env, blobId, cb) { var blobPath = makeBlobPath(Env, blobId); var archivePath = prependArchive(Env, blobPath); Fse.move(archivePath, blobPath, cb); + restoreMetadata(Env, blobId, () => {}); restoreActivity(Env, blobId, () => {}); clearPlaceholder(Env, blobId, () => {}); }; -// archiveProof -var archiveProof = function (Env, safeKey, blobId, cb) { - var proofPath = makeProofPath(Env, safeKey, blobId); - var archivePath = prependArchive(Env, proofPath); - Fse.move(proofPath, archivePath, { overwrite: true }, cb); -}; - -var removeArchivedProof = function (Env, safeKey, blobId, cb) { - var archivedPath = prependArchive(Env, makeProofPath(Env, safeKey, blobId)); - Fs.unlink(archivedPath, cb); -}; - -// restoreProof -var restoreProof = function (Env, safeKey, blobId, cb) { - var proofPath = makeProofPath(Env, safeKey, blobId); - var archivePath = prependArchive(Env, proofPath); - Fse.move(archivePath, proofPath, cb); -}; - var makeWalker = function (n, handleChild, done) { if (!n || typeof(n) !== 'number' || n < 2) { n = 2; } @@ -553,7 +513,7 @@ var makeWalker = function (n, handleChild, done) { } if (!stats.isDirectory()) { w.abort(); - if (/\.activity$/.test(path)) { + if (/\.activity$/.test(path)) { // NOTE: some activity files were created for deleted blobs due to // a bug. We're going to detect them here in order to be able to clean // them. @@ -586,46 +546,6 @@ var makeWalker = function (n, handleChild, done) { return recurse; }; -var listProofs = function (root, handler, cb) { - Fs.readdir(root, function (err, dir) { - if (err) { return void cb(err); } - - var walk = makeWalker(20, function (err, path, next, loneActivity) { - if (loneActivity) { return void next(); } - // path is the path to a child node on the filesystem - - // next handles the next job in a queue - - // iterate over proofs - // check for presence of corresponding files - Fs.stat(path, function (err, stats) { - if (err) { - return void handler(err, void 0, next); - } - - var parsed = parseProofPath(path); - handler(void 0, { - path: path, - blobId: parsed.blobId, - safeKey: parsed.safeKey, - atime: stats.atime, - ctime: stats.ctime, - mtime: stats.mtime, - }, next); - }); - }, function () { - // called when there are no more directories or children to process - cb(); - }); - - dir.forEach(function (d) { - // ignore directories that aren't 3 characters long... - if (d.length !== 3) { return; } - walk(Path.join(root, d)); - }); - }); -}; - var getActivityStat = function (path, base, cb) { var suffix = base ? '' : '.activity'; Fs.stat(path+suffix, function (err, stats) { @@ -633,32 +553,91 @@ var getActivityStat = function (path, base, cb) { cb(err, stats); }); }; -var listBlobs = function (root, handler, cb) { - // iterate over files - Fs.readdir(root, function (err, dir) { - if (err) { return void cb(err); } - var walk = makeWalker(20, function (err, path, next, loneActivity) { - if (loneActivity) { return void next(); } - getActivityStat(path, false, function (err, stats) { - if (err) { - return void handler(err, void 0, next); - } - handler(void 0, { - blobId: Path.basename(path), - atime: stats.atime, - ctime: stats.ctime, - mtime: stats.mtime, - }, next); - }); - }, function () { - cb(); - }); +let blobRegex = /^[0-9a-fA-F]{48}(\.metadata)*(\.ndjson)*$/; +var listBlobs = function (root, handler, fast, cb) { + var dirList = []; - dir.forEach(function (d) { - if (d.length !== 2) { return; } - walk(Path.join(root, d)); + nThen(function (w) { + // the root of your datastore contains nested directories... + Fs.readdir(root, w(function (err, list) { + if (err) { + w.abort(); + // TODO check if we normally return strings or errors + return void cb(err); + } + dirList = list; + })); + }).nThen(function (waitFor) { + // search inside the nested directories + // stream it so you don't put unnecessary data in memory + var n = nThen; + dirList.forEach(function (dir) { + if (dir.length !== 2) { return; } + // Handle one directory at a time to save some memory + n = n(function (w) { + // do twenty things at a time + var sema = Semaphore.create(20); + var nestedDirPath = Path.join(root, dir); + Fs.readdir(nestedDirPath, w(function (err, list) { + if (err) { return void handler(err); } // Is this correct? + list.forEach(function (item) { + // ignore hidden files + if (/^\./.test(item)) { return; } + // ignore anything that isn't channel or metadata + if (!blobRegex.test(item)) { return; } + + var isLonelyMetadata = false; + var blobName; + var metadataName; + + // if the current file is not the channel data, then it must be metadata + if (!/^[0-9a-fA-F]{48}$/.test(item)) { + metadataName = item; + blobName = item.replace(/\.metadata\.ndjson/, ''); + // check if blob already exists + if (list.indexOf(blobName) !== -1) { return; } + // otherwise set a flag indicating that we should + // handle the metadata on its own + isLonelyMetadata = true; + } else { + blobName = item; + metadataName = blobName + '.metadata.ndjson'; + } + if (blobName.length !== 48) { return; } + + sema.take(function (give) { + var next = w(give()); + + if (fast) { + return void handler(void 0, { + blobId: blobName + }, next); + } + + var filePath = Path.join(nestedDirPath, blobName); + if (isLonelyMetadata) { + // Set time to 0 to delete this + // lonely metadata file + return void handler(void 0, { + blobId: blobName, + mtime: 0, + atime: 0, + ctime: 0 + }, next); + } + return void getActivityStat(filePath, false, (err, data) => { + data.blobId = blobName; + handler(err, data, next); + }); + }); + }); + })); + }).nThen; }); + n(waitFor()); + }).nThen(function () { + cb(); }); }; @@ -741,12 +720,6 @@ BlobStore.create = function (config, _cb) { upload_cancel(Env, safeKey, fileSize, cb); }, - isOwnedBy: function (safeKey, blobId, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } - isOwnedBy(Env, safeKey, blobId, cb); - }, - readMetadata: (blobId, handler, cb) => { if (!isValidId(blobId)) { return void cb("INVALID_ID"); } readBlobMetadata(Env, blobId, handler, cb); @@ -762,24 +735,12 @@ BlobStore.create = function (config, _cb) { if (!isValidId(blobId)) { return void cb("INVALID_ID"); } remove(Env, blobId, cb); }, - proof: function (safeKey, blobId, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } - if (!isValidId(blobId)) { return void cb("INVALID_ID"); } - removeProof(Env, safeKey, blobId, cb); - }, archived: { blob: function (blobId, _cb) { var cb = Util.once(Util.mkAsync(_cb)); if (!isValidId(blobId)) { return void cb("INVALID_ID"); } removeArchivedBlob(Env, blobId, cb); }, - proof: function (safeKey, blobId, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } - if (!isValidId(blobId)) { return void cb("INVALID_ID"); } - removeArchivedProof(Env, safeKey, blobId, cb); - }, }, loneActivity: function (_cb) { var cb = Util.once(Util.mkAsync(_cb)); @@ -793,12 +754,6 @@ BlobStore.create = function (config, _cb) { if (!isValidId(blobId)) { return void cb("INVALID_ID"); } archiveBlob(Env, blobId, reason, cb); }, - proof: function (safeKey, blobId, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } - if (!isValidId(blobId)) { return void cb("INVALID_ID"); } - archiveProof(Env, safeKey, blobId, cb); - }, }, restore: { @@ -807,12 +762,6 @@ BlobStore.create = function (config, _cb) { if (!isValidId(blobId)) { return void cb("INVALID_ID"); } restoreBlob(Env, blobId, cb); }, - proof: function (safeKey, blobId, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } - if (!isValidId(blobId)) { return void cb("INVALID_ID"); } - restoreProof(Env, safeKey, blobId, cb); - }, }, isBlobAvailable: function (blobId, _cb) { @@ -865,22 +814,14 @@ BlobStore.create = function (config, _cb) { }, list: { - blobs: function (handler, _cb) { + blobs: function (handler, _cb, fast) { var cb = Util.once(Util.mkAsync(_cb)); - listBlobs(Env.blobPath, handler, cb); - }, - proofs: function (handler, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - listProofs(Env.blobPath, handler, cb); + listBlobs(Env.blobPath, handler, fast, cb); }, archived: { - proofs: function (handler, _cb) { + blobs: function (handler, _cb, fast) { var cb = Util.once(Util.mkAsync(_cb)); - listProofs(prependArchive(Env, Env.blobPath), handler, cb); - }, - blobs: function (handler, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - listBlobs(prependArchive(Env, Env.blobPath), handler, cb); + listBlobs(prependArchive(Env, Env.blobPath), handler, fast, cb); }, } }, diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index ddd49ccc0..d1a0fe962 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -570,12 +570,6 @@ const removeOwnedBlob = function (data, cb) { nThen(function (w) { // check if you have permissions - blobStore.isOwnedBy(safeKey, blobId, w(function (err, owned) { - if (err || !owned) { - w.abort(); - return void cb("INSUFFICIENT_PERMISSIONS"); - } - })); computeMetadata({channel: blobId}, w((err, meta) => { if (err || !meta) { w.abort(); From 1e2f57a28f30dfc1280cc443b355534bf1391ddf Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 17 Dec 2024 17:03:27 +0100 Subject: [PATCH 17/41] Blob metadata --- lib/eviction.js | 85 +++++++++++++++++---------------------------- lib/storage/blob.js | 9 +++++ 2 files changed, 40 insertions(+), 54 deletions(-) diff --git a/lib/eviction.js b/lib/eviction.js index 8975e3fdd..b75fc78a6 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -51,7 +51,6 @@ var evictArchived = function (Env, cb) { var report = { // archivedChannelsRemoved, // archivedAccountsRemoved, - // archivedBlobProofsRemoved, // archivedBlobsRemoved, // totalChannels, @@ -237,37 +236,6 @@ var evictArchived = function (Env, cb) { store.listArchivedChannels(handler, w(done)); }; - var removeArchivedBlobProofs = function (w) { - if (typeof(Env.archiveRetentionTime) !== "number") { return; } - // Iterate over archive blob ownership proofs and remove them - // if they are older than the specified retention time - var removed = 0; - blobs.list.archived.proofs(function (err, item, next) { - next = Util.mkAsync(next, THROTTLE_FACTOR); - if (err) { - Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err); - return void next(); - } - if (item && item.ctime > retentionTime) { return void next(); } - if (Env.DRY_RUN) { - removed++; - return void Log.info("EVICT_ARCHIVED_BLOB_PROOF_DRY_RUN", item, next); - } - blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) { - if (err) { - Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item); - return void next(); - } - Log.info("EVICT_ARCHIVED_BLOB_PROOF", item); - removed++; - next(); - })); - }, w(function () { - report.archivedBlobProofsRemoved = removed; - Log.info('EVICT_ARCHIVED_BLOB_PROOFS_REMOVED', removed); - })); - }; - var removeArchivedBlobs = function (w) { if (typeof(Env.archiveRetentionTime) !== "number") { return; } // Iterate over archived blobs and remove them @@ -303,7 +271,6 @@ var evictArchived = function (Env, cb) { nThen(loadStorage) .nThen(migrateIncorrectBlobs) .nThen(removeArchivedChannels) - .nThen(removeArchivedBlobProofs) .nThen(removeArchivedBlobs) .nThen(function () { cb(void 0, report); @@ -315,7 +282,6 @@ module.exports = function (Env, cb) { var report = { // archivedChannelsRemoved, // archivedAccountsRemoved, - // archivedBlobProofsRemoved, // archivedBlobsRemoved, // totalChannels, @@ -612,34 +578,45 @@ module.exports = function (Env, cb) { if (pinnedDocs.test(item.blobId)) { return void next(); } if (activeDocs.test(item.blobId)) { return void next(); } - // This seems redundant because we're already checking the bloom filter - // but we can't implement a 'fast mode' for the iterator - // unless we address this race condition with this last-minute double-check - if (item.mtime > inactiveTime) { return void next(); } - - if (Env.DRY_RUN) { - removed++; - return void Log.info("EVICT_ARCHIVE_BLOB_DRY_RUN", { - item: item, - }, next); - } - blobs.archive.blob(item.blobId, 'INACTIVE', function (err) { - if (err) { - return Log.error("EVICT_ARCHIVE_BLOB_ERROR", { - error: err, + // NOTE: fast mode allows us to skip getStats for + // the pinned and active channels + nThen(function (w) { + // double check that the channel really is inactive before archiving it + // because it might have been created after the initial activity scan + blobs.getStats(item.blobId, w(function (err, newerItem) { + if (err) { return; } + if (newerItem && getNewestTime(newerItem) > retentionTime) { + // it's actually active, so don't archive it. + w.abort(); + cb(); + } + // else fall through to the archival + })); + }).nThen(function () { + if (Env.DRY_RUN) { + removed++; + return void Log.info("EVICT_ARCHIVE_BLOB_DRY_RUN", { item: item, }, next); } - removed++; - Log.info("EVICT_ARCHIVE_BLOB", { - item: item, - }, next); + blobs.archive.blob(item.blobId, 'INACTIVE', function (err) { + if (err) { + return Log.error("EVICT_ARCHIVE_BLOB_ERROR", { + error: err, + item: item, + }, next); + } + removed++; + Log.info("EVICT_ARCHIVE_BLOB", { + item: item, + }, next); + }); }); }, w(function () { report.totalBlobs = total; report.activeBlobs = total - removed; Log.info('EVICT_BLOBS_REMOVED', removed, w()); - })); + }), true); }; var archiveInactiveChannels = function (w) { diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 629dfd9d3..86b3fbeeb 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -553,6 +553,10 @@ var getActivityStat = function (path, base, cb) { cb(err, stats); }); }; +var getStats = function (Env, blobId, cb) { + var path = makeBlobPath(Env, blobId); + getActivityStat(path, false, cb); +}; let blobRegex = /^[0-9a-fA-F]{48}(\.metadata)*(\.ndjson)*$/; var listBlobs = function (root, handler, fast, cb) { @@ -812,6 +816,11 @@ BlobStore.create = function (config, _cb) { if (!isValidId(id)) { return void cb("INVALID_ID"); } getActivity(Env, id, cb); }, + getStats: function (id, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(id)) { return void cb("INVALID_ID"); } + getStats(Env, id, cb); + }, list: { blobs: function (handler, _cb, fast) { From 04cdd0a54aa42212810ba8b8eeb8c2eba228fb4a Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Wed, 18 Dec 2024 11:13:15 +0100 Subject: [PATCH 18/41] Only one (responses) file downloaded per Form --- www/common/make-backup.js | 6 +++--- www/form/export.js | 27 +++++++++++---------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index bf73d6c3a..7394be98a 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -32,7 +32,7 @@ define([ return n; }; - var transform = function (ctx, parsed, sjson, cb, padData, zip, existingNames) { + var transform = function (ctx, parsed, sjson, cb, padData) { var result = { data: sjson, ext: '.json', @@ -49,7 +49,7 @@ define([ result.ext = _ext || Exporter.ext || ''; result.data = data; cb(result); - }, null, ctx.sframeChan, padData, zip, sanitize, getUnique, existingNames); + }, null, ctx.sframeChan, padData); }, function () { cb(result); }); @@ -247,7 +247,7 @@ define([ }, { hash: parsed.hash, password: fData.password - }, zip, existingNames); + }); }); }; diff --git a/www/form/export.js b/www/form/export.js index 0f48b22ef..e4f81fb1b 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -182,7 +182,7 @@ define([ return order; }; - Export.main = function (content, cb, ext, sframeChan, parsed, zip, sanitize, getUnique, existingNames) { + Export.main = function (content, cb, ext, sframeChan, parsed) { if (sframeChan && content.form) { var _answers = content["answers"]; _answers['href'] = parsed.hash; @@ -191,24 +191,19 @@ define([ var answers; sframeChan.query("Q_FORM_FETCH_ANSWERS", _answers, function (err, obj) { answers = obj && obj.results; - var rawName = content.metadata.title || 'File'; - var fileName = getUnique(sanitize(rawName + ' (answers)'), '.json', existingNames); - existingNames.push(fileName.toLowerCase()); var types = {input: {}, textarea: {}, radio: {}, multiradio: {}, date: {}, checkbox: {}, multicheck: {}, sort: {}, poll: {}}; - var arr = Export.results(content, answers, types, getFullOrder(content), "json"); - var results = new Blob([arr], { type : "application/json" }); - var opts = { - binary: true, - }; - zip.file(fileName, results, opts); + var arr = Export.results(content, answers, types, getFullOrder(content), "json"); + cb(new Blob([arr], { + type: 'application/json;charset=utf-8' + })); }); + } else { + var json = Util.clone(content || {}); + delete json.answers; + cb(new Blob([JSON.stringify(json, 0, 2)], { + type: 'application/json;charset=utf-8' + })); } - - var json = Util.clone(content || {}); - delete json.answers; - cb(new Blob([JSON.stringify(json, 0, 2)], { - type: 'application/json;charset=utf-8' - })); }; return Export; From 53c647f7ca136690085de4240e1841e7d0a69094 Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Wed, 18 Dec 2024 11:32:58 +0100 Subject: [PATCH 19/41] Date format --- www/form/export.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/www/form/export.js b/www/form/export.js index e4f81fb1b..792fd5dc4 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -93,7 +93,11 @@ define([ data[id] = TYPES[type].exportCSV(msg[key], form[key]); return; } - data[id] = msg[key]; + if (type === 'date') { + data[id] = new Date(msg[key]).toISOString(); + } else { + data[id] = msg[key]; + } }); r.push(data); }); From ec394257913b3b4f17134ffca835274ada69ab53 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 18 Dec 2024 12:56:22 +0100 Subject: [PATCH 20/41] Fix encrypted edit links when downloading shared folders --- www/common/make-backup.js | 13 +++++++++++-- www/common/userObject.js | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/www/common/make-backup.js b/www/common/make-backup.js index 7394be98a..fd9330178 100644 --- a/www/common/make-backup.js +++ b/www/common/make-backup.js @@ -10,12 +10,13 @@ define([ '/common/common-interface.js', '/common/hyperscript.js', '/common/common-feedback.js', + '/common/userObject.js', '/common/inner/cache.js', '/customize/messages.js', '/components/nthen/index.js', '/components/saferphore/index.js', '/components/jszip/dist/jszip.min.js', -], function ($, FileCrypto, Hash, Util, UI, h, Feedback, +], function ($, FileCrypto, Hash, Util, UI, h, Feedback, UO, Cache, Messages, nThen, Saferphore, JsZip) { var saveAs = window.saveAs; @@ -170,7 +171,7 @@ define([ }); } - var href = (fData.href && fData.href.indexOf('#') !== -1) ? fData.href : fData.roHref; + let href = UO.getHref(fData, ctx.currentCryptor); var parsed = Hash.parsePadUrl(href); if (['pad', 'file'].indexOf(parsed.hashData.type) === -1) { return; } @@ -294,9 +295,17 @@ define([ if (typeof el === "object" && el.metadata !== true) { // if folder var fName = getUnique(sanitize(k), '', existingNames); existingNames.push(fName.toLowerCase()); + ctx.currentCryptor = undefined; return void makeFolder(ctx, el, zip.folder(fName), fd); } if (ctx.data.sharedFolders[el]) { // if shared folder + let obj = ctx.data.sharedFolders[el]; + let parsed = Hash.parsePadUrl(obj.href || obj.roHref); + var secret = Hash.getSecrets('drive', parsed.hash, obj.password); + let cryptor = secret.keys?.secondaryKey ? UO.createCryptor(secret.keys?.secondaryKey) + : undefined; + ctx.currentCryptor = cryptor; + var sfData = ctx.sf[el].metadata; var sfName = getUnique(sanitize((sfData && sfData.title) || 'Folder'), '', existingNames); existingNames.push(sfName.toLowerCase()); diff --git a/www/common/userObject.js b/www/common/userObject.js index aeb446487..c83cbbaa6 100644 --- a/www/common/userObject.js +++ b/www/common/userObject.js @@ -72,7 +72,7 @@ define([ // Href exists and is not encrypted: return href return pad.href; } - if (pad.href) { + if (pad.href && cryptor) { // Href exists and is encrypted var d = cryptor.decrypt(pad.href); // If we can decrypt, return the decrypted value, otherwise continue and return roHref From a2b7729d8277ae8b73f8bd58fe9ef00145ea1a6c Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Fri, 20 Dec 2024 10:54:28 +0100 Subject: [PATCH 21/41] Navigation and submission block now work as expected with pagebreak + conditional section --- www/form/inner.js | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/www/form/inner.js b/www/form/inner.js index 661b0ee98..77cd34b6d 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -4374,6 +4374,7 @@ define([ var shownPages = checkPages[1]; var shownLength = shownContent.length; $(state).text(Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownLength])); + togglePageArrows(shownLength) }); var left = h('button.btn.btn-secondary.cp-prev', [ h('i.fa.fa-arrow-left'), @@ -4385,6 +4386,29 @@ define([ if (shownPages.indexOf(_content[current-1])+1 === shownContent.length) { $(right).css('visibility', 'hidden'); } if (current === 1) { $(left).css('visibility', 'hidden'); } + var togglePageArrows = function(shownLength) { + var checkPages = checkEmptyPages(); + var shownContent = checkPages[0]; + var shownPages = checkPages[1]; + var shownLength = shownContent.length; + if (shownPages.indexOf(_content[current-1])+1 === shownLength) { + $(right).css('visibility', 'hidden'); + } else { + $(right).css('visibility', 'visible') + } + + if (current === 1) {$(left).css('visibility', 'hidden');} + $container.find('.cp-form-page').hide() + $($container.find('.cp-form-page').get(current-1)).show() + if (current !== shownLength) { + $container.find('.cp-form-send-container').hide() + } else { + $container.find('.cp-form-send-container').show() + } + } + + togglePageArrows() + $(left).click(function () { refreshPage(current - 1, 'prev'); }); @@ -4392,14 +4416,6 @@ define([ refreshPage(current + 1, 'next'); }); $page.append([left, state, right]); - - $container.find('.cp-form-page').hide(); - $($container.find('.cp-form-page').get(current-1)).show(); - if (current !== pages) { - $container.find('.cp-form-send-container').hide(); - } else { - $container.find('.cp-form-send-container').show(); - } }; setTimeout(refreshPage); } From bf01f3ff66d8151338c261d6bba9bdc534e90f7e Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Fri, 20 Dec 2024 11:23:18 +0100 Subject: [PATCH 22/41] Refactor --- www/form/inner.js | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/www/form/inner.js b/www/form/inner.js index 77cd34b6d..8eb4e7c75 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -4346,11 +4346,8 @@ define([ $page.empty(); if (!current || current < 1) { current = 1; } - var checkPages = checkEmptyPages(); - var shownContent = checkPages[0]; - var shownPages = checkPages[1]; - var shownLength = shownContent.length; - + var shownContent = checkEmptyPages()[0]; + var shownPages = checkEmptyPages()[1]; if (pgcontent[(current - 1)] && pgcontent[current-1].empty) { if (direction === 'next') { current++; @@ -4367,14 +4364,13 @@ define([ } } - var state = h('span', Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownLength])); + var state = h('span', Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownContent.length])); evOnChange.reg(function(){ - var checkPages = checkEmptyPages(); - var shownContent = checkPages[0]; - var shownPages = checkPages[1]; - var shownLength = shownContent.length; - $(state).text(Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownLength])); - togglePageArrows(shownLength) + togglePageArrows() + var shownContent = checkEmptyPages()[0]; + var shownPages = checkEmptyPages()[1]; + $(state).text(Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownContent.length])); + }); var left = h('button.btn.btn-secondary.cp-prev', [ h('i.fa.fa-arrow-left'), @@ -4386,12 +4382,10 @@ define([ if (shownPages.indexOf(_content[current-1])+1 === shownContent.length) { $(right).css('visibility', 'hidden'); } if (current === 1) { $(left).css('visibility', 'hidden'); } - var togglePageArrows = function(shownLength) { - var checkPages = checkEmptyPages(); - var shownContent = checkPages[0]; - var shownPages = checkPages[1]; - var shownLength = shownContent.length; - if (shownPages.indexOf(_content[current-1])+1 === shownLength) { + var togglePageArrows = function() { + var shownContent = checkEmptyPages()[0]; + var shownPages = checkEmptyPages()[1]; + if (shownPages.indexOf(_content[current-1])+1 === shownContent.length) { $(right).css('visibility', 'hidden'); } else { $(right).css('visibility', 'visible') @@ -4400,7 +4394,7 @@ define([ if (current === 1) {$(left).css('visibility', 'hidden');} $container.find('.cp-form-page').hide() $($container.find('.cp-form-page').get(current-1)).show() - if (current !== shownLength) { + if (current !== shownContent.length) { $container.find('.cp-form-send-container').hide() } else { $container.find('.cp-form-send-container').show() From 0de7fa52ff180c9f5921c150a53564853f303558 Mon Sep 17 00:00:00 2001 From: zuzanna-maria Date: Fri, 20 Dec 2024 11:29:53 +0100 Subject: [PATCH 23/41] Linting --- www/form/inner.js | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/www/form/inner.js b/www/form/inner.js index 8eb4e7c75..c426090a6 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -4364,14 +4364,6 @@ define([ } } - var state = h('span', Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownContent.length])); - evOnChange.reg(function(){ - togglePageArrows() - var shownContent = checkEmptyPages()[0]; - var shownPages = checkEmptyPages()[1]; - $(state).text(Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownContent.length])); - - }); var left = h('button.btn.btn-secondary.cp-prev', [ h('i.fa.fa-arrow-left'), ]); @@ -4379,29 +4371,38 @@ define([ h('i.fa.fa-arrow-right'), ]); - if (shownPages.indexOf(_content[current-1])+1 === shownContent.length) { $(right).css('visibility', 'hidden'); } - if (current === 1) { $(left).css('visibility', 'hidden'); } - var togglePageArrows = function() { var shownContent = checkEmptyPages()[0]; var shownPages = checkEmptyPages()[1]; if (shownPages.indexOf(_content[current-1])+1 === shownContent.length) { $(right).css('visibility', 'hidden'); } else { - $(right).css('visibility', 'visible') + $(right).css('visibility', 'visible'); } if (current === 1) {$(left).css('visibility', 'hidden');} - $container.find('.cp-form-page').hide() - $($container.find('.cp-form-page').get(current-1)).show() + $container.find('.cp-form-page').hide(); + $($container.find('.cp-form-page').get(current-1)).show(); if (current !== shownContent.length) { - $container.find('.cp-form-send-container').hide() + $container.find('.cp-form-send-container').hide(); } else { - $container.find('.cp-form-send-container').show() + $container.find('.cp-form-send-container').show(); } - } + }; - togglePageArrows() + var state = h('span', Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownContent.length])); + evOnChange.reg(function(){ + togglePageArrows(); + var shownContent = checkEmptyPages()[0]; + var shownPages = checkEmptyPages()[1]; + $(state).text(Messages._getKey('form_page', [shownPages.indexOf(_content[current-1])+1, shownContent.length])); + + }); + + if (shownPages.indexOf(_content[current-1])+1 === shownContent.length) { $(right).css('visibility', 'hidden'); } + if (current === 1) { $(left).css('visibility', 'hidden'); } + + togglePageArrows(); $(left).click(function () { refreshPage(current - 1, 'prev'); From ededf142d93b4b959b527e2c11a4c8848b7201d4 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 6 Jan 2025 16:52:59 +0100 Subject: [PATCH 24/41] Blob metadata and migration --- lib/storage/blob.js | 8 +- scripts/migrations/migrate-blob-proofs.js | 164 ++++++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 scripts/migrations/migrate-blob-proofs.js diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 86b3fbeeb..63932a2ce 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -226,7 +226,7 @@ var archiveMetadata = (Env, blobId, cb) => { // if we fail to delete the metadata file, it can still be removed later by the eviction script Fse.move(path, archivePath, { overwrite: true }, cb); }; -var restoreActivity = function (Env, blobId, cb) { +var restoreMetadata = function (Env, blobId, cb) { var path = mkMetadataPath(Env, blobId); var archivePath = prependArchive(Env, path); Fse.move(archivePath, path, cb); @@ -732,6 +732,12 @@ BlobStore.create = function (config, _cb) { if (!isValidId(blobId)) { return void cb("INVALID_ID"); } writeMetadata(Env, blobId, data, cb); }, + hasMetadata: (blobId, _cb) => { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + var path = mkMetadataPath(Env, blobId); + isFile(path, cb); + }, remove: { blob: function (blobId, _cb) { diff --git a/scripts/migrations/migrate-blob-proofs.js b/scripts/migrations/migrate-blob-proofs.js new file mode 100644 index 000000000..8c977693c --- /dev/null +++ b/scripts/migrations/migrate-blob-proofs.js @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team and contributors +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const Path = require('node:path'); +const nThen = require("nthen"); +const Semaphore = require("saferphore"); +const Logger = require("../../lib/log"); +const config = require("../../lib/load-config"); +const BlobStorage = require("../../lib/storage/blob"); +const Fs = require('node:fs'); + + +const blobPath = config.blobPath || './blob'; +let Log = {}; + +// XXX NOTE: in cleaning mode, we DON'T migrate +// (we suppose data has already been migrated) +const DRY_RUN = true; +const CLEAN_OLD = false; + +const start = (clean) => { + let dirList = []; + let blobStore; + nThen(w => { + Logger.create(config, w(function (_log) { + Log = _log; + })); + }).nThen(w => { + config.getSession = function () {}; + BlobStorage.create(config, w(function (err, _store) { + if (err) { + w.abort(); + return void Log.error("ERR_BLOB_STORE", err); + } + blobStore = _store; + })); + }).nThen(w => { + Fs.readdir(blobPath, w((err, list) => { + if (err) { + w.abort(); + return void Log.error("ERR_READING_ROOT", err); + } + dirList = list; + })); + }).nThen(() => { + let n = nThen; + dirList.forEach(dir => { + if (dir.length !== 3) { return; } + // ./blob/abc + const nestedDirPath = Path.join(blobPath, dir); + + if (clean) { + n = n(ww => { + Log.info("REMOVING_DIR", nestedDirPath); + if (DRY_RUN) { return; } + Fs.rm(nestedDirPath, { + recursive: true, force: true + }, ww(err => { + if (err) { + Log.error("ERR_REMOVE_DIR", { + path: nestedDirPath, + err + }); + } + })); + }).nThen; + return; + } + + n = n(w => { + // One user at a time + const sema = Semaphore.create(1); + let nestedDirList = []; + nThen(ww => { + Fs.readdir(nestedDirPath, ww((err, list) => { + if (err) { + w.abort(); + ww.abort(); + return Log.error("ERR_READING_DIR", { + path: nestedDirPath, + err + }); + } + nestedDirList = list; + })); + }).nThen(ww => { + nestedDirList.forEach(key => { + // ./blob/abc/abcdefg... + const keyPath = Path.join(nestedDirPath, key); + sema.take(give => { + let edPublic = key.replace(/\-/g, '/'); + let md = JSON.stringify({ owners: [edPublic] }); + Log.info("START_USER", edPublic); + Fs.readdir(keyPath, ww((err, list) => { + if (err) { + w.abort(); + ww.abort(); + return Log.error("ERR_READING_DIR", { + path: keyPath, + err + }); + } + let blobs = [] + nThen(www => { + list.forEach(dir => { + // ./blob/abc/abcdefg.../01 + const path = Path.join(keyPath, dir); + Fs.readdir(path, www((err, blobsList) => { + if (err) { + w.abort(); + ww.abort(); + www.abort(); + return Log.error("ERR_READING_DIR", { + path, err + }); + } + Array.prototype.push.apply(blobs, blobsList); + })); + }); + }).nThen(www => { + // migrate 20 blobs at a time for a given user + const sema = Semaphore.create(20); + blobs.forEach(blobId => { + sema.take(ggive => { + blobStore.isBlobAvailable(blobId, www((err, blobExists) => { + blobStore.hasMetadata(blobId, www((err, exists) => { + // If blob is not available or metadata already + // exists, don't write md file + if (!blobExists || exists) { return void ggive(); } + Log.info('WRITE_METADATA', blobId); + if (DRY_RUN) { return void ggive(); } + blobStore.writeMetadata(blobId, md, www(e => { + if (e) { + w.abort(); + ww.abort(); + www.abort(); + return Log.error("ERR_WRITING_MD", { blobId }); + } + ggive(); + })); + + })); + })); + }) + }); + }).nThen(ww(give(() => { + Log.info("END_USER", edPublic); + }))); + })); + }); + + }); + }).nThen(w()); + }).nThen; + }); + n(() => { + Log.info("DONE"); + process.exit(0); + }); + }); +}; + +start(CLEAN_OLD); From 267f6c56d48b583aa4b0a73b77a8f8012e0c9f25 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 10 Jan 2025 15:49:31 +0100 Subject: [PATCH 25/41] User stats script --- lib/pins.js | 9 ++- scripts/user-statistics.js | 133 +++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 scripts/user-statistics.js diff --git a/lib/pins.js b/lib/pins.js index 2201e561c..d2504077f 100644 --- a/lib/pins.js +++ b/lib/pins.js @@ -33,6 +33,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { // it's a weird API but it's faster than unpinning manually var pins = ref.pins = {}; ref.index = 0; + ref.first = 0; ref.latest = 0; // the latest message (timestamp in ms) ref.surplus = 0; // how many lines exist behind a reset @@ -58,7 +59,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { return sanitized; }; - return function (line) { + return function (line, i) { ref.index++; if (!Boolean(line)) { return; } @@ -74,6 +75,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { } if (typeof(l[2]) === 'number') { + if (!ref.first) { ref.first = l[2]; } ref.latest = l[2]; // date } @@ -109,6 +111,11 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { default: errorHandler("PIN_LINE_UNSUPPORTED_COMMAND", l); } + + if (i === 0) { // First line when using Pins.load + if (l[0] === 'PIN' || ref.block) { ref.user = true; } // teams always start with RESET + } + }; }; diff --git a/scripts/user-statistics.js b/scripts/user-statistics.js new file mode 100644 index 000000000..bc6b16227 --- /dev/null +++ b/scripts/user-statistics.js @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team and contributors +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const Path = require('node:path'); +const nThen = require("nthen"); +const Semaphore = require("saferphore"); +const Logger = require("../lib/log"); +const Pins = require("../lib/pins"); +const config = require("../lib/load-config"); +const BlobStorage = require("../lib/storage/blob"); +const Store = require("../lib/storage/file"); +const Fs = require('node:fs'); +const Quota = require("../lib/commands/quota"); +const Environment = require('../lib/env'); +const Env = Environment.create(config); + +const CSV = true; + +config.logPath = false; +config.logToStdout = true; + +const start = () => { + let time = +new Date(); + let Log = {}; + let all = {}; + let blobStore, pinStore, store; + nThen(w => { + Logger.create(config, w(function (_log) { + Env.Log = Log = _log; + })); + }).nThen(w => { + config.getSession = function () {}; + Store.create(config, w(function (err, _store) { + if (err) { + w.abort(); + return void Log.error("ERR_PAD_STORE", err); + } + store = _store; + })); + BlobStorage.create(config, w(function (err, _store) { + if (err) { + w.abort(); + return void Log.error("ERR_BLOB_STORE", err); + } + blobStore = _store; + })); + }).nThen(w => { + Quota.updateCachedLimits(Env, w((err) => { + if (err) { + return Env.Log.warn('UPDATE_QUOTA_ERR', err); + } + Env.Log.info('QUOTA_UPDATED', {}); + })); + }).nThen(w => { + Env.Log.info('START_LOADING_PINS'); + const handlePinLog = (content, id, next) => { + const sema = Semaphore.create(20); + const data = all[id] = { + size: 0, + n_pads: 0, + n_blobs: 0, + n_total: 0, + first: content.first, + last: content.latest + }; + if (!content.user) { + data.maybeTeam = true; + } + nThen(ww => { + Object.keys(content.pins).forEach(id => { + sema.take(give => { + let addSize = ww(give((e, s) => { + if (typeof(s) !== "number") { + return; // XXX + } + data.size += s; + data.n_total++; + if (id.length === 32) { + data.n_pads++; + } else { + data.n_blobs++; + } + })); + if (id.length === 32) { // PAD + return store.getChannelSize(id, addSize); + } + blobStore.size(id, addSize); + }); + }); + }).nThen(() => { + let key = id.replace(/-/g, '/'); + if (Env.limits[key]) { + let sub = Env.limits[key]; + data.premium = sub?.plan; + } + Env.Log.info('PIN_LOG_HANDLED', key); + next(); + }); + }; + + Pins.load(w(() => { + let duration = +new Date() - time; + Env.Log.info('ALL_PINS_LOADED', duration); + }), { + pinPath: config.pinPath, + handler: handlePinLog, + }); + }).nThen(() => { + if (!CSV) { return console.log(all); } + let csv = `"User key","Premium plan","Bytes","Number pads","Number blobs","First activity","Last activity","May be a team"\n`; + Object.keys(all).sort((a,b) => { + return all[b].size - all[a].size; + }).forEach(k => { + const data = all[k]; + k = k.replace(/-/g, '/'); + let first = new Date(data.first).toISOString().slice(0,10); + let last = new Date(data.last).toISOString().slice(0,10); + let plan = data.premium || ''; + let t = String(!!data.maybeTeam); + csv += `"${k}","${plan}","${data.size}","${data.n_pads}","${data.n_blobs}","${first}","${last}","${t}"\n`; + }); + let filename = `../${new Date().toISOString().slice(0,10)}-stats.csv`; + Fs.writeFile(filename, csv, err => { + if (err) { + console.error(err); + } else { + console.log('CSV available at', filename); + } + }); + }); +}; +start(); From 18e8f057cb2661389bca517ac307af437e777c9c Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 10 Jan 2025 16:54:37 +0100 Subject: [PATCH 26/41] Start blob proofs migration automatically --- lib/api.js | 25 ++++++++++++ lib/commands/admin-rpc.js | 2 +- lib/decrees.js | 8 ++++ lib/storage/blob.js | 13 ++++++ scripts/migrations/migrate-blob-proofs.js | 50 +++++++++++++++++++---- 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/lib/api.js b/lib/api.js index df3206b1e..d8313521c 100644 --- a/lib/api.js +++ b/lib/api.js @@ -27,6 +27,31 @@ nThen(function (w) { console.error(err); } })); +}).nThen(function (w) { + if (Env.proofsMigrated) { return; } + const { Worker } = require('node:worker_threads'); + const Admin = require("./commands/admin-rpc"); + + const worker = new Worker('./scripts/migrations/migrate-blob-proofs.js'); + + worker.on('message', message => { + if (message === 'READY') { + log.info('BLOB_PROOFS_MIGRATION'); + return void worker.postMessage({ + start: 1, + }); + } + if (message === 'MIGRATED') { + return void log.info('BLOB_PROOFS_DELETION'); + } + if (message === 'CLEANED') { + log.info('BLOB_PROOFS_MIGRATED'); + Admin.sendDecree(Env, null, function (err) { + if (err) { return void log.error('BLOB_PROOF', err); } + Env.flushCache(); + }, ['PROOFS_MIGRATED', ['PROOFS_MIGRATED', 1]], 'server'); + } + }); }).nThen(function (w) { let admins = Env.admins || []; diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 9030159f8..d029fc079 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -409,7 +409,7 @@ var getChannelMetadata = function (Env, Server, cb, data) { }; // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['RESTRICT_REGISTRATION', [true]]], console.log) -var adminDecree = function (Env, Server, cb, data, unsafeKey) { +var adminDecree = Admin.sendDecree = function (Env, Server, cb, data, unsafeKey) { var value = data[1]; if (!Array.isArray(value)) { return void cb('INVALID_DECREE'); } diff --git a/lib/decrees.js b/lib/decrees.js index 0124d9d90..c52476865 100644 --- a/lib/decrees.js +++ b/lib/decrees.js @@ -395,6 +395,14 @@ commands.ADD_ADMIN_KEY = function (Env, args) { return true; }; +commands.PROOFS_MIGRATED = function (Env, args) { + if (args !== 1) { + throw new Error("INVALID_ARGS"); + } + Env.proofsMigrated = true; + return true; +}; + commands.SET_BEARER_SECRET = function (Env, args) { if (!args_isString(args) || args.length !== 1 || !args[0]) { throw new Error("INVALID_ARGS"); diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 63932a2ce..f3d44c5d7 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -40,6 +40,7 @@ var makeBlobPath = function (Env, blobId) { return Path.join(Env.blobPath, blobId.slice(0, 2), blobId); }; + var makeActivityPath = function (Env, blobId) { return makeBlobPath(Env, blobId) + '.activity'; }; @@ -116,6 +117,18 @@ var isFile = function (filePath, cb) { }); }; +// PROOFS +// DEPRECATED, keep for compatibility +// /blob//// +var makeProofPath = function (Env, safeKey, blobId) { + return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId); +}; +// isOwnedBy(id, safeKey) +var isOwnedBy = function (Env, safeKey, blobId, cb) { + var proofPath = makeProofPath(Env, safeKey, blobId); + isFile(proofPath, cb); +}; + var makeFileStream = function (full, _cb) { var cb = Util.once(Util.mkAsync(_cb)); Fse.mkdirp(Path.dirname(full), function (e) { diff --git a/scripts/migrations/migrate-blob-proofs.js b/scripts/migrations/migrate-blob-proofs.js index 8c977693c..c8f87bb3a 100644 --- a/scripts/migrations/migrate-blob-proofs.js +++ b/scripts/migrations/migrate-blob-proofs.js @@ -2,24 +2,24 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later +const { parentPort } = require('node:worker_threads'); const Path = require('node:path'); +const Fs = require('node:fs'); const nThen = require("nthen"); const Semaphore = require("saferphore"); const Logger = require("../../lib/log"); -const config = require("../../lib/load-config"); const BlobStorage = require("../../lib/storage/blob"); -const Fs = require('node:fs'); +let config = require("../../lib/load-config"); const blobPath = config.blobPath || './blob'; let Log = {}; -// XXX NOTE: in cleaning mode, we DON'T migrate +// NOTE: in cleaning mode, we DON'T migrate // (we suppose data has already been migrated) -const DRY_RUN = true; -const CLEAN_OLD = false; +const start = (clean, dry, cb) => { + const DRY_RUN = dry; -const start = (clean) => { let dirList = []; let blobStore; nThen(w => { @@ -156,9 +156,43 @@ const start = (clean) => { }); n(() => { Log.info("DONE"); - process.exit(0); + cb(); }); }); }; -start(CLEAN_OLD); +if (parentPort) { + // Loaded as worker script + config = JSON.parse(JSON.stringify(config)); + config.logToStdout = false; + parentPort.on('message', (message) => { + let parsed = message; //JSON.parse(message); + if (!parsed?.start) { return; } + // Migrate + start(false, false, () => { + parentPort.postMessage('MIGRATED'); + // If success, clean + start(true, false, () => { + parentPort.postMessage('CLEANED'); + }); + }); + }); + parentPort.postMessage('READY'); +} else if (require.main === module) { + // Loaded from command-line + let dry = false; + let clean = false; + process.argv.forEach(key => { + if (key === '--dry') { + dry = true; + return; + } + if (key === '--clean') { + clean = true; + return; + } + }); + start(clean, dry, () => { + process.exit(0); + }); +} From 5fbcd78ab3b069dac57ab2eeb294c06b92218bb2 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 10 Jan 2025 17:23:06 +0100 Subject: [PATCH 27/41] Fallback to owners proofs during migration --- lib/env.js | 1 + lib/storage/blob.js | 5 +++++ lib/workers/db-worker.js | 20 ++++++++++++++++++++ lib/workers/index.js | 5 ++++- server.js | 10 +++++++++- 5 files changed, 39 insertions(+), 2 deletions(-) diff --git a/lib/env.js b/lib/env.js index d3748750f..a3593c7a1 100644 --- a/lib/env.js +++ b/lib/env.js @@ -415,6 +415,7 @@ const BAD = [ 'limits', 'customLimits', 'scheduleDecree', + 'plugins', 'httpServer', diff --git a/lib/storage/blob.js b/lib/storage/blob.js index f3d44c5d7..62719cefe 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -737,6 +737,11 @@ BlobStore.create = function (config, _cb) { upload_cancel(Env, safeKey, fileSize, cb); }, + isOwnedBy: function (safeKey, blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } + isOwnedBy(Env, safeKey, blobId, cb); + }, readMetadata: (blobId, handler, cb) => { if (!isValidId(blobId)) { return void cb("INVALID_ID"); } readBlobMetadata(Env, blobId, handler, cb); diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index d1a0fe962..9a4a39903 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -158,6 +158,12 @@ const isValidOffsetNumber = function (n) { return typeof(n) === 'number' && n >= 0; }; +const updateEnv = data => { + const {value} = data; + let env = Util.tryParse(value) || {}; + Env.proofsMigrated = env?.proofsMigrated; +}; + const computeIndexFromOffset = function (channelName, offset, cb) { let cpIndex = []; let messageBuf = []; @@ -576,6 +582,16 @@ const removeOwnedBlob = function (data, cb) { return void cb("INSUFFICIENT_PERMISSIONS"); } let owners = meta.owners; + if (!owners && !Env.proofsMigrated) { + // Check old proofs during migration + blobStore.isOwnedBy(safeKey, blobId, w((e, owned) => { + if (e || !owned) { + w.abort(); + return void cb("INSUFFICIENT_PERMISSIONS"); + } + })) + return; + } if (!owners || !owners.includes(unsafeKey)) { w.abort(); return void cb("INSUFFICIENT_PERMISSIONS"); @@ -693,6 +709,7 @@ const getLastChannelTime = function (data, cb) { }; const COMMANDS = { + ENV_UPDATE: updateEnv, COMPUTE_INDEX: computeIndex, COMPUTE_METADATA: computeMetadata, GET_OLDER_HISTORY: getOlderHistory, @@ -848,6 +865,9 @@ process.on('message', function (data) { }; if (!ready) { + if (data.env) { + updateEnv({value:data.env}); + } return void init(data.config, function (err) { if (err) { return void cb(Util.serializeError(err)); } ready = true; diff --git a/lib/workers/index.js b/lib/workers/index.js index 5bdbd5d7a..bc1780cc5 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -9,6 +9,7 @@ const { fork } = require('child_process'); const Workers = module.exports; const PID = process.pid; const Block = require("../storage/block"); +const Environment = require('../env'); const DB_PATH = 'lib/workers/db-worker'; const MAX_JOBS = 16; @@ -256,6 +257,7 @@ Workers.initialize = function (Env, config, _cb) { pid: PID, txid: txid, config: config, + env: Environment.serialize(Env) }); worker.on('message', function (res) { @@ -342,7 +344,8 @@ Workers.initialize = function (Env, config, _cb) { type: 'broadcast', pid: PID, command: data.command, - txid: data.txid + txid: data.txid, + value: data.value }); }); return workers; diff --git a/server.js b/server.js index 407f10f43..ca2b2ae77 100644 --- a/server.js +++ b/server.js @@ -190,7 +190,15 @@ nThen(function (w) { var throttledEnvChange = Util.throttle(function () { Env.Log.info('WORKER_ENV_UPDATE', 'Updating HTTP workers with latest state'); - broadcast('ENV_UPDATE', Environment.serialize(Env)); + let serialized = Environment.serialize(Env); + broadcast('ENV_UPDATE', serialized); + if (Env.broadcastWorkerCommand) { + Env.broadcastWorkerCommand({ + command: 'ENV_UPDATE', + value: serialized, + txid: Util.uid() + }); + } }, 250); // NOTE: changing this value will impact lib/commands/admin-rpc.js#adminDecree callback var throttledCacheFlush = Util.throttle(function () { From 7186f3e2ef2aa222fae06474f83960dd70693693 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 10 Jan 2025 17:28:05 +0100 Subject: [PATCH 28/41] lint compliance --- lib/api.js | 2 +- lib/hk-util.js | 2 +- lib/storage/blob.js | 10 ---------- lib/workers/db-worker.js | 2 +- scripts/migrations/migrate-blob-proofs.js | 4 ++-- scripts/user-statistics.js | 3 +-- 6 files changed, 6 insertions(+), 17 deletions(-) diff --git a/lib/api.js b/lib/api.js index d8313521c..d613df78e 100644 --- a/lib/api.js +++ b/lib/api.js @@ -27,7 +27,7 @@ nThen(function (w) { console.error(err); } })); -}).nThen(function (w) { +}).nThen(function () { if (Env.proofsMigrated) { return; } const { Worker } = require('node:worker_threads'); const Admin = require("./commands/admin-rpc"); diff --git a/lib/hk-util.js b/lib/hk-util.js index 3355f1c09..0ee685bba 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -42,7 +42,7 @@ const ADMIN_CHANNEL_LENGTH = HK.ADMIN_CHANNEL_LENGTH = 33; // with a 34 character id const EPHEMERAL_CHANNEL_LENGTH = HK.EPHEMERAL_CHANNEL_LENGTH = 34; -const BLOB_ID_LENGTH = HK.BLOB_ID_LENGTH = 48; +HK.BLOB_ID_LENGTH = 48; // Temporary channels are archived X ms after everyone has left them const TEMPORARY_CHANNEL_LIFETIME = 30 * 1000; diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 62719cefe..a7a053038 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -10,7 +10,6 @@ var BlobStore = module.exports; var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../common-util"); -var Meta = require("../metadata"); const PERMISSIVE = 511; const readFileBin = require("../stream-file").readFileBin; @@ -443,12 +442,6 @@ var owned_upload_complete = function (Env, safeKey, id, cb) { // move the existing file to its new path Fse.move(oldPath, finalPath, w(function (e) { if (e) { - // if there's an error putting the file into its final location... - // ... you should remove the ownership file - Fs.unlink(finalOwnPath, function () { - // but if you can't, it's not catestrophic - // we can clean it up later - }); w.abort(); return void cb(e.code); } @@ -606,11 +599,9 @@ var listBlobs = function (root, handler, fast, cb) { var isLonelyMetadata = false; var blobName; - var metadataName; // if the current file is not the channel data, then it must be metadata if (!/^[0-9a-fA-F]{48}$/.test(item)) { - metadataName = item; blobName = item.replace(/\.metadata\.ndjson/, ''); // check if blob already exists if (list.indexOf(blobName) !== -1) { return; } @@ -619,7 +610,6 @@ var listBlobs = function (root, handler, fast, cb) { isLonelyMetadata = true; } else { blobName = item; - metadataName = blobName + '.metadata.ndjson'; } if (blobName.length !== 48) { return; } diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index 9a4a39903..88fcd5989 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -589,7 +589,7 @@ const removeOwnedBlob = function (data, cb) { w.abort(); return void cb("INSUFFICIENT_PERMISSIONS"); } - })) + })); return; } if (!owners || !owners.includes(unsafeKey)) { diff --git a/scripts/migrations/migrate-blob-proofs.js b/scripts/migrations/migrate-blob-proofs.js index c8f87bb3a..d38c94bb0 100644 --- a/scripts/migrations/migrate-blob-proofs.js +++ b/scripts/migrations/migrate-blob-proofs.js @@ -101,7 +101,7 @@ const start = (clean, dry, cb) => { err }); } - let blobs = [] + let blobs = []; nThen(www => { list.forEach(dir => { // ./blob/abc/abcdefg.../01 @@ -142,7 +142,7 @@ const start = (clean, dry, cb) => { })); })); - }) + }); }); }).nThen(ww(give(() => { Log.info("END_USER", edPublic); diff --git a/scripts/user-statistics.js b/scripts/user-statistics.js index bc6b16227..02a74ef89 100644 --- a/scripts/user-statistics.js +++ b/scripts/user-statistics.js @@ -2,7 +2,6 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -const Path = require('node:path'); const nThen = require("nthen"); const Semaphore = require("saferphore"); const Logger = require("../lib/log"); @@ -24,7 +23,7 @@ const start = () => { let time = +new Date(); let Log = {}; let all = {}; - let blobStore, pinStore, store; + let blobStore, store; nThen(w => { Logger.create(config, w(function (_log) { Env.Log = Log = _log; From b135e815e7bd353ac7240ecffbdfb904c5cf7711 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 14 Jan 2025 18:13:45 +0100 Subject: [PATCH 29/41] Allow new types of decrees in different files --- lib/commands/admin-rpc.js | 13 ++ lib/decrees-core.js | 141 +++++++++++++++++ lib/decrees.js | 324 ++++++++++++-------------------------- lib/env.js | 2 +- lib/http-worker.js | 15 +- lib/load-config.js | 7 - package-lock.json | 16 +- package.json | 2 +- www/admin/inner.js | 22 ++- 9 files changed, 287 insertions(+), 255 deletions(-) create mode 100644 lib/decrees-core.js diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 9030159f8..b682bbf62 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -1138,6 +1138,19 @@ Admin.command = function (Env, safeKey, data, _cb, Server) { var command = commands[data[0]]; + Object.keys(Env.plugins || {}).forEach(name => { + let plugin = Env.plugins[name]; + if (!plugin.addAdminCommands) { return; } + try { + let c = plugin.addAdminCommands(Env); + Object.keys(c || {}).forEach(cmd => { + if (typeof(c[cmd]) !== "function") { return; } + if (commands[cmd]) { return; } + commands[cmd] = c[cmd]; + }); + } catch (e) {} + }); + if (typeof(command) === 'function') { return void command(Env, Server, cb, data, unsafeKey); } diff --git a/lib/decrees-core.js b/lib/decrees-core.js new file mode 100644 index 000000000..a8320a922 --- /dev/null +++ b/lib/decrees-core.js @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team and contributors +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +var Decrees = module.exports; +var Util = require("./common-util"); +var Fs = require("fs"); +var Path = require("path"); +var readFileBin = require("./stream-file").readFileBin; +var Schedule = require("./schedule"); +var Fse = require("fs-extra"); +var nThen = require("nthen"); + + +const Utils = Decrees.Utils = {}; +var isString = (str) => { + return typeof(str) === "string"; +}; +var isInteger = function (n) { + return !(typeof(n) !== 'number' || isNaN(n) || (n % 1) !== 0); +}; +Utils.args_isBoolean = function (args) { + return !(!Array.isArray(args) || typeof(args[0]) !== 'boolean'); +}; +Utils.args_isString = function (args) { + return !(!Array.isArray(args) || !isString(args[0])); +}; +Utils.args_isInteger = function (args) { + return !(!Array.isArray(args) || !isInteger(args[0])); +}; +Utils.args_isPositiveInteger = function (args) { + return Array.isArray(args) && isInteger(args[0]) && args[0] > 0; +}; + + +Decrees.create = (name, commands) => { + // [, , ,