The_Great_Archive/cryptpad
1
0
Fork 0
mirror of https://github.com/cryptpad/cryptpad.git synced 2026-09-14 11:05:41 +05:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): linked documents - trim history

Browse Source
This commit is contained in:
yflory 2026-06-23 16:53:09 +02:00
parent 316e17b356
commit ab2534ead9
17 changed files with 215 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/commands/channel.js
View File

@ -8,6 +8,7 @@ const Util = require("../common-util");
const nThen = require("nthen");
const Core = require("./core");
const Metadata = require("./metadata");
const Linked = require("./linked");
const HK = require("../hk-util");
const Nacl = require("tweetnacl/nacl-fast");
@ -200,7 +201,11 @@ Channel.trimHistory = function (Env, safeKey, data, cb) {
}
// else fall through to the next block
}));
}).nThen(function (w) {
// Archive old checkpoints
Linked.trimHistory(Env, { channel: channelId }, w());
}).nThen(function () {
// Trim chainpad doc:
Env.msgStore.trimChannel(channelId, hash, function (err) {
Env.Log.info('HK_TRIM_HISTORY', {
unsafeKey: unsafeKey,

77
lib/commands/linked.js
View File

@ -21,12 +21,14 @@ const getMetadata = (Env, channel, _cb) => {
MetaRPC.getMetadataRaw(Env, channel, (err, metadata) => {
if (err) { return void cb(err); }
if (metadata?.channel !== channel) {
if (metadata?.channel !== channel && channel.length !== HK.BLOB_ID_LENGTH) {
return cb();
}
// cache it
Env.metadata_cache[channel] = metadata;
if (channel.length !== HK.BLOB_ID_LENGTH) {
Env.metadata_cache[channel] = metadata;
}
cb(undefined, metadata);
});
};
@ -65,6 +67,23 @@ Linked.listLinkedDocuments = (Env, channel, cb) => {
});
};
Linked.listOldCheckpoints = (Env, channel, cb) => {
const list = new Set();
Linked.getLinkedDocuments(Env, { channel }, (err, json) => {
if (err) { return void cb(err); }
const cps = json.checkpoints || [];
cps.pop(); // preserve last cp
cps.forEach(obj => {
if (obj?.rtChannel) { list.add(obj.rtChannel); }
if (obj?.blob) { list.add(obj.blob); }
});
cb(void 0, Array.from(list));
});
};
const checkContent = (content, user) => {
const { type, data } = content;
@ -286,6 +305,8 @@ Linked.getHistorySize = (Env, data, _cb) => {
let linked;
let channelTotalSize = 0;
let size = 0;
let start = 0;
let hash;
nThen(waitFor => {
Linked.getLinkedDocuments(Env, data, waitFor((err, json) => {
@ -311,9 +332,19 @@ Linked.getHistorySize = (Env, data, _cb) => {
waitFor.abort();
return void cb(err);
}
start = offset;
const chanSize = channelTotalSize - offset;
size += chanSize;
}));
}).nThen(waitFor => {
// Get oldest hash of non-history data
Env.store.readMessagesBin(channel, start, (msgObj, readMore, abort) => {
const parsed = Util.tryParse(msgObj.buff.toString('utf8'));
if (!parsed) { return void readMore(); }
hash = HK.getHash(parsed[4]);
abort();
}, waitFor());
}).nThen(waitFor => {
// Get last checkpoint size (blob + rtChannel)
// Note: blob may be falsy if no checkpoint
@ -340,6 +371,46 @@ Linked.getHistorySize = (Env, data, _cb) => {
size += _size;
}), true);
}).nThen(() => {
cb(void 0, size);
cb(void 0, {
size, hash
});
});
};
Linked.trimHistory = (Env, data, cb) => {
const channel = data.channel;
let linked;
// if we reach this step, it means this user is an owner of "channel"
// so we can also delete any document linked to "channel" (from metadata)
nThen(waitFor => {
// List all but the current checkpoints
Linked.listOldCheckpoints(Env, channel, waitFor((err, channels) => {
if (err) {
waitFor.abort();
return void cb(err);
}
linked = channels || [];
}));
}).nThen(() => {
let n = nThen;
linked.forEach(chan => {
n = n(w => {
// If channel is "linked", we can archive all but last cp
getMetadata(Env, chan, w((err, md) => {
if (md?.linked !== channel) { return; }
// This is an old checkpoint linked to our document,
// we can archive it
const reason = "TRIM_HISTORY";
if (chan.length === HK.BLOB_ID_LENGTH) {
return Env.blobStore.archive.blob(chan, reason, w());
}
Env.store.archiveChannel(chan, reason, w());
}));
}).nThen;
});
n(() => {
cb();
});
});
};

6
lib/commands/upload.js
View File

@ -69,7 +69,9 @@ Upload.status = function (Env, safeKey, data, _cb) { // FIXME FILES
var user = Core.getSession(Env.Sessions, safeKey);
user.pendingUploadSize = filesize;
user.currentUploadSize = 0;
user.linked = data.linked;
console.error('UPLOAD STATUS', user.linked);
cb(void 0, false);
});
});
@ -88,7 +90,9 @@ var completeUpload = function (owned) {
Env.blobStore.closeBlobstage(safeKey);
var user = Core.getSession(Env.Sessions, safeKey);
var size = user.pendingUploadSize;
Env.completeUpload(safeKey, arg, Boolean(owned), size, cb);
var linked = user.linked;
console.error('COMPLETE USER', user.linked);
Env.completeUpload(safeKey, arg, Boolean(owned), size, linked, cb);
};
};

24
lib/storage/blob.js
View File

@ -378,7 +378,7 @@ var upload_cancel = function (Env, safeKey, fileSize, cb) {
};
// upload_complete
var upload_complete = function (Env, safeKey, id, cb) {
var upload_complete = function (Env, safeKey, id, cb, linked) {
closeBlobstage(Env, safeKey);
var oldPath = makeStagePath(Env, safeKey);
@ -405,6 +405,12 @@ var upload_complete = function (Env, safeKey, id, cb) {
}
cb(void 0, id);
}));
}).nThen(function (w) {
if (!linked) { return; }
// Write the metadata
let meta = { linked };
let md = JSON.stringify(meta);
writeMetadata(Env, id, md, w());
}).nThen(function () {
// finally, move the old file to the new path
// FIXME we could just move and handle the EEXISTS instead of the above block
@ -438,7 +444,7 @@ var tryId = function (path, cb) {
let unescapeKeyCharacters = function (key) {
return key.replace(/\-/g, '/');
};
var owned_upload_complete = function (Env, safeKey, id, cb) {
var owned_upload_complete = function (Env, safeKey, id, cb, linked) {
closeBlobstage(Env, safeKey);
if (!isValidId(id)) {
return void cb('EINVAL_ID');
@ -472,9 +478,9 @@ var owned_upload_complete = function (Env, safeKey, id, cb) {
}));
}).nThen(function (w) {
// Write the metadata
let md = JSON.stringify({
owners: [unsafeKey]
});
let meta = { owners: [unsafeKey] };
if (linked) { meta.linked = linked; }
let md = JSON.stringify(meta);
writeMetadata(Env, id, md, w((e) => {
if (e) {
w.abort();
@ -883,17 +889,17 @@ BlobStore.create = function (config, _cb) {
closeBlobstage: function (safeKey) {
closeBlobstage(Env, safeKey);
},
complete: function (safeKey, id, _cb) {
complete: function (safeKey, id, _cb, linked) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(id)) { return void cb("INVALID_ID"); }
upload_complete(Env, safeKey, id, cb);
upload_complete(Env, safeKey, id, cb, linked);
},
completeOwned: function (safeKey, id, _cb) {
completeOwned: function (safeKey, id, _cb, linked) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(id)) { return void cb("INVALID_ID"); }
owned_upload_complete(Env, safeKey, id, cb);
owned_upload_complete(Env, safeKey, id, cb, linked);
},
size: function (id, _cb) {
var cb = Util.once(Util.mkAsync(_cb));

2
lib/workers/db-worker.js
View File

@ -702,7 +702,7 @@ const completeUpload = function (data, cb) {
Env.blobStore[method](safeKey, arg, function (err, id) {
reportStatus(Env, label, safeKey, err, id, size);
cb(err, id);
});
}, data.linked);
};
const getPinActivity = function (data, cb) {

3
lib/workers/index.js
View File

@ -584,9 +584,10 @@ Workers.initialize = function (Env, config, _cb) {
}, cb);
};
Env.completeUpload = function (safeKey, arg, owned, size, cb) {
Env.completeUpload = function (safeKey, arg, owned, size, linked, cb) {
sendCommand({
command: "COMPLETE_UPLOAD",
linked,
owned: owned, // Boolean
safeKey: safeKey, // String (public key)
arg: arg, // String (file id)

3
src/worker/async-store.js
View File

@ -393,7 +393,8 @@ const factory = (Sortify, UserObject, ProxyManager,
if (!s.rpc) { return void cb({error: 'RPC_NOT_READY'}); }
s.rpc.uploadStatus({
id: data.id,
size: data.size
size: data.size,
linked: data.linked
}, function (err, res) {
if (err) { return void cb({error:err}); }
cb(res);

5
src/worker/modules/history.js
View File

@ -139,7 +139,9 @@ const factory = (Util, Hash, UserObject, nThen) => {
waitFor.abort();
return void cb(obj);
}
dataSize = obj[0] || 0;
let value = obj[0];
dataSize = value?.size || 0;
hash = value?.hash;
}));
} else {
Store.getHistory(null, {
@ -239,6 +241,7 @@ const factory = (Util, Hash, UserObject, nThen) => {
});
};
// XXX
commands.TRIM_HISTORY = function (ctx, data, cId, cb) {
if (!ctx.store.loggedIn || !ctx.store.rpc) { return void cb({ error: 'INSUFFICIENT_PERMISSIONS' }); }
var channels = data.channels;

4
www/common/cryptpad-common.js
View File

@ -635,8 +635,8 @@ define([
});
};
common.uploadStatus = function (teamId, id, size, cb) {
postMessage("UPLOAD_STATUS", {teamId, id, size}, function (obj) {
common.uploadStatus = function (data, cb) {
postMessage("UPLOAD_STATUS", data, function (obj) {
if (obj && obj.error) { return void cb(obj.error); }
cb(null, obj);
});

27
www/common/inner/properties.js
View File

@ -19,6 +19,7 @@ define([
var getPadProperties = function (Env, data, opts, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
var common = Env.common;
var sframeChan = common.getSframeChannel();
opts = opts || {};
var $d = $('<div>');
if (!data) { return void cb(void 0, $d); }
@ -175,23 +176,29 @@ define([
spinner.spin();
history.execCommand('TRIM_HISTORY', {
pad: true,
linked: data?.linked,
channels: trimChannels,
teamId: typeof(owned) === "number" && owned
}, function (obj) {
spinner.hide();
if (obj && obj.error || obj.warning) {
if (obj && obj.error || obj.warning) {
console.error(obj.warning);
$(size).append(h('div.alert.alert-danger', Messages.trimHistory_error));
spinner.hide();
return;
}
$(size).remove();
var formatted = UIElements.prettySize(bytes - historyBytes);
$d.append(h('div.cp-app-prop', [
Messages.upload_size,
h('br'),
h('span.cp-app-prop-content', formatted)
]));
$d.append(h('div.alert.alert-success', Messages.trimHistory_success));
sframeChan.query('Q_TRIM_HISTORY', {
href: data.href
}, function () {
spinner.hide();
$(size).remove();
var formatted = UIElements.prettySize(bytes - historyBytes);
$d.append(h('div.cp-app-prop', [
Messages.upload_size,
h('br'),
h('span.cp-app-prop-content', formatted)
]));
$d.append(h('div.alert.alert-success', Messages.trimHistory_success));
});
});
});

9
www/common/onlyoffice/history.js
View File

@ -9,11 +9,16 @@ define([
'/common/hyperscript.js',
'/common/common-icons.js',
'/common/common-util.js',
], function ($, UI, UIElements, h, Icons, Util) {
//var ChainPad = window.ChainPad;
var History = {};
History.sortCpIndex = function (hashes) {
return Object.keys(hashes).map(Number).sort(function (a, b) {
return a-b;
});
};
History.loadHistoryData = (cfg) => {
const {
sframeChan, mainRtChannel, downloadId,
@ -100,7 +105,7 @@ define([
const hashes = config.onlyoffice.hashes;
const mainRtChannel = config.onlyoffice.channel;
const sortedCp = config.sortCpIndex(hashes);
const sortedCp = History.sortCpIndex(hashes);
let cpIdx = sortedCp.length - 1;
let msgIdx = 0;

22
www/common/onlyoffice/inner.js
View File

@ -260,11 +260,7 @@ define([
});
};
var sortCpIndex = function (hashes) {
return Object.keys(hashes).map(Number).sort(function (a, b) {
return a-b;
});
};
var sortCpIndex = History.sortCpIndex;
const addLinkedCheckpoint = (cpData, cb) => {
let parsed = Hash.parsePadUrl(cpData.file);
@ -305,7 +301,7 @@ define([
});
});
// If < 10, add initial channel
if (sortedCp.length < 10) {
if (sortedCp.length < 10 && content.channel) {
value.checkpoints.unshift({
blob: 0,
rtChannel: content.channel
@ -618,7 +614,7 @@ define([
})
}, () => {});
};
const onRtChannelError = (err) => {
const onRtChannelError = (err, channel) => {
const wasReadOnly = readOnly;
readOnly = true;
offline = true;
@ -627,7 +623,7 @@ define([
error: err?.error,
reason: err?.reason,
channel: privateData.channel,
rtChannel: content.channel
rtChannel: channel
}, 0, 2);
let txt = Messages.oo_rtChannelMissing;
@ -694,7 +690,7 @@ define([
sframeChan.on('EV_OO_EVENT', function (obj) {
switch (obj.ev) {
case 'ERROR':
onRtChannelError(obj.data);
onRtChannelError(obj.data, channel);
cb();
break;
case 'READY':
@ -819,6 +815,8 @@ define([
resetData(blob, file, cpData);
};
// XXX
blob.linked = privateData.channel;
APP.FM.handleFile(blob, data);
};
@ -2224,9 +2222,10 @@ define([
// Check if history can/should be trimmed
var cp = getLastCp();
if (cp && cp.file && cp.hash) {
if (cp?.file) {
// XXX trim history to test
var channels = [{
channel: content.channel,
channel: cp.rtChannel || content.channel,
lastKnownHash: cp.hash
}];
common.checkTrimHistory(channels);
@ -3358,7 +3357,6 @@ Uncaught TypeError: Cannot read property 'calculatedType' of null
onRevert: commit,
setHistory: setHistoryMode,
makeSnapshot,
sortCpIndex,
onlyoffice: {
hashes: content.hashes || {},
channel: content.channel,

29
www/common/onlyoffice/trim-history.js Normal file
View File

@ -0,0 +1,29 @@
const factory = () => {
const sortCpIndex = (hashes) => {
return Object.keys(hashes).map(Number).sort(function (a, b) {
return a-b;
});
};
const trim = (content) => {
let hashes = content?.content?.hashes || {};
if (!hashes) { return content; }
const sortedCp = sortCpIndex(hashes);
const lastIdx = sortedCp.pop();
if (!lastIdx) { return content; }
const lastCp = hashes[lastIdx];
if (!lastCp) { return content; }
content.content.hashes = hashes = {};
hashes[lastIdx] = lastCp;
return content;
};
return { trim };
};
if (typeof(module) !== 'undefined' && module.exports) {
module.exports = factory();
} else if ((typeof(define) !== 'undefined' && define !== null) && (define.amd !== null)) {
define([], factory);
}

7
www/common/outer/upload.js
View File

@ -20,6 +20,7 @@ define([
var u8 = data.u8;
var metadata = data.metadata;
var key = data.key;
var linked = data.linked;
var onError = data.onError || function () {};
var onPending = data.onPending || function () {};
@ -141,7 +142,9 @@ define([
});
};
common.uploadStatus(teamId, id, estimate, function (e, pending) {
common.uploadStatus({
teamId, id, linked, size: estimate
}, function (e, pending) {
if (e) {
console.error(e);
onError(e);
@ -166,6 +169,7 @@ define([
module.upload = function (file, noStore, common, updateProgress, onComplete, onError, onPending) {
var u8 = file.blob; // This is not a blob but a uint8array
var metadata = file.metadata;
var linked = file.linked;
var owned = file.owned;
var teamId = file.teamId;
@ -209,6 +213,7 @@ define([
module.uploadU8(common, {
teamId: teamId,
u8: u8,
linked,
metadata: metadata,
key: key,
id: id,

1
www/common/sframe-common-file.js
View File

@ -485,6 +485,7 @@ define([
queue.push({
blob: file_arraybuffer,
metadata: metadata,
linked: file.linked,
password: password,
owned: owned,
forceSave: forceSave,

34
www/common/sframe-common-outer.js
View File

@ -1537,6 +1537,40 @@ define([
});
});
});
sframeChan.on('Q_TRIM_HISTORY', function (data, cb) {
const { href } = data;
const parsed = Utils.Hash.parsePadUrl(href);
let type = parsed.type;
if (['sheet', 'doc', 'presentation'].includes(type)) {
type = 'common/onlyoffice';
}
const path = `/${type}/trim-history.js`;
const cfg = {
password: data.password
};
require([path], (Trimming) => {
nThen(waitFor => {
Cryptpad.getAccessKeys(waitFor((keys) => {
cfg.accessKeys = keys;
}));
}).nThen(function () {
Cryptget.get(parsed.hash, (err, val) => {
if (err) { return void cb(); }
const json = Utils.Util.tryParse(val);
if (!json) { return void cb(); }
const newJson = Trimming.trim(json);
if (!newJson) { return void cb(); }
console.error(newJson);
Cryptget.put(parsed.hash, JSON.stringify(newJson), () => {
cb();
}, cfg);
}, cfg);
});
}, () => {
cb();
});
});
};
addCommonRpc(sframeChan, isSafe);

2
www/common/worker.bundle.min.js vendored
View File

File diff suppressed because one or more lines are too long