The_Great_Archive/cryptpad
1
0
Fork 0
mirror of https://github.com/cryptpad/cryptpad.git synced 2026-09-13 18:45:43 +05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'staging' into worker-loading

Browse Source
This commit is contained in:
yflory 2025-03-12 18:02:25 +01:00
commit 280cd3dc26
11 changed files with 273 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
customize.dist/src/less2/include/colortheme-dark.less
View File

@ -164,6 +164,7 @@
@cp_buttons-hover: @cryptpad_color_brand_fadest;
@cp_buttons-default: @cryptpad_color_grey_700;
@cp_buttons-default-color: @cryptpad_text_col;
@cp_buttons-default-alt-color: @cryptpad_color_grey_700;
@cp_buttons-default-border: @cryptpad_text_col;
@cp_buttons-red: #E55236;
@cp_buttons-red-text: @cryptpad_color_light_red;

1
customize.dist/src/less2/include/colortheme.less
View File

@ -164,6 +164,7 @@
@cp_buttons-default: #CCC;
@cp_buttons-default-color: @cryptpad_text_col;
@cp_buttons-default-border: @cryptpad_text_col;
@cp_buttons-default-alt-color: @cryptpad_color_grey_50;
@cp_buttons-red: #E55236;
@cp_buttons-red-text: @cp_buttons-red;
@cp_buttons-red-color: #FFF;

10
customize.dist/src/less2/include/forms.less
View File

@ -199,6 +199,16 @@
background-color: @cp_buttons-default;
}
}
&.btn-default-alt {
border-color: @cp_buttons-default-alt-color;
color: @cp_buttons-default-alt-color;
background-color: @cp_buttons-default-color;
&:hover, &:not(:disabled):active, &:focus {
border-color: @cp_buttons-default-color;
color: @cp_buttons-default-color;
background-color: @cp_toolbar-fade3;
}
}
&.danger, &.btn-danger {
background-color: @cp_buttons-red;

46
lib/hk-util.js
View File

@ -767,15 +767,16 @@ const handleGetHistoryRange = function (Env, Server, seq, userId, parsed) {
var channelName = parsed[1];
var map = parsed[2];
const HISTORY_KEEPER_ID = Env.id;
const store = Env.store;
if (!(map && typeof(map) === 'object')) {
return void Server.send(userId, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]);
}
var oldestKnownHash = map.from;
var untilHash = map.to;
var desiredMessages = map.count;
var desiredCheckpoint = map.cpCount;
var oldestKnownHash = map.from; // last known hash
var untilHash = map.to; // oldest hash (unknown), start point if defined
var desiredMessages = map.count; // nb messages before lkh
var desiredCheckpoint = map.cpCount; // nb cp before lkh
var txid = map.txid;
if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number' && !untilHash) {
return void Server.send(userId, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]);
@ -786,6 +787,43 @@ const handleGetHistoryRange = function (Env, Server, seq, userId, parsed) {
}
Server.send(userId, [seq, 'ACK']);
if (untilHash) {
// Get all messages between untilHash (oldest but unknown)
// and oldestKnownHesh (last known hash) (or until the end if undefined)
// Messages can be streamed since we instantly know the start point
let found = false;
store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {
const parsed = tryParse(Env, msgObj.buff.toString('utf8'));
if (!parsed) { return void readMore(); }
if (isMetadataMessage(parsed)) { return void readMore(); }
const content = parsed[4];
if (typeof(content) !== 'string') { return void readMore(); }
const hash = getHash(content);
if (hash === untilHash || untilHash === 'NONE') { found = true; }
let then = hash === oldestKnownHash ? abort : readMore;
if (found) {
Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId,
JSON.stringify(['HISTORY_RANGE', txid, parsed])], then);
}
return void readMore();
}, function (err, reason) {
if (err) {
Env.Log.error("HK_GET_OLDER_HISTORY", channelName, err, reason);
Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId,
JSON.stringify(['HISTORY_RANGE_ERROR', txid, err])
]);
return;
}
Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId,
JSON.stringify(['HISTORY_RANGE_END', txid, channelName])
]);
});
return;
}
// If desiredCp or desiredMsg are defined, we can't stream and must
// get a list of messages to send from a worker
Env.getOlderHistory(channelName, oldestKnownHash, untilHash, desiredMessages, desiredCheckpoint, function (err, toSend) {
if (err && err.code !== 'ENOENT') {
Env.Log.error("HK_GET_OLDER_HISTORY", err);

2
lib/storage/file.js
View File

@ -366,7 +366,7 @@ var readMessages = function (path, msgHandler, _cb) {
return readFileBin(stream, function (msgObj, readMore) {
collector.keepAlive();
msgHandler(msgObj.buff.toString('utf8'));
readMore();
setTimeout(readMore);
}, function (err) {
cb(err);
});

44
lib/workers/db-worker.js
View File

@ -40,7 +40,7 @@ Logger.levels.forEach(function (level) {
};
});
const HISTORY_SIZE_LIMIT = 1024 * 1024 * 1024; // XXX 1GB
//const HISTORY_SIZE_LIMIT = 1024 * 1024 * 1024; // XXX 1GB
var DETAIL = 1000;
var round = function (n) {
@ -396,11 +396,50 @@ const getFileSize = function (data, cb) {
const getOlderHistory = function (data, cb) {
const oldestKnownHash = data.hash;
const untilHash = data.toHash;
const channelName = data.channel;
const desiredMessages = data.desiredMessages;
const desiredCheckpoint = data.desiredCheckpoint;
let messages = [];
store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {
const parsed = HK.tryParse(Env, msgObj.buff.toString('utf8'));
if (!parsed) { return void readMore(); }
if (HK.isMetadataMessage(parsed)) { return void readMore(); }
const content = parsed[4];
if (typeof(content) !== 'string') { return void readMore(); }
const hash = HK.getHash(content);
messages.push(parsed);
// "X" messages before oldestKnownHash
if (typeof (desiredMessages) === "number") {
messages = messages.slice(-desiredMessages);
if (hash === oldestKnownHash) { return void abort(); }
return void readMore();
}
// "X" checkpoints before oldestKnownHash
if (hash === oldestKnownHash) { return void abort(); }
if (/^cp\|/.test(content)) { // clean whenever we push a cp
let foundCp = 0;
const idx = messages.findLastIndex(parsed => {
let isCp = /^cp\|/.test(parsed[4]);
if (!isCp) { return; }
foundCp++;
return foundCp >= desiredCheckpoint;
});
if (idx > 0) {
messages = messages.slice(idx);
}
}
readMore();
}, function (err, reason) {
if (err) { return void cb(err, reason); }
cb(void 0, messages);
});
/*
const untilHash = data.toHash;
var next = () => {
var messages = [];
var found = false;
@ -456,6 +495,7 @@ const getOlderHistory = function (data, cb) {
}
next();
});
*/
};
const getPinState = function (data, cb) {

81
lib/workers/index.js
View File

@ -53,16 +53,20 @@ Workers.initialize = function (Env, config, _cb) {
//return Object.keys(workers[index].tasks || {}).length;
};
const WORKER_TASK_LIMIT = 1000; // XXX
//const WORKER_TASK_LIMIT = 100; // XXX
var workerOffset = -1;
var queue = [];
var getAvailableWorkerIndex = function () {
var getAvailableWorkerIndex = function (isQueue) {
// If there is already a backlog of tasks you can avoid some work
// by going to the end of the line
if (queue.length) { return -1; }
// by going to the end of the line (unless we're trying to
// empty the queue)
if (queue.length && !isQueue) { return -1; }
var L = workers.length;
if (L === 0) {
Log.error('NO_WORKERS_AVAILABLE', {
Log.warn('NO_WORKERS_AVAILABLE', {
queue: queue.length,
});
return -1;
@ -94,7 +98,7 @@ Workers.initialize = function (Env, config, _cb) {
};
var drained = true;
var sendCommand = function (msg, _cb, opt) {
var sendCommand = function (msg, _cb, opt, isQueue) {
if (!_cb) {
return void Log.error('WORKER_COMMAND_MISSING_CB', {
msg: msg,
@ -103,7 +107,7 @@ Workers.initialize = function (Env, config, _cb) {
}
opt = opt || {};
var index = getAvailableWorkerIndex();
var index = getAvailableWorkerIndex(isQueue);
var state = workers[index];
// if there is no worker available:
@ -115,7 +119,7 @@ Workers.initialize = function (Env, config, _cb) {
});
if (drained) {
drained = false;
Log.error('WORKER_QUEUE_BACKLOG', {
Log.warn('WORKER_QUEUE_BACKLOG', {
workers: workers.length,
});
}
@ -128,13 +132,14 @@ Workers.initialize = function (Env, config, _cb) {
var cb = Util.once(Util.mkAsync(Util.both(_cb, function (err /*, value */) {
incrementTime(msg && msg.command, start);
if (err !== 'TIMEOUT') { return; }
Log.debug("WORKER_TIMEOUT_CAUSE", msg);
Log.warn("WORKER_TIMEOUT_CAUSE", msg);
// in the event of a timeout the user will receive an error
// but the state used to resend a query in the event of a worker crash
// won't be cleared. This also leaks a slot that could be used to keep
// an upper bound on the amount of parallelism for any given worker.
// if you run out of slots then the worker locks up.
delete state.tasks[txid];
state.checkTasks();
})));
if (!msg) {
@ -165,6 +170,12 @@ Workers.initialize = function (Env, config, _cb) {
msg._cb = _cb;
msg._opt = opt;
});
state.count++;
if (state.count > WORKER_TASK_LIMIT) {
// Remove from list and spawn new one
if (state.replaceWorker) { state.replaceWorker(); }
}
};
const pluginsResponses = {};
@ -208,6 +219,8 @@ Workers.initialize = function (Env, config, _cb) {
if (!res.txid) { return; }
response.handle(res.txid, [res.error, res.value]);
delete state.tasks[res.txid];
state.checkTasks();
if (!queue.length) {
if (!drained) {
drained = true;
@ -235,7 +248,7 @@ Workers.initialize = function (Env, config, _cb) {
to the back because the following msg took its place. OR, in an
even worse scenario, we cycle through the queue but don't run anything.
*/
sendCommand(nextMsg.msg, nextMsg.cb);
sendCommand(nextMsg.msg, nextMsg.cb, {}, true);
};
const initWorker = function (worker, cb) {
@ -244,13 +257,60 @@ Workers.initialize = function (Env, config, _cb) {
const state = {
worker: worker,
tasks: {},
count: Math.floor(Math.random()*(WORKER_TASK_LIMIT/10)),
pid: worker.pid, // store the child process's id in an easily accessible location
};
state.replaceWorker = () => {
let index = workers.indexOf(state);
if (index === -1) { return; }
// Remove old
workers.splice(index, 1);
// Create new
state.complete = true;
const w = fork(DB_PATH);
Log.info('WORKER_REPLACE_START', {
from: state.worker.pid,
to: w.pid
});
initWorker(w, function (err) {
if (err) {
throw new Error(err);
}
});
};
// If we've reached the limit, kill the worker once
// all the tasks are complete or timed out
state.checkTasks = () => {
// Check limit
if (!state.complete || !state.worker) { return; }
// Check remaining tasks
if (Object.keys(state.tasks).length) { return; }
// Kill
Log.info('WORKER_KILL', {
worker: state.worker.pid,
count: state.count
});
delete state.worker;
worker.kill();
};
response.expect(txid, function (err) {
if (err) { return void cb(err); }
workers.push(state);
cb(void 0, state);
// We just pushed a new worker, available to receive
// a task, so we can empty the queue if necessary
if (queue.length) {
const nextMsg = queue.shift();
if (!nextMsg || !nextMsg.msg) {
return Log.error('WORKER_QUEUE_EMPTY_MESSAGE', {
item: nextMsg,
});
}
sendCommand(nextMsg.msg, nextMsg.cb, {}, true);
}
}, 15000);
worker.send({
@ -298,18 +358,21 @@ Workers.initialize = function (Env, config, _cb) {
});
worker.on('exit', function () {
if (!state.worker) { return; } // Manually killed
substituteWorker();
Env.Log.error("DB_WORKER_EXIT", {
pid: state.pid,
});
});
worker.on('close', function () {
if (!state.worker) { return; } // Manually killed
substituteWorker();
Env.Log.error("DB_WORKER_CLOSE", {
pid: state.pid,
});
});
worker.on('error', function (err) {
if (!state.worker) { return; } // Manually killed
substituteWorker();
Env.Log.error("DB_WORKER_ERROR", {
pid: state.pid,

10
www/common/onlyoffice/inner.js
View File

@ -1720,11 +1720,14 @@ define([
"documentType": file.doc,
"editorConfig": {
customization: {
compactHeader: true,
chat: false,
logo: {
url: "/bounce/#" + encodeURIComponent('https://www.onlyoffice.com')
},
comments: !lock && !readOnly
comments: !lock && !readOnly,
hideRightMenu: true,
uiTheme: window.CryptPad_theme === "dark" ? "theme-dark" : "theme-classic-light"
},
"user": {
"id": String(myOOId), //"c0c3bf82-20d7-4663-bf6d-7fa39c598b1d",
@ -2183,8 +2186,11 @@ Uncaught TypeError: Cannot read property 'calculatedType' of null
}
c.forcesave = true;
}
console.error('updated config', APP.ooconfig);
// Always hide right menu
localStorage?.original.removeItem('sse-hide-right-settings');
localStorage?.original.removeItem('de-hide-right-settings');
localStorage?.original.removeItem('pe-hide-right-settings');
APP.docEditor = new window.DocsAPI.DocEditor("cp-app-oo-placeholder-a", APP.ooconfig);
ooLoaded = true;

11
www/common/sframe-boot2.js
View File

@ -25,16 +25,19 @@ define([
delete ls[k];
});
};
var mkFakeStore = function () {
var mkFakeStore = function (original) {
var fakeStorage = {
getItem: function (k) { return fakeStorage[k]; },
setItem: function (k, v) { fakeStorage[k] = v; return v; },
removeItem: function (k) { delete fakeStorage[k]; }
removeItem: function (k) { delete fakeStorage[k]; },
original
};
return fakeStorage;
};
window.__defineGetter__('localStorage', function () { return mkFakeStore(); });
window.__defineGetter__('sessionStorage', function () { return mkFakeStore(); });
let loc = localStorage;
let ses = sessionStorage;
window.__defineGetter__('localStorage', function () { return mkFakeStore(loc); });
window.__defineGetter__('sessionStorage', function () { return mkFakeStore(ses); });
window.CRYPTPAD_INSIDE = true;

38
www/kanban/app-kanban.less
View File

@ -145,6 +145,17 @@
margin-bottom: 15px;
}
.cp-kanban-toggle-container.cp-kanban-container-flex {
flex: 1;
}
.cp-kanban-toggle-tags {
text-transform: unset;
margin-right: 0.5rem;
padding: 3px 10px;
span {
font: @colortheme_app-font;
}
}
#cp-kanban-edit-tags {
.tokenfield {
margin: 0;
@ -152,10 +163,6 @@
}
margin-bottom: 15px;
}
.kanban-tag-btn-toggle {
margin-top: 10px;
margin-left: 10px
}
#cp-app-kanban-container {
flex: 1;
display: flex;
@ -430,12 +437,10 @@
justify-content: space-between;
position: relative;
min-height: 50px;
align-items: center;
.cp-kanban-filterTags {
@media (min-width: 505px) {
display: inline-flex;
.kanban-tag-btn-toggle {
margin-right: 10px;
}
}
align-items: center;
flex: 1;
@ -449,16 +454,8 @@
}
flex-flow: column;
flex-shrink: 0;
& > * {
visibility: hidden;
}
& > span {
display: inline-block;
height: 38px;
line-height: 38px;
}
& > button {
margin-top: -38px;
}
}
button.cp-kanban-filterTags-reset {
@ -571,7 +568,7 @@
display: flex;
min-height: 0;
.kanban-container {
padding: 30px 5px;
padding: 0px 5px;
flex: 1;
display: flex;
max-height: 100%;
@ -700,6 +697,15 @@
}
}
@media (pointer: none), (pointer:coarse) {
.kanban-container-outer {
.kanban-container {
padding: 30px 5px;
}
}
}
&.cp-app-readonly {
.kanban-item, .kanban-title-board {
cursor: default !important;

124
www/kanban/inner.js
View File

@ -65,6 +65,9 @@ define([
var onCursorUpdate = Util.mkEvent();
var remoteCursors = {};
let getCursor = () => {};
let restoreCursor = () => {};
var setValueAndCursor = function (input, val, _cursor) {
if (!input) { return; }
var $input = $(input);
@ -149,13 +152,17 @@ define([
var addEditItemButton = function () {};
var onRemoteChange = Util.mkEvent();
var now = function () { return +new Date(); };
var _lastUpdate = 0;
var _updateBoards = function (framework, kanban, boards) {
_lastUpdate = now();
var cursor = getCursor();
kanban.setBoards(Util.clone(boards));
kanban.inEditMode = false;
addEditItemButton(framework, kanban);
restoreCursor(cursor);
onRemoteChange.fire();
};
var _updateBoardsThrottle = Util.throttle(_updateBoards, 1000);
var updateBoards = function (framework, kanban, boards) {
@ -166,7 +173,6 @@ define([
_updateBoardsThrottle(framework, kanban, boards);
};
var onRemoteChange = Util.mkEvent();
var editModal;
var PROPERTIES = ['title', 'body', 'tags', 'color'];
var BOARD_PROPERTIES = ['title', 'color'];
@ -700,8 +706,11 @@ define([
var item = kanban.getItemJSON(eid);
item.title = name;
kanban.onChange();
// Unlock edit mode
kanban.inEditMode = false;
// Unlock edit mode unless we're already editing
// something else
if (kanban.inEditMode === eid) {
kanban.inEditMode = false;
}
onCursorUpdate.fire({});
};
$input.blur(save);
@ -764,7 +773,9 @@ define([
kanban.getBoardJSON(boardId).title = name;
kanban.onChange();
// Unlock edit mode
kanban.inEditMode = false;
if (kanban.inEditMode === boardId) {
kanban.inEditMode = false;
}
onCursorUpdate.fire({});
};
$input.blur(save);
@ -819,7 +830,9 @@ define([
});
var save = function () {
$item.remove();
kanban.inEditMode = false;
if (kanban.inEditMode === "new") {
kanban.inEditMode = false;
}
onCursorUpdate.fire({});
if (!$input.val()) { return; }
var id = Util.createRandomInteger();
@ -930,18 +943,15 @@ define([
//framework._.sfCommon.setPadAttribute('quickMode', false);
});
var toggleTagsButton = h('button.btn.btn-default.kanban-tag-btn-toggle', Messages.kanban_showTags);
// Tags filter
var existing = getExistingTags(kanban.options.boards);
var list = h('div.cp-kanban-filterTags-list');
var reset = h('button.btn.btn-cancel.cp-kanban-filterTags-reset', [
var reset = h('button.btn.btn-cancel.cp-kanban-filterTags-reset.cp-kanban-toggle-tags', [
h('i.fa.fa-times'),
Messages.kanban_clearFilter
h('span', Messages.kanban_clearFilter)
]);
var hint = h('span.cp-kanban-filterTags-name', Messages.kanban_tags);
var tags = h('div.cp-kanban-filterTags', [
h('span.cp-kanban-filterTags-toggle', [
hint,
reset,
@ -954,8 +964,16 @@ define([
var $hint = $(hint);
var setTagFilterState = function (bool) {
//$hint.toggle(!bool);
//$reset.toggle(!!bool);
$hint.css('visibility', bool? 'hidden': 'visible');
$hint.css('height', bool ? 0 : '');
$hint.css('padding-top', bool ? 0 : '');
$hint.css('padding-bottom', bool ? 0 : '');
$reset.css('visibility', bool? 'visible': 'hidden');
$reset.css('height', !bool ? 0 : '');
$reset.css('padding-top', !bool ? 0 : '');
$reset.css('padding-bottom', !bool ? 0 : '');
};
setTagFilterState();
@ -1022,41 +1040,48 @@ define([
commitTags();
});
let toggleTagsButton = h('button.btn.btn-default.cp-kanban-toggle-tags', [
h('i.fa.fa-tags'),
h('span', Messages.fm_tagsName)
]);
let toggleContainer = h('div.cp-kanban-toggle-container', toggleTagsButton);
if ($(window).width() < 500) {
let toggleClicked = false;
let $tags = $(tags);
let $toggleBtn = $(toggleTagsButton);
let toggle = () => {
$tags.toggle();
let visible = $tags.is(':visible');
$(toggleContainer).toggleClass('cp-kanban-container-flex', !visible);
$toggleBtn.toggleClass('btn-default', visible);
$toggleBtn.toggleClass('btn-default-alt', !visible);
};
$toggleBtn.click(function() {
toggleClicked = true;
toggle();
});
$(tags).append(toggleTagsButton);
var hideTags = function () {
for (var tag of list.children) {
if (existing.indexOf(tag.innerHTML) > 10) {
$(tag).hide();
}
const resizeTags = () => {
if (toggleClicked) { return; }
let visible = $tags.is(':visible');
// Small screen and visible: hide
if ($(window).width() < 600) {
if (visible) {
$(tags).show();
toggle();
}
};
hideTags();
var toggleTags = function () {
for (var tag of list.children) {
if (existing.indexOf(tag.innerHTML) > 10 && kanban.options.tags.indexOf(tag.innerHTML) === -1) {
if ($(tag).is(":visible")) {
$(tag).hide();
$(toggleTagsButton).text(Messages.kanban_showTags);
} else {
$(tag).show();
$(toggleTagsButton).text(Messages.kanban_hideTags);
}
}
}
};
$(toggleTagsButton).click(function() {
toggleTags();
});
}
return;
}
// Large screen: make visible by default
if (visible) { return; }
$(tags).hide();
toggle();
};
$(window).on('resize', resizeTags);
var container = h('div#cp-kanban-controls', [
toggleContainer,
tags,
h('div.cp-kanban-changeView', [
small,
@ -1065,18 +1090,6 @@ define([
]);
$container.before(container);
var common = framework._.sfCommon;
var $button = common.createButton('toggle', true, {
element: $(container),
icon: 'fa-tags',
text: Messages.fm_tagsName,
}, function () {
$button.toggleClass('cp-toolbar-button-active');
});
$button.addClass('cp-toolbar-button-active');
framework._.toolbar.$bottomL.append($button);
onRedraw.reg(function () {
// Redraw if new tags have been added to items
var old = Sortify(existing);
@ -1191,7 +1204,7 @@ define([
$container.find('.kanban-edit-item').remove();
});
var getCursor = function () {
getCursor = function () {
if (!kanban || !kanban.inEditMode) { return; }
try {
var id = kanban.inEditMode;
@ -1232,7 +1245,7 @@ define([
return {};
}
};
var restoreCursor = function (data) {
restoreCursor = function (data) {
if (!data) { return; }
try {
var id = data.id;
@ -1296,12 +1309,9 @@ define([
var remoteContent = newContent.content;
if (Sortify(currentContent) !== Sortify(remoteContent)) {
var cursor = getCursor();
verbose("Content is different.. Applying content");
kanban.options.boards = remoteContent;
updateBoards(framework, kanban, remoteContent);
restoreCursor(cursor);
onRemoteChange.fire();
}
});