Merge branch 'staging' into ooapi-dl

.dockerignore

@@ -3,14 +3,18 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 .dockerignore
+.editorconfig
 .git
 .gitignore
 .gitmodules
 .github
+.reuse
+.stylelintrc.js
+.eslint.config.js
 docker-compose.yml
-traefik2.yml
 Dockerfile*
 *.png
+*.md
 /onlyoffice-builds.git/
 /www/common/onlyoffice/dist/
 blob
@@ -18,3 +22,4 @@ block
 customize
 data
 datastore
+docs

CHANGELOG.md

@@ -4,6 +4,71 @@ SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and cont
 SPDX-License-Identifier: AGPL-3.0-or-later
 -->
 
+
+# ❄️ Winter release (2024.12.0)
+
+## Goals
+
+This version delivers fixes and improvements across CryptPad. We are particularly happy to release a fix of our OnlyOffice integration that could address long-standing issues with documents becoming corrupted. If confirmed at scale, this fix could dramatically improve the use of OnlyOffice apps in CryptPad.
+
+## Improvements
+
+- OnlyOffice integration
+  - Fix bug resulting in corrupted documents [#1736](https://github.com/cryptpad/cryptpad/pull/1736)
+- Drive
+  - Links included in Drive exports [#1695](https://github.com/cryptpad/cryptpad/pull/1695)
+- Rich Text
+  - Formatted tables and strikethrough text in Pad .md exports [#1720](https://github.com/cryptpad/cryptpad/pull/1720)
+- Forms
+  - Form password warning [#1690](https://github.com/cryptpad/cryptpad/pull/1690)
+- Performance improvements (1/3) to example-advanced.nginx.conf [#1709](https://github.com/cryptpad/cryptpad/pull/1709)
+- Enable toggle in and out of calendars on small screens [#1584](https://github.com/cryptpad/cryptpad/pull/1584)
+
+## Fixes
+
+- Accessibility
+  - Kanban Focus order fix [#1708](https://github.com/cryptpad/cryptpad/pull/1708)
+  - Change iframe title [#1706](https://github.com/cryptpad/cryptpad/pull/1706)
+  - Fix keyboard trap inside Form description [#1672](https://github.com/cryptpad/cryptpad/pull/1672)
+  - Disable arrow key navigation in the drive while modal is active [#1669](https://github.com/cryptpad/cryptpad/pull/1669)
+  - Simulate click action for keyboard users in Ctrl+E modal [#1726](https://github.com/cryptpad/cryptpad/pull/1726)
+- Drive
+  - Prevent links in trash from disappearing after drive reload [#1697](https://github.com/cryptpad/cryptpad/pull/1697)
+  - Restore multiple files/directories [#1692](https://github.com/cryptpad/cryptpad/pull/1692)
+  - Stop selection of all other trashed files when restoring single file [#1681](https://github.com/cryptpad/cryptpad/pull/1681)
+- Notifications
+  - Calendar reminders in notification panel [#1721](https://github.com/cryptpad/cryptpad/pull/1721)
+  - Add Notifications padding [#1688](https://github.com/cryptpad/cryptpad/pull/1688)
+  - Notification fixes [#1674](https://github.com/cryptpad/cryptpad/pull/1674)
+- Forms
+  - Fix some form storage-related bugs [#1723](https://github.com/cryptpad/cryptpad/pull/1723)
+  - Fix spacing issues in Forms  [#1682](https://github.com/cryptpad/cryptpad/pull/1682)
+  - Fix padding in form questions [#1670](https://github.com/cryptpad/cryptpad/pull/1670)
+- Helpdesk
+  - Fix "Closed" support tickets remaining in Inbox [#1719](https://github.com/cryptpad/cryptpad/pull/1719)
+- 'Request edit' button [#1680](https://github.com/cryptpad/cryptpad/pull/1680)
+- Fix example-code-typo [#1703](https://github.com/cryptpad/cryptpad/pull/1703)
+
+## Upgrade notes
+
+If you are upgrading from a version older than `2024.9.1` please read the upgrade notes of all versions between yours and `2024.9.1` to avoid configuration issues.
+
+To upgrade:
+
+1. Stop your server
+2. Get the latest code with git
+
+```bash
+git fetch origin --tags
+git checkout 2024.12.0
+npm ci
+npm run install:components
+```
+
+3. Restart your server
+4. Review your instance's checkup page to ensure that you are passing all tests
+
+
 # Autumn release (2024.9.0)
 
 ## Goals

customize.dist/messages.js

@@ -134,6 +134,24 @@ define(req, function(AppConfig, Default, Language) {
             return text;
         }
     };
+
+        // XXX
+        Messages.admin_cat_admins = "Administrators";
+        Messages.admin_admin = "Admin";
+        Messages.admin_listAdminsTitle = "Current administrators";
+        Messages.admin_listAdminsHint = "View and remove administrators";
+        Messages.admin_addAdminsTitle = "Add administrators";
+        Messages.admin_addAdminsHint = "Add administrators from their public key or from your contacts list";
+        Messages.admin_addAdminsAdd = "Promote a contact to admin";
+        Messages.admin_addKeyLabel = "Add an admin using their public key";
+
+        Messages.admin_listName = "Admin name";
+        Messages.admin_listKey = "Admin key";
+        Messages.admin_listAction = "Remove admin rights";
+
+        Messages.admin_listHardcoded = "Admin added into config.js. Can only be removed by editing the config file.";
+        Messages.admin_listConfirm = "Are you sure you want to remove the admin rights of this user?";
+
     return Messages;
 
 });

customize.dist/src/less2/include/colortheme-dark.less

@@ -164,6 +164,7 @@
 @cp_buttons-hover: @cryptpad_color_brand_fadest;
 @cp_buttons-default: @cryptpad_color_grey_700;
 @cp_buttons-default-color: @cryptpad_text_col;
+@cp_buttons-default-alt-color: @cryptpad_color_grey_700;
 @cp_buttons-default-border: @cryptpad_text_col;
 @cp_buttons-red: #E55236;
 @cp_buttons-red-text: @cryptpad_color_light_red;

customize.dist/src/less2/include/colortheme.less

@@ -164,6 +164,7 @@
 @cp_buttons-default: #CCC;
 @cp_buttons-default-color: @cryptpad_text_col;
 @cp_buttons-default-border: @cryptpad_text_col;
+@cp_buttons-default-alt-color: @cryptpad_color_grey_50;
 @cp_buttons-red: #E55236;
 @cp_buttons-red-text: @cp_buttons-red;
 @cp_buttons-red-color: #FFF;

customize.dist/src/less2/include/forms.less

@@ -199,6 +199,16 @@
                 background-color: @cp_buttons-default;
             }
         }
+        &.btn-default-alt {
+            border-color: @cp_buttons-default-alt-color;
+            color: @cp_buttons-default-alt-color;
+            background-color: @cp_buttons-default-color;
+            &:hover, &:not(:disabled):active, &:focus {
+                border-color: @cp_buttons-default-color;
+                color: @cp_buttons-default-color;
+                background-color: @cp_toolbar-fade3;
+            }
+        }
 
         &.danger, &.btn-danger {
             background-color: @cp_buttons-red;

customize.dist/src/less2/include/sidebar-layout.less

@@ -7,6 +7,7 @@
 @import (reference) "/customize/src/less2/include/colortheme-all.less";
 @import (reference) "/customize/src/less2/include/leftside-menu.less";
 @import (reference) "/customize/src/less2/include/browser.less";
+@import (reference) "/customize/src/less2/include/variables.less";
 
 @sidebar_block-width: 25rem;;
 @sidebar_base-margin: 0.5rem;
@@ -121,6 +122,12 @@
                         flex-flow: column;
                         align-items: baseline;
                     }
+                    .cp-usergrid-container {
+                        margin-bottom: 0px !important;
+                        .cp-usergrid-grid {
+                            margin-bottom: -3px;
+                        }
+                    }
                     label {
                         margin-bottom: 0;
                     }
@@ -128,6 +135,9 @@
                         font-family: inherit;
                         max-width: @sidebar_block-width;
                     }
+                    input:invalid {
+                        border: 1px solid red;
+                    }
                     [type="color"] {
                         width: @sidebar_block-width/5;
                         padding: 3px;
@@ -214,7 +224,7 @@
                             border-top-left-radius: 0px;
                             border-bottom-left-radius: 0px;
                             border-left: 0px;
-                            height: 40px;
+                            height: @variables_input-height;
                             margin: 0 !important;
                         }
                     }
@@ -236,6 +246,12 @@
                         margin-bottom: 0;
                     }
                 }
+                .cp-sidebarlayout-description-item {
+                    display: block;
+                    color: @cp_sidebar-hint;
+                    margin-top: @sidebar_base-margin;
+                    margin-bottom: 0;
+                }
                 label.noTitle {
                     display: inline-flex;
                     .fa {

docker-compose.yml

@@ -5,7 +5,7 @@
 ---
 services:
   cryptpad:
-    image: "cryptpad/cryptpad:version-2024.12.0"
+    image: "cryptpad/cryptpad:version-2025.3.0"
     hostname: cryptpad
 
     environment:

lib/api.js

@@ -27,6 +27,31 @@ nThen(function (w) {
             console.error(err);
         }
     }));
+}).nThen(function () {
+    if (Env.proofsMigrated) { return; }
+    const { Worker } = require('node:worker_threads');
+    const Admin = require("./commands/admin-rpc");
+
+    const worker = new Worker('./scripts/migrations/migrate-blob-proofs.js');
+
+    worker.on('message', message => {
+        if (message === 'READY') {
+            log.info('BLOB_PROOFS_MIGRATION');
+            return void worker.postMessage({
+                start: 1,
+            });
+        }
+        if (message === 'MIGRATED') {
+            return void log.info('BLOB_PROOFS_DELETION');
+        }
+        if (message === 'CLEANED') {
+            log.info('BLOB_PROOFS_MIGRATED');
+            Admin.sendDecree(Env, null, function (err) {
+                if (err) { return void log.error('BLOB_PROOF', err); }
+                Env.flushCache();
+            }, ['PROOFS_MIGRATED', ['PROOFS_MIGRATED', 1]], 'server');
+        }
+    });
 }).nThen(function (w) {
     let admins = Env.admins || [];
 

lib/archive-account.js

@@ -13,6 +13,7 @@ const Metadata = require("./commands/metadata");
 const Meta = require("./metadata");
 const Logger = require("./log");
 const plugins = require("./plugin-manager");
+const HK = require('./hk-util');
 
 let SSOUtils = plugins.SSO && plugins.SSO.utils;
 
@@ -53,7 +54,13 @@ const init = (cb) => {
     Env.computeMetadata = function (channel, cb) {
         const ref = {};
         const lineHandler = Meta.createLineHandler(ref, (err) => { console.log(err); });
-        return void Env.store.readChannelMetadata(channel, lineHandler, function (err) {
+
+        let f = Env.store.readChannelMetadata;
+        if (channel.length === HK.BLOB_ID_LENGTH) {
+            f = Env.blobStore.readMetadata;
+        }
+
+        return void f(channel, lineHandler, function (err) {
             if (err) {
                 // stream errors?
                 return void cb(err);
@@ -132,9 +139,12 @@ COMMANDS.start = (edPublic, blockId, reason) => {
             n = n((w) => {
                 // Blobs
                 if (Env.blobStore.isFileId(chanId)) {
-                    return void Env.blobStore.isOwnedBy(safeKey, chanId, w((err, owned) => {
-                        if (err || !owned) { return; }
-                        blobsToArchive.push(chanId);
+                    return Env.computeMetadata(chanId, w((e, md) => {
+                        if (e || !md) { return; }
+                        if (md && md.owners
+                            && md.owners.includes(edPublic)) {
+                            blobsToArchive.push(chanId);
+                        }
                     }));
                 }
                 // Pads

lib/commands/admin-rpc.js

@@ -18,9 +18,11 @@ const MFA = require("../storage/mfa");
 const ArchiveAccount = require('../archive-account');
 const { Worker } = require('node:worker_threads');
 const Fse = require("fs-extra");
-
 const Fs = require("fs");
 
+const config = require("../load-config");
+const Keys = require("../keys");
+
 var Admin = module.exports;
 
 var getFileDescriptorCount = function (Env, server, cb) {
@@ -409,7 +411,7 @@ var getChannelMetadata = function (Env, Server, cb, data) {
 };
 
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['RESTRICT_REGISTRATION', [true]]], console.log)
-var adminDecree = function (Env, Server, cb, data, unsafeKey) {
+var adminDecree = Admin.sendDecree = function (Env, Server, cb, data, unsafeKey) {
     var value = data[1];
     if (!Array.isArray(value)) { return void cb('INVALID_DECREE'); }
 
@@ -468,7 +470,29 @@ var setLastEviction = function (Env, Server, cb, data, unsafeKey) {
 };
 
 // CryptPad_AsyncStore.rpc.send('ADMIN', ['INSTANCE_STATUS], console.log)
+const getAdminsData = (Env) => {
+    return Env.adminsData.map(str => {
+        // str is either a full public key or just the ed part
+        const edPublic = Keys.canonicalize(str);
+        const hardcoded = Array.isArray(config?.adminKeys) &&
+                config.adminKeys.some(key => {
+                    return Keys.canonicalize(key) === edPublic;
+                });
+        if (str.length === 44) {
+            return { edPublic, first: true, hardcoded };
+        }
+        let name;
+        try {
+            const parsed = Keys.parseUser(str);
+            name = parsed.user;
+        } catch (e) {}
+        return {
+            edPublic, hardcoded, name
+        };
+    });
+};
 var instanceStatus = function (Env, Server, cb) {
+
     cb(void 0, {
 
         appsToDisable: Env.appsToDisable,
@@ -514,6 +538,8 @@ var instanceStatus = function (Env, Server, cb) {
         instanceName: Env.instanceName,
         instanceNotice: Env.instanceNotice,
         enforceMFA: Env.enforceMFA,
+
+        admins: getAdminsData(Env)
     });
 };
 
@@ -1138,6 +1164,19 @@ Admin.command = function (Env, safeKey, data, _cb, Server) {
 
     var command = commands[data[0]];
 
+    Object.keys(Env.plugins || {}).forEach(name => {
+        let plugin = Env.plugins[name];
+        if (!plugin.addAdminCommands) { return; }
+        try {
+            let c = plugin.addAdminCommands(Env);
+            Object.keys(c || {}).forEach(cmd => {
+                if (typeof(c[cmd]) !== "function") { return; }
+                if (commands[cmd]) { return; }
+                commands[cmd] = c[cmd];
+            });
+        } catch (e) {}
+    });
+
     if (typeof(command) === 'function') {
         return void command(Env, Server, cb, data, unsafeKey);
     }

lib/commands/metadata.js

@@ -13,7 +13,8 @@ Data.getMetadataRaw = function (Env, channel /* channelName */, _cb) {
     const cb = Util.once(Util.mkAsync(_cb));
     if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); }
     if (channel.length !== HK.STANDARD_CHANNEL_LENGTH &&
-        channel.length !== HK.ADMIN_CHANNEL_LENGTH) { return cb("INVALID_CHAN_LENGTH"); }
+        channel.length !== HK.ADMIN_CHANNEL_LENGTH &&
+        channel.length !== HK.BLOB_ID_LENGTH) { return cb("INVALID_CHAN_LENGTH"); }
 
     // return synthetic metadata for admin broadcast channels as a safety net
     // in case anybody manages to write metadata
@@ -79,6 +80,7 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) {
 
     var channel = data.channel;
     var command = data.command;
+    // XXX BLOBMD allow blobs
     if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); }
     if (!command || typeof (command) !== 'string') { return void cb('INVALID_COMMAND'); }
     if (Meta.commands.indexOf(command) === -1) { return void cb('UNSUPPORTED_COMMAND'); }
@@ -140,6 +142,7 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) {
                 cb(void 0, metadata);
                 return void next();
             }
+            // XXX BLOBMD use correct store for blobs
             Env.msgStore.writeMetadata(channel, JSON.stringify(line), function (e) {
                 if (e) {
                     cb(e);
@@ -155,6 +158,7 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) {
 
                 // update the cached metadata
                 metadata_cache[channel] = metadata;
+                Env.checkCache(channel); // XXX ???
 
                 // it's easy to check if the channel is restricted
                 const isRestricted = metadata.restricted;

lib/decrees-core.js

@@ -0,0 +1,140 @@
+// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+var Decrees = module.exports;
+var Util = require("./common-util");
+var Fs = require("fs");
+var Path = require("path");
+var readFileBin = require("./stream-file").readFileBin;
+var Schedule = require("./schedule");
+var Fse = require("fs-extra");
+var nThen = require("nthen");
+
+
+const Utils = Decrees.Utils = {};
+var isString = (str) => {
+    return typeof(str) === "string";
+};
+var isInteger = function (n) {
+    return !(typeof(n) !== 'number' || isNaN(n) || (n % 1) !== 0);
+};
+Utils.args_isBoolean = function (args) {
+    return !(!Array.isArray(args) || typeof(args[0]) !== 'boolean');
+};
+Utils.args_isString = function (args) {
+    return !(!Array.isArray(args) || !isString(args[0]));
+};
+Utils.args_isInteger = function (args) {
+    return !(!Array.isArray(args) || !isInteger(args[0]));
+};
+Utils.args_isPositiveInteger = function (args) {
+    return Array.isArray(args) && isInteger(args[0]) && args[0] > 0;
+};
+
+
+Decrees.create = (name, commands) => {
+    // [<command>, <args>, <author>, <time>]
+    const handleCommand = function (Env, line) {
+        var command = line[0];
+        var args = line[1];
+
+        if (typeof(commands[command]) !== 'function') {
+            throw new Error("DECREE_UNSUPPORTED_COMMAND");
+        }
+
+        var outcome = commands[command](Env, args);
+        if (outcome) {
+            // trigger Env change event...
+            Env.envUpdated.fire();
+        }
+        return outcome;
+    };
+
+    const createLineHandler = function (Env) {
+        var Log = Env.Log;
+
+        var index = -1;
+
+        return function (err, line) {
+            index++;
+            if (err) {
+                // Log the error and bail out
+                return void Log.error("DECREE_LINE_ERR", {
+                    error: err.message,
+                    index: index,
+                    line: line,
+                });
+            }
+
+            if (Array.isArray(line)) {
+                try {
+                    return void handleCommand(Env, line);
+                } catch (err2) {
+                    return void Log.error("DECREE_COMMAND_ERR", {
+                        error: err2.message,
+                        index: index,
+                        line: line,
+                    });
+                }
+            }
+
+            Log.error("DECREE_HANDLER_WEIRD_LINE", {
+                line: line,
+                index: index,
+            });
+        };
+    };
+
+    const load = function (Env, _cb) {
+        Env.scheduleDecree = Env.scheduleDecree || Schedule();
+
+        var cb = Util.once(Util.mkAsync(function (err) {
+            if (err && err.code !== 'ENOENT') {
+                return void _cb(err);
+            }
+            _cb();
+        }));
+
+        Env.scheduleDecree.blocking('', function (unblock) {
+            var done = Util.once(Util.both(cb, unblock));
+            nThen(function (w) {
+                // ensure that the path to the decree log exists
+                Fse.mkdirp(Env.paths.decree, w(function (err) {
+                    if (!err) { return; }
+                    w.abort();
+                    done(err);
+                }));
+            }).nThen(function () {
+                var decreeName = Path.join(Env.paths.decree, name);
+                var stream = Fs.createReadStream(decreeName, {start: 0});
+                var handler = createLineHandler(Env);
+                readFileBin(stream, function (msgObj, next) {
+                    var text = msgObj.buff.toString('utf8');
+                    try {
+                        handler(void 0, JSON.parse(text));
+                    } catch (err) {
+                        handler(err, text);
+                    }
+                    next();
+                }, function (err) {
+                    done(err);
+                });
+            });
+        });
+    };
+
+    const write = function (Env, decree, _cb) {
+        var path = Path.join(Env.paths.decree, name);
+        Env.scheduleDecree.ordered('', function (next) {
+            var cb = Util.both(Util.mkAsync(_cb), next);
+            Fs.appendFile(path, JSON.stringify(decree) + '\n', cb);
+        });
+    };
+
+    return {
+        handleCommand,
+        load,
+        write
+    };
+};

lib/decrees.js

@@ -1,9 +1,91 @@
-// SPDX-FileCopyrightText: 2023 XWiki CryptPad Team <contact@cryptpad.org> and contributors
+// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team <contact@cryptpad.org> and contributors
 //
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-var Decrees = module.exports;
-var Core = require("./commands/core");
+const Core = require("./commands/core");
+const DecreesCore = require("./decrees-core");
+const config = require('./load-config');
+
+const Quota = require("./commands/quota");
+const Keys = require("./keys");
+
+const DECREE_NAME = 'decree.ndjson';
+
+const {
+    args_isBoolean,
+    args_isString,
+    args_isInteger,
+    args_isPositiveInteger
+} = DecreesCore.Utils;
+
+// Toggles a simple boolean
+const makeBooleanSetter = function (attr) {
+    return function (Env, args) {
+        if (!args_isBoolean(args)) {
+            throw new Error('INVALID_ARGS');
+        }
+        var bool = args[0];
+        if (bool === Env[attr]) { return false; }
+        Env[attr] = bool;
+        return true;
+    };
+};
+
+const default_validator = function () { return true; };
+const makeGenericSetter = function (attr, validator) {
+    validator = validator || default_validator;
+    return function (Env, args) {
+        if (!validator(args)) {
+            throw new Error("INVALID_ARGS");
+        }
+        var value = args[0];
+        if (value === Env[attr]) { return false; }
+        Env[attr] = value;
+        return true;
+    };
+};
+
+const makeIntegerSetter = function (attr) {
+    return makeGenericSetter(attr, args_isInteger);
+};
+
+const makeTranslation = function (attr) {
+    return function (Env, args) {
+        if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); }
+        var value = args[0];
+        var state = Env[attr];
+
+        if (typeof(value) === 'string') {
+            if (state.default === value) { return false; }
+            state.default = value;
+            return true;
+        }
+        if (value && typeof(value) === 'object') {
+            var changed = false;
+            Object.keys(value).forEach(function (lang) {
+                if (state[lang] === value[lang]) { return; }
+                state[lang] = value[lang];
+                changed = true;
+            });
+            return changed;
+        }
+        return false;
+    };
+};
+
+/*  commands have a simple API:
+
+* they receive the global Env and the arguments to be applied
+* if the arguments are invalid the operation will not be applied
+  * the command throws
+* if the arguments are valid but do not result in a change, the operation is redundant.
+  * return false
+* if the arguments are valid and will result in a change, the operation should be applied
+  * apply it
+  * return true to indicate that it was applied
+
+*/
+const commands = {};
 
 /*  Admin decrees which modify global server state
 
@@ -75,33 +157,26 @@ RM_ADMIN_KEY
 
 */
 
-var commands = {};
-/*  commands have a simple API:
-
-* they receive the global Env and the arguments to be applied
-* if the arguments are invalid the operation will not be applied
-  * the command throws
-* if the arguments are valid but do not result in a change, the operation is redundant.
-  * return false
-* if the arguments are valid and will result in a change, the operation should be applied
-  * apply it
-  * return true to indicate that it was applied
-
-*/
-
-var args_isBoolean = function (args) {
-    return !(!Array.isArray(args) || typeof(args[0]) !== 'boolean');
+// Maintenance: Empty string or an object with a start and end time
+const isNumber = function (value) {
+    return typeof(value) === "number" && !isNaN(value);
+};
+const args_isMaintenance = function (args) {
+    return Array.isArray(args) && args[0] &&
+        (args[0] === "" || (isNumber(args[0].end) && isNumber(args[0].start)));
 };
 
-// Toggles a simple boolean
-var makeBooleanSetter = function (attr) {
+// we anticipate that we'll add language-specific surveys in the future
+// whenever that happens we can relax validation a bit to support more formats
+const makeBroadcastSetter = function (attr, validation) {
     return function (Env, args) {
-        if (!args_isBoolean(args)) {
+        if ((validation && !validation(args)) && !args_isString(args)) {
             throw new Error('INVALID_ARGS');
         }
-        var bool = args[0];
-        if (bool === Env[attr]) { return false; }
-        Env[attr] = bool;
+        var str = args[0];
+        if (str === Env[attr]) { return false; }
+        Env[attr] = str;
+        Env.broadcastCache = {};
         return true;
     };
 };
@@ -139,49 +214,6 @@ commands.REMOVE_DONATE_BUTTON = makeBooleanSetter('removeDonateButton');
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['BLOCK_DAILY_CHECK', [true]]], console.log)
 commands.BLOCK_DAILY_CHECK = makeBooleanSetter('blockDailyCheck');
 
-/*
-var isNonNegativeNumber = function (n) {
-    return !(typeof(n) !== 'number' || isNaN(n) || n < 0);
-};
-*/
-
-var default_validator = function () { return true; };
-var makeGenericSetter = function (attr, validator) {
-    validator = validator || default_validator;
-    return function (Env, args) {
-        if (!validator(args)) {
-            throw new Error("INVALID_ARGS");
-        }
-        var value = args[0];
-        if (value === Env[attr]) { return false; }
-        Env[attr] = value;
-        return true;
-    };
-};
-
-var isString = (str) => {
-    return typeof(str) === "string";
-};
-var isInteger = function (n) {
-    return !(typeof(n) !== 'number' || isNaN(n) || (n % 1) !== 0);
-};
-
-var args_isString = function (args) {
-    return !(!Array.isArray(args) || !isString(args[0]));
-};
-
-var args_isInteger = function (args) {
-    return !(!Array.isArray(args) || !isInteger(args[0]));
-};
-
-var makeIntegerSetter = function (attr) {
-    return makeGenericSetter(attr, args_isInteger);
-};
-
-var arg_isPositiveInteger = function (args) {
-    return Array.isArray(args) && isInteger(args[0]) && args[0] > 0;
-};
-
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_LOGO_MIME', ['image/png']]], console.log)
 commands.SET_LOGO_MIME = makeGenericSetter('logoMimeType', args_isString);
 
@@ -192,7 +224,7 @@ commands.SET_ACCENT_COLOR = makeGenericSetter('accentColor', args_isString);
 commands.ENABLE_PROFILING = makeBooleanSetter('enableProfiling');
 
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_PROFILING_WINDOW', [10000]]], console.log)
-commands.SET_PROFILING_WINDOW = makeGenericSetter('profilingWindow', arg_isPositiveInteger);
+commands.SET_PROFILING_WINDOW = makeGenericSetter('profilingWindow', args_isPositiveInteger);
 
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAX_UPLOAD_SIZE', [50 * 1024 * 1024]]], console.log)
 commands.SET_MAX_UPLOAD_SIZE = makeIntegerSetter('maxUploadSize');
@@ -246,30 +278,6 @@ commands.SET_SUPPORT_KEYS = function (Env, args) {
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_PURPOSE', ["development"]]], console.log)
 commands.SET_INSTANCE_PURPOSE = makeGenericSetter('instancePurpose', args_isString);
 
-var makeTranslation = function (attr) {
-    return function (Env, args) {
-        if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); }
-        var value = args[0];
-        var state = Env[attr];
-
-        if (typeof(value) === 'string') {
-            if (state.default === value) { return false; }
-            state.default = value;
-            return true;
-        }
-        if (value && typeof(value) === 'object') {
-            var changed = false;
-            Object.keys(value).forEach(function (lang) {
-                if (state[lang] === value[lang]) { return; }
-                state[lang] = value[lang];
-                changed = true;
-            });
-            return changed;
-        }
-        return false;
-    };
-};
-
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_JURISDICTION', ['France']]], console.log)
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_JURISDICTION', [{default:'France',de:'Frankreich'}]]], console.log)
 commands.SET_INSTANCE_JURISDICTION = makeTranslation('instanceJurisdiction');
@@ -286,30 +294,6 @@ commands.SET_INSTANCE_DESCRIPTION = makeTranslation('instanceDescription');
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_NOTICE', [{default:'Our hosting costs have increased during the pandemic. Please consider donating!',fr:'Nos coûts d'hébergement ont augmenté pendant la pandémie. Veuillez envisager de faire un don !']]], console.log)
 commands.SET_INSTANCE_NOTICE = makeTranslation('instanceNotice');
 
-// Maintenance: Empty string or an object with a start and end time
-var isNumber = function (value) {
-    return typeof(value) === "number" && !isNaN(value);
-};
-var args_isMaintenance = function (args) {
-    return Array.isArray(args) && args[0] &&
-        (args[0] === "" || (isNumber(args[0].end) && isNumber(args[0].start)));
-};
-
-// we anticipate that we'll add language-specific surveys in the future
-// whenever that happens we can relax validation a bit to support more formats
-var makeBroadcastSetter = function (attr, validation) {
-    return function (Env, args) {
-        if ((validation && !validation(args)) && !args_isString(args)) {
-            throw new Error('INVALID_ARGS');
-        }
-        var str = args[0];
-        if (str === Env[attr]) { return false; }
-        Env[attr] = str;
-        Env.broadcastCache = {};
-        return true;
-    };
-};
-
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_LAST_BROADCAST_HASH', [hash]]], console.log)
 commands.SET_LAST_BROADCAST_HASH = makeBroadcastSetter('lastBroadcastHash');
 
@@ -320,10 +304,6 @@ commands.SET_SURVEY_URL = makeBroadcastSetter('surveyURL');
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAINTENANCE', [""]]], console.log)
 commands.SET_MAINTENANCE = makeBroadcastSetter('maintenance', args_isMaintenance);
 
-var Quota = require("./commands/quota");
-var Keys = require("./keys");
-var Util = require("./common-util");
-
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_QUOTA', ['[user@box:3000/VzeS4vP1DF+tXGuq1i50DKYuBL+09Yqy8kGxoUKRzhA=]', { limit: 2 * 1024 * 1024 * 1024, plan: 'buddy', note: "you're welcome" } ] ] ], console.log)
 commands.SET_QUOTA = function (Env, args) {
     if (!Array.isArray(args) || args.length !== 2) {
@@ -387,11 +367,53 @@ commands.ADD_ADMIN_KEY = function (Env, args) {
 
     Env.admins = Env.admins || [];
 
-
     var key = Keys.canonicalize(args[0]);
     if (!key) { throw new Error("INVALID_KEY"); }
-    Env.admins.push(key);
 
+    if (Env.admins.includes(key)) { // Nothing to change
+        return false;
+    }
+
+    Env.admins.push(key);
+    Env.adminsData.push(args[0]);
+    return true;
+};
+commands.RM_ADMIN_KEY = function (Env, args) {
+    if (!Array.isArray(args) || args.length !== 1 || !args[0]) {
+        throw new Error("INVALID_ARGS");
+    }
+
+    const key = Keys.canonicalize(args[0]);
+    if (!key) { throw new Error("INVALID_KEY"); }
+
+    Env.admins = Env.admins || [];
+    if (!Env.admins.includes(key)) { // Nothing to change
+        return false;
+    }
+
+    // NOTE prevent removing config.js hardcoded admin keys
+    if (Array.isArray(config?.adminKeys) && config.adminKeys.includes(key)) {
+        throw new Error("CANT_REMOVE_CONFIG");
+    }
+
+    let idx = Env.admins.indexOf(key);
+    if (idx < 0) { return false; } // should never happen
+
+    if (Env.admins.length === 1) { throw new Error("CANT_REMOVE_LAST_ADMIN"); }
+    Env.admins.splice(idx, 1);
+    Env.adminsData = Env.adminsData.filter(str => {
+        const ed = Keys.canonicalize(str);
+        if (!ed) { return true; }
+        return ed !== key;
+    });
+    return true;
+};
+
+commands.PROOFS_MIGRATED = function (Env, args) {
+    if (args !== 1) {
+        throw new Error("INVALID_ARGS");
+    }
+    Env.proofsMigrated = true;
     return true;
 };
 
@@ -406,107 +428,6 @@ commands.SET_BEARER_SECRET = function (Env, args) {
     return true;
 };
 
-// [<command>, <args>, <author>, <time>]
-var handleCommand = Decrees.handleCommand = function (Env, line) {
-    var command = line[0];
-    var args = line[1];
 
-    if (typeof(commands[command]) !== 'function') {
-        throw new Error("DECREE_UNSUPPORTED_COMMAND");
-    }
+module.exports = DecreesCore.create(DECREE_NAME, commands);
 
-    var outcome = commands[command](Env, args);
-    if (outcome) {
-        // trigger Env change event...
-        Env.envUpdated.fire();
-    }
-    return outcome;
-};
-
-Decrees.createLineHandler = function (Env) {
-    var Log = Env.Log;
-
-    var index = -1;
-
-    return function (err, line) {
-        index++;
-        if (err) {
-            // Log the error and bail out
-            return void Log.error("DECREE_LINE_ERR", {
-                error: err.message,
-                index: index,
-                line: line,
-            });
-        }
-
-        if (Array.isArray(line)) {
-            try {
-                return void handleCommand(Env, line);
-            } catch (err2) {
-                return void Log.error("DECREE_COMMAND_ERR", {
-                    error: err2.message,
-                    index: index,
-                    line: line,
-                });
-            }
-        }
-
-        Log.error("DECREE_HANDLER_WEIRD_LINE", {
-            line: line,
-            index: index,
-        });
-    };
-};
-
-var Fs = require("fs");
-var Path = require("path");
-var readFileBin = require("./stream-file").readFileBin;
-var Schedule = require("./schedule");
-var Fse = require("fs-extra");
-var nThen = require("nthen");
-
-Decrees.load = function (Env, _cb) {
-    Env.scheduleDecree = Env.scheduleDecree || Schedule();
-
-    var cb = Util.once(Util.mkAsync(function (err) {
-        if (err && err.code !== 'ENOENT') {
-            return void _cb(err);
-        }
-        _cb();
-    }));
-
-    Env.scheduleDecree.blocking('', function (unblock) {
-        var done = Util.once(Util.both(cb, unblock));
-        nThen(function (w) {
-            // ensure that the path to the decree log exists
-            Fse.mkdirp(Env.paths.decree, w(function (err) {
-                if (!err) { return; }
-                w.abort();
-                done(err);
-            }));
-        }).nThen(function () {
-            var decreeName = Path.join(Env.paths.decree, 'decree.ndjson');
-            var stream = Fs.createReadStream(decreeName, {start: 0});
-            var handler = Decrees.createLineHandler(Env);
-            readFileBin(stream, function (msgObj, next) {
-                var text = msgObj.buff.toString('utf8');
-                try {
-                    handler(void 0, JSON.parse(text));
-                } catch (err) {
-                    handler(err, text);
-                }
-                next();
-            }, function (err) {
-                done(err);
-            });
-        });
-    });
-};
-
-Decrees.write = function (Env, decree, _cb) {
-    var path = Path.join(Env.paths.decree, 'decree.ndjson');
-    Env.scheduleDecree.ordered('', function (next) {
-        var cb = Util.both(Util.mkAsync(_cb), next);
-        Fs.appendFile(path, JSON.stringify(decree) + '\n', cb);
-    });
-};

lib/env.js

@@ -239,7 +239,7 @@ module.exports.create = function (config) {
         evictionReport: {},
         commandTimers: {},
 
-        sso: config.sso,
+        sso: plugins?.SSO?.config || {},
         enforceMFA: config.enforceMFA,
 
         onlyOffice: {
@@ -372,6 +372,7 @@ module.exports.create = function (config) {
         Core.DEFAULT_LIMIT;
 
     try {
+        Env.adminsData = (config.adminKeys || []).slice();
         Env.admins = (config.adminKeys || []).map(function (k) {
             try {
                 return Keys.canonicalize(k);
@@ -415,6 +416,7 @@ const BAD = [
     'limits',
     'customLimits',
     'scheduleDecree',
+    'plugins',
 
     'httpServer',
 

lib/eviction.js

@@ -51,7 +51,6 @@ var evictArchived = function (Env, cb) {
     var report = {
         // archivedChannelsRemoved,
         // archivedAccountsRemoved,
-        // archivedBlobProofsRemoved,
         // archivedBlobsRemoved,
 
         // totalChannels,
@@ -237,37 +236,6 @@ var evictArchived = function (Env, cb) {
         store.listArchivedChannels(handler, w(done));
     };
 
-    var removeArchivedBlobProofs = function (w) {
-        if (typeof(Env.archiveRetentionTime) !== "number") { return; }
-    // Iterate over archive blob ownership proofs and remove them
-    // if they are older than the specified retention time
-        var removed = 0;
-        blobs.list.archived.proofs(function (err, item, next) {
-            next = Util.mkAsync(next, THROTTLE_FACTOR);
-            if (err) {
-                Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
-                return void next();
-            }
-            if (item && item.ctime > retentionTime) { return void next(); }
-            if (Env.DRY_RUN) {
-                removed++;
-                return void Log.info("EVICT_ARCHIVED_BLOB_PROOF_DRY_RUN", item, next);
-            }
-            blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) {
-                if (err) {
-                    Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item);
-                    return void next();
-                }
-                Log.info("EVICT_ARCHIVED_BLOB_PROOF", item);
-                removed++;
-                next();
-            }));
-        }, w(function () {
-            report.archivedBlobProofsRemoved = removed;
-            Log.info('EVICT_ARCHIVED_BLOB_PROOFS_REMOVED', removed);
-        }));
-    };
-
     var removeArchivedBlobs = function (w) {
         if (typeof(Env.archiveRetentionTime) !== "number") { return; }
     // Iterate over archived blobs and remove them
@@ -303,7 +271,6 @@ var evictArchived = function (Env, cb) {
     nThen(loadStorage)
     .nThen(migrateIncorrectBlobs)
     .nThen(removeArchivedChannels)
-    .nThen(removeArchivedBlobProofs)
     .nThen(removeArchivedBlobs)
     .nThen(function () {
         cb(void 0, report);
@@ -315,7 +282,6 @@ module.exports = function (Env, cb) {
     var report = {
         // archivedChannelsRemoved,
         // archivedAccountsRemoved,
-        // archivedBlobProofsRemoved,
         // archivedBlobsRemoved,
 
         // totalChannels,
@@ -612,92 +578,45 @@ module.exports = function (Env, cb) {
             if (pinnedDocs.test(item.blobId)) { return void next(); }
             if (activeDocs.test(item.blobId)) { return void next(); }
 
-            // This seems redundant because we're already checking the bloom filter
-            // but we can't implement a 'fast mode' for the iterator
-            // unless we address this race condition with this last-minute double-check
-            if (item.mtime > inactiveTime) { return void next(); }
-
-            if (Env.DRY_RUN) {
-                removed++;
-                return void Log.info("EVICT_ARCHIVE_BLOB_DRY_RUN", {
-                    item: item,
-                }, next);
-            }
-            blobs.archive.blob(item.blobId, 'INACTIVE', function (err) {
-                if (err) {
-                    return Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
-                        error: err,
+            // NOTE: fast mode allows us to skip getStats for
+            // the pinned and active channels
+            nThen(function (w) {
+                // double check that the channel really is inactive before archiving it
+                // because it might have been created after the initial activity scan
+                blobs.getStats(item.blobId, w(function (err, newerItem) {
+                    if (err) { return; }
+                    if (newerItem && getNewestTime(newerItem) > retentionTime) {
+                        // it's actually active, so don't archive it.
+                        w.abort();
+                        cb();
+                    }
+                    // else fall through to the archival
+                }));
+            }).nThen(function () {
+                if (Env.DRY_RUN) {
+                    removed++;
+                    return void Log.info("EVICT_ARCHIVE_BLOB_DRY_RUN", {
                         item: item,
                     }, next);
                 }
-                removed++;
-                Log.info("EVICT_ARCHIVE_BLOB", {
-                    item: item,
-                }, next);
+                blobs.archive.blob(item.blobId, 'INACTIVE', function (err) {
+                    if (err) {
+                        return Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
+                            error: err,
+                            item: item,
+                        }, next);
+                    }
+                    removed++;
+                    Log.info("EVICT_ARCHIVE_BLOB", {
+                        item: item,
+                    }, next);
+                });
             });
         }, w(function () {
             report.totalBlobs = total;
             report.activeBlobs = total - removed;
             Log.info('EVICT_BLOBS_REMOVED', removed, w());
-        }));
-    };
-
-    var archiveInactiveBlobProofs = function (w) {
-    // iterate over blob proofs and remove them
-    // if they don't correspond to a pinned or active file
-        var removed = 0;
-        var total = 0;
-
-        Log.info("EVICT_ARCHIVE_INACTIVE_BLOB_PROOFS_START", {});
-        blobs.list.proofs(function (err, item, next) {
-            next = Util.mkAsync(next, THROTTLE_FACTOR);
-            if (err) {
-                return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next);
-            }
-            if (!item) {
-                return void Log.error('EVICT_BLOB_LIST_PROOFS_NO_ITEM', item, next);
-            }
-            total++;
-
-            if (total % PROGRESS_FACTOR === 0) {
-                Log.info('EVICT_BLOB_PROOF_PROGRESS', {
-                    proofs: total,
-                });
-            }
-
-            if (pinnedDocs.test(item.blobId)) { return void next(); }
-            if (item.mtime > inactiveTime) { return void next(); }
-            nThen(function (w) {
-                blobs.size(item.blobId, w(function (err, size) {
-                    if (err && err === 'ENOENT') { return; } // XXX delete the proof
-                    if (err) {
-                        w.abort();
-                        return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err, next);
-                    }
-                    if (size !== 0) {
-                        w.abort();
-                        next();
-                    }
-                }));
-            }).nThen(function () {
-                if (Env.DRY_RUN) {
-                    removed++;
-                    return void Log.info("EVICT_BLOB_PROOF_LONELY_DRY_RUN", item, next);
-                }
-                blobs.remove.proof(item.safeKey, item.blobId, function (err) {
-                    if (err) {
-                        return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item, next);
-                    }
-                    removed++;
-                    return Log.info("EVICT_BLOB_PROOF_LONELY", item, next);
-                });
-            });
-        }, w(function () {
-            Log.info("EVICT_BLOB_PROOFS_REMOVED", {
-                removed,
-                total,
-            }, w());
-        }));
+        }), true);
     };
 
     var archiveInactiveChannels = function (w) {
@@ -802,7 +721,6 @@ module.exports = function (Env, cb) {
         // (documents which are not in either bloom filter)
 
     .nThen(archiveInactiveBlobs)
-    .nThen(archiveInactiveBlobProofs)
     .nThen(archiveInactiveChannels)
     .nThen(function () {
         var runningTime = report.runningTime = msSinceStart();

lib/hk-util.js

@@ -42,6 +42,8 @@ const ADMIN_CHANNEL_LENGTH = HK.ADMIN_CHANNEL_LENGTH = 33;
 // with a 34 character id
 const EPHEMERAL_CHANNEL_LENGTH = HK.EPHEMERAL_CHANNEL_LENGTH = 34;
 
+HK.BLOB_ID_LENGTH = 48;
+
 // Temporary channels are archived X ms after everyone has left them
 const TEMPORARY_CHANNEL_LIFETIME = 30 * 1000;
 
@@ -763,15 +765,16 @@ const handleGetHistoryRange = function (Env, Server, seq, userId, parsed) {
     var channelName = parsed[1];
     var map = parsed[2];
     const HISTORY_KEEPER_ID = Env.id;
+    const store = Env.store;
 
     if (!(map && typeof(map) === 'object')) {
         return void Server.send(userId, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]);
     }
 
-    var oldestKnownHash = map.from;
-    var untilHash = map.to;
-    var desiredMessages = map.count;
-    var desiredCheckpoint = map.cpCount;
+    var oldestKnownHash = map.from; // last known hash
+    var untilHash = map.to; // oldest hash (unknown), start point if defined
+    var desiredMessages = map.count; // nb messages before lkh
+    var desiredCheckpoint = map.cpCount; // nb cp before lkh
     var txid = map.txid;
     if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number' && !untilHash) {
         return void Server.send(userId, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]);
@@ -782,6 +785,43 @@ const handleGetHistoryRange = function (Env, Server, seq, userId, parsed) {
     }
 
     Server.send(userId, [seq, 'ACK']);
+
+    if (untilHash) {
+        // Get all messages between untilHash (oldest but unknown)
+        // and oldestKnownHesh (last known hash) (or until the end if undefined)
+        // Messages can be streamed since we instantly know the start point
+        let found = false;
+        store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {
+            const parsed = tryParse(Env, msgObj.buff.toString('utf8'));
+            if (!parsed) { return void readMore(); }
+            if (isMetadataMessage(parsed)) { return void readMore(); }
+            const content = parsed[4];
+            if (typeof(content) !== 'string') { return void readMore(); }
+
+            const hash = getHash(content);
+            if (hash === untilHash || untilHash === 'NONE') { found = true; }
+            let then = hash === oldestKnownHash ? abort : readMore;
+            if (found) {
+                Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId,
+                    JSON.stringify(['HISTORY_RANGE', txid, parsed])], then);
+            }
+            return void readMore();
+        }, function (err, reason) {
+            if (err) {
+                Env.Log.error("HK_GET_OLDER_HISTORY", channelName, err, reason);
+                Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId,
+                    JSON.stringify(['HISTORY_RANGE_ERROR', txid, err])
+                ]);
+                return;
+            }
+            Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId,
+                JSON.stringify(['HISTORY_RANGE_END', txid, channelName])
+            ]);
+        });
+        return;
+    }
+    // If desiredCp or desiredMsg are defined, we can't stream and must
+    // get a list of messages to send from a worker
     Env.getOlderHistory(channelName, oldestKnownHash, untilHash, desiredMessages, desiredCheckpoint, function (err, toSend) {
         if (err && err.code !== 'ENOENT') {
             Env.Log.error("HK_GET_OLDER_HISTORY", err);

lib/http-worker.js

@@ -584,14 +584,15 @@ var makeRouteCache = function (template, cacheName) {
     };
 };
 
-const ssoList = Env.sso && Env.sso.enabled && Array.isArray(Env.sso.list) &&
-                Env.sso.list.map(function (obj) { return obj.name; }) || [];
-const ssoCfg = (SSOUtils && ssoList.length) ? {
-    force: (Env.sso && Env.sso.enforced && 1) || 0,
-    password: (Env.sso && Env.sso.cpPassword && (Env.sso.forceCpPassword ? 2 : 1)) || 0,
-    list: ssoList
-} : false;
 var serveConfig = makeRouteCache(function () {
+    const ssoList = Env.sso && Env.sso.enabled && Array.isArray(Env.sso.list) &&
+                    Env.sso.list.map(function (obj) { return obj.name; }) || [];
+    const ssoCfg = (SSOUtils && ssoList.length) ? {
+        force: (Env.sso && Env.sso.enforced && 1) || 0,
+        password: (Env.sso && Env.sso.cpPassword && (Env.sso.forceCpPassword ? 2 : 1)) || 0,
+        list: ssoList
+    } : false;
+
     return [
         'define(function(){',
         'return ' + JSON.stringify({

lib/load-config.js

@@ -46,11 +46,4 @@ if (!isPositiveNumber(config.premiumUploadSize) || config.premiumUploadSize < co
     delete config.premiumUploadSize;
 }
 
-config.sso = {};
-try {
-    config.sso = require("../config/sso");
-} catch (e) {
-    //console.log("SSO config not found");
-}
-
 module.exports = config;

lib/pins.js

@@ -33,6 +33,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
     // it's a weird API but it's faster than unpinning manually
     var pins = ref.pins = {};
     ref.index = 0;
+    ref.first = 0;
     ref.latest = 0; // the latest message (timestamp in ms)
     ref.surplus = 0; // how many lines exist behind a reset
 
@@ -58,7 +59,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
         return sanitized;
     };
 
-    return function (line) {
+    return function (line, i) {
         ref.index++;
         if (!Boolean(line)) { return; }
 
@@ -74,6 +75,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
         }
 
         if (typeof(l[2]) === 'number') {
+            if (!ref.first) { ref.first = l[2]; }
             ref.latest = l[2]; // date
         }
 
@@ -109,6 +111,11 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
             default:
                 errorHandler("PIN_LINE_UNSUPPORTED_COMMAND", l);
         }
+
+        if (i === 0) { // First line when using Pins.load
+            if (l[0] === 'PIN' || ref.block) { ref.user = true; } // teams always start with RESET
+        }
+
     };
 };
 

lib/storage/blob.js

@@ -10,6 +10,9 @@ var BlobStore = module.exports;
 var nThen = require("nthen");
 var Semaphore = require("saferphore");
 var Util = require("../common-util");
+const PERMISSIVE = 511;
+
+const readFileBin = require("../stream-file").readFileBin;
 
 const BLOB_LENGTH = 48;
 
@@ -31,37 +34,30 @@ var prependArchive = function (Env, path) {
     return Path.join(Env.archivePath, 'blob', relativePathToBlob);
 };
 
-// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
+// /blob/<blobPrefix>/<blobId>
 var makeBlobPath = function (Env, blobId) {
     return Path.join(Env.blobPath, blobId.slice(0, 2), blobId);
 };
 
+
 var makeActivityPath = function (Env, blobId) {
     return makeBlobPath(Env, blobId) + '.activity';
 };
 
+// /blob/<blobPrefix>/<blobId>.metadata.ndjson
+var mkMetadataPath = function (Env, blobId) {
+    return Path.join(Env.blobPath, blobId.slice(0, 2), blobId) + '.metadata.ndjson';
+};
+
 // /blobstate/<safeKeyPrefix>/<safeKey>
 var makeStagePath = function (Env, safeKey) {
     return Path.join(Env.blobStagingPath, safeKey.slice(0, 2), safeKey);
 };
 
-// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
-var makeProofPath = function (Env, safeKey, blobId) {
-    return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId);
-};
-
 var mkPlaceholderPath = function (Env, blobId) {
     return makeBlobPath(Env, blobId) + '.placeholder';
 };
 
-var parseProofPath = function (path) {
-    var parts = path.split('/');
-    return {
-        blobId: parts[parts.length -1],
-        safeKey: parts[parts.length - 3],
-    };
-};
-
 // Placeholder for deleted files
 var addPlaceholder = function (Env, blobId, reason, cb) {
     if (!reason) { return cb(); }
@@ -120,6 +116,18 @@ var isFile = function (filePath, cb) {
     });
 };
 
+// PROOFS
+// DEPRECATED, keep for compatibility
+// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
+var makeProofPath = function (Env, safeKey, blobId) {
+    return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId);
+};
+// isOwnedBy(id, safeKey)
+var isOwnedBy = function (Env, safeKey, blobId, cb) {
+    var proofPath = makeProofPath(Env, safeKey, blobId);
+    isFile(proofPath, cb);
+};
+
 var makeFileStream = function (full, _cb) {
     var cb = Util.once(Util.mkAsync(_cb));
     Fse.mkdirp(Path.dirname(full), function (e) {
@@ -190,6 +198,76 @@ var getActivity = function (Env, blobId, cb) {
     });
 };
 
+// destroyStream && createIdleStreamCollector
+// copied from lib/storage/file.js
+// see comments there
+const STREAM_CLOSE_TIMEOUT = 120000;
+const STREAM_DESTROY_TIMEOUT = 30000;
+const destroyStream = function (stream) {
+    if (!stream) { return; }
+    try {
+        stream.close();
+        if (stream.closed && stream.fd === null) { return; }
+    } catch (err) {
+        console.error(err);
+    }
+    setTimeout(function () {
+        try { stream.destroy(); } catch (err) { console.error(err); }
+    }, STREAM_DESTROY_TIMEOUT);
+};
+const createIdleStreamCollector = function (stream) {
+    var collector = Util.once(Util.mkAsync(Util.bake(destroyStream, [stream])));
+    collector.keepAlive = Util.throttle(collector, STREAM_CLOSE_TIMEOUT);
+    collector.keepAlive();
+    return collector;
+};
+
+//  writeMetadata appends to the dedicated log of metadata amendments
+var writeMetadata = function (env, channelId, data, cb) {
+    var path = mkMetadataPath(env, channelId);
+
+    Fse.mkdirp(Path.dirname(path), PERMISSIVE, function (err) {
+        if (err && err.code !== 'EEXIST') { return void cb(err); }
+        Fs.appendFile(path, data + '\n', cb);
+    });
+};
+var archiveMetadata = (Env, blobId, cb) => {
+    var path = mkMetadataPath(Env, blobId);
+    var archivePath = prependArchive(Env, path);
+    // XXX eviction clean lone md files
+    // if we fail to delete the metadata file, it can still be removed later by the eviction script
+    Fse.move(path, archivePath, { overwrite: true }, cb);
+};
+var restoreMetadata = function (Env, blobId, cb) {
+    var path = mkMetadataPath(Env, blobId);
+    var archivePath = prependArchive(Env, path);
+    Fse.move(archivePath, path, cb);
+};
+var readBlobMetadata = function (env, blobId, handler, _cb) {
+    var metadataPath = mkMetadataPath(env, blobId);
+    var stream = Fs.createReadStream(metadataPath, {start: 0});
+
+    const collector = createIdleStreamCollector(stream);
+    var cb = Util.both(_cb, collector);
+
+    readFileBin(stream, function (msgObj, readMore) {
+        collector.keepAlive();
+        var line = msgObj.buff.toString('utf8');
+        try {
+            var parsed = JSON.parse(line);
+            handler(null, parsed);
+        } catch (err) {
+            handler(err, line);
+        }
+        readMore();
+    }, function (err) {
+        // ENOENT => there is no metadata log
+        if (!err || err.code === 'ENOENT') { return void cb(); }
+        // otherwise stream errors?
+        cb(err);
+    });
+};
+
 /**********  METHODS **************/
 
 var upload = function (Env, safeKey, content, cb) {
@@ -313,6 +391,9 @@ var tryId = function (path, cb) {
 };
 
 // owned_upload_complete
+let unescapeKeyCharacters = function (key) {
+    return key.replace(/\-/g, '/');
+};
 var owned_upload_complete = function (Env, safeKey, id, cb) {
     closeBlobstage(Env, safeKey);
     if (!isValidId(id)) {
@@ -325,12 +406,9 @@ var owned_upload_complete = function (Env, safeKey, id, cb) {
     }
 
     var finalPath = makeBlobPath(Env, id);
+    let unsafeKey = unescapeKeyCharacters(safeKey);
 
-    var finalOwnPath = makeProofPath(Env, safeKey, id);
-
-    // the user wants to move it into blob and create a empty file with the same id
-    // in their own space:
-    // /blob/safeKeyPrefix/safeKey/blobPrefix/blobID
+    // the user wants to move it into blob and create a metadata log with an owner
 
     nThen(function (w) {
         // make the requisite directory structure using Mkdirp
@@ -340,12 +418,6 @@ var owned_upload_complete = function (Env, safeKey, id, cb) {
                 return void cb(e.code);
             }
         }));
-        Fse.mkdirp(Path.dirname(finalOwnPath), w(function (e /*, path */) {
-            if (e) { // does not throw error if the directory already existed
-                w.abort();
-                return void cb(e.code);
-            }
-        }));
     }).nThen(function (w) {
         // make sure the id does not collide with another
         tryId(finalPath, w(function (e) {
@@ -355,8 +427,11 @@ var owned_upload_complete = function (Env, safeKey, id, cb) {
             }
         }));
     }).nThen(function (w) {
-        // Create the empty file proving ownership
-        Fs.writeFile(finalOwnPath, '', w(function (e) {
+        // Write the metadata
+        let md = JSON.stringify({
+            owners: [unsafeKey]
+        });
+        writeMetadata(Env, id, md, w((e) => {
             if (e) {
                 w.abort();
                 return void cb(e.code);
@@ -367,12 +442,6 @@ var owned_upload_complete = function (Env, safeKey, id, cb) {
         // move the existing file to its new path
         Fse.move(oldPath, finalPath, w(function (e) {
             if (e) {
-                // if there's an error putting the file into its final location...
-                // ... you should remove the ownership file
-                Fs.unlink(finalOwnPath, function () {
-                    // but if you can't, it's not catestrophic
-                    // we can clean it up later
-                });
                 w.abort();
                 return void cb(e.code);
             }
@@ -393,24 +462,12 @@ var remove = function (Env, blobId, cb) {
     clearActivity(Env, blobId, () => {});
 };
 
-// removeProof
-var removeProof = function (Env, safeKey, blobId, cb) {
-    var proofPath = makeProofPath(Env, safeKey, blobId);
-    Fs.unlink(proofPath, cb);
-};
-
-// isOwnedBy(id, safeKey)
-var isOwnedBy = function (Env, safeKey, blobId, cb) {
-    var proofPath = makeProofPath(Env, safeKey, blobId);
-    isFile(proofPath, cb);
-};
-
-
 // archiveBlob
 var archiveBlob = function (Env, blobId, reason, cb) {
     var blobPath = makeBlobPath(Env, blobId);
     var archivePath = prependArchive(Env, blobPath);
     Fse.move(blobPath, archivePath, { overwrite: true }, cb);
+    archiveMetadata(Env, blobId, () => {});
     archiveActivity(Env, blobId, () => {});
     addPlaceholder(Env, blobId, reason, () => {});
 };
@@ -426,29 +483,11 @@ var restoreBlob = function (Env, blobId, cb) {
     var blobPath = makeBlobPath(Env, blobId);
     var archivePath = prependArchive(Env, blobPath);
     Fse.move(archivePath, blobPath, cb);
+    restoreMetadata(Env, blobId, () => {});
     restoreActivity(Env, blobId, () => {});
     clearPlaceholder(Env, blobId, () => {});
 };
 
-// archiveProof
-var archiveProof = function (Env, safeKey, blobId, cb) {
-    var proofPath = makeProofPath(Env, safeKey, blobId);
-    var archivePath = prependArchive(Env, proofPath);
-    Fse.move(proofPath, archivePath, { overwrite: true }, cb);
-};
-
-var removeArchivedProof = function (Env, safeKey, blobId, cb) {
-    var archivedPath = prependArchive(Env, makeProofPath(Env, safeKey, blobId));
-    Fs.unlink(archivedPath, cb);
-};
-
-// restoreProof
-var restoreProof = function (Env, safeKey, blobId, cb) {
-    var proofPath = makeProofPath(Env, safeKey, blobId);
-    var archivePath = prependArchive(Env, proofPath);
-    Fse.move(archivePath, proofPath, cb);
-};
-
 var makeWalker = function (n, handleChild, done) {
     if (!n || typeof(n) !== 'number' || n < 2) { n = 2; }
 
@@ -480,7 +519,7 @@ var makeWalker = function (n, handleChild, done) {
                     }
                     if (!stats.isDirectory()) {
                         w.abort();
-                        if (/\.activity$/.test(path)) {
+                       if (/\.activity$/.test(path)) {
                             // NOTE: some activity files were created for deleted blobs due to
                             // a bug. We're going to detect them here in order to be able to clean
                             // them.
@@ -513,46 +552,6 @@ var makeWalker = function (n, handleChild, done) {
     return recurse;
 };
 
-var listProofs = function (root, handler, cb) {
-    Fs.readdir(root, function (err, dir) {
-        if (err) { return void cb(err); }
-
-        var walk = makeWalker(20, function (err, path, next, loneActivity) {
-            if (loneActivity) { return void next(); }
-            // path is the path to a child node on the filesystem
-
-            // next handles the next job in a queue
-
-                // iterate over proofs
-                // check for presence of corresponding files
-            Fs.stat(path, function (err, stats) {
-                if (err) {
-                    return void handler(err, void 0, next);
-                }
-
-                var parsed = parseProofPath(path);
-                handler(void 0, {
-                    path: path,
-                    blobId: parsed.blobId,
-                    safeKey: parsed.safeKey,
-                    atime: stats.atime,
-                    ctime: stats.ctime,
-                    mtime: stats.mtime,
-                }, next);
-            });
-        }, function () {
-            // called when there are no more directories or children to process
-            cb();
-        });
-
-        dir.forEach(function (d) {
-            // ignore directories that aren't 3 characters long...
-            if (d.length !== 3) { return; }
-            walk(Path.join(root, d));
-        });
-    });
-};
-
 var getActivityStat = function (path, base, cb) {
     var suffix = base ? '' : '.activity';
     Fs.stat(path+suffix, function (err, stats) {
@@ -560,32 +559,92 @@ var getActivityStat = function (path, base, cb) {
         cb(err, stats);
     });
 };
-var listBlobs = function (root, handler, cb) {
-    // iterate over files
-    Fs.readdir(root, function (err, dir) {
-        if (err) { return void cb(err); }
-        var walk = makeWalker(20, function (err, path, next, loneActivity) {
-            if (loneActivity) { return void next(); }
-            getActivityStat(path, false, function (err, stats) {
-                if (err) {
-                    return void handler(err, void 0, next);
-                }
+var getStats = function (Env, blobId, cb) {
+    var path = makeBlobPath(Env, blobId);
+    getActivityStat(path, false, cb);
+};
 
-                handler(void 0, {
-                    blobId: Path.basename(path),
-                    atime: stats.atime,
-                    ctime: stats.ctime,
-                    mtime: stats.mtime,
-                }, next);
-            });
-        }, function () {
-            cb();
-        });
+let blobRegex = /^[0-9a-fA-F]{48}(\.metadata)*(\.ndjson)*$/;
+var listBlobs = function (root, handler, fast, cb) {
+    var dirList = [];
 
-        dir.forEach(function (d) {
-            if (d.length !== 2) { return; }
-            walk(Path.join(root, d));
+    nThen(function (w) {
+        // the root of your datastore contains nested directories...
+        Fs.readdir(root, w(function (err, list) {
+            if (err) {
+                w.abort();
+                // TODO check if we normally return strings or errors
+                return void cb(err);
+            }
+            dirList = list;
+        }));
+    }).nThen(function (waitFor) {
+        // search inside the nested directories
+        // stream it so you don't put unnecessary data in memory
+        var n = nThen;
+        dirList.forEach(function (dir) {
+            if (dir.length !== 2) { return; }
+            // Handle one directory at a time to save some memory
+            n = n(function (w) {
+                // do twenty things at a time
+                var sema = Semaphore.create(20);
+                var nestedDirPath = Path.join(root, dir);
+                Fs.readdir(nestedDirPath, w(function (err, list) {
+                    if (err) { return void handler(err); } // Is this correct?
+                    list.forEach(function (item) {
+                        // ignore hidden files
+                        if (/^\./.test(item)) { return; }
+                        // ignore anything that isn't channel or metadata
+                        if (!blobRegex.test(item)) { return; }
+
+                        var isLonelyMetadata = false;
+                        var blobName;
+
+                        // if the current file is not the channel data, then it must be metadata
+                        if (!/^[0-9a-fA-F]{48}$/.test(item)) {
+                            blobName = item.replace(/\.metadata\.ndjson/, '');
+                            // check if blob already exists
+                            if (list.indexOf(blobName) !== -1) { return; }
+                            // otherwise set a flag indicating that we should
+                            // handle the metadata on its own
+                            isLonelyMetadata = true;
+                        } else {
+                            blobName = item;
+                        }
+                        if (blobName.length !== 48) { return; }
+
+                        sema.take(function (give) {
+                            var next = w(give());
+
+                            if (fast) {
+                                return void handler(void 0, {
+                                    blobId: blobName
+                                }, next);
+                            }
+
+                            var filePath = Path.join(nestedDirPath, blobName);
+                            if (isLonelyMetadata) {
+                                // Set time to 0 to delete this
+                                // lonely metadata file
+                                return void handler(void 0, {
+                                    blobId: blobName,
+                                    mtime: 0,
+                                    atime: 0,
+                                    ctime: 0
+                                }, next);
+                            }
+                            return void getActivityStat(filePath, false, (err, data) => {
+                                data.blobId = blobName;
+                                handler(err, data, next);
+                            });
+                        });
+                    });
+                }));
+            }).nThen;
         });
+        n(waitFor());
+    }).nThen(function () {
+        cb();
     });
 };
 
@@ -673,6 +732,20 @@ BlobStore.create = function (config, _cb) {
                 if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
                 isOwnedBy(Env, safeKey, blobId, cb);
             },
+            readMetadata: (blobId, handler, cb) => {
+                if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
+                readBlobMetadata(Env, blobId, handler, cb);
+            },
+            writeMetadata: (blobId, data, cb) => {
+                if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
+                writeMetadata(Env, blobId, data, cb);
+            },
+            hasMetadata: (blobId, _cb) => {
+                var cb = Util.once(Util.mkAsync(_cb));
+                if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
+                var path = mkMetadataPath(Env, blobId);
+                isFile(path, cb);
+            },
 
             remove: {
                 blob: function (blobId, _cb) {
@@ -680,24 +753,12 @@ BlobStore.create = function (config, _cb) {
                     if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
                     remove(Env, blobId, cb);
                 },
-                proof: function (safeKey, blobId, _cb) {
-                    var cb = Util.once(Util.mkAsync(_cb));
-                    if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
-                    if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
-                    removeProof(Env, safeKey, blobId, cb);
-                },
                 archived: {
                     blob: function (blobId, _cb) {
                         var cb = Util.once(Util.mkAsync(_cb));
                         if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
                         removeArchivedBlob(Env, blobId, cb);
                     },
-                    proof:  function (safeKey, blobId, _cb) {
-                        var cb = Util.once(Util.mkAsync(_cb));
-                        if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
-                        if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
-                        removeArchivedProof(Env, safeKey, blobId, cb);
-                    },
                 },
                 loneActivity: function (_cb) {
                     var cb = Util.once(Util.mkAsync(_cb));
@@ -711,12 +772,6 @@ BlobStore.create = function (config, _cb) {
                     if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
                     archiveBlob(Env, blobId, reason, cb);
                 },
-                proof: function (safeKey, blobId, _cb) {
-                    var cb = Util.once(Util.mkAsync(_cb));
-                    if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
-                    if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
-                    archiveProof(Env, safeKey, blobId, cb);
-                },
             },
 
             restore: {
@@ -725,12 +780,6 @@ BlobStore.create = function (config, _cb) {
                     if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
                     restoreBlob(Env, blobId, cb);
                 },
-                proof: function (safeKey, blobId, _cb) {
-                    var cb = Util.once(Util.mkAsync(_cb));
-                    if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
-                    if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
-                    restoreProof(Env, safeKey, blobId, cb);
-                },
             },
 
             isBlobAvailable: function (blobId, _cb) {
@@ -781,24 +830,21 @@ BlobStore.create = function (config, _cb) {
                 if (!isValidId(id)) { return void cb("INVALID_ID"); }
                 getActivity(Env, id, cb);
             },
+            getStats: function (id, _cb) {
+                var cb = Util.once(Util.mkAsync(_cb));
+                if (!isValidId(id)) { return void cb("INVALID_ID"); }
+                getStats(Env, id, cb);
+            },
 
             list: {
-                blobs: function (handler, _cb) {
+                blobs: function (handler, _cb, fast) {
                     var cb = Util.once(Util.mkAsync(_cb));
-                    listBlobs(Env.blobPath, handler, cb);
-                },
-                proofs: function (handler, _cb) {
-                    var cb = Util.once(Util.mkAsync(_cb));
-                    listProofs(Env.blobPath, handler, cb);
+                    listBlobs(Env.blobPath, handler, fast, cb);
                 },
                 archived: {
-                    proofs: function (handler, _cb) {
+                    blobs: function (handler, _cb, fast) {
                         var cb = Util.once(Util.mkAsync(_cb));
-                        listProofs(prependArchive(Env, Env.blobPath), handler, cb);
-                    },
-                    blobs: function (handler, _cb) {
-                        var cb = Util.once(Util.mkAsync(_cb));
-                        listBlobs(prependArchive(Env, Env.blobPath), handler, cb);
+                        listBlobs(prependArchive(Env, Env.blobPath), handler, fast, cb);
                     },
                 }
             },

lib/storage/file.js

@@ -366,7 +366,7 @@ var readMessages = function (path, msgHandler, _cb) {
     return readFileBin(stream, function (msgObj, readMore) {
         collector.keepAlive();
         msgHandler(msgObj.buff.toString('utf8'));
-        readMore();
+        setTimeout(readMore);
     }, function (err) {
         cb(err);
     });

lib/workers/db-worker.js

@@ -40,7 +40,7 @@ Logger.levels.forEach(function (level) {
     };
 });
 
-const HISTORY_SIZE_LIMIT = 1024 * 1024 * 1024; // XXX 1GB
+//const HISTORY_SIZE_LIMIT = 1024 * 1024 * 1024; // XXX 1GB
 
 var DETAIL = 1000;
 var round = function (n) {
@@ -158,6 +158,12 @@ const isValidOffsetNumber = function (n) {
     return typeof(n) === 'number' && n >= 0;
 };
 
+const updateEnv = data => {
+    const {value} = data;
+    let env = Util.tryParse(value) || {};
+    Env.proofsMigrated = env?.proofsMigrated;
+};
+
 const computeIndexFromOffset = function (channelName, offset, cb) {
     let cpIndex = [];
     let messageBuf = [];
@@ -338,7 +344,13 @@ const computeMetadata = function (data, cb) {
     const ref = {};
     const lineHandler = Meta.createLineHandler(ref, Env.Log.error);
     monitoringIncrement('computeMetadata');
-    return void store.readChannelMetadata(data.channel, lineHandler, function (err) {
+
+    let f = store.readChannelMetadata;
+    if (data.channel.length === HK.BLOB_ID_LENGTH) {
+        f = blobStore.readMetadata;
+    }
+
+    return void f(data.channel, lineHandler, function (err) {
         if (err) {
             // stream errors?
             return void cb(err);
@@ -384,11 +396,50 @@ const getFileSize = function (data, cb) {
 
 const getOlderHistory = function (data, cb) {
     const oldestKnownHash = data.hash;
-    const untilHash = data.toHash;
     const channelName = data.channel;
     const desiredMessages = data.desiredMessages;
     const desiredCheckpoint = data.desiredCheckpoint;
 
+    let messages = [];
+    store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {
+        const parsed = HK.tryParse(Env, msgObj.buff.toString('utf8'));
+        if (!parsed) { return void readMore(); }
+        if (HK.isMetadataMessage(parsed)) { return void readMore(); }
+        const content = parsed[4];
+        if (typeof(content) !== 'string') { return void readMore(); }
+        const hash = HK.getHash(content);
+
+        messages.push(parsed);
+
+        // "X" messages before oldestKnownHash
+        if (typeof (desiredMessages) === "number") {
+            messages = messages.slice(-desiredMessages);
+            if (hash === oldestKnownHash) { return void abort(); }
+            return void readMore();
+        }
+
+        // "X" checkpoints before oldestKnownHash
+        if (hash === oldestKnownHash) { return void abort(); }
+        if (/^cp\|/.test(content)) { // clean whenever we push a cp
+            let foundCp = 0;
+            const idx = messages.findLastIndex(parsed => {
+                let isCp = /^cp\|/.test(parsed[4]);
+                if (!isCp) { return; }
+                foundCp++;
+                return foundCp >= desiredCheckpoint;
+            });
+            if (idx > 0) {
+                messages = messages.slice(idx);
+            }
+        }
+        readMore();
+    }, function (err, reason) {
+        if (err) { return void cb(err, reason); }
+        cb(void 0, messages);
+    });
+
+    /*
+    const untilHash = data.toHash;
     var next = () => {
     var messages = [];
     var found = false;
@@ -444,6 +495,7 @@ const getOlderHistory = function (data, cb) {
         }
         next();
     });
+    */
 };
 
 const getPinState = function (data, cb) {
@@ -558,16 +610,33 @@ const removeOwnedBlob = function (data, cb) {
     if (typeof(data.safeKey) !== 'string') { return void cb("INVALID_KEY"); }
     const blobId = data.blobId;
     const safeKey = Util.escapeKeyCharacters(data.safeKey);
+    const unsafeKey = Util.unescapeKeyCharacters(data.safeKey);
 
     const reason = data.reason || 'ARCHIVE_OWNED';
 
     nThen(function (w) {
         // check if you have permissions
-        blobStore.isOwnedBy(safeKey, blobId, w(function (err, owned) {
-            if (err || !owned) {
+        computeMetadata({channel: blobId}, w((err, meta) => {
+            if (err || !meta) {
                 w.abort();
                 return void cb("INSUFFICIENT_PERMISSIONS");
             }
+            let owners = meta.owners;
+            if (!owners && !Env.proofsMigrated) {
+                // Check old proofs during migration
+                blobStore.isOwnedBy(safeKey, blobId, w((e, owned) => {
+                    if (e || !owned) {
+                        w.abort();
+                        return void cb("INSUFFICIENT_PERMISSIONS");
+                    }
+                }));
+                return;
+            }
+            if (!owners || !owners.includes(unsafeKey)) {
+                w.abort();
+                return void cb("INSUFFICIENT_PERMISSIONS");
+            }
+            // Owned, continue
         }));
     }).nThen(function (w) {
         // remove the blob
@@ -581,20 +650,8 @@ const removeOwnedBlob = function (data, cb) {
                 w.abort();
                 return void cb(err);
             }
-        }));
-    }).nThen(function () {
-        // archive the proof
-        blobStore.archive.proof(safeKey, blobId, function (err) {
-            Env.Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", {
-                safeKey: safeKey,
-                blobId: blobId,
-                status: err? String(err): 'SUCCESS',
-            });
-            if (err) {
-                return void cb("E_PROOF_REMOVAL");
-            }
             cb(void 0, 'OK');
-        });
+        }));
     });
 };
 
@@ -692,6 +749,7 @@ const getLastChannelTime = function (data, cb) {
 };
 
 const COMMANDS = {
+    ENV_UPDATE: updateEnv,
     COMPUTE_INDEX: computeIndex,
     COMPUTE_METADATA: computeMetadata,
     GET_OLDER_HISTORY: getOlderHistory,
@@ -847,6 +905,9 @@ process.on('message', function (data) {
     };
 
     if (!ready) {
+        if (data.env) {
+            updateEnv({value:data.env});
+        }
         return void init(data.config, function (err) {
             if (err) { return void cb(Util.serializeError(err)); }
             ready = true;

lib/workers/index.js

@@ -9,6 +9,7 @@ const { fork } = require('child_process');
 const Workers = module.exports;
 const PID = process.pid;
 const Block = require("../storage/block");
+const Environment = require('../env');
 
 const DB_PATH = 'lib/workers/db-worker';
 const MAX_JOBS = 16;
@@ -52,16 +53,20 @@ Workers.initialize = function (Env, config, _cb) {
         //return Object.keys(workers[index].tasks || {}).length;
     };
 
+    const WORKER_TASK_LIMIT = 1000; // XXX
+    //const WORKER_TASK_LIMIT = 100; // XXX
+
     var workerOffset = -1;
     var queue = [];
-    var getAvailableWorkerIndex = function () {
+    var getAvailableWorkerIndex = function (isQueue) {
 //  If there is already a backlog of tasks you can avoid some work
-//  by going to the end of the line
-        if (queue.length) { return -1; }
+//  by going to the end of the line (unless we're trying to
+//  empty the queue)
+        if (queue.length && !isQueue) { return -1; }
 
         var L = workers.length;
         if (L === 0) {
-            Log.error('NO_WORKERS_AVAILABLE', {
+            Log.warn('NO_WORKERS_AVAILABLE', {
                 queue: queue.length,
             });
             return -1;
@@ -93,7 +98,7 @@ Workers.initialize = function (Env, config, _cb) {
     };
 
     var drained = true;
-    var sendCommand = function (msg, _cb, opt) {
+    var sendCommand = function (msg, _cb, opt, isQueue) {
         if (!_cb) {
             return void Log.error('WORKER_COMMAND_MISSING_CB', {
                 msg: msg,
@@ -102,7 +107,7 @@ Workers.initialize = function (Env, config, _cb) {
         }
 
         opt = opt || {};
-        var index = getAvailableWorkerIndex();
+        var index = getAvailableWorkerIndex(isQueue);
 
         var state = workers[index];
         // if there is no worker available:
@@ -114,7 +119,7 @@ Workers.initialize = function (Env, config, _cb) {
             });
             if (drained) {
                 drained = false;
-                Log.error('WORKER_QUEUE_BACKLOG', {
+                Log.warn('WORKER_QUEUE_BACKLOG', {
                     workers: workers.length,
                 });
             }
@@ -127,13 +132,14 @@ Workers.initialize = function (Env, config, _cb) {
         var cb = Util.once(Util.mkAsync(Util.both(_cb, function (err /*, value */) {
             incrementTime(msg && msg.command, start);
             if (err !== 'TIMEOUT') { return; }
-            Log.debug("WORKER_TIMEOUT_CAUSE", msg);
+            Log.warn("WORKER_TIMEOUT_CAUSE", msg);
             // in the event of a timeout the user will receive an error
             // but the state used to resend a query in the event of a worker crash
             // won't be cleared. This also leaks a slot that could be used to keep
             // an upper bound on the amount of parallelism for any given worker.
             // if you run out of slots then the worker locks up.
             delete state.tasks[txid];
+            state.checkTasks();
         })));
 
         if (!msg) {
@@ -164,6 +170,12 @@ Workers.initialize = function (Env, config, _cb) {
             msg._cb = _cb;
             msg._opt = opt;
         });
+
+        state.count++;
+        if (state.count > WORKER_TASK_LIMIT) {
+            // Remove from list and spawn new one
+            if (state.replaceWorker) { state.replaceWorker(); }
+        }
     };
 
     const pluginsResponses = {};
@@ -207,6 +219,8 @@ Workers.initialize = function (Env, config, _cb) {
         if (!res.txid) { return; }
         response.handle(res.txid, [res.error, res.value]);
         delete state.tasks[res.txid];
+        state.checkTasks();
+
         if (!queue.length) {
             if (!drained) {
                 drained = true;
@@ -234,7 +248,7 @@ Workers.initialize = function (Env, config, _cb) {
     to the back because the following msg took its place. OR, in an
     even worse scenario, we cycle through the queue but don't run anything.
 */
-        sendCommand(nextMsg.msg, nextMsg.cb);
+        sendCommand(nextMsg.msg, nextMsg.cb, {}, true);
     };
 
     const initWorker = function (worker, cb) {
@@ -243,19 +257,67 @@ Workers.initialize = function (Env, config, _cb) {
         const state = {
             worker: worker,
             tasks: {},
+            count: Math.floor(Math.random()*(WORKER_TASK_LIMIT/10)),
             pid: worker.pid, // store the child process's id in an easily accessible location
         };
 
+        state.replaceWorker = () => {
+            let index = workers.indexOf(state);
+            if (index === -1) { return; }
+            // Remove old
+            workers.splice(index, 1);
+            // Create new
+            state.complete = true;
+            const w = fork(DB_PATH);
+            Log.info('WORKER_REPLACE_START', {
+                from: state.worker.pid,
+                to: w.pid
+            });
+            initWorker(w, function (err) {
+                if (err) {
+                    throw new Error(err);
+                }
+            });
+        };
+
+        // If we've reached the limit, kill the worker once
+        // all the tasks are complete or timed out
+        state.checkTasks = () => {
+            // Check limit
+            if (!state.complete || !state.worker) { return; }
+            // Check remaining tasks
+            if (Object.keys(state.tasks).length) { return; }
+            // Kill
+            Log.info('WORKER_KILL', {
+                worker: state.worker.pid,
+                count: state.count
+            });
+            delete state.worker;
+            worker.kill();
+        };
+
         response.expect(txid, function (err) {
             if (err) { return void cb(err); }
             workers.push(state);
             cb(void 0, state);
+            // We just pushed a new worker, available to receive
+            // a task, so we can empty the queue if necessary
+            if (queue.length) {
+                const nextMsg = queue.shift();
+                if (!nextMsg || !nextMsg.msg) {
+                    return Log.error('WORKER_QUEUE_EMPTY_MESSAGE', {
+                        item: nextMsg,
+                    });
+                }
+                sendCommand(nextMsg.msg, nextMsg.cb, {}, true);
+            }
         }, 15000);
 
         worker.send({
             pid: PID,
             txid: txid,
             config: config,
+            env: Environment.serialize(Env)
         });
 
         worker.on('message', function (res) {
@@ -296,18 +358,21 @@ Workers.initialize = function (Env, config, _cb) {
         });
 
         worker.on('exit', function () {
+            if (!state.worker) { return; } // Manually killed
             substituteWorker();
             Env.Log.error("DB_WORKER_EXIT", {
                 pid: state.pid,
             });
         });
         worker.on('close', function () {
+            if (!state.worker) { return; } // Manually killed
             substituteWorker();
             Env.Log.error("DB_WORKER_CLOSE", {
                 pid: state.pid,
             });
         });
         worker.on('error', function (err) {
+            if (!state.worker) { return; } // Manually killed
             substituteWorker();
             Env.Log.error("DB_WORKER_ERROR", {
                 pid: state.pid,
@@ -342,7 +407,8 @@ Workers.initialize = function (Env, config, _cb) {
                     type: 'broadcast',
                     pid: PID,
                     command: data.command,
-                    txid: data.txid
+                    txid: data.txid,
+                    value: data.value
                 });
             });
             return workers;

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cryptpad",
-  "version": "2024.12.0",
+  "version": "2025.3.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
@@ -45,7 +45,7 @@
         "notp": "^2.0.3",
         "nthen": "0.1.8",
         "open-sans-fontface": "^1.4.0",
-        "openid-client": "^5.4.2",
+        "openid-client": "^5.7.0",
         "pako": "^2.1.0",
         "prompt-confirm": "^2.0.4",
         "pull-stream": "^3.6.1",
@@ -2981,9 +2981,9 @@
       }
     },
     "node_modules/jose": {
-      "version": "4.15.5",
-      "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.5.tgz",
-      "integrity": "sha512-jc7BFxgKPKi94uOvEmzlSWFFe2+vASyXaKUpdQKatWAESU2MWjDfFf0fdfc83CDKcA5QecabZeNLyfhe3yKNkg==",
+      "version": "4.15.9",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz",
+      "integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==",
       "funding": {
         "url": "https://github.com/sponsors/panva"
       }
@@ -3728,11 +3728,11 @@
       "integrity": "sha512-d1VXrt1qPScsZnDHbZTOf1SmUnanr3KQgQM6+ye6KoFgrLo8a8mkX/J/ZJ2+w7vf0sCC02lRia5SAiaz0JPEog=="
     },
     "node_modules/openid-client": {
-      "version": "5.6.1",
-      "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.6.1.tgz",
-      "integrity": "sha512-PtrWsY+dXg6y8mtMPyL/namZSYVz8pjXz3yJiBNZsEdCnu9miHLB4ELVC85WvneMKo2Rg62Ay7NkuCpM0bgiLQ==",
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.7.1.tgz",
+      "integrity": "sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==",
       "dependencies": {
-        "jose": "^4.15.1",
+        "jose": "^4.15.9",
         "lru-cache": "^6.0.0",
         "object-hash": "^2.2.0",
         "oidc-token-hash": "^5.0.3"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "cryptpad",
   "description": "a collaborative office suite that is end-to-end encrypted and open-source",
-  "version": "2024.12.0",
+  "version": "2025.3.0",
   "license": "AGPL-3.0+",
   "repository": {
     "type": "git",
@@ -48,7 +48,7 @@
     "notp": "^2.0.3",
     "nthen": "0.1.8",
     "open-sans-fontface": "^1.4.0",
-    "openid-client": "^5.4.2",
+    "openid-client": "^5.7.1",
     "pako": "^2.1.0",
     "prompt-confirm": "^2.0.4",
     "pull-stream": "^3.6.1",

scripts/migrations/migrate-blob-proofs.js

@@ -0,0 +1,198 @@
+// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team <contact@cryptpad.org> and contributors
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+const { parentPort } = require('node:worker_threads');
+const Path = require('node:path');
+const Fs = require('node:fs');
+const nThen = require("nthen");
+const Semaphore = require("saferphore");
+const Logger = require("../../lib/log");
+const BlobStorage = require("../../lib/storage/blob");
+let config = require("../../lib/load-config");
+
+
+const blobPath = config.blobPath || './blob';
+let Log = {};
+
+// NOTE: in cleaning mode, we DON'T migrate
+// (we suppose data has already been migrated)
+const start = (clean, dry, cb) => {
+    const DRY_RUN = dry;
+
+    let dirList = [];
+    let blobStore;
+    nThen(w => {
+        Logger.create(config, w(function (_log) {
+            Log = _log;
+        }));
+    }).nThen(w => {
+        config.getSession = function () {};
+        BlobStorage.create(config, w(function (err, _store) {
+            if (err) {
+                w.abort();
+                return void Log.error("ERR_BLOB_STORE", err);
+            }
+            blobStore = _store;
+        }));
+    }).nThen(w => {
+        Fs.readdir(blobPath, w((err, list) => {
+            if (err) {
+                w.abort();
+                return void Log.error("ERR_READING_ROOT", err);
+            }
+            dirList = list;
+        }));
+    }).nThen(() => {
+        let n = nThen;
+        dirList.forEach(dir => {
+            if (dir.length !== 3) { return; }
+            // ./blob/abc
+            const nestedDirPath = Path.join(blobPath, dir);
+
+            if (clean) {
+                n = n(ww => {
+                    Log.info("REMOVING_DIR", nestedDirPath);
+                    if (DRY_RUN) { return; }
+                    Fs.rm(nestedDirPath, {
+                        recursive: true, force: true
+                    }, ww(err => {
+                        if (err) {
+                            Log.error("ERR_REMOVE_DIR", {
+                                path: nestedDirPath,
+                                err
+                            });
+                        }
+                    }));
+                }).nThen;
+                return;
+            }
+
+            n = n(w => {
+                // One user at a time
+                const sema = Semaphore.create(1);
+                let nestedDirList = [];
+                nThen(ww => {
+                    Fs.readdir(nestedDirPath, ww((err, list) => {
+                        if (err) {
+                            w.abort();
+                            ww.abort();
+                            return Log.error("ERR_READING_DIR", {
+                                path: nestedDirPath,
+                                err
+                            });
+                        }
+                        nestedDirList = list;
+                    }));
+                }).nThen(ww => {
+                    nestedDirList.forEach(key => {
+                        // ./blob/abc/abcdefg...
+                        const keyPath = Path.join(nestedDirPath, key);
+                        sema.take(give => {
+                        let edPublic = key.replace(/\-/g, '/');
+                        let md = JSON.stringify({ owners: [edPublic] });
+                        Log.info("START_USER", edPublic);
+                        Fs.readdir(keyPath, ww((err, list) => {
+                            if (err) {
+                                w.abort();
+                                ww.abort();
+                                return Log.error("ERR_READING_DIR", {
+                                    path: keyPath,
+                                    err
+                                });
+                            }
+                            let blobs = [];
+                            nThen(www => {
+                                list.forEach(dir => {
+                                    // ./blob/abc/abcdefg.../01
+                                    const path = Path.join(keyPath, dir);
+                                    Fs.readdir(path, www((err, blobsList) => {
+                                        if (err) {
+                                            w.abort();
+                                            ww.abort();
+                                            www.abort();
+                                            return Log.error("ERR_READING_DIR", {
+                                                path, err
+                                            });
+                                        }
+                                        Array.prototype.push.apply(blobs, blobsList);
+                                    }));
+                                });
+                            }).nThen(www => {
+                                // migrate 20 blobs at a time for a given user
+                                const sema = Semaphore.create(20);
+                                blobs.forEach(blobId => {
+                                    sema.take(ggive => {
+                                        blobStore.isBlobAvailable(blobId, www((err, blobExists) => {
+                                        blobStore.hasMetadata(blobId, www((err, exists) => {
+                                            // If blob is not available or metadata already
+                                            // exists, don't write md file
+                                            if (!blobExists || exists) { return void ggive(); }
+                                            Log.info('WRITE_METADATA', blobId);
+                                            if (DRY_RUN) { return void ggive(); }
+                                            blobStore.writeMetadata(blobId, md, www(e => {
+                                                if (e) {
+                                                    w.abort();
+                                                    ww.abort();
+                                                    www.abort();
+                                                    return Log.error("ERR_WRITING_MD", { blobId });
+                                                }
+                                                ggive();
+                                            }));
+
+                                        }));
+                                        }));
+                                    });
+                                });
+                            }).nThen(ww(give(() => {
+                                Log.info("END_USER", edPublic);
+                            })));
+                        }));
+                        });
+
+                    });
+                }).nThen(w());
+            }).nThen;
+        });
+        n(() => {
+            Log.info("DONE");
+            cb();
+        });
+    });
+};
+
+if (parentPort) {
+    // Loaded as worker script
+    config = JSON.parse(JSON.stringify(config));
+    config.logToStdout = false;
+    parentPort.on('message', (message) => {
+        let parsed = message; //JSON.parse(message);
+        if (!parsed?.start) { return; }
+        // Migrate
+        start(false, false, () => {
+            parentPort.postMessage('MIGRATED');
+            // If success, clean
+            start(true, false, () => {
+                parentPort.postMessage('CLEANED');
+            });
+        });
+    });
+    parentPort.postMessage('READY');
+} else if (require.main === module) {
+    // Loaded from command-line
+    let dry = false;
+    let clean = false;
+    process.argv.forEach(key => {
+        if (key === '--dry') {
+            dry = true;
+            return;
+        }
+        if (key === '--clean') {
+            clean = true;
+            return;
+        }
+    });
+    start(clean, dry, () => {
+        process.exit(0);
+    });
+}

scripts/user-statistics.js

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: 2025 XWiki CryptPad Team <contact@cryptpad.org> and contributors
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+const nThen = require("nthen");
+const Semaphore = require("saferphore");
+const Logger = require("../lib/log");
+const Pins = require("../lib/pins");
+const config = require("../lib/load-config");
+const BlobStorage = require("../lib/storage/blob");
+const Store = require("../lib/storage/file");
+const Fs = require('node:fs');
+const Quota = require("../lib/commands/quota");
+const Environment = require('../lib/env');
+const Env = Environment.create(config);
+
+const CSV = true;
+
+config.logPath = false;
+config.logToStdout = true;
+
+const start = () => {
+    let time = +new Date();
+    let Log = {};
+    let all = {};
+    let blobStore, store;
+    nThen(w => {
+        Logger.create(config, w(function (_log) {
+            Env.Log = Log = _log;
+        }));
+    }).nThen(w => {
+        config.getSession = function () {};
+        Store.create(config, w(function (err, _store) {
+            if (err) {
+                w.abort();
+                return void Log.error("ERR_PAD_STORE", err);
+            }
+            store = _store;
+        }));
+        BlobStorage.create(config, w(function (err, _store) {
+            if (err) {
+                w.abort();
+                return void Log.error("ERR_BLOB_STORE", err);
+            }
+            blobStore = _store;
+        }));
+    }).nThen(w => {
+        Quota.updateCachedLimits(Env, w((err) => {
+            if (err) {
+                return Env.Log.warn('UPDATE_QUOTA_ERR', err);
+            }
+            Env.Log.info('QUOTA_UPDATED', {});
+        }));
+    }).nThen(w => {
+        Env.Log.info('START_LOADING_PINS');
+        const handlePinLog = (content, id, next) => {
+            const sema = Semaphore.create(20);
+            const data = all[id] = {
+                size: 0,
+                n_pads: 0,
+                n_blobs: 0,
+                n_total: 0,
+                first: content.first,
+                last: content.latest
+            };
+            if (!content.user) {
+                data.maybeTeam = true;
+            }
+            nThen(ww => {
+                Object.keys(content.pins).forEach(id => {
+                    sema.take(give => {
+                        let addSize = ww(give((e, s) => {
+                            if (typeof(s) !== "number") {
+                                return; // XXX
+                            }
+                            data.size += s;
+                            data.n_total++;
+                            if (id.length === 32) {
+                                data.n_pads++;
+                            } else {
+                                data.n_blobs++;
+                            }
+                        }));
+                        if (id.length === 32) { // PAD
+                            return store.getChannelSize(id, addSize);
+                        }
+                        blobStore.size(id, addSize);
+                    });
+                });
+            }).nThen(() => {
+                let key = id.replace(/-/g, '/');
+                if (Env.limits[key]) {
+                    let sub = Env.limits[key];
+                    data.premium = sub?.plan;
+                }
+                Env.Log.info('PIN_LOG_HANDLED', key);
+                next();
+            });
+        };
+
+        Pins.load(w(() => {
+            let duration = +new Date() - time;
+            Env.Log.info('ALL_PINS_LOADED', duration);
+        }), {
+            pinPath: config.pinPath,
+            handler: handlePinLog,
+        });
+    }).nThen(() => {
+        if (!CSV) { return console.log(all); }
+        let csv = `"User key","Premium plan","Bytes","Number pads","Number blobs","First activity","Last activity","May be a team"\n`;
+        Object.keys(all).sort((a,b) => {
+            return all[b].size - all[a].size;
+        }).forEach(k => {
+            const data = all[k];
+            k = k.replace(/-/g, '/');
+            let first = new Date(data.first).toISOString().slice(0,10);
+            let last = new Date(data.last).toISOString().slice(0,10);
+            let plan = data.premium || '';
+            let t = String(!!data.maybeTeam);
+            csv += `"${k}","${plan}","${data.size}","${data.n_pads}","${data.n_blobs}","${first}","${last}","${t}"\n`;
+        });
+        let filename = `../${new Date().toISOString().slice(0,10)}-stats.csv`;
+        Fs.writeFile(filename, csv, err => {
+            if (err) {
+                console.error(err);
+            } else {
+                console.log('CSV available at', filename);
+            }
+        });
+    });
+};
+start();

server.js

@@ -190,7 +190,15 @@ nThen(function (w) {
 
     var throttledEnvChange = Util.throttle(function () {
         Env.Log.info('WORKER_ENV_UPDATE', 'Updating HTTP workers with latest state');
-        broadcast('ENV_UPDATE', Environment.serialize(Env));
+        let serialized = Environment.serialize(Env);
+        broadcast('ENV_UPDATE', serialized);
+        if (Env.broadcastWorkerCommand) {
+            Env.broadcastWorkerCommand({
+                command: 'ENV_UPDATE',
+                value: serialized,
+                txid: Util.uid()
+            });
+        }
     }, 250); // NOTE: changing this value will impact lib/commands/admin-rpc.js#adminDecree callback
 
     var throttledCacheFlush = Util.throttle(function () {

www/admin/app-admin.less

@@ -61,6 +61,11 @@
                         }
                     }
                 }
+                &[data-item="add-admins"] {
+                    .cp-sidebar-form:not(:last-child) {
+                        margin-bottom: 1em;
+                    }
+                }
             }
         }
 }

www/admin/inner.js

@@ -67,7 +67,7 @@ define([
                     'description',
                     'email',
                     'jurisdiction',
-                    'flush-cache'
+                    'flush-cache',
                 ]
             },
             'customize': { // Msg.admin_cat_customize
@@ -77,6 +77,13 @@ define([
                     'color',
                 ]
             },
+            'admins': {
+                icon: 'fa fa-users',
+                content: [
+                    'list-admins',
+                    'add-admins'
+                ]
+            },
             'broadcast' : { // Msg.admin_cat_broadcast
                 icon: 'fa fa-bullhorn',
                 content : [
@@ -94,7 +101,7 @@ define([
                 ]
             },
             'apps': { // Msg.admin_cat_apps
-                icon: 'fa  fa-wrench',
+                icon: 'fa fa-wrench',
                 content: [
                     'apps',
                 ]
@@ -226,6 +233,193 @@ define([
             cb(button);
         });
 
+        const evRefreshAdmins = Util.mkEvent();
+        sidebar.addItem('list-admins', cb => {
+            const removeAdmin = (edPublic, _cb) => {
+                const cb = Util.mkAsync(_cb);
+                sFrameChan.query('Q_ADMIN_RPC', {
+                    cmd: 'ADMIN_DECREE',
+                    data: ['RM_ADMIN_KEY', [edPublic]]
+                }, function (e, response) {
+                    if (e || response.error) {
+                        UI.warn(Messages.error);
+                        console.error(e, response);
+                        return void cb('ERROR');
+                    }
+                    if (typeof(cb) === "function") { cb(); }
+                });
+            };
+
+            const header = [
+                Messages.admin_listName,
+                Messages.admin_listKey,
+                Messages.admin_listAction
+            ];
+            var list = blocks.table(header, []);
+            list.setAttribute('id', 'cp-admin-table');
+            let div = blocks.block([list]);
+            div.setAttribute('id', 'cp-admin-table-container');
+
+            const refreshTable = () => {
+                const admins = APP.instanceStatus.admins || [];
+                const newRows = admins.map(obj => {
+                    let { name, edPublic, hardcoded } = obj;
+                    name = name || Messages.admin_admin;
+                    let button = blocks.button('danger','fa-ban', Messages.admin_usersRemove);
+                    let $b = $(button);
+                    Util.onClickEnter($b, () => {
+                        $b.prop('disabled', 'disabled');
+                        UI.confirm(Messages.admin_listConfirm, yes => {
+                            if (!yes) { return $b.prop('disabled', false); }
+                            removeAdmin(edPublic, err => {
+                                $b.prop('disabled', false);
+                                if (err) { return; }
+                                APP.updateStatus(function () {
+                                    flushCache();
+                                    evRefreshAdmins.fire();
+                                });
+                            });
+                        });
+                    });
+                    let note = Messages.admin_listHardcoded;
+                    let action = hardcoded ? note : button;
+
+                    return [name, edPublic, action];
+                });
+                list.updateContent(newRows);
+            };
+            refreshTable();
+            evRefreshAdmins.reg(() => {
+                refreshTable();
+            });
+
+            cb(div);
+        });
+        sidebar.addItem('add-admins', cb => {
+            const content = blocks.block();
+            const metadataMgr = common.getMetadataMgr();
+            const privateData = metadataMgr.getPrivateData();
+            const $div = $(content);
+
+            const addAdmin = (data, _cb) => {
+                const cb = Util.mkAsync(_cb);
+                const { ed, name } = data;
+                const key = Hash.getPublicSigningKeyString(privateData.origin, name, ed);
+                sFrameChan.query('Q_ADMIN_RPC', {
+                    cmd: 'ADMIN_DECREE',
+                    data: ['ADD_ADMIN_KEY', [key]]
+                }, function (e, response) {
+                    if (e || response.error) {
+                        UI.warn(Messages.error);
+                        console.error(e, response);
+                        return void cb('ERROR');
+                    }
+                    if (typeof(cb) === "function") { cb(); }
+                    flushCache();
+                });
+            };
+
+            const keyInput = blocks.input({
+                placeholder: Messages.admin_accountMetadataPlaceholder
+            });
+            const keyLabel = blocks.labelledInput(Messages.admin_addKeyLabel, keyInput);
+            const keyButton = blocks.button('primary', 'fa-plus', Messages.tag_add);
+            const keyForm = blocks.form([keyLabel], blocks.nav([keyButton]));
+            const $keyInput = $(keyInput).on('input', () => {
+                let val = $keyInput.val().trim();
+                if (!val) {
+                    keyInput.setCustomValidity('');
+                    return;
+                }
+                let key = Keys.canonicalize(val);
+
+                if (keyInput.setCustomValidity) {
+                    if (!key) {
+                        const msg = Messages.admin_invalKey;
+                        keyInput.setCustomValidity(msg);
+                    } else {
+                        keyInput.setCustomValidity('');
+                    }
+                }
+            });
+            const $keyBtn = $(keyButton);
+            Util.onClickEnter($keyBtn, () => {
+                let val = $keyInput.val().trim();
+                let key = Keys.canonicalize(val);
+                if (!key) { return; }
+                // We have a valid key
+                let name = Messages.admin_admin;
+                try {
+                    let parsed = Keys.parseUser(val);
+                    name = parsed.user;
+                } catch (e) {}
+                $keyBtn.prop('disabled', 'disabled');
+                addAdmin({ ed:key, name }, (err) => {
+                    $keyBtn.prop('disabled', false);
+                    if (!err) { $keyInput.val(''); }
+                    // refresh
+                    APP.updateStatus(function () {
+                        evRefreshAdmins.fire();
+                    });
+                });
+            });
+
+            const drawContacts = () => {
+                $div.empty();
+                const members = {};
+                const admins = APP.instanceStatus.admins || [];
+                admins.forEach(obj => {
+                    const { edPublic, name, hardcoded, first } = obj;
+                    members[edPublic] = { name, hardcoded, first };
+                });
+
+                // Remove admins from contacts list
+                const friends = Util.clone(common.getFriends(false));
+                Object.keys(friends).forEach((curve) => {
+                    const ed = friends[curve]?.edPublic;
+                    if (members[ed]) { delete friends[curve]; }
+                });
+                let contactsGrid = UIElements.getUserGrid(Messages.admin_addAdminsAdd, {
+                    common: common,
+                    list: true,
+                    large: true,
+                    data: friends
+                }, function () {});
+                let addBtn = blocks.button('primary', 'fa-plus', Messages.tag_add);
+                Util.onClickEnter($(addBtn), () => {
+                    var $sel = $(contactsGrid.div).find('.cp-usergrid-user.cp-selected');
+                    nThen((waitFor) => {
+                        $sel.each((i, el) => {
+                            const $el = $(el);
+                            let ed = $el.attr('data-ed');
+                            let name = $el.attr('data-name');
+                            if (!ed || !name) {
+                                console.error('Missing data on selected user', el);
+                                return void UI.warn(Messages.error);
+                            }
+                            addAdmin({ed, name}, waitFor());
+                        });
+                    }).nThen(() => {
+                        APP.updateStatus(function () {
+                            evRefreshAdmins.fire();
+                            drawContacts();
+                        });
+                    });
+                });
+                evRefreshAdmins.reg(() => {
+                    drawContacts();
+                });
+
+                const list = blocks.form([
+                    //currentList.div,
+                    contactsGrid.div,
+                ], blocks.nav([addBtn]));
+                $div.append([keyForm, list]);
+            };
+            drawContacts();
+            cb(content);
+        });
+
         var isHex = s => !/[^0-9a-f]/.test(s);
 
         var sframeCommand = function (command, data, cb) {
@@ -3928,7 +4122,7 @@ define([
 
         // EXTENSION_POINT:ADMIN_ITEM
         let utils = {
-            h, Util, Hash, UIElements
+            $, h, Util, Hash, UIElements, UI, APP
         };
         common.getExtensionsSync('ADMIN_ITEM').forEach(ext => {
             if (!ext || !ext.id || typeof(ext.getContent) !== "function") {
@@ -3952,13 +4146,19 @@ define([
         sidebar.makeLeftside(categories);
     };
     var updateStatus = APP.updateStatus = function (cb) {
-        sFrameChan.query('Q_ADMIN_RPC', {
-            cmd: 'INSTANCE_STATUS',
-        }, function (e, data) {
-            if (e) { console.error(e); return void cb(e); }
-            if (!Array.isArray(data)) { return void cb('EINVAL'); }
-            APP.instanceStatus = data[0];
-            console.log("Status", APP.instanceStatus);
+        nThen(w => {
+            sFrameChan.query('Q_ADMIN_RPC', {
+                cmd: 'INSTANCE_STATUS',
+            }, w(function (e, data) {
+                if (e) { console.error(e); return void cb(e); }
+                if (!Array.isArray(data)) { return void cb('EINVAL'); }
+                APP.instanceStatus = data[0];
+                console.log("Status", APP.instanceStatus);
+            }));
+            require([`/api/config?${+new Date()}`], w(ApiConfig => {
+                APP.instanceConfig = ApiConfig;
+            }));
+        }).nThen(() => {
             cb();
         });
     };

www/common/common-interface.js

@@ -1237,6 +1237,7 @@ define([
         var addTippy = function (i, el) {
             if (el._tippy) { return; }
             if (!el.getAttribute('title')) { return; }
+            if (el.getAttribute('data-notippy')) { return; }
             if (el.nodeName === 'IFRAME') { return; }
             var opts = {
                 distance: 15

www/common/inner/sidebar-layout.js

@@ -156,6 +156,35 @@ define([
             return box;
         };
 
+        // opts.values = { key1:label1, key2:label2 }
+        blocks.radio = (key, state, opts, onChange) => {
+            if (!opts?.values) {
+                return void console.error('NO_VALUES');
+            }
+            let all = Object.keys(opts.values).map(k => {
+                let v = opts.values[k];
+                let r = UI.createRadio(
+                    `cp-${app}-${key}`,
+                    `cp-${app}-${key}-${k}`,
+                    v, state === k, {
+                        input: { value: k },
+                        label: { class: 'noTitle' }
+                    }
+                );
+                if (typeof(onChange) === "function"){
+                    $(r).find('input').on('change', function() {
+                        onChange(k);
+                    });
+                }
+                return r;
+            });
+            let block = h('div.cp-sidebar-flex-block', all);
+            if (opts && opts.spinner) {
+                block.spinner = UI.makeSpinner($(block));
+            }
+            return block;
+        };
+
         blocks.table = function (header, entries) {
             const table = h('table.cp-sidebar-table');
             if (header) {
@@ -255,6 +284,13 @@ define([
             return box;
         };
 
+        blocks.hintItem = (hint, item) => {
+            return blocks.form([
+                h('span.cp-sidebarlayout-description-item', hint),
+                item
+            ]);
+        };
+
         return blocks;
     };
 

www/common/onlyoffice/inner.js

@@ -1720,11 +1720,14 @@ define([
                 "documentType": file.doc,
                 "editorConfig": {
                     customization: {
+                        compactHeader: true,
                         chat: false,
                         logo: {
                             url: "/bounce/#" + encodeURIComponent('https://www.onlyoffice.com')
                         },
-                        comments: !lock && !readOnly
+                        comments: !lock && !readOnly,
+                        hideRightMenu: true,
+                        uiTheme: window.CryptPad_theme === "dark" ? "theme-dark" : "theme-classic-light"
                     },
                     "user": {
                         "id": String(myOOId), //"c0c3bf82-20d7-4663-bf6d-7fa39c598b1d",
@@ -2183,8 +2186,11 @@ Uncaught TypeError: Cannot read property 'calculatedType' of null
                 }
                 c.forcesave = true;
             }
-            console.error('updated config', APP.ooconfig);
 
+            // Always hide right menu
+            localStorage?.original.removeItem('sse-hide-right-settings');
+            localStorage?.original.removeItem('de-hide-right-settings');
+            localStorage?.original.removeItem('pe-hide-right-settings');
 
             APP.docEditor = new window.DocsAPI.DocEditor("cp-app-oo-placeholder-a", APP.ooconfig);
             ooLoaded = true;

www/common/sframe-boot2.js

@@ -25,16 +25,19 @@ define([
             delete ls[k];
         });
     };
-    var mkFakeStore = function () {
+    var mkFakeStore = function (original) {
         var fakeStorage = {
             getItem: function (k) { return fakeStorage[k]; },
             setItem: function (k, v) { fakeStorage[k] = v; return v; },
-            removeItem: function (k) { delete fakeStorage[k]; }
+            removeItem: function (k) { delete fakeStorage[k]; },
+            original
         };
         return fakeStorage;
     };
-    window.__defineGetter__('localStorage', function () { return mkFakeStore(); });
-    window.__defineGetter__('sessionStorage', function () { return mkFakeStore(); });
+    let loc = localStorage;
+    let ses = sessionStorage;
+    window.__defineGetter__('localStorage', function () { return mkFakeStore(loc); });
+    window.__defineGetter__('sessionStorage', function () { return mkFakeStore(ses); });
 
     window.CRYPTPAD_INSIDE = true;
 

www/common/translations/messages.pl.json

@@ -444,7 +444,7 @@
     "support_formContentError": "Błąd: treść jest pusta",
     "support_formTitleError": "Błąd: tytuł jest pusty",
     "support_formButton": "Wyślij",
-    "support_formHint": "Użyj tego formularza, aby bezpiecznie skontaktować się z administratorami.<br>Proszę zauważyć, że niektóre kwestie/pytania mogą być już poruszone w <a>Podręczniku Użytkownika CryptPad</a>. Prosimy nie tworzyć nowego zgłoszenia, jeśli masz już otwarte zgłoszenie dotyczące tego samego problemu. Zamiast tego odpowiedz na oryginalną wiadomość, podając dodatkowe informacje.",
+    "support_formHint": "Użyj tego formularza, aby bezpiecznie skontaktować się z administratorami.<br>Zwróć uwagę, że niektóre kwestie/pytania mogą być już poruszone w <a>Podręczniku Użytkownika CryptPad</a>. Prosimy nie tworzyć nowego zgłoszenia, jeśli masz już otwarte zgłoszenie dotyczące tego samego problemu. Zamiast tego odpowiedz na oryginalną wiadomość, podając dodatkowe informacje.",
     "support_formTitle": "Nowe Zgłoszenie",
     "support_cat_new": "Nowe zgłoszenie",
     "support_disabledHint": "Ta instancja CryptPad nie jest jeszcze skonfigurowana do korzystania z formularza pomocy technicznej.",
@@ -1783,5 +1783,9 @@
     "admin_onboardingOptionsHint": "Zaznacz właściwą opcję dla Twojej instancji.<br>To ustawienie może być później zmienione w panelu administratora.",
     "team_autoTrim": "Zmniejszanie historii dysku zespołu... Proszę czekać.",
     "admin_mfa_confirm_enable": "Czy na pewno chcesz włączyć uwierzytelnianie wieloetapowe?",
-    "admin_mfa_confirm_disable": "Czy na pewno chcesz wyłączyć uwierzytelnianie wieloetapowe?"
+    "admin_mfa_confirm_disable": "Czy na pewno chcesz wyłączyć uwierzytelnianie wieloetapowe?",
+    "form_passwordWarning": "Zwróć uwagę, że hasło do formularza można ustawić tylko teraz, podczas tworzenia go, i nie może być ono zmienione później.",
+    "fm_restoreMultipleDialog": "Czy na pewni chcesz przywrócić {0} plików i/lub folderów do ich poprzedniej lokalizacji?",
+    "calendar_show": "Pokaż kalendarze",
+    "calendar_hide": "Ukryj kalendarze"
 }

www/common/translations/messages.zh.json

@@ -1783,5 +1783,9 @@
     "dph_tmp_pw": "此模板受新密码保护。从您的云盘打开它以输入新密码。",
     "duplicate": "重复",
     "admin_logoButton": "上传新的",
-    "install_notes": "<ul class=\"cp-notes-list\"><li>在此页面上创建您的第一个管理员账号。管理员管理实例设置，包括存储配额，并有权访问审核工具。</li><li>您的密码是加密此实例上所有文档和管理员权限的密钥。<span class=\"red\">如果您丢失了密码，我们将无法恢复您的数据。</span></li><li>如果您使用的是共享计算机，<span class=\"red\">请记住在使用完后登出</span>。只关闭浏览器窗口会暴露您的账号。</li></ul>"
+    "install_notes": "<ul class=\"cp-notes-list\"><li>在此页面上创建您的第一个管理员账号。管理员管理实例设置，包括存储配额，并有权访问审核工具。</li><li>您的密码是加密此实例上所有文档和管理员权限的密钥。<span class=\"red\">如果您丢失了密码，我们将无法恢复您的数据。</span></li><li>如果您使用的是共享计算机，<span class=\"red\">请记住在使用完后登出</span>。只关闭浏览器窗口会暴露您的账号。</li></ul>",
+    "fm_restoreMultipleDialog": "是否确定要将 {0} 个文件和/或文件夹还原到其以前的位置？",
+    "calendar_hide": "隐藏日历",
+    "form_passwordWarning": "请注意，表单密码只能在创建时设置，以后不能更改。",
+    "calendar_show": "显示日历"
 }

www/kanban/app-kanban.less

@@ -145,6 +145,17 @@
         margin-bottom: 15px;
     }
 
+    .cp-kanban-toggle-container.cp-kanban-container-flex {
+        flex: 1;
+    }
+    .cp-kanban-toggle-tags {
+        text-transform: unset;
+        margin-right: 0.5rem;
+        padding: 3px 10px;
+        span {
+            font: @colortheme_app-font;
+        }
+    }
     #cp-kanban-edit-tags {
         .tokenfield {
             margin: 0;
@@ -152,10 +163,6 @@
         }
         margin-bottom: 15px;
     }
-    .kanban-tag-btn-toggle {
-        margin-top: 10px;
-        margin-left: 10px
-    }
     #cp-app-kanban-container {
         flex: 1;
         display: flex;
@@ -430,12 +437,10 @@
         justify-content: space-between;
         position: relative;
         min-height: 50px;
+        align-items: center;
         .cp-kanban-filterTags {
             @media (min-width: 505px) {
                 display: inline-flex;
-                .kanban-tag-btn-toggle {
-                    margin-right: 10px;
-                }
             }
             align-items: center;
             flex: 1;
@@ -449,16 +454,8 @@
                 }
                 flex-flow: column;
                 flex-shrink: 0;
-                & > * {
-                    visibility: hidden;
-                }
                 & > span {
                     display: inline-block;
-                    height: 38px;
-                    line-height: 38px;
-                }
-                & > button {
-                    margin-top: -38px;
                 }
             }
             button.cp-kanban-filterTags-reset {
@@ -571,7 +568,7 @@
             display: flex;
             min-height: 0;
             .kanban-container {
-                padding: 30px 5px;
+                padding: 0px 5px;
                 flex: 1;
                 display: flex;
                 max-height: 100%;
@@ -700,6 +697,15 @@
             }
         }
 
+        @media (pointer: none), (pointer:coarse) {
+            .kanban-container-outer {
+                .kanban-container {
+                    padding: 30px 5px;
+                }
+            }
+        }
+
+
         &.cp-app-readonly {
             .kanban-item, .kanban-title-board {
                 cursor: default !important;

www/kanban/inner.js

@@ -65,6 +65,9 @@ define([
     var onCursorUpdate = Util.mkEvent();
     var remoteCursors = {};
 
+    let getCursor = () => {};
+    let restoreCursor = () => {};
+
     var setValueAndCursor = function (input, val, _cursor) {
         if (!input) { return; }
         var $input = $(input);
@@ -153,9 +156,12 @@ define([
     var _lastUpdate = 0;
     var _updateBoards = function (framework, kanban, boards) {
         _lastUpdate = now();
+        var cursor = getCursor();
         kanban.setBoards(Util.clone(boards));
         kanban.inEditMode = false;
         addEditItemButton(framework, kanban);
+        restoreCursor(cursor);
+        onRemoteChange.fire();
     };
     var _updateBoardsThrottle = Util.throttle(_updateBoards, 1000);
     var updateBoards = function (framework, kanban, boards) {
@@ -700,8 +706,11 @@ define([
                     var item = kanban.getItemJSON(eid);
                     item.title = name;
                     kanban.onChange();
-                    // Unlock edit mode
-                    kanban.inEditMode = false;
+                    // Unlock edit mode unless we're already editing
+                    // something else
+                    if (kanban.inEditMode === eid) {
+                        kanban.inEditMode = false;
+                    }
                     onCursorUpdate.fire({});
                 };
                 $input.blur(save);
@@ -764,7 +773,9 @@ define([
                     kanban.getBoardJSON(boardId).title = name;
                     kanban.onChange();
                     // Unlock edit mode
-                    kanban.inEditMode = false;
+                    if (kanban.inEditMode === boardId) {
+                        kanban.inEditMode = false;
+                    }
                     onCursorUpdate.fire({});
                 };
                 $input.blur(save);
@@ -819,7 +830,9 @@ define([
                 });
                 var save = function () {
                     $item.remove();
-                    kanban.inEditMode = false;
+                    if (kanban.inEditMode === "new") {
+                        kanban.inEditMode = false;
+                    }
                     onCursorUpdate.fire({});
                     if (!$input.val()) { return; }
                     var id = Util.createRandomInteger();
@@ -930,18 +943,15 @@ define([
                 //framework._.sfCommon.setPadAttribute('quickMode', false);
             });
 
-            var toggleTagsButton = h('button.btn.btn-default.kanban-tag-btn-toggle', Messages.kanban_showTags);
-
             // Tags filter
             var existing = getExistingTags(kanban.options.boards);
             var list = h('div.cp-kanban-filterTags-list');
-            var reset = h('button.btn.btn-cancel.cp-kanban-filterTags-reset', [
+            var reset = h('button.btn.btn-cancel.cp-kanban-filterTags-reset.cp-kanban-toggle-tags', [
                 h('i.fa.fa-times'),
-                Messages.kanban_clearFilter
+                h('span', Messages.kanban_clearFilter)
             ]);
             var hint = h('span.cp-kanban-filterTags-name', Messages.kanban_tags);
             var tags = h('div.cp-kanban-filterTags', [
-
                 h('span.cp-kanban-filterTags-toggle', [
                     hint,
                     reset,
@@ -954,8 +964,16 @@ define([
             var $hint = $(hint);
 
             var setTagFilterState = function (bool) {
+                //$hint.toggle(!bool);
+                //$reset.toggle(!!bool);
                 $hint.css('visibility', bool? 'hidden': 'visible');
+                $hint.css('height', bool ? 0 : '');
+                $hint.css('padding-top', bool ? 0 : '');
+                $hint.css('padding-bottom', bool ? 0 : '');
                 $reset.css('visibility', bool? 'visible': 'hidden');
+                $reset.css('height', !bool ? 0 : '');
+                $reset.css('padding-top', !bool ? 0 : '');
+                $reset.css('padding-bottom', !bool ? 0 : '');
             };
             setTagFilterState();
 
@@ -1022,41 +1040,47 @@ define([
                 commitTags();
             });
 
+            let toggleTagsButton = h('button.btn.btn-default.cp-kanban-toggle-tags', [
+                h('i.fa.fa-tags'),
+                h('span', Messages.fm_tagsName)
+            ]);
+            let toggleContainer = h('div.cp-kanban-toggle-container', toggleTagsButton);
 
-            if ($(window).width() < 500) {
+            let toggleClicked = false;
+            let $tags = $(tags);
+            let toggle = () => {
+                $tags.toggle();
+                let visible = $tags.is(':visible');
+                $(toggleContainer).toggleClass('cp-kanban-container-flex', !visible);
+                $toggleBtn.toggleClass('btn-default', visible);
+                $toggleBtn.toggleClass('btn-default-alt', !visible);
+            };
+            let $toggleBtn = $(toggleTagsButton).click(function() {
+                toggleClicked = true;
+                toggle();
+            });
 
-                $(tags).append(toggleTagsButton);
-
-                var hideTags = function () {
-                    for (var tag of list.children) {
-                        if (existing.indexOf(tag.innerHTML) > 10) {
-                            $(tag).hide();                    
-                        }
+            const resizeTags = () => {
+                if (toggleClicked) { return; }
+                let visible = $tags.is(':visible');
+                // Small screen and visible: hide
+                if ($(window).width() < 600) {
+                    if (visible) {
+                        $(tags).show();
+                        toggle();
                     }
-                };
-                hideTags();
-    
-                var toggleTags = function () {
-                    for (var tag of list.children) {
-                        if (existing.indexOf(tag.innerHTML) > 10 && kanban.options.tags.indexOf(tag.innerHTML) === -1) {
-                            if ($(tag).is(":visible")) { 
-                                $(tag).hide();
-                                $(toggleTagsButton).text(Messages.kanban_showTags);
-                            } else {
-                                $(tag).show();
-                                $(toggleTagsButton).text(Messages.kanban_hideTags);
-                            }
-                        }
-                    }
-                };
-    
-                $(toggleTagsButton).click(function() {
-                    toggleTags();
-                });
-    
-            }
+                    return;
+                }
+                // Large screen: make visible by default
+                if (visible) { return; }
+                $(tags).hide();
+                toggle();
+            };
+
+            $(window).on('resize', resizeTags);
 
             var container = h('div#cp-kanban-controls', [
+                toggleContainer,
                 tags,
                 h('div.cp-kanban-changeView', [
                     small,
@@ -1065,18 +1089,6 @@ define([
             ]);
             $container.before(container);
 
-            var common = framework._.sfCommon;
-            var $button = common.createButton('toggle', true, {
-                element: $(container),
-                icon: 'fa-tags',
-                text: Messages.fm_tagsName,
-            }, function () {
-                $button.toggleClass('cp-toolbar-button-active');
-
-            });
-            $button.addClass('cp-toolbar-button-active');
-            framework._.toolbar.$bottomL.append($button);
-
             onRedraw.reg(function () {
                 // Redraw if new tags have been added to items
                 var old = Sortify(existing);
@@ -1191,7 +1203,7 @@ define([
             $container.find('.kanban-edit-item').remove();
         });
 
-        var getCursor = function () {
+        getCursor = function () {
             if (!kanban || !kanban.inEditMode) { return; }
             try {
                 var id = kanban.inEditMode;
@@ -1232,7 +1244,7 @@ define([
                 return {};
             }
         };
-        var restoreCursor = function (data) {
+        restoreCursor = function (data) {
             if (!data) { return; }
             try {
                 var id = data.id;
@@ -1296,12 +1308,9 @@ define([
             var remoteContent = newContent.content;
 
             if (Sortify(currentContent) !== Sortify(remoteContent)) {
-                var cursor = getCursor();
                 verbose("Content is different.. Applying content");
                 kanban.options.boards = remoteContent;
                 updateBoards(framework, kanban, remoteContent);
-                restoreCursor(cursor);
-                onRemoteChange.fire();
             }
         });
 

www/notifications/app-notifications.less

@@ -78,6 +78,7 @@
                 justify-content: flex-start;
                 align-items: center;
                 background-color: @cp_notif-bg;
+                border-top: 1px solid @cp_notif-table-border;
                 &.no-notifications {
                     display: none;
                     padding: 1rem 1rem;
@@ -95,12 +96,14 @@
                         background-color: @cp_notif-hover;
                     }
                 }
+                .cp-avatar-calendar {
+                    font-size: 45px;
+                    padding: 0 12px;
+                    overflow: hidden;
+                }
                 &.cp-app-notification-archived {
                     background-color: @cp_notif-bg;
                 }
-                &:not(:first-child) {
-                    border-top: 1px solid @cp_notif-table-border;
-                }
                 &.dismissed {
                     display: none;
                 }

www/notifications/inner.js

@@ -87,6 +87,8 @@ define([
             // if the type of notification correspond
             if (filterTypes.indexOf(data.content.msg.type) !== -1) {
                 notifsData.push(data);
+                var icon = $(el).find(".cp-reminder");
+                $(icon).addClass('cp-avatar-calendar');
                 $(notifsList).prepend(el);
             }
         };
@@ -143,7 +145,7 @@ define([
             $(loadmore).click();
         }
 
-        common.mailbox.subscribe(["notifications"], {
+        common.mailbox.subscribe(["notifications", "reminders"], {
             onMessage: function (data, el) {
                 addNotification(data, el);
             },