From 44aee13a56135063f5ef1ea6448254e18ae8400c Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 15 Jul 2024 12:02:07 +0200 Subject: [PATCH] Compute new metadata first and fallback to old system if needed --- lib/storage/file.js | 29 +++++++-- .../00000000000000000000000000000000.ndjson | 7 +++ ...00000000000000000000000001.metadata.ndjson | 5 ++ .../00000000000000000000000000000001.ndjson | 7 +++ ...00000000000000000000000002.metadata.ndjson | 1 + .../00000000000000000000000000000002.ndjson | 6 ++ ...00000000000000000000000003.metadata.ndjson | 6 ++ .../00000000000000000000000000000003.ndjson | 6 ++ .../00000000000000000000000000000004.ndjson | 6 ++ scripts/tests/test-metadata.js | 60 +++++++++++++++++++ 10 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000000.ndjson create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000001.metadata.ndjson create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000001.ndjson create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000002.metadata.ndjson create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000002.ndjson create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000003.metadata.ndjson create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000003.ndjson create mode 100644 scripts/tests/test-data/00/00000000000000000000000000000004.ndjson create mode 100644 scripts/tests/test-metadata.js diff --git a/lib/storage/file.js b/lib/storage/file.js index 2c2888f7a..d33c42f5f 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -429,7 +429,27 @@ How to proceed */ + let requiresChannel = true; + let all = []; nThen(function (w) { + let first = true; + getDedicatedMetadata(env, channelId, (err, line) => { + if (first && !err) { + if (!Array.isArray(line)) { + requiresChannel = false; + } + first = false; + } + all.push({err, line}); + }, w(function (err) { + if (err) { + // stream errors? + w.abort(); + return void cb(err); + } + })); + }).nThen(function (w) { + if (!requiresChannel) { return; } // returns the first line of a channel, parsed... getChannelMetadata(env, channelId, w(function (err, data) { if (err) { @@ -445,13 +465,10 @@ How to proceed handler(null, data); })); }).nThen(function () { - getDedicatedMetadata(env, channelId, handler, function (err) { - if (err) { - // stream errors? - return void cb(err); - } - cb(); + all.forEach(({err, line}) => { + handler(err, line); }); + cb(); }); }; diff --git a/scripts/tests/test-data/00/00000000000000000000000000000000.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000000.ndjson new file mode 100644 index 000000000..cdbaaefa2 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000000.ndjson @@ -0,0 +1,7 @@ +{"owners":["TestOwner"],"validateKey":"TestKey","channel":"0","created":1721035117462} +[0,"test","MSG","test",1721035117969] +[0,"test","MSG","test2",1721035117969] +[0,"test","MSG","test3",1721035117969] +[0,"test","MSG","test4",1721035117969] +[0,"test","MSG","test5",1721035117969] +[0,"test","MSG","test6",1721035117969] diff --git a/scripts/tests/test-data/00/00000000000000000000000000000001.metadata.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000001.metadata.ndjson new file mode 100644 index 000000000..1cafb44c2 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000001.metadata.ndjson @@ -0,0 +1,5 @@ +["RESTRICT_ACCESS",[true],1721035145090] +["ADD_ALLOWED",["NewAllowedKeyNewAllowedKeyNewAllowedKeyNewAl"],1721035148115] +["ADD_PENDING_OWNERS",["PendingOwner"],1721035151295] +["RESTRICT_ACCESS",[false],1721035155836] +["RESTRICT_ACCESS",[true],1721035156728] diff --git a/scripts/tests/test-data/00/00000000000000000000000000000001.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000001.ndjson new file mode 100644 index 000000000..cdbaaefa2 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000001.ndjson @@ -0,0 +1,7 @@ +{"owners":["TestOwner"],"validateKey":"TestKey","channel":"0","created":1721035117462} +[0,"test","MSG","test",1721035117969] +[0,"test","MSG","test2",1721035117969] +[0,"test","MSG","test3",1721035117969] +[0,"test","MSG","test4",1721035117969] +[0,"test","MSG","test5",1721035117969] +[0,"test","MSG","test6",1721035117969] diff --git a/scripts/tests/test-data/00/00000000000000000000000000000002.metadata.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000002.metadata.ndjson new file mode 100644 index 000000000..7c47d09d3 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000002.metadata.ndjson @@ -0,0 +1 @@ +{"owners":["TestOwner"],"validateKey":"TestKey","channel":"0","created":1721035117462} diff --git a/scripts/tests/test-data/00/00000000000000000000000000000002.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000002.ndjson new file mode 100644 index 000000000..99973c454 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000002.ndjson @@ -0,0 +1,6 @@ +[0,"test","MSG","test",1721035117969] +[0,"test","MSG","test2",1721035117969] +[0,"test","MSG","test3",1721035117969] +[0,"test","MSG","test4",1721035117969] +[0,"test","MSG","test5",1721035117969] +[0,"test","MSG","test6",1721035117969] diff --git a/scripts/tests/test-data/00/00000000000000000000000000000003.metadata.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000003.metadata.ndjson new file mode 100644 index 000000000..ffcb25082 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000003.metadata.ndjson @@ -0,0 +1,6 @@ +{"owners":["TestOwner"],"validateKey":"TestKey","channel":"0","created":1721035117462} +["RESTRICT_ACCESS",[true],1721035145090] +["ADD_ALLOWED",["NewAllowedKeyNewAllowedKeyNewAllowedKeyNewAl"],1721035148115] +["ADD_PENDING_OWNERS",["PendingOwner"],1721035151295] +["RESTRICT_ACCESS",[false],1721035155836] +["RESTRICT_ACCESS",[true],1721035156728] diff --git a/scripts/tests/test-data/00/00000000000000000000000000000003.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000003.ndjson new file mode 100644 index 000000000..99973c454 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000003.ndjson @@ -0,0 +1,6 @@ +[0,"test","MSG","test",1721035117969] +[0,"test","MSG","test2",1721035117969] +[0,"test","MSG","test3",1721035117969] +[0,"test","MSG","test4",1721035117969] +[0,"test","MSG","test5",1721035117969] +[0,"test","MSG","test6",1721035117969] diff --git a/scripts/tests/test-data/00/00000000000000000000000000000004.ndjson b/scripts/tests/test-data/00/00000000000000000000000000000004.ndjson new file mode 100644 index 000000000..99973c454 --- /dev/null +++ b/scripts/tests/test-data/00/00000000000000000000000000000004.ndjson @@ -0,0 +1,6 @@ +[0,"test","MSG","test",1721035117969] +[0,"test","MSG","test2",1721035117969] +[0,"test","MSG","test3",1721035117969] +[0,"test","MSG","test4",1721035117969] +[0,"test","MSG","test5",1721035117969] +[0,"test","MSG","test6",1721035117969] diff --git a/scripts/tests/test-metadata.js b/scripts/tests/test-metadata.js new file mode 100644 index 000000000..30d5ddc8c --- /dev/null +++ b/scripts/tests/test-metadata.js @@ -0,0 +1,60 @@ +const Store = require("../../lib/storage/file"); +const Meta = require("../../lib/metadata"); +const nThen = require('nthen'); + +let chanOld = '00000000000000000000000000000000'; +let chanOldUpdated = '00000000000000000000000000000001'; +let chanNew = '00000000000000000000000000000002'; +let chanNewUpdated = '00000000000000000000000000000003'; +let chanNoMeta = '00000000000000000000000000000004'; + +Store.create({ + filePath: './test-data/' +}, (err, store) => { + if (err) { return void console.error(err); } + + const readMetadata = (channel, cb) => { + const ref = {}; + const h = Meta.createLineHandler(ref, console.error); + store.readChannelMetadata(channel, h, () => { + cb(ref && ref.meta); + }); + }; + + nThen(w => { + readMetadata(chanOld, w(meta => { + if (!meta || meta.validateKey !== "TestKey") { + console.log('OldChanNoUpdate', meta); + throw new Error("Error with old channel without metadata update"); + } + })); + readMetadata(chanOldUpdated, w(meta => { + if (!meta || meta.validateKey !== "TestKey" || !meta.restricted || !meta.allowed.includes('NewAllowedKeyNewAllowedKeyNewAllowedKeyNewAl')) { + console.log('OldChanUpdate', meta); + throw new Error("Error with old channel with metadata updates"); + } + })); + readMetadata(chanNew, w(meta => { + if (!meta || meta.validateKey !== "TestKey") { + console.log('NewChanNoUpdate', meta); + throw new Error("Error with new channel without metadata update"); + } + })); + readMetadata(chanNewUpdated, w(meta => { + if (!meta || meta.validateKey !== "TestKey" || !meta.restricted || !meta.allowed.includes('NewAllowedKeyNewAllowedKeyNewAllowedKeyNewAl')) { + console.log('NewChanUpdate', meta); + throw new Error("Error with new channel with metadata updates"); + } + })); + readMetadata(chanNoMeta, w(meta => { + if (meta && Object.keys(meta).length) { + console.log('NoMetadataChan', meta); + throw new Error("Error with channel without metadata"); + } + })); + }).nThen(() => { + console.log('Success'); + process.exit(1); + }); +}) +