From 5807b4dddf2bef8c28c0d9f18f5b50d0e67d015b Mon Sep 17 00:00:00 2001
From: yflory <yann.flory@xwiki.com>
Date: Mon, 21 Aug 2023 12:45:18 +0200
Subject: [PATCH] Reduce memory usage for the eviction script

---
 lib/eviction.js     |  2 +-
 lib/storage/file.js | 23 +++++++++++------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/lib/eviction.js b/lib/eviction.js
index 73f43ae45..6a631de8c 100644
--- a/lib/eviction.js
+++ b/lib/eviction.js
@@ -367,7 +367,7 @@ module.exports = function (Env, cb) {
 
         TODO make this configurable ?
     */
-    var BLOOM_CAPACITY = (1 << 22) - 1; // over two million items
+    var BLOOM_CAPACITY = (1 << 24) - 1; // over two million items
     var BLOOM_ERROR = 1 / 10000;  // an error rate of one in ten thousand
 
     // we'll use one filter for the set of active documents
diff --git a/lib/storage/file.js b/lib/storage/file.js
index ee023df7f..5f915f5c8 100644
--- a/lib/storage/file.js
+++ b/lib/storage/file.js
@@ -584,8 +584,6 @@ var getStats = function (env, channelName, cb) {
 
 // TODO use ../plan.js for a smaller memory footprint
 var listChannels = function (root, handler, cb, fast) {
-    // do twenty things at a time
-    var sema = Semaphore.create(20);
 
     var dirList = [];
 
@@ -599,17 +597,18 @@ var listChannels = function (root, handler, cb, fast) {
             }
             dirList = list;
         }));
-    }).nThen(function (w) {
+    }).nThen(function (waitFor) {
         // search inside the nested directories
         // stream it so you don't put unnecessary data in memory
-        var wait = w();
+        //var wait = w();
+        var n = nThen;
         dirList.forEach(function (dir) {
-            sema.take(function (give) {
-    // TODO modify the asynchronous bits here to keep less in memory at any given time
-    // list a directory -> process its contents with semaphores until less than N jobs are running
-    // then list the next directory...
+            // Handle one directory at a time to save some memory
+            n = n(function (w) {
+                // do twenty things at a time
+                var sema = Semaphore.create(20);
                 var nestedDirPath = Path.join(root, dir);
-                Fs.readdir(nestedDirPath, w(give(function (err, list) {
+                Fs.readdir(nestedDirPath, w(function (err, list) {
                     if (err) { return void handler(err); } // Is this correct?
 
                     list.forEach(function (item) {
@@ -660,10 +659,10 @@ var listChannels = function (root, handler, cb, fast) {
                             }, isLonelyMetadata);
                         });
                     });
-                })));
-            });
+                }));
+            }).nThen;
         });
-        wait();
+        n(waitFor());
     }).nThen(function () {
         cb();
     });