fix runner stdin and update maintenance lifecycle

archivebox/cli/archivebox_run.py

@@ -342,7 +342,8 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
 @click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool, no_stdin: bool):
     """
     Process queued work.
 
@@ -394,7 +395,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
         if maintenance_only:
             sys.exit(run_runner(daemon=daemon, maintenance_only=True))
 
-        if not sys.stdin.isatty():
+        if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
             sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))

archivebox/cli/archivebox_update.py

@@ -342,7 +342,7 @@ def update(
                             (
                                 stats_combined["phase1"].get("queued", 0),
                                 stats_combined["phase2"].get("queued", 0),
-                                stats_combined["phase2"].get("crawls_queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
                             ),
                         )
                         runner_work_queued = runner_work_queued or maintenance_work_queued
@@ -726,7 +726,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
         "updated_db": 0,
         "queued": 0,
         "sealed": 0,
-        "crawls_queued": 0,
+        "crawls_sealed": 0,
     }
     current_fs_version = Snapshot._fs_current_version()
 
@@ -833,7 +833,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     queue_stale_fs_batch()
 
     now = timezone.now()
-    stats["crawls_queued"] = (
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
         Crawl.objects.filter(
             status__in=Crawl.RUNNABLE_STATES,
         )
@@ -841,11 +844,12 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
             snapshot_set__status__in=Snapshot.OPEN_STATES,
         )
         .update(
-            retry_at=now,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
             modified_at=now,
         )
     )
-    stats["updated_db"] += stats["crawls_queued"]
+    stats["updated_db"] += stats["crawls_sealed"]
     return stats
 
 
@@ -983,7 +987,7 @@ Phase 2 (Process DB):
   Updated JSON:     {s2.get("updated_json", 0)}
   Updated DB rows:  {s2.get("updated_db", 0)}
   Sealed snapshots: {s2.get("sealed", 0)}
-  Queued crawls:    {s2.get("crawls_queued", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)
 
 

archivebox/tests/conftest.py

@@ -202,6 +202,7 @@ def run_archivebox_cmd(
         if disable_extractors:
             run_env.update(
                 {
+                    "PLUGINS": "__archivebox_test_no_plugins__",
                     "SAVE_ARCHIVEDOTORG": "False",
                     "SAVE_TITLE": "False",
                     "SAVE_FAVICON": "False",
@@ -592,9 +593,11 @@ def cli_env(
     if disable_extractors or live or server:
         env.update(
             {
+                "PLUGINS": "__archivebox_test_no_plugins__",
                 "SAVE_ARCHIVEDOTORG": "False",
                 "SAVE_TITLE": "False",
                 "SAVE_FAVICON": "False",
+                "SAVE_WGET": "False",
                 "SAVE_WARC": "False",
                 "SAVE_PDF": "False",
                 "SAVE_SCREENSHOT": "False",

archivebox/workers/supervisord_util.py

@@ -170,7 +170,7 @@ RUNNER_WORKER = {
 RUNNER_ONCE_WORKER = lambda args, name="worker_runner_once": {
     **RUNNER_WORKER,
     "name": name,
-    "command": _shell_join([sys.executable, "-m", "archivebox", "run", *args]),
+    "command": _shell_join([sys.executable, "-m", "archivebox", "run", "--no-stdin", *args]),
     "environment": 'PYTHONUNBUFFERED="1",COLUMNS="200"',
     "autorestart": "false",
     "stopwaitsecs": "1",