Use lightweight supervisord parent watchdog

archivebox/tests/test_cli_server.py

@@ -252,6 +252,31 @@ def test_daphne_worker_uses_default_application_close_timeout():
     assert "--application-close-timeout=0" not in command
 
 
+def test_supervisord_parent_watchdog_does_not_start_another_archivebox_runtime():
+    from archivebox.workers import supervisord_util
+
+    worker = supervisord_util.SUPERVISORD_PARENT_WATCHDOG_WORKER(
+        owner_pid=101,
+        owner_started_at=1001.5,
+        supervisord_pid=202,
+        supervisord_started_at=2002.5,
+    )
+    command = shlex.split(worker["command"])
+    watchdog_script = Path(supervisord_util.__file__).with_name("supervisord_parent_watchdog.py")
+
+    assert command == [
+        sys.executable,
+        str(watchdog_script),
+        "--owner-pid=101",
+        "--owner-started-at=1001.5",
+        "--supervisord-pid=202",
+        "--supervisord-started-at=2002.5",
+    ]
+    assert Path(command[1]).name == "supervisord_parent_watchdog.py"
+    assert "archivebox" not in Path(command[0]).name
+    assert "manage" not in command
+
+
 def test_reload_workers_use_active_archivebox_module():
     from archivebox.workers.supervisord_util import RUNNER_WATCH_WORKER, RUNSERVER_WORKER, archivebox_cmd
 

archivebox/workers/supervisord_parent_watchdog.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import time
+
+import psutil
+
+
+def matching_process(pid: int, started_at: float) -> psutil.Process | None:
+    try:
+        process = psutil.Process(pid)
+        if process.status() == psutil.STATUS_ZOMBIE:
+            return None
+        if abs(process.create_time() - started_at) > 0.01:
+            return None
+        return process
+    except (psutil.NoSuchProcess, psutil.AccessDenied):
+        return None
+
+
+def run_watchdog(
+    *,
+    owner_pid: int,
+    owner_started_at: float,
+    supervisord_pid: int,
+    supervisord_started_at: float,
+    interval: float = 1.0,
+) -> None:
+    while matching_process(owner_pid, owner_started_at) is not None:
+        time.sleep(max(0.2, interval))
+
+    supervisord = matching_process(supervisord_pid, supervisord_started_at)
+    if supervisord is None:
+        return
+
+    try:
+        children = [child for child in supervisord.children(recursive=True) if child.pid != os.getpid()]
+        supervisord.terminate()
+        for child in children:
+            try:
+                child.terminate()
+            except psutil.NoSuchProcess:
+                pass
+        _gone, alive = psutil.wait_procs([supervisord, *children], timeout=5)
+        for process in alive:
+            try:
+                process.kill()
+            except psutil.NoSuchProcess:
+                pass
+        psutil.wait_procs(alive, timeout=2)
+    except psutil.NoSuchProcess:
+        pass
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--owner-pid", type=int, required=True)
+    parser.add_argument("--owner-started-at", type=float, required=True)
+    parser.add_argument("--supervisord-pid", type=int, required=True)
+    parser.add_argument("--supervisord-started-at", type=float, required=True)
+    parser.add_argument("--interval", type=float, default=1.0)
+    run_watchdog(**vars(parser.parse_args()))
+
+
+if __name__ == "__main__":
+    main()

archivebox/workers/supervisord_util.py

@@ -234,19 +234,36 @@ RUNNER_WATCH_WORKER = lambda bind_url: {
     "redirect_stderr": "true",
 }
 
-SUPERVISORD_PARENT_WATCHDOG_WORKER = lambda supervisord_process_id: {
-    "name": "worker_supervisord_parent_watchdog",
-    "command": _shell_join(
-        archivebox_cmd("manage", "supervisord_watchdog", f"--supervisord-process-id={supervisord_process_id}"),
-    ),
-    "autostart": "false",
-    "autorestart": "false",
-    "stopasgroup": "true",
-    "killasgroup": "true",
-    "stopwaitsecs": "1",
-    "stdout_logfile": "logs/worker_supervisord_parent_watchdog.log",
-    "redirect_stderr": "true",
-}
+
+def SUPERVISORD_PARENT_WATCHDOG_WORKER(
+    *,
+    owner_pid: int,
+    owner_started_at: float,
+    supervisord_pid: int,
+    supervisord_started_at: float,
+):
+    watchdog_script = Path(__file__).with_name("supervisord_parent_watchdog.py")
+    return {
+        "name": "worker_supervisord_parent_watchdog",
+        "command": _shell_join(
+            [
+                sys.executable,
+                str(watchdog_script),
+                f"--owner-pid={owner_pid}",
+                f"--owner-started-at={owner_started_at}",
+                f"--supervisord-pid={supervisord_pid}",
+                f"--supervisord-started-at={supervisord_started_at}",
+            ],
+        ),
+        "autostart": "false",
+        "autorestart": "false",
+        "stopasgroup": "true",
+        "killasgroup": "true",
+        "stopwaitsecs": "1",
+        "stdout_logfile": "logs/worker_supervisord_parent_watchdog.log",
+        "redirect_stderr": "true",
+    }
+
 
 SERVER_WORKER = lambda host, port: {
     "name": "worker_daphne",
@@ -460,7 +477,7 @@ def create_worker_config(daemon):
     worker_conf.write_text(worker_str)
 
 
-def _current_foreground_supervisord_process_id():
+def _current_foreground_supervisord_watchdog_args():
     if not _supervisord_proc or _supervisord_proc.poll() is not None:
         return None
 
@@ -477,7 +494,14 @@ def _current_foreground_supervisord_process_id():
             parent=current,
         ).iterator(chunk_size=10):
             if process.is_running:
-                return process.id
+                owner = psutil.Process(current.pid)
+                supervisord = psutil.Process(process.pid)
+                return {
+                    "owner_pid": owner.pid,
+                    "owner_started_at": owner.create_time(),
+                    "supervisord_pid": supervisord.pid,
+                    "supervisord_started_at": supervisord.create_time(),
+                }
     except _PROCESS_STATE_ERRORS:
         return None
     return None
@@ -498,9 +522,9 @@ def sync_supervisord_workers(supervisor, workers: list[tuple[dict[str, str], boo
 
     global _desired_supervisord_workers
 
-    supervisord_process_id = _current_foreground_supervisord_process_id()
-    if supervisord_process_id is not None:
-        watchdog = SUPERVISORD_PARENT_WATCHDOG_WORKER(supervisord_process_id)
+    watchdog_args = _current_foreground_supervisord_watchdog_args()
+    if watchdog_args is not None:
+        watchdog = SUPERVISORD_PARENT_WATCHDOG_WORKER(**watchdog_args)
         if all(worker["name"] != watchdog["name"] for worker, _lazy in workers):
             workers = [*workers, (watchdog, False)]