From 68c53d59349c08cfae8f16c4e391d723aedebfa0 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 25 Jul 2026 21:32:09 -0700 Subject: [PATCH] Fix CI runtime binary resolution and docs install drift --- AGENTS.md | 2 +- archivebox/progressmonitor/views.py | 20 ++-- archivebox/workers/supervisord_util.py | 142 +++++++++++++------------ docs/codeblocks.toml | 12 +-- pyproject.toml | 4 +- skills/archivebox/SKILL.md | 2 +- uv.lock | 2 +- 7 files changed, 95 insertions(+), 89 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ba7d52d0..bb1c9258 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,7 @@ uv run --project .. archivebox run Recommended CLI install: ```bash -uv tool install --python 3.13 --prerelease allow archivebox +uv tool install --python 3.13 --prerelease allow --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev' mkdir -p ~/archivebox/data cd ~/archivebox/data archivebox init --install diff --git a/archivebox/progressmonitor/views.py b/archivebox/progressmonitor/views.py index d1d4c0ef..30753fbc 100644 --- a/archivebox/progressmonitor/views.py +++ b/archivebox/progressmonitor/views.py @@ -4,20 +4,20 @@ from functools import lru_cache from pathlib import Path from typing import Literal +from abx_dl.events import PROCESS_EXIT_SKIPPED from django.conf import settings +from django.db import DatabaseError from django.db.models import CharField, Count, Q, Sum from django.db.models.functions import Cast from django.http import HttpResponse, JsonResponse from django.utils import timezone -from abx_dl.events import PROCESS_EXIT_SKIPPED - from archivebox.config import CONSTANTS from archivebox.config.common import get_config -from archivebox.core.routes_util import build_snapshot_url, build_web_url, get_api_base_url from archivebox.core.permissions import can_view_snapshot, is_admin_user -from archivebox.plugins.discovery import discover_plugin_configs +from archivebox.core.routes_util import build_snapshot_url, build_web_url, get_api_base_url from archivebox.misc.logging_util import printable_filesize +from archivebox.plugins.discovery import discover_plugin_configs def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_id: object | None = None) -> str: @@ -42,9 +42,9 @@ def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]: def live_progress_view(request): """Simple JSON endpoint for live progress status - used by admin progress monitor.""" try: + from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl - from archivebox.core.models import Snapshot, ArchiveResult - from archivebox.machine.models import Process, Machine + from archivebox.machine.models import Machine, Process snapshot_id_filter = (request.GET.get("snapshot_id") or "").strip().replace("-", "") crawl_id_filter = (request.GET.get("crawl_id") or "").strip().replace("-", "") @@ -251,7 +251,7 @@ def live_progress_view(request): supervisor = get_existing_supervisord_process(quiet=True) runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None - except Exception: + except (OSError, RuntimeError, TimeoutError): runner_worker = None runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING")) @@ -517,14 +517,14 @@ def live_progress_view(request): def find_snapshot_for_process(proc_pwd: Path) -> Snapshot | None: for path_part in reversed(proc_pwd.parts): - snapshot = snapshots_by_id.get(path_part) + snapshot = snapshots_by_id.get(path_part.replace("-", "")) if snapshot: return snapshot return None def find_crawl_for_process(proc_pwd: Path) -> Crawl | None: for path_part in reversed(proc_pwd.parts): - crawl = crawls_by_id.get(path_part) + crawl = crawls_by_id.get(path_part.replace("-", "")) if crawl: return crawl return None @@ -938,7 +938,7 @@ def live_progress_view(request): return HttpResponse(ujson.dumps(payload), content_type="application/json") except ImportError: return JsonResponse(payload) - except Exception as e: + except (DatabaseError, OSError, RuntimeError, TypeError, ValueError) as e: error_payload = { "error": str(e), "orchestrator_running": False, diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py index 58e8e092..e07c8963 100644 --- a/archivebox/workers/supervisord_util.py +++ b/archivebox/workers/supervisord_util.py @@ -1,24 +1,25 @@ __package__ = "archivebox.workers" -import sys -import time -import socket -import os import csv import json -import psutil -import shutil -import subprocess +import os import shlex +import shutil import signal - -from typing import cast -from pathlib import Path +import socket +import subprocess +import sys +import time from functools import cache - -from supervisor.xmlrpc import SupervisorTransport +from pathlib import Path +from typing import cast +from xmlrpc.client import Error as XmlRpcError from xmlrpc.client import Fault, ServerProxy +import psutil +from django.db import DatabaseError +from supervisor.xmlrpc import SupervisorTransport + from archivebox.config import CONSTANTS from archivebox.config.common import rprint as print from archivebox.config.paths import SUPERVISORD_SOCKET_FILENAME, get_or_create_working_tmp_dir @@ -42,21 +43,25 @@ _supervisord_proc = None _desired_supervisord_workers: dict[str, dict[str, str]] = {} _ACTIVE_WORKER_STATES = {"STARTING", "RUNNING", "BACKOFF"} _RUNTIME_COMPONENT_ORDER = ("orchestrator", "server", "sonic") +_SUPERVISORD_ERRORS = (XmlRpcError, OSError, RuntimeError, TimeoutError) +_PROCESS_STATE_ERRORS = (DatabaseError, OSError, RuntimeError, ValueError, psutil.Error) def _shell_join(args: list[str]) -> str: return shlex.join(args) +def _warn_background_cleanup(context: str, err: BaseException) -> None: + STDERR.print(f"[yellow][!] {context}: {err!s}[/yellow]") + + def archivebox_cmd(*args: str) -> list[str]: - executable = Path(sys.argv[0]).resolve() if sys.argv and sys.argv[0] else None - if executable and executable.name == "archivebox" and executable.is_file() and os.access(executable, os.X_OK): - return [str(executable), *args] - return [sys.executable, "-m", "archivebox", *args] + return [str(resolve_env_binary("archivebox")), *args] def resolve_env_binary(name: str) -> Path: from abxpkg import EnvProvider + from archivebox.config.common import get_config lib_dir = Path(os.environ.get("ABXPKG_LIB_DIR") or get_config().ABXPKG_LIB_DIR) @@ -79,7 +84,9 @@ def resolve_env_binary(name: str) -> Path: def _record_supervisord_process(proc: subprocess.Popen, config_file: Path, supervisord_binary: Path) -> None: try: from datetime import datetime + from django.utils import timezone + from archivebox.machine.models import Machine, Process try: @@ -99,8 +106,8 @@ def _record_supervisord_process(proc: subprocess.Popen, config_file: Path, super status=Process.StatusChoices.RUNNING, timeout=CONSTANTS.MAX_HOOK_RUNTIME_SECONDS, ) - except Exception: - pass + except _PROCESS_STATE_ERRORS as err: + _warn_background_cleanup("Could not record supervisord process", err) def _fallback_supervisord_process_from_db(): @@ -117,7 +124,7 @@ def _fallback_supervisord_process_from_db(): if proc is not None: return proc process.mark_exited(exit_code=0) - except Exception: + except _PROCESS_STATE_ERRORS: return None return None @@ -149,7 +156,7 @@ def _live_supervisord_processes_from_db(): else: process.mark_exited(exit_code=0) return live - except Exception: + except _PROCESS_STATE_ERRORS: return [] @@ -461,7 +468,7 @@ def _current_foreground_supervisord_process_id(): ).iterator(chunk_size=10): if process.is_running: return process.id - except Exception: + except _PROCESS_STATE_ERRORS: return None return None @@ -503,14 +510,14 @@ def sync_supervisord_workers(supervisor, workers: list[tuple[dict[str, str], boo for group in removed: try: supervisor.stopProcessGroup(group) - except Exception: - pass + except _SUPERVISORD_ERRORS as err: + _warn_background_cleanup(f"Could not stop removed supervisord group {group}", err) supervisor.removeProcessGroup(group) for group in changed: try: supervisor.stopProcessGroup(group) - except Exception: - pass + except _SUPERVISORD_ERRORS as err: + _warn_background_cleanup(f"Could not stop changed supervisord group {group}", err) supervisor.removeProcessGroup(group) supervisor.addProcessGroup(group) for group in added: @@ -548,7 +555,7 @@ def sync_supervisord_workers(supervisor, workers: list[tuple[dict[str, str], boo procs_by_name[worker_name] = proc break else: - raise Exception(f"Failed to sync worker {worker_name}! Only found: {supervisor.getAllProcessInfo()}") + raise RuntimeError(f"Failed to sync worker {worker_name}! Only found: {supervisor.getAllProcessInfo()}") return procs_by_name @@ -575,11 +582,11 @@ def get_existing_supervisord_process(*, quiet: bool = False): print(f"[🦸‍♂️] Supervisord is already shutting down via unix://{pretty_path(SOCK_FILE)}.") return None if not quiet: - print(f"Error connecting to existing supervisord: {str(err)}") + print(f"Error connecting to existing supervisord: {err!s}") return None - except Exception as e: + except _SUPERVISORD_ERRORS as e: if not quiet: - print(f"Error connecting to existing supervisord: {str(e)}") + print(f"Error connecting to existing supervisord: {e!s}") return None @@ -598,7 +605,7 @@ class SupervisordConnectionCache: try: self.supervisor.getPID() return self.supervisor - except Exception: + except _SUPERVISORD_ERRORS: self.supervisor = None supervisor = get_existing_supervisord_process(quiet=self.quiet) @@ -648,7 +655,7 @@ def stop_existing_supervisord_process(): if supervisor is not None: try: supervisor_pid = supervisor.getPID() - except Exception: + except _SUPERVISORD_ERRORS: supervisor_pid = None # Ask supervisord to stop each worker first so child shutdown follows # each worker's own stopasgroup/killasgroup/stopwaitsecs settings. The @@ -664,9 +671,9 @@ def stop_existing_supervisord_process(): time.sleep(0.2) except Fault as err: if err.faultCode != 6 or "SHUTDOWN_STATE" not in str(err): - print(f"Error stopping supervisord workers: {str(err)}") - except Exception as err: - print(f"Error stopping supervisord workers: {str(err)}") + print(f"Error stopping supervisord workers: {err!s}") + except _SUPERVISORD_ERRORS as err: + print(f"Error stopping supervisord workers: {err!s}") try: supervisor.shutdown() @@ -675,8 +682,8 @@ def stop_existing_supervisord_process(): if err.faultCode == 6 and "SHUTDOWN_STATE" in str(err): supervisor_shutdown_requested = True else: - print(f"Error shutting down supervisord: {str(err)}") - except Exception: + print(f"Error shutting down supervisord: {err!s}") + except _SUPERVISORD_ERRORS: supervisor_shutdown_requested = True try: @@ -733,8 +740,8 @@ def stop_existing_supervisord_process(): # clear PID file and socket file PID_FILE.unlink(missing_ok=True) get_sock_file().unlink(missing_ok=True) - except BaseException: - pass + except OSError as err: + _warn_background_cleanup("Could not clear supervisord pid/socket files", err) def stop_own_supervisord_process(*, record_exit: bool = True): @@ -780,8 +787,8 @@ def stop_own_supervisord_process(*, record_exit: bool = True): pid=stopped_pid, ).iterator(chunk_size=10): process.mark_exited(exit_code=0) - except Exception: - pass + except _PROCESS_STATE_ERRORS as err: + _warn_background_cleanup("Could not mark supervisord process exited", err) except (BrokenPipeError, OSError, psutil.TimeoutExpired): pass finally: @@ -791,8 +798,8 @@ def stop_own_supervisord_process(*, record_exit: bool = True): if PID_FILE.exists() and PID_FILE.read_text().strip() == str(stopped_pid): PID_FILE.unlink(missing_ok=True) SOCK_FILE.unlink(missing_ok=True) - except Exception: - pass + except (OSError, RuntimeError, ValueError) as err: + _warn_background_cleanup("Could not clear owned supervisord pid/socket files", err) _supervisord_proc = None return True @@ -835,24 +842,24 @@ def start_new_supervisord_process(daemonize=False): # Open log file for supervisord output LOG_FILE.parent.mkdir(parents=True, exist_ok=True) - log_handle = open(LOG_FILE, "a") supervisord_binary = resolve_env_binary("supervisord") - if daemonize: - # Start supervisord in background (daemon mode) - proc = subprocess.Popen( - [str(supervisord_binary), f"--configuration={CONFIG_FILE}"], - stdin=None, - stdout=log_handle, - stderr=log_handle, - start_new_session=True, - ) - current_started_at = psutil.Process(proc.pid).create_time() - _record_supervisord_process(proc, CONFIG_FILE, supervisord_binary) - supervisor = wait_for_supervisord_ready() - _stop_older_supervisord_processes(current_pid=proc.pid, current_started_at=current_started_at, timeout=stop_grace_seconds) - return supervisor - else: + with open(LOG_FILE, "a") as log_handle: + if daemonize: + # Start supervisord in background (daemon mode) + proc = subprocess.Popen( + [str(supervisord_binary), f"--configuration={CONFIG_FILE}"], + stdin=None, + stdout=log_handle, + stderr=log_handle, + start_new_session=True, + ) + current_started_at = psutil.Process(proc.pid).create_time() + _record_supervisord_process(proc, CONFIG_FILE, supervisord_binary) + supervisor = wait_for_supervisord_ready() + _stop_older_supervisord_processes(current_pid=proc.pid, current_started_at=current_started_at, timeout=stop_grace_seconds) + return supervisor + # Keep supervisord foreground-owned by this process, but isolate it # from terminal Ctrl+C. The ArchiveBox parent owns user-facing server # signals and stops supervisord explicitly, so Ctrl+C does not also @@ -997,8 +1004,8 @@ def run_runner_worker( def get_worker(supervisor, daemon_name): try: return supervisor.getProcessInfo(daemon_name) - except Exception: - pass + except _SUPERVISORD_ERRORS as err: + _warn_background_cleanup(f"Could not get supervisord worker {daemon_name}", err) return None @@ -1029,7 +1036,7 @@ def active_supervisord_runtime_components(*, config=None, supervisor=None) -> li return [] try: worker_names = {proc.get("name") for proc in supervisor.getAllProcessInfo() if proc.get("statename") in _ACTIVE_WORKER_STATES} - except Exception: + except _SUPERVISORD_ERRORS: return [] return runtime_components_for_worker_names({str(name) for name in worker_names if name}, config=config) @@ -1055,7 +1062,7 @@ def build_server_worker_plan(*, config, host: str, port: str, debug: bool, reloa try: current_sonic = get_worker(supervisor, sonic_worker["name"]) if supervisor is not None else None supervisor_pid = supervisor.getPID() if supervisor is not None else None - except Exception: + except _SUPERVISORD_ERRORS: current_sonic = None supervisor_pid = None sonic_host = str(config.SEARCH_BACKEND_SONIC_HOST_NAME or "127.0.0.1") @@ -1099,7 +1106,7 @@ def stop_worker(supervisor, daemon_name): time.sleep(0.5) proc = get_worker(supervisor, daemon_name) - raise Exception(f"Failed to stop worker {daemon_name}!") + raise RuntimeError(f"Failed to stop worker {daemon_name}!") def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep_running=None): @@ -1131,13 +1138,13 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep file_handles = [] for log_path in log_paths: try: - f = open(log_path) + f = log_path.open() # Seek to end - only show NEW logs from now on, not old logs f.seek(0, 2) # Go to end file_handles.append((log_path, f)) print(f" [tailing {log_path.name}]") - except Exception as e: + except OSError as e: sys.stderr.write(f"Warning: Could not open {log_path}: {e}\n") if not file_handles: @@ -1183,8 +1190,8 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep for _, f in file_handles: try: f.close() - except Exception: - pass + except OSError as err: + _warn_background_cleanup("Could not close worker log file", err) return "stopped" @@ -1396,7 +1403,6 @@ def start_server_workers( except SystemExit: if daemonize: raise - pass except BaseException as e: if daemonize: raise diff --git a/docs/codeblocks.toml b/docs/codeblocks.toml index 588691fc..a0b20799 100644 --- a/docs/codeblocks.toml +++ b/docs/codeblocks.toml @@ -52,7 +52,7 @@ version = 2 # AGENTS.md "2695777473e3a217-1" = "run" "16f996904f0eaa96-1" = "run" -"93dd34b605cd6775-1" = "run" +"b894a7c4f93e4e92-1" = "run" "6a6234b63cd3d0ae-1" = "run" "396490664ce459e9-1" = "run" @@ -67,7 +67,7 @@ version = 2 # skills/archivebox/SKILL.md "2695777473e3a217-2" = "run" -"93dd34b605cd6775-2" = "run" +"b894a7c4f93e4e92-2" = "run" "46fe9d81940ea385-1" = "run" "396490664ce459e9-2" = "run" @@ -267,7 +267,7 @@ version = 2 "fab7ffa7e26bc3e5-1" = "collection" "2695777473e3a217-1" = "project" "16f996904f0eaa96-1" = "project" -"93dd34b605cd6775-1" = "system" +"b894a7c4f93e4e92-1" = "system" "6a6234b63cd3d0ae-1" = "collection" "396490664ce459e9-1" = "project" "d9ff84abeb97d380-1" = "collection" @@ -275,7 +275,7 @@ version = 2 "399c370c70523428-1" = "collection" "7d33d72cc96df65b-1" = "collection" "2695777473e3a217-2" = "project" -"93dd34b605cd6775-2" = "system" +"b894a7c4f93e4e92-2" = "system" "46fe9d81940ea385-1" = "collection" "396490664ce459e9-2" = "project" "201988374bff6d14-1" = "collection" @@ -348,7 +348,7 @@ version = 2 "fab7ffa7e26bc3e5-1" = "ubuntu" "2695777473e3a217-1" = "ubuntu" "16f996904f0eaa96-1" = "ubuntu" -"93dd34b605cd6775-1" = "ubuntu" +"b894a7c4f93e4e92-1" = "ubuntu" "6a6234b63cd3d0ae-1" = "ubuntu" "396490664ce459e9-1" = "ubuntu" "d9ff84abeb97d380-1" = "ubuntu" @@ -356,7 +356,7 @@ version = 2 "399c370c70523428-1" = "ubuntu" "7d33d72cc96df65b-1" = "ubuntu" "2695777473e3a217-2" = "ubuntu" -"93dd34b605cd6775-2" = "ubuntu" +"b894a7c4f93e4e92-2" = "ubuntu" "46fe9d81940ea385-1" = "ubuntu" "396490664ce459e9-2" = "ubuntu" "201988374bff6d14-1" = "ubuntu" diff --git a/pyproject.toml b/pyproject.toml index 541cf210..7022bc85 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc140" +version = "0.9.35rc141" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -348,7 +348,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc138" +current_version = "v0.9.35rc141" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/skills/archivebox/SKILL.md b/skills/archivebox/SKILL.md index 5d8bddc0..31aefb19 100644 --- a/skills/archivebox/SKILL.md +++ b/skills/archivebox/SKILL.md @@ -33,7 +33,7 @@ uv run --project .. archivebox init --install Recommended CLI install: ```bash -uv tool install --python 3.13 --prerelease allow archivebox +uv tool install --python 3.13 --prerelease allow --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev' mkdir -p ~/archivebox/data cd ~/archivebox/data archivebox init --install diff --git a/uv.lock b/uv.lock index dc3a6500..4d1bd364 100644 --- a/uv.lock +++ b/uv.lock @@ -123,7 +123,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc140" +version = "0.9.35rc141" source = { editable = "." } dependencies = [ { name = "abx-dl" },