From 52b37d48bf43aaf8268efd6bdebbbb9be913f437 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 28 May 2026 14:32:28 -0700 Subject: [PATCH] release: archivebox 0.9.33rc26 --- archivebox/cli/archivebox_run.py | 3 +- archivebox/cli/archivebox_server.py | 25 +++++++-- archivebox/core/views.py | 39 ++++++-------- archivebox/machine/models.py | 7 +-- archivebox/misc/db.py | 8 ++- archivebox/services/runner.py | 44 ++++++--------- archivebox/services/supervision_service.py | 11 ++-- .../templates/admin/progress_monitor.html | 54 +++++++++---------- archivebox/workers/supervisord_util.py | 5 +- etc/package.json | 2 +- pyproject.toml | 2 +- 11 files changed, 100 insertions(+), 100 deletions(-) diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index 8aef2193..7ff05752 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -42,6 +42,7 @@ __command__ = "archivebox run" import os import sys +import asyncio from collections import defaultdict import rich_click as click @@ -266,7 +267,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon): run_pending_crawls(daemon=daemon, crawl_id=crawl_id, maintenance_only=maintenance_only) return 0 - except KeyboardInterrupt: + except (KeyboardInterrupt, asyncio.CancelledError): return 0 except Exception as e: rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr) diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 537f389d..d8a351c8 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -22,7 +22,7 @@ def server( nothreading: bool = False, ) -> None: """Run the ArchiveBox HTTP server""" - from archivebox.config.common import get_config + from archivebox.config.common import get_config, rprint config = get_config() runserver_args = list(runserver_args or (config.BIND_ADDR,)) @@ -76,6 +76,7 @@ def server( from archivebox.services.supervision_service import ( command_owns_runtime_stack, current_command, + runtime_stack_owner, standby_until_runtime_stack_needed, ) from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals @@ -95,6 +96,14 @@ def server( bind_url = f"http://{host}:{port}" command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url) + def still_owns_runtime_stack() -> bool: + from django.db import connections + + try: + return command_owns_runtime_stack(command, data_dir=config.DATA_DIR) + finally: + connections.close_all() + try: with foreground_shutdown_signals(), foreground_parent_watchdog(): while True: @@ -114,13 +123,19 @@ def server( debug=run_in_debug, reload=reload, nothreading=nothreading, - keep_running=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR), - should_stop_supervisord=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR), + keep_running=still_owns_runtime_stack, + should_stop_supervisord=still_owns_runtime_stack, ) if result == "interrupted": break - if not command_owns_runtime_stack(command, data_dir=config.DATA_DIR): - print("[yellow][*] Another ArchiveBox command took over the runtime stack; standing by.[/yellow]") + if not still_owns_runtime_stack(): + owner = runtime_stack_owner(data_dir=config.DATA_DIR) + owner_pid = owner.pid if owner else "unknown" + rprint( + "[yellow][*] A newer archivebox process took over the runner " + f"(pid={owner_pid}). Work will continue there, and will continue here if the other process is stopped and work still remains.[/yellow]", + file=sys.stderr, + ) continue if result == "exited": print("[yellow][*] Runtime stack exited while this parent is still leader; restarting...[/yellow]") diff --git a/archivebox/core/views.py b/archivebox/core/views.py index fd56ce75..c35798df 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1560,6 +1560,17 @@ def live_progress_view(request): url = str(url or "") return url if len(url) <= 96 else f"{url[:93]}..." + def screencast_frame_url(object_id: str) -> str: + frame_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / object_id / "latest.jpg" + try: + frame_stat = frame_path.stat() + except OSError: + return "" + if frame_stat.st_size <= 0: + return "" + token = SCREENCAST_SIGNER.sign(object_id) + return f"/admin/live-progress/screencast/{object_id}.jpg?v={frame_stat.st_mtime_ns}&token={quote(token)}" + machine_id = Machine.current().id orchestrator_proc = ( Process.objects.filter( @@ -1971,19 +1982,8 @@ def live_progress_view(request): crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded") crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed") crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued") - crawl_screencast_url = "" - crawl_screencast_link = "" - live_crawl_preview_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / crawl_id / "latest.jpg" - try: - live_crawl_preview_stat = live_crawl_preview_path.stat() - except OSError: - live_crawl_preview_stat = None - if live_crawl_preview_stat and live_crawl_preview_stat.st_size > 0: - token = SCREENCAST_SIGNER.sign(crawl_id) - crawl_screencast_url = ( - f"/admin/live-progress/screencast/{crawl_id}.jpg?v={live_crawl_preview_stat.st_mtime_ns}&token={quote(token)}" - ) - crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id}/change/" + crawl_screencast_url = screencast_frame_url(crawl_id) + crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id}/change/" if crawl_screencast_url else "" # Get active snapshots for this crawl (already prefetched) active_snapshots_for_crawl = [] @@ -2034,17 +2034,8 @@ def live_progress_view(request): snapshot_preview_url = snapshot_favicon_url if snapshot["status"] == Snapshot.StatusChoices.STARTED: - live_preview_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / str(snapshot["id"]) / "latest.jpg" - try: - live_preview_stat = live_preview_path.stat() - except OSError: - live_preview_stat = None - if live_preview_stat and live_preview_stat.st_size > 0: - token = SCREENCAST_SIGNER.sign(str(snapshot["id"])) - snapshot_screencast_url = ( - f"/admin/live-progress/screencast/{snapshot['id']}.jpg?v={live_preview_stat.st_mtime_ns}&token={quote(token)}" - ) - snapshot_screencast_link = snapshot_view_url(snapshot) + snapshot_screencast_url = screencast_frame_url(str(snapshot["id"])) + snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else "" def plugin_sort_key(ar): status_order = { diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index d6e8dce1..a2363eae 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -812,8 +812,10 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): print(f"Failed to create LIB_BIN_DIR {lib_bin_dir}: {e}", file=sys.stderr) return None - # Get binary name (last component of path) - binary_name = binary_abspath.name + # Expose the canonical Binary.name in LIB_BIN_DIR. Some providers point + # abspath at implementation files like cli.js or manifest.json; those + # are valid targets, but they are not user-facing binary names. + binary_name = _canonical_binary_name(self.name) or binary_abspath.name symlink_path = lib_bin_dir / binary_name if app_index != -1 and len(binary_parts) > app_index + 2 and binary_parts[app_index + 1 : app_index + 3] == ("Contents", "MacOS"): @@ -842,7 +844,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): # Create new symlink try: symlink_path.symlink_to(binary_abspath) - print(f"Symlinked {binary_name} -> {symlink_path}", file=sys.stderr) return symlink_path except (OSError, PermissionError) as e: print(f"Failed to create symlink {symlink_path} -> {binary_abspath}: {e}", file=sys.stderr) diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index 1803511b..7221d032 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -34,6 +34,12 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str] import psutil db_path = db_path.resolve() + db_sidecars = { + db_path, + db_path.with_name(f"{db_path.name}-wal"), + db_path.with_name(f"{db_path.name}-shm"), + db_path.with_name(f"{db_path.name}-journal"), + } holders: list[str] = [] for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline", "status"]): try: @@ -45,7 +51,7 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str] open_path = Path(open_file.path).resolve() except (OSError, RuntimeError): continue - if open_path == db_path or open_path.name in {f"{db_path.name}-wal", f"{db_path.name}-shm", f"{db_path.name}-journal"}: + if open_path in db_sidecars: info = proc.info cmdline = compact_command(info.get("cmdline"), fallback=info.get("name") or "") holders.append(f"pid={info['pid']} ppid={info['ppid']} {info['status']} {cmdline}") diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 80838115..d9cd382b 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -231,49 +231,30 @@ class CrawlRunner: if threading.current_thread() is not threading.main_thread(): return [] - loop = asyncio.get_running_loop() installed: list[tuple[signal.Signals, Any, bool]] = [] for sig in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): previous = signal.getsignal(sig) - def request_abort(sig=sig) -> None: + def request_abort(_signum, _frame, sig=sig) -> None: + os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name}, stopping gracefully...\n".encode()) self._request_abort_from_signal(sig) + raise KeyboardInterrupt - try: - loop.add_signal_handler(sig, request_abort) - installed.append((sig, previous, True)) - except (NotImplementedError, RuntimeError): - signal.signal(sig, lambda _signum, _frame, sig=sig: self._request_abort_from_signal(sig)) - installed.append((sig, previous, False)) + signal.signal(sig, request_abort) + installed.append((sig, previous, False)) return installed def _restore_signal_handlers(self, installed: list[tuple[signal.Signals, Any, bool]]) -> None: - loop = asyncio.get_running_loop() for sig, previous, installed_on_loop in reversed(installed): if installed_on_loop: - loop.remove_signal_handler(sig) + asyncio.get_running_loop().remove_signal_handler(sig) signal.signal(sig, previous) - def _request_abort_from_signal(self, sig: signal.Signals) -> None: - if self._signal_abort_requested: - if self._run_task is not None and not self._run_task.done(): - self._run_task.cancel() - return + def _request_abort_from_signal(self, _sig: signal.Signals) -> None: self._signal_abort_requested = True self._skip_wait_until_idle = True - asyncio.create_task(self.abort_from_signal(sig.name)) - - async def abort_from_signal(self, signal_name: str) -> None: - from archivebox.crawls.models import Crawl - - await sync_to_async( - Crawl.objects.filter(id=self.crawl.id).exclude(status=Crawl.StatusChoices.SEALED).update, - thread_sensitive=False, - )( - status=Crawl.StatusChoices.STARTED, - retry_at=timezone.now(), - modified_at=timezone.now(), - ) + if self._run_task is not None and not self._run_task.done(): + self._run_task.cancel() async def crawl_is_cancelled(self) -> bool: from archivebox.crawls.models import Crawl @@ -380,7 +361,12 @@ class CrawlRunner: async def stop_snapshot_tasks(self) -> None: if not self.snapshot_tasks: return - done, pending = await asyncio.wait(list(self.snapshot_tasks.values()), timeout=5.0) + tasks = list(self.snapshot_tasks.values()) + if self._signal_abort_requested: + done = {task for task in tasks if task.done()} + pending = set(tasks) - done + else: + done, pending = await asyncio.wait(tasks, timeout=5.0) for task in pending: task.cancel() await asyncio.gather(*done, *pending, return_exceptions=True) diff --git a/archivebox/services/supervision_service.py b/archivebox/services/supervision_service.py index 6ae2f8e4..93ee2086 100644 --- a/archivebox/services/supervision_service.py +++ b/archivebox/services/supervision_service.py @@ -1,6 +1,7 @@ from __future__ import annotations import time +import sys from pathlib import Path from django.utils import timezone @@ -29,7 +30,6 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None): from archivebox.machine.models import Machine, Process - Process.cleanup_stale_running(machine=Machine.current()) qs = Process.objects.filter( machine=Machine.current(), process_type=process_type, @@ -54,7 +54,6 @@ def command_is_newest(command, *, process_type: str, data_dir: str | Path, url: def runtime_stack_owner(*, data_dir: str | Path): from archivebox.machine.models import Machine, Process - Process.cleanup_stale_running(machine=Machine.current()) base_qs = Process.objects.filter( machine=Machine.current(), status=Process.StatusChoices.RUNNING, @@ -117,7 +116,6 @@ def healthy_orchestrator(*, data_dir: str | Path): from archivebox.machine.models import Machine, Process from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker - Process.cleanup_stale_running(machine=Machine.current()) supervisor = get_existing_supervisord_process() worker = get_worker(supervisor, "worker_runner") if supervisor else None if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"): @@ -145,7 +143,6 @@ def standby_until_leader_needed(command, *, process_type: str, data_dir: str | P leader_pid = leader.pid if leader else "unknown" rprint(f"[yellow][*] Standing by; newer ArchiveBox parent pid={leader_pid} is running the orchestrator and server.[/yellow]") announced = True - command.heartbeat() time.sleep(interval) command.modified_at = timezone.now() command.save(update_fields=["modified_at"]) @@ -161,11 +158,11 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva owner = runtime_stack_owner(data_dir=data_dir) owner_pid = owner.pid if owner else "unknown" rprint( - "[yellow][*] Runner is now owned by newer archivebox process " - f"pid={owner_pid}, processing will continue there and will resume here if the other process is stopped and work still remains[/yellow]", + "[yellow][*] A newer archivebox process took over the runner " + f"(pid={owner_pid}). Work will continue there, and will continue here if the other process is stopped and work still remains.[/yellow]", + file=sys.stderr, ) announced = True - command.heartbeat() time.sleep(interval) command.modified_at = timezone.now() command.save(update_fields=["modified_at"]) diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html index cf8fc2de..c012f3a7 100644 --- a/archivebox/templates/admin/progress_monitor.html +++ b/archivebox/templates/admin/progress_monitor.html @@ -1190,12 +1190,9 @@ }); } - function setupHookMatches(item, pattern) { - const text = `${item.plugin || ''} ${item.hook_name || ''} ${item.label || ''}`.toLowerCase(); - return text.includes(pattern); - } - function activeScreencastTarget(data) { + const hookText = (item) => `${item.plugin || ''} ${item.hook_name || ''} ${item.label || ''}`.toLowerCase(); + let fallback = null; for (const crawl of data.active_crawls || []) { for (const snapshot of crawl.active_snapshots || []) { if (!Array.isArray(snapshot) && snapshot.screencast_url) { @@ -1218,12 +1215,12 @@ }, }; } - } - for (const crawl of data.active_crawls || []) { + + if (fallback) continue; + const setupPlugins = crawl.setup_plugins || []; - const startedSetup = setupPlugins.filter((item) => item.status === 'started'); - if (startedSetup.some((item) => setupHookMatches(item, 'chrome launch'))) { - return { + if (setupPlugins.some((item) => item.status === 'started' && hookText(item).includes('chrome launch'))) { + fallback = { type: 'placeholder', crawl, state: 'launching', @@ -1231,24 +1228,27 @@ title: 'Launching browser', subtitle: 'Chrome is starting and publishing its CDP session.', }; + continue; } - for (const snapshot of crawl.active_snapshots || []) { - if (Array.isArray(snapshot)) continue; - const startedSnapshotPlugins = (snapshot.all_plugins || []).filter((item) => item.status === 'started'); - if (startedSnapshotPlugins.some((item) => setupHookMatches(item, 'chrome tab'))) { - return { - type: 'placeholder', - crawl, - snapshot, - state: 'launching', - icon: '▣', - title: 'Opening browser tab', - subtitle: 'The snapshot tab is being attached for capture.', - }; - } + const openingSnapshot = (crawl.active_snapshots || []).find((snapshot) => { + return !Array.isArray(snapshot) && (snapshot.all_plugins || []).some((item) => ( + item.status === 'started' && hookText(item).includes('chrome tab') + )); + }); + if (openingSnapshot) { + fallback = { + type: 'placeholder', + crawl, + snapshot: openingSnapshot, + state: 'launching', + icon: '▣', + title: 'Opening browser tab', + subtitle: 'The snapshot tab is being attached for capture.', + }; + continue; } - if ((crawl.status === 'queued' || crawl.status === 'started') && !setupPlugins.some((item) => setupHookMatches(item, 'chrome launch'))) { - return { + if ((crawl.status === 'queued' || crawl.status === 'started') && !setupPlugins.some((item) => hookText(item).includes('chrome launch'))) { + fallback = { type: 'placeholder', crawl, state: 'not-launched', @@ -1258,7 +1258,7 @@ }; } } - return null; + return fallback; } function updateScreencastPanel(data) { diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py index ecc683cf..68ca6f38 100644 --- a/archivebox/workers/supervisord_util.py +++ b/archivebox/workers/supervisord_util.py @@ -422,7 +422,7 @@ def stop_existing_supervisord_process(): PID_FILE = SOCK_FILE.parent / PID_FILE_NAME stop_grace_seconds = configured_stopwaitsecs(tuple(_desired_supervisord_workers.values())) - supervisor = get_existing_supervisord_process() + supervisor = get_existing_supervisord_process(quiet=True) supervisor_pid = None supervisor_shutdown_requested = False if supervisor is not None: @@ -955,6 +955,9 @@ def start_server_workers( if daemonize: return None + from django.db import connections + + connections.close_all() try: with foreground_shutdown_signals() as shutdown_state: # Tail worker logs while supervisord runs. diff --git a/etc/package.json b/etc/package.json index f01f2861..8a6a14ca 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.33rc25", + "version": "0.9.33rc26", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 107c8ee6..479e0794 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.33rc25" +version = "0.9.33rc26" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]