From 4124f78bdca858ed080dfc6155438996eb83e930 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 27 May 2026 15:11:40 -0700 Subject: [PATCH] release: archivebox 0.9.32rc34 --- archivebox/cli/archivebox_run.py | 8 +-- archivebox/cli/archivebox_server.py | 4 -- archivebox/core/apps.py | 4 +- .../archivebox/api/migrations/__init__.py | 0 .../archivebox/crawls/migrations/__init__.py | 0 .../archivebox/machine/migrations/__init__.py | 0 archivebox/core/views.py | 52 ++++++++----------- archivebox/services/runner.py | 28 +++++++--- archivebox/tests/test_admin_views.py | 25 +++------ archivebox/tests/test_cli_server.py | 5 +- .../management/commands/runner_watch.py | 2 - etc/package.json | 2 +- pyproject.toml | 17 ++++-- 13 files changed, 67 insertions(+), 80 deletions(-) delete mode 100644 archivebox/core/migrations/archivebox/api/migrations/__init__.py delete mode 100644 archivebox/core/migrations/archivebox/crawls/migrations/__init__.py delete mode 100644 archivebox/core/migrations/archivebox/machine/migrations/__init__.py diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index e17f5709..5d0b11ad 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -255,13 +255,9 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int: """ from django.utils import timezone from archivebox.machine.models import Machine, Process - from archivebox.services.runner import recover_orphaned_crawls, recover_orphaned_snapshots, run_pending_crawls + from archivebox.services.runner import cleanup_orchestrator_state, run_pending_crawls - Process.cleanup_stale_running() - Process.cleanup_orphaned_workers() - Process.cleanup_orphaned_chrome() - recover_orphaned_snapshots() - recover_orphaned_crawls() + cleanup_orchestrator_state(include_chrome=True) Machine.current() current = Process.current() if current.process_type != Process.TypeChoices.ORCHESTRATOR: diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index d0a1743e..57abfc49 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -15,9 +15,6 @@ from archivebox.config.common import get_config def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int: """Stop any existing orchestrator process so the server can take ownership.""" - process_model.cleanup_stale_running(machine=machine) - process_model.cleanup_orphaned_workers() - running_runners = list( process_model.objects.filter( machine=machine, @@ -47,7 +44,6 @@ def stop_existing_background_runner(*, machine, process_model, supervisor=None, except Exception: pass - process_model.cleanup_stale_running(machine=machine) return len(running_runners) diff --git a/archivebox/core/apps.py b/archivebox/core/apps.py index 701af2c6..9316377c 100644 --- a/archivebox/core/apps.py +++ b/archivebox/core/apps.py @@ -45,8 +45,6 @@ class CoreConfig(AppConfig): return False if _should_prepare_runtime(): - from archivebox.machine.models import Process, Machine + from archivebox.machine.models import Machine - Process.cleanup_stale_running() - Process.cleanup_orphaned_workers() Machine.current() diff --git a/archivebox/core/migrations/archivebox/api/migrations/__init__.py b/archivebox/core/migrations/archivebox/api/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/core/migrations/archivebox/crawls/migrations/__init__.py b/archivebox/core/migrations/archivebox/crawls/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/core/migrations/archivebox/machine/migrations/__init__.py b/archivebox/core/migrations/archivebox/machine/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 95aab31b..1c7a4522 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1523,13 +1523,13 @@ def live_progress_view(request): if status == Crawl.StatusChoices.SEALED: status_qs = status_qs.filter(modified_at__gte=recently_cancelled_after) active_crawl_candidates.extend( - status_qs.values(*active_crawl_fields).order_by("-modified_at")[:10], + status_qs.values(*active_crawl_fields).order_by("-modified_at"), ) active_crawls_list = sorted( {str(crawl["id"]): crawl for crawl in active_crawl_candidates}.values(), key=lambda crawl: crawl["modified_at"], reverse=True, - )[:10] + ) persona_details_by_id: dict[str, dict[str, str]] = {} persona_details_by_name: dict[str, dict[str, str]] = {} persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]} @@ -1573,36 +1573,12 @@ def live_progress_view(request): ): crawl_output_sizes_by_crawl[str(row["snapshot__crawl_id"])] = int(row["total_size"] or 0) - if machine_id is not None: - running_processes = Process.objects.filter( - machine_id=machine_id, - status=Process.StatusChoices.RUNNING, - process_type__in=[ - Process.TypeChoices.HOOK, - Process.TypeChoices.BINARY, - ], - ).values("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "modified_at") - recent_processes = ( - Process.objects.filter( - machine_id=machine_id, - process_type__in=[ - Process.TypeChoices.HOOK, - Process.TypeChoices.BINARY, - ], - modified_at__gte=now - timedelta(minutes=10), - ) - .values("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "modified_at") - .order_by("-modified_at") - ) - else: - running_processes = Process.objects.none() - recent_processes = Process.objects.none() crawl_process_pids: dict[str, int] = {} snapshot_process_pids: dict[str, int] = {} process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {} process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {} seen_process_records: set[str] = set() - active_snapshot_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED} + active_snapshot_statuses = {Snapshot.StatusChoices.STARTED} recently_cancelled_snapshots_q = Q( status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True, @@ -1624,17 +1600,33 @@ def live_progress_view(request): "fs_version", "status", ) - .order_by("crawl_id", "status", "modified_at")[:100], + .order_by("crawl_id", "status", "modified_at"), ) snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots} displayed_snapshots_by_crawl: dict[str, list[Snapshot]] = {str(crawl_id): [] for crawl_id in active_crawl_ids} for snapshot in snapshots: crawl_snapshots = displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), []) - if len(crawl_snapshots) < 5: - crawl_snapshots.append(snapshot) + crawl_snapshots.append(snapshot) displayed_snapshot_ids = [ snapshot["id"] for crawl_snapshots in displayed_snapshots_by_crawl.values() for snapshot in crawl_snapshots ] + process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "modified_at") + if active_crawl_ids or displayed_snapshot_ids: + process_scope = Process.objects.filter( + machine_id=machine_id, + process_type__in=[ + Process.TypeChoices.HOOK, + Process.TypeChoices.BINARY, + ], + ) + running_processes = process_scope.filter(status=Process.StatusChoices.RUNNING).values(*process_value_fields) + recent_processes = ( + process_scope.filter(modified_at__gte=now - timedelta(minutes=10)).values(*process_value_fields).order_by("-modified_at") + ) + else: + running_processes = Process.objects.none() + recent_processes = Process.objects.none() + archiveresults_by_snapshot: dict[str, list[ArchiveResult]] = {str(snapshot_id): [] for snapshot_id in displayed_snapshot_ids} if displayed_snapshot_ids: displayed_archiveresults = ( diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 87312838..e5bc2f55 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -146,14 +146,13 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool: if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"): return False - Process.cleanup_stale_running() - Process.cleanup_orphaned_workers() machine = Machine.current() - if Process.objects.filter( + running_orchestrators = Process.objects.filter( machine=machine, status=Process.StatusChoices.RUNNING, process_type=Process.TypeChoices.ORCHESTRATOR, - ).exists(): + ) + if any(proc.is_running for proc in running_orchestrators): return False log_path = CONSTANTS.LOGS_DIR / "errors.log" @@ -1233,6 +1232,22 @@ def recover_orphaned_snapshots() -> int: return recovered +def cleanup_orchestrator_state(*, recover: bool = True, include_chrome: bool = False) -> dict[str, int]: + from archivebox.machine.models import Process + + cleaned = { + "stale_processes": Process.cleanup_stale_running(), + "orphaned_processes": Process.cleanup_orphaned_workers(), + "orphaned_chrome": Process.cleanup_orphaned_chrome() if include_chrome else 0, + "orphaned_snapshots": 0, + "orphaned_crawls": 0, + } + if recover: + cleaned["orphaned_snapshots"] = recover_orphaned_snapshots() + cleaned["orphaned_crawls"] = recover_orphaned_crawls() + return cleaned + + def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> int: from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.core.models import ArchiveResult, Snapshot @@ -1244,10 +1259,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> now_monotonic = time.monotonic() if daemon: if now_monotonic - last_recovery_at >= 30.0: - Process.cleanup_stale_running() - Process.cleanup_orphaned_workers() - recover_orphaned_snapshots() - recover_orphaned_crawls() + cleanup_orchestrator_state() last_recovery_at = now_monotonic if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0): for model in (ArchiveResult, Snapshot, Crawl, Process): diff --git a/archivebox/tests/test_admin_views.py b/archivebox/tests/test_admin_views.py index b1423065..ce531865 100644 --- a/archivebox/tests/test_admin_views.py +++ b/archivebox/tests/test_admin_views.py @@ -14,7 +14,6 @@ import uuid from pathlib import Path from types import SimpleNamespace from typing import cast -from unittest.mock import patch from django.test import override_settings from django.test.client import RequestFactory from django.urls import reverse @@ -1291,7 +1290,7 @@ class TestLiveProgressView: assert payload["active_crawls"] == [] assert payload["total_workers"] == 0 - def test_live_progress_cleans_stale_running_processes(self, client, admin_user, db): + def test_live_progress_does_not_clean_stale_running_processes(self, client, admin_user, db): from datetime import timedelta import archivebox.machine.models as machine_models from archivebox.machine.models import Machine, Process @@ -1313,8 +1312,8 @@ class TestLiveProgressView: assert response.status_code == 200 proc.refresh_from_db() - assert proc.status == Process.StatusChoices.EXITED - assert proc.ended_at is not None + assert proc.status == Process.StatusChoices.RUNNING + assert proc.ended_at is None assert response.json()["total_workers"] == 0 def test_live_progress_routes_crawl_process_rows_to_crawl_setup(self, client, admin_user, snapshot, db): @@ -1336,11 +1335,7 @@ class TestLiveProgressView: ) client.login(username="testadmin", password="testpassword") - with ( - patch.object(Process, "cleanup_stale_running", return_value=0), - patch.object(Process, "cleanup_orphaned_workers", return_value=0), - ): - response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST) + response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST) assert response.status_code == 200 payload = response.json() @@ -1371,11 +1366,7 @@ class TestLiveProgressView: ) client.login(username="testadmin", password="testpassword") - with ( - patch.object(Process, "cleanup_stale_running", return_value=0), - patch.object(Process, "cleanup_orphaned_workers", return_value=0), - ): - response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST) + response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST) assert response.status_code == 200 payload = response.json() @@ -1410,11 +1401,7 @@ class TestLiveProgressView: ) client.login(username="testadmin", password="testpassword") - with ( - patch.object(Process, "cleanup_stale_running", return_value=0), - patch.object(Process, "cleanup_orphaned_workers", return_value=0), - ): - response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST) + response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST) assert response.status_code == 200 payload = response.json() diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index 7faa51ea..99ce4152 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -165,7 +165,7 @@ def test_sonic_daemon_event_handler_requires_running_supervised_worker(monkeypat asyncio.run(run_test()) -def test_stop_existing_background_runner_cleans_up_and_stops_orchestrators(): +def test_stop_existing_background_runner_stops_orchestrators_without_row_healing(): from archivebox.cli.archivebox_server import stop_existing_background_runner runner_a = Mock() @@ -195,7 +195,8 @@ def test_stop_existing_background_runner_cleans_up_and_stops_orchestrators(): ) assert stopped == 2 - assert process_model.cleanup_stale_running.call_count == 2 + process_model.cleanup_stale_running.assert_not_called() + process_model.cleanup_orphaned_workers.assert_not_called() stop_worker.assert_any_call(supervisor, "worker_runner") stop_worker.assert_any_call(supervisor, "worker_runner_watch") runner_a.kill_tree.assert_called_once_with(graceful_timeout=2.0) diff --git a/archivebox/workers/management/commands/runner_watch.py b/archivebox/workers/management/commands/runner_watch.py index 671ce2dc..ab784d50 100644 --- a/archivebox/workers/management/commands/runner_watch.py +++ b/archivebox/workers/management/commands/runner_watch.py @@ -71,8 +71,6 @@ class Command(BaseCommand): start_worker(get_supervisor(), RUNNER_WORKER, lazy=True) def restart_runner() -> None: - Process.cleanup_stale_running() - Process.cleanup_orphaned_workers() machine = Machine.current() running = Process.objects.filter( diff --git a/etc/package.json b/etc/package.json index c1e34825..e7a14e12 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.32rc32", + "version": "0.9.32rc34", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index aee09d58..d863698a 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.32rc32" +version = "0.9.32rc34" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -80,9 +80,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.7", # EventBus API - "abxpkg>=1.11.16", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.19", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.19", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.18", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.21", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.21", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ] @@ -166,16 +166,23 @@ build-backend = "pdm.backend" [tool.pdm.build] includes = ["archivebox/"] -source-includes = [] +source-includes = [ + ".git/HEAD", + ".git/refs/heads/dev", +] excludes = [ "archivebox/**/.DS_Store", + "archivebox/**/.flake8", "archivebox/**/__pycache__", "archivebox/**/*.pyc", "archivebox/**/*.pyo", "archivebox/**/*.sqlite3", "archivebox/**/*.sqlite3-*", + "archivebox/mcp/README.md", "archivebox/tests/", "archivebox/tests/**", + "archivebox/**/tests/", + "archivebox/**/tests/**", "tests/", "tests/**", ]