The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-13 18:46:17 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

release: archivebox 0.9.32rc34

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-27 15:11:40 -07:00
parent 09f7c8bba0
commit 4124f78bdc
No known key found for this signature in database
13 changed files with 67 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
archivebox/cli/archivebox_run.py
View File

@ -255,13 +255,9 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
"""
from django.utils import timezone
from archivebox.machine.models import Machine, Process
from archivebox.services.runner import recover_orphaned_crawls, recover_orphaned_snapshots, run_pending_crawls
from archivebox.services.runner import cleanup_orchestrator_state, run_pending_crawls
Process.cleanup_stale_running()
Process.cleanup_orphaned_workers()
Process.cleanup_orphaned_chrome()
recover_orphaned_snapshots()
recover_orphaned_crawls()
cleanup_orchestrator_state(include_chrome=True)
Machine.current()
current = Process.current()
if current.process_type != Process.TypeChoices.ORCHESTRATOR:

4
archivebox/cli/archivebox_server.py
View File

@ -15,9 +15,6 @@ from archivebox.config.common import get_config
def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
"""Stop any existing orchestrator process so the server can take ownership."""
process_model.cleanup_stale_running(machine=machine)
process_model.cleanup_orphaned_workers()
running_runners = list(
process_model.objects.filter(
machine=machine,
@ -47,7 +44,6 @@ def stop_existing_background_runner(*, machine, process_model, supervisor=None,
except Exception:
pass
process_model.cleanup_stale_running(machine=machine)
return len(running_runners)

4
archivebox/core/apps.py
View File

@ -45,8 +45,6 @@ class CoreConfig(AppConfig):
return False
if _should_prepare_runtime():
from archivebox.machine.models import Process, Machine
from archivebox.machine.models import Machine
Process.cleanup_stale_running()
Process.cleanup_orphaned_workers()
Machine.current()

0
archivebox/core/migrations/archivebox/api/migrations/__init__.py
View File

0
archivebox/core/migrations/archivebox/crawls/migrations/__init__.py
View File

0
archivebox/core/migrations/archivebox/machine/migrations/__init__.py
View File

52
archivebox/core/views.py
View File

@ -1523,13 +1523,13 @@ def live_progress_view(request):
if status == Crawl.StatusChoices.SEALED:
status_qs = status_qs.filter(modified_at__gte=recently_cancelled_after)
active_crawl_candidates.extend(
status_qs.values(*active_crawl_fields).order_by("-modified_at")[:10],
status_qs.values(*active_crawl_fields).order_by("-modified_at"),
)
active_crawls_list = sorted(
{str(crawl["id"]): crawl for crawl in active_crawl_candidates}.values(),
key=lambda crawl: crawl["modified_at"],
reverse=True,
)[:10]
)
persona_details_by_id: dict[str, dict[str, str]] = {}
persona_details_by_name: dict[str, dict[str, str]] = {}
persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
@ -1573,36 +1573,12 @@ def live_progress_view(request):
):
crawl_output_sizes_by_crawl[str(row["snapshot__crawl_id"])] = int(row["total_size"] or 0)
if machine_id is not None:
running_processes = Process.objects.filter(
machine_id=machine_id,
status=Process.StatusChoices.RUNNING,
process_type__in=[
Process.TypeChoices.HOOK,
Process.TypeChoices.BINARY,
],
).values("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "modified_at")
recent_processes = (
Process.objects.filter(
machine_id=machine_id,
process_type__in=[
Process.TypeChoices.HOOK,
Process.TypeChoices.BINARY,
],
modified_at__gte=now - timedelta(minutes=10),
)
.values("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "modified_at")
.order_by("-modified_at")
)
else:
running_processes = Process.objects.none()
recent_processes = Process.objects.none()
crawl_process_pids: dict[str, int] = {}
snapshot_process_pids: dict[str, int] = {}
process_records_by_crawl: dict[str, list[tuple[dict[str, object], object | None]]] = {}
process_records_by_snapshot: dict[str, list[tuple[dict[str, object], object | None]]] = {}
seen_process_records: set[str] = set()
active_snapshot_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED}
active_snapshot_statuses = {Snapshot.StatusChoices.STARTED}
recently_cancelled_snapshots_q = Q(
status=Snapshot.StatusChoices.SEALED,
downloaded_at__isnull=True,
@ -1624,17 +1600,33 @@ def live_progress_view(request):
"fs_version",
"status",
)
.order_by("crawl_id", "status", "modified_at")[:100],
.order_by("crawl_id", "status", "modified_at"),
)
snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots}
displayed_snapshots_by_crawl: dict[str, list[Snapshot]] = {str(crawl_id): [] for crawl_id in active_crawl_ids}
for snapshot in snapshots:
crawl_snapshots = displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), [])
if len(crawl_snapshots) < 5:
crawl_snapshots.append(snapshot)
crawl_snapshots.append(snapshot)
displayed_snapshot_ids = [
snapshot["id"] for crawl_snapshots in displayed_snapshots_by_crawl.values() for snapshot in crawl_snapshots
]
process_value_fields = ("id", "process_type", "status", "pwd", "cmd", "pid", "exit_code", "started_at", "modified_at")
if active_crawl_ids or displayed_snapshot_ids:
process_scope = Process.objects.filter(
machine_id=machine_id,
process_type__in=[
Process.TypeChoices.HOOK,
Process.TypeChoices.BINARY,
],
)
running_processes = process_scope.filter(status=Process.StatusChoices.RUNNING).values(*process_value_fields)
recent_processes = (
process_scope.filter(modified_at__gte=now - timedelta(minutes=10)).values(*process_value_fields).order_by("-modified_at")
)
else:
running_processes = Process.objects.none()
recent_processes = Process.objects.none()
archiveresults_by_snapshot: dict[str, list[ArchiveResult]] = {str(snapshot_id): [] for snapshot_id in displayed_snapshot_ids}
if displayed_snapshot_ids:
displayed_archiveresults = (

28
archivebox/services/runner.py
View File

@ -146,14 +146,13 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
return False
Process.cleanup_stale_running()
Process.cleanup_orphaned_workers()
machine = Machine.current()
if Process.objects.filter(
running_orchestrators = Process.objects.filter(
machine=machine,
status=Process.StatusChoices.RUNNING,
process_type=Process.TypeChoices.ORCHESTRATOR,
).exists():
)
if any(proc.is_running for proc in running_orchestrators):
return False
log_path = CONSTANTS.LOGS_DIR / "errors.log"
@ -1233,6 +1232,22 @@ def recover_orphaned_snapshots() -> int:
return recovered
def cleanup_orchestrator_state(*, recover: bool = True, include_chrome: bool = False) -> dict[str, int]:
from archivebox.machine.models import Process
cleaned = {
"stale_processes": Process.cleanup_stale_running(),
"orphaned_processes": Process.cleanup_orphaned_workers(),
"orphaned_chrome": Process.cleanup_orphaned_chrome() if include_chrome else 0,
"orphaned_snapshots": 0,
"orphaned_crawls": 0,
}
if recover:
cleaned["orphaned_snapshots"] = recover_orphaned_snapshots()
cleaned["orphaned_crawls"] = recover_orphaned_crawls()
return cleaned
def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> int:
from archivebox.crawls.models import Crawl, CrawlSchedule
from archivebox.core.models import ArchiveResult, Snapshot
@ -1244,10 +1259,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
now_monotonic = time.monotonic()
if daemon:
if now_monotonic - last_recovery_at >= 30.0:
Process.cleanup_stale_running()
Process.cleanup_orphaned_workers()
recover_orphaned_snapshots()
recover_orphaned_crawls()
cleanup_orchestrator_state()
last_recovery_at = now_monotonic
if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
for model in (ArchiveResult, Snapshot, Crawl, Process):

25
archivebox/tests/test_admin_views.py
View File

@ -14,7 +14,6 @@ import uuid
from pathlib import Path
from types import SimpleNamespace
from typing import cast
from unittest.mock import patch
from django.test import override_settings
from django.test.client import RequestFactory
from django.urls import reverse
@ -1291,7 +1290,7 @@ class TestLiveProgressView:
assert payload["active_crawls"] == []
assert payload["total_workers"] == 0
def test_live_progress_cleans_stale_running_processes(self, client, admin_user, db):
def test_live_progress_does_not_clean_stale_running_processes(self, client, admin_user, db):
from datetime import timedelta
import archivebox.machine.models as machine_models
from archivebox.machine.models import Machine, Process
@ -1313,8 +1312,8 @@ class TestLiveProgressView:
assert response.status_code == 200
proc.refresh_from_db()
assert proc.status == Process.StatusChoices.EXITED
assert proc.ended_at is not None
assert proc.status == Process.StatusChoices.RUNNING
assert proc.ended_at is None
assert response.json()["total_workers"] == 0
def test_live_progress_routes_crawl_process_rows_to_crawl_setup(self, client, admin_user, snapshot, db):
@ -1336,11 +1335,7 @@ class TestLiveProgressView:
)
client.login(username="testadmin", password="testpassword")
with (
patch.object(Process, "cleanup_stale_running", return_value=0),
patch.object(Process, "cleanup_orphaned_workers", return_value=0),
):
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST)
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST)
assert response.status_code == 200
payload = response.json()
@ -1371,11 +1366,7 @@ class TestLiveProgressView:
)
client.login(username="testadmin", password="testpassword")
with (
patch.object(Process, "cleanup_stale_running", return_value=0),
patch.object(Process, "cleanup_orphaned_workers", return_value=0),
):
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST)
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST)
assert response.status_code == 200
payload = response.json()
@ -1410,11 +1401,7 @@ class TestLiveProgressView:
)
client.login(username="testadmin", password="testpassword")
with (
patch.object(Process, "cleanup_stale_running", return_value=0),
patch.object(Process, "cleanup_orphaned_workers", return_value=0),
):
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST)
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST)
assert response.status_code == 200
payload = response.json()

5
archivebox/tests/test_cli_server.py
View File

@ -165,7 +165,7 @@ def test_sonic_daemon_event_handler_requires_running_supervised_worker(monkeypat
asyncio.run(run_test())
def test_stop_existing_background_runner_cleans_up_and_stops_orchestrators():
def test_stop_existing_background_runner_stops_orchestrators_without_row_healing():
from archivebox.cli.archivebox_server import stop_existing_background_runner
runner_a = Mock()
@ -195,7 +195,8 @@ def test_stop_existing_background_runner_cleans_up_and_stops_orchestrators():
)
assert stopped == 2
assert process_model.cleanup_stale_running.call_count == 2
process_model.cleanup_stale_running.assert_not_called()
process_model.cleanup_orphaned_workers.assert_not_called()
stop_worker.assert_any_call(supervisor, "worker_runner")
stop_worker.assert_any_call(supervisor, "worker_runner_watch")
runner_a.kill_tree.assert_called_once_with(graceful_timeout=2.0)

2
archivebox/workers/management/commands/runner_watch.py
View File

@ -71,8 +71,6 @@ class Command(BaseCommand):
start_worker(get_supervisor(), RUNNER_WORKER, lazy=True)
def restart_runner() -> None:
Process.cleanup_stale_running()
Process.cleanup_orphaned_workers()
machine = Machine.current()
running = Process.objects.filter(

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.32rc32",
"version": "0.9.32rc34",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

17
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.32rc32"
version = "0.9.32rc34"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -80,9 +80,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.7", # EventBus API
"abxpkg>=1.11.16", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.19", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.19", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.18", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.21", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.21", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]
@ -166,16 +166,23 @@ build-backend = "pdm.backend"
[tool.pdm.build]
includes = ["archivebox/"]
source-includes = []
source-includes = [
".git/HEAD",
".git/refs/heads/dev",
]
excludes = [
"archivebox/**/.DS_Store",
"archivebox/**/.flake8",
"archivebox/**/__pycache__",
"archivebox/**/*.pyc",
"archivebox/**/*.pyo",
"archivebox/**/*.sqlite3",
"archivebox/**/*.sqlite3-*",
"archivebox/mcp/README.md",
"archivebox/tests/",
"archivebox/tests/**",
"archivebox/**/tests/",
"archivebox/**/tests/**",
"tests/",
"tests/**",
]