The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

release: archivebox 0.9.33rc26

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-28 14:32:28 -07:00
parent f25bd809cb
commit 52b37d48bf
No known key found for this signature in database
11 changed files with 100 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
archivebox/cli/archivebox_run.py
View File

@ -42,6 +42,7 @@ __command__ = "archivebox run"
import os
import sys
import asyncio
from collections import defaultdict
import rich_click as click
@ -266,7 +267,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
run_pending_crawls(daemon=daemon, crawl_id=crawl_id, maintenance_only=maintenance_only)
return 0
except KeyboardInterrupt:
except (KeyboardInterrupt, asyncio.CancelledError):
return 0
except Exception as e:
rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)

25
archivebox/cli/archivebox_server.py
View File

@ -22,7 +22,7 @@ def server(
nothreading: bool = False,
) -> None:
"""Run the ArchiveBox HTTP server"""
from archivebox.config.common import get_config
from archivebox.config.common import get_config, rprint
config = get_config()
runserver_args = list(runserver_args or (config.BIND_ADDR,))
@ -76,6 +76,7 @@ def server(
from archivebox.services.supervision_service import (
command_owns_runtime_stack,
current_command,
runtime_stack_owner,
standby_until_runtime_stack_needed,
)
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@ -95,6 +96,14 @@ def server(
bind_url = f"http://{host}:{port}"
command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
def still_owns_runtime_stack() -> bool:
from django.db import connections
try:
return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
finally:
connections.close_all()
try:
with foreground_shutdown_signals(), foreground_parent_watchdog():
while True:
@ -114,13 +123,19 @@ def server(
debug=run_in_debug,
reload=reload,
nothreading=nothreading,
keep_running=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR),
should_stop_supervisord=lambda: command_owns_runtime_stack(command, data_dir=config.DATA_DIR),
keep_running=still_owns_runtime_stack,
should_stop_supervisord=still_owns_runtime_stack,
)
if result == "interrupted":
break
if not command_owns_runtime_stack(command, data_dir=config.DATA_DIR):
print("[yellow][*] Another ArchiveBox command took over the runtime stack; standing by.[/yellow]")
if not still_owns_runtime_stack():
owner = runtime_stack_owner(data_dir=config.DATA_DIR)
owner_pid = owner.pid if owner else "unknown"
rprint(
"[yellow][*] A newer archivebox process took over the runner "
f"(pid={owner_pid}). Work will continue there, and will continue here if the other process is stopped and work still remains.[/yellow]",
file=sys.stderr,
)
continue
if result == "exited":
print("[yellow][*] Runtime stack exited while this parent is still leader; restarting...[/yellow]")

39
archivebox/core/views.py
View File

@ -1560,6 +1560,17 @@ def live_progress_view(request):
url = str(url or "")
return url if len(url) <= 96 else f"{url[:93]}..."
def screencast_frame_url(object_id: str) -> str:
frame_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / object_id / "latest.jpg"
try:
frame_stat = frame_path.stat()
except OSError:
return ""
if frame_stat.st_size <= 0:
return ""
token = SCREENCAST_SIGNER.sign(object_id)
return f"/admin/live-progress/screencast/{object_id}.jpg?v={frame_stat.st_mtime_ns}&token={quote(token)}"
machine_id = Machine.current().id
orchestrator_proc = (
Process.objects.filter(
@ -1971,19 +1982,8 @@ def live_progress_view(request):
crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
crawl_screencast_url = ""
crawl_screencast_link = ""
live_crawl_preview_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / crawl_id / "latest.jpg"
try:
live_crawl_preview_stat = live_crawl_preview_path.stat()
except OSError:
live_crawl_preview_stat = None
if live_crawl_preview_stat and live_crawl_preview_stat.st_size > 0:
token = SCREENCAST_SIGNER.sign(crawl_id)
crawl_screencast_url = (
f"/admin/live-progress/screencast/{crawl_id}.jpg?v={live_crawl_preview_stat.st_mtime_ns}&token={quote(token)}"
)
crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id}/change/"
crawl_screencast_url = screencast_frame_url(crawl_id)
crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id}/change/" if crawl_screencast_url else ""
# Get active snapshots for this crawl (already prefetched)
active_snapshots_for_crawl = []
@ -2034,17 +2034,8 @@ def live_progress_view(request):
snapshot_preview_url = snapshot_favicon_url
if snapshot["status"] == Snapshot.StatusChoices.STARTED:
live_preview_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / str(snapshot["id"]) / "latest.jpg"
try:
live_preview_stat = live_preview_path.stat()
except OSError:
live_preview_stat = None
if live_preview_stat and live_preview_stat.st_size > 0:
token = SCREENCAST_SIGNER.sign(str(snapshot["id"]))
snapshot_screencast_url = (
f"/admin/live-progress/screencast/{snapshot['id']}.jpg?v={live_preview_stat.st_mtime_ns}&token={quote(token)}"
)
snapshot_screencast_link = snapshot_view_url(snapshot)
snapshot_screencast_url = screencast_frame_url(str(snapshot["id"]))
snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
def plugin_sort_key(ar):
status_order = {

7
archivebox/machine/models.py
View File

@ -812,8 +812,10 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
print(f"Failed to create LIB_BIN_DIR {lib_bin_dir}: {e}", file=sys.stderr)
return None
# Get binary name (last component of path)
binary_name = binary_abspath.name
# Expose the canonical Binary.name in LIB_BIN_DIR. Some providers point
# abspath at implementation files like cli.js or manifest.json; those
# are valid targets, but they are not user-facing binary names.
binary_name = _canonical_binary_name(self.name) or binary_abspath.name
symlink_path = lib_bin_dir / binary_name
if app_index != -1 and len(binary_parts) > app_index + 2 and binary_parts[app_index + 1 : app_index + 3] == ("Contents", "MacOS"):
@ -842,7 +844,6 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
# Create new symlink
try:
symlink_path.symlink_to(binary_abspath)
print(f"Symlinked {binary_name} -> {symlink_path}", file=sys.stderr)
return symlink_path
except (OSError, PermissionError) as e:
print(f"Failed to create symlink {symlink_path} -> {binary_abspath}: {e}", file=sys.stderr)

8
archivebox/misc/db.py
View File

@ -34,6 +34,12 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
import psutil
db_path = db_path.resolve()
db_sidecars = {
db_path,
db_path.with_name(f"{db_path.name}-wal"),
db_path.with_name(f"{db_path.name}-shm"),
db_path.with_name(f"{db_path.name}-journal"),
}
holders: list[str] = []
for proc in psutil.process_iter(["pid", "ppid", "name", "cmdline", "status"]):
try:
@ -45,7 +51,7 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
open_path = Path(open_file.path).resolve()
except (OSError, RuntimeError):
continue
if open_path == db_path or open_path.name in {f"{db_path.name}-wal", f"{db_path.name}-shm", f"{db_path.name}-journal"}:
if open_path in db_sidecars:
info = proc.info
cmdline = compact_command(info.get("cmdline"), fallback=info.get("name") or "")
holders.append(f"pid={info['pid']} ppid={info['ppid']} {info['status']} {cmdline}")

44
archivebox/services/runner.py
View File

@ -231,49 +231,30 @@ class CrawlRunner:
if threading.current_thread() is not threading.main_thread():
return []
loop = asyncio.get_running_loop()
installed: list[tuple[signal.Signals, Any, bool]] = []
for sig in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM):
previous = signal.getsignal(sig)
def request_abort(sig=sig) -> None:
def request_abort(_signum, _frame, sig=sig) -> None:
os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name}, stopping gracefully...\n".encode())
self._request_abort_from_signal(sig)
raise KeyboardInterrupt
try:
loop.add_signal_handler(sig, request_abort)
installed.append((sig, previous, True))
except (NotImplementedError, RuntimeError):
signal.signal(sig, lambda _signum, _frame, sig=sig: self._request_abort_from_signal(sig))
installed.append((sig, previous, False))
signal.signal(sig, request_abort)
installed.append((sig, previous, False))
return installed
def _restore_signal_handlers(self, installed: list[tuple[signal.Signals, Any, bool]]) -> None:
loop = asyncio.get_running_loop()
for sig, previous, installed_on_loop in reversed(installed):
if installed_on_loop:
loop.remove_signal_handler(sig)
asyncio.get_running_loop().remove_signal_handler(sig)
signal.signal(sig, previous)
def _request_abort_from_signal(self, sig: signal.Signals) -> None:
if self._signal_abort_requested:
if self._run_task is not None and not self._run_task.done():
self._run_task.cancel()
return
def _request_abort_from_signal(self, _sig: signal.Signals) -> None:
self._signal_abort_requested = True
self._skip_wait_until_idle = True
asyncio.create_task(self.abort_from_signal(sig.name))
async def abort_from_signal(self, signal_name: str) -> None:
from archivebox.crawls.models import Crawl
await sync_to_async(
Crawl.objects.filter(id=self.crawl.id).exclude(status=Crawl.StatusChoices.SEALED).update,
thread_sensitive=False,
)(
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
modified_at=timezone.now(),
)
if self._run_task is not None and not self._run_task.done():
self._run_task.cancel()
async def crawl_is_cancelled(self) -> bool:
from archivebox.crawls.models import Crawl
@ -380,7 +361,12 @@ class CrawlRunner:
async def stop_snapshot_tasks(self) -> None:
if not self.snapshot_tasks:
return
done, pending = await asyncio.wait(list(self.snapshot_tasks.values()), timeout=5.0)
tasks = list(self.snapshot_tasks.values())
if self._signal_abort_requested:
done = {task for task in tasks if task.done()}
pending = set(tasks) - done
else:
done, pending = await asyncio.wait(tasks, timeout=5.0)
for task in pending:
task.cancel()
await asyncio.gather(*done, *pending, return_exceptions=True)

11
archivebox/services/supervision_service.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import time
import sys
from pathlib import Path
from django.utils import timezone
@ -29,7 +30,6 @@ def current_command(process_type: str, *, data_dir: str | Path, url: str | None
def live_processes(*, process_type: str, data_dir: str | Path, url: str | None = None):
from archivebox.machine.models import Machine, Process
Process.cleanup_stale_running(machine=Machine.current())
qs = Process.objects.filter(
machine=Machine.current(),
process_type=process_type,
@ -54,7 +54,6 @@ def command_is_newest(command, *, process_type: str, data_dir: str | Path, url:
def runtime_stack_owner(*, data_dir: str | Path):
from archivebox.machine.models import Machine, Process
Process.cleanup_stale_running(machine=Machine.current())
base_qs = Process.objects.filter(
machine=Machine.current(),
status=Process.StatusChoices.RUNNING,
@ -117,7 +116,6 @@ def healthy_orchestrator(*, data_dir: str | Path):
from archivebox.machine.models import Machine, Process
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
Process.cleanup_stale_running(machine=Machine.current())
supervisor = get_existing_supervisord_process()
worker = get_worker(supervisor, "worker_runner") if supervisor else None
if isinstance(worker, dict) and worker.get("statename") in ("STARTING", "RUNNING"):
@ -145,7 +143,6 @@ def standby_until_leader_needed(command, *, process_type: str, data_dir: str | P
leader_pid = leader.pid if leader else "unknown"
rprint(f"[yellow][*] Standing by; newer ArchiveBox parent pid={leader_pid} is running the orchestrator and server.[/yellow]")
announced = True
command.heartbeat()
time.sleep(interval)
command.modified_at = timezone.now()
command.save(update_fields=["modified_at"])
@ -161,11 +158,11 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
owner = runtime_stack_owner(data_dir=data_dir)
owner_pid = owner.pid if owner else "unknown"
rprint(
"[yellow][*] Runner is now owned by newer archivebox process "
f"pid={owner_pid}, processing will continue there and will resume here if the other process is stopped and work still remains[/yellow]",
"[yellow][*] A newer archivebox process took over the runner "
f"(pid={owner_pid}). Work will continue there, and will continue here if the other process is stopped and work still remains.[/yellow]",
file=sys.stderr,
)
announced = True
command.heartbeat()
time.sleep(interval)
command.modified_at = timezone.now()
command.save(update_fields=["modified_at"])

54
archivebox/templates/admin/progress_monitor.html
View File

@ -1190,12 +1190,9 @@
});
}
function setupHookMatches(item, pattern) {
const text = `${item.plugin || ''} ${item.hook_name || ''} ${item.label || ''}`.toLowerCase();
return text.includes(pattern);
}
function activeScreencastTarget(data) {
const hookText = (item) => `${item.plugin || ''} ${item.hook_name || ''} ${item.label || ''}`.toLowerCase();
let fallback = null;
for (const crawl of data.active_crawls || []) {
for (const snapshot of crawl.active_snapshots || []) {
if (!Array.isArray(snapshot) && snapshot.screencast_url) {
@ -1218,12 +1215,12 @@
},
};
}
}
for (const crawl of data.active_crawls || []) {
if (fallback) continue;
const setupPlugins = crawl.setup_plugins || [];
const startedSetup = setupPlugins.filter((item) => item.status === 'started');
if (startedSetup.some((item) => setupHookMatches(item, 'chrome launch'))) {
return {
if (setupPlugins.some((item) => item.status === 'started' && hookText(item).includes('chrome launch'))) {
fallback = {
type: 'placeholder',
crawl,
state: 'launching',
@ -1231,24 +1228,27 @@
title: 'Launching browser',
subtitle: 'Chrome is starting and publishing its CDP session.',
};
continue;
}
for (const snapshot of crawl.active_snapshots || []) {
if (Array.isArray(snapshot)) continue;
const startedSnapshotPlugins = (snapshot.all_plugins || []).filter((item) => item.status === 'started');
if (startedSnapshotPlugins.some((item) => setupHookMatches(item, 'chrome tab'))) {
return {
type: 'placeholder',
crawl,
snapshot,
state: 'launching',
icon: '▣',
title: 'Opening browser tab',
subtitle: 'The snapshot tab is being attached for capture.',
};
}
const openingSnapshot = (crawl.active_snapshots || []).find((snapshot) => {
return !Array.isArray(snapshot) && (snapshot.all_plugins || []).some((item) => (
item.status === 'started' && hookText(item).includes('chrome tab')
));
});
if (openingSnapshot) {
fallback = {
type: 'placeholder',
crawl,
snapshot: openingSnapshot,
state: 'launching',
icon: '▣',
title: 'Opening browser tab',
subtitle: 'The snapshot tab is being attached for capture.',
};
continue;
}
if ((crawl.status === 'queued' || crawl.status === 'started') && !setupPlugins.some((item) => setupHookMatches(item, 'chrome launch'))) {
return {
if ((crawl.status === 'queued' || crawl.status === 'started') && !setupPlugins.some((item) => hookText(item).includes('chrome launch'))) {
fallback = {
type: 'placeholder',
crawl,
state: 'not-launched',
@ -1258,7 +1258,7 @@
};
}
}
return null;
return fallback;
}
function updateScreencastPanel(data) {

5
archivebox/workers/supervisord_util.py
View File

@ -422,7 +422,7 @@ def stop_existing_supervisord_process():
PID_FILE = SOCK_FILE.parent / PID_FILE_NAME
stop_grace_seconds = configured_stopwaitsecs(tuple(_desired_supervisord_workers.values()))
supervisor = get_existing_supervisord_process()
supervisor = get_existing_supervisord_process(quiet=True)
supervisor_pid = None
supervisor_shutdown_requested = False
if supervisor is not None:
@ -955,6 +955,9 @@ def start_server_workers(
if daemonize:
return None
from django.db import connections
connections.close_all()
try:
with foreground_shutdown_signals() as shutdown_state:
# Tail worker logs while supervisord runs.

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.33rc25",
"version": "0.9.33rc26",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

2
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.33rc25"
version = "0.9.33rc26"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]