The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

release: archivebox 0.9.33rc24

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-28 13:59:18 -07:00
parent 65dc252189
commit 08df41abdb
No known key found for this signature in database
6 changed files with 72 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
archivebox/services/runner.py
View File

@ -1208,11 +1208,17 @@ def run_snapshot_maintenance(snapshot_id: str) -> bool:
snapshot.write_index_jsonl()
snapshot.write_json_details()
snapshot.write_html_details()
print(
f"[runner] Snapshot {str(snapshot.id)[-12:]} maintenance complete "
f"status={snapshot.status} fs_version={snapshot.fs_version} queued_results={'yes' if has_queued_results else 'no'}",
flush=True,
)
return True
def run_due_crawl(crawl, *, lock_seconds: int) -> bool:
if crawl.is_paused:
print(f"[runner] Crawl {str(crawl.id)[-12:]} paused; skipping until resumed", flush=True)
return True
if crawl.status in (crawl.StatusChoices.QUEUED, crawl.StatusChoices.STARTED):
from archivebox.core.models import Snapshot
@ -1268,14 +1274,17 @@ def run_due_crawl(crawl, *, lock_seconds: int) -> bool:
return True
if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
return False
print(f"[runner] Crawl {str(crawl.id)[-12:]} running status={crawl.status} snapshots={snapshot_count}", flush=True)
run_crawl(str(crawl.id), process_discovered_snapshots_inline=True)
return True
if crawl.status == crawl.StatusChoices.SEALED:
print(f"[runner] Crawl {str(crawl.id)[-12:]} sealed; clearing retry tick", flush=True)
crawl.retry_at = None
crawl.save(update_fields=["retry_at", "modified_at"])
return True
print(f"[runner] Crawl {str(crawl.id)[-12:]} status={crawl.status}; clearing retry tick", flush=True)
crawl.retry_at = None
crawl.save(update_fields=["retry_at", "modified_at"])
return True
@ -1287,6 +1296,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool:
if snapshot.is_paused:
selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
if snapshot.fs_migration_needed and Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
print(f"[runner] Snapshot {str(snapshot.id)[-12:]} paused maintenance fs_version={snapshot.fs_version}", flush=True)
run_snapshot_maintenance(str(snapshot.id))
if not selected_plugins:
# No targeted plugin rows remain, so put paused snapshots back
@ -1307,6 +1317,10 @@ def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool:
if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
return False
try:
print(
f"[runner] Snapshot {str(snapshot.id)[-12:]} paused targeted plugins={','.join(selected_plugins)} url={snapshot.url}",
flush=True,
)
# Explicit maintenance, e.g. `archivebox update --index-only`, may
# need to run search/index hooks for a paused snapshot. That should
# not resume the crawl or make unrelated queued work runnable, so
@ -1333,9 +1347,17 @@ def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool:
# migration. Run the filesystem/json save path before queued search
# backfill rows so both maintenance streams stay ordered without
# changing Snapshot.status away from SEALED.
print(
f"[runner] Snapshot {str(snapshot.id)[-12:]} sealed maintenance fs_version={snapshot.fs_version} url={snapshot.url}",
flush=True,
)
return run_snapshot_maintenance(str(snapshot.id))
selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
if selected_plugins:
print(
f"[runner] Snapshot {str(snapshot.id)[-12:]} sealed targeted plugins={','.join(selected_plugins)} url={snapshot.url}",
flush=True,
)
run_crawl(
str(snapshot.crawl_id),
snapshot_ids=[str(snapshot.id)],
@ -1348,11 +1370,13 @@ def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool:
if snapshot.status == Snapshot.StatusChoices.STARTED:
_reset_count, running_count = reset_abandoned_snapshot_results(snapshot)
if running_count:
print(f"[runner] Snapshot {str(snapshot.id)[-12:]} still has {running_count} running ArchiveResults", flush=True)
snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS))
return True
if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
return False
print(f"[runner] Snapshot {str(snapshot.id)[-12:]} running status={snapshot.status} url={snapshot.url}", flush=True)
run_crawl(
str(snapshot.crawl_id),
snapshot_ids=[str(snapshot.id)],

6
archivebox/services/supervision_service.py
View File

@ -160,8 +160,10 @@ def standby_until_runtime_stack_needed(command, *, data_dir: str | Path, interva
if not announced:
owner = runtime_stack_owner(data_dir=data_dir)
owner_pid = owner.pid if owner else "unknown"
owner_type = owner.process_type if owner else "unknown"
rprint(f"[yellow][*] Standing by; ArchiveBox {owner_type} pid={owner_pid} owns the runtime stack.[/yellow]")
rprint(
"[yellow][*] Runner is now owned by newer archivebox process "
f"pid={owner_pid}, processing will continue there and will resume here if the other process is stopped and work still remains[/yellow]",
)
announced = True
command.heartbeat()
time.sleep(interval)

12
archivebox/templates/admin/progress_monitor.html
View File

@ -203,7 +203,7 @@
#progress-monitor .screencast-frame {
display: block;
width: 100%;
aspect-ratio: 16 / 10;
height: 310px;
background: #010409;
color: #6e7681;
text-decoration: none;
@ -211,8 +211,11 @@
}
#progress-monitor .screencast-frame img {
display: block;
width: 100%;
height: 100%;
width: 120%;
height: auto;
min-height: 100%;
max-width: none;
margin-left: -10%;
object-fit: cover;
object-position: top center;
}
@ -924,6 +927,9 @@
width: 100%;
align-self: stretch;
}
#progress-monitor .screencast-frame {
height: 300px;
}
#progress-monitor .crawl-header {
display: grid;
grid-template-columns: minmax(0, 1fr) auto;

37
archivebox/workers/supervisord_util.py
View File

@ -156,6 +156,7 @@ RUNNER_WORKER = {
"command": _shell_join([sys.executable, "-m", "archivebox", "run", "--daemon"]),
"autostart": "false",
"autorestart": "true",
"environment": 'PYTHONUNBUFFERED="1",COLUMNS="200"',
"stopasgroup": "true",
"killasgroup": "true",
"stopwaitsecs": "30",
@ -693,15 +694,37 @@ def start_worker(supervisor, daemon, lazy=False):
def run_runner_worker(args: list[str], *, name: str = "worker_runner_once") -> int:
supervisor = get_or_create_supervisord_process(daemonize=False)
worker = RUNNER_ONCE_WORKER(args, name=name)
log_path = Path(worker["stdout_logfile"])
if not log_path.is_absolute():
log_path = CONSTANTS.DATA_DIR / log_path
log_path.parent.mkdir(parents=True, exist_ok=True)
log_path.touch()
log_handle = log_path.open()
log_handle.seek(0, 2)
sync_supervisord_workers(supervisor, [(worker, False)], prune=False)
final_states = {"STOPPED", "EXITED", "FATAL", "UNKNOWN"}
while True:
proc = get_worker(supervisor, name)
if proc is None:
return 1
if proc["statename"] in final_states:
return int(proc.get("exitstatus") or 0) if proc["statename"] == "EXITED" else 1
time.sleep(0.5)
try:
while True:
while True:
line = log_handle.readline()
if not line:
break
sys.stdout.write(line)
sys.stdout.flush()
proc = get_worker(supervisor, name)
if proc is None:
return 1
if proc["statename"] in final_states:
while True:
line = log_handle.readline()
if not line:
break
sys.stdout.write(line)
sys.stdout.flush()
return int(proc.get("exitstatus") or 0) if proc["statename"] == "EXITED" else 1
time.sleep(0.5)
finally:
log_handle.close()
def get_worker(supervisor, daemon_name):

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.33rc23",
"version": "0.9.33rc24",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

8
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.33rc23"
version = "0.9.33rc24"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.7", # EventBus API
"abxpkg>=1.11.49", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.56", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.55", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.50", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.57", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.57", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]