The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

chore: apply release hook fixes

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-28 06:42:53 -07:00
parent b2fa839788
commit fe09f5a517
No known key found for this signature in database
10 changed files with 124 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
archivebox/cli/archivebox_add.py
View File

@ -111,7 +111,7 @@ def add(
from archivebox.personas.models import Persona
from archivebox.misc.logging_util import printable_filesize
from archivebox.misc.system import get_dir_size
from archivebox.core.shutdown_util import foreground_parent_watchdog
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.services.runner import run_crawl
from django.utils import timezone
@ -217,7 +217,7 @@ def add(
else:
# Foreground mode: run full crawl runner until all work is done
print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
with foreground_parent_watchdog():
with foreground_shutdown_signals(), foreground_parent_watchdog():
run_crawl(str(crawl.id))
# Print summary for foreground runs
@ -300,26 +300,29 @@ def add(
def main(**kwargs):
"""Add a new URL or list of URLs to your archive"""
raw_urls = kwargs.pop("urls")
urls = _collect_input_urls(raw_urls)
if not urls:
raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
if int(kwargs.get("max_urls") or 0) < 0:
raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
if int(kwargs.get("crawl_timeout") or 0) < 0:
raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
try:
kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
except ValueError as err:
raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
try:
kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
except ValueError as err:
raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
add(urls=urls, **kwargs)
with foreground_shutdown_signals(), foreground_parent_watchdog():
raw_urls = kwargs.pop("urls")
urls = _collect_input_urls(raw_urls)
if not urls:
raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
if int(kwargs.get("max_urls") or 0) < 0:
raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
if int(kwargs.get("crawl_timeout") or 0) < 0:
raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
try:
kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
except ValueError as err:
raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
try:
kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
except ValueError as err:
raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
add(urls=urls, **kwargs)
if __name__ == "__main__":

12
archivebox/cli/archivebox_run.py
View File

@ -77,7 +77,7 @@ def process_stdin_records() -> int:
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import Snapshot, ArchiveResult
from archivebox.crawls.models import Crawl
from archivebox.core.shutdown_util import foreground_parent_watchdog
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.machine.models import Binary
from archivebox.services.runner import run_binary, run_crawl
@ -229,7 +229,7 @@ def process_stdin_records() -> int:
if not crawl.claim_processing_lock(lock_seconds=10):
rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr)
return 1
with foreground_parent_watchdog():
with foreground_shutdown_signals(), foreground_parent_watchdog():
run_crawl(
crawl_id,
snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
@ -248,7 +248,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
Returns exit code (0 = success, 1 = error).
"""
from archivebox.config import CONSTANTS
from archivebox.core.shutdown_util import foreground_parent_watchdog
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.machine.models import Machine, Process
from archivebox.services.supervision_service import healthy_orchestrator
from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
@ -263,7 +263,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
return 0
current.mark_running(process_type=Process.TypeChoices.ORCHESTRATOR, pwd=str(CONSTANTS.DATA_DIR), timeout=0)
try:
with foreground_parent_watchdog(enabled=not daemon):
with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
run_pending_crawls(daemon=daemon, crawl_id=crawl_id)
return 0
except KeyboardInterrupt:
@ -324,14 +324,14 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
def run_snapshot_worker(snapshot_id: str) -> int:
from archivebox.core.shutdown_util import foreground_parent_watchdog
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.core.models import Snapshot
from archivebox.services.runner import run_due_snapshot
from django.utils import timezone
snapshot = None
try:
with foreground_parent_watchdog():
with foreground_shutdown_signals(), foreground_parent_watchdog():
snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
if snapshot.retry_at is None:
Snapshot.objects.filter(pk=snapshot.pk).update(retry_at=timezone.now(), modified_at=timezone.now())

36
archivebox/cli/archivebox_update.py
View File

@ -102,6 +102,7 @@ def reindex_snapshots(
search_plugins: list[str],
batch_size: int,
collect_ids: bool = False,
wait_for_turn=None,
) -> dict[str, Any]:
from archivebox.cli.archivebox_extract import run_plugins
@ -114,6 +115,8 @@ def reindex_snapshots(
def run_batch() -> None:
if not records:
return
if wait_for_turn:
wait_for_turn()
batch_records = list(records)
# Index-only backfill intentionally queues only search ArchiveResult
# rows. The extract runner bumps Snapshot.retry_at so the orchestrator
@ -201,11 +204,15 @@ def update(
setup_django()
from archivebox.machine.models import Process
from archivebox.core.shutdown_util import foreground_parent_watchdog
from archivebox.services.supervision_service import current_command, ensure_daemon_stack
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
from archivebox.services.supervision_service import current_command, ensure_daemon_stack, standby_until_runtime_stack_needed
from archivebox.workers.supervisord_util import stop_existing_supervisord_process
command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
def wait_for_turn() -> None:
standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
is_filtered_update = any(
(
filter_patterns,
@ -229,10 +236,11 @@ def update(
# Run migrations first to ensure DB schema is up-to-date
print("[*] Checking for pending migrations...")
check_migrations(auto_apply=True)
wait_for_turn()
if stop_daemon_stack:
stop_existing_supervisord_process()
with foreground_parent_watchdog():
with foreground_shutdown_signals(), foreground_parent_watchdog():
while True:
do_migrate = migrate_only or not index_only
do_index = index_only or not migrate_only
@ -269,6 +277,7 @@ def update(
resume=resume,
batch_size=batch_size,
queue_for_archiving=do_run_until_idle,
wait_for_turn=wait_for_turn,
)
print_stats(stats)
touched_snapshot_ids.update(stats.get("snapshot_ids", []))
@ -282,7 +291,11 @@ def update(
)
print("[*] Phase 2: Processing all database snapshots (most recent first)...")
stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume)
stats_combined["phase2"] = process_all_db_snapshots(
batch_size=batch_size,
resume=resume,
wait_for_turn=wait_for_turn,
)
print_combined_stats(stats_combined)
if do_index:
@ -311,6 +324,7 @@ def update(
search_plugins=search_plugins,
batch_size=batch_size,
collect_ids=is_filtered_update,
wait_for_turn=wait_for_turn,
)
print_index_stats(stats)
touched_snapshot_ids.update(stats.get("snapshot_ids", []))
@ -540,7 +554,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100
return stats
def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -> dict[str, int]:
def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
"""
O(n) scan over entire DB from most recent to least recent.
@ -577,6 +591,8 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
updated = 0
checked = 0
while True:
if wait_for_turn:
wait_for_turn()
ids = list(rows.order_by("-timestamp").values_list("id", flat=True)[:batch_size])
if not ids:
if updated:
@ -614,6 +630,8 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -
def queue_stale_fs_batch() -> None:
if not stale_batch:
return
if wait_for_turn:
wait_for_turn()
now = timezone.now()
snapshot_ids = [snapshot_id for snapshot_id, _crawl_id, _timestamp in stale_batch]
# Do not bump fs_version here. The orchestrator calls Snapshot.save(),
@ -676,6 +694,7 @@ def process_filtered_snapshots(
resume: str | None,
batch_size: int,
queue_for_archiving: bool = True,
wait_for_turn=None,
) -> dict[str, Any]:
"""Process snapshots matching filters (DB query only)."""
from archivebox.core.models import Snapshot
@ -703,6 +722,8 @@ def process_filtered_snapshots(
print(f"[*] Found {total} matching snapshots")
for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
if wait_for_turn and stats["processed"] % batch_size == 0:
wait_for_turn()
stats["processed"] += 1
# Skip snapshots with missing crawl references
@ -815,7 +836,10 @@ def print_index_stats(stats: dict[str, Any]) -> None:
@click.argument("filter_patterns", nargs=-1)
@docstring(update.__doc__)
def main(**kwargs):
update(**kwargs)
from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
with foreground_shutdown_signals(), foreground_parent_watchdog():
update(**kwargs)
if __name__ == "__main__":

54
archivebox/core/migrations/0027_copy_archiveresult_to_process.py
View File

@ -140,35 +140,33 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
cursor.execute("PRAGMA table_info(machine_binary)")
binary_cols = {row[1] for row in cursor.fetchall()}
# Use only columns that exist in current schema
# 0.8.x schema: id, created_at, modified_at, machine_id, name, binprovider, abspath, version, sha256, num_uses_failed, num_uses_succeeded
# 0.9.x schema adds: binproviders, overrides, status, retry_at, output_dir
if "binproviders" in binary_cols:
# 0.9.x schema
cursor.execute(
"""
INSERT INTO machine_binary (
id, created_at, modified_at, machine_id,
name, binproviders, overrides, binprovider, abspath, version, sha256,
status, retry_at, output_dir,
num_uses_failed, num_uses_succeeded
) VALUES (?, ?, ?, ?, ?, 'env', '{}', 'env', ?, ?, '',
'succeeded', NULL, '', 0, 0)
values_by_col = {
"id": binary_id,
"created_at": now,
"modified_at": now,
"machine_id": machine_id,
"name": name,
"binproviders": "env",
"overrides": "{}",
"binprovider": "env",
"abspath": abspath,
"version": version,
"sha256": "",
"status": "installed",
"retry_at": None,
"output_dir": "",
"num_uses_failed": 0,
"num_uses_succeeded": 0,
}
insert_cols = [col for col in values_by_col if col in binary_cols]
placeholders = ", ".join(["?"] * len(insert_cols))
cursor.execute(
f"""
INSERT INTO machine_binary ({", ".join(insert_cols)})
VALUES ({placeholders})
""",
[binary_id, now, now, machine_id, name, abspath, version],
)
else:
# 0.8.x schema (simpler)
cursor.execute(
"""
INSERT INTO machine_binary (
id, created_at, modified_at, machine_id,
name, binprovider, abspath, version, sha256,
num_uses_failed, num_uses_succeeded
) VALUES (?, ?, ?, ?, ?, 'env', ?, ?, '', 0, 0)
""",
[binary_id, now, now, machine_id, name, abspath, version],
)
[values_by_col[col] for col in insert_cols],
)
return binary_id

7
archivebox/core/shutdown_util.py
View File

@ -1,9 +1,9 @@
from __future__ import annotations
import os
import signal
import subprocess
import sys
import os
import threading
from collections.abc import Iterator
from contextlib import contextmanager
@ -101,8 +101,7 @@ def foreground_shutdown_signals(
def raise_keyboard_interrupt(signum, _frame):
state.signal_name = signal.Signals(signum).name
sys.stdout.write(f"\n[🛑] Got {state.signal_name}, stopping gracefully...\n")
sys.stdout.flush()
os.write(sys.stdout.fileno(), f"\n[🛑] Got {state.signal_name}, stopping gracefully...\n".encode())
raise KeyboardInterrupt
try:
@ -119,7 +118,7 @@ def foreground_parent_watchdog(
*,
enabled: bool = True,
check_interval: float = 2.0,
shutdown_signal: signal.Signals = signal.SIGINT,
shutdown_signal: signal.Signals = signal.SIGTERM,
) -> Iterator[None]:
"""Ask a foreground command to exit if its launcher/wrapper disappears.

26
archivebox/machine/migrations/0005_converge_binary_model.py
View File

@ -23,8 +23,8 @@ def converge_binary_table(apps, schema_editor):
print("✓ Dropping machine_installedbinary table (0.8.6rc0 divergence)")
cursor.execute("DROP TABLE IF EXISTS machine_installedbinary")
# Create Binary table if it doesn't exist
# This handles the case where 0.8.6rc0's 0001_initial didn't create it
# Create Binary table if it doesn't exist.
# This handles the case where 0.8.6rc0's 0001_initial didn't create it.
if "machine_binary" not in existing_tables:
print("✓ Creating machine_binary table with correct schema")
cursor.execute("""
@ -56,6 +56,28 @@ def converge_binary_table(apps, schema_editor):
print("✓ machine_binary table created")
else:
print("✓ machine_binary table already exists")
cursor.execute("PRAGMA table_info(machine_binary)")
binary_cols = {row[1] for row in cursor.fetchall()}
# Old 0.8.x data dirs already have machine_binary, but with the
# pre-abxpkg shape. Converge it here before later migrations and
# runtime code expect Binary.binproviders / Binary.status to exist.
if "binproviders" not in binary_cols:
cursor.execute("ALTER TABLE machine_binary ADD COLUMN binproviders VARCHAR(255) NOT NULL DEFAULT 'env'")
if "overrides" not in binary_cols:
cursor.execute("ALTER TABLE machine_binary ADD COLUMN overrides TEXT NOT NULL DEFAULT '{}'")
if "status" not in binary_cols:
cursor.execute("ALTER TABLE machine_binary ADD COLUMN status VARCHAR(16) NOT NULL DEFAULT 'installed'")
if "retry_at" not in binary_cols:
cursor.execute("ALTER TABLE machine_binary ADD COLUMN retry_at DATETIME NULL")
if "output_dir" not in binary_cols:
cursor.execute("ALTER TABLE machine_binary ADD COLUMN output_dir VARCHAR(255) NOT NULL DEFAULT ''")
cursor.execute(
"UPDATE machine_binary SET binproviders = COALESCE(NULLIF(binproviders, ''), COALESCE(NULLIF(binprovider, ''), 'env'))",
)
cursor.execute("UPDATE machine_binary SET overrides = COALESCE(NULLIF(overrides, ''), '{}')")
cursor.execute("UPDATE machine_binary SET status = COALESCE(NULLIF(status, ''), 'installed')")
class Migration(migrations.Migration):

4
bin/fuzz_test.sh
View File

@ -150,7 +150,7 @@ run_with_timeout() {
) &
local watchdog=$!
trap 'kill_tree "$child"; kill "$watchdog" >/dev/null 2>&1 || true' INT TERM
trap '[[ -n "${child:-}" ]] && kill_tree "$child"; [[ -n "${watchdog:-}" ]] && kill "$watchdog" >/dev/null 2>&1 || true' INT TERM
wait "$child"
local code=$?
@ -190,7 +190,7 @@ run_server_for_a_bit() {
) >> "$logfile" 2>&1 &
local child=$!
trap 'kill_tree "$child"' INT TERM
trap '[[ -n "${child:-}" ]] && kill_tree "$child"' INT TERM
sleep "$hold"
echo "[$(ts)] STOP label=$label pid=$child after=${hold}s" | tee -a "$logfile"

4
bin/take_screenshot.js
View File

@ -23,6 +23,7 @@ Environment:
SCREENSHOT_FULL_PAGE Set to 1 to capture the full page, defaults to viewport only
SCREENSHOT_SCROLL_SELECTOR Scroll this selector into view before capture
SCREENSHOT_WAIT_SELECTOR Wait for this selector before capture
SCREENSHOT_HOST_RESOLVER_RULES Chrome host resolver rules
SCREENSHOT_SNAPSHOT_VIEW Set to list or grid before loading the page
SCREENSHOT_RESET_FILTERS Set to 1 to clear the admin filter collapsed preference
`);
@ -60,6 +61,9 @@ async function main() {
headless: true,
defaultViewport: { width, height },
};
if (process.env.SCREENSHOT_HOST_RESOLVER_RULES) {
launchOptions.args = [`--host-resolver-rules=${process.env.SCREENSHOT_HOST_RESOLVER_RULES}`];
}
const executablePath = chromePath();
if (executablePath) {
launchOptions.executablePath = executablePath;

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.33rc13",
"version": "0.9.33rc14",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

8
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.33rc13"
version = "0.9.33rc14"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.7", # EventBus API
"abxpkg>=1.11.37", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.43", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.43", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.38", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.44", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.44", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]