diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index f612c7d8..8a72fa2c 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -111,7 +111,7 @@ def add( from archivebox.personas.models import Persona from archivebox.misc.logging_util import printable_filesize from archivebox.misc.system import get_dir_size - from archivebox.core.shutdown_util import foreground_parent_watchdog + from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals from archivebox.services.runner import run_crawl from django.utils import timezone @@ -217,7 +217,7 @@ def add( else: # Foreground mode: run full crawl runner until all work is done print("[green]\\[*] Starting crawl runner to process crawl...[/green]") - with foreground_parent_watchdog(): + with foreground_shutdown_signals(), foreground_parent_watchdog(): run_crawl(str(crawl.id)) # Print summary for foreground runs @@ -300,26 +300,29 @@ def add( def main(**kwargs): """Add a new URL or list of URLs to your archive""" - raw_urls = kwargs.pop("urls") - urls = _collect_input_urls(raw_urls) - if not urls: - raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.") - if int(kwargs.get("max_urls") or 0) < 0: - raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls") - if int(kwargs.get("crawl_timeout") or 0) < 0: - raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout") - try: - kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size")) - except ValueError as err: - raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err - try: - kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size")) - except ValueError as err: - raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err - if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1: - raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots") + from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals - add(urls=urls, **kwargs) + with foreground_shutdown_signals(), foreground_parent_watchdog(): + raw_urls = kwargs.pop("urls") + urls = _collect_input_urls(raw_urls) + if not urls: + raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.") + if int(kwargs.get("max_urls") or 0) < 0: + raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls") + if int(kwargs.get("crawl_timeout") or 0) < 0: + raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout") + try: + kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size")) + except ValueError as err: + raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err + try: + kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size")) + except ValueError as err: + raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err + if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1: + raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots") + + add(urls=urls, **kwargs) if __name__ == "__main__": diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index efe95fe3..9e99133d 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -77,7 +77,7 @@ def process_stdin_records() -> int: from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.core.models import Snapshot, ArchiveResult from archivebox.crawls.models import Crawl - from archivebox.core.shutdown_util import foreground_parent_watchdog + from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals from archivebox.machine.models import Binary from archivebox.services.runner import run_binary, run_crawl @@ -229,7 +229,7 @@ def process_stdin_records() -> int: if not crawl.claim_processing_lock(lock_seconds=10): rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr) return 1 - with foreground_parent_watchdog(): + with foreground_shutdown_signals(), foreground_parent_watchdog(): run_crawl( crawl_id, snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]), @@ -248,7 +248,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int: Returns exit code (0 = success, 1 = error). """ from archivebox.config import CONSTANTS - from archivebox.core.shutdown_util import foreground_parent_watchdog + from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals from archivebox.machine.models import Machine, Process from archivebox.services.supervision_service import healthy_orchestrator from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls @@ -263,7 +263,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int: return 0 current.mark_running(process_type=Process.TypeChoices.ORCHESTRATOR, pwd=str(CONSTANTS.DATA_DIR), timeout=0) try: - with foreground_parent_watchdog(enabled=not daemon): + with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon): run_pending_crawls(daemon=daemon, crawl_id=crawl_id) return 0 except KeyboardInterrupt: @@ -324,14 +324,14 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str): def run_snapshot_worker(snapshot_id: str) -> int: - from archivebox.core.shutdown_util import foreground_parent_watchdog + from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals from archivebox.core.models import Snapshot from archivebox.services.runner import run_due_snapshot from django.utils import timezone snapshot = None try: - with foreground_parent_watchdog(): + with foreground_shutdown_signals(), foreground_parent_watchdog(): snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id) if snapshot.retry_at is None: Snapshot.objects.filter(pk=snapshot.pk).update(retry_at=timezone.now(), modified_at=timezone.now()) diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 707f5752..eda1d1c8 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -102,6 +102,7 @@ def reindex_snapshots( search_plugins: list[str], batch_size: int, collect_ids: bool = False, + wait_for_turn=None, ) -> dict[str, Any]: from archivebox.cli.archivebox_extract import run_plugins @@ -114,6 +115,8 @@ def reindex_snapshots( def run_batch() -> None: if not records: return + if wait_for_turn: + wait_for_turn() batch_records = list(records) # Index-only backfill intentionally queues only search ArchiveResult # rows. The extract runner bumps Snapshot.retry_at so the orchestrator @@ -201,11 +204,15 @@ def update( setup_django() from archivebox.machine.models import Process - from archivebox.core.shutdown_util import foreground_parent_watchdog - from archivebox.services.supervision_service import current_command, ensure_daemon_stack + from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals + from archivebox.services.supervision_service import current_command, ensure_daemon_stack, standby_until_runtime_stack_needed from archivebox.workers.supervisord_util import stop_existing_supervisord_process command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR) + + def wait_for_turn() -> None: + standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR) + is_filtered_update = any( ( filter_patterns, @@ -229,10 +236,11 @@ def update( # Run migrations first to ensure DB schema is up-to-date print("[*] Checking for pending migrations...") check_migrations(auto_apply=True) + wait_for_turn() if stop_daemon_stack: stop_existing_supervisord_process() - with foreground_parent_watchdog(): + with foreground_shutdown_signals(), foreground_parent_watchdog(): while True: do_migrate = migrate_only or not index_only do_index = index_only or not migrate_only @@ -269,6 +277,7 @@ def update( resume=resume, batch_size=batch_size, queue_for_archiving=do_run_until_idle, + wait_for_turn=wait_for_turn, ) print_stats(stats) touched_snapshot_ids.update(stats.get("snapshot_ids", [])) @@ -282,7 +291,11 @@ def update( ) print("[*] Phase 2: Processing all database snapshots (most recent first)...") - stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume) + stats_combined["phase2"] = process_all_db_snapshots( + batch_size=batch_size, + resume=resume, + wait_for_turn=wait_for_turn, + ) print_combined_stats(stats_combined) if do_index: @@ -311,6 +324,7 @@ def update( search_plugins=search_plugins, batch_size=batch_size, collect_ids=is_filtered_update, + wait_for_turn=wait_for_turn, ) print_index_stats(stats) touched_snapshot_ids.update(stats.get("snapshot_ids", [])) @@ -540,7 +554,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100 return stats -def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -> dict[str, int]: +def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None, wait_for_turn=None) -> dict[str, int]: """ O(n) scan over entire DB from most recent to least recent. @@ -577,6 +591,8 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - updated = 0 checked = 0 while True: + if wait_for_turn: + wait_for_turn() ids = list(rows.order_by("-timestamp").values_list("id", flat=True)[:batch_size]) if not ids: if updated: @@ -614,6 +630,8 @@ def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) - def queue_stale_fs_batch() -> None: if not stale_batch: return + if wait_for_turn: + wait_for_turn() now = timezone.now() snapshot_ids = [snapshot_id for snapshot_id, _crawl_id, _timestamp in stale_batch] # Do not bump fs_version here. The orchestrator calls Snapshot.save(), @@ -676,6 +694,7 @@ def process_filtered_snapshots( resume: str | None, batch_size: int, queue_for_archiving: bool = True, + wait_for_turn=None, ) -> dict[str, Any]: """Process snapshots matching filters (DB query only).""" from archivebox.core.models import Snapshot @@ -703,6 +722,8 @@ def process_filtered_snapshots( print(f"[*] Found {total} matching snapshots") for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size): + if wait_for_turn and stats["processed"] % batch_size == 0: + wait_for_turn() stats["processed"] += 1 # Skip snapshots with missing crawl references @@ -815,7 +836,10 @@ def print_index_stats(stats: dict[str, Any]) -> None: @click.argument("filter_patterns", nargs=-1) @docstring(update.__doc__) def main(**kwargs): - update(**kwargs) + from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals + + with foreground_shutdown_signals(), foreground_parent_watchdog(): + update(**kwargs) if __name__ == "__main__": diff --git a/archivebox/core/migrations/0027_copy_archiveresult_to_process.py b/archivebox/core/migrations/0027_copy_archiveresult_to_process.py index 37c4f8df..4c1e32f3 100644 --- a/archivebox/core/migrations/0027_copy_archiveresult_to_process.py +++ b/archivebox/core/migrations/0027_copy_archiveresult_to_process.py @@ -140,35 +140,33 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version): cursor.execute("PRAGMA table_info(machine_binary)") binary_cols = {row[1] for row in cursor.fetchall()} - # Use only columns that exist in current schema - # 0.8.x schema: id, created_at, modified_at, machine_id, name, binprovider, abspath, version, sha256, num_uses_failed, num_uses_succeeded - # 0.9.x schema adds: binproviders, overrides, status, retry_at, output_dir - if "binproviders" in binary_cols: - # 0.9.x schema - cursor.execute( - """ - INSERT INTO machine_binary ( - id, created_at, modified_at, machine_id, - name, binproviders, overrides, binprovider, abspath, version, sha256, - status, retry_at, output_dir, - num_uses_failed, num_uses_succeeded - ) VALUES (?, ?, ?, ?, ?, 'env', '{}', 'env', ?, ?, '', - 'succeeded', NULL, '', 0, 0) + values_by_col = { + "id": binary_id, + "created_at": now, + "modified_at": now, + "machine_id": machine_id, + "name": name, + "binproviders": "env", + "overrides": "{}", + "binprovider": "env", + "abspath": abspath, + "version": version, + "sha256": "", + "status": "installed", + "retry_at": None, + "output_dir": "", + "num_uses_failed": 0, + "num_uses_succeeded": 0, + } + insert_cols = [col for col in values_by_col if col in binary_cols] + placeholders = ", ".join(["?"] * len(insert_cols)) + cursor.execute( + f""" + INSERT INTO machine_binary ({", ".join(insert_cols)}) + VALUES ({placeholders}) """, - [binary_id, now, now, machine_id, name, abspath, version], - ) - else: - # 0.8.x schema (simpler) - cursor.execute( - """ - INSERT INTO machine_binary ( - id, created_at, modified_at, machine_id, - name, binprovider, abspath, version, sha256, - num_uses_failed, num_uses_succeeded - ) VALUES (?, ?, ?, ?, ?, 'env', ?, ?, '', 0, 0) - """, - [binary_id, now, now, machine_id, name, abspath, version], - ) + [values_by_col[col] for col in insert_cols], + ) return binary_id diff --git a/archivebox/core/shutdown_util.py b/archivebox/core/shutdown_util.py index a51de6b0..030a0b1c 100644 --- a/archivebox/core/shutdown_util.py +++ b/archivebox/core/shutdown_util.py @@ -1,9 +1,9 @@ from __future__ import annotations +import os import signal import subprocess import sys -import os import threading from collections.abc import Iterator from contextlib import contextmanager @@ -101,8 +101,7 @@ def foreground_shutdown_signals( def raise_keyboard_interrupt(signum, _frame): state.signal_name = signal.Signals(signum).name - sys.stdout.write(f"\n[🛑] Got {state.signal_name}, stopping gracefully...\n") - sys.stdout.flush() + os.write(sys.stdout.fileno(), f"\n[🛑] Got {state.signal_name}, stopping gracefully...\n".encode()) raise KeyboardInterrupt try: @@ -119,7 +118,7 @@ def foreground_parent_watchdog( *, enabled: bool = True, check_interval: float = 2.0, - shutdown_signal: signal.Signals = signal.SIGINT, + shutdown_signal: signal.Signals = signal.SIGTERM, ) -> Iterator[None]: """Ask a foreground command to exit if its launcher/wrapper disappears. diff --git a/archivebox/machine/migrations/0005_converge_binary_model.py b/archivebox/machine/migrations/0005_converge_binary_model.py index 593d9637..7e861968 100644 --- a/archivebox/machine/migrations/0005_converge_binary_model.py +++ b/archivebox/machine/migrations/0005_converge_binary_model.py @@ -23,8 +23,8 @@ def converge_binary_table(apps, schema_editor): print("✓ Dropping machine_installedbinary table (0.8.6rc0 divergence)") cursor.execute("DROP TABLE IF EXISTS machine_installedbinary") - # Create Binary table if it doesn't exist - # This handles the case where 0.8.6rc0's 0001_initial didn't create it + # Create Binary table if it doesn't exist. + # This handles the case where 0.8.6rc0's 0001_initial didn't create it. if "machine_binary" not in existing_tables: print("✓ Creating machine_binary table with correct schema") cursor.execute(""" @@ -56,6 +56,28 @@ def converge_binary_table(apps, schema_editor): print("✓ machine_binary table created") else: print("✓ machine_binary table already exists") + cursor.execute("PRAGMA table_info(machine_binary)") + binary_cols = {row[1] for row in cursor.fetchall()} + + # Old 0.8.x data dirs already have machine_binary, but with the + # pre-abxpkg shape. Converge it here before later migrations and + # runtime code expect Binary.binproviders / Binary.status to exist. + if "binproviders" not in binary_cols: + cursor.execute("ALTER TABLE machine_binary ADD COLUMN binproviders VARCHAR(255) NOT NULL DEFAULT 'env'") + if "overrides" not in binary_cols: + cursor.execute("ALTER TABLE machine_binary ADD COLUMN overrides TEXT NOT NULL DEFAULT '{}'") + if "status" not in binary_cols: + cursor.execute("ALTER TABLE machine_binary ADD COLUMN status VARCHAR(16) NOT NULL DEFAULT 'installed'") + if "retry_at" not in binary_cols: + cursor.execute("ALTER TABLE machine_binary ADD COLUMN retry_at DATETIME NULL") + if "output_dir" not in binary_cols: + cursor.execute("ALTER TABLE machine_binary ADD COLUMN output_dir VARCHAR(255) NOT NULL DEFAULT ''") + + cursor.execute( + "UPDATE machine_binary SET binproviders = COALESCE(NULLIF(binproviders, ''), COALESCE(NULLIF(binprovider, ''), 'env'))", + ) + cursor.execute("UPDATE machine_binary SET overrides = COALESCE(NULLIF(overrides, ''), '{}')") + cursor.execute("UPDATE machine_binary SET status = COALESCE(NULLIF(status, ''), 'installed')") class Migration(migrations.Migration): diff --git a/bin/fuzz_test.sh b/bin/fuzz_test.sh index ae2e5336..c508ac47 100755 --- a/bin/fuzz_test.sh +++ b/bin/fuzz_test.sh @@ -150,7 +150,7 @@ run_with_timeout() { ) & local watchdog=$! - trap 'kill_tree "$child"; kill "$watchdog" >/dev/null 2>&1 || true' INT TERM + trap '[[ -n "${child:-}" ]] && kill_tree "$child"; [[ -n "${watchdog:-}" ]] && kill "$watchdog" >/dev/null 2>&1 || true' INT TERM wait "$child" local code=$? @@ -190,7 +190,7 @@ run_server_for_a_bit() { ) >> "$logfile" 2>&1 & local child=$! - trap 'kill_tree "$child"' INT TERM + trap '[[ -n "${child:-}" ]] && kill_tree "$child"' INT TERM sleep "$hold" echo "[$(ts)] STOP label=$label pid=$child after=${hold}s" | tee -a "$logfile" diff --git a/bin/take_screenshot.js b/bin/take_screenshot.js index 47d235da..9b0f8771 100755 --- a/bin/take_screenshot.js +++ b/bin/take_screenshot.js @@ -23,6 +23,7 @@ Environment: SCREENSHOT_FULL_PAGE Set to 1 to capture the full page, defaults to viewport only SCREENSHOT_SCROLL_SELECTOR Scroll this selector into view before capture SCREENSHOT_WAIT_SELECTOR Wait for this selector before capture + SCREENSHOT_HOST_RESOLVER_RULES Chrome host resolver rules SCREENSHOT_SNAPSHOT_VIEW Set to list or grid before loading the page SCREENSHOT_RESET_FILTERS Set to 1 to clear the admin filter collapsed preference `); @@ -60,6 +61,9 @@ async function main() { headless: true, defaultViewport: { width, height }, }; + if (process.env.SCREENSHOT_HOST_RESOLVER_RULES) { + launchOptions.args = [`--host-resolver-rules=${process.env.SCREENSHOT_HOST_RESOLVER_RULES}`]; + } const executablePath = chromePath(); if (executablePath) { launchOptions.executablePath = executablePath; diff --git a/etc/package.json b/etc/package.json index 93bf47c6..671abc01 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.33rc13", + "version": "0.9.33rc14", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index dc0698c9..b8775775 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.33rc13" +version = "0.9.33rc14" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.7", # EventBus API - "abxpkg>=1.11.37", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.43", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.43", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.38", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.44", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.44", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]