From 5f8098c6329b3aeca2d9acf8eeb063c10aacf4b4 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 13 May 2026 08:13:33 -0700 Subject: [PATCH] fix recursive crawls and docker runtime --- Dockerfile | 7 +- archivebox/core/settings.py | 6 +- archivebox/crawls/models.py | 91 ++++++++++++++++++++++++ archivebox/services/runner.py | 41 ++++------- archivebox/services/snapshot_service.py | 34 ++------- archivebox/tests/test_crawl_admin.py | 24 +++++++ archivebox/tests/test_recursive_crawl.py | 57 +++++++++++++++ archivebox/tests/test_runner.py | 6 +- bin/docker_entrypoint.sh | 16 ++++- docker-compose.yml | 3 +- pyproject.toml | 9 +-- uv.lock | 30 ++++---- 12 files changed, 244 insertions(+), 80 deletions(-) diff --git a/Dockerfile b/Dockerfile index 12702dca..62047de1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -133,7 +133,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T # 1. packaging dependencies apt-transport-https ca-certificates apt-utils gnupg2 curl wget \ # 2. docker and init system dependencies - zlib1g-dev dumb-init gosu cron unzip grep dnsutils \ + zlib1g-dev dumb-init gosu cron unzip grep dnsutils python3.12-venv \ # 3. frivolous CLI helpers to make debugging failed archiving easier tree nano iputils-ping \ # nano iputils-ping dnsutils htop procps jq yq @@ -302,7 +302,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T # Install Node extractor dependencies ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH" \ PERSONAS_DIR=/data/personas \ - NODE_PATH="/home/$ARCHIVEBOX_USER/.npm/lib/node_modules:/usr/lib/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules" \ + NODE_PATH="/home/$ARCHIVEBOX_USER/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules" \ PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser \ CHROME_BIN=/usr/bin/chromium-browser \ CHROME_BINARY=/usr/bin/chromium-browser \ @@ -378,7 +378,8 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$T # Setup ArchiveBox runtime config ENV TMP_DIR=/tmp/archivebox \ - LIB_DIR=/usr/share/archivebox/lib \ + LIB_DIR=/data/lib \ + PIP_VENV_PYTHON=/usr/bin/python3.12 \ GOOGLE_API_KEY=no \ GOOGLE_DEFAULT_CLIENT_ID=no \ GOOGLE_DEFAULT_CLIENT_SECRET=no diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index d3f8ef02..802036b6 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -226,6 +226,8 @@ TEMPLATES = [ # CACHE_DB_TABLE = 'django_cache' DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE)) +SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "TRUNCATE" if CONSTANTS.IN_DOCKER else "WAL") +SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728") SQLITE_CONNECTION_OPTIONS = { "ENGINE": "django.db.backends.sqlite3", @@ -240,10 +242,10 @@ SQLITE_CONNECTION_OPTIONS = { "init_command": ( "PRAGMA foreign_keys=ON;" "PRAGMA busy_timeout = 30000;" - "PRAGMA journal_mode = WAL;" + f"PRAGMA journal_mode = {SQLITE_JOURNAL_MODE};" "PRAGMA synchronous = NORMAL;" "PRAGMA temp_store = MEMORY;" - "PRAGMA mmap_size = 134217728;" + f"PRAGMA mmap_size = {SQLITE_MMAP_SIZE};" "PRAGMA journal_size_limit = 67108864;" "PRAGMA cache_size = 2000;" ), diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index 4b5e58d5..e4327549 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -450,6 +450,40 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith entries.append((raw_line.rstrip(), stripped)) return entries + def count_urls_for_limit(self) -> int: + """ + Count unique URLs already queued or snapshotted for this crawl. + + max_urls is a crawl-wide cap on snapshots, so direct URL entries and + recursively discovered snapshots both have to consume the same budget. + """ + from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url + + urls = set(self.snapshot_set.values_list("url", flat=True)) + for _raw_line, raw_url in self._iter_url_lines(): + url = sanitize_extracted_url(fix_url_from_markdown(str(raw_url or "").strip())) + if url: + urls.add(url) + return len(urls) + + def remaining_url_capacity(self) -> int | None: + if self.max_urls <= 0: + return None + return max(self.max_urls - self.count_urls_for_limit(), 0) + + def has_remaining_url_capacity(self) -> bool: + remaining = self.remaining_url_capacity() + return remaining is None or remaining > 0 + + def remaining_snapshot_capacity(self) -> int | None: + if self.max_urls <= 0: + return None + return max(self.max_urls - self.snapshot_set.count(), 0) + + def has_remaining_snapshot_capacity(self) -> bool: + remaining = self.remaining_snapshot_capacity() + return remaining is None or remaining > 0 + def prune_urls(self, predicate) -> list[str]: kept_lines: list[str] = [] removed_urls: list[str] = [] @@ -562,6 +596,9 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith if url in existing_urls: return False + if not self.has_remaining_url_capacity(): + return False + # Append as JSONL entry = {**entry, "url": url} jsonl_entry = json.dumps(entry) @@ -609,6 +646,10 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith if depth > self.max_depth: continue + # Stop creating new snapshots once the crawl-wide URL cap is reached. + if not self.has_remaining_snapshot_capacity(): + break + # Create snapshot if doesn't exist snapshot, created = Snapshot.objects.get_or_create( url=url, @@ -637,6 +678,56 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith return created_snapshots + def create_discovered_snapshot( + self, + parent_snapshot, + *, + url: str, + depth: int, + title: str = "", + tags: str = "", + created_by_id: int | None = None, + ): + """Create one child snapshot if it passes crawl filters and limits.""" + from archivebox.core.models import Snapshot + from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url + + url = sanitize_extracted_url(fix_url_from_markdown(str(url or "").strip())) + if not url: + return None + if depth > self.max_depth: + return None + if not self.url_passes_filters(url, snapshot=parent_snapshot): + return None + if self.snapshot_set.filter(url=url).exists(): + return None + if not self.has_remaining_snapshot_capacity(): + return None + + snapshot = Snapshot.from_json( + { + "url": url, + "depth": depth, + "title": title, + "tags": tags, + "parent_snapshot_id": str(parent_snapshot.id), + "crawl_id": str(self.id), + }, + overrides={ + "crawl": self, + "snapshot": parent_snapshot, + "created_by_id": created_by_id or self.created_by_id, + }, + queue_for_extraction=False, + ) + if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED: + return None + + snapshot.status = Snapshot.StatusChoices.QUEUED + snapshot.retry_at = timezone.now() + snapshot.save(update_fields=["status", "retry_at", "modified_at"]) + return snapshot + def install_declared_binaries(self, binary_names: set[str], machine=None) -> None: """ Install crawl-declared Binary rows without violating the retry_at lock lifecycle. diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index c2d972c7..2b622576 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -400,30 +400,18 @@ class CrawlRunner: url = str(record.get("url") or "").strip() if not url: continue - passes_filters = await sync_to_async(self.crawl.url_passes_filters, thread_sensitive=True)(url, snapshot=parent_snapshot) - if not passes_filters: - continue - child_snapshot = await sync_to_async(Snapshot.from_json, thread_sensitive=True)( - { - "url": url, - "depth": parent_snapshot.depth + 1, - "title": str(record.get("title") or "").strip(), - "tags": str(record.get("tags") or "").strip(), - "parent_snapshot_id": str(parent_snapshot.id), - "crawl_id": str(self.crawl.id), - }, - overrides={ - "crawl": self.crawl, - "snapshot": parent_snapshot, - "created_by_id": self.crawl.created_by_id, - }, - queue_for_extraction=False, + child_snapshot = await sync_to_async(self.crawl.create_discovered_snapshot, thread_sensitive=True)( + parent_snapshot, + url=url, + depth=parent_snapshot.depth + 1, + title=str(record.get("title") or "").strip(), + tags=str(record.get("tags") or "").strip(), ) - if child_snapshot is None or child_snapshot.status == child_snapshot.StatusChoices.SEALED: - continue - child_snapshot.status = child_snapshot.StatusChoices.QUEUED - child_snapshot.retry_at = timezone.now() - await child_snapshot.asave(update_fields=["status", "retry_at", "modified_at"]) + if child_snapshot is None: + has_capacity = await sync_to_async(self.crawl.has_remaining_snapshot_capacity, thread_sensitive=True)() + if has_capacity: + continue + break if self.process_discovered_snapshots_inline: await self.enqueue_snapshot(str(child_snapshot.id)) @@ -584,6 +572,7 @@ class CrawlRunner: event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), ), ).now() + await self.bus.wait_until_idle() await self.enqueue_discovered_snapshots_from_outputs(snapshot) finally: current_task = asyncio.current_task() @@ -921,7 +910,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> if queued_crawl is not None: if not queued_crawl.claim_processing_lock(lock_seconds=60): continue - run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=False) + run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=True) continue if crawl_id is None: @@ -938,7 +927,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> run_crawl( str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], - process_discovered_snapshots_inline=False, + process_discovered_snapshots_inline=True, ) continue @@ -975,4 +964,4 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> if not crawl.claim_processing_lock(lock_seconds=60): continue - run_crawl(str(crawl.id), process_discovered_snapshots_inline=False) + run_crawl(str(crawl.id), process_discovered_snapshots_inline=True) diff --git a/archivebox/services/snapshot_service.py b/archivebox/services/snapshot_service.py index a82b2d74..f12994ab 100644 --- a/archivebox/services/snapshot_service.py +++ b/archivebox/services/snapshot_service.py @@ -4,7 +4,6 @@ from pathlib import Path from asgiref.sync import sync_to_async from django.utils import timezone - from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent from abx_dl.limits import CrawlLimitState from abx_dl.services.base import BaseService @@ -22,40 +21,21 @@ class SnapshotService(BaseService): self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent) async def _upsert_discovered_snapshot(self, parent_snapshot, *, url: str, depth: int, title: str = "", tags: str = "") -> str | None: - from archivebox.core.models import Snapshot - crawl = parent_snapshot.crawl if depth > crawl.max_depth: return None stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(crawl) if stop_reason == "max_size": return None - passes_filters = await sync_to_async(crawl.url_passes_filters, thread_sensitive=True)(url, snapshot=parent_snapshot) - if not passes_filters: - return None - - snapshot = await sync_to_async(Snapshot.from_json, thread_sensitive=True)( - { - "url": url, - "depth": depth, - "title": title, - "tags": tags, - "parent_snapshot_id": str(parent_snapshot.id), - "crawl_id": str(crawl.id), - }, - overrides={ - "crawl": crawl, - "snapshot": parent_snapshot, - "created_by_id": crawl.created_by_id, - }, - queue_for_extraction=False, + snapshot = await sync_to_async(crawl.create_discovered_snapshot, thread_sensitive=True)( + parent_snapshot, + url=url, + depth=depth, + title=title, + tags=tags, ) - if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED: + if snapshot is None: return None - - snapshot.status = Snapshot.StatusChoices.QUEUED - snapshot.retry_at = timezone.now() - await snapshot.asave(update_fields=["status", "retry_at", "modified_at"]) return str(snapshot.id) async def on_SnapshotEvent(self, event: SnapshotEvent) -> None: diff --git a/archivebox/tests/test_crawl_admin.py b/archivebox/tests/test_crawl_admin.py index 62d666f4..e3ab7933 100644 --- a/archivebox/tests/test_crawl_admin.py +++ b/archivebox/tests/test_crawl_admin.py @@ -200,6 +200,30 @@ def test_create_snapshots_from_urls_respects_url_allowlist_and_denylist(admin_us assert [snapshot.url for snapshot in created] == ["https://example.com/root"] +def test_create_snapshots_from_urls_respects_max_urls(admin_user): + crawl = Crawl.objects.create( + urls="\n".join( + [ + "https://example.com/root", + "https://example.com/about", + "https://example.com/contact", + ], + ), + max_urls=2, + created_by=admin_user, + ) + + created = crawl.create_snapshots_from_urls() + + assert [snapshot.url for snapshot in created] == [ + "https://example.com/root", + "https://example.com/about", + ] + assert crawl.snapshot_set.count() == 2 + assert crawl.remaining_snapshot_capacity() == 0 + assert crawl.add_url({"url": "https://example.com/extra", "depth": 1}) is False + + def test_url_filter_regex_lists_preserve_commas_and_split_on_newlines_only(admin_user): crawl = Crawl.objects.create( urls="\n".join( diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 58876b44..5fcfdf65 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -314,6 +314,63 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, process, disable_extract assert max_depth_found <= 1, f"Max depth should not exceed 1, got {max_depth_found}. Depth distribution: {depth_counts}" +def test_recursive_crawl_respects_max_urls(tmp_path, process, disable_extractors_dict, recursive_test_site): + """Test that recursive discovery stops creating snapshots at max_urls.""" + os.chdir(tmp_path) + + env = disable_extractors_dict.copy() + env.update( + { + "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*", + "SAVE_WGET": "true", + "USE_CHROME": "false", + "USE_COLOR": "false", + "SHOW_PROGRESS": "false", + }, + ) + + result = subprocess.run( + [ + "archivebox", + "add", + "--depth=2", + "--max-urls=4", + "--plugins=wget,parse_html_urls", + recursive_test_site["root_url"], + ], + capture_output=True, + text=True, + env=env, + timeout=120, + ) + stdout, stderr = result.stdout, result.stderr + + if stderr: + print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n") + if stdout: + print(f"\n=== STDOUT (last 2000 chars) ===\n{stdout[-2000:]}\n=== END STDOUT ===\n") + + assert result.returncode == 0, result.stderr + + conn = sqlite3.connect("index.sqlite3") + c = conn.cursor() + + crawl = c.execute( + "SELECT max_depth, max_urls, json_extract(config, '$.MAX_URLS') FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", + ).fetchone() + snapshot_rows = c.execute("SELECT url, depth, parent_snapshot_id FROM core_snapshot ORDER BY depth, url").fetchall() + depth_counts = dict(c.execute("SELECT depth, COUNT(*) FROM core_snapshot GROUP BY depth ORDER BY depth").fetchall()) + + conn.close() + + assert crawl == (2, 4, 4) + assert len(snapshot_rows) == 4 + assert depth_counts.get(0, 0) == 1 + assert depth_counts.get(1, 0) == 3 + assert depth_counts.get(2, 0) == 0 + assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 1}) + + def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_path, process, recursive_test_site): """Run a real depth=2 crawl and verify DB, output files, and process side effects.""" os.chdir(tmp_path) diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py index 32721015..92ffb647 100644 --- a/archivebox/tests/test_runner.py +++ b/archivebox/tests/test_runner.py @@ -781,7 +781,7 @@ def test_run_pending_crawls_runs_due_snapshot_in_place(monkeypatch): result = runner_module.run_pending_crawls(daemon=False) assert result == 0 - assert run_calls == [(str(crawl.id), [str(snapshot.id)], False)] + assert run_calls == [(str(crawl.id), [str(snapshot.id)], True)] def test_run_pending_crawls_prioritizes_new_queued_crawl_before_snapshot_backlog(monkeypatch): @@ -826,7 +826,7 @@ def test_run_pending_crawls_prioritizes_new_queued_crawl_before_snapshot_backlog with pytest.raises(_StopScheduling): runner_module.run_pending_crawls(daemon=False) - assert run_calls == [(str(newer_crawl.id), None, False)] + assert run_calls == [(str(newer_crawl.id), None, True)] def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_backlog(monkeypatch): @@ -870,5 +870,5 @@ def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_bac with pytest.raises(_StopScheduling): runner_module.run_pending_crawls(daemon=False) - assert run_calls == [(str(queued_crawl.id), None, False)] + assert run_calls == [(str(queued_crawl.id), None, True)] assert binary_calls == [] diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh index aeba701d..299d95ee 100755 --- a/bin/docker_entrypoint.sh +++ b/bin/docker_entrypoint.sh @@ -27,7 +27,7 @@ set -o pipefail # Load global invariants (set by Dockerfile during image build time, not intended to be customized by users at runtime) export DATA_DIR="${DATA_DIR:-/data}" export TMP_DIR="${TMP_DIR:-/tmp/archivebox}" -export LIB_DIR="${LIB_DIR:-/usr/share/archivebox/lib}" +export LIB_DIR="${LIB_DIR:-$DATA_DIR/lib}" export ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}" # Global default PUID and PGID if data dir is empty and no intended PUID+PGID is set manually by user @@ -49,7 +49,7 @@ export DETECTED_PGID="$(stat -c '%g' "$DATA_DIR/logs/errors.log" 2>/dev/null || # If data directory exists but is owned by root, use defaults instead of root because root is not allowed [[ "$DETECTED_PUID" == "0" ]] && export DETECTED_PUID="$DEFAULT_PUID" -# (GUID / DETECTED_GUID is allowed to be 0 though) +[[ "$DETECTED_PGID" == "0" ]] && export DETECTED_PGID="$DEFAULT_PGID" # Set archivebox user and group ids to desired PUID/PGID usermod -o -u "${PUID:-$DETECTED_PUID}" "$ARCHIVEBOX_USER" > /dev/null 2>&1 @@ -106,6 +106,17 @@ if ! chown $PUID:$PGID "$DATA_DIR"/* > /dev/null 2>&1; then find "$DATA_DIR" -type d -not -path "$DATA_DIR/archive*" -exec chown $PUID:$PGID {} \; > /dev/null 2>&1 find "$DATA_DIR" -type f -not -path "$DATA_DIR/archive/*" -exec chown $PUID:$PGID {} \; > /dev/null 2>&1 fi +chmod -R a+rwX \ + "$DATA_DIR" \ + "$DATA_DIR"/logs \ + "$DATA_DIR"/users \ + "$DATA_DIR"/sources \ + "$DATA_DIR"/archive \ + "$DATA_DIR"/personas \ + "$DATA_DIR"/lib \ + "$DATA_DIR"/index.sqlite3 \ + "$DATA_DIR"/ArchiveBox.conf \ + 2>/dev/null || true # Active browser processes do not survive container restarts, but their lock # files can. Clear stale browser state before dropping privileges. @@ -134,6 +145,7 @@ fi # also create and chown tmp dir and lib dir (and their default equivalents inside data/) # mkdir -p "$DATA_DIR"/lib/bin # chown $PUID:$PGID "$DATA_DIR"/lib "$DATA_DIR"/lib/* +mkdir -p "$LIB_DIR" chown $PUID:$PGID "$LIB_DIR" 2>/dev/null chown $PUID:$PGID "$LIB_DIR/*" 2>/dev/null & diff --git a/docker-compose.yml b/docker-compose.yml index 56b9405a..c99986e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,8 @@ services: # - SEARCH_BACKEND_HOST_NAME=127.0.0.1 - SEARCH_BACKEND_PASSWORD=SomeSecretPassword - PERSONAS_DIR=/data/personas - - NODE_PATH=/home/archivebox/.npm/lib/node_modules:/usr/lib/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules + - LIB_DIR=/data/lib + - NODE_PATH=/home/archivebox/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser - CHROME_BIN=/usr/bin/chromium-browser - CHROME_BINARY=/usr/bin/chromium-browser diff --git a/pyproject.toml b/pyproject.toml index 5c1711b9..06205df6 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.30rc2" +version = "0.9.30rc3" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus @ git+https://github.com/ArchiveBox/abxbus.git@event-wait-now-results", # PR #22 EventBus API - "abxpkg>=1.10.5", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.10.31", # shared ArchiveBox plugin package with install_args-only overrides - "abx-dl>=1.10.31", # shared ArchiveBox downloader package with install_args-only overrides + "abxpkg>=1.10.7", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.10.32", # shared ArchiveBox plugin package with install_args-only overrides + "abx-dl>=1.10.34", # shared ArchiveBox downloader package with install_args-only overrides ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ] @@ -159,6 +159,7 @@ exclude-newer-package = { abx-plugins = "1 second", abx-dl = "1 second", abxpkg # compile-bytecode = true [tool.uv.sources] +abxpkg = { git = "https://github.com/ArchiveBox/abxpkg.git", branch = "main" } abx-plugins = { git = "https://github.com/ArchiveBox/abx-plugins.git", branch = "main" } abx-dl = { git = "https://github.com/ArchiveBox/abx-dl.git", branch = "main" } diff --git a/uv.lock b/uv.lock index 7249f868..d7f8a7e9 100644 --- a/uv.lock +++ b/uv.lock @@ -12,10 +12,20 @@ supported-markers = [ "sys_platform == 'linux'", ] +[options] +exclude-newer = "2026-05-08T11:17:01.494298Z" +exclude-newer-span = "P5D" + +[options.exclude-newer-package] +abxbus = { timestamp = "2026-05-13T11:17:00.494317Z", span = "PT1S" } +abx-plugins = { timestamp = "2026-05-13T11:17:00.494305Z", span = "PT1S" } +abx-dl = { timestamp = "2026-05-13T11:17:00.494316Z", span = "PT1S" } +abxpkg = { timestamp = "2026-05-13T11:17:00.494317Z", span = "PT1S" } + [[package]] name = "abx-dl" -version = "1.10.31" -source = { git = "https://github.com/ArchiveBox/abx-dl.git?branch=main#a0748a58b8d40be8c61fb745791480e29600fa17" } +version = "1.10.34" +source = { git = "https://github.com/ArchiveBox/abx-dl.git?branch=main#c184cc89895d472ec84e2d46e92e6cc8ec779f8b" } dependencies = [ { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -32,8 +42,8 @@ dependencies = [ [[package]] name = "abx-plugins" -version = "1.10.31" -source = { git = "https://github.com/ArchiveBox/abx-plugins.git?branch=main#6adb9736f9bd6ec5ba7f7c0d22d29227e1cae4f8" } +version = "1.10.32" +source = { git = "https://github.com/ArchiveBox/abx-plugins.git?branch=main#7cbb2acf618a22d1cb0d8bcab6656fec2862151e" } dependencies = [ { name = "abxpkg", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "jambo", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -55,8 +65,8 @@ dependencies = [ [[package]] name = "abxpkg" -version = "1.10.6" -source = { registry = "https://pypi.org/simple" } +version = "1.10.7" +source = { git = "https://github.com/ArchiveBox/abxpkg.git?branch=main#9be0672e2a835e2fc2292f91d0ebf744849e72ef" } dependencies = [ { name = "pip", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -64,10 +74,6 @@ dependencies = [ { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/6f/f97bd4f0a3da6f4167f1ef855f9ca41aaccc9a184cffb74d1f17c92bdb40/abxpkg-1.10.6.tar.gz", hash = "sha256:c4dc61fae2ec74cea434fbbca213a8d634b21bf44086d08bc702129b790962b5", size = 187769, upload-time = "2026-04-15T03:12:00.827Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/92/9bb3e2c9d16d4385a181e439fb53a2ccfa7ae04e765740e7d7445724acc3/abxpkg-1.10.6-py3-none-any.whl", hash = "sha256:6f0239a8eba1c51c953d1abb7ed0351f06382928d26b8c8ddb1a9716b22487c9", size = 205413, upload-time = "2026-04-15T03:11:59.462Z" }, -] [[package]] name = "aiofiles" @@ -110,7 +116,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.30rc2" +version = "0.9.30rc3" source = { editable = "." } dependencies = [ { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -207,7 +213,7 @@ requires-dist = [ { name = "abx-dl", git = "https://github.com/ArchiveBox/abx-dl.git?branch=main" }, { name = "abx-plugins", git = "https://github.com/ArchiveBox/abx-plugins.git?branch=main" }, { name = "abxbus", git = "https://github.com/ArchiveBox/abxbus.git?rev=event-wait-now-results" }, - { name = "abxpkg", specifier = ">=1.10.5" }, + { name = "abxpkg", git = "https://github.com/ArchiveBox/abxpkg.git?branch=main" }, { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" }, { name = "atomicwrites", specifier = "==1.4.1" }, { name = "base32-crockford", specifier = ">=0.3.0" },