The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

fix recursive crawls and docker runtime

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-13 08:13:33 -07:00
parent 5d60098618
commit 5f8098c632
No known key found for this signature in database
12 changed files with 244 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Dockerfile
View File

@ -133,7 +133,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
# 1. packaging dependencies
apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
# 2. docker and init system dependencies
zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
zlib1g-dev dumb-init gosu cron unzip grep dnsutils python3.12-venv \
# 3. frivolous CLI helpers to make debugging failed archiving easier
tree nano iputils-ping \
# nano iputils-ping dnsutils htop procps jq yq
@ -302,7 +302,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
# Install Node extractor dependencies
ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH" \
PERSONAS_DIR=/data/personas \
NODE_PATH="/home/$ARCHIVEBOX_USER/.npm/lib/node_modules:/usr/lib/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules" \
NODE_PATH="/home/$ARCHIVEBOX_USER/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules" \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser \
CHROME_BIN=/usr/bin/chromium-browser \
CHROME_BINARY=/usr/bin/chromium-browser \
@ -378,7 +378,8 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$T
# Setup ArchiveBox runtime config
ENV TMP_DIR=/tmp/archivebox \
LIB_DIR=/usr/share/archivebox/lib \
LIB_DIR=/data/lib \
PIP_VENV_PYTHON=/usr/bin/python3.12 \
GOOGLE_API_KEY=no \
GOOGLE_DEFAULT_CLIENT_ID=no \
GOOGLE_DEFAULT_CLIENT_SECRET=no

6
archivebox/core/settings.py
View File

@ -226,6 +226,8 @@ TEMPLATES = [
# CACHE_DB_TABLE = 'django_cache'
DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE))
SQLITE_JOURNAL_MODE = os.environ.get("ARCHIVEBOX_SQLITE_JOURNAL_MODE", "TRUNCATE" if CONSTANTS.IN_DOCKER else "WAL")
SQLITE_MMAP_SIZE = os.environ.get("ARCHIVEBOX_SQLITE_MMAP_SIZE", "0" if CONSTANTS.IN_DOCKER else "134217728")
SQLITE_CONNECTION_OPTIONS = {
"ENGINE": "django.db.backends.sqlite3",
@ -240,10 +242,10 @@ SQLITE_CONNECTION_OPTIONS = {
"init_command": (
"PRAGMA foreign_keys=ON;"
"PRAGMA busy_timeout = 30000;"
"PRAGMA journal_mode = WAL;"
f"PRAGMA journal_mode = {SQLITE_JOURNAL_MODE};"
"PRAGMA synchronous = NORMAL;"
"PRAGMA temp_store = MEMORY;"
"PRAGMA mmap_size = 134217728;"
f"PRAGMA mmap_size = {SQLITE_MMAP_SIZE};"
"PRAGMA journal_size_limit = 67108864;"
"PRAGMA cache_size = 2000;"
),

91
archivebox/crawls/models.py
View File

@ -450,6 +450,40 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
entries.append((raw_line.rstrip(), stripped))
return entries
def count_urls_for_limit(self) -> int:
"""
Count unique URLs already queued or snapshotted for this crawl.
max_urls is a crawl-wide cap on snapshots, so direct URL entries and
recursively discovered snapshots both have to consume the same budget.
"""
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
urls = set(self.snapshot_set.values_list("url", flat=True))
for _raw_line, raw_url in self._iter_url_lines():
url = sanitize_extracted_url(fix_url_from_markdown(str(raw_url or "").strip()))
if url:
urls.add(url)
return len(urls)
def remaining_url_capacity(self) -> int | None:
if self.max_urls <= 0:
return None
return max(self.max_urls - self.count_urls_for_limit(), 0)
def has_remaining_url_capacity(self) -> bool:
remaining = self.remaining_url_capacity()
return remaining is None or remaining > 0
def remaining_snapshot_capacity(self) -> int | None:
if self.max_urls <= 0:
return None
return max(self.max_urls - self.snapshot_set.count(), 0)
def has_remaining_snapshot_capacity(self) -> bool:
remaining = self.remaining_snapshot_capacity()
return remaining is None or remaining > 0
def prune_urls(self, predicate) -> list[str]:
kept_lines: list[str] = []
removed_urls: list[str] = []
@ -562,6 +596,9 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
if url in existing_urls:
return False
if not self.has_remaining_url_capacity():
return False
# Append as JSONL
entry = {**entry, "url": url}
jsonl_entry = json.dumps(entry)
@ -609,6 +646,10 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
if depth > self.max_depth:
continue
# Stop creating new snapshots once the crawl-wide URL cap is reached.
if not self.has_remaining_snapshot_capacity():
break
# Create snapshot if doesn't exist
snapshot, created = Snapshot.objects.get_or_create(
url=url,
@ -637,6 +678,56 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
return created_snapshots
def create_discovered_snapshot(
self,
parent_snapshot,
*,
url: str,
depth: int,
title: str = "",
tags: str = "",
created_by_id: int | None = None,
):
"""Create one child snapshot if it passes crawl filters and limits."""
from archivebox.core.models import Snapshot
from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
url = sanitize_extracted_url(fix_url_from_markdown(str(url or "").strip()))
if not url:
return None
if depth > self.max_depth:
return None
if not self.url_passes_filters(url, snapshot=parent_snapshot):
return None
if self.snapshot_set.filter(url=url).exists():
return None
if not self.has_remaining_snapshot_capacity():
return None
snapshot = Snapshot.from_json(
{
"url": url,
"depth": depth,
"title": title,
"tags": tags,
"parent_snapshot_id": str(parent_snapshot.id),
"crawl_id": str(self.id),
},
overrides={
"crawl": self,
"snapshot": parent_snapshot,
"created_by_id": created_by_id or self.created_by_id,
},
queue_for_extraction=False,
)
if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED:
return None
snapshot.status = Snapshot.StatusChoices.QUEUED
snapshot.retry_at = timezone.now()
snapshot.save(update_fields=["status", "retry_at", "modified_at"])
return snapshot
def install_declared_binaries(self, binary_names: set[str], machine=None) -> None:
"""
Install crawl-declared Binary rows without violating the retry_at lock lifecycle.

41
archivebox/services/runner.py
View File

@ -400,30 +400,18 @@ class CrawlRunner:
url = str(record.get("url") or "").strip()
if not url:
continue
passes_filters = await sync_to_async(self.crawl.url_passes_filters, thread_sensitive=True)(url, snapshot=parent_snapshot)
if not passes_filters:
continue
child_snapshot = await sync_to_async(Snapshot.from_json, thread_sensitive=True)(
{
"url": url,
"depth": parent_snapshot.depth + 1,
"title": str(record.get("title") or "").strip(),
"tags": str(record.get("tags") or "").strip(),
"parent_snapshot_id": str(parent_snapshot.id),
"crawl_id": str(self.crawl.id),
},
overrides={
"crawl": self.crawl,
"snapshot": parent_snapshot,
"created_by_id": self.crawl.created_by_id,
},
queue_for_extraction=False,
child_snapshot = await sync_to_async(self.crawl.create_discovered_snapshot, thread_sensitive=True)(
parent_snapshot,
url=url,
depth=parent_snapshot.depth + 1,
title=str(record.get("title") or "").strip(),
tags=str(record.get("tags") or "").strip(),
)
if child_snapshot is None or child_snapshot.status == child_snapshot.StatusChoices.SEALED:
continue
child_snapshot.status = child_snapshot.StatusChoices.QUEUED
child_snapshot.retry_at = timezone.now()
await child_snapshot.asave(update_fields=["status", "retry_at", "modified_at"])
if child_snapshot is None:
has_capacity = await sync_to_async(self.crawl.has_remaining_snapshot_capacity, thread_sensitive=True)()
if has_capacity:
continue
break
if self.process_discovered_snapshots_inline:
await self.enqueue_snapshot(str(child_snapshot.id))
@ -584,6 +572,7 @@ class CrawlRunner:
event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
),
).now()
await self.bus.wait_until_idle()
await self.enqueue_discovered_snapshots_from_outputs(snapshot)
finally:
current_task = asyncio.current_task()
@ -921,7 +910,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
if queued_crawl is not None:
if not queued_crawl.claim_processing_lock(lock_seconds=60):
continue
run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=False)
run_crawl(str(queued_crawl.id), process_discovered_snapshots_inline=True)
continue
if crawl_id is None:
@ -938,7 +927,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
run_crawl(
str(snapshot.crawl_id),
snapshot_ids=[str(snapshot.id)],
process_discovered_snapshots_inline=False,
process_discovered_snapshots_inline=True,
)
continue
@ -975,4 +964,4 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
if not crawl.claim_processing_lock(lock_seconds=60):
continue
run_crawl(str(crawl.id), process_discovered_snapshots_inline=False)
run_crawl(str(crawl.id), process_discovered_snapshots_inline=True)

34
archivebox/services/snapshot_service.py
View File

@ -4,7 +4,6 @@ from pathlib import Path
from asgiref.sync import sync_to_async
from django.utils import timezone
from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
from abx_dl.limits import CrawlLimitState
from abx_dl.services.base import BaseService
@ -22,40 +21,21 @@ class SnapshotService(BaseService):
self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent)
async def _upsert_discovered_snapshot(self, parent_snapshot, *, url: str, depth: int, title: str = "", tags: str = "") -> str | None:
from archivebox.core.models import Snapshot
crawl = parent_snapshot.crawl
if depth > crawl.max_depth:
return None
stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(crawl)
if stop_reason == "max_size":
return None
passes_filters = await sync_to_async(crawl.url_passes_filters, thread_sensitive=True)(url, snapshot=parent_snapshot)
if not passes_filters:
return None
snapshot = await sync_to_async(Snapshot.from_json, thread_sensitive=True)(
{
"url": url,
"depth": depth,
"title": title,
"tags": tags,
"parent_snapshot_id": str(parent_snapshot.id),
"crawl_id": str(crawl.id),
},
overrides={
"crawl": crawl,
"snapshot": parent_snapshot,
"created_by_id": crawl.created_by_id,
},
queue_for_extraction=False,
snapshot = await sync_to_async(crawl.create_discovered_snapshot, thread_sensitive=True)(
parent_snapshot,
url=url,
depth=depth,
title=title,
tags=tags,
)
if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED:
if snapshot is None:
return None
snapshot.status = Snapshot.StatusChoices.QUEUED
snapshot.retry_at = timezone.now()
await snapshot.asave(update_fields=["status", "retry_at", "modified_at"])
return str(snapshot.id)
async def on_SnapshotEvent(self, event: SnapshotEvent) -> None:

24
archivebox/tests/test_crawl_admin.py
View File

@ -200,6 +200,30 @@ def test_create_snapshots_from_urls_respects_url_allowlist_and_denylist(admin_us
assert [snapshot.url for snapshot in created] == ["https://example.com/root"]
def test_create_snapshots_from_urls_respects_max_urls(admin_user):
crawl = Crawl.objects.create(
urls="\n".join(
[
"https://example.com/root",
"https://example.com/about",
"https://example.com/contact",
],
),
max_urls=2,
created_by=admin_user,
)
created = crawl.create_snapshots_from_urls()
assert [snapshot.url for snapshot in created] == [
"https://example.com/root",
"https://example.com/about",
]
assert crawl.snapshot_set.count() == 2
assert crawl.remaining_snapshot_capacity() == 0
assert crawl.add_url({"url": "https://example.com/extra", "depth": 1}) is False
def test_url_filter_regex_lists_preserve_commas_and_split_on_newlines_only(admin_user):
crawl = Crawl.objects.create(
urls="\n".join(

57
archivebox/tests/test_recursive_crawl.py
View File

@ -314,6 +314,63 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, process, disable_extract
assert max_depth_found <= 1, f"Max depth should not exceed 1, got {max_depth_found}. Depth distribution: {depth_counts}"
def test_recursive_crawl_respects_max_urls(tmp_path, process, disable_extractors_dict, recursive_test_site):
"""Test that recursive discovery stops creating snapshots at max_urls."""
os.chdir(tmp_path)
env = disable_extractors_dict.copy()
env.update(
{
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
"SAVE_WGET": "true",
"USE_CHROME": "false",
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
},
)
result = subprocess.run(
[
"archivebox",
"add",
"--depth=2",
"--max-urls=4",
"--plugins=wget,parse_html_urls",
recursive_test_site["root_url"],
],
capture_output=True,
text=True,
env=env,
timeout=120,
)
stdout, stderr = result.stdout, result.stderr
if stderr:
print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n")
if stdout:
print(f"\n=== STDOUT (last 2000 chars) ===\n{stdout[-2000:]}\n=== END STDOUT ===\n")
assert result.returncode == 0, result.stderr
conn = sqlite3.connect("index.sqlite3")
c = conn.cursor()
crawl = c.execute(
"SELECT max_depth, max_urls, json_extract(config, '$.MAX_URLS') FROM crawls_crawl ORDER BY created_at DESC LIMIT 1",
).fetchone()
snapshot_rows = c.execute("SELECT url, depth, parent_snapshot_id FROM core_snapshot ORDER BY depth, url").fetchall()
depth_counts = dict(c.execute("SELECT depth, COUNT(*) FROM core_snapshot GROUP BY depth ORDER BY depth").fetchall())
conn.close()
assert crawl == (2, 4, 4)
assert len(snapshot_rows) == 4
assert depth_counts.get(0, 0) == 1
assert depth_counts.get(1, 0) == 3
assert depth_counts.get(2, 0) == 0
assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 1})
def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_path, process, recursive_test_site):
"""Run a real depth=2 crawl and verify DB, output files, and process side effects."""
os.chdir(tmp_path)

6
archivebox/tests/test_runner.py
View File

@ -781,7 +781,7 @@ def test_run_pending_crawls_runs_due_snapshot_in_place(monkeypatch):
result = runner_module.run_pending_crawls(daemon=False)
assert result == 0
assert run_calls == [(str(crawl.id), [str(snapshot.id)], False)]
assert run_calls == [(str(crawl.id), [str(snapshot.id)], True)]
def test_run_pending_crawls_prioritizes_new_queued_crawl_before_snapshot_backlog(monkeypatch):
@ -826,7 +826,7 @@ def test_run_pending_crawls_prioritizes_new_queued_crawl_before_snapshot_backlog
with pytest.raises(_StopScheduling):
runner_module.run_pending_crawls(daemon=False)
assert run_calls == [(str(newer_crawl.id), None, False)]
assert run_calls == [(str(newer_crawl.id), None, True)]
def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_backlog(monkeypatch):
@ -870,5 +870,5 @@ def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_bac
with pytest.raises(_StopScheduling):
runner_module.run_pending_crawls(daemon=False)
assert run_calls == [(str(queued_crawl.id), None, False)]
assert run_calls == [(str(queued_crawl.id), None, True)]
assert binary_calls == []

16
bin/docker_entrypoint.sh
View File

@ -27,7 +27,7 @@ set -o pipefail
# Load global invariants (set by Dockerfile during image build time, not intended to be customized by users at runtime)
export DATA_DIR="${DATA_DIR:-/data}"
export TMP_DIR="${TMP_DIR:-/tmp/archivebox}"
export LIB_DIR="${LIB_DIR:-/usr/share/archivebox/lib}"
export LIB_DIR="${LIB_DIR:-$DATA_DIR/lib}"
export ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
# Global default PUID and PGID if data dir is empty and no intended PUID+PGID is set manually by user
@ -49,7 +49,7 @@ export DETECTED_PGID="$(stat -c '%g' "$DATA_DIR/logs/errors.log" 2>/dev/null ||
# If data directory exists but is owned by root, use defaults instead of root because root is not allowed
[[ "$DETECTED_PUID" == "0" ]] && export DETECTED_PUID="$DEFAULT_PUID"
# (GUID / DETECTED_GUID is allowed to be 0 though)
[[ "$DETECTED_PGID" == "0" ]] && export DETECTED_PGID="$DEFAULT_PGID"
# Set archivebox user and group ids to desired PUID/PGID
usermod -o -u "${PUID:-$DETECTED_PUID}" "$ARCHIVEBOX_USER" > /dev/null 2>&1
@ -106,6 +106,17 @@ if ! chown $PUID:$PGID "$DATA_DIR"/* > /dev/null 2>&1; then
find "$DATA_DIR" -type d -not -path "$DATA_DIR/archive*" -exec chown $PUID:$PGID {} \; > /dev/null 2>&1
find "$DATA_DIR" -type f -not -path "$DATA_DIR/archive/*" -exec chown $PUID:$PGID {} \; > /dev/null 2>&1
fi
chmod -R a+rwX \
"$DATA_DIR" \
"$DATA_DIR"/logs \
"$DATA_DIR"/users \
"$DATA_DIR"/sources \
"$DATA_DIR"/archive \
"$DATA_DIR"/personas \
"$DATA_DIR"/lib \
"$DATA_DIR"/index.sqlite3 \
"$DATA_DIR"/ArchiveBox.conf \
2>/dev/null || true
# Active browser processes do not survive container restarts, but their lock
# files can. Clear stale browser state before dropping privileges.
@ -134,6 +145,7 @@ fi
# also create and chown tmp dir and lib dir (and their default equivalents inside data/)
# mkdir -p "$DATA_DIR"/lib/bin
# chown $PUID:$PGID "$DATA_DIR"/lib "$DATA_DIR"/lib/*
mkdir -p "$LIB_DIR"
chown $PUID:$PGID "$LIB_DIR" 2>/dev/null
chown $PUID:$PGID "$LIB_DIR/*" 2>/dev/null &

3
docker-compose.yml
View File

@ -30,7 +30,8 @@ services:
# - SEARCH_BACKEND_HOST_NAME=127.0.0.1
- SEARCH_BACKEND_PASSWORD=SomeSecretPassword
- PERSONAS_DIR=/data/personas
- NODE_PATH=/home/archivebox/.npm/lib/node_modules:/usr/lib/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules
- LIB_DIR=/data/lib
- NODE_PATH=/home/archivebox/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules
- PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
- CHROME_BIN=/usr/bin/chromium-browser
- CHROME_BINARY=/usr/bin/chromium-browser

9
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.30rc2"
version = "0.9.30rc3"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus @ git+https://github.com/ArchiveBox/abxbus.git@event-wait-now-results", # PR #22 EventBus API
"abxpkg>=1.10.5", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.10.31", # shared ArchiveBox plugin package with install_args-only overrides
"abx-dl>=1.10.31", # shared ArchiveBox downloader package with install_args-only overrides
"abxpkg>=1.10.7", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.10.32", # shared ArchiveBox plugin package with install_args-only overrides
"abx-dl>=1.10.34", # shared ArchiveBox downloader package with install_args-only overrides
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]
@ -159,6 +159,7 @@ exclude-newer-package = { abx-plugins = "1 second", abx-dl = "1 second", abxpkg
# compile-bytecode = true
[tool.uv.sources]
abxpkg = { git = "https://github.com/ArchiveBox/abxpkg.git", branch = "main" }
abx-plugins = { git = "https://github.com/ArchiveBox/abx-plugins.git", branch = "main" }
abx-dl = { git = "https://github.com/ArchiveBox/abx-dl.git", branch = "main" }

30
uv.lock
View File

@ -12,10 +12,20 @@ supported-markers = [
"sys_platform == 'linux'",
]
[options]
exclude-newer = "2026-05-08T11:17:01.494298Z"
exclude-newer-span = "P5D"
[options.exclude-newer-package]
abxbus = { timestamp = "2026-05-13T11:17:00.494317Z", span = "PT1S" }
abx-plugins = { timestamp = "2026-05-13T11:17:00.494305Z", span = "PT1S" }
abx-dl = { timestamp = "2026-05-13T11:17:00.494316Z", span = "PT1S" }
abxpkg = { timestamp = "2026-05-13T11:17:00.494317Z", span = "PT1S" }
[[package]]
name = "abx-dl"
version = "1.10.31"
source = { git = "https://github.com/ArchiveBox/abx-dl.git?branch=main#a0748a58b8d40be8c61fb745791480e29600fa17" }
version = "1.10.34"
source = { git = "https://github.com/ArchiveBox/abx-dl.git?branch=main#c184cc89895d472ec84e2d46e92e6cc8ec779f8b" }
dependencies = [
{ name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -32,8 +42,8 @@ dependencies = [
[[package]]
name = "abx-plugins"
version = "1.10.31"
source = { git = "https://github.com/ArchiveBox/abx-plugins.git?branch=main#6adb9736f9bd6ec5ba7f7c0d22d29227e1cae4f8" }
version = "1.10.32"
source = { git = "https://github.com/ArchiveBox/abx-plugins.git?branch=main#7cbb2acf618a22d1cb0d8bcab6656fec2862151e" }
dependencies = [
{ name = "abxpkg", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "jambo", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -55,8 +65,8 @@ dependencies = [
[[package]]
name = "abxpkg"
version = "1.10.6"
source = { registry = "https://pypi.org/simple" }
version = "1.10.7"
source = { git = "https://github.com/ArchiveBox/abxpkg.git?branch=main#9be0672e2a835e2fc2292f91d0ebf744849e72ef" }
dependencies = [
{ name = "pip", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -64,10 +74,6 @@ dependencies = [
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/28/6f/f97bd4f0a3da6f4167f1ef855f9ca41aaccc9a184cffb74d1f17c92bdb40/abxpkg-1.10.6.tar.gz", hash = "sha256:c4dc61fae2ec74cea434fbbca213a8d634b21bf44086d08bc702129b790962b5", size = 187769, upload-time = "2026-04-15T03:12:00.827Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/12/92/9bb3e2c9d16d4385a181e439fb53a2ccfa7ae04e765740e7d7445724acc3/abxpkg-1.10.6-py3-none-any.whl", hash = "sha256:6f0239a8eba1c51c953d1abb7ed0351f06382928d26b8c8ddb1a9716b22487c9", size = 205413, upload-time = "2026-04-15T03:11:59.462Z" },
]
[[package]]
name = "aiofiles"
@ -110,7 +116,7 @@ wheels = [
[[package]]
name = "archivebox"
version = "0.9.30rc2"
version = "0.9.30rc3"
source = { editable = "." }
dependencies = [
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@ -207,7 +213,7 @@ requires-dist = [
{ name = "abx-dl", git = "https://github.com/ArchiveBox/abx-dl.git?branch=main" },
{ name = "abx-plugins", git = "https://github.com/ArchiveBox/abx-plugins.git?branch=main" },
{ name = "abxbus", git = "https://github.com/ArchiveBox/abxbus.git?rev=event-wait-now-results" },
{ name = "abxpkg", specifier = ">=1.10.5" },
{ name = "abxpkg", git = "https://github.com/ArchiveBox/abxpkg.git?branch=main" },
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
{ name = "atomicwrites", specifier = "==1.4.1" },
{ name = "base32-crockford", specifier = ">=0.3.0" },