diff --git a/archivebox/cli/archivebox_pluginmap.py b/archivebox/cli/archivebox_pluginmap.py
index fa0edbe7..f17ddbb9 100644
--- a/archivebox/cli/archivebox_pluginmap.py
+++ b/archivebox/cli/archivebox_pluginmap.py
@@ -14,11 +14,10 @@ EVENT_FLOW_DIAGRAM = """
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ InstallEvent │
-│ └─ on_Install__* │
-│ └─ BinaryRequest records │
-│ └─ BinaryRequestEvent │
-│ └─ on_BinaryRequest__* │
-│ └─ BinaryEvent / MachineEvent │
+│ └─ config.json > required_binaries │
+│ └─ BinaryRequestEvent │
+│ └─ on_BinaryRequest__* │
+│ └─ BinaryEvent │
│ │
│ CrawlEvent │
│ └─ CrawlSetupEvent │
@@ -70,15 +69,15 @@ def pluginmap(
event_phases = {
"InstallEvent": {
- "description": "Pre-run dependency phase. on_Install hooks request binaries and update machine config.",
- "emits": ["BinaryRequestEvent", "BinaryEvent", "MachineEvent", "ProcessEvent"],
+ "description": "Pre-run dependency phase. Enabled plugins emit BinaryRequest events from config.json required_binaries.",
+ "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
},
"BinaryRequestEvent": {
"description": "Provider phase. on_BinaryRequest hooks resolve or install requested binaries.",
- "emits": ["BinaryEvent", "MachineEvent", "ProcessEvent"],
+ "emits": ["BinaryEvent", "ProcessEvent"],
},
"BinaryEvent": {
- "description": "Resolved binary metadata event. Projected into the DB/runtime config.",
+ "description": "Resolved binary metadata event. Projected into the DB binary cache.",
"emits": [],
},
"CrawlEvent": {
@@ -87,11 +86,11 @@ def pluginmap(
},
"CrawlSetupEvent": {
"description": "Crawl-scoped setup phase. on_CrawlSetup hooks launch/configure shared daemons and runtime state.",
- "emits": ["MachineEvent", "ProcessEvent"],
+ "emits": ["ProcessEvent"],
},
"SnapshotEvent": {
- "description": "Per-snapshot extraction phase. on_Snapshot hooks emit ArchiveResult, Snapshot, Tag, Machine, and BinaryRequest records.",
- "emits": ["ArchiveResultEvent", "SnapshotEvent", "TagEvent", "MachineEvent", "BinaryRequestEvent", "ProcessEvent"],
+ "description": "Per-snapshot extraction phase. on_Snapshot hooks emit ArchiveResult, Snapshot, Tag, and BinaryRequest records.",
+ "emits": ["ArchiveResultEvent", "SnapshotEvent", "TagEvent", "BinaryRequestEvent", "ProcessEvent"],
},
"SnapshotCleanupEvent": {
"description": "Internal snapshot cleanup phase.",
diff --git a/archivebox/cli/archivebox_version.py b/archivebox/cli/archivebox_version.py
index a9ec19d6..8d61c64c 100755
--- a/archivebox/cli/archivebox_version.py
+++ b/archivebox/cli/archivebox_version.py
@@ -5,7 +5,6 @@ __package__ = "archivebox.cli"
import sys
import os
import platform
-import logging
from pathlib import Path
from collections.abc import Iterable
@@ -124,17 +123,19 @@ def version(
setup_django()
from archivebox.machine.models import Machine, Binary
- from archivebox.config.views import KNOWN_BINARIES, canonical_binary_name
- from abx_dl.dependencies import load_binary
machine = Machine.current()
- requested_names = {canonical_binary_name(name) for name in binaries} if binaries else set()
+ if isinstance(binaries, str):
+ requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+ else:
+ requested_names = {name for name in (binaries or ()) if name}
- db_binaries = {
- canonical_binary_name(binary.name): binary for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at")
- }
- all_binary_names = sorted(set(KNOWN_BINARIES) | set(db_binaries.keys()))
+ db_binaries: dict[str, Binary] = {}
+ for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+ db_binaries.setdefault(binary.name, binary)
+
+ all_binary_names = sorted(requested_names or set(db_binaries.keys()))
if not all_binary_names:
prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
@@ -163,37 +164,10 @@ def version(
any_available = True
continue
- loaded = None
- try:
- abx_pkg_logger = logging.getLogger("abx_pkg")
- previous_level = abx_pkg_logger.level
- abx_pkg_logger.setLevel(logging.CRITICAL)
- try:
- loaded = load_binary({"name": name, "binproviders": "env,pip,npm,brew,apt"})
- finally:
- abx_pkg_logger.setLevel(previous_level)
- except Exception:
- loaded = None
-
- if loaded and loaded.is_valid and loaded.loaded_abspath:
- display_path = str(loaded.loaded_abspath).replace(str(DATA_DIR), ".").replace(str(Path("~").expanduser()), "~")
- version_str = str(loaded.loaded_version or "unknown")[:15]
- provider = str(getattr(getattr(loaded, "loaded_binprovider", None), "name", "") or "env")[:8]
- prnt(
- "",
- "[green]√[/green]",
- "",
- name.ljust(18),
- version_str.ljust(16),
- provider.ljust(8),
- display_path,
- overflow="ignore",
- crop=False,
- )
- any_available = True
- continue
-
- prnt("", "[red]X[/red]", "", name.ljust(18), "[grey53]not installed[/grey53]", overflow="ignore", crop=False)
+ status = (
+ "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+ )
+ prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
failures.append(name)
if not any_available:
diff --git a/archivebox/config/configset.py b/archivebox/config/configset.py
index 869d1cf0..d14e1bfc 100644
--- a/archivebox/config/configset.py
+++ b/archivebox/config/configset.py
@@ -138,10 +138,9 @@ def get_config(
3. Per-user config (user.config JSON field)
4. Per-persona config (persona.get_derived_config() - includes CHROME_USER_DATA_DIR etc.)
5. Environment variables
- 6. Per-machine config (machine.config JSON field - resolved binary paths)
- 7. Config file (ArchiveBox.conf)
- 8. Plugin schema defaults (config.json)
- 9. Core config defaults
+ 6. Config file (ArchiveBox.conf)
+ 7. Plugin schema defaults (config.json)
+ 8. Core config defaults
Args:
defaults: Default values to start with
@@ -150,7 +149,7 @@ def get_config(
crawl: Crawl object with config JSON field
snapshot: Snapshot object with config JSON field
archiveresult: ArchiveResult object (auto-fetches snapshot)
- machine: Machine object with config JSON field (defaults to Machine.current())
+ machine: Unused legacy argument kept for call compatibility
Note: Objects are auto-fetched from relationships if not provided:
- snapshot auto-fetched from archiveresult.snapshot
@@ -221,19 +220,6 @@ def get_config(
file_config = BaseConfigSet.load_from_file(config_file)
config.update(file_config)
- # Apply machine config overrides (cached binary paths, etc.)
- if machine is None:
- # Default to current machine if not provided
- try:
- from archivebox.machine.models import Machine
-
- machine = Machine.current()
- except Exception:
- pass # Machine might not be available during early init
-
- if machine and hasattr(machine, "config") and machine.config:
- config.update(machine.config)
-
# Override with environment variables (for keys that exist in config)
for key in config:
env_val = os.environ.get(key)
diff --git a/archivebox/config/views.py b/archivebox/config/views.py
index 7c2f99dd..23dbbe59 100644
--- a/archivebox/config/views.py
+++ b/archivebox/config/views.py
@@ -29,42 +29,6 @@ ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
-# Common binaries to check for
-KNOWN_BINARIES = [
- "wget",
- "curl",
- "chromium",
- "chrome",
- "google-chrome",
- "google-chrome-stable",
- "node",
- "npm",
- "npx",
- "yt-dlp",
- "git",
- "singlefile",
- "readability-extractor",
- "mercury-parser",
- "python3",
- "python",
- "bash",
- "zsh",
- "ffmpeg",
- "ripgrep",
- "rg",
- "sonic",
- "archivebox",
-]
-
-CANONICAL_BINARY_ALIASES = {
- "youtube-dl": "yt-dlp",
- "ytdlp": "yt-dlp",
- "ripgrep": "rg",
- "singlefile": "single-file",
- "mercury-parser": "postlight-parser",
-}
-
-
def is_superuser(request: HttpRequest) -> bool:
return bool(getattr(request.user, "is_superuser", False))
@@ -131,13 +95,12 @@ def get_environment_binary_url(name: str) -> str:
return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
-def get_installed_binary_change_url(name: str, binary: Any) -> str | None:
- binary_id = getattr(binary, "id", None)
- if not binary_id:
+def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | None:
+ if binary is None or not binary.id:
return None
- base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary_id}/change/"
- changelist_filters = urlencode({"q": canonical_binary_name(name)})
+ base_url = binary.admin_change_url or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
+ changelist_filters = urlencode({"q": name})
return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
@@ -168,11 +131,14 @@ def render_code_tag_list(values: list[str]) -> str:
def render_plugin_metadata_html(config: dict[str, Any]) -> str:
+ required_binaries = [
+ str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name")
+ ]
rows = (
("Title", config.get("title") or "(none)"),
("Description", config.get("description") or "(none)"),
("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))),
- ("Required Binaries", mark_safe(render_link_tag_list(config.get("required_binaries") or [], get_environment_binary_url))),
+ ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))),
("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))),
)
@@ -383,10 +349,6 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
return f" {str(obj)}"
-def canonical_binary_name(name: str) -> str:
- return CANONICAL_BINARY_ALIASES.get(name, name)
-
-
def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
return (
int(binary.status == Binary.StatusChoices.INSTALLED),
@@ -399,24 +361,11 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
def get_db_binaries_by_name() -> dict[str, Binary]:
grouped: dict[str, list[Binary]] = {}
for binary in Binary.objects.all():
- grouped.setdefault(canonical_binary_name(binary.name), []).append(binary)
+ grouped.setdefault(binary.name, []).append(binary)
return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}
-def serialize_binary_record(name: str, binary: Binary | None) -> dict[str, Any]:
- is_installed = bool(binary and binary.status == Binary.StatusChoices.INSTALLED)
- return {
- "name": canonical_binary_name(name),
- "version": str(getattr(binary, "version", "") or ""),
- "binprovider": str(getattr(binary, "binprovider", "") or ""),
- "abspath": str(getattr(binary, "abspath", "") or ""),
- "sha256": str(getattr(binary, "sha256", "") or ""),
- "status": str(getattr(binary, "status", "") or ""),
- "is_available": is_installed and bool(getattr(binary, "abspath", "") or ""),
- }
-
-
def get_filesystem_plugins() -> dict[str, dict[str, Any]]:
"""Discover plugins from filesystem directories."""
import json
@@ -474,14 +423,14 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
all_binary_names = sorted(db_binaries.keys())
for name in all_binary_names:
- merged = serialize_binary_record(name, db_binaries.get(name))
+ binary = db_binaries.get(name)
rows["Binary Name"].append(ItemLink(name, key=name))
- if merged["is_available"]:
- rows["Found Version"].append(f"✅ {merged['version']}" if merged["version"] else "✅ found")
- rows["Provided By"].append(merged["binprovider"] or "-")
- rows["Found Abspath"].append(merged["abspath"] or "-")
+ if binary and binary.is_valid:
+ rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
+ rows["Provided By"].append(binary.binprovider or "-")
+ rows["Found Abspath"].append(binary.abspath or "-")
else:
rows["Found Version"].append("❌ missing")
rows["Provided By"].append("-")
@@ -496,22 +445,20 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert is_superuser(request), "Must be a superuser to view configuration settings."
- key = canonical_binary_name(key)
db_binary = get_db_binaries_by_name().get(key)
- merged = serialize_binary_record(key, db_binary)
-
- if merged["is_available"]:
+ if db_binary and db_binary.is_valid:
+ binary_data = db_binary.to_json()
section: SectionData = {
"name": key,
- "description": mark_safe(render_binary_detail_description(key, merged, db_binary)),
+ "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),
"fields": {
"name": key,
- "binprovider": merged["binprovider"] or "-",
- "abspath": merged["abspath"] or "not found",
- "version": merged["version"] or "unknown",
- "sha256": merged["sha256"],
- "status": merged["status"],
+ "binprovider": db_binary.binprovider or "-",
+ "abspath": db_binary.abspath or "not found",
+ "version": db_binary.version or "unknown",
+ "sha256": db_binary.sha256,
+ "status": db_binary.status,
},
"help_texts": {},
}
@@ -526,10 +473,10 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
"description": "No persisted Binary record found",
"fields": {
"name": key,
- "binprovider": merged["binprovider"] or "not recorded",
- "abspath": merged["abspath"] or "not recorded",
- "version": merged["version"] or "N/A",
- "status": merged["status"] or "unrecorded",
+ "binprovider": db_binary.binprovider if db_binary else "not recorded",
+ "abspath": db_binary.abspath if db_binary else "not recorded",
+ "version": db_binary.version if db_binary else "N/A",
+ "status": db_binary.status if db_binary else "unrecorded",
},
"help_texts": {},
}
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index 0c11ec49..e356fea4 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -1226,7 +1226,7 @@ def live_progress_view(request):
return (plugin, plugin, "unknown", "")
phase = "unknown"
- if normalized_hook_name.startswith("on_Install__"):
+ if normalized_hook_name == "InstallEvent":
phase = "install"
elif normalized_hook_name.startswith("on_CrawlSetup__"):
phase = "crawl"
@@ -1966,7 +1966,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
Priority order (highest to lowest):
- Environment - Environment variables
- - Machine - Machine-specific overrides (e.g., resolved binary paths)
+
- Machine - Machine-specific overrides
{f'
→ Edit {key} in Machine.config for this server' if machine_admin_url else ""}
- Config File - data/ArchiveBox.conf
diff --git a/archivebox/hooks.py b/archivebox/hooks.py
index af763c3a..2ff94ba2 100644
--- a/archivebox/hooks.py
+++ b/archivebox/hooks.py
@@ -9,11 +9,14 @@ ArchiveBox no longer drives plugin execution itself during normal crawls.
- parses hook stdout JSONL records into ArchiveBox models when needed
Hook-backed event families are discovered from filenames like:
- on_Install__*
on_BinaryRequest__*
on_CrawlSetup__*
on_Snapshot__*
+InstallEvent itself is still part of the runtime lifecycle, but it has no
+corresponding hook family. Its dependency declarations come directly from each
+plugin's `config.json > required_binaries`.
+
Lifecycle event names like `InstallEvent` or `SnapshotCleanupEvent` are
normalized to the corresponding `on_{EventFamily}__*` prefix by a simple
string transform. If no scripts exist for that prefix, discovery returns `[]`.
@@ -212,7 +215,7 @@ def discover_hooks(
pattern_direct = f"on_{hook_event_name}__*.{ext}"
hooks.extend(base_dir.glob(pattern_direct))
- # Binary install hooks are provider hooks, not end-user extractors. They
+ # Binary provider hooks are not end-user extractors. They
# self-filter via `binproviders`, so applying the PLUGINS whitelist here
# can hide the very installer needed by a selected plugin (e.g.
# `--plugins=singlefile` still needs the `npm` BinaryRequest hook).
@@ -394,54 +397,14 @@ def run_hook(
# Derive LIB_BIN_DIR from LIB_DIR if not set
lib_bin_dir = Path(lib_dir) / "bin"
- # Build PATH with proper precedence:
- # 1. LIB_BIN_DIR (highest priority - local symlinked binaries)
- # 2. Machine.config.PATH (pip/npm bin dirs from providers)
- # 3. os.environ['PATH'] (system PATH)
-
- if lib_bin_dir:
- lib_bin_dir = str(lib_bin_dir)
- env["LIB_BIN_DIR"] = lib_bin_dir
-
- # Start with base PATH
- current_path = env.get("PATH", "")
-
- # Prepend Machine.config.PATH if it exists (treat as extra entries, not replacement)
- try:
- from archivebox.machine.models import Machine
-
- machine = Machine.current()
- if machine and machine.config:
- machine_path = machine.config.get("PATH")
- if machine_path:
- # Prepend machine_path to current PATH
- current_path = f"{machine_path}:{current_path}" if current_path else machine_path
- except Exception:
- pass
-
- # Finally prepend LIB_BIN_DIR to the front (highest priority)
- if lib_bin_dir:
- if not current_path.startswith(f"{lib_bin_dir}:"):
- env["PATH"] = f"{lib_bin_dir}:{current_path}" if current_path else lib_bin_dir
- else:
- env["PATH"] = current_path
- else:
- env["PATH"] = current_path
-
- # Set NODE_PATH for Node.js module resolution
- # Priority: config dict > Machine.config > derive from LIB_DIR
+ # Set NODE_PATH for Node.js module resolution.
+ # Priority: config dict > derive from LIB_DIR
node_path = config.get("NODE_PATH")
if not node_path and lib_dir:
# Derive from LIB_DIR/npm/node_modules (create if needed)
node_modules_dir = Path(lib_dir) / "npm" / "node_modules"
node_modules_dir.mkdir(parents=True, exist_ok=True)
node_path = str(node_modules_dir)
- if not node_path:
- try:
- # Fallback to Machine.config
- node_path = machine.config.get("NODE_MODULES_DIR")
- except Exception:
- pass
if node_path:
env["NODE_PATH"] = node_path
env["NODE_MODULES_DIR"] = node_path # For backwards compatibility
@@ -472,6 +435,41 @@ def run_hook(
else:
env[key] = str(value)
+ # Build PATH with proper precedence:
+ # 1. path-like *_BINARY parents (explicit binary overrides / cached abspaths)
+ # 2. LIB_BIN_DIR (local symlinked binaries)
+ # 3. existing PATH
+ runtime_bin_dirs: list[str] = []
+ if lib_bin_dir:
+ lib_bin_dir = str(lib_bin_dir)
+ env["LIB_BIN_DIR"] = lib_bin_dir
+ for key, raw_value in env.items():
+ if not key.endswith("_BINARY"):
+ continue
+ value = str(raw_value or "").strip()
+ if not value:
+ continue
+ path_value = Path(value).expanduser()
+ if not (path_value.is_absolute() or "/" in value or "\\" in value):
+ continue
+ binary_dir = str(path_value.resolve(strict=False).parent)
+ if binary_dir and binary_dir not in runtime_bin_dirs:
+ runtime_bin_dirs.append(binary_dir)
+ if lib_bin_dir and lib_bin_dir not in runtime_bin_dirs:
+ runtime_bin_dirs.append(lib_bin_dir)
+ uv_value = str(env.get("UV") or "").strip()
+ if uv_value:
+ uv_bin_dir = str(Path(uv_value).expanduser().resolve(strict=False).parent)
+ if uv_bin_dir and uv_bin_dir not in runtime_bin_dirs:
+ runtime_bin_dirs.append(uv_bin_dir)
+
+ current_path = env.get("PATH", "")
+ path_parts = [part for part in current_path.split(os.pathsep) if part]
+ for extra_dir in reversed(runtime_bin_dirs):
+ if extra_dir not in path_parts:
+ path_parts.insert(0, extra_dir)
+ env["PATH"] = os.pathsep.join(path_parts)
+
# Create output directory if needed
output_dir.mkdir(parents=True, exist_ok=True)
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
index 1d8f75f6..3479114d 100755
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@@ -101,8 +101,6 @@ def _get_process_binary_env_keys(plugin_name: str, hook_path: str, env: dict[str
schema_keys.sort(
key=lambda key: (
key != f"{plugin_key}_BINARY",
- key.endswith("_NODE_BINARY"),
- key.endswith("_CHROME_BINARY"),
key,
),
)
@@ -117,8 +115,6 @@ def _get_process_binary_env_keys(plugin_name: str, hook_path: str, env: dict[str
hook_suffix = Path(hook_path).suffix.lower()
if hook_suffix == ".js":
- if plugin_key:
- add(f"{plugin_key}_NODE_BINARY")
add("NODE_BINARY")
return keys
@@ -160,7 +156,7 @@ class Machine(ModelWithHealthStats):
default=dict,
null=True,
blank=True,
- help_text="Machine-specific config overrides (e.g., resolved binary paths like WGET_BINARY)",
+ help_text="Machine-specific config overrides.",
)
num_uses_failed = models.PositiveIntegerField(default=0)
num_uses_succeeded = models.PositiveIntegerField(default=0)
@@ -176,24 +172,13 @@ class Machine(ModelWithHealthStats):
global _CURRENT_MACHINE
if _CURRENT_MACHINE:
if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
- return cls._sanitize_config(cls._hydrate_config_from_sibling(_CURRENT_MACHINE))
+ return cls._sanitize_config(_CURRENT_MACHINE)
_CURRENT_MACHINE = None
_CURRENT_MACHINE, _ = cls.objects.update_or_create(
guid=get_host_guid(),
defaults={"hostname": socket.gethostname(), **get_os_info(), **get_vm_info(), "stats": get_host_stats()},
)
- return cls._sanitize_config(cls._hydrate_config_from_sibling(_CURRENT_MACHINE))
-
- @classmethod
- def _hydrate_config_from_sibling(cls, machine: Machine) -> Machine:
- if machine.config:
- return machine
-
- sibling = cls.objects.exclude(pk=machine.pk).filter(hostname=machine.hostname).exclude(config={}).order_by("-modified_at").first()
- if sibling and sibling.config:
- machine.config = dict(sibling.config)
- machine.save(update_fields=["config", "modified_at"])
- return machine
+ return cls._sanitize_config(_CURRENT_MACHINE)
@classmethod
def _sanitize_config(cls, machine: Machine) -> Machine:
@@ -622,12 +607,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
from archivebox.config.configset import get_config
# Get merged config (Binary doesn't have crawl/snapshot context).
- # Binary workers can install several dependencies in one process, so
- # refresh from the latest persisted machine config before each hook run.
config = get_config()
- current_machine = Machine.current()
- if current_machine.config:
- config.update(current_machine.config)
# ArchiveBox installs the puppeteer package and Chromium in separate
# hook phases. Suppress puppeteer's bundled browser download during the
@@ -760,6 +740,11 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
binary_abspath = Path(self.abspath).resolve()
lib_bin_dir = Path(lib_bin_dir).resolve()
+ binary_parts = binary_abspath.parts
+ try:
+ app_index = next(index for index, part in enumerate(binary_parts) if part.endswith(".app"))
+ except StopIteration:
+ app_index = -1
# Create LIB_BIN_DIR if it doesn't exist
try:
@@ -772,6 +757,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
binary_name = binary_abspath.name
symlink_path = lib_bin_dir / binary_name
+ if app_index != -1 and len(binary_parts) > app_index + 2 and binary_parts[app_index + 1 : app_index + 3] == ("Contents", "MacOS"):
+ if symlink_path.exists() or symlink_path.is_symlink():
+ try:
+ symlink_path.unlink()
+ except (OSError, PermissionError) as e:
+ print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr)
+ return None
+ return binary_abspath
+
# Remove existing symlink/file if it exists
if symlink_path.exists() or symlink_path.is_symlink():
try:
diff --git a/archivebox/services/__init__.py b/archivebox/services/__init__.py
index 1e6c5b63..8b41348b 100644
--- a/archivebox/services/__init__.py
+++ b/archivebox/services/__init__.py
@@ -2,7 +2,6 @@ from .archive_result_service import ArchiveResultService
from .binary_service import BinaryService
from .crawl_service import CrawlService
from .machine_service import MachineService
-from .process_request_service import ProcessRequestService
from .process_service import ProcessService
from .runner import run_binary, run_crawl, run_install, run_pending_crawls
from .snapshot_service import SnapshotService
@@ -13,7 +12,6 @@ __all__ = [
"BinaryService",
"CrawlService",
"MachineService",
- "ProcessRequestService",
"ProcessService",
"SnapshotService",
"TagService",
diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py
index 71097e0e..184c2b9f 100644
--- a/archivebox/services/binary_service.py
+++ b/archivebox/services/binary_service.py
@@ -14,6 +14,23 @@ class BinaryService(BaseService):
async def on_BinaryRequestEvent__Outer(self, event: BinaryRequestEvent) -> None:
await run_db_op(self._project_binary, event)
+ cached = await run_db_op(self._load_cached_binary, event)
+ if cached is not None:
+ await self.bus.emit(
+ BinaryEvent(
+ name=event.name,
+ plugin_name=event.plugin_name,
+ hook_name=event.hook_name,
+ abspath=cached["abspath"],
+ version=cached["version"],
+ sha256=cached["sha256"],
+ binproviders=event.binproviders or cached["binproviders"],
+ binprovider=cached["binprovider"],
+ overrides=event.overrides or cached["overrides"],
+ binary_id=event.binary_id,
+ machine_id=event.machine_id or cached["machine_id"],
+ ),
+ )
async def on_BinaryEvent__Outer(self, event: BinaryEvent) -> None:
resolved = await asyncio.to_thread(self._resolve_installed_binary_metadata, event)
@@ -44,6 +61,29 @@ class BinaryService(BaseService):
},
)
+ def _load_cached_binary(self, event: BinaryRequestEvent) -> dict[str, str] | None:
+ from archivebox.machine.models import Binary, Machine
+
+ machine = Machine.current()
+ installed = (
+ Binary.objects.filter(machine=machine, name=event.name, status=Binary.StatusChoices.INSTALLED)
+ .exclude(abspath="")
+ .exclude(abspath__isnull=True)
+ .order_by("-modified_at")
+ .first()
+ )
+ if installed is None:
+ return None
+ return {
+ "abspath": installed.abspath,
+ "version": installed.version or "",
+ "sha256": installed.sha256 or "",
+ "binproviders": installed.binproviders or "",
+ "binprovider": installed.binprovider or "",
+ "machine_id": str(installed.machine_id),
+ "overrides": installed.overrides or {},
+ }
+
def _resolve_installed_binary_metadata(self, event: BinaryEvent) -> dict[str, str]:
resolved = {
"abspath": event.abspath or "",
@@ -77,12 +117,11 @@ class BinaryService(BaseService):
"overrides": event.overrides or {},
}
binary = load_binary(spec)
- resolved["abspath"] = str(getattr(binary, "abspath", None) or resolved["abspath"] or "")
- resolved["version"] = str(getattr(binary, "version", None) or resolved["version"] or "")
- resolved["sha256"] = str(getattr(binary, "sha256", None) or resolved["sha256"] or "")
- provider_name = getattr(getattr(binary, "loaded_binprovider", None), "name", None)
- if provider_name:
- resolved["binprovider"] = str(provider_name)
+ resolved["abspath"] = str(binary.abspath or resolved["abspath"] or "")
+ resolved["version"] = str(binary.version or resolved["version"] or "")
+ resolved["sha256"] = str(binary.sha256 or resolved["sha256"] or "")
+ if binary.loaded_binprovider is not None and binary.loaded_binprovider.name:
+ resolved["binprovider"] = str(binary.loaded_binprovider.name)
except Exception:
pass
diff --git a/archivebox/services/machine_service.py b/archivebox/services/machine_service.py
index 574893ee..e90975ad 100644
--- a/archivebox/services/machine_service.py
+++ b/archivebox/services/machine_service.py
@@ -14,13 +14,13 @@ class MachineService(BaseService):
await run_db_op(self._project, event)
def _project(self, event: MachineEvent) -> None:
- from archivebox.machine.models import Machine
+ from archivebox.machine.models import Machine, _sanitize_machine_config
machine = Machine.current()
config = dict(machine.config or {})
if event.config is not None:
- config.update(event.config)
+ config.update(_sanitize_machine_config(event.config))
elif event.method == "update":
key = event.key.replace("config/", "", 1).strip()
if key:
@@ -28,5 +28,5 @@ class MachineService(BaseService):
else:
return
- machine.config = config
+ machine.config = _sanitize_machine_config(config)
machine.save(update_fields=["config", "modified_at"])
diff --git a/archivebox/services/process_request_service.py b/archivebox/services/process_request_service.py
deleted file mode 100644
index cbad168b..00000000
--- a/archivebox/services/process_request_service.py
+++ /dev/null
@@ -1,179 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from datetime import datetime, timezone
-import json
-from pathlib import Path
-import shlex
-import socket
-import time
-from typing import ClassVar
-
-from abxbus import BaseEvent
-from abx_dl.events import ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, ProcessStdoutEvent
-from abx_dl.services.base import BaseService
-
-
-def _is_port_listening(host: str, port: int) -> bool:
- if not host or not port:
- return False
- try:
- with socket.create_connection((host, port), timeout=0.5):
- return True
- except OSError:
- return False
-
-
-def _supervisor_env(env: dict[str, str]) -> str:
- pairs = []
- for key, value in env.items():
- escaped = value.replace('"', '\\"')
- pairs.append(f'{key}="{escaped}"')
- return ",".join(pairs)
-
-
-def _iso_from_epoch(value: object) -> str:
- if not isinstance(value, (int, float)) or value <= 0:
- return ""
- return datetime.fromtimestamp(value, tz=timezone.utc).isoformat()
-
-
-def _ensure_worker(process_event: ProcessEvent) -> dict[str, object]:
- from archivebox.workers.supervisord_util import get_or_create_supervisord_process, get_worker, start_worker
-
- output_dir = Path(process_event.output_dir)
- output_dir.mkdir(parents=True, exist_ok=True)
- worker_name = process_event.hook_name
- supervisor = get_or_create_supervisord_process(daemonize=True)
-
- existing = get_worker(supervisor, worker_name)
- if (
- isinstance(existing, dict)
- and existing.get("statename") == "RUNNING"
- and (
- not process_event.daemon_startup_host
- or not process_event.daemon_startup_port
- or _is_port_listening(process_event.daemon_startup_host, process_event.daemon_startup_port)
- )
- ):
- return existing
-
- daemon = {
- "name": worker_name,
- "command": shlex.join([process_event.hook_path, *process_event.hook_args]),
- "directory": str(output_dir),
- "autostart": "false",
- "autorestart": "true",
- "stdout_logfile": str(output_dir / f"{worker_name}.stdout.log"),
- "redirect_stderr": "true",
- }
- if process_event.env:
- daemon["environment"] = _supervisor_env(process_event.env)
-
- proc = start_worker(supervisor, daemon)
- deadline = time.monotonic() + max(float(process_event.daemon_startup_timeout), 0.5)
- while time.monotonic() < deadline:
- current = get_worker(supervisor, worker_name)
- if isinstance(current, dict) and current.get("statename") == "RUNNING":
- if (
- not process_event.daemon_startup_host
- or not process_event.daemon_startup_port
- or _is_port_listening(process_event.daemon_startup_host, process_event.daemon_startup_port)
- ):
- return current
- time.sleep(0.1)
- return proc if isinstance(proc, dict) else {}
-
-
-class ProcessRequestService(BaseService):
- LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [ProcessStdoutEvent]
- EMITS: ClassVar[list[type[BaseEvent]]] = [ProcessEvent, ProcessStartedEvent, ProcessCompletedEvent]
-
- async def on_ProcessStdoutEvent(self, event: ProcessStdoutEvent) -> None:
- try:
- record = json.loads(event.line)
- except (json.JSONDecodeError, ValueError):
- return
- if not isinstance(record, dict) or record.pop("type", "") != "ProcessEvent":
- return
-
- process_event = ProcessEvent(
- plugin_name=record.get("plugin_name") or event.plugin_name,
- hook_name=record.get("hook_name") or "process_request",
- hook_path=record["hook_path"],
- hook_args=[str(arg) for arg in record.get("hook_args", [])],
- is_background=bool(record.get("is_background", True)),
- output_dir=record.get("output_dir") or event.output_dir,
- env={str(key): str(value) for key, value in (record.get("env") or {}).items()},
- snapshot_id=record.get("snapshot_id") or event.snapshot_id,
- timeout=int(record.get("timeout") or 60),
- daemon=bool(record.get("daemon", False)),
- daemon_startup_host=str(record.get("daemon_startup_host") or ""),
- daemon_startup_port=int(record.get("daemon_startup_port") or 0),
- daemon_startup_timeout=float(record.get("daemon_startup_timeout") or 0.0),
- process_type=str(record.get("process_type") or ""),
- worker_type=str(record.get("worker_type") or ""),
- event_timeout=float(record.get("event_timeout") or 360.0),
- event_handler_timeout=float(record.get("event_handler_timeout") or 390.0),
- )
- if not process_event.daemon:
- await self.bus.emit(process_event)
- return
-
- proc = await asyncio.to_thread(_ensure_worker, process_event)
- process_id = str(record.get("process_id") or f"worker:{process_event.hook_name}")
- start_ts = _iso_from_epoch(proc.get("start"))
- pid = int(proc.get("pid") or 0)
- statename = str(proc.get("statename") or "")
- exitstatus = int(proc.get("exitstatus") or 0)
- process_type = process_event.process_type or "worker"
- worker_type = process_event.worker_type or process_event.plugin_name
-
- if statename == "RUNNING" and pid:
- await self.bus.emit(
- ProcessStartedEvent(
- plugin_name=process_event.plugin_name,
- hook_name=process_event.hook_name,
- hook_path=process_event.hook_path,
- hook_args=process_event.hook_args,
- output_dir=process_event.output_dir,
- env=process_event.env,
- timeout=process_event.timeout,
- pid=pid,
- process_id=process_id,
- snapshot_id=process_event.snapshot_id,
- is_background=True,
- process_type=process_type,
- worker_type=worker_type,
- start_ts=start_ts,
- ),
- )
- return
-
- stderr = (
- f"Worker {process_event.hook_name} failed to start"
- if not statename
- else f"Worker {process_event.hook_name} state={statename} exitstatus={exitstatus}"
- )
- await self.bus.emit(
- ProcessCompletedEvent(
- plugin_name=process_event.plugin_name,
- hook_name=process_event.hook_name,
- hook_path=process_event.hook_path,
- hook_args=process_event.hook_args,
- env=process_event.env,
- stdout="",
- stderr=stderr,
- exit_code=exitstatus or 1,
- output_dir=process_event.output_dir,
- is_background=True,
- process_id=process_id,
- snapshot_id=process_event.snapshot_id,
- pid=pid,
- process_type=process_type,
- worker_type=worker_type,
- start_ts=start_ts,
- end_ts=datetime.now(tz=timezone.utc).isoformat(),
- ),
- )
- raise RuntimeError(stderr)
diff --git a/archivebox/services/process_service.py b/archivebox/services/process_service.py
index c550d2ff..21547b19 100644
--- a/archivebox/services/process_service.py
+++ b/archivebox/services/process_service.py
@@ -1,11 +1,19 @@
from __future__ import annotations
-from datetime import datetime
-from typing import TYPE_CHECKING
+import asyncio
+from datetime import datetime, timezone as datetime_timezone
+import json
+from pathlib import Path
+import shlex
+import socket
+import time
+from typing import TYPE_CHECKING, Any, ClassVar
+from urllib.parse import urlparse
from django.utils import timezone
-from abx_dl.events import ProcessCompletedEvent, ProcessStartedEvent
+from abxbus import BaseEvent
+from abx_dl.events import ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, ProcessStdoutEvent
from abx_dl.services.base import BaseService
from .db import run_db_op
@@ -14,6 +22,9 @@ if TYPE_CHECKING:
from archivebox.machine.models import Process
+WORKER_READY_TIMEOUT = 10.0
+
+
def parse_event_datetime(value: str | None):
if not value:
return None
@@ -26,14 +37,218 @@ def parse_event_datetime(value: str | None):
return dt
+def _is_port_listening(host: str, port: int) -> bool:
+ if not host or not port:
+ return False
+ try:
+ with socket.create_connection((host, port), timeout=0.5):
+ return True
+ except OSError:
+ return False
+
+
+def _worker_socket_from_url(url: str) -> tuple[str, int] | None:
+ if not url:
+ return None
+ parsed = urlparse(url)
+ if parsed.scheme != "tcp" or not parsed.hostname or not parsed.port:
+ return None
+ return parsed.hostname, parsed.port
+
+
+def _supervisor_env(env: dict[str, str]) -> str:
+ pairs = []
+ for key, value in env.items():
+ escaped = value.replace('"', '\\"')
+ pairs.append(f'{key}="{escaped}"')
+ return ",".join(pairs)
+
+
+def _iso_from_epoch(value: object) -> str:
+ if not isinstance(value, (int, float)) or value <= 0:
+ return ""
+ return datetime.fromtimestamp(value, tz=datetime_timezone.utc).isoformat()
+
+
+def _int_from_object(value: object) -> int:
+ if isinstance(value, bool):
+ return int(value)
+ if isinstance(value, int):
+ return value
+ if isinstance(value, float):
+ return int(value)
+ if isinstance(value, str):
+ try:
+ return int(value)
+ except ValueError:
+ return 0
+ return 0
+
+
+def _ensure_worker(process_event: ProcessEvent) -> dict[str, object]:
+ from archivebox.workers.supervisord_util import get_or_create_supervisord_process, get_worker, start_worker
+
+ output_dir = Path(process_event.output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+ worker_name = process_event.hook_name
+ supervisor = get_or_create_supervisord_process(daemonize=True)
+ worker_socket = _worker_socket_from_url(getattr(process_event, "url", ""))
+
+ existing = get_worker(supervisor, worker_name)
+ if (
+ isinstance(existing, dict)
+ and existing.get("statename") == "RUNNING"
+ and (worker_socket is None or _is_port_listening(*worker_socket))
+ ):
+ return existing
+
+ daemon = {
+ "name": worker_name,
+ "command": shlex.join([process_event.hook_path, *process_event.hook_args]),
+ "directory": str(output_dir),
+ "autostart": "false",
+ "autorestart": "true",
+ "stdout_logfile": str(output_dir / f"{worker_name}.stdout.log"),
+ "redirect_stderr": "true",
+ }
+ if process_event.env:
+ daemon["environment"] = _supervisor_env(process_event.env)
+
+ proc = start_worker(supervisor, daemon)
+ deadline = time.monotonic() + WORKER_READY_TIMEOUT
+ while time.monotonic() < deadline:
+ current = get_worker(supervisor, worker_name)
+ if isinstance(current, dict) and current.get("statename") == "RUNNING":
+ if worker_socket is None or _is_port_listening(*worker_socket):
+ return current
+ time.sleep(0.1)
+ return proc if isinstance(proc, dict) else {}
+
+
class ProcessService(BaseService):
- LISTENS_TO = [ProcessStartedEvent, ProcessCompletedEvent]
- EMITS = []
+ LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [ProcessStdoutEvent, ProcessStartedEvent, ProcessCompletedEvent]
+ EMITS: ClassVar[list[type[BaseEvent]]] = [ProcessEvent, ProcessStartedEvent, ProcessCompletedEvent]
def __init__(self, bus):
self.process_ids: dict[str, str] = {}
super().__init__(bus)
+ async def on_ProcessStdoutEvent(self, event: ProcessStdoutEvent) -> None:
+ try:
+ record = json.loads(event.line)
+ except (json.JSONDecodeError, ValueError):
+ return
+ if not isinstance(record, dict) or record.get("type") != "ProcessEvent":
+ return
+
+ passthrough_fields: dict[str, Any] = {
+ key: value
+ for key, value in record.items()
+ if key
+ not in {
+ "type",
+ "plugin_name",
+ "hook_name",
+ "hook_path",
+ "hook_args",
+ "is_background",
+ "output_dir",
+ "env",
+ "snapshot_id",
+ "process_id",
+ "url",
+ "timeout",
+ "daemon",
+ "process_type",
+ "worker_type",
+ "event_timeout",
+ "event_handler_timeout",
+ }
+ }
+ process_event = ProcessEvent(
+ plugin_name=record.get("plugin_name") or event.plugin_name,
+ hook_name=record.get("hook_name") or "process",
+ hook_path=record["hook_path"],
+ hook_args=[str(arg) for arg in record.get("hook_args", [])],
+ is_background=bool(record.get("is_background", True)),
+ output_dir=record.get("output_dir") or event.output_dir,
+ env={str(key): str(value) for key, value in (record.get("env") or {}).items()},
+ snapshot_id=record.get("snapshot_id") or event.snapshot_id,
+ timeout=int(record.get("timeout") or 60),
+ daemon=bool(record.get("daemon", False)),
+ url=str(record.get("url") or ""),
+ process_type=str(record.get("process_type") or ""),
+ worker_type=str(record.get("worker_type") or ""),
+ event_timeout=float(record.get("event_timeout") or 360.0),
+ event_handler_timeout=float(record.get("event_handler_timeout") or 390.0),
+ **passthrough_fields,
+ )
+ if not process_event.daemon:
+ await self.bus.emit(process_event)
+ return
+
+ proc = await asyncio.to_thread(_ensure_worker, process_event)
+ process_id = str(record.get("process_id") or f"worker:{process_event.hook_name}")
+ start_ts = _iso_from_epoch(proc.get("start"))
+ pid = _int_from_object(proc.get("pid"))
+ statename = str(proc.get("statename") or "")
+ exitstatus = _int_from_object(proc.get("exitstatus"))
+ process_type = process_event.process_type or "worker"
+ worker_type = process_event.worker_type or process_event.plugin_name
+
+ if statename == "RUNNING" and pid:
+ await self.bus.emit(
+ ProcessStartedEvent(
+ plugin_name=process_event.plugin_name,
+ hook_name=process_event.hook_name,
+ hook_path=process_event.hook_path,
+ hook_args=process_event.hook_args,
+ output_dir=process_event.output_dir,
+ env=process_event.env,
+ timeout=process_event.timeout,
+ pid=pid,
+ process_id=process_id,
+ snapshot_id=process_event.snapshot_id,
+ is_background=True,
+ url=process_event.url,
+ process_type=process_type,
+ worker_type=worker_type,
+ start_ts=start_ts,
+ **passthrough_fields,
+ ),
+ )
+ return
+
+ stderr = (
+ f"Worker {process_event.hook_name} failed to start"
+ if not statename
+ else f"Worker {process_event.hook_name} state={statename} exitstatus={exitstatus}"
+ )
+ await self.bus.emit(
+ ProcessCompletedEvent(
+ plugin_name=process_event.plugin_name,
+ hook_name=process_event.hook_name,
+ hook_path=process_event.hook_path,
+ hook_args=process_event.hook_args,
+ env=process_event.env,
+ stdout="",
+ stderr=stderr,
+ exit_code=exitstatus or 1,
+ output_dir=process_event.output_dir,
+ is_background=True,
+ process_id=process_id,
+ snapshot_id=process_event.snapshot_id,
+ pid=pid,
+ url=process_event.url,
+ process_type=process_type,
+ worker_type=worker_type,
+ start_ts=start_ts,
+ end_ts=datetime.now(tz=datetime_timezone.utc).isoformat(),
+ **passthrough_fields,
+ ),
+ )
+ raise RuntimeError(stderr)
+
async def on_ProcessStartedEvent__Outer(self, event: ProcessStartedEvent) -> None:
await run_db_op(self._project_started, event)
@@ -51,7 +266,7 @@ class ProcessService(BaseService):
if db_process_id:
process = Process.objects.filter(id=db_process_id).first()
if process is not None:
- if process.iface_id != iface.id or process.machine_id != iface.machine_id:
+ if getattr(process, "iface_id", None) != iface.id or process.machine_id != iface.machine_id:
process.iface = iface
process.machine = iface.machine
process.save(update_fields=["iface", "machine", "modified_at"])
@@ -84,6 +299,7 @@ class ProcessService(BaseService):
env=event.env,
timeout=getattr(event, "timeout", 60),
pid=event.pid or None,
+ url=getattr(event, "url", "") or None,
started_at=parse_event_datetime(getattr(event, "start_ts", "")),
status=Process.StatusChoices.RUNNING,
retry_at=None,
@@ -98,6 +314,7 @@ class ProcessService(BaseService):
process.env = event.env
process.timeout = event.timeout
process.pid = event.pid or None
+ process.url = getattr(event, "url", "") or process.url
process.process_type = getattr(event, "process_type", "") or process.process_type
process.worker_type = getattr(event, "worker_type", "") or process.worker_type
process.started_at = parse_event_datetime(event.start_ts) or process.started_at or timezone.now()
@@ -113,6 +330,7 @@ class ProcessService(BaseService):
process.cmd = [event.hook_path, *event.hook_args]
process.env = event.env
process.pid = event.pid or process.pid
+ process.url = getattr(event, "url", "") or process.url
process.process_type = getattr(event, "process_type", "") or process.process_type
process.worker_type = getattr(event, "worker_type", "") or process.worker_type
process.started_at = parse_event_datetime(event.start_ts) or process.started_at
diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py
index b1d0619b..fdcb15cf 100644
--- a/archivebox/services/runner.py
+++ b/archivebox/services/runner.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import json
import os
+import re
import shutil
import subprocess
import sys
@@ -28,8 +29,6 @@ from abx_dl.orchestrator import (
from .archive_result_service import ArchiveResultService
from .binary_service import BinaryService
from .crawl_service import CrawlService
-from .machine_service import MachineService
-from .process_request_service import ProcessRequestService
from .process_service import ProcessService
from .snapshot_service import SnapshotService
from .tag_service import TagService
@@ -58,28 +57,34 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
)
-def _binary_env_key(name: str) -> str:
- normalized = "".join(ch if ch.isalnum() else "_" for ch in name).upper()
- return f"{normalized}_BINARY"
+_TEMPLATE_NAME_RE = re.compile(r"^\{([A-Z0-9_]+)\}$")
-def _binary_config_keys_for_plugins(plugins: dict[str, Plugin], binary_name: str) -> list[str]:
+def _binary_config_keys_for_plugins(plugins: dict[str, Plugin], binary_name: str, config: dict[str, Any]) -> list[str]:
keys: list[str] = []
- if binary_name != "postlight-parser":
- keys.append(_binary_env_key(binary_name))
for plugin in plugins.values():
+ for spec in plugin.binaries:
+ template_name = str(spec.get("name") or "").strip()
+ match = _TEMPLATE_NAME_RE.fullmatch(template_name)
+ if match is None:
+ continue
+ key = match.group(1)
+ configured_value = config.get(key)
+ if configured_value is not None and str(configured_value).strip() == binary_name:
+ keys.append(key)
for key, prop in plugin.config_schema.items():
if key.endswith("_BINARY") and prop.get("default") == binary_name:
- keys.insert(0, key)
+ keys.append(key)
return list(dict.fromkeys(keys))
-def _installed_binary_config_overrides(plugins: dict[str, Plugin]) -> dict[str, str]:
+def _installed_binary_config_overrides(plugins: dict[str, Plugin], config: dict[str, Any] | None = None) -> dict[str, str]:
from archivebox.machine.models import Binary, Machine
machine = Machine.current()
+ active_config = dict(config or {})
overrides: dict[str, str] = {}
shared_lib_dir: Path | None = None
pip_home: Path | None = None
@@ -98,7 +103,7 @@ def _installed_binary_config_overrides(plugins: dict[str, Plugin]) -> dict[str,
continue
if not resolved_path.is_file() or not os.access(resolved_path, os.X_OK):
continue
- for key in _binary_config_keys_for_plugins(plugins, binary.name):
+ for key in _binary_config_keys_for_plugins(plugins, binary.name, active_config):
overrides[key] = binary.abspath
if resolved_path.parent.name == ".bin" and resolved_path.parent.parent.name == "node_modules":
@@ -231,10 +236,8 @@ class CrawlRunner:
self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
self.plugins = discover_plugins()
self.process_service = ProcessService(self.bus)
- self.machine_service = MachineService(self.bus)
self.binary_service = BinaryService(self.bus)
self.tag_service = TagService(self.bus)
- self.process_request_service = ProcessRequestService(self.bus)
self.crawl_service = CrawlService(self.bus, crawl_id=str(crawl.id))
self.process_discovered_snapshots_inline = process_discovered_snapshots_inline
self.snapshot_service = SnapshotService(
@@ -250,32 +253,10 @@ class CrawlRunner:
self.abx_services = None
self.persona = None
self.base_config: dict[str, Any] = {}
+ self.derived_config: dict[str, Any] = {}
self.primary_url = ""
self._live_stream = None
- def _create_projector_bus(self, *, identifier: str, config_overrides: dict[str, Any]):
- bus = create_bus(name=_bus_name("ArchiveBox", identifier), total_timeout=3600.0)
- process_service = ProcessService(bus)
- MachineService(bus)
- BinaryService(bus)
- TagService(bus)
- ProcessRequestService(bus)
- CrawlService(bus, crawl_id=str(self.crawl.id))
- SnapshotService(
- bus,
- crawl_id=str(self.crawl.id),
- schedule_snapshot=self.enqueue_snapshot if self.process_discovered_snapshots_inline else self.leave_snapshot_queued,
- )
- ArchiveResultService(bus, process_service=process_service)
- abx_services = setup_abx_services(
- bus,
- plugins=self.plugins,
- config_overrides=config_overrides,
- auto_install=True,
- emit_jsonl=False,
- )
- return bus, abx_services
-
async def run(self) -> None:
from asgiref.sync import sync_to_async
from archivebox.crawls.models import Crawl
@@ -292,6 +273,8 @@ class CrawlRunner:
**self.base_config,
"ABX_RUNTIME": "archivebox",
},
+ derived_config_overrides=self.derived_config,
+ persist_derived=False,
auto_install=True,
emit_jsonl=False,
)
@@ -369,7 +352,7 @@ class CrawlRunner:
current_process.save(update_fields=["iface", "machine", "modified_at"])
self.persona = self.crawl.resolve_persona()
self.base_config = get_config(crawl=self.crawl)
- self.base_config.update(_installed_binary_config_overrides(self.plugins))
+ self.derived_config = _installed_binary_config_overrides(self.plugins, self.base_config)
self.base_config["ABX_RUNTIME"] = "archivebox"
if self.selected_plugins is None:
self.selected_plugins = _selected_plugins_from_config(self.base_config)
@@ -473,7 +456,6 @@ class CrawlRunner:
plugins=self.plugins,
output_dir=Path(snapshot["output_dir"]),
selected_plugins=self.selected_plugins,
- config_overrides=snapshot["config"],
bus=self.bus,
emit_jsonl=False,
snapshot=setup_snapshot,
@@ -501,7 +483,6 @@ class CrawlRunner:
plugins=self.plugins,
output_dir=Path(snapshot["output_dir"]),
selected_plugins=self.selected_plugins,
- config_overrides=snapshot["config"],
bus=self.bus,
emit_jsonl=False,
snapshot=cleanup_snapshot,
@@ -530,31 +511,22 @@ class CrawlRunner:
parent_snapshot_id=snapshot["parent_snapshot_id"],
crawl_id=str(self.crawl.id),
)
- snapshot_bus, snapshot_services = self._create_projector_bus(
- identifier=f"{self.crawl.id}_{snapshot['id']}",
- config_overrides=snapshot["config"],
- )
try:
- _attach_bus_trace(snapshot_bus)
await download(
url=snapshot["url"],
plugins=self.plugins,
output_dir=Path(snapshot["output_dir"]),
selected_plugins=self.selected_plugins,
- config_overrides=snapshot["config"],
- bus=snapshot_bus,
+ bus=self.bus,
emit_jsonl=False,
snapshot=abx_snapshot,
skip_crawl_setup=True,
skip_crawl_cleanup=True,
)
- await snapshot_services.process.wait_for_background_monitors()
finally:
current_task = asyncio.current_task()
if current_task is not None and self.snapshot_tasks.get(snapshot_id) is current_task:
self.snapshot_tasks.pop(snapshot_id, None)
- await _stop_bus_trace(snapshot_bus)
- await snapshot_bus.stop()
def _load_snapshot_run_data(self, snapshot_id: str):
from archivebox.core.models import Snapshot
@@ -615,19 +587,19 @@ async def _run_binary(binary_id: str) -> None:
binary = await sync_to_async(Binary.objects.get, thread_sensitive=True)(id=binary_id)
plugins = discover_plugins()
config = get_config()
- config.update(await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins))
+ derived_config = await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins, config)
config["ABX_RUNTIME"] = "archivebox"
bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
process_service = ProcessService(bus)
- MachineService(bus)
BinaryService(bus)
TagService(bus)
- ProcessRequestService(bus)
ArchiveResultService(bus, process_service=process_service)
setup_abx_services(
bus,
plugins=plugins,
config_overrides=config,
+ derived_config_overrides=derived_config,
+ persist_derived=False,
auto_install=True,
emit_jsonl=False,
)
@@ -662,19 +634,19 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
plugins = discover_plugins()
config = get_config()
- config.update(await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins))
+ derived_config = await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins, config)
config["ABX_RUNTIME"] = "archivebox"
bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
process_service = ProcessService(bus)
- MachineService(bus)
BinaryService(bus)
TagService(bus)
- ProcessRequestService(bus)
ArchiveResultService(bus, process_service=process_service)
abx_services = setup_abx_services(
bus,
plugins=plugins,
config_overrides=config,
+ derived_config_overrides=derived_config,
+ persist_derived=False,
auto_install=True,
emit_jsonl=False,
)
diff --git a/archivebox/tests/test_archive_result_service.py b/archivebox/tests/test_archive_result_service.py
index 808a844b..3c3aaad5 100644
--- a/archivebox/tests/test_archive_result_service.py
+++ b/archivebox/tests/test_archive_result_service.py
@@ -518,7 +518,6 @@ def test_binary_event_reuses_existing_installed_binary_row(monkeypatch):
event = BinaryRequestEvent(
name="wget",
plugin_name="wget",
- hook_name="on_Install__10_wget.finite.bg",
output_dir="/tmp/wget",
binproviders="provider",
)
diff --git a/archivebox/tests/test_config_views.py b/archivebox/tests/test_config_views.py
index 123677ac..632258b0 100644
--- a/archivebox/tests/test_config_views.py
+++ b/archivebox/tests/test_config_views.py
@@ -133,7 +133,13 @@ def test_plugin_detail_view_renders_config_in_dedicated_sections(monkeypatch):
"description": "Example config used to verify plugin metadata rendering.",
"type": "object",
"required_plugins": ["chrome"],
- "required_binaries": ["example-cli"],
+ "required_binaries": [
+ {
+ "name": "example-cli",
+ "binproviders": "env,apt,brew",
+ "min_version": None,
+ },
+ ],
"output_mimetypes": ["text/plain", "application/json"],
"properties": {
"EXAMPLE_ENABLED": {
diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py
index 62b114fe..14983a27 100755
--- a/archivebox/tests/test_hooks.py
+++ b/archivebox/tests/test_hooks.py
@@ -3,7 +3,7 @@
Unit tests for the ArchiveBox hook architecture.
Tests hook discovery, execution, JSONL parsing, background hook detection,
-binary lookup, and install hook XYZ_BINARY env var handling.
+binary lookup, and required_binaries XYZ_BINARY passthrough handling.
Run with:
sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_hooks.py -v'
@@ -126,8 +126,8 @@ not json at all
self.assertEqual(records[0]["type"], "ArchiveResult")
-class TestInstallHookEnvVarHandling(unittest.TestCase):
- """Test that install hooks respect XYZ_BINARY env vars."""
+class TestRequiredBinaryConfigHandling(unittest.TestCase):
+ """Test that required_binaries keep configured XYZ_BINARY values intact."""
def setUp(self):
"""Set up test environment."""
@@ -139,39 +139,28 @@ class TestInstallHookEnvVarHandling(unittest.TestCase):
shutil.rmtree(self.work_dir, ignore_errors=True)
def test_binary_env_var_absolute_path_handling(self):
- """Install hooks should handle absolute paths in XYZ_BINARY."""
- # Test the logic that install hooks use
+ """Absolute binary paths should pass through unchanged."""
configured_binary = "/custom/path/to/wget2"
- if "/" in configured_binary:
- bin_name = Path(configured_binary).name
- else:
- bin_name = configured_binary
+ binary_name = configured_binary
- self.assertEqual(bin_name, "wget2")
+ self.assertEqual(binary_name, "/custom/path/to/wget2")
def test_binary_env_var_name_only_handling(self):
- """Install hooks should handle binary names in XYZ_BINARY."""
- # Test the logic that install hooks use
+ """Binary command names should pass through unchanged."""
configured_binary = "wget2"
- if "/" in configured_binary:
- bin_name = Path(configured_binary).name
- else:
- bin_name = configured_binary
+ binary_name = configured_binary
- self.assertEqual(bin_name, "wget2")
+ self.assertEqual(binary_name, "wget2")
def test_binary_env_var_empty_default(self):
- """Install hooks should use default when XYZ_BINARY is empty."""
+ """Empty configured values should fall back to config defaults."""
configured_binary = ""
if configured_binary:
- if "/" in configured_binary:
- bin_name = Path(configured_binary).name
- else:
- bin_name = configured_binary
+ binary_name = configured_binary
else:
- bin_name = "wget" # default
+ binary_name = "wget"
- self.assertEqual(bin_name, "wget")
+ self.assertEqual(binary_name, "wget")
class TestHookDiscovery(unittest.TestCase):
@@ -187,7 +176,7 @@ class TestHookDiscovery(unittest.TestCase):
wget_dir = self.plugins_dir / "wget"
wget_dir.mkdir()
(wget_dir / "on_Snapshot__50_wget.py").write_text("# test hook")
- (wget_dir / "on_Install__10_wget.finite.bg.py").write_text("# install hook")
+ (wget_dir / "on_BinaryRequest__10_wget.py").write_text("# binary request hook")
chrome_dir = self.plugins_dir / "chrome"
chrome_dir.mkdir(exist_ok=True)
@@ -299,7 +288,7 @@ class TestHookDiscovery(unittest.TestCase):
self.assertIn("on_BinaryRequest__10_npm.py", hook_names)
def test_discover_hooks_accepts_event_class_names(self):
- """discover_hooks should accept InstallEvent / SnapshotEvent class names."""
+ """discover_hooks should accept BinaryRequestEvent / SnapshotEvent class names."""
from archivebox import hooks as hooks_module
hooks_module.get_plugins.cache_clear()
@@ -307,10 +296,10 @@ class TestHookDiscovery(unittest.TestCase):
patch.object(hooks_module, "BUILTIN_PLUGINS_DIR", self.plugins_dir),
patch.object(hooks_module, "USER_PLUGINS_DIR", self.test_dir / "user_plugins"),
):
- install_hooks = hooks_module.discover_hooks("InstallEvent", filter_disabled=False)
+ binary_hooks = hooks_module.discover_hooks("BinaryRequestEvent", filter_disabled=False)
snapshot_hooks = hooks_module.discover_hooks("SnapshotEvent", filter_disabled=False)
- self.assertIn("on_Install__10_wget.finite.bg.py", [hook.name for hook in install_hooks])
+ self.assertIn("on_BinaryRequest__10_wget.py", [hook.name for hook in binary_hooks])
self.assertIn("on_Snapshot__50_wget.py", [hook.name for hook in snapshot_hooks])
def test_discover_hooks_returns_empty_for_non_hook_lifecycle_events(self):
@@ -325,44 +314,6 @@ class TestHookDiscovery(unittest.TestCase):
self.assertEqual(hooks_module.discover_hooks("BinaryEvent", filter_disabled=False), [])
self.assertEqual(hooks_module.discover_hooks("CrawlCleanupEvent", filter_disabled=False), [])
- def test_discover_install_hooks_only_include_declared_plugin_dependencies(self):
- """Install hook discovery should include required_plugins without broadening to provider plugins."""
- responses_dir = self.plugins_dir / "responses"
- responses_dir.mkdir()
- (responses_dir / "config.json").write_text(
- json.dumps(
- {
- "type": "object",
- "required_plugins": ["chrome"],
- "properties": {},
- },
- ),
- )
-
- chrome_dir = self.plugins_dir / "chrome"
- chrome_dir.mkdir(exist_ok=True)
- (chrome_dir / "config.json").write_text('{"type": "object", "properties": {}}')
- (chrome_dir / "on_Install__70_chrome.finite.bg.py").write_text("# chrome install hook")
-
- npm_dir = self.plugins_dir / "npm"
- npm_dir.mkdir()
- (npm_dir / "on_BinaryRequest__10_npm.py").write_text("# npm binary hook")
- (npm_dir / "on_Install__00_npm.py").write_text("# npm install hook")
- (npm_dir / "config.json").write_text('{"type": "object", "properties": {}}')
-
- from archivebox import hooks as hooks_module
-
- hooks_module.get_plugins.cache_clear()
- with (
- patch.object(hooks_module, "BUILTIN_PLUGINS_DIR", self.plugins_dir),
- patch.object(hooks_module, "USER_PLUGINS_DIR", self.test_dir / "user_plugins"),
- ):
- hooks = hooks_module.discover_hooks("Install", config={"PLUGINS": "responses"})
-
- hook_names = [hook.name for hook in hooks]
- self.assertIn("on_Install__70_chrome.finite.bg.py", hook_names)
- self.assertNotIn("on_Install__00_npm.py", hook_names)
-
class TestGetExtractorName(unittest.TestCase):
"""Test get_extractor_name() function."""
@@ -484,8 +435,8 @@ print(json.dumps({"type": "ArchiveResult", "status": "succeeded", "url": args.ge
self.assertEqual(records[0]["url"], "https://example.com")
-class TestInstallHookOutput(unittest.TestCase):
- """Test install hook output format compliance."""
+class TestDependencyRecordOutput(unittest.TestCase):
+ """Test dependency record output format compliance."""
def setUp(self):
"""Set up test environment."""
@@ -495,8 +446,8 @@ class TestInstallHookOutput(unittest.TestCase):
"""Clean up test environment."""
shutil.rmtree(self.work_dir, ignore_errors=True)
- def test_install_hook_outputs_binary(self):
- """Install hook should output Binary JSONL when binary found."""
+ def test_dependency_record_outputs_binary(self):
+ """Dependency resolution should output Binary JSONL when binary is found."""
hook_output = json.dumps(
{
"type": "Binary",
@@ -515,8 +466,8 @@ class TestInstallHookOutput(unittest.TestCase):
self.assertEqual(data["name"], "wget")
self.assertTrue(data["abspath"].startswith("/"))
- def test_install_hook_outputs_machine_config(self):
- """Install hook should output Machine config update JSONL."""
+ def test_dependency_record_outputs_machine_config(self):
+ """Dependency resolution should output Machine config update JSONL."""
hook_output = json.dumps(
{
"type": "Machine",
diff --git a/archivebox/tests/test_process_service.py b/archivebox/tests/test_process_service.py
new file mode 100644
index 00000000..89ddd88f
--- /dev/null
+++ b/archivebox/tests/test_process_service.py
@@ -0,0 +1,69 @@
+import asyncio
+import json
+
+import pytest
+
+from abx_dl.events import ProcessStartedEvent, ProcessStdoutEvent
+from abx_dl.orchestrator import create_bus
+
+
+pytestmark = pytest.mark.django_db
+
+
+def test_process_service_emits_process_started_from_inline_process_event(monkeypatch):
+ from archivebox.services import process_service as process_service_module
+ from archivebox.services.process_service import ProcessService
+
+ bus = create_bus(name="test_process_service_inline_process_event")
+ ProcessService(bus)
+
+ monkeypatch.setattr(
+ process_service_module,
+ "_ensure_worker",
+ lambda event: {
+ "pid": 4321,
+ "start": 1711111111.0,
+ "statename": "RUNNING",
+ "exitstatus": 0,
+ },
+ )
+
+ async def run_test():
+ await bus.emit(
+ ProcessStdoutEvent(
+ line=json.dumps(
+ {
+ "type": "ProcessEvent",
+ "plugin_name": "search_backend_sonic",
+ "hook_name": "worker_sonic",
+ "hook_path": "/usr/bin/sonic",
+ "hook_args": ["-c", "/tmp/sonic/config.cfg"],
+ "is_background": True,
+ "daemon": True,
+ "url": "tcp://127.0.0.1:1491",
+ "output_dir": "/tmp/sonic",
+ "env": {},
+ "process_type": "worker",
+ "worker_type": "sonic",
+ "process_id": "worker:sonic",
+ "output_str": "127.0.0.1:1491",
+ },
+ ),
+ plugin_name="search_backend_sonic",
+ hook_name="on_CrawlSetup__55_sonic_start.py",
+ output_dir="/tmp/search_backend_sonic",
+ snapshot_id="snap-1",
+ process_id="proc-hook",
+ ),
+ )
+ started = await bus.find(ProcessStartedEvent, process_id="worker:sonic")
+ await bus.stop()
+ return started
+
+ started = asyncio.run(run_test())
+ assert started is not None
+ assert started.hook_name == "worker_sonic"
+ assert started.process_type == "worker"
+ assert started.worker_type == "sonic"
+ assert getattr(started, "url", "") == "tcp://127.0.0.1:1491"
+ assert getattr(started, "output_str", "") == "127.0.0.1:1491"
diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py
index 43b4dcd3..93c9069b 100644
--- a/archivebox/tests/test_runner.py
+++ b/archivebox/tests/test_runner.py
@@ -46,7 +46,7 @@ async def _call_sync(func, *args, **kwargs):
return func(*args, **kwargs)
-def test_run_snapshot_uses_isolated_bus_per_snapshot(monkeypatch):
+def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
@@ -87,13 +87,13 @@ def test_run_snapshot_uses_isolated_bus_per_snapshot(monkeypatch):
download_calls = []
- async def fake_download(*, url, bus, config_overrides, snapshot, **kwargs):
+ async def fake_download(*, url, bus, snapshot, **kwargs):
download_calls.append(
{
"url": url,
"bus": bus,
- "snapshot_id": config_overrides["SNAPSHOT_ID"],
- "source_url": config_overrides["SOURCE_URL"],
+ "snapshot_id": snapshot.id,
+ "source_url": snapshot.url,
"abx_snapshot_id": snapshot.id,
},
)
@@ -146,8 +146,8 @@ def test_run_snapshot_uses_isolated_bus_per_snapshot(monkeypatch):
assert len(download_calls) == 2
assert {call["snapshot_id"] for call in download_calls} == {str(snapshot_a.id), str(snapshot_b.id)}
assert {call["source_url"] for call in download_calls} == {snapshot_a.url, snapshot_b.url}
- assert len({id(call["bus"]) for call in download_calls}) == 2
- assert len(created_buses) == 3 # 1 crawl bus + 2 isolated snapshot buses
+ assert len({id(call["bus"]) for call in download_calls}) == 1
+ assert len(created_buses) == 1
def test_ensure_background_runner_starts_when_none_running(monkeypatch):
@@ -353,6 +353,62 @@ def test_installed_binary_config_overrides_include_valid_installed_binaries(monk
assert overrides["NODE_PATH"] == "/tmp/shared-lib/npm/node_modules"
+def test_installed_binary_config_overrides_do_not_map_hardcoded_artifacts_to_configurable_binary_keys(monkeypatch):
+ from archivebox.machine.models import Binary, Machine
+ from archivebox.services import runner as runner_module
+ from abx_dl.models import Plugin
+
+ machine = Machine.objects.create(
+ guid="test-guid-runner-singlefile-cache",
+ hostname="runner-host-singlefile",
+ hw_in_docker=False,
+ hw_in_vm=False,
+ hw_manufacturer="Test",
+ hw_product="Test Product",
+ hw_uuid="test-hw-runner-singlefile-cache",
+ os_arch="arm64",
+ os_family="darwin",
+ os_platform="macOS",
+ os_release="14.0",
+ os_kernel="Darwin",
+ stats={},
+ config={},
+ )
+ singlefile_extension = Binary.objects.create(
+ machine=machine,
+ name="singlefile",
+ abspath="/tmp/shared-lib/bin/singlefile",
+ version="1.0.0",
+ binprovider="chromewebstore",
+ binproviders="chromewebstore",
+ status=Binary.StatusChoices.INSTALLED,
+ )
+
+ monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine))
+ monkeypatch.setattr(Path, "is_file", lambda self: str(self) == singlefile_extension.abspath)
+ monkeypatch.setattr(runner_module.os, "access", lambda path, mode: str(path) == singlefile_extension.abspath)
+
+ overrides = runner_module._installed_binary_config_overrides(
+ {
+ "singlefile": Plugin(
+ name="singlefile",
+ path=Path("."),
+ hooks=[],
+ config_schema={"SINGLEFILE_BINARY": {"type": "string", "default": "single-file"}},
+ binaries=[
+ {"name": "{SINGLEFILE_BINARY}", "binproviders": "env,npm"},
+ {"name": "singlefile", "binproviders": "chromewebstore"},
+ ],
+ ),
+ },
+ config={"SINGLEFILE_BINARY": "single-file"},
+ )
+
+ assert "SINGLEFILE_BINARY" not in overrides
+ assert overrides["LIB_DIR"] == "/tmp/shared-lib"
+ assert overrides["LIB_BIN_DIR"] == "/tmp/shared-lib/bin"
+
+
def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch):
import asgiref.sync
@@ -700,11 +756,9 @@ def test_crawl_runner_calls_crawl_cleanup_after_snapshot_phase(monkeypatch):
"_run_crawl_cleanup",
lambda self, snapshot_id: cleanup_calls.append("abx_cleanup") or asyncio.sleep(0),
)
- monkeypatch.setattr(crawl, "cleanup", lambda: cleanup_calls.append("crawl_cleanup"))
-
asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run())
- assert cleanup_calls == ["crawl_cleanup", "abx_cleanup"]
+ assert cleanup_calls == ["abx_cleanup"]
def test_abx_process_service_background_monitor_finishes_after_process_exit(monkeypatch, tmp_path):
@@ -765,6 +819,9 @@ def test_abx_process_service_background_monitor_finishes_after_process_exit(monk
timeout=60,
snapshot_id="snap-1",
is_background=True,
+ url="https://example.org/",
+ process_type="hook",
+ worker_type="hook",
)
async def run_test():
diff --git a/uv.lock b/uv.lock
index ab25cc19..d9dd1bc6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -23,6 +23,7 @@ dependencies = [
{ name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "psutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "pydantic-settings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -39,6 +40,7 @@ requires-dist = [
{ name = "platformdirs", specifier = ">=4.0.0" },
{ name = "psutil", specifier = ">=7.2.1" },
{ name = "pydantic", specifier = ">=2.0.0" },
+ { name = "pydantic-settings", specifier = ">=2.0.0" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
{ name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" },
{ name = "requests", specifier = ">=2.28.0" },
@@ -105,16 +107,18 @@ version = "1.10.19"
source = { editable = "../abx-plugins" }
dependencies = [
{ name = "abx-pkg", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
- { name = "pydantic-settings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "jambo", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
[package.metadata]
requires-dist = [
{ name = "abx-pkg", editable = "../abx-pkg" },
+ { name = "abxbus", editable = "../abxbus" },
{ name = "feedparser", marker = "extra == 'dev'", specifier = ">=6.0.0" },
+ { name = "jambo", specifier = ">=0.1.7" },
{ name = "jinja2", marker = "extra == 'dev'", specifier = ">=3.1.0" },
- { name = "pydantic-settings", specifier = ">=2.0.0" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.408" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" },
{ name = "pytest-httpserver", marker = "extra == 'dev'", specifier = ">=1.1.0" },
@@ -135,7 +139,6 @@ source = { editable = "../abxbus" }
dependencies = [
{ name = "aiofiles", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
- { name = "portalocker", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -149,7 +152,6 @@ requires-dist = [
{ name = "asyncpg", marker = "extra == 'postgres'", specifier = ">=0.31.0" },
{ name = "nats-py", marker = "extra == 'bridges'", specifier = ">=2.13.1" },
{ name = "nats-py", marker = "extra == 'nats'", specifier = ">=2.13.1" },
- { name = "portalocker", specifier = ">=2.7.0" },
{ name = "pydantic", specifier = ">=2.11.5" },
{ name = "redis", marker = "extra == 'bridges'", specifier = ">=7.1.1" },
{ name = "redis", marker = "extra == 'redis'", specifier = ">=7.1.1" },
@@ -1000,6 +1002,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/36/85/c4e42d21cf748c696b8c05316bbd8e8666f17eeda0cf1743056f4cf7622b/djdt_flamegraph-0.2.13-py2.py3-none-any.whl", hash = "sha256:b3252b8cc9b586829166cc158b26952626cd6f41a3ffa92dceef2f5dbe5b99a0", size = 15256, upload-time = "2020-01-17T05:40:37.799Z" },
]
+[[package]]
+name = "dnspython"
+version = "2.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" },
+]
+
[[package]]
name = "docutils"
version = "0.22.4"
@@ -1009,6 +1020,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" },
]
+[[package]]
+name = "email-validator"
+version = "2.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "dnspython", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "idna", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" },
+]
+
[[package]]
name = "et-xmlfile"
version = "2.0.0"
@@ -1196,6 +1220,17 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" },
]
+[[package]]
+name = "jambo"
+version = "0.1.7"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "email-validator", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "jsonschema", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/91/f5/74de157c7aece6a070f99f18201a0e2f46cdfd0f9e337efd411745ed9b22/jambo-0.1.7.tar.gz", hash = "sha256:df89ab8209ebdf7a6e92252ec925979cd3d32811bf4a8182a97dc35b7df58f74", size = 137822, upload-time = "2026-01-14T19:17:30.302Z" }
+
[[package]]
name = "jedi"
version = "0.19.2"
@@ -1220,6 +1255,33 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },
]
+[[package]]
+name = "jsonschema"
+version = "4.26.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "attrs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "jsonschema-specifications", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "referencing", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "rpds-py", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" },
+]
+
+[[package]]
+name = "jsonschema-specifications"
+version = "2025.9.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "referencing", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
+]
+
[[package]]
name = "lexid"
version = "2021.1006"
@@ -1716,15 +1778,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
]
-[[package]]
-name = "portalocker"
-version = "3.2.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" },
-]
-
[[package]]
name = "prek"
version = "0.3.8"
@@ -2171,6 +2224,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c6/77/ed589c75db5d02a77a1d5d2d9abc63f29676467d396c64277f98b50b79c2/recommonmark-0.7.1-py2.py3-none-any.whl", hash = "sha256:1b1db69af0231efce3fa21b94ff627ea33dee7079a01dd0a7f8482c3da148b3f", size = 10214, upload-time = "2020-12-17T19:24:55.137Z" },
]
+[[package]]
+name = "referencing"
+version = "0.37.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "attrs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+ { name = "rpds-py", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" },
+]
+
[[package]]
name = "regex"
version = "2026.2.28"
@@ -2294,6 +2360,62 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" },
]
+[[package]]
+name = "rpds-py"
+version = "0.30.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" },
+ { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" },
+ { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" },
+ { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" },
+ { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" },
+ { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" },
+ { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" },
+ { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" },
+ { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" },
+ { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" },
+ { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" },
+ { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" },
+ { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" },
+ { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" },
+ { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" },
+ { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" },
+ { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" },
+ { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" },
+ { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" },
+ { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" },
+ { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" },
+ { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" },
+ { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" },
+ { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" },
+ { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" },
+]
+
[[package]]
name = "ruff"
version = "0.15.7"