diff --git a/archivebox/cli/archivebox_pluginmap.py b/archivebox/cli/archivebox_pluginmap.py index fa0edbe7..f17ddbb9 100644 --- a/archivebox/cli/archivebox_pluginmap.py +++ b/archivebox/cli/archivebox_pluginmap.py @@ -14,11 +14,10 @@ EVENT_FLOW_DIAGRAM = """ ├─────────────────────────────────────────────────────────────────────────────┤ │ │ │ InstallEvent │ -│ └─ on_Install__* │ -│ └─ BinaryRequest records │ -│ └─ BinaryRequestEvent │ -│ └─ on_BinaryRequest__* │ -│ └─ BinaryEvent / MachineEvent │ +│ └─ config.json > required_binaries │ +│ └─ BinaryRequestEvent │ +│ └─ on_BinaryRequest__* │ +│ └─ BinaryEvent │ │ │ │ CrawlEvent │ │ └─ CrawlSetupEvent │ @@ -70,15 +69,15 @@ def pluginmap( event_phases = { "InstallEvent": { - "description": "Pre-run dependency phase. on_Install hooks request binaries and update machine config.", - "emits": ["BinaryRequestEvent", "BinaryEvent", "MachineEvent", "ProcessEvent"], + "description": "Pre-run dependency phase. Enabled plugins emit BinaryRequest events from config.json required_binaries.", + "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"], }, "BinaryRequestEvent": { "description": "Provider phase. on_BinaryRequest hooks resolve or install requested binaries.", - "emits": ["BinaryEvent", "MachineEvent", "ProcessEvent"], + "emits": ["BinaryEvent", "ProcessEvent"], }, "BinaryEvent": { - "description": "Resolved binary metadata event. Projected into the DB/runtime config.", + "description": "Resolved binary metadata event. Projected into the DB binary cache.", "emits": [], }, "CrawlEvent": { @@ -87,11 +86,11 @@ def pluginmap( }, "CrawlSetupEvent": { "description": "Crawl-scoped setup phase. on_CrawlSetup hooks launch/configure shared daemons and runtime state.", - "emits": ["MachineEvent", "ProcessEvent"], + "emits": ["ProcessEvent"], }, "SnapshotEvent": { - "description": "Per-snapshot extraction phase. on_Snapshot hooks emit ArchiveResult, Snapshot, Tag, Machine, and BinaryRequest records.", - "emits": ["ArchiveResultEvent", "SnapshotEvent", "TagEvent", "MachineEvent", "BinaryRequestEvent", "ProcessEvent"], + "description": "Per-snapshot extraction phase. on_Snapshot hooks emit ArchiveResult, Snapshot, Tag, and BinaryRequest records.", + "emits": ["ArchiveResultEvent", "SnapshotEvent", "TagEvent", "BinaryRequestEvent", "ProcessEvent"], }, "SnapshotCleanupEvent": { "description": "Internal snapshot cleanup phase.", diff --git a/archivebox/cli/archivebox_version.py b/archivebox/cli/archivebox_version.py index a9ec19d6..8d61c64c 100755 --- a/archivebox/cli/archivebox_version.py +++ b/archivebox/cli/archivebox_version.py @@ -5,7 +5,6 @@ __package__ = "archivebox.cli" import sys import os import platform -import logging from pathlib import Path from collections.abc import Iterable @@ -124,17 +123,19 @@ def version( setup_django() from archivebox.machine.models import Machine, Binary - from archivebox.config.views import KNOWN_BINARIES, canonical_binary_name - from abx_dl.dependencies import load_binary machine = Machine.current() - requested_names = {canonical_binary_name(name) for name in binaries} if binaries else set() + if isinstance(binaries, str): + requested_names = {name.strip() for name in binaries.split(",") if name.strip()} + else: + requested_names = {name for name in (binaries or ()) if name} - db_binaries = { - canonical_binary_name(binary.name): binary for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at") - } - all_binary_names = sorted(set(KNOWN_BINARIES) | set(db_binaries.keys())) + db_binaries: dict[str, Binary] = {} + for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"): + db_binaries.setdefault(binary.name, binary) + + all_binary_names = sorted(requested_names or set(db_binaries.keys())) if not all_binary_names: prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]") @@ -163,37 +164,10 @@ def version( any_available = True continue - loaded = None - try: - abx_pkg_logger = logging.getLogger("abx_pkg") - previous_level = abx_pkg_logger.level - abx_pkg_logger.setLevel(logging.CRITICAL) - try: - loaded = load_binary({"name": name, "binproviders": "env,pip,npm,brew,apt"}) - finally: - abx_pkg_logger.setLevel(previous_level) - except Exception: - loaded = None - - if loaded and loaded.is_valid and loaded.loaded_abspath: - display_path = str(loaded.loaded_abspath).replace(str(DATA_DIR), ".").replace(str(Path("~").expanduser()), "~") - version_str = str(loaded.loaded_version or "unknown")[:15] - provider = str(getattr(getattr(loaded, "loaded_binprovider", None), "name", "") or "env")[:8] - prnt( - "", - "[green]√[/green]", - "", - name.ljust(18), - version_str.ljust(16), - provider.ljust(8), - display_path, - overflow="ignore", - crop=False, - ) - any_available = True - continue - - prnt("", "[red]X[/red]", "", name.ljust(18), "[grey53]not installed[/grey53]", overflow="ignore", crop=False) + status = ( + "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]" + ) + prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False) failures.append(name) if not any_available: diff --git a/archivebox/config/configset.py b/archivebox/config/configset.py index 869d1cf0..d14e1bfc 100644 --- a/archivebox/config/configset.py +++ b/archivebox/config/configset.py @@ -138,10 +138,9 @@ def get_config( 3. Per-user config (user.config JSON field) 4. Per-persona config (persona.get_derived_config() - includes CHROME_USER_DATA_DIR etc.) 5. Environment variables - 6. Per-machine config (machine.config JSON field - resolved binary paths) - 7. Config file (ArchiveBox.conf) - 8. Plugin schema defaults (config.json) - 9. Core config defaults + 6. Config file (ArchiveBox.conf) + 7. Plugin schema defaults (config.json) + 8. Core config defaults Args: defaults: Default values to start with @@ -150,7 +149,7 @@ def get_config( crawl: Crawl object with config JSON field snapshot: Snapshot object with config JSON field archiveresult: ArchiveResult object (auto-fetches snapshot) - machine: Machine object with config JSON field (defaults to Machine.current()) + machine: Unused legacy argument kept for call compatibility Note: Objects are auto-fetched from relationships if not provided: - snapshot auto-fetched from archiveresult.snapshot @@ -221,19 +220,6 @@ def get_config( file_config = BaseConfigSet.load_from_file(config_file) config.update(file_config) - # Apply machine config overrides (cached binary paths, etc.) - if machine is None: - # Default to current machine if not provided - try: - from archivebox.machine.models import Machine - - machine = Machine.current() - except Exception: - pass # Machine might not be available during early init - - if machine and hasattr(machine, "config") and machine.config: - config.update(machine.config) - # Override with environment variables (for keys that exist in config) for key in config: env_val = os.environ.get(key) diff --git a/archivebox/config/views.py b/archivebox/config/views.py index 7c2f99dd..23dbbe59 100644 --- a/archivebox/config/views.py +++ b/archivebox/config/views.py @@ -29,42 +29,6 @@ ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/" INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/" -# Common binaries to check for -KNOWN_BINARIES = [ - "wget", - "curl", - "chromium", - "chrome", - "google-chrome", - "google-chrome-stable", - "node", - "npm", - "npx", - "yt-dlp", - "git", - "singlefile", - "readability-extractor", - "mercury-parser", - "python3", - "python", - "bash", - "zsh", - "ffmpeg", - "ripgrep", - "rg", - "sonic", - "archivebox", -] - -CANONICAL_BINARY_ALIASES = { - "youtube-dl": "yt-dlp", - "ytdlp": "yt-dlp", - "ripgrep": "rg", - "singlefile": "single-file", - "mercury-parser": "postlight-parser", -} - - def is_superuser(request: HttpRequest) -> bool: return bool(getattr(request.user, "is_superuser", False)) @@ -131,13 +95,12 @@ def get_environment_binary_url(name: str) -> str: return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/" -def get_installed_binary_change_url(name: str, binary: Any) -> str | None: - binary_id = getattr(binary, "id", None) - if not binary_id: +def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | None: + if binary is None or not binary.id: return None - base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary_id}/change/" - changelist_filters = urlencode({"q": canonical_binary_name(name)}) + base_url = binary.admin_change_url or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/" + changelist_filters = urlencode({"q": name}) return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}" @@ -168,11 +131,14 @@ def render_code_tag_list(values: list[str]) -> str: def render_plugin_metadata_html(config: dict[str, Any]) -> str: + required_binaries = [ + str(item.get("name")) for item in (config.get("required_binaries") or []) if isinstance(item, dict) and item.get("name") + ] rows = ( ("Title", config.get("title") or "(none)"), ("Description", config.get("description") or "(none)"), ("Required Plugins", mark_safe(render_link_tag_list(config.get("required_plugins") or [], get_plugin_docs_url))), - ("Required Binaries", mark_safe(render_link_tag_list(config.get("required_binaries") or [], get_environment_binary_url))), + ("Required Binaries", mark_safe(render_link_tag_list(required_binaries, get_environment_binary_url))), ("Output MIME Types", mark_safe(render_code_tag_list(config.get("output_mimetypes") or []))), ) @@ -383,10 +349,6 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str: return f" {str(obj)}" -def canonical_binary_name(name: str) -> str: - return CANONICAL_BINARY_ALIASES.get(name, name) - - def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]: return ( int(binary.status == Binary.StatusChoices.INSTALLED), @@ -399,24 +361,11 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]: def get_db_binaries_by_name() -> dict[str, Binary]: grouped: dict[str, list[Binary]] = {} for binary in Binary.objects.all(): - grouped.setdefault(canonical_binary_name(binary.name), []).append(binary) + grouped.setdefault(binary.name, []).append(binary) return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()} -def serialize_binary_record(name: str, binary: Binary | None) -> dict[str, Any]: - is_installed = bool(binary and binary.status == Binary.StatusChoices.INSTALLED) - return { - "name": canonical_binary_name(name), - "version": str(getattr(binary, "version", "") or ""), - "binprovider": str(getattr(binary, "binprovider", "") or ""), - "abspath": str(getattr(binary, "abspath", "") or ""), - "sha256": str(getattr(binary, "sha256", "") or ""), - "status": str(getattr(binary, "status", "") or ""), - "is_available": is_installed and bool(getattr(binary, "abspath", "") or ""), - } - - def get_filesystem_plugins() -> dict[str, dict[str, Any]]: """Discover plugins from filesystem directories.""" import json @@ -474,14 +423,14 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext: all_binary_names = sorted(db_binaries.keys()) for name in all_binary_names: - merged = serialize_binary_record(name, db_binaries.get(name)) + binary = db_binaries.get(name) rows["Binary Name"].append(ItemLink(name, key=name)) - if merged["is_available"]: - rows["Found Version"].append(f"✅ {merged['version']}" if merged["version"] else "✅ found") - rows["Provided By"].append(merged["binprovider"] or "-") - rows["Found Abspath"].append(merged["abspath"] or "-") + if binary and binary.is_valid: + rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found") + rows["Provided By"].append(binary.binprovider or "-") + rows["Found Abspath"].append(binary.abspath or "-") else: rows["Found Version"].append("❌ missing") rows["Provided By"].append("-") @@ -496,22 +445,20 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext: @render_with_item_view def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: assert is_superuser(request), "Must be a superuser to view configuration settings." - key = canonical_binary_name(key) db_binary = get_db_binaries_by_name().get(key) - merged = serialize_binary_record(key, db_binary) - - if merged["is_available"]: + if db_binary and db_binary.is_valid: + binary_data = db_binary.to_json() section: SectionData = { "name": key, - "description": mark_safe(render_binary_detail_description(key, merged, db_binary)), + "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)), "fields": { "name": key, - "binprovider": merged["binprovider"] or "-", - "abspath": merged["abspath"] or "not found", - "version": merged["version"] or "unknown", - "sha256": merged["sha256"], - "status": merged["status"], + "binprovider": db_binary.binprovider or "-", + "abspath": db_binary.abspath or "not found", + "version": db_binary.version or "unknown", + "sha256": db_binary.sha256, + "status": db_binary.status, }, "help_texts": {}, } @@ -526,10 +473,10 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: "description": "No persisted Binary record found", "fields": { "name": key, - "binprovider": merged["binprovider"] or "not recorded", - "abspath": merged["abspath"] or "not recorded", - "version": merged["version"] or "N/A", - "status": merged["status"] or "unrecorded", + "binprovider": db_binary.binprovider if db_binary else "not recorded", + "abspath": db_binary.abspath if db_binary else "not recorded", + "version": db_binary.version if db_binary else "N/A", + "status": db_binary.status if db_binary else "unrecorded", }, "help_texts": {}, } diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 0c11ec49..e356fea4 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1226,7 +1226,7 @@ def live_progress_view(request): return (plugin, plugin, "unknown", "") phase = "unknown" - if normalized_hook_name.startswith("on_Install__"): + if normalized_hook_name == "InstallEvent": phase = "install" elif normalized_hook_name.startswith("on_CrawlSetup__"): phase = "crawl" @@ -1966,7 +1966,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont Priority order (highest to lowest):
  1. Environment - Environment variables
  2. -
  3. Machine - Machine-specific overrides (e.g., resolved binary paths) +
  4. Machine - Machine-specific overrides {f'
    → Edit {key} in Machine.config for this server' if machine_admin_url else ""}
  5. Config File - data/ArchiveBox.conf
  6. diff --git a/archivebox/hooks.py b/archivebox/hooks.py index af763c3a..2ff94ba2 100644 --- a/archivebox/hooks.py +++ b/archivebox/hooks.py @@ -9,11 +9,14 @@ ArchiveBox no longer drives plugin execution itself during normal crawls. - parses hook stdout JSONL records into ArchiveBox models when needed Hook-backed event families are discovered from filenames like: - on_Install__* on_BinaryRequest__* on_CrawlSetup__* on_Snapshot__* +InstallEvent itself is still part of the runtime lifecycle, but it has no +corresponding hook family. Its dependency declarations come directly from each +plugin's `config.json > required_binaries`. + Lifecycle event names like `InstallEvent` or `SnapshotCleanupEvent` are normalized to the corresponding `on_{EventFamily}__*` prefix by a simple string transform. If no scripts exist for that prefix, discovery returns `[]`. @@ -212,7 +215,7 @@ def discover_hooks( pattern_direct = f"on_{hook_event_name}__*.{ext}" hooks.extend(base_dir.glob(pattern_direct)) - # Binary install hooks are provider hooks, not end-user extractors. They + # Binary provider hooks are not end-user extractors. They # self-filter via `binproviders`, so applying the PLUGINS whitelist here # can hide the very installer needed by a selected plugin (e.g. # `--plugins=singlefile` still needs the `npm` BinaryRequest hook). @@ -394,54 +397,14 @@ def run_hook( # Derive LIB_BIN_DIR from LIB_DIR if not set lib_bin_dir = Path(lib_dir) / "bin" - # Build PATH with proper precedence: - # 1. LIB_BIN_DIR (highest priority - local symlinked binaries) - # 2. Machine.config.PATH (pip/npm bin dirs from providers) - # 3. os.environ['PATH'] (system PATH) - - if lib_bin_dir: - lib_bin_dir = str(lib_bin_dir) - env["LIB_BIN_DIR"] = lib_bin_dir - - # Start with base PATH - current_path = env.get("PATH", "") - - # Prepend Machine.config.PATH if it exists (treat as extra entries, not replacement) - try: - from archivebox.machine.models import Machine - - machine = Machine.current() - if machine and machine.config: - machine_path = machine.config.get("PATH") - if machine_path: - # Prepend machine_path to current PATH - current_path = f"{machine_path}:{current_path}" if current_path else machine_path - except Exception: - pass - - # Finally prepend LIB_BIN_DIR to the front (highest priority) - if lib_bin_dir: - if not current_path.startswith(f"{lib_bin_dir}:"): - env["PATH"] = f"{lib_bin_dir}:{current_path}" if current_path else lib_bin_dir - else: - env["PATH"] = current_path - else: - env["PATH"] = current_path - - # Set NODE_PATH for Node.js module resolution - # Priority: config dict > Machine.config > derive from LIB_DIR + # Set NODE_PATH for Node.js module resolution. + # Priority: config dict > derive from LIB_DIR node_path = config.get("NODE_PATH") if not node_path and lib_dir: # Derive from LIB_DIR/npm/node_modules (create if needed) node_modules_dir = Path(lib_dir) / "npm" / "node_modules" node_modules_dir.mkdir(parents=True, exist_ok=True) node_path = str(node_modules_dir) - if not node_path: - try: - # Fallback to Machine.config - node_path = machine.config.get("NODE_MODULES_DIR") - except Exception: - pass if node_path: env["NODE_PATH"] = node_path env["NODE_MODULES_DIR"] = node_path # For backwards compatibility @@ -472,6 +435,41 @@ def run_hook( else: env[key] = str(value) + # Build PATH with proper precedence: + # 1. path-like *_BINARY parents (explicit binary overrides / cached abspaths) + # 2. LIB_BIN_DIR (local symlinked binaries) + # 3. existing PATH + runtime_bin_dirs: list[str] = [] + if lib_bin_dir: + lib_bin_dir = str(lib_bin_dir) + env["LIB_BIN_DIR"] = lib_bin_dir + for key, raw_value in env.items(): + if not key.endswith("_BINARY"): + continue + value = str(raw_value or "").strip() + if not value: + continue + path_value = Path(value).expanduser() + if not (path_value.is_absolute() or "/" in value or "\\" in value): + continue + binary_dir = str(path_value.resolve(strict=False).parent) + if binary_dir and binary_dir not in runtime_bin_dirs: + runtime_bin_dirs.append(binary_dir) + if lib_bin_dir and lib_bin_dir not in runtime_bin_dirs: + runtime_bin_dirs.append(lib_bin_dir) + uv_value = str(env.get("UV") or "").strip() + if uv_value: + uv_bin_dir = str(Path(uv_value).expanduser().resolve(strict=False).parent) + if uv_bin_dir and uv_bin_dir not in runtime_bin_dirs: + runtime_bin_dirs.append(uv_bin_dir) + + current_path = env.get("PATH", "") + path_parts = [part for part in current_path.split(os.pathsep) if part] + for extra_dir in reversed(runtime_bin_dirs): + if extra_dir not in path_parts: + path_parts.insert(0, extra_dir) + env["PATH"] = os.pathsep.join(path_parts) + # Create output directory if needed output_dir.mkdir(parents=True, exist_ok=True) diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 1d8f75f6..3479114d 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -101,8 +101,6 @@ def _get_process_binary_env_keys(plugin_name: str, hook_path: str, env: dict[str schema_keys.sort( key=lambda key: ( key != f"{plugin_key}_BINARY", - key.endswith("_NODE_BINARY"), - key.endswith("_CHROME_BINARY"), key, ), ) @@ -117,8 +115,6 @@ def _get_process_binary_env_keys(plugin_name: str, hook_path: str, env: dict[str hook_suffix = Path(hook_path).suffix.lower() if hook_suffix == ".js": - if plugin_key: - add(f"{plugin_key}_NODE_BINARY") add("NODE_BINARY") return keys @@ -160,7 +156,7 @@ class Machine(ModelWithHealthStats): default=dict, null=True, blank=True, - help_text="Machine-specific config overrides (e.g., resolved binary paths like WGET_BINARY)", + help_text="Machine-specific config overrides.", ) num_uses_failed = models.PositiveIntegerField(default=0) num_uses_succeeded = models.PositiveIntegerField(default=0) @@ -176,24 +172,13 @@ class Machine(ModelWithHealthStats): global _CURRENT_MACHINE if _CURRENT_MACHINE: if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL): - return cls._sanitize_config(cls._hydrate_config_from_sibling(_CURRENT_MACHINE)) + return cls._sanitize_config(_CURRENT_MACHINE) _CURRENT_MACHINE = None _CURRENT_MACHINE, _ = cls.objects.update_or_create( guid=get_host_guid(), defaults={"hostname": socket.gethostname(), **get_os_info(), **get_vm_info(), "stats": get_host_stats()}, ) - return cls._sanitize_config(cls._hydrate_config_from_sibling(_CURRENT_MACHINE)) - - @classmethod - def _hydrate_config_from_sibling(cls, machine: Machine) -> Machine: - if machine.config: - return machine - - sibling = cls.objects.exclude(pk=machine.pk).filter(hostname=machine.hostname).exclude(config={}).order_by("-modified_at").first() - if sibling and sibling.config: - machine.config = dict(sibling.config) - machine.save(update_fields=["config", "modified_at"]) - return machine + return cls._sanitize_config(_CURRENT_MACHINE) @classmethod def _sanitize_config(cls, machine: Machine) -> Machine: @@ -622,12 +607,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): from archivebox.config.configset import get_config # Get merged config (Binary doesn't have crawl/snapshot context). - # Binary workers can install several dependencies in one process, so - # refresh from the latest persisted machine config before each hook run. config = get_config() - current_machine = Machine.current() - if current_machine.config: - config.update(current_machine.config) # ArchiveBox installs the puppeteer package and Chromium in separate # hook phases. Suppress puppeteer's bundled browser download during the @@ -760,6 +740,11 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): binary_abspath = Path(self.abspath).resolve() lib_bin_dir = Path(lib_bin_dir).resolve() + binary_parts = binary_abspath.parts + try: + app_index = next(index for index, part in enumerate(binary_parts) if part.endswith(".app")) + except StopIteration: + app_index = -1 # Create LIB_BIN_DIR if it doesn't exist try: @@ -772,6 +757,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine): binary_name = binary_abspath.name symlink_path = lib_bin_dir / binary_name + if app_index != -1 and len(binary_parts) > app_index + 2 and binary_parts[app_index + 1 : app_index + 3] == ("Contents", "MacOS"): + if symlink_path.exists() or symlink_path.is_symlink(): + try: + symlink_path.unlink() + except (OSError, PermissionError) as e: + print(f"Failed to remove existing file at {symlink_path}: {e}", file=sys.stderr) + return None + return binary_abspath + # Remove existing symlink/file if it exists if symlink_path.exists() or symlink_path.is_symlink(): try: diff --git a/archivebox/services/__init__.py b/archivebox/services/__init__.py index 1e6c5b63..8b41348b 100644 --- a/archivebox/services/__init__.py +++ b/archivebox/services/__init__.py @@ -2,7 +2,6 @@ from .archive_result_service import ArchiveResultService from .binary_service import BinaryService from .crawl_service import CrawlService from .machine_service import MachineService -from .process_request_service import ProcessRequestService from .process_service import ProcessService from .runner import run_binary, run_crawl, run_install, run_pending_crawls from .snapshot_service import SnapshotService @@ -13,7 +12,6 @@ __all__ = [ "BinaryService", "CrawlService", "MachineService", - "ProcessRequestService", "ProcessService", "SnapshotService", "TagService", diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py index 71097e0e..184c2b9f 100644 --- a/archivebox/services/binary_service.py +++ b/archivebox/services/binary_service.py @@ -14,6 +14,23 @@ class BinaryService(BaseService): async def on_BinaryRequestEvent__Outer(self, event: BinaryRequestEvent) -> None: await run_db_op(self._project_binary, event) + cached = await run_db_op(self._load_cached_binary, event) + if cached is not None: + await self.bus.emit( + BinaryEvent( + name=event.name, + plugin_name=event.plugin_name, + hook_name=event.hook_name, + abspath=cached["abspath"], + version=cached["version"], + sha256=cached["sha256"], + binproviders=event.binproviders or cached["binproviders"], + binprovider=cached["binprovider"], + overrides=event.overrides or cached["overrides"], + binary_id=event.binary_id, + machine_id=event.machine_id or cached["machine_id"], + ), + ) async def on_BinaryEvent__Outer(self, event: BinaryEvent) -> None: resolved = await asyncio.to_thread(self._resolve_installed_binary_metadata, event) @@ -44,6 +61,29 @@ class BinaryService(BaseService): }, ) + def _load_cached_binary(self, event: BinaryRequestEvent) -> dict[str, str] | None: + from archivebox.machine.models import Binary, Machine + + machine = Machine.current() + installed = ( + Binary.objects.filter(machine=machine, name=event.name, status=Binary.StatusChoices.INSTALLED) + .exclude(abspath="") + .exclude(abspath__isnull=True) + .order_by("-modified_at") + .first() + ) + if installed is None: + return None + return { + "abspath": installed.abspath, + "version": installed.version or "", + "sha256": installed.sha256 or "", + "binproviders": installed.binproviders or "", + "binprovider": installed.binprovider or "", + "machine_id": str(installed.machine_id), + "overrides": installed.overrides or {}, + } + def _resolve_installed_binary_metadata(self, event: BinaryEvent) -> dict[str, str]: resolved = { "abspath": event.abspath or "", @@ -77,12 +117,11 @@ class BinaryService(BaseService): "overrides": event.overrides or {}, } binary = load_binary(spec) - resolved["abspath"] = str(getattr(binary, "abspath", None) or resolved["abspath"] or "") - resolved["version"] = str(getattr(binary, "version", None) or resolved["version"] or "") - resolved["sha256"] = str(getattr(binary, "sha256", None) or resolved["sha256"] or "") - provider_name = getattr(getattr(binary, "loaded_binprovider", None), "name", None) - if provider_name: - resolved["binprovider"] = str(provider_name) + resolved["abspath"] = str(binary.abspath or resolved["abspath"] or "") + resolved["version"] = str(binary.version or resolved["version"] or "") + resolved["sha256"] = str(binary.sha256 or resolved["sha256"] or "") + if binary.loaded_binprovider is not None and binary.loaded_binprovider.name: + resolved["binprovider"] = str(binary.loaded_binprovider.name) except Exception: pass diff --git a/archivebox/services/machine_service.py b/archivebox/services/machine_service.py index 574893ee..e90975ad 100644 --- a/archivebox/services/machine_service.py +++ b/archivebox/services/machine_service.py @@ -14,13 +14,13 @@ class MachineService(BaseService): await run_db_op(self._project, event) def _project(self, event: MachineEvent) -> None: - from archivebox.machine.models import Machine + from archivebox.machine.models import Machine, _sanitize_machine_config machine = Machine.current() config = dict(machine.config or {}) if event.config is not None: - config.update(event.config) + config.update(_sanitize_machine_config(event.config)) elif event.method == "update": key = event.key.replace("config/", "", 1).strip() if key: @@ -28,5 +28,5 @@ class MachineService(BaseService): else: return - machine.config = config + machine.config = _sanitize_machine_config(config) machine.save(update_fields=["config", "modified_at"]) diff --git a/archivebox/services/process_request_service.py b/archivebox/services/process_request_service.py deleted file mode 100644 index cbad168b..00000000 --- a/archivebox/services/process_request_service.py +++ /dev/null @@ -1,179 +0,0 @@ -from __future__ import annotations - -import asyncio -from datetime import datetime, timezone -import json -from pathlib import Path -import shlex -import socket -import time -from typing import ClassVar - -from abxbus import BaseEvent -from abx_dl.events import ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, ProcessStdoutEvent -from abx_dl.services.base import BaseService - - -def _is_port_listening(host: str, port: int) -> bool: - if not host or not port: - return False - try: - with socket.create_connection((host, port), timeout=0.5): - return True - except OSError: - return False - - -def _supervisor_env(env: dict[str, str]) -> str: - pairs = [] - for key, value in env.items(): - escaped = value.replace('"', '\\"') - pairs.append(f'{key}="{escaped}"') - return ",".join(pairs) - - -def _iso_from_epoch(value: object) -> str: - if not isinstance(value, (int, float)) or value <= 0: - return "" - return datetime.fromtimestamp(value, tz=timezone.utc).isoformat() - - -def _ensure_worker(process_event: ProcessEvent) -> dict[str, object]: - from archivebox.workers.supervisord_util import get_or_create_supervisord_process, get_worker, start_worker - - output_dir = Path(process_event.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - worker_name = process_event.hook_name - supervisor = get_or_create_supervisord_process(daemonize=True) - - existing = get_worker(supervisor, worker_name) - if ( - isinstance(existing, dict) - and existing.get("statename") == "RUNNING" - and ( - not process_event.daemon_startup_host - or not process_event.daemon_startup_port - or _is_port_listening(process_event.daemon_startup_host, process_event.daemon_startup_port) - ) - ): - return existing - - daemon = { - "name": worker_name, - "command": shlex.join([process_event.hook_path, *process_event.hook_args]), - "directory": str(output_dir), - "autostart": "false", - "autorestart": "true", - "stdout_logfile": str(output_dir / f"{worker_name}.stdout.log"), - "redirect_stderr": "true", - } - if process_event.env: - daemon["environment"] = _supervisor_env(process_event.env) - - proc = start_worker(supervisor, daemon) - deadline = time.monotonic() + max(float(process_event.daemon_startup_timeout), 0.5) - while time.monotonic() < deadline: - current = get_worker(supervisor, worker_name) - if isinstance(current, dict) and current.get("statename") == "RUNNING": - if ( - not process_event.daemon_startup_host - or not process_event.daemon_startup_port - or _is_port_listening(process_event.daemon_startup_host, process_event.daemon_startup_port) - ): - return current - time.sleep(0.1) - return proc if isinstance(proc, dict) else {} - - -class ProcessRequestService(BaseService): - LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [ProcessStdoutEvent] - EMITS: ClassVar[list[type[BaseEvent]]] = [ProcessEvent, ProcessStartedEvent, ProcessCompletedEvent] - - async def on_ProcessStdoutEvent(self, event: ProcessStdoutEvent) -> None: - try: - record = json.loads(event.line) - except (json.JSONDecodeError, ValueError): - return - if not isinstance(record, dict) or record.pop("type", "") != "ProcessEvent": - return - - process_event = ProcessEvent( - plugin_name=record.get("plugin_name") or event.plugin_name, - hook_name=record.get("hook_name") or "process_request", - hook_path=record["hook_path"], - hook_args=[str(arg) for arg in record.get("hook_args", [])], - is_background=bool(record.get("is_background", True)), - output_dir=record.get("output_dir") or event.output_dir, - env={str(key): str(value) for key, value in (record.get("env") or {}).items()}, - snapshot_id=record.get("snapshot_id") or event.snapshot_id, - timeout=int(record.get("timeout") or 60), - daemon=bool(record.get("daemon", False)), - daemon_startup_host=str(record.get("daemon_startup_host") or ""), - daemon_startup_port=int(record.get("daemon_startup_port") or 0), - daemon_startup_timeout=float(record.get("daemon_startup_timeout") or 0.0), - process_type=str(record.get("process_type") or ""), - worker_type=str(record.get("worker_type") or ""), - event_timeout=float(record.get("event_timeout") or 360.0), - event_handler_timeout=float(record.get("event_handler_timeout") or 390.0), - ) - if not process_event.daemon: - await self.bus.emit(process_event) - return - - proc = await asyncio.to_thread(_ensure_worker, process_event) - process_id = str(record.get("process_id") or f"worker:{process_event.hook_name}") - start_ts = _iso_from_epoch(proc.get("start")) - pid = int(proc.get("pid") or 0) - statename = str(proc.get("statename") or "") - exitstatus = int(proc.get("exitstatus") or 0) - process_type = process_event.process_type or "worker" - worker_type = process_event.worker_type or process_event.plugin_name - - if statename == "RUNNING" and pid: - await self.bus.emit( - ProcessStartedEvent( - plugin_name=process_event.plugin_name, - hook_name=process_event.hook_name, - hook_path=process_event.hook_path, - hook_args=process_event.hook_args, - output_dir=process_event.output_dir, - env=process_event.env, - timeout=process_event.timeout, - pid=pid, - process_id=process_id, - snapshot_id=process_event.snapshot_id, - is_background=True, - process_type=process_type, - worker_type=worker_type, - start_ts=start_ts, - ), - ) - return - - stderr = ( - f"Worker {process_event.hook_name} failed to start" - if not statename - else f"Worker {process_event.hook_name} state={statename} exitstatus={exitstatus}" - ) - await self.bus.emit( - ProcessCompletedEvent( - plugin_name=process_event.plugin_name, - hook_name=process_event.hook_name, - hook_path=process_event.hook_path, - hook_args=process_event.hook_args, - env=process_event.env, - stdout="", - stderr=stderr, - exit_code=exitstatus or 1, - output_dir=process_event.output_dir, - is_background=True, - process_id=process_id, - snapshot_id=process_event.snapshot_id, - pid=pid, - process_type=process_type, - worker_type=worker_type, - start_ts=start_ts, - end_ts=datetime.now(tz=timezone.utc).isoformat(), - ), - ) - raise RuntimeError(stderr) diff --git a/archivebox/services/process_service.py b/archivebox/services/process_service.py index c550d2ff..21547b19 100644 --- a/archivebox/services/process_service.py +++ b/archivebox/services/process_service.py @@ -1,11 +1,19 @@ from __future__ import annotations -from datetime import datetime -from typing import TYPE_CHECKING +import asyncio +from datetime import datetime, timezone as datetime_timezone +import json +from pathlib import Path +import shlex +import socket +import time +from typing import TYPE_CHECKING, Any, ClassVar +from urllib.parse import urlparse from django.utils import timezone -from abx_dl.events import ProcessCompletedEvent, ProcessStartedEvent +from abxbus import BaseEvent +from abx_dl.events import ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, ProcessStdoutEvent from abx_dl.services.base import BaseService from .db import run_db_op @@ -14,6 +22,9 @@ if TYPE_CHECKING: from archivebox.machine.models import Process +WORKER_READY_TIMEOUT = 10.0 + + def parse_event_datetime(value: str | None): if not value: return None @@ -26,14 +37,218 @@ def parse_event_datetime(value: str | None): return dt +def _is_port_listening(host: str, port: int) -> bool: + if not host or not port: + return False + try: + with socket.create_connection((host, port), timeout=0.5): + return True + except OSError: + return False + + +def _worker_socket_from_url(url: str) -> tuple[str, int] | None: + if not url: + return None + parsed = urlparse(url) + if parsed.scheme != "tcp" or not parsed.hostname or not parsed.port: + return None + return parsed.hostname, parsed.port + + +def _supervisor_env(env: dict[str, str]) -> str: + pairs = [] + for key, value in env.items(): + escaped = value.replace('"', '\\"') + pairs.append(f'{key}="{escaped}"') + return ",".join(pairs) + + +def _iso_from_epoch(value: object) -> str: + if not isinstance(value, (int, float)) or value <= 0: + return "" + return datetime.fromtimestamp(value, tz=datetime_timezone.utc).isoformat() + + +def _int_from_object(value: object) -> int: + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) + if isinstance(value, str): + try: + return int(value) + except ValueError: + return 0 + return 0 + + +def _ensure_worker(process_event: ProcessEvent) -> dict[str, object]: + from archivebox.workers.supervisord_util import get_or_create_supervisord_process, get_worker, start_worker + + output_dir = Path(process_event.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + worker_name = process_event.hook_name + supervisor = get_or_create_supervisord_process(daemonize=True) + worker_socket = _worker_socket_from_url(getattr(process_event, "url", "")) + + existing = get_worker(supervisor, worker_name) + if ( + isinstance(existing, dict) + and existing.get("statename") == "RUNNING" + and (worker_socket is None or _is_port_listening(*worker_socket)) + ): + return existing + + daemon = { + "name": worker_name, + "command": shlex.join([process_event.hook_path, *process_event.hook_args]), + "directory": str(output_dir), + "autostart": "false", + "autorestart": "true", + "stdout_logfile": str(output_dir / f"{worker_name}.stdout.log"), + "redirect_stderr": "true", + } + if process_event.env: + daemon["environment"] = _supervisor_env(process_event.env) + + proc = start_worker(supervisor, daemon) + deadline = time.monotonic() + WORKER_READY_TIMEOUT + while time.monotonic() < deadline: + current = get_worker(supervisor, worker_name) + if isinstance(current, dict) and current.get("statename") == "RUNNING": + if worker_socket is None or _is_port_listening(*worker_socket): + return current + time.sleep(0.1) + return proc if isinstance(proc, dict) else {} + + class ProcessService(BaseService): - LISTENS_TO = [ProcessStartedEvent, ProcessCompletedEvent] - EMITS = [] + LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [ProcessStdoutEvent, ProcessStartedEvent, ProcessCompletedEvent] + EMITS: ClassVar[list[type[BaseEvent]]] = [ProcessEvent, ProcessStartedEvent, ProcessCompletedEvent] def __init__(self, bus): self.process_ids: dict[str, str] = {} super().__init__(bus) + async def on_ProcessStdoutEvent(self, event: ProcessStdoutEvent) -> None: + try: + record = json.loads(event.line) + except (json.JSONDecodeError, ValueError): + return + if not isinstance(record, dict) or record.get("type") != "ProcessEvent": + return + + passthrough_fields: dict[str, Any] = { + key: value + for key, value in record.items() + if key + not in { + "type", + "plugin_name", + "hook_name", + "hook_path", + "hook_args", + "is_background", + "output_dir", + "env", + "snapshot_id", + "process_id", + "url", + "timeout", + "daemon", + "process_type", + "worker_type", + "event_timeout", + "event_handler_timeout", + } + } + process_event = ProcessEvent( + plugin_name=record.get("plugin_name") or event.plugin_name, + hook_name=record.get("hook_name") or "process", + hook_path=record["hook_path"], + hook_args=[str(arg) for arg in record.get("hook_args", [])], + is_background=bool(record.get("is_background", True)), + output_dir=record.get("output_dir") or event.output_dir, + env={str(key): str(value) for key, value in (record.get("env") or {}).items()}, + snapshot_id=record.get("snapshot_id") or event.snapshot_id, + timeout=int(record.get("timeout") or 60), + daemon=bool(record.get("daemon", False)), + url=str(record.get("url") or ""), + process_type=str(record.get("process_type") or ""), + worker_type=str(record.get("worker_type") or ""), + event_timeout=float(record.get("event_timeout") or 360.0), + event_handler_timeout=float(record.get("event_handler_timeout") or 390.0), + **passthrough_fields, + ) + if not process_event.daemon: + await self.bus.emit(process_event) + return + + proc = await asyncio.to_thread(_ensure_worker, process_event) + process_id = str(record.get("process_id") or f"worker:{process_event.hook_name}") + start_ts = _iso_from_epoch(proc.get("start")) + pid = _int_from_object(proc.get("pid")) + statename = str(proc.get("statename") or "") + exitstatus = _int_from_object(proc.get("exitstatus")) + process_type = process_event.process_type or "worker" + worker_type = process_event.worker_type or process_event.plugin_name + + if statename == "RUNNING" and pid: + await self.bus.emit( + ProcessStartedEvent( + plugin_name=process_event.plugin_name, + hook_name=process_event.hook_name, + hook_path=process_event.hook_path, + hook_args=process_event.hook_args, + output_dir=process_event.output_dir, + env=process_event.env, + timeout=process_event.timeout, + pid=pid, + process_id=process_id, + snapshot_id=process_event.snapshot_id, + is_background=True, + url=process_event.url, + process_type=process_type, + worker_type=worker_type, + start_ts=start_ts, + **passthrough_fields, + ), + ) + return + + stderr = ( + f"Worker {process_event.hook_name} failed to start" + if not statename + else f"Worker {process_event.hook_name} state={statename} exitstatus={exitstatus}" + ) + await self.bus.emit( + ProcessCompletedEvent( + plugin_name=process_event.plugin_name, + hook_name=process_event.hook_name, + hook_path=process_event.hook_path, + hook_args=process_event.hook_args, + env=process_event.env, + stdout="", + stderr=stderr, + exit_code=exitstatus or 1, + output_dir=process_event.output_dir, + is_background=True, + process_id=process_id, + snapshot_id=process_event.snapshot_id, + pid=pid, + url=process_event.url, + process_type=process_type, + worker_type=worker_type, + start_ts=start_ts, + end_ts=datetime.now(tz=datetime_timezone.utc).isoformat(), + **passthrough_fields, + ), + ) + raise RuntimeError(stderr) + async def on_ProcessStartedEvent__Outer(self, event: ProcessStartedEvent) -> None: await run_db_op(self._project_started, event) @@ -51,7 +266,7 @@ class ProcessService(BaseService): if db_process_id: process = Process.objects.filter(id=db_process_id).first() if process is not None: - if process.iface_id != iface.id or process.machine_id != iface.machine_id: + if getattr(process, "iface_id", None) != iface.id or process.machine_id != iface.machine_id: process.iface = iface process.machine = iface.machine process.save(update_fields=["iface", "machine", "modified_at"]) @@ -84,6 +299,7 @@ class ProcessService(BaseService): env=event.env, timeout=getattr(event, "timeout", 60), pid=event.pid or None, + url=getattr(event, "url", "") or None, started_at=parse_event_datetime(getattr(event, "start_ts", "")), status=Process.StatusChoices.RUNNING, retry_at=None, @@ -98,6 +314,7 @@ class ProcessService(BaseService): process.env = event.env process.timeout = event.timeout process.pid = event.pid or None + process.url = getattr(event, "url", "") or process.url process.process_type = getattr(event, "process_type", "") or process.process_type process.worker_type = getattr(event, "worker_type", "") or process.worker_type process.started_at = parse_event_datetime(event.start_ts) or process.started_at or timezone.now() @@ -113,6 +330,7 @@ class ProcessService(BaseService): process.cmd = [event.hook_path, *event.hook_args] process.env = event.env process.pid = event.pid or process.pid + process.url = getattr(event, "url", "") or process.url process.process_type = getattr(event, "process_type", "") or process.process_type process.worker_type = getattr(event, "worker_type", "") or process.worker_type process.started_at = parse_event_datetime(event.start_ts) or process.started_at diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index b1d0619b..fdcb15cf 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import json import os +import re import shutil import subprocess import sys @@ -28,8 +29,6 @@ from abx_dl.orchestrator import ( from .archive_result_service import ArchiveResultService from .binary_service import BinaryService from .crawl_service import CrawlService -from .machine_service import MachineService -from .process_request_service import ProcessRequestService from .process_service import ProcessService from .snapshot_service import SnapshotService from .tag_service import TagService @@ -58,28 +57,34 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str ) -def _binary_env_key(name: str) -> str: - normalized = "".join(ch if ch.isalnum() else "_" for ch in name).upper() - return f"{normalized}_BINARY" +_TEMPLATE_NAME_RE = re.compile(r"^\{([A-Z0-9_]+)\}$") -def _binary_config_keys_for_plugins(plugins: dict[str, Plugin], binary_name: str) -> list[str]: +def _binary_config_keys_for_plugins(plugins: dict[str, Plugin], binary_name: str, config: dict[str, Any]) -> list[str]: keys: list[str] = [] - if binary_name != "postlight-parser": - keys.append(_binary_env_key(binary_name)) for plugin in plugins.values(): + for spec in plugin.binaries: + template_name = str(spec.get("name") or "").strip() + match = _TEMPLATE_NAME_RE.fullmatch(template_name) + if match is None: + continue + key = match.group(1) + configured_value = config.get(key) + if configured_value is not None and str(configured_value).strip() == binary_name: + keys.append(key) for key, prop in plugin.config_schema.items(): if key.endswith("_BINARY") and prop.get("default") == binary_name: - keys.insert(0, key) + keys.append(key) return list(dict.fromkeys(keys)) -def _installed_binary_config_overrides(plugins: dict[str, Plugin]) -> dict[str, str]: +def _installed_binary_config_overrides(plugins: dict[str, Plugin], config: dict[str, Any] | None = None) -> dict[str, str]: from archivebox.machine.models import Binary, Machine machine = Machine.current() + active_config = dict(config or {}) overrides: dict[str, str] = {} shared_lib_dir: Path | None = None pip_home: Path | None = None @@ -98,7 +103,7 @@ def _installed_binary_config_overrides(plugins: dict[str, Plugin]) -> dict[str, continue if not resolved_path.is_file() or not os.access(resolved_path, os.X_OK): continue - for key in _binary_config_keys_for_plugins(plugins, binary.name): + for key in _binary_config_keys_for_plugins(plugins, binary.name, active_config): overrides[key] = binary.abspath if resolved_path.parent.name == ".bin" and resolved_path.parent.parent.name == "node_modules": @@ -231,10 +236,8 @@ class CrawlRunner: self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0) self.plugins = discover_plugins() self.process_service = ProcessService(self.bus) - self.machine_service = MachineService(self.bus) self.binary_service = BinaryService(self.bus) self.tag_service = TagService(self.bus) - self.process_request_service = ProcessRequestService(self.bus) self.crawl_service = CrawlService(self.bus, crawl_id=str(crawl.id)) self.process_discovered_snapshots_inline = process_discovered_snapshots_inline self.snapshot_service = SnapshotService( @@ -250,32 +253,10 @@ class CrawlRunner: self.abx_services = None self.persona = None self.base_config: dict[str, Any] = {} + self.derived_config: dict[str, Any] = {} self.primary_url = "" self._live_stream = None - def _create_projector_bus(self, *, identifier: str, config_overrides: dict[str, Any]): - bus = create_bus(name=_bus_name("ArchiveBox", identifier), total_timeout=3600.0) - process_service = ProcessService(bus) - MachineService(bus) - BinaryService(bus) - TagService(bus) - ProcessRequestService(bus) - CrawlService(bus, crawl_id=str(self.crawl.id)) - SnapshotService( - bus, - crawl_id=str(self.crawl.id), - schedule_snapshot=self.enqueue_snapshot if self.process_discovered_snapshots_inline else self.leave_snapshot_queued, - ) - ArchiveResultService(bus, process_service=process_service) - abx_services = setup_abx_services( - bus, - plugins=self.plugins, - config_overrides=config_overrides, - auto_install=True, - emit_jsonl=False, - ) - return bus, abx_services - async def run(self) -> None: from asgiref.sync import sync_to_async from archivebox.crawls.models import Crawl @@ -292,6 +273,8 @@ class CrawlRunner: **self.base_config, "ABX_RUNTIME": "archivebox", }, + derived_config_overrides=self.derived_config, + persist_derived=False, auto_install=True, emit_jsonl=False, ) @@ -369,7 +352,7 @@ class CrawlRunner: current_process.save(update_fields=["iface", "machine", "modified_at"]) self.persona = self.crawl.resolve_persona() self.base_config = get_config(crawl=self.crawl) - self.base_config.update(_installed_binary_config_overrides(self.plugins)) + self.derived_config = _installed_binary_config_overrides(self.plugins, self.base_config) self.base_config["ABX_RUNTIME"] = "archivebox" if self.selected_plugins is None: self.selected_plugins = _selected_plugins_from_config(self.base_config) @@ -473,7 +456,6 @@ class CrawlRunner: plugins=self.plugins, output_dir=Path(snapshot["output_dir"]), selected_plugins=self.selected_plugins, - config_overrides=snapshot["config"], bus=self.bus, emit_jsonl=False, snapshot=setup_snapshot, @@ -501,7 +483,6 @@ class CrawlRunner: plugins=self.plugins, output_dir=Path(snapshot["output_dir"]), selected_plugins=self.selected_plugins, - config_overrides=snapshot["config"], bus=self.bus, emit_jsonl=False, snapshot=cleanup_snapshot, @@ -530,31 +511,22 @@ class CrawlRunner: parent_snapshot_id=snapshot["parent_snapshot_id"], crawl_id=str(self.crawl.id), ) - snapshot_bus, snapshot_services = self._create_projector_bus( - identifier=f"{self.crawl.id}_{snapshot['id']}", - config_overrides=snapshot["config"], - ) try: - _attach_bus_trace(snapshot_bus) await download( url=snapshot["url"], plugins=self.plugins, output_dir=Path(snapshot["output_dir"]), selected_plugins=self.selected_plugins, - config_overrides=snapshot["config"], - bus=snapshot_bus, + bus=self.bus, emit_jsonl=False, snapshot=abx_snapshot, skip_crawl_setup=True, skip_crawl_cleanup=True, ) - await snapshot_services.process.wait_for_background_monitors() finally: current_task = asyncio.current_task() if current_task is not None and self.snapshot_tasks.get(snapshot_id) is current_task: self.snapshot_tasks.pop(snapshot_id, None) - await _stop_bus_trace(snapshot_bus) - await snapshot_bus.stop() def _load_snapshot_run_data(self, snapshot_id: str): from archivebox.core.models import Snapshot @@ -615,19 +587,19 @@ async def _run_binary(binary_id: str) -> None: binary = await sync_to_async(Binary.objects.get, thread_sensitive=True)(id=binary_id) plugins = discover_plugins() config = get_config() - config.update(await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins)) + derived_config = await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins, config) config["ABX_RUNTIME"] = "archivebox" bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0) process_service = ProcessService(bus) - MachineService(bus) BinaryService(bus) TagService(bus) - ProcessRequestService(bus) ArchiveResultService(bus, process_service=process_service) setup_abx_services( bus, plugins=plugins, config_overrides=config, + derived_config_overrides=derived_config, + persist_derived=False, auto_install=True, emit_jsonl=False, ) @@ -662,19 +634,19 @@ async def _run_install(plugin_names: list[str] | None = None) -> None: plugins = discover_plugins() config = get_config() - config.update(await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins)) + derived_config = await sync_to_async(_installed_binary_config_overrides, thread_sensitive=True)(plugins, config) config["ABX_RUNTIME"] = "archivebox" bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0) process_service = ProcessService(bus) - MachineService(bus) BinaryService(bus) TagService(bus) - ProcessRequestService(bus) ArchiveResultService(bus, process_service=process_service) abx_services = setup_abx_services( bus, plugins=plugins, config_overrides=config, + derived_config_overrides=derived_config, + persist_derived=False, auto_install=True, emit_jsonl=False, ) diff --git a/archivebox/tests/test_archive_result_service.py b/archivebox/tests/test_archive_result_service.py index 808a844b..3c3aaad5 100644 --- a/archivebox/tests/test_archive_result_service.py +++ b/archivebox/tests/test_archive_result_service.py @@ -518,7 +518,6 @@ def test_binary_event_reuses_existing_installed_binary_row(monkeypatch): event = BinaryRequestEvent( name="wget", plugin_name="wget", - hook_name="on_Install__10_wget.finite.bg", output_dir="/tmp/wget", binproviders="provider", ) diff --git a/archivebox/tests/test_config_views.py b/archivebox/tests/test_config_views.py index 123677ac..632258b0 100644 --- a/archivebox/tests/test_config_views.py +++ b/archivebox/tests/test_config_views.py @@ -133,7 +133,13 @@ def test_plugin_detail_view_renders_config_in_dedicated_sections(monkeypatch): "description": "Example config used to verify plugin metadata rendering.", "type": "object", "required_plugins": ["chrome"], - "required_binaries": ["example-cli"], + "required_binaries": [ + { + "name": "example-cli", + "binproviders": "env,apt,brew", + "min_version": None, + }, + ], "output_mimetypes": ["text/plain", "application/json"], "properties": { "EXAMPLE_ENABLED": { diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py index 62b114fe..14983a27 100755 --- a/archivebox/tests/test_hooks.py +++ b/archivebox/tests/test_hooks.py @@ -3,7 +3,7 @@ Unit tests for the ArchiveBox hook architecture. Tests hook discovery, execution, JSONL parsing, background hook detection, -binary lookup, and install hook XYZ_BINARY env var handling. +binary lookup, and required_binaries XYZ_BINARY passthrough handling. Run with: sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_hooks.py -v' @@ -126,8 +126,8 @@ not json at all self.assertEqual(records[0]["type"], "ArchiveResult") -class TestInstallHookEnvVarHandling(unittest.TestCase): - """Test that install hooks respect XYZ_BINARY env vars.""" +class TestRequiredBinaryConfigHandling(unittest.TestCase): + """Test that required_binaries keep configured XYZ_BINARY values intact.""" def setUp(self): """Set up test environment.""" @@ -139,39 +139,28 @@ class TestInstallHookEnvVarHandling(unittest.TestCase): shutil.rmtree(self.work_dir, ignore_errors=True) def test_binary_env_var_absolute_path_handling(self): - """Install hooks should handle absolute paths in XYZ_BINARY.""" - # Test the logic that install hooks use + """Absolute binary paths should pass through unchanged.""" configured_binary = "/custom/path/to/wget2" - if "/" in configured_binary: - bin_name = Path(configured_binary).name - else: - bin_name = configured_binary + binary_name = configured_binary - self.assertEqual(bin_name, "wget2") + self.assertEqual(binary_name, "/custom/path/to/wget2") def test_binary_env_var_name_only_handling(self): - """Install hooks should handle binary names in XYZ_BINARY.""" - # Test the logic that install hooks use + """Binary command names should pass through unchanged.""" configured_binary = "wget2" - if "/" in configured_binary: - bin_name = Path(configured_binary).name - else: - bin_name = configured_binary + binary_name = configured_binary - self.assertEqual(bin_name, "wget2") + self.assertEqual(binary_name, "wget2") def test_binary_env_var_empty_default(self): - """Install hooks should use default when XYZ_BINARY is empty.""" + """Empty configured values should fall back to config defaults.""" configured_binary = "" if configured_binary: - if "/" in configured_binary: - bin_name = Path(configured_binary).name - else: - bin_name = configured_binary + binary_name = configured_binary else: - bin_name = "wget" # default + binary_name = "wget" - self.assertEqual(bin_name, "wget") + self.assertEqual(binary_name, "wget") class TestHookDiscovery(unittest.TestCase): @@ -187,7 +176,7 @@ class TestHookDiscovery(unittest.TestCase): wget_dir = self.plugins_dir / "wget" wget_dir.mkdir() (wget_dir / "on_Snapshot__50_wget.py").write_text("# test hook") - (wget_dir / "on_Install__10_wget.finite.bg.py").write_text("# install hook") + (wget_dir / "on_BinaryRequest__10_wget.py").write_text("# binary request hook") chrome_dir = self.plugins_dir / "chrome" chrome_dir.mkdir(exist_ok=True) @@ -299,7 +288,7 @@ class TestHookDiscovery(unittest.TestCase): self.assertIn("on_BinaryRequest__10_npm.py", hook_names) def test_discover_hooks_accepts_event_class_names(self): - """discover_hooks should accept InstallEvent / SnapshotEvent class names.""" + """discover_hooks should accept BinaryRequestEvent / SnapshotEvent class names.""" from archivebox import hooks as hooks_module hooks_module.get_plugins.cache_clear() @@ -307,10 +296,10 @@ class TestHookDiscovery(unittest.TestCase): patch.object(hooks_module, "BUILTIN_PLUGINS_DIR", self.plugins_dir), patch.object(hooks_module, "USER_PLUGINS_DIR", self.test_dir / "user_plugins"), ): - install_hooks = hooks_module.discover_hooks("InstallEvent", filter_disabled=False) + binary_hooks = hooks_module.discover_hooks("BinaryRequestEvent", filter_disabled=False) snapshot_hooks = hooks_module.discover_hooks("SnapshotEvent", filter_disabled=False) - self.assertIn("on_Install__10_wget.finite.bg.py", [hook.name for hook in install_hooks]) + self.assertIn("on_BinaryRequest__10_wget.py", [hook.name for hook in binary_hooks]) self.assertIn("on_Snapshot__50_wget.py", [hook.name for hook in snapshot_hooks]) def test_discover_hooks_returns_empty_for_non_hook_lifecycle_events(self): @@ -325,44 +314,6 @@ class TestHookDiscovery(unittest.TestCase): self.assertEqual(hooks_module.discover_hooks("BinaryEvent", filter_disabled=False), []) self.assertEqual(hooks_module.discover_hooks("CrawlCleanupEvent", filter_disabled=False), []) - def test_discover_install_hooks_only_include_declared_plugin_dependencies(self): - """Install hook discovery should include required_plugins without broadening to provider plugins.""" - responses_dir = self.plugins_dir / "responses" - responses_dir.mkdir() - (responses_dir / "config.json").write_text( - json.dumps( - { - "type": "object", - "required_plugins": ["chrome"], - "properties": {}, - }, - ), - ) - - chrome_dir = self.plugins_dir / "chrome" - chrome_dir.mkdir(exist_ok=True) - (chrome_dir / "config.json").write_text('{"type": "object", "properties": {}}') - (chrome_dir / "on_Install__70_chrome.finite.bg.py").write_text("# chrome install hook") - - npm_dir = self.plugins_dir / "npm" - npm_dir.mkdir() - (npm_dir / "on_BinaryRequest__10_npm.py").write_text("# npm binary hook") - (npm_dir / "on_Install__00_npm.py").write_text("# npm install hook") - (npm_dir / "config.json").write_text('{"type": "object", "properties": {}}') - - from archivebox import hooks as hooks_module - - hooks_module.get_plugins.cache_clear() - with ( - patch.object(hooks_module, "BUILTIN_PLUGINS_DIR", self.plugins_dir), - patch.object(hooks_module, "USER_PLUGINS_DIR", self.test_dir / "user_plugins"), - ): - hooks = hooks_module.discover_hooks("Install", config={"PLUGINS": "responses"}) - - hook_names = [hook.name for hook in hooks] - self.assertIn("on_Install__70_chrome.finite.bg.py", hook_names) - self.assertNotIn("on_Install__00_npm.py", hook_names) - class TestGetExtractorName(unittest.TestCase): """Test get_extractor_name() function.""" @@ -484,8 +435,8 @@ print(json.dumps({"type": "ArchiveResult", "status": "succeeded", "url": args.ge self.assertEqual(records[0]["url"], "https://example.com") -class TestInstallHookOutput(unittest.TestCase): - """Test install hook output format compliance.""" +class TestDependencyRecordOutput(unittest.TestCase): + """Test dependency record output format compliance.""" def setUp(self): """Set up test environment.""" @@ -495,8 +446,8 @@ class TestInstallHookOutput(unittest.TestCase): """Clean up test environment.""" shutil.rmtree(self.work_dir, ignore_errors=True) - def test_install_hook_outputs_binary(self): - """Install hook should output Binary JSONL when binary found.""" + def test_dependency_record_outputs_binary(self): + """Dependency resolution should output Binary JSONL when binary is found.""" hook_output = json.dumps( { "type": "Binary", @@ -515,8 +466,8 @@ class TestInstallHookOutput(unittest.TestCase): self.assertEqual(data["name"], "wget") self.assertTrue(data["abspath"].startswith("/")) - def test_install_hook_outputs_machine_config(self): - """Install hook should output Machine config update JSONL.""" + def test_dependency_record_outputs_machine_config(self): + """Dependency resolution should output Machine config update JSONL.""" hook_output = json.dumps( { "type": "Machine", diff --git a/archivebox/tests/test_process_service.py b/archivebox/tests/test_process_service.py new file mode 100644 index 00000000..89ddd88f --- /dev/null +++ b/archivebox/tests/test_process_service.py @@ -0,0 +1,69 @@ +import asyncio +import json + +import pytest + +from abx_dl.events import ProcessStartedEvent, ProcessStdoutEvent +from abx_dl.orchestrator import create_bus + + +pytestmark = pytest.mark.django_db + + +def test_process_service_emits_process_started_from_inline_process_event(monkeypatch): + from archivebox.services import process_service as process_service_module + from archivebox.services.process_service import ProcessService + + bus = create_bus(name="test_process_service_inline_process_event") + ProcessService(bus) + + monkeypatch.setattr( + process_service_module, + "_ensure_worker", + lambda event: { + "pid": 4321, + "start": 1711111111.0, + "statename": "RUNNING", + "exitstatus": 0, + }, + ) + + async def run_test(): + await bus.emit( + ProcessStdoutEvent( + line=json.dumps( + { + "type": "ProcessEvent", + "plugin_name": "search_backend_sonic", + "hook_name": "worker_sonic", + "hook_path": "/usr/bin/sonic", + "hook_args": ["-c", "/tmp/sonic/config.cfg"], + "is_background": True, + "daemon": True, + "url": "tcp://127.0.0.1:1491", + "output_dir": "/tmp/sonic", + "env": {}, + "process_type": "worker", + "worker_type": "sonic", + "process_id": "worker:sonic", + "output_str": "127.0.0.1:1491", + }, + ), + plugin_name="search_backend_sonic", + hook_name="on_CrawlSetup__55_sonic_start.py", + output_dir="/tmp/search_backend_sonic", + snapshot_id="snap-1", + process_id="proc-hook", + ), + ) + started = await bus.find(ProcessStartedEvent, process_id="worker:sonic") + await bus.stop() + return started + + started = asyncio.run(run_test()) + assert started is not None + assert started.hook_name == "worker_sonic" + assert started.process_type == "worker" + assert started.worker_type == "sonic" + assert getattr(started, "url", "") == "tcp://127.0.0.1:1491" + assert getattr(started, "output_str", "") == "127.0.0.1:1491" diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py index 43b4dcd3..93c9069b 100644 --- a/archivebox/tests/test_runner.py +++ b/archivebox/tests/test_runner.py @@ -46,7 +46,7 @@ async def _call_sync(func, *args, **kwargs): return func(*args, **kwargs) -def test_run_snapshot_uses_isolated_bus_per_snapshot(monkeypatch): +def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot @@ -87,13 +87,13 @@ def test_run_snapshot_uses_isolated_bus_per_snapshot(monkeypatch): download_calls = [] - async def fake_download(*, url, bus, config_overrides, snapshot, **kwargs): + async def fake_download(*, url, bus, snapshot, **kwargs): download_calls.append( { "url": url, "bus": bus, - "snapshot_id": config_overrides["SNAPSHOT_ID"], - "source_url": config_overrides["SOURCE_URL"], + "snapshot_id": snapshot.id, + "source_url": snapshot.url, "abx_snapshot_id": snapshot.id, }, ) @@ -146,8 +146,8 @@ def test_run_snapshot_uses_isolated_bus_per_snapshot(monkeypatch): assert len(download_calls) == 2 assert {call["snapshot_id"] for call in download_calls} == {str(snapshot_a.id), str(snapshot_b.id)} assert {call["source_url"] for call in download_calls} == {snapshot_a.url, snapshot_b.url} - assert len({id(call["bus"]) for call in download_calls}) == 2 - assert len(created_buses) == 3 # 1 crawl bus + 2 isolated snapshot buses + assert len({id(call["bus"]) for call in download_calls}) == 1 + assert len(created_buses) == 1 def test_ensure_background_runner_starts_when_none_running(monkeypatch): @@ -353,6 +353,62 @@ def test_installed_binary_config_overrides_include_valid_installed_binaries(monk assert overrides["NODE_PATH"] == "/tmp/shared-lib/npm/node_modules" +def test_installed_binary_config_overrides_do_not_map_hardcoded_artifacts_to_configurable_binary_keys(monkeypatch): + from archivebox.machine.models import Binary, Machine + from archivebox.services import runner as runner_module + from abx_dl.models import Plugin + + machine = Machine.objects.create( + guid="test-guid-runner-singlefile-cache", + hostname="runner-host-singlefile", + hw_in_docker=False, + hw_in_vm=False, + hw_manufacturer="Test", + hw_product="Test Product", + hw_uuid="test-hw-runner-singlefile-cache", + os_arch="arm64", + os_family="darwin", + os_platform="macOS", + os_release="14.0", + os_kernel="Darwin", + stats={}, + config={}, + ) + singlefile_extension = Binary.objects.create( + machine=machine, + name="singlefile", + abspath="/tmp/shared-lib/bin/singlefile", + version="1.0.0", + binprovider="chromewebstore", + binproviders="chromewebstore", + status=Binary.StatusChoices.INSTALLED, + ) + + monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) + monkeypatch.setattr(Path, "is_file", lambda self: str(self) == singlefile_extension.abspath) + monkeypatch.setattr(runner_module.os, "access", lambda path, mode: str(path) == singlefile_extension.abspath) + + overrides = runner_module._installed_binary_config_overrides( + { + "singlefile": Plugin( + name="singlefile", + path=Path("."), + hooks=[], + config_schema={"SINGLEFILE_BINARY": {"type": "string", "default": "single-file"}}, + binaries=[ + {"name": "{SINGLEFILE_BINARY}", "binproviders": "env,npm"}, + {"name": "singlefile", "binproviders": "chromewebstore"}, + ], + ), + }, + config={"SINGLEFILE_BINARY": "single-file"}, + ) + + assert "SINGLEFILE_BINARY" not in overrides + assert overrides["LIB_DIR"] == "/tmp/shared-lib" + assert overrides["LIB_BIN_DIR"] == "/tmp/shared-lib/bin" + + def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch): import asgiref.sync @@ -700,11 +756,9 @@ def test_crawl_runner_calls_crawl_cleanup_after_snapshot_phase(monkeypatch): "_run_crawl_cleanup", lambda self, snapshot_id: cleanup_calls.append("abx_cleanup") or asyncio.sleep(0), ) - monkeypatch.setattr(crawl, "cleanup", lambda: cleanup_calls.append("crawl_cleanup")) - asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run()) - assert cleanup_calls == ["crawl_cleanup", "abx_cleanup"] + assert cleanup_calls == ["abx_cleanup"] def test_abx_process_service_background_monitor_finishes_after_process_exit(monkeypatch, tmp_path): @@ -765,6 +819,9 @@ def test_abx_process_service_background_monitor_finishes_after_process_exit(monk timeout=60, snapshot_id="snap-1", is_background=True, + url="https://example.org/", + process_type="hook", + worker_type="hook", ) async def run_test(): diff --git a/uv.lock b/uv.lock index ab25cc19..d9dd1bc6 100644 --- a/uv.lock +++ b/uv.lock @@ -23,6 +23,7 @@ dependencies = [ { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "psutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pydantic-settings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -39,6 +40,7 @@ requires-dist = [ { name = "platformdirs", specifier = ">=4.0.0" }, { name = "psutil", specifier = ">=7.2.1" }, { name = "pydantic", specifier = ">=2.0.0" }, + { name = "pydantic-settings", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "requests", specifier = ">=2.28.0" }, @@ -105,16 +107,18 @@ version = "1.10.19" source = { editable = "../abx-plugins" } dependencies = [ { name = "abx-pkg", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "pydantic-settings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "jambo", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] [package.metadata] requires-dist = [ { name = "abx-pkg", editable = "../abx-pkg" }, + { name = "abxbus", editable = "../abxbus" }, { name = "feedparser", marker = "extra == 'dev'", specifier = ">=6.0.0" }, + { name = "jambo", specifier = ">=0.1.7" }, { name = "jinja2", marker = "extra == 'dev'", specifier = ">=3.1.0" }, - { name = "pydantic-settings", specifier = ">=2.0.0" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.408" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, { name = "pytest-httpserver", marker = "extra == 'dev'", specifier = ">=1.1.0" }, @@ -135,7 +139,6 @@ source = { editable = "../abxbus" } dependencies = [ { name = "aiofiles", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, - { name = "portalocker", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -149,7 +152,6 @@ requires-dist = [ { name = "asyncpg", marker = "extra == 'postgres'", specifier = ">=0.31.0" }, { name = "nats-py", marker = "extra == 'bridges'", specifier = ">=2.13.1" }, { name = "nats-py", marker = "extra == 'nats'", specifier = ">=2.13.1" }, - { name = "portalocker", specifier = ">=2.7.0" }, { name = "pydantic", specifier = ">=2.11.5" }, { name = "redis", marker = "extra == 'bridges'", specifier = ">=7.1.1" }, { name = "redis", marker = "extra == 'redis'", specifier = ">=7.1.1" }, @@ -1000,6 +1002,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/85/c4e42d21cf748c696b8c05316bbd8e8666f17eeda0cf1743056f4cf7622b/djdt_flamegraph-0.2.13-py2.py3-none-any.whl", hash = "sha256:b3252b8cc9b586829166cc158b26952626cd6f41a3ffa92dceef2f5dbe5b99a0", size = 15256, upload-time = "2020-01-17T05:40:37.799Z" }, ] +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + [[package]] name = "docutils" version = "0.22.4" @@ -1009,6 +1020,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] +[[package]] +name = "email-validator" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "idna", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, +] + [[package]] name = "et-xmlfile" version = "2.0.0" @@ -1196,6 +1220,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, ] +[[package]] +name = "jambo" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "email-validator", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "jsonschema", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/f5/74de157c7aece6a070f99f18201a0e2f46cdfd0f9e337efd411745ed9b22/jambo-0.1.7.tar.gz", hash = "sha256:df89ab8209ebdf7a6e92252ec925979cd3d32811bf4a8182a97dc35b7df58f74", size = 137822, upload-time = "2026-01-14T19:17:30.302Z" } + [[package]] name = "jedi" version = "0.19.2" @@ -1220,6 +1255,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "jsonschema-specifications", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "referencing", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "rpds-py", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "lexid" version = "2021.1006" @@ -1716,15 +1778,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "portalocker" -version = "3.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644, upload-time = "2025-06-14T13:20:40.03Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424, upload-time = "2025-06-14T13:20:38.083Z" }, -] - [[package]] name = "prek" version = "0.3.8" @@ -2171,6 +2224,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/77/ed589c75db5d02a77a1d5d2d9abc63f29676467d396c64277f98b50b79c2/recommonmark-0.7.1-py2.py3-none-any.whl", hash = "sha256:1b1db69af0231efce3fa21b94ff627ea33dee7079a01dd0a7f8482c3da148b3f", size = 10214, upload-time = "2020-12-17T19:24:55.137Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "rpds-py", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "regex" version = "2026.2.28" @@ -2294,6 +2360,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, ] +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, +] + [[package]] name = "ruff" version = "0.15.7"