diff --git a/archivebox/config/common.py b/archivebox/config/common.py index e345494a..0ab5553c 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -478,7 +478,7 @@ class SearchBackendConfig(BaseConfigSet): toml_section_header: str = "SEARCH_BACKEND_CONFIG" _scope: str = PrivateAttr(default=_SCOPE_SERVER) - SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep") + SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION}) def _plugin_user_config_value(value: Any) -> str: @@ -598,18 +598,13 @@ class ArchiveBoxBaseConfig( if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"): scope = str(prop_schema["x-scope"]) elif scope is None: - if str(plugin_name).startswith("search_backend_"): - scope = _SCOPE_SERVER - else: - scope = _SCOPE_CRAWL_FROZEN + scope = _SCOPE_CRAWL_FROZEN return scope @classmethod @lru_cache(maxsize=None) def scope_for_key(cls, key: str) -> str: for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items(): - if str(plugin_name).startswith("search_backend_"): - continue properties = schema.get("properties") if isinstance(schema, dict) else None if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties: return _SCOPE_CRAWL_EXECUTION @@ -684,6 +679,10 @@ class ArchiveBoxBaseConfig( model_fields = type(self).model_fields for key in type(self).runtime_derived_config_keys(): config.pop(key, None) + # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model + # validation to derive the selected backend's *_ENABLED flag. Hooks + # only receive the backend-local flags, never the selector itself. + config.pop("SEARCH_BACKEND_ENGINE", None) if persona is not None: for key, value in persona.get_derived_config().items(): if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION: @@ -714,7 +713,6 @@ class ArchiveBoxBaseConfig( context.update(dict(extra_context)) config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True) - _derive_plugin_enabled_config(config) return config @model_validator(mode="after") @@ -733,6 +731,18 @@ class ArchiveBoxBaseConfig( return self + @model_validator(mode="after") + def derive_plugin_enabled_config(self): + plugin_names = _normalize_plugins_config_value(self.PLUGINS) + selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set() + search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower() + if search_backend: + selected_plugins.add(f"search_backend_{search_backend}") + for plugin_name, enabled_key in _plugin_enabled_config_keys().items(): + if plugin_names or plugin_name in selected_plugins: + setattr(self, enabled_key, plugin_name in selected_plugins) + return self + def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]: core_fields = set(ArchiveBoxBaseConfig.model_fields) @@ -805,15 +815,6 @@ def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]: return selected -def _derive_plugin_enabled_config(config: dict[str, Any]) -> None: - plugin_names = _normalize_plugins_config_value(config.get("PLUGINS")) - if not plugin_names: - return - selected_plugins = _plugins_with_required_plugins(plugin_names) - for plugin_name, enabled_key in _plugin_enabled_config_keys().items(): - config[enabled_key] = plugin_name in selected_plugins - - def get_live_config_url(key: str) -> str: return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/" @@ -1089,8 +1090,6 @@ def get_config( config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False)) config_data.update(archivebox_scope_overrides) - _derive_plugin_enabled_config(config_data) - # Decode JSON-encoded complex values (dict/list fields) that came from # string-only sources before validation. ``IniConfigSettingsSource`` does # this for the ArchiveBox.conf path, but Machine.config (mirrored from the diff --git a/archivebox/core/models.py b/archivebox/core/models.py index d66566ff..86f72761 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -14,7 +14,7 @@ from urllib.parse import urlparse from statemachine import State, registry from django.db import models, transaction -from django.db.models import Case, Q, QuerySet, Sum, Value, When +from django.db.models import Case, F, Q, QuerySet, Sum, Value, When from django.db.models.functions import Coalesce, Concat from django.db.models.fields.json import KT from django.utils.functional import cached_property @@ -3803,21 +3803,49 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes): or "snapshot_id" in update_fields ) old_snapshot_id = None + old_output_size = 0 if refresh_snapshot_size and not is_new: - old_snapshot_id = type(self).objects.filter(pk=self.pk).values_list("snapshot_id", flat=True).first() - - update_fields = kwargs.get("update_fields") - if self.delete_at is None: - self.set_delete_at_from_config() - if self.delete_at is not None and update_fields is not None: - kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"])) + old_values = type(self).objects.filter(pk=self.pk).values("snapshot_id", "output_size").first() + if old_values: + old_snapshot_id = old_values["snapshot_id"] + old_output_size = int(old_values["output_size"] or 0) + # ArchiveResult rows are updated on every plugin event. Resolving + # DELETE_AFTER here is deceptively expensive because the effective + # value lives on Snapshot/Crawl config, so a save of an already-loaded + # result can still materialize parent objects and parse config. The + # orchestrator owns the repair pass for these rows instead: it fills + # missing delete_at values from fresh Snapshot/Crawl config when the + # queue is idle, outside the hook-result write hot path. # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories # Call the Django Model.save() directly instead models.Model.save(self, *args, **kwargs) if refresh_snapshot_size: - snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id} - transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids)) + current_snapshot_id = self.snapshot_id + snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, current_snapshot_id) if snapshot_id} + current_output_size = int(self.output_size or 0) + if len(snapshot_ids) > 1: + # Moving an ArchiveResult between Snapshots is rare and cannot + # be represented as a single delta on one parent row. Keep the + # conservative aggregate fallback for that shape. + transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids)) + elif current_snapshot_id: + # Hook-result projection updates ArchiveResult rows at very + # high frequency during indexing. Re-aggregating every sibling + # row for the parent Snapshot on each save turns those short + # writes into a table-scan hot path. For the common case where + # the result stays attached to the same Snapshot, the persisted + # parent total is exactly the old total plus this row's size + # delta; F() keeps that update atomic with concurrent result + # saves for other plugins on the same Snapshot. + size_delta = current_output_size if is_new else current_output_size - old_output_size + if size_delta: + transaction.on_commit( + lambda: Snapshot.objects.filter(pk=current_snapshot_id).update( + output_size=F("output_size") + size_delta, + modified_at=timezone.now(), + ), + ) if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields: transaction.on_commit(type(self).clear_majority_status_cache) diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 03babadd..0530ed4e 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -412,14 +412,25 @@ class NetworkInterface(ModelWithHealthStats): def current(cls, refresh: bool = False) -> NetworkInterface: global _CURRENT_INTERFACE machine = Machine.current(refresh=refresh) - if _CURRENT_INTERFACE: - if ( - not refresh - and _CURRENT_INTERFACE.machine_id == machine.id - and timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL) - ): + if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id: + if not refresh: + # Callers that pass refresh=False are asking for attribution to + # the currently known interface, not for public-IP/ISP probing. + # Maintenance paths create many short-lived services per run; + # expiring this in-memory object by age forced every crawl to + # hit external network APIs even though Process rows only need + # a stable existing FK. Active downloading paths opt into live + # detection with refresh=True. return _CURRENT_INTERFACE - _CURRENT_INTERFACE = None + if timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL): + return _CURRENT_INTERFACE + _CURRENT_INTERFACE = None + + if not refresh: + _CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first() + if _CURRENT_INTERFACE is not None: + return _CURRENT_INTERFACE + net_info = get_host_network() lookup = dict( machine=machine, diff --git a/archivebox/plugins/discovery.py b/archivebox/plugins/discovery.py index 46d95484..c05262c2 100644 --- a/archivebox/plugins/discovery.py +++ b/archivebox/plugins/discovery.py @@ -195,12 +195,15 @@ def get_search_backends() -> dict[str, Any]: ) +@lru_cache(maxsize=1) def discover_plugin_configs() -> dict[str, dict[str, Any]]: """ Discover all plugin config.json schemas. Each plugin can define a config.json file with JSONSchema defining - its configuration options. + its configuration options. This is intentionally cached because these + schemas are plugin package metadata, not live user config; runtime values + still come from env/db config at each callsite. """ configs = {} diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 8dcaf39e..c27d2296 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -10,7 +10,7 @@ import threading import time from contextlib import nullcontext from datetime import timedelta -from functools import lru_cache +from functools import lru_cache, wraps from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -74,6 +74,39 @@ from .tag_service import TagService QUEUED_PLUGIN_RESULT_BATCH_SIZE = 100 +def _perf_trace(label): + def decorator(func): + if asyncio.iscoroutinefunction(func): + + @wraps(func) + async def async_wrapper(*args, **kwargs): + if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1": + return await func(*args, **kwargs) + started_at = time.perf_counter() + try: + return await func(*args, **kwargs) + finally: + elapsed_ms = (time.perf_counter() - started_at) * 1000 + print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True) + + return async_wrapper + + @wraps(func) + def sync_wrapper(*args, **kwargs): + if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1": + return func(*args, **kwargs) + started_at = time.perf_counter() + try: + return func(*args, **kwargs) + finally: + elapsed_ms = (time.perf_counter() - started_at) * 1000 + print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True) + + return sync_wrapper + + return decorator + + def _bus_name(prefix: str, identifier: str) -> str: normalized = "".join(ch if ch.isalnum() else "_" for ch in identifier) return f"{prefix}_{normalized}" @@ -1728,24 +1761,42 @@ def _run_due_queued_plugin_result( if not plugin_names: return False - queued_snapshot_ids = ArchiveResult.objects.filter( + now = timezone.now() + queued_results = ArchiveResult.objects.filter( + snapshot_id=OuterRef("pk"), status=ArchiveResult.StatusChoices.QUEUED, plugin__in=plugin_names, - ).values("snapshot_id") - due_snapshots = Snapshot.objects.filter( - id__in=queued_snapshot_ids, - retry_at__lte=timezone.now(), - status=Snapshot.StatusChoices.SEALED, ) + first_due_results = list( + ArchiveResult.objects.filter( + status=ArchiveResult.StatusChoices.QUEUED, + plugin__in=plugin_names, + snapshot__retry_at__lte=now, + snapshot__status=Snapshot.StatusChoices.SEALED, + ) + .filter(**({"snapshot__crawl_id": crawl_id} if crawl_id else {})) + .values("snapshot_id", "snapshot__crawl_id")[:1], + ) + if not first_due_results: + return False + root_crawl_id = str(first_due_results[0]["snapshot__crawl_id"]) + + due_snapshots = Snapshot.objects.filter( + retry_at__lte=now, + status=Snapshot.StatusChoices.SEALED, + ).filter(Exists(queued_results)) if crawl_id: due_snapshots = due_snapshots.filter(crawl_id=crawl_id) - due_snapshots = due_snapshots.only("id", "crawl_id", "retry_at", "status").order_by("retry_at", "created_at") - first_due_snapshot = due_snapshots.first() - if first_due_snapshot is None: - return False - root_crawl_id = str(first_due_snapshot.crawl_id) + batch_candidates = list( - due_snapshots.filter(crawl_id=root_crawl_id)[:QUEUED_PLUGIN_RESULT_BATCH_SIZE], + # The crawl picker above starts from enabled queued ArchiveResult rows + # and uses a sliced LIMIT 1. Do not use QuerySet.first() here: it adds + # ordering and can turn this hot scheduler check into a temp-sort over + # hundreds of thousands of plugin rows. Once a crawl is selected, + # sibling order is irrelevant; the crawl_id/status index can fetch this + # small local batch directly while EXISTS proves the enabled queued + # plugin rows via the existing ArchiveResult unique index. + due_snapshots.filter(crawl_id=root_crawl_id).order_by()[:QUEUED_PLUGIN_RESULT_BATCH_SIZE], ) if not batch_candidates: return False @@ -1753,7 +1804,9 @@ def _run_due_queued_plugin_result( selected_plugins: list[str] | None = None claimed_snapshot_ids: list[str] = [] for snapshot in batch_candidates: - snapshot_selected_plugins = queued_plugins_for_snapshot(str(snapshot.id)) + snapshot_selected_plugins = [ + plugin_name for plugin_name in (queued_plugins_for_snapshot(str(snapshot.id)) or []) if plugin_name in plugin_names + ] if not snapshot_selected_plugins: continue if selected_plugins is None: @@ -1789,6 +1842,7 @@ def _run_due_queued_plugin_result( queued_results = ArchiveResult.objects.filter( snapshot_id=OuterRef("pk"), status=ArchiveResult.StatusChoices.QUEUED, + plugin__in=selected_plugins, ) Snapshot.objects.filter( id__in=claimed_snapshot_ids, @@ -1881,6 +1935,7 @@ def run_pending_crawls( from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.core.models import ArchiveResult, Snapshot from archivebox.plugins.discovery import discover_plugin_configs + from archivebox.plugins.hooks import discover_hooks from archivebox.machine.models import Process crawl_claim_lock_seconds = 10 @@ -1891,9 +1946,9 @@ def run_pending_crawls( for plugin_name, plugin_config in plugin_configs.items() if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_") ) - search_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_")) last_recovery_at = 0.0 last_retention_at = 0.0 + last_retention_repair_at = 0.0 last_analyze_at = 0.0 analyze_queue: list[str] | None = None analyze_sweep_started_at = 0.0 @@ -1903,10 +1958,12 @@ def run_pending_crawls( now_monotonic = time.monotonic() if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0): for model in (ArchiveResult, Snapshot, Crawl, Process): - # The runner hot path must only touch indexed scheduler/retention - # columns before claiming work. delete_at is hydrated when rows - # are saved, while missing_delete_at_candidates() may inspect JSON - # config across large tables and can stall worker startup. + # Keep the tight scheduler loop anchored on indexed delete_at + # columns only. Backfilling missing delete_at values has to read + # config JSON for models whose retention policy is scoped to a + # Crawl/Snapshot/Process. That repair is still required for + # correctness, but it belongs in the idle maintenance block + # below, not ahead of every claim attempt. model.delete_expired(batch_size=100, backfill_missing=False) last_retention_at = now_monotonic @@ -2019,6 +2076,17 @@ def run_pending_crawls( ): continue + # Search backend selection is live crawl-execution config, not an + # installed-plugin list. Old queued rows for a backend that is disabled + # by the current Machine/Crawl/Snapshot config must remain queued so + # they can run if the user re-enables that backend, but they should not + # launch a standalone hook process just to skip after imports/config + # hydration. Refreshing here preserves mid-run config edits while using + # the same enabled-hook discovery path that created ArchiveResult rows. + runtime_config = get_config() + search_plugin_names = frozenset( + hook.parent.name for hook in discover_hooks("Snapshot", config=runtime_config) if hook.parent.name.startswith("search_backend_") + ) if _run_due_queued_plugin_result( search_plugin_names, crawl_id=crawl_id, @@ -2067,6 +2135,18 @@ def run_pending_crawls( if _run_due_binary(): continue + now_monotonic = time.monotonic() + if now_monotonic - last_retention_repair_at >= (60.0 if daemon else 0.0): + for model in (ArchiveResult, Snapshot, Crawl, Process): + # No runnable work was found on this scheduler pass. This is + # the bounded repair point for missing retention deadlines, + # including ArchiveResult rows intentionally saved without + # delete_at in the plugin-result hot path. Running it here keeps + # DELETE_AFTER resolution fresh without making every hook event + # load parent Snapshot/Crawl config. + model.delete_expired(batch_size=100, backfill_missing=True) + last_retention_repair_at = now_monotonic + if daemon: now_monotonic = time.monotonic() if now_monotonic - last_recovery_at >= 30.0: diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index 4f8ab00f..3fcf9c61 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -231,7 +231,7 @@ def initialized_archive(isolated_data_dir): @pytest.fixture -def archivebox_daemon_server(tmp_path, process, unused_tcp_port_factory): +def archivebox_daemon_server(tmp_path, process, free_tcp_port_factory): """ Start a real daemonized ArchiveBox server in this test's DATA_DIR and always stop its supervisord before the test exits. @@ -246,11 +246,11 @@ def archivebox_daemon_server(tmp_path, process, unused_tcp_port_factory): "USE_COLOR": "False", "SHOW_PROGRESS": "False", "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1", - "SEARCH_BACKEND_SONIC_PORT": str(unused_tcp_port_factory()), + "SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()), **{key: str(value) for key, value in env_overrides.items()}, }, ) - port = unused_tcp_port_factory() + port = free_tcp_port_factory() result = subprocess.run( [sys.executable, "-m", "archivebox", "server", "--daemonize", f"127.0.0.1:{port}"], cwd=tmp_path, diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 5067236a..c15b9047 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -1388,6 +1388,41 @@ class TestRecoverOrchestratorState: @pytest.mark.django_db class TestRunDueCrawlState: + def test_idle_maintenance_repairs_archive_result_delete_at(self): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import run_pending_crawls + + crawl = Crawl.objects.create( + urls="https://example.com/retention-repair", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + config={"DELETE_AFTER": "2h"}, + ) + snapshot = Snapshot.objects.create( + url="https://example.com/retention-repair", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=None, + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="search_backend_sqlite", + hook_name="on_Snapshot__90_index_sqlite.py", + status=ArchiveResult.StatusChoices.SUCCEEDED, + ) + + # ArchiveResult saves are the plugin-event hot path. They intentionally + # do not resolve parent Snapshot/Crawl config on every write; the real + # runner's idle maintenance pass owns missing delete_at repair. + assert result.delete_at is None + assert run_pending_crawls(daemon=False, maintenance_only=True) == 0 + + result.refresh_from_db() + assert result.delete_at is not None + def test_maintenance_only_runner_does_not_start_regular_queued_crawls(self): from django.utils import timezone diff --git a/archivebox/tests/test_cli_server.py b/archivebox/tests/test_cli_server.py index 8d2dd002..73964a9e 100644 --- a/archivebox/tests/test_cli_server.py +++ b/archivebox/tests/test_cli_server.py @@ -199,13 +199,83 @@ def test_server_daemon_restarts_runner_killed_by_signal(archivebox_daemon_server assert state["worker_daphne"]["statename"] == "RUNNING", state -def test_sonic_worker_is_disabled_when_sonic_disabled_and_engine_not_sonic(tmp_path): +def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_runtime(archivebox_daemon_server): + server = archivebox_daemon_server(SEARCH_BACKEND_ENGINE="ripgrep") + server.wait_for_workers(("worker_daphne", "worker_runner")) + + setup_result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import django;" + "django.setup();" + "from archivebox.base_models.models import get_or_create_system_user_pk;" + "from archivebox.crawls.models import Crawl;" + "from archivebox.core.models import Snapshot;" + "from archivebox.machine.models import Machine;" + "machine = Machine.current(refresh=True);" + "machine.config = {**dict(machine.config or {}), 'SEARCH_BACKEND_ENGINE': 'sqlite'};" + "machine.save(update_fields=['config', 'modified_at']);" + "crawl = Crawl.objects.create(" + "urls='https://example.com/live-machine-search-config'," + "created_by_id=get_or_create_system_user_pk()," + "config={}," + ");" + "snapshot = Snapshot.objects.create(" + "url='https://example.com/live-machine-search-config'," + "crawl=crawl," + ");" + "print(snapshot.id)" + ), + ], + cwd=server.data_dir, + env=server.env, + capture_output=True, + text=True, + timeout=30, + ) + assert setup_result.returncode == 0, setup_result.stderr or setup_result.stdout + snapshot_id = setup_result.stdout.strip().splitlines()[-1] + + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import django,json;" + "django.setup();" + "from archivebox.core.models import Snapshot;" + "from archivebox.config.common import get_config;" + f"snapshot = Snapshot.objects.select_related('crawl').get(id='{snapshot_id}');" + "runtime = get_config(snapshot=snapshot).for_crawl_runtime(" + "crawl=snapshot.crawl," + "snapshot=snapshot," + "extra_context={'snapshot_id': str(snapshot.id)}," + ");" + "print(json.dumps({" + "'sqlite_enabled': runtime.get('SEARCH_BACKEND_SQLITE_ENABLED')," + "'engine_in_runtime': 'SEARCH_BACKEND_ENGINE' in runtime," + "}))" + ), + ], + cwd=server.data_dir, + env=server.env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr or result.stdout + resolved = json.loads(result.stdout.strip().splitlines()[-1]) + assert resolved == {"sqlite_enabled": True, "engine_in_runtime": False} + + +def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path): from archivebox.workers.supervisord_util import get_sonic_supervisord_worker_from_plugin worker = get_sonic_supervisord_worker_from_plugin( SimpleNamespace( DATA_DIR=str(tmp_path), - SEARCH_BACKEND_ENGINE="ripgrep", SEARCH_BACKEND_SONIC_ENABLED=False, SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1", SEARCH_BACKEND_SONIC_PORT=_free_port(), @@ -234,7 +304,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo daemon_event = prepare_sonic_daemon( SimpleNamespace( DATA_DIR=str(server.data_dir), - SEARCH_BACKEND_ENGINE="sonic", + SEARCH_BACKEND_SONIC_ENABLED=True, SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1", SEARCH_BACKEND_SONIC_PORT=sonic_port, SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword", diff --git a/archivebox/tests/test_frozen_crawl_config.py b/archivebox/tests/test_frozen_crawl_config.py index 33792991..2d48b1d9 100644 --- a/archivebox/tests/test_frozen_crawl_config.py +++ b/archivebox/tests/test_frozen_crawl_config.py @@ -131,11 +131,44 @@ def test_config_scopes_are_derived_from_section_and_field_metadata(): assert ArchiveBoxConfig.scope_for_key("DEBUG") == "crawl_execution" assert ArchiveBoxConfig.scope_for_key("DEFAULT_PERSONA") == "crawl_execution" assert ArchiveBoxConfig.scope_for_key("WGET_ENABLED") == "crawl_execution" + assert ArchiveBoxConfig.scope_for_key("SEARCH_BACKEND_SQLITE_ENABLED") == "crawl_execution" + assert ArchiveBoxConfig.scope_for_key("SEARCH_BACKEND_ENGINE") == "crawl_execution" assert ArchiveBoxConfig.scope_for_key("WGET_WARC_ENABLED") == "crawl_frozen" assert ArchiveBoxConfig.scope_for_key("SECRET_KEY") == "server" assert ArchiveBoxConfig.scope_for_key("DATABASE_NAME") == "server" +def test_search_backend_engine_derives_default_backend_enabled_without_entering_hook_env(): + from archivebox.config.common import ArchiveBoxConfig + + default_runtime_config = ArchiveBoxConfig().for_crawl_runtime(extra_context={"snapshot_id": "default-runtime-config"}) + assert default_runtime_config["SEARCH_BACKEND_RIPGREP_ENABLED"] is True + assert default_runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is False + assert default_runtime_config["SEARCH_BACKEND_SONIC_ENABLED"] is True + + sqlite_runtime_config = ArchiveBoxConfig(SEARCH_BACKEND_ENGINE="sqlite").for_crawl_runtime( + extra_context={"snapshot_id": "sqlite-runtime-config"}, + ) + assert sqlite_runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is True + + config = ArchiveBoxConfig( + SEARCH_BACKEND_ENGINE="ripgrep", + SEARCH_BACKEND_SQLITE_ENABLED=False, + SEARCH_BACKEND_SONIC_ENABLED=True, + SECRET_KEY="server-secret", + DATABASE_NAME="server-db.sqlite3", + ) + + runtime_config = config.for_crawl_runtime(extra_context={"snapshot_id": "runtime-config"}) + + assert "SEARCH_BACKEND_ENGINE" not in runtime_config + assert runtime_config["SEARCH_BACKEND_RIPGREP_ENABLED"] is True + assert runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is False + assert runtime_config["SEARCH_BACKEND_SONIC_ENABLED"] is True + assert "SECRET_KEY" not in runtime_config + assert "DATABASE_NAME" not in runtime_config + + def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_env_overridden(archivebox_db, monkeypatch): from archivebox.config.common import get_config from archivebox.crawls.models import Crawl diff --git a/etc/package.json b/etc/package.json index 1d566d37..2b1f4c3f 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.34rc28", + "version": "0.9.34rc29", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index bc7f6ef6..6b9e29fe 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.34rc28" +version = "0.9.34rc29" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.9", # EventBus API - "abxpkg>=1.11.142", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.145", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.145", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.143", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.146", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.146", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]