release: archivebox 0.9.34rc29

archivebox/config/common.py

@@ -478,7 +478,7 @@ class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
+    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
 
 def _plugin_user_config_value(value: Any) -> str:
@@ -598,18 +598,13 @@ class ArchiveBoxBaseConfig(
             if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
                 scope = str(prop_schema["x-scope"])
             elif scope is None:
-                if str(plugin_name).startswith("search_backend_"):
-                    scope = _SCOPE_SERVER
-                else:
-                    scope = _SCOPE_CRAWL_FROZEN
+                scope = _SCOPE_CRAWL_FROZEN
         return scope
 
     @classmethod
     @lru_cache(maxsize=None)
     def scope_for_key(cls, key: str) -> str:
         for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
-            if str(plugin_name).startswith("search_backend_"):
-                continue
             properties = schema.get("properties") if isinstance(schema, dict) else None
             if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties:
                 return _SCOPE_CRAWL_EXECUTION
@@ -684,6 +679,10 @@ class ArchiveBoxBaseConfig(
         model_fields = type(self).model_fields
         for key in type(self).runtime_derived_config_keys():
             config.pop(key, None)
+        # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model
+        # validation to derive the selected backend's *_ENABLED flag. Hooks
+        # only receive the backend-local flags, never the selector itself.
+        config.pop("SEARCH_BACKEND_ENGINE", None)
         if persona is not None:
             for key, value in persona.get_derived_config().items():
                 if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
@@ -714,7 +713,6 @@ class ArchiveBoxBaseConfig(
             context.update(dict(extra_context))
             config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True)
 
-        _derive_plugin_enabled_config(config)
         return config
 
     @model_validator(mode="after")
@@ -733,6 +731,18 @@ class ArchiveBoxBaseConfig(
 
         return self
 
+    @model_validator(mode="after")
+    def derive_plugin_enabled_config(self):
+        plugin_names = _normalize_plugins_config_value(self.PLUGINS)
+        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
+        if search_backend:
+            selected_plugins.add(f"search_backend_{search_backend}")
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+            if plugin_names or plugin_name in selected_plugins:
+                setattr(self, enabled_key, plugin_name in selected_plugins)
+        return self
+
 
 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
     core_fields = set(ArchiveBoxBaseConfig.model_fields)
@@ -805,15 +815,6 @@ def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
     return selected
 
 
-def _derive_plugin_enabled_config(config: dict[str, Any]) -> None:
-    plugin_names = _normalize_plugins_config_value(config.get("PLUGINS"))
-    if not plugin_names:
-        return
-    selected_plugins = _plugins_with_required_plugins(plugin_names)
-    for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
-        config[enabled_key] = plugin_name in selected_plugins
-
-
 def get_live_config_url(key: str) -> str:
     return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
 
@@ -1089,8 +1090,6 @@ def get_config(
             config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
 
-    _derive_plugin_enabled_config(config_data)
-
     # Decode JSON-encoded complex values (dict/list fields) that came from
     # string-only sources before validation. ``IniConfigSettingsSource`` does
     # this for the ArchiveBox.conf path, but Machine.config (mirrored from the

archivebox/core/models.py

@@ -14,7 +14,7 @@ from urllib.parse import urlparse
 from statemachine import State, registry
 
 from django.db import models, transaction
-from django.db.models import Case, Q, QuerySet, Sum, Value, When
+from django.db.models import Case, F, Q, QuerySet, Sum, Value, When
 from django.db.models.functions import Coalesce, Concat
 from django.db.models.fields.json import KT
 from django.utils.functional import cached_property
@@ -3803,21 +3803,49 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             or "snapshot_id" in update_fields
         )
         old_snapshot_id = None
+        old_output_size = 0
         if refresh_snapshot_size and not is_new:
-            old_snapshot_id = type(self).objects.filter(pk=self.pk).values_list("snapshot_id", flat=True).first()
-
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+            old_values = type(self).objects.filter(pk=self.pk).values("snapshot_id", "output_size").first()
+            if old_values:
+                old_snapshot_id = old_values["snapshot_id"]
+                old_output_size = int(old_values["output_size"] or 0)
 
+        # ArchiveResult rows are updated on every plugin event. Resolving
+        # DELETE_AFTER here is deceptively expensive because the effective
+        # value lives on Snapshot/Crawl config, so a save of an already-loaded
+        # result can still materialize parent objects and parse config. The
+        # orchestrator owns the repair pass for these rows instead: it fills
+        # missing delete_at values from fresh Snapshot/Crawl config when the
+        # queue is idle, outside the hook-result write hot path.
         # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories
         # Call the Django Model.save() directly instead
         models.Model.save(self, *args, **kwargs)
         if refresh_snapshot_size:
-            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id}
-            transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            current_snapshot_id = self.snapshot_id
+            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, current_snapshot_id) if snapshot_id}
+            current_output_size = int(self.output_size or 0)
+            if len(snapshot_ids) > 1:
+                # Moving an ArchiveResult between Snapshots is rare and cannot
+                # be represented as a single delta on one parent row. Keep the
+                # conservative aggregate fallback for that shape.
+                transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            elif current_snapshot_id:
+                # Hook-result projection updates ArchiveResult rows at very
+                # high frequency during indexing. Re-aggregating every sibling
+                # row for the parent Snapshot on each save turns those short
+                # writes into a table-scan hot path. For the common case where
+                # the result stays attached to the same Snapshot, the persisted
+                # parent total is exactly the old total plus this row's size
+                # delta; F() keeps that update atomic with concurrent result
+                # saves for other plugins on the same Snapshot.
+                size_delta = current_output_size if is_new else current_output_size - old_output_size
+                if size_delta:
+                    transaction.on_commit(
+                        lambda: Snapshot.objects.filter(pk=current_snapshot_id).update(
+                            output_size=F("output_size") + size_delta,
+                            modified_at=timezone.now(),
+                        ),
+                    )
         if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields:
             transaction.on_commit(type(self).clear_majority_status_cache)
 

archivebox/machine/models.py

@@ -412,14 +412,25 @@ class NetworkInterface(ModelWithHealthStats):
     def current(cls, refresh: bool = False) -> NetworkInterface:
         global _CURRENT_INTERFACE
         machine = Machine.current(refresh=refresh)
-        if _CURRENT_INTERFACE:
-            if (
-                not refresh
-                and _CURRENT_INTERFACE.machine_id == machine.id
-                and timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL)
-            ):
+        if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
+            if not refresh:
+                # Callers that pass refresh=False are asking for attribution to
+                # the currently known interface, not for public-IP/ISP probing.
+                # Maintenance paths create many short-lived services per run;
+                # expiring this in-memory object by age forced every crawl to
+                # hit external network APIs even though Process rows only need
+                # a stable existing FK. Active downloading paths opt into live
+                # detection with refresh=True.
                 return _CURRENT_INTERFACE
-            _CURRENT_INTERFACE = None
+            if timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
+                return _CURRENT_INTERFACE
+        _CURRENT_INTERFACE = None
+
+        if not refresh:
+            _CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first()
+            if _CURRENT_INTERFACE is not None:
+                return _CURRENT_INTERFACE
+
         net_info = get_host_network()
         lookup = dict(
             machine=machine,

archivebox/plugins/discovery.py

@@ -195,12 +195,15 @@ def get_search_backends() -> dict[str, Any]:
     )
 
 
+@lru_cache(maxsize=1)
 def discover_plugin_configs() -> dict[str, dict[str, Any]]:
     """
     Discover all plugin config.json schemas.
 
     Each plugin can define a config.json file with JSONSchema defining
-    its configuration options.
+    its configuration options. This is intentionally cached because these
+    schemas are plugin package metadata, not live user config; runtime values
+    still come from env/db config at each callsite.
     """
     configs = {}
 

archivebox/services/runner.py

@@ -10,7 +10,7 @@ import threading
 import time
 from contextlib import nullcontext
 from datetime import timedelta
-from functools import lru_cache
+from functools import lru_cache, wraps
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
@@ -74,6 +74,39 @@ from .tag_service import TagService
 QUEUED_PLUGIN_RESULT_BATCH_SIZE = 100
 
 
+def _perf_trace(label):
+    def decorator(func):
+        if asyncio.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                    return await func(*args, **kwargs)
+                started_at = time.perf_counter()
+                try:
+                    return await func(*args, **kwargs)
+                finally:
+                    elapsed_ms = (time.perf_counter() - started_at) * 1000
+                    print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+            return async_wrapper
+
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                return func(*args, **kwargs)
+            started_at = time.perf_counter()
+            try:
+                return func(*args, **kwargs)
+            finally:
+                elapsed_ms = (time.perf_counter() - started_at) * 1000
+                print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+        return sync_wrapper
+
+    return decorator
+
+
 def _bus_name(prefix: str, identifier: str) -> str:
     normalized = "".join(ch if ch.isalnum() else "_" for ch in identifier)
     return f"{prefix}_{normalized}"
@@ -1728,24 +1761,42 @@ def _run_due_queued_plugin_result(
 
     if not plugin_names:
         return False
-    queued_snapshot_ids = ArchiveResult.objects.filter(
+    now = timezone.now()
+    queued_results = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
         status=ArchiveResult.StatusChoices.QUEUED,
         plugin__in=plugin_names,
-    ).values("snapshot_id")
-    due_snapshots = Snapshot.objects.filter(
-        id__in=queued_snapshot_ids,
-        retry_at__lte=timezone.now(),
-        status=Snapshot.StatusChoices.SEALED,
     )
+    first_due_results = list(
+        ArchiveResult.objects.filter(
+            status=ArchiveResult.StatusChoices.QUEUED,
+            plugin__in=plugin_names,
+            snapshot__retry_at__lte=now,
+            snapshot__status=Snapshot.StatusChoices.SEALED,
+        )
+        .filter(**({"snapshot__crawl_id": crawl_id} if crawl_id else {}))
+        .values("snapshot_id", "snapshot__crawl_id")[:1],
+    )
+    if not first_due_results:
+        return False
+    root_crawl_id = str(first_due_results[0]["snapshot__crawl_id"])
+
+    due_snapshots = Snapshot.objects.filter(
+        retry_at__lte=now,
+        status=Snapshot.StatusChoices.SEALED,
+    ).filter(Exists(queued_results))
     if crawl_id:
         due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
-    due_snapshots = due_snapshots.only("id", "crawl_id", "retry_at", "status").order_by("retry_at", "created_at")
-    first_due_snapshot = due_snapshots.first()
-    if first_due_snapshot is None:
-        return False
-    root_crawl_id = str(first_due_snapshot.crawl_id)
+
     batch_candidates = list(
-        due_snapshots.filter(crawl_id=root_crawl_id)[:QUEUED_PLUGIN_RESULT_BATCH_SIZE],
+        # The crawl picker above starts from enabled queued ArchiveResult rows
+        # and uses a sliced LIMIT 1. Do not use QuerySet.first() here: it adds
+        # ordering and can turn this hot scheduler check into a temp-sort over
+        # hundreds of thousands of plugin rows. Once a crawl is selected,
+        # sibling order is irrelevant; the crawl_id/status index can fetch this
+        # small local batch directly while EXISTS proves the enabled queued
+        # plugin rows via the existing ArchiveResult unique index.
+        due_snapshots.filter(crawl_id=root_crawl_id).order_by()[:QUEUED_PLUGIN_RESULT_BATCH_SIZE],
     )
     if not batch_candidates:
         return False
@@ -1753,7 +1804,9 @@ def _run_due_queued_plugin_result(
     selected_plugins: list[str] | None = None
     claimed_snapshot_ids: list[str] = []
     for snapshot in batch_candidates:
-        snapshot_selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
+        snapshot_selected_plugins = [
+            plugin_name for plugin_name in (queued_plugins_for_snapshot(str(snapshot.id)) or []) if plugin_name in plugin_names
+        ]
         if not snapshot_selected_plugins:
             continue
         if selected_plugins is None:
@@ -1789,6 +1842,7 @@ def _run_due_queued_plugin_result(
         queued_results = ArchiveResult.objects.filter(
             snapshot_id=OuterRef("pk"),
             status=ArchiveResult.StatusChoices.QUEUED,
+            plugin__in=selected_plugins,
         )
         Snapshot.objects.filter(
             id__in=claimed_snapshot_ids,
@@ -1881,6 +1935,7 @@ def run_pending_crawls(
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.machine.models import Process
 
     crawl_claim_lock_seconds = 10
@@ -1891,9 +1946,9 @@ def run_pending_crawls(
         for plugin_name, plugin_config in plugin_configs.items()
         if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
     )
-    search_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_"))
     last_recovery_at = 0.0
     last_retention_at = 0.0
+    last_retention_repair_at = 0.0
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
@@ -1903,10 +1958,12 @@ def run_pending_crawls(
         now_monotonic = time.monotonic()
         if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
             for model in (ArchiveResult, Snapshot, Crawl, Process):
-                # The runner hot path must only touch indexed scheduler/retention
-                # columns before claiming work. delete_at is hydrated when rows
-                # are saved, while missing_delete_at_candidates() may inspect JSON
-                # config across large tables and can stall worker startup.
+                # Keep the tight scheduler loop anchored on indexed delete_at
+                # columns only. Backfilling missing delete_at values has to read
+                # config JSON for models whose retention policy is scoped to a
+                # Crawl/Snapshot/Process. That repair is still required for
+                # correctness, but it belongs in the idle maintenance block
+                # below, not ahead of every claim attempt.
                 model.delete_expired(batch_size=100, backfill_missing=False)
             last_retention_at = now_monotonic
 
@@ -2019,6 +2076,17 @@ def run_pending_crawls(
         ):
             continue
 
+        # Search backend selection is live crawl-execution config, not an
+        # installed-plugin list. Old queued rows for a backend that is disabled
+        # by the current Machine/Crawl/Snapshot config must remain queued so
+        # they can run if the user re-enables that backend, but they should not
+        # launch a standalone hook process just to skip after imports/config
+        # hydration. Refreshing here preserves mid-run config edits while using
+        # the same enabled-hook discovery path that created ArchiveResult rows.
+        runtime_config = get_config()
+        search_plugin_names = frozenset(
+            hook.parent.name for hook in discover_hooks("Snapshot", config=runtime_config) if hook.parent.name.startswith("search_backend_")
+        )
         if _run_due_queued_plugin_result(
             search_plugin_names,
             crawl_id=crawl_id,
@@ -2067,6 +2135,18 @@ def run_pending_crawls(
             if _run_due_binary():
                 continue
 
+        now_monotonic = time.monotonic()
+        if now_monotonic - last_retention_repair_at >= (60.0 if daemon else 0.0):
+            for model in (ArchiveResult, Snapshot, Crawl, Process):
+                # No runnable work was found on this scheduler pass. This is
+                # the bounded repair point for missing retention deadlines,
+                # including ArchiveResult rows intentionally saved without
+                # delete_at in the plugin-result hot path. Running it here keeps
+                # DELETE_AFTER resolution fresh without making every hook event
+                # load parent Snapshot/Crawl config.
+                model.delete_expired(batch_size=100, backfill_missing=True)
+            last_retention_repair_at = now_monotonic
+
         if daemon:
             now_monotonic = time.monotonic()
             if now_monotonic - last_recovery_at >= 30.0:

archivebox/tests/conftest.py

@@ -231,7 +231,7 @@ def initialized_archive(isolated_data_dir):
 
 
 @pytest.fixture
-def archivebox_daemon_server(tmp_path, process, unused_tcp_port_factory):
+def archivebox_daemon_server(tmp_path, process, free_tcp_port_factory):
     """
     Start a real daemonized ArchiveBox server in this test's DATA_DIR and
     always stop its supervisord before the test exits.
@@ -246,11 +246,11 @@ def archivebox_daemon_server(tmp_path, process, unused_tcp_port_factory):
                 "USE_COLOR": "False",
                 "SHOW_PROGRESS": "False",
                 "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
-                "SEARCH_BACKEND_SONIC_PORT": str(unused_tcp_port_factory()),
+                "SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
                 **{key: str(value) for key, value in env_overrides.items()},
             },
         )
-        port = unused_tcp_port_factory()
+        port = free_tcp_port_factory()
         result = subprocess.run(
             [sys.executable, "-m", "archivebox", "server", "--daemonize", f"127.0.0.1:{port}"],
             cwd=tmp_path,

archivebox/tests/test_cli_run.py

@@ -1388,6 +1388,41 @@ class TestRecoverOrchestratorState:
 
 @pytest.mark.django_db
 class TestRunDueCrawlState:
+    def test_idle_maintenance_repairs_archive_result_delete_at(self):
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import run_pending_crawls
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com/retention-repair",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+            config={"DELETE_AFTER": "2h"},
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com/retention-repair",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        result = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="search_backend_sqlite",
+            hook_name="on_Snapshot__90_index_sqlite.py",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+        )
+
+        # ArchiveResult saves are the plugin-event hot path. They intentionally
+        # do not resolve parent Snapshot/Crawl config on every write; the real
+        # runner's idle maintenance pass owns missing delete_at repair.
+        assert result.delete_at is None
+        assert run_pending_crawls(daemon=False, maintenance_only=True) == 0
+
+        result.refresh_from_db()
+        assert result.delete_at is not None
+
     def test_maintenance_only_runner_does_not_start_regular_queued_crawls(self):
         from django.utils import timezone
 

archivebox/tests/test_cli_server.py

@@ -199,13 +199,83 @@ def test_server_daemon_restarts_runner_killed_by_signal(archivebox_daemon_server
     assert state["worker_daphne"]["statename"] == "RUNNING", state
 
 
-def test_sonic_worker_is_disabled_when_sonic_disabled_and_engine_not_sonic(tmp_path):
+def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_runtime(archivebox_daemon_server):
+    server = archivebox_daemon_server(SEARCH_BACKEND_ENGINE="ripgrep")
+    server.wait_for_workers(("worker_daphne", "worker_runner"))
+
+    setup_result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import django;"
+                "django.setup();"
+                "from archivebox.base_models.models import get_or_create_system_user_pk;"
+                "from archivebox.crawls.models import Crawl;"
+                "from archivebox.core.models import Snapshot;"
+                "from archivebox.machine.models import Machine;"
+                "machine = Machine.current(refresh=True);"
+                "machine.config = {**dict(machine.config or {}), 'SEARCH_BACKEND_ENGINE': 'sqlite'};"
+                "machine.save(update_fields=['config', 'modified_at']);"
+                "crawl = Crawl.objects.create("
+                "urls='https://example.com/live-machine-search-config',"
+                "created_by_id=get_or_create_system_user_pk(),"
+                "config={},"
+                ");"
+                "snapshot = Snapshot.objects.create("
+                "url='https://example.com/live-machine-search-config',"
+                "crawl=crawl,"
+                ");"
+                "print(snapshot.id)"
+            ),
+        ],
+        cwd=server.data_dir,
+        env=server.env,
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    assert setup_result.returncode == 0, setup_result.stderr or setup_result.stdout
+    snapshot_id = setup_result.stdout.strip().splitlines()[-1]
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import django,json;"
+                "django.setup();"
+                "from archivebox.core.models import Snapshot;"
+                "from archivebox.config.common import get_config;"
+                f"snapshot = Snapshot.objects.select_related('crawl').get(id='{snapshot_id}');"
+                "runtime = get_config(snapshot=snapshot).for_crawl_runtime("
+                "crawl=snapshot.crawl,"
+                "snapshot=snapshot,"
+                "extra_context={'snapshot_id': str(snapshot.id)},"
+                ");"
+                "print(json.dumps({"
+                "'sqlite_enabled': runtime.get('SEARCH_BACKEND_SQLITE_ENABLED'),"
+                "'engine_in_runtime': 'SEARCH_BACKEND_ENGINE' in runtime,"
+                "}))"
+            ),
+        ],
+        cwd=server.data_dir,
+        env=server.env,
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    resolved = json.loads(result.stdout.strip().splitlines()[-1])
+    assert resolved == {"sqlite_enabled": True, "engine_in_runtime": False}
+
+
+def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path):
     from archivebox.workers.supervisord_util import get_sonic_supervisord_worker_from_plugin
 
     worker = get_sonic_supervisord_worker_from_plugin(
         SimpleNamespace(
             DATA_DIR=str(tmp_path),
-            SEARCH_BACKEND_ENGINE="ripgrep",
             SEARCH_BACKEND_SONIC_ENABLED=False,
             SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
             SEARCH_BACKEND_SONIC_PORT=_free_port(),
@@ -234,7 +304,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo
     daemon_event = prepare_sonic_daemon(
         SimpleNamespace(
             DATA_DIR=str(server.data_dir),
-            SEARCH_BACKEND_ENGINE="sonic",
+            SEARCH_BACKEND_SONIC_ENABLED=True,
             SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
             SEARCH_BACKEND_SONIC_PORT=sonic_port,
             SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword",

archivebox/tests/test_frozen_crawl_config.py

@@ -131,11 +131,44 @@ def test_config_scopes_are_derived_from_section_and_field_metadata():
     assert ArchiveBoxConfig.scope_for_key("DEBUG") == "crawl_execution"
     assert ArchiveBoxConfig.scope_for_key("DEFAULT_PERSONA") == "crawl_execution"
     assert ArchiveBoxConfig.scope_for_key("WGET_ENABLED") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("SEARCH_BACKEND_SQLITE_ENABLED") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("SEARCH_BACKEND_ENGINE") == "crawl_execution"
     assert ArchiveBoxConfig.scope_for_key("WGET_WARC_ENABLED") == "crawl_frozen"
     assert ArchiveBoxConfig.scope_for_key("SECRET_KEY") == "server"
     assert ArchiveBoxConfig.scope_for_key("DATABASE_NAME") == "server"
 
 
+def test_search_backend_engine_derives_default_backend_enabled_without_entering_hook_env():
+    from archivebox.config.common import ArchiveBoxConfig
+
+    default_runtime_config = ArchiveBoxConfig().for_crawl_runtime(extra_context={"snapshot_id": "default-runtime-config"})
+    assert default_runtime_config["SEARCH_BACKEND_RIPGREP_ENABLED"] is True
+    assert default_runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is False
+    assert default_runtime_config["SEARCH_BACKEND_SONIC_ENABLED"] is True
+
+    sqlite_runtime_config = ArchiveBoxConfig(SEARCH_BACKEND_ENGINE="sqlite").for_crawl_runtime(
+        extra_context={"snapshot_id": "sqlite-runtime-config"},
+    )
+    assert sqlite_runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is True
+
+    config = ArchiveBoxConfig(
+        SEARCH_BACKEND_ENGINE="ripgrep",
+        SEARCH_BACKEND_SQLITE_ENABLED=False,
+        SEARCH_BACKEND_SONIC_ENABLED=True,
+        SECRET_KEY="server-secret",
+        DATABASE_NAME="server-db.sqlite3",
+    )
+
+    runtime_config = config.for_crawl_runtime(extra_context={"snapshot_id": "runtime-config"})
+
+    assert "SEARCH_BACKEND_ENGINE" not in runtime_config
+    assert runtime_config["SEARCH_BACKEND_RIPGREP_ENABLED"] is True
+    assert runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is False
+    assert runtime_config["SEARCH_BACKEND_SONIC_ENABLED"] is True
+    assert "SECRET_KEY" not in runtime_config
+    assert "DATABASE_NAME" not in runtime_config
+
+
 def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_env_overridden(archivebox_db, monkeypatch):
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.34rc28",
+  "version": "0.9.34rc29",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.34rc28"
+version = "0.9.34rc29"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.9",          # EventBus API
-    "abxpkg>=1.11.142",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.145",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.145",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.143",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.146",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.146",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]