Fix real binary resolution in CI tests

archivebox/progressmonitor/views.py

@@ -743,7 +743,7 @@ def live_progress_view(request):
                     seen_plugin_keys.add(str(process.id) if process else f"{ar.plugin}:{hook_name}")
 
                 for proc_payload, proc_started_at in process_records_by_snapshot.get(str(snapshot["id"]), []):
-                    if not is_current_run_timestamp(proc_started_at, snapshot_run_started_at):
+                    if not is_current_run_timestamp(proc_started_at, snapshot.get("created_at")):
                         continue
                     proc_key = str(proc_payload.get("process_id") or f"{proc_payload.get('plugin')}:{proc_payload.get('hook_name')}")
                     if proc_key in seen_plugin_keys:

archivebox/tests/test_search.py

@@ -1,10 +1,11 @@
 import os
 import re
 import time
+from datetime import timedelta
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from itertools import pairwise
 from pathlib import Path
 from threading import Thread
-from datetime import timedelta
 from urllib.parse import urlencode
 
 import pytest
@@ -14,20 +15,19 @@ from django.contrib.auth import get_user_model
 from django.core.cache import cache
 from django.urls import reverse
 
-from archivebox.misc.logging import AttrDict
 from archivebox.machine.models import Machine
+from archivebox.misc.logging import AttrDict
 from archivebox.tests.conftest import (
     cli_env,
     create_admin_and_token,
     get_free_port,
-    run_archivebox_cmd,
+    get_http_response,
     resolve_abxpkg_binary_env,
+    run_archivebox_cmd,
     start_archivebox_server,
     stop_archivebox_process,
-    get_http_response,
 )
 
-
 pytestmark = pytest.mark.django_db(transaction=True)
 
 User = get_user_model()
@@ -99,6 +99,25 @@ def populate_admin_search_cache(client, path, params):
     return client.get(path, params, HTTP_HOST=ADMIN_HOST)
 
 
+def configure_ripgrep_search_backend(lib_dir: Path) -> None:
+    from abx_plugins import get_plugins_dir
+
+    binary_env = resolve_abxpkg_binary_env(
+        lib_dir,
+        deps_from=Path(get_plugins_dir()) / "search_backend_ripgrep" / "config.json",
+    )
+    rg_binary = Path(binary_env["RIPGREP_BINARY"])
+    assert rg_binary.is_file()
+    Machine.from_json(
+        {
+            "config": {
+                "SEARCH_BACKEND_ENGINE": "ripgrep",
+                "RIPGREP_BINARY": str(rg_binary),
+            },
+        },
+    )
+
+
 def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
     from archivebox.search.backends import search_backend_env
 
@@ -225,10 +244,16 @@ class TestAdminSnapshotSearch:
         assert b'id="changelist"' in response.content
         assert b"search-mode-contents" in response.content
 
-    def test_admin_search_stream_uses_real_ripgrep_backend_for_deep_results(self, client, admin_user, crawl):
+    def test_admin_search_stream_uses_real_ripgrep_backend_for_deep_results(
+        self,
+        client,
+        admin_user,
+        crawl,
+        cached_abxpkg_lib_dir,
+    ):
         from archivebox.core.models import Snapshot
 
-        Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}})
+        configure_ripgrep_search_backend(cached_abxpkg_lib_dir)
         fulltext_snapshot = Snapshot.objects.create(
             url="https://example.com/fulltext-only",
             title="Unrelated Title",
@@ -282,10 +307,16 @@ class TestAdminSnapshotSearch:
         assert result_ids[:3] == [prefix_snapshot.pk, title_snapshot.pk, contains_snapshot.pk]
         assert {title_snapshot.pk, contains_snapshot.pk, prefix_snapshot.pk}.issubset(result_ids)
 
-    def test_admin_contents_search_stream_uses_real_backend_results(self, client, admin_user, crawl):
+    def test_admin_contents_search_stream_uses_real_backend_results(
+        self,
+        client,
+        admin_user,
+        crawl,
+        cached_abxpkg_lib_dir,
+    ):
         from archivebox.core.models import Snapshot
 
-        Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}})
+        configure_ripgrep_search_backend(cached_abxpkg_lib_dir)
         metadata_snapshot = Snapshot.objects.create(
             url="https://example.com/google-meta",
             title="Google Metadata Match",
@@ -410,10 +441,15 @@ class TestPublicIndexSearch:
         assert b'value="deep:ripgrep"' in response.content
         assert b">deep:ripgrep<" in response.content
 
-    def test_public_search_uses_streamed_metadata_order(self, client, crawl):
+    def test_public_search_uses_streamed_metadata_order(
+        self,
+        client,
+        crawl,
+        cached_abxpkg_lib_dir,
+    ):
         from archivebox.core.models import Snapshot
 
-        Machine.from_json({"config": {"SEARCH_BACKEND_ENGINE": "ripgrep"}})
+        configure_ripgrep_search_backend(cached_abxpkg_lib_dir)
         metadata_snapshot = Snapshot.objects.create(
             url="https://public-example.com/google-meta",
             title="Google Metadata Match",
@@ -1152,8 +1188,7 @@ class TestSearchBackendsE2E:
                     # there are multiple positive events, still verify progress stays live.
                     if len(positive_events) > 1:
                         max_progress_gap = max(
-                            later_elapsed - earlier_elapsed
-                            for (_, earlier_elapsed), (_, later_elapsed) in zip(positive_events, positive_events[1:])
+                            later_elapsed - earlier_elapsed for (_, earlier_elapsed), (_, later_elapsed) in pairwise(positive_events)
                         )
                         assert max_progress_gap < 1.0, (surface_name, search_mode, max_progress_gap, positive_events[:10])
                     assert first_positive_elapsed is not None and first_positive_elapsed < 0.75, (

archivebox/workers/supervisord_util.py

@@ -66,7 +66,16 @@ def resolve_env_binary(name: str) -> Path:
 
     lib_dir = Path(os.environ.get("ABXPKG_LIB_DIR") or get_config().ABXPKG_LIB_DIR)
     env_root = lib_dir / "env"
-    provider = EnvProvider(install_root=env_root, PATH=os.environ["PATH"])
+    runtime_bin_dir = Path(sys.executable).parent
+    provider_path = os.pathsep.join(
+        str(path)
+        for path in (
+            *os.environ.get("PATH", "").split(os.pathsep),
+            str(runtime_bin_dir),
+        )
+        if path
+    )
+    provider = EnvProvider(install_root=env_root, PATH=provider_path)
     if name == "daphne":
         from importlib.metadata import version