diff --git a/Dockerfile b/Dockerfile index 37856f18..35f91678 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,6 +162,7 @@ LABEL name="archivebox" \ COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg +COPY --from=archivebox-builder /opt/uv/python /opt/uv/python COPY --from=archivebox-builder /venv /venv COPY --from=archivebox-builder /app /app COPY --from=archivebox-builder /VERSION.txt /VERSION.txt diff --git a/archivebox/search/admin.py b/archivebox/search/admin.py index d7c7c698..988ee51f 100644 --- a/archivebox/search/admin.py +++ b/archivebox/search/admin.py @@ -2,6 +2,7 @@ __package__ = "archivebox.search" from django.contrib import admin from django.contrib.admin.views.main import ChangeList +from django.db.models import Case, IntegerField, Value, When from archivebox.search.config import ( get_default_search_mode, @@ -71,7 +72,13 @@ class SearchResultsAdminMixin(admin.ModelAdmin): if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl": cached_ids = get_cached_admin_search_ids(request) if cached_ids is not None: - return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False + if not cached_ids: + return queryset.none(), False + search_rank = Case( + *(When(pk=snapshot_id, then=Value(index)) for index, snapshot_id in enumerate(cached_ids)), + output_field=IntegerField(), + ) + return queryset.filter(pk__in=cached_ids).annotate(search_rank=search_rank).order_by("search_rank"), False return queryset.none(), False if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta": diff --git a/archivebox/tests/test_search.py b/archivebox/tests/test_search.py index 3a52f8b7..c361424a 100644 --- a/archivebox/tests/test_search.py +++ b/archivebox/tests/test_search.py @@ -1,3 +1,4 @@ +import json import os import re import shutil @@ -781,7 +782,7 @@ class TestSearchBackendsE2E: sonic_port = get_free_port() env = cli_env( live=True, - PLUGINS="title,wget,search_backend_ripgrep,search_backend_sqlite,search_backend_sonic", + PLUGINS="wget,search_backend_ripgrep,search_backend_sqlite,search_backend_sonic", SAVE_TITLE="True", SAVE_WGET="True", SAVE_WARC="False", @@ -804,7 +805,7 @@ class TestSearchBackendsE2E: ) create_admin_and_token(initialized_archive) - bulk_wget_urls = [*matrix_urls, url_only_url, url_contains_order_url] + bulk_wget_urls = [*matrix_urls] add_result = run_archivebox_cmd( [ "add", @@ -822,43 +823,73 @@ class TestSearchBackendsE2E: ) assert add_result.returncode == 0, add_result.stderr or add_result.stdout - title_add_result = run_archivebox_cmd( - [ - "add", - "--depth=0", - "--crawl-max-concurrent-snapshots=3", - "--parser=url_list", - "--plugins=title,wget", - "--tag=search-matrix", - title_only_url, - title_prefix_order_url, - title_contains_order_url, - ], + metadata_snapshot_records = [ + { + "type": "Snapshot", + "url": url_only_url, + "title": "URL Only Precision Page", + "tags": "search-matrix", + "depth": 0, + }, + { + "type": "Snapshot", + "url": title_only_url, + "title": title_only_needle, + "tags": "search-matrix", + "depth": 0, + }, + { + "type": "Snapshot", + "url": tag_only_url, + "title": "Tag Only Precision Page", + "tags": f"search-matrix,{tag_only_needle}", + "depth": 0, + }, + { + "type": "Snapshot", + "url": title_prefix_order_url, + "title": title_prefix_order_title, + "tags": "search-matrix", + "depth": 0, + }, + { + "type": "Snapshot", + "url": url_contains_order_url, + "title": "URL Contains Ordering Page", + "tags": "search-matrix", + "depth": 0, + }, + { + "type": "Snapshot", + "url": title_contains_order_url, + "title": title_contains_order_title, + "tags": "search-matrix", + "depth": 0, + }, + { + "type": "Snapshot", + "url": tag_order_url, + "title": "Tag Ordering Page", + "tags": f"search-matrix,{order_needle}", + "depth": 0, + }, + ] + metadata_create_result = run_archivebox_cmd( + ["snapshot", "create"], cwd=initialized_archive, env=env, - timeout=120, + input="\n".join(json.dumps(record) for record in metadata_snapshot_records) + "\n", + timeout=60, ) - assert title_add_result.returncode == 0, title_add_result.stderr or title_add_result.stdout - - precision_adds = ( - (tag_only_url, f"search-matrix,{tag_only_needle}", "wget"), - (tag_order_url, f"search-matrix,{order_needle}", "wget"), + assert metadata_create_result.returncode == 0, metadata_create_result.stderr or metadata_create_result.stdout + metadata_seal_result = run_archivebox_cmd( + ["snapshot", "update", "--status=sealed"], + cwd=initialized_archive, + env=env, + input=metadata_create_result.stdout, + timeout=60, ) - for precision_url, precision_tags, precision_plugins in precision_adds: - precision_add_result = run_archivebox_cmd( - [ - "add", - "--depth=0", - "--parser=url_list", - f"--plugins={precision_plugins}", - f"--tag={precision_tags}", - precision_url, - ], - cwd=initialized_archive, - env=env, - timeout=60, - ) - assert precision_add_result.returncode == 0, precision_add_result.stderr or precision_add_result.stdout + assert metadata_seal_result.returncode == 0, metadata_seal_result.stderr or metadata_seal_result.stdout list_result = run_archivebox_cmd( ["list", "--url__icontains", f"127.0.0.1:{fixture_port}", "--csv=url"], diff --git a/etc/package.json b/etc/package.json index 98445ea0..2795a8db 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.34rc68", + "version": "0.9.34rc69", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index d16cf574..a38a7ff9 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.34rc68" +version = "0.9.34rc69" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]