release: archivebox 0.9.33rc80

archivebox/core/admin_snapshots.py

@@ -11,21 +11,23 @@ from urllib.parse import urlsplit
 from uuid import UUID
 
 from django.contrib import admin, messages
+from django.contrib.admin.views.main import IncorrectLookupParameters
 from django.urls import path, reverse
 from django.shortcuts import get_object_or_404, redirect
 from django.core.cache import cache
+from django.core.paginator import InvalidPage
 from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, QueryDict, StreamingHttpResponse
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
-from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
+from django.db.models import Q, Count, Exists, F, IntegerField, OuterRef, Prefetch, Subquery
 from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
 
 from archivebox.config.common import get_config
 from archivebox.misc.util import htmldecode, urldecode
-from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.misc.paginators import AcceleratedPaginator, CountlessPaginator
 from archivebox.misc.logging_util import printable_filesize
 from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key
 from archivebox.core.host_util import build_snapshot_url, build_web_url
@@ -266,6 +268,7 @@ class SnapshotRetryListFilter(admin.SimpleListFilter):
 class SnapshotResultHealthListFilter(admin.SimpleListFilter):
     title = "ArchiveResult status"
     parameter_name = "archiveresult_status"
+    SNAPSHOT_FIRST_VALUES = {"succeeded"}
 
     def lookups(self, request, model_admin):
         return (
@@ -279,6 +282,47 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
             ("noresults", ">50% noresults"),
         )
 
+    @staticmethod
+    def _snapshot_total_count_subquery(outer_ref: str = "pk"):
+        return (
+            ArchiveResult.objects.filter(snapshot_id=OuterRef(outer_ref))
+            .order_by()
+            .values("snapshot_id")
+            .annotate(count=Count("pk"))
+            .values("count")
+        )
+
+    @staticmethod
+    def _snapshot_status_count_subquery(status: str, outer_ref: str = "pk"):
+        return (
+            ArchiveResult.objects.filter(snapshot_id=OuterRef(outer_ref), status=status)
+            .order_by()
+            .values("snapshot_id")
+            .annotate(count=Count("pk"))
+            .values("count")
+        )
+
+    def _filter_snapshot_first(self, queryset, status: str):
+        return queryset.annotate(
+            total_results=Subquery(self._snapshot_total_count_subquery(), output_field=IntegerField()),
+            matching_results=Subquery(self._snapshot_status_count_subquery(status), output_field=IntegerField()),
+        ).filter(matching_results__gt=F("total_results") / 2)
+
+    def _filter_status_first(self, queryset, status: str):
+        total_results = self._snapshot_total_count_subquery("snapshot_id")
+        matching_snapshot_ids = (
+            ArchiveResult.objects.filter(status=status)
+            .order_by()
+            .values("snapshot_id")
+            .annotate(
+                matching_results=Count("pk"),
+                total_results=Subquery(total_results, output_field=IntegerField()),
+            )
+            .filter(matching_results__gt=F("total_results") / 2)
+            .values("snapshot_id")
+        )
+        return queryset.filter(pk__in=matching_snapshot_ids)
+
     def queryset(self, request, queryset):
         value = self.value()
         if value:
@@ -296,14 +340,10 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                 "noresults": ArchiveResult.StatusChoices.NORESULTS,
             }
             if value in status_by_value:
-                queryset = queryset.annotate(
-                    total_results=Count("archiveresult"),
-                    matching_results=Count(
-                        "archiveresult",
-                        filter=Q(archiveresult__status=status_by_value[value]),
-                    ),
-                )
-                return queryset.filter(matching_results__gt=F("total_results") / 2)
+                status = status_by_value[value]
+                if value in self.SNAPSHOT_FIRST_VALUES:
+                    return self._filter_snapshot_first(queryset, status)
+                return self._filter_status_first(queryset, status)
         return queryset
 
 
@@ -316,7 +356,28 @@ class SnapshotChangeList(SearchResultsChangeList):
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
 
+    def _uses_expensive_archiveresult_filter(self, request) -> bool:
+        return bool(request.GET.get(SnapshotResultHealthListFilter.parameter_name))
+
     def get_results(self, request):
+        if self._uses_expensive_archiveresult_filter(request):
+            paginator = CountlessPaginator(self.queryset, self.list_per_page)
+            try:
+                page = paginator.page(self.page_num)
+            except InvalidPage:
+                raise IncorrectLookupParameters
+
+            self.result_count = paginator.count
+            self.show_full_result_count = False
+            self.show_admin_actions = True
+            self.full_result_count = None
+            self.result_list = page.object_list
+            self.can_show_all = False
+            self.multi_page = page.has_next() or self.page_num > 1
+            self.paginator = paginator
+            self.show_search_index_hint = False
+            return
+
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
@@ -1261,6 +1322,33 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ordering="ar_succeeded_count",
     )
     def files(self, obj):
+        stats = self._get_progress_stats(obj)
+        if obj.status == Snapshot.StatusChoices.STARTED and stats["total"] > 0:
+            succeeded = stats["succeeded"]
+            failed = stats["failed"]
+            skipped = stats["skipped"]
+            completed = succeeded + failed + skipped + stats["noresults"]
+            return format_html(
+                """<div style="min-width: 96px;">
+                    <div style="display: flex; align-items: center; gap: 6px; margin-bottom: 4px;">
+                        <span class="snapshot-progress-spinner"></span>
+                        <span style="font-size: 11px; color: #64748b;">{}/{} hooks</span>
+                    </div>
+                    <div class="snapshot-progress-bar">
+                        <div class="snapshot-progress-bar-fill" style="background: #3b82f6; width: {}%;"></div>
+                    </div>
+                    <div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">
+                        ✓{} ✗{} ⏳{}
+                    </div>
+                </div>""",
+                completed,
+                stats["total"],
+                stats["percent"],
+                succeeded,
+                failed,
+                stats["running"],
+            )
+
         results = self._get_prefetched_results(obj)
         if results is None:
             results = obj.archiveresult_set.only("plugin", "status", "output_size")

archivebox/core/migrations/0047_archiveresult_status_snapshot_index.py

@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0046_repair_snapshot_permissions"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="CREATE INDEX IF NOT EXISTS archiveresult_status_snap_idx ON core_archiveresult (status, snapshot_id)",
+                    reverse_sql="DROP INDEX IF EXISTS archiveresult_status_snap_idx",
+                ),
+            ],
+            state_operations=[
+                migrations.AddIndex(
+                    model_name="archiveresult",
+                    index=models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"),
+                ),
+            ],
+        ),
+        migrations.RunSQL(
+            sql="ANALYZE core_archiveresult",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
+    ]

archivebox/core/models.py

@@ -1948,8 +1948,44 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         cache_key = f"result_icons:{self.pk}:{'compact' if compact_icons else 'full'}:{(self.downloaded_at or self.modified_at or self.created_at or self.bookmarked_at).timestamp()}"
 
         def calc_icons():
+            if compact_icons and self.status == self.StatusChoices.STARTED:
+                progress_stats = getattr(self, "_icons_progress_stats", None) or self.get_progress_stats()
+                total = int(progress_stats.get("total") or 0)
+                succeeded = int(progress_stats.get("succeeded") or 0)
+                failed = int(progress_stats.get("failed") or 0)
+                skipped = int(progress_stats.get("skipped") or 0)
+                noresults = int(progress_stats.get("noresults") or 0)
+                running = int(progress_stats.get("running") or 0)
+                completed = succeeded + failed + skipped + noresults
+                percent = int((completed / total * 100) if total > 0 else 0)
+                return format_html(
+                    '<div class="snapshot-files-progress" title="{} of {} hooks complete" style="min-width: 96px;">'
+                    '<div style="display: flex; align-items: center; gap: 6px; margin-bottom: 4px;">'
+                    '<span class="snapshot-progress-spinner" style="display: inline-block; width: 12px; height: 12px; border: 2px solid #e2e8f0; border-top-color: #3b82f6; border-radius: 50%; animation: snapshot-spin 0.8s linear infinite;"></span>'
+                    '<span style="font-size: 11px; color: #64748b;">{}/{} hooks</span>'
+                    "</div>"
+                    '<div style="background: #e2e8f0; border-radius: 4px; height: 6px; overflow: hidden;">'
+                    '<div style="background: #3b82f6; width: {}%; height: 100%; transition: width 0.3s;"></div>'
+                    "</div>"
+                    '<div style="font-size: 10px; color: #94a3b8; margin-top: 2px;">'
+                    "✓{} ✗{} ⏳{}"
+                    "</div>"
+                    "</div>",
+                    completed,
+                    total,
+                    completed,
+                    total,
+                    percent,
+                    succeeded,
+                    failed,
+                    running,
+                )
+
+            precomputed_archive_results = getattr(self, "_icons_archive_results", None)
             prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
-            if "archiveresult_set" in prefetched_cache:
+            if precomputed_archive_results is not None and compact_icons:
+                archive_results = {plugin: True for plugin in precomputed_archive_results}
+            elif "archiveresult_set" in prefetched_cache:
                 archive_results = {
                     r.plugin: r
                     for r in self.archiveresult_set.all()
@@ -1979,7 +2015,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
             for plugin in ordered_plugins:
                 result = archive_results.get(plugin)
-                existing = bool(result and result.status == "succeeded" and (compact_icons or result.output_files or result.output_str))
+                existing = result is True or bool(
+                    result and result.status == "succeeded" and (compact_icons or result.output_files or result.output_str),
+                )
                 if not existing:
                     continue
                 icon = mark_safe(get_plugin_icon(plugin))
@@ -2004,6 +2042,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 mark_safe(output),
             )
 
+        if compact_icons and self.status == self.StatusChoices.STARTED:
+            return calc_icons()
+
         cache_result = cache.get(cache_key)
         if cache_result:
             return cache_result
@@ -3468,6 +3509,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         verbose_name_plural = "Archive Results Log"
         indexes = [
             models.Index(fields=["snapshot", "status"], name="archiveresult_snap_status_idx"),
+            models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"),
             models.Index(fields=["-start_ts", "-id"], name="archiveresult_start_idx"),
         ]
         constraints = [

archivebox/core/views.py

@@ -1071,9 +1071,49 @@ class PublicIndexView(ListView):
             and search_mode_backend
             and getattr(context.get("paginator"), "count", 0) == 0,
         )
-        for snapshot in context.get("object_list") or ():
+        snapshots = list(context.get("object_list") or ())
+        icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
+        progress_by_snapshot: dict[str, dict[str, int]] = {
+            str(snapshot.id): {
+                "total": 0,
+                "succeeded": 0,
+                "failed": 0,
+                "running": 0,
+                "skipped": 0,
+                "noresults": 0,
+            }
+            for snapshot in snapshots
+        }
+        if icons_by_snapshot:
+            for snapshot_id, plugin, status in (
+                ArchiveResult.objects.filter(
+                    snapshot_id__in=icons_by_snapshot.keys(),
+                )
+                .exclude(plugin="")
+                .values_list("snapshot_id", "plugin", "status")
+                .iterator(chunk_size=1000)
+            ):
+                snapshot_key = str(snapshot_id)
+                progress = progress_by_snapshot[snapshot_key]
+                progress["total"] += 1
+                if status == ArchiveResult.StatusChoices.SUCCEEDED:
+                    icons_by_snapshot[snapshot_key].add(plugin)
+                    progress["succeeded"] += 1
+                elif status == ArchiveResult.StatusChoices.FAILED:
+                    progress["failed"] += 1
+                elif status == ArchiveResult.StatusChoices.STARTED:
+                    progress["running"] += 1
+                elif status == ArchiveResult.StatusChoices.SKIPPED:
+                    progress["skipped"] += 1
+                elif status == ArchiveResult.StatusChoices.NORESULTS:
+                    progress["noresults"] += 1
+
+        for snapshot in snapshots:
             snapshot._icons_compact = True
+            snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set())
+            snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {})
             snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED)
+        context["object_list"] = snapshots
         return context
 
     def get_queryset(self, **kwargs):

archivebox/templates/admin/snapshots_grid.html

@@ -130,7 +130,7 @@
       grid-template-columns: 18px minmax(0, 1fr) auto;
       gap: 8px;
       align-items: center;
-      text-align: left;
+      text-align: center;
   }
   .cards .card .card-meta .card-date,
   .cards .card .card-meta .card-size {

archivebox/templates/core/index_row.html

@@ -69,13 +69,7 @@
     </td>
     <td class="snapshot-files-cell">
         <span data-number-for="{{ link.url }}" title="{{ link.num_outputs }} successful outputs">
-            {% if compact_public_index %}
-                {% if link.num_outputs %}
-                    {{ link.num_outputs }} output{{ link.num_outputs|pluralize }}
-                {% else %}
-                    <span class="empty-value">...</span>
-                {% endif %}
-            {% elif link.icons %}
+            {% if link.icons %}
                 {{ link.icons }}
             {% else %}
                 <span class="empty-value">...</span>

archivebox/templates/core/public_index.html

@@ -349,6 +349,10 @@
         max-width: 200px;
     }
 
+    @keyframes snapshot-spin {
+        to { transform: rotate(360deg); }
+    }
+
     .snapshot-size-cell {
         width: 102px;
         white-space: nowrap;

archivebox/templates/static/admin.css

@@ -1893,7 +1893,8 @@ tbody .output-link:hover {opacity: 1;}
 
 .model-snapshot.change-list .cards .card .card-info.card-meta > a {
     grid-column: auto;
-    justify-self: start;
+    justify-self: center;
+    text-align: center;
 }
 
 .model-snapshot.change-list .cards .card .card-info .timestamp {

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.33rc79",
+  "version": "0.9.33rc80",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.33rc79"
+version = "0.9.33rc80"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.8",          # EventBus API
-    "abxpkg>=1.11.111",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.114",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.114",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.112",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.115",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.115",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]