diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index e73b1d57..985e0b04 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -11,21 +11,23 @@ from urllib.parse import urlsplit from uuid import UUID from django.contrib import admin, messages +from django.contrib.admin.views.main import IncorrectLookupParameters from django.urls import path, reverse from django.shortcuts import get_object_or_404, redirect from django.core.cache import cache +from django.core.paginator import InvalidPage from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, QueryDict, StreamingHttpResponse from django.utils import timezone from django.utils.html import format_html, format_html_join from django.utils.safestring import mark_safe -from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch +from django.db.models import Q, Count, Exists, F, IntegerField, OuterRef, Prefetch, Subquery from django import forms from django.template import Template, RequestContext from django.contrib.admin.helpers import ActionForm from archivebox.config.common import get_config from archivebox.misc.util import htmldecode, urldecode -from archivebox.misc.paginators import AcceleratedPaginator +from archivebox.misc.paginators import AcceleratedPaginator, CountlessPaginator from archivebox.misc.logging_util import printable_filesize from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key from archivebox.core.host_util import build_snapshot_url, build_web_url @@ -266,6 +268,7 @@ class SnapshotRetryListFilter(admin.SimpleListFilter): class SnapshotResultHealthListFilter(admin.SimpleListFilter): title = "ArchiveResult status" parameter_name = "archiveresult_status" + SNAPSHOT_FIRST_VALUES = {"succeeded"} def lookups(self, request, model_admin): return ( @@ -279,6 +282,47 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter): ("noresults", ">50% noresults"), ) + @staticmethod + def _snapshot_total_count_subquery(outer_ref: str = "pk"): + return ( + ArchiveResult.objects.filter(snapshot_id=OuterRef(outer_ref)) + .order_by() + .values("snapshot_id") + .annotate(count=Count("pk")) + .values("count") + ) + + @staticmethod + def _snapshot_status_count_subquery(status: str, outer_ref: str = "pk"): + return ( + ArchiveResult.objects.filter(snapshot_id=OuterRef(outer_ref), status=status) + .order_by() + .values("snapshot_id") + .annotate(count=Count("pk")) + .values("count") + ) + + def _filter_snapshot_first(self, queryset, status: str): + return queryset.annotate( + total_results=Subquery(self._snapshot_total_count_subquery(), output_field=IntegerField()), + matching_results=Subquery(self._snapshot_status_count_subquery(status), output_field=IntegerField()), + ).filter(matching_results__gt=F("total_results") / 2) + + def _filter_status_first(self, queryset, status: str): + total_results = self._snapshot_total_count_subquery("snapshot_id") + matching_snapshot_ids = ( + ArchiveResult.objects.filter(status=status) + .order_by() + .values("snapshot_id") + .annotate( + matching_results=Count("pk"), + total_results=Subquery(total_results, output_field=IntegerField()), + ) + .filter(matching_results__gt=F("total_results") / 2) + .values("snapshot_id") + ) + return queryset.filter(pk__in=matching_snapshot_ids) + def queryset(self, request, queryset): value = self.value() if value: @@ -296,14 +340,10 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter): "noresults": ArchiveResult.StatusChoices.NORESULTS, } if value in status_by_value: - queryset = queryset.annotate( - total_results=Count("archiveresult"), - matching_results=Count( - "archiveresult", - filter=Q(archiveresult__status=status_by_value[value]), - ), - ) - return queryset.filter(matching_results__gt=F("total_results") / 2) + status = status_by_value[value] + if value in self.SNAPSHOT_FIRST_VALUES: + return self._filter_snapshot_first(queryset, status) + return self._filter_status_first(queryset, status) return queryset @@ -316,7 +356,28 @@ class SnapshotChangeList(SearchResultsChangeList): resolver_name == "grid" or request.path.rstrip("/").endswith("/grid") ) + def _uses_expensive_archiveresult_filter(self, request) -> bool: + return bool(request.GET.get(SnapshotResultHealthListFilter.parameter_name)) + def get_results(self, request): + if self._uses_expensive_archiveresult_filter(request): + paginator = CountlessPaginator(self.queryset, self.list_per_page) + try: + page = paginator.page(self.page_num) + except InvalidPage: + raise IncorrectLookupParameters + + self.result_count = paginator.count + self.show_full_result_count = False + self.show_admin_actions = True + self.full_result_count = None + self.result_list = page.object_list + self.can_show_all = False + self.multi_page = page.has_next() or self.page_num > 1 + self.paginator = paginator + self.show_search_index_hint = False + return + super().get_results(request) if request.GET.get("_embedded") == "crawl": self.full_result_count = self.result_count @@ -1261,6 +1322,33 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): ordering="ar_succeeded_count", ) def files(self, obj): + stats = self._get_progress_stats(obj) + if obj.status == Snapshot.StatusChoices.STARTED and stats["total"] > 0: + succeeded = stats["succeeded"] + failed = stats["failed"] + skipped = stats["skipped"] + completed = succeeded + failed + skipped + stats["noresults"] + return format_html( + """
+
+ + {}/{} hooks +
+
+
+
+
+ ✓{} ✗{} ⏳{} +
+
""", + completed, + stats["total"], + stats["percent"], + succeeded, + failed, + stats["running"], + ) + results = self._get_prefetched_results(obj) if results is None: results = obj.archiveresult_set.only("plugin", "status", "output_size") diff --git a/archivebox/core/migrations/0047_archiveresult_status_snapshot_index.py b/archivebox/core/migrations/0047_archiveresult_status_snapshot_index.py new file mode 100644 index 00000000..2c31aaeb --- /dev/null +++ b/archivebox/core/migrations/0047_archiveresult_status_snapshot_index.py @@ -0,0 +1,28 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0046_repair_snapshot_permissions"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunSQL( + sql="CREATE INDEX IF NOT EXISTS archiveresult_status_snap_idx ON core_archiveresult (status, snapshot_id)", + reverse_sql="DROP INDEX IF EXISTS archiveresult_status_snap_idx", + ), + ], + state_operations=[ + migrations.AddIndex( + model_name="archiveresult", + index=models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"), + ), + ], + ), + migrations.RunSQL( + sql="ANALYZE core_archiveresult", + reverse_sql=migrations.RunSQL.noop, + ), + ] diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 2af3a4ed..f1250b26 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -1948,8 +1948,44 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW cache_key = f"result_icons:{self.pk}:{'compact' if compact_icons else 'full'}:{(self.downloaded_at or self.modified_at or self.created_at or self.bookmarked_at).timestamp()}" def calc_icons(): + if compact_icons and self.status == self.StatusChoices.STARTED: + progress_stats = getattr(self, "_icons_progress_stats", None) or self.get_progress_stats() + total = int(progress_stats.get("total") or 0) + succeeded = int(progress_stats.get("succeeded") or 0) + failed = int(progress_stats.get("failed") or 0) + skipped = int(progress_stats.get("skipped") or 0) + noresults = int(progress_stats.get("noresults") or 0) + running = int(progress_stats.get("running") or 0) + completed = succeeded + failed + skipped + noresults + percent = int((completed / total * 100) if total > 0 else 0) + return format_html( + '
' + '
' + '' + '{}/{} hooks' + "
" + '
' + '
' + "
" + '
' + "✓{} ✗{} ⏳{}" + "
" + "
", + completed, + total, + completed, + total, + percent, + succeeded, + failed, + running, + ) + + precomputed_archive_results = getattr(self, "_icons_archive_results", None) prefetched_cache = getattr(self, "_prefetched_objects_cache", {}) - if "archiveresult_set" in prefetched_cache: + if precomputed_archive_results is not None and compact_icons: + archive_results = {plugin: True for plugin in precomputed_archive_results} + elif "archiveresult_set" in prefetched_cache: archive_results = { r.plugin: r for r in self.archiveresult_set.all() @@ -1979,7 +2015,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW for plugin in ordered_plugins: result = archive_results.get(plugin) - existing = bool(result and result.status == "succeeded" and (compact_icons or result.output_files or result.output_str)) + existing = result is True or bool( + result and result.status == "succeeded" and (compact_icons or result.output_files or result.output_str), + ) if not existing: continue icon = mark_safe(get_plugin_icon(plugin)) @@ -2004,6 +2042,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW mark_safe(output), ) + if compact_icons and self.status == self.StatusChoices.STARTED: + return calc_icons() + cache_result = cache.get(cache_key) if cache_result: return cache_result @@ -3468,6 +3509,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M verbose_name_plural = "Archive Results Log" indexes = [ models.Index(fields=["snapshot", "status"], name="archiveresult_snap_status_idx"), + models.Index(fields=["status", "snapshot"], name="archiveresult_status_snap_idx"), models.Index(fields=["-start_ts", "-id"], name="archiveresult_start_idx"), ] constraints = [ diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 948a7015..2811d303 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1071,9 +1071,49 @@ class PublicIndexView(ListView): and search_mode_backend and getattr(context.get("paginator"), "count", 0) == 0, ) - for snapshot in context.get("object_list") or (): + snapshots = list(context.get("object_list") or ()) + icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots} + progress_by_snapshot: dict[str, dict[str, int]] = { + str(snapshot.id): { + "total": 0, + "succeeded": 0, + "failed": 0, + "running": 0, + "skipped": 0, + "noresults": 0, + } + for snapshot in snapshots + } + if icons_by_snapshot: + for snapshot_id, plugin, status in ( + ArchiveResult.objects.filter( + snapshot_id__in=icons_by_snapshot.keys(), + ) + .exclude(plugin="") + .values_list("snapshot_id", "plugin", "status") + .iterator(chunk_size=1000) + ): + snapshot_key = str(snapshot_id) + progress = progress_by_snapshot[snapshot_key] + progress["total"] += 1 + if status == ArchiveResult.StatusChoices.SUCCEEDED: + icons_by_snapshot[snapshot_key].add(plugin) + progress["succeeded"] += 1 + elif status == ArchiveResult.StatusChoices.FAILED: + progress["failed"] += 1 + elif status == ArchiveResult.StatusChoices.STARTED: + progress["running"] += 1 + elif status == ArchiveResult.StatusChoices.SKIPPED: + progress["skipped"] += 1 + elif status == ArchiveResult.StatusChoices.NORESULTS: + progress["noresults"] += 1 + + for snapshot in snapshots: snapshot._icons_compact = True + snapshot._icons_archive_results = icons_by_snapshot.get(str(snapshot.id), set()) + snapshot._icons_progress_stats = progress_by_snapshot.get(str(snapshot.id), {}) snapshot._is_archived_cached = bool(snapshot.downloaded_at or snapshot.status == Snapshot.StatusChoices.SEALED) + context["object_list"] = snapshots return context def get_queryset(self, **kwargs): diff --git a/archivebox/templates/admin/snapshots_grid.html b/archivebox/templates/admin/snapshots_grid.html index f2c190d4..3eea2a1e 100644 --- a/archivebox/templates/admin/snapshots_grid.html +++ b/archivebox/templates/admin/snapshots_grid.html @@ -130,7 +130,7 @@ grid-template-columns: 18px minmax(0, 1fr) auto; gap: 8px; align-items: center; - text-align: left; + text-align: center; } .cards .card .card-meta .card-date, .cards .card .card-meta .card-size { diff --git a/archivebox/templates/core/index_row.html b/archivebox/templates/core/index_row.html index b617edfe..6ad92662 100644 --- a/archivebox/templates/core/index_row.html +++ b/archivebox/templates/core/index_row.html @@ -69,13 +69,7 @@ - {% if compact_public_index %} - {% if link.num_outputs %} - {{ link.num_outputs }} output{{ link.num_outputs|pluralize }} - {% else %} - ... - {% endif %} - {% elif link.icons %} + {% if link.icons %} {{ link.icons }} {% else %} ... diff --git a/archivebox/templates/core/public_index.html b/archivebox/templates/core/public_index.html index 3f8cda01..cdff82ff 100644 --- a/archivebox/templates/core/public_index.html +++ b/archivebox/templates/core/public_index.html @@ -349,6 +349,10 @@ max-width: 200px; } + @keyframes snapshot-spin { + to { transform: rotate(360deg); } + } + .snapshot-size-cell { width: 102px; white-space: nowrap; diff --git a/archivebox/templates/static/admin.css b/archivebox/templates/static/admin.css index eb330227..01b9d32b 100755 --- a/archivebox/templates/static/admin.css +++ b/archivebox/templates/static/admin.css @@ -1893,7 +1893,8 @@ tbody .output-link:hover {opacity: 1;} .model-snapshot.change-list .cards .card .card-info.card-meta > a { grid-column: auto; - justify-self: start; + justify-self: center; + text-align: center; } .model-snapshot.change-list .cards .card .card-info .timestamp { diff --git a/etc/package.json b/etc/package.json index b8a32677..e51c4e2d 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.33rc79", + "version": "0.9.33rc80", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index fb224983..607e2f33 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.33rc79" +version = "0.9.33rc80" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.8", # EventBus API - "abxpkg>=1.11.111", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.114", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.114", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.112", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.115", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.115", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]