release: archivebox 0.9.32rc36

Dockerfile

@@ -251,6 +251,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
 # installed into LIB_DIR below by archivebox init --install and resolved from
 # LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
 ENV PERSONAS_DIR=/data/personas \
+    CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \
     CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
     CHROME_HEADLESS=true \
     CHROME_SANDBOX=false \
@@ -309,17 +310,28 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
     --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
     --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/opt/archivebox/lib,sharing=locked,id=archivebox-lib-$TARGETARCH$TARGETVARIANT \
     echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
+    && export PERSONAS_DIR="$LIB_DIR/personas" \
+    && export CHROME_EXTENSIONS_DIR="$LIB_DIR/chrome_extensions" \
+    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
+    && mkdir -p "$LIB_DIR" "$LIB_DIR/chrome_extensions" \
     && apt-get update -qq \
     && if [ "$TARGETARCH" = "arm64" ]; then \
         abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
         abxpkg install --no-cache --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
     fi \
     && PUID=0 PGID=0 abx-dl plugins --install \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
+    && mkdir -p /opt/archivebox/lib-layer \
+    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
     && rm -rf /var/lib/apt/lists/* \
-    && (chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR"/personas 2>/dev/null || true) \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
+
+RUN rm -rf "$LIB_DIR" \
+    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
     && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
 
 # Install ArchiveBox Python package from the checked-out source.

archivebox/core/views.py

@@ -18,11 +18,12 @@ from django.utils.safestring import mark_safe
 from django.views import View
 from django.views.generic.list import ListView
 from django.views.generic import FormView
-from django.db.models import CharField, Count, Q, Prefetch
+from django.db.models import CharField, Count, Q, Prefetch, Sum
 from django.db.models.functions import Cast
 from django.contrib import messages
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.gzip import gzip_page
 from django.utils.decorators import method_decorator
 
 from admin_data_views.typing import TableContext, ItemContext, SectionData
@@ -1294,6 +1295,7 @@ class HealthCheckView(View):
         return HttpResponse("OK", content_type="text/plain", status=200)
 
 
+@gzip_page
 def live_progress_view(request):
     """Simple JSON endpoint for live progress status - used by admin progress monitor."""
     try:
@@ -1436,6 +1438,10 @@ def live_progress_view(request):
                 config=request_config,
             )
 
+        def snapshot_display_url(url: str) -> str:
+            url = str(url or "")
+            return url if len(url) <= 96 else f"{url[:93]}..."
+
         machine_id = Machine.current().id
         orchestrator_proc = (
             Process.objects.filter(
@@ -1555,6 +1561,27 @@ def live_progress_view(request):
         snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
         cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
         crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
+        active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
+        if active_crawl_ids:
+            for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
+                snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
+
+            for row in (
+                active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
+                .values("crawl_id")
+                .annotate(count=Count("id"))
+            ):
+                cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
+
+            for row in (
+                archiveresult_scope.filter(
+                    snapshot__crawl_id__in=active_crawl_ids,
+                    snapshot__status=Snapshot.StatusChoices.SEALED,
+                )
+                .values("snapshot__crawl_id")
+                .annotate(size=Sum("output_size"))
+            ):
+                crawl_output_sizes_by_crawl[str(row["snapshot__crawl_id"])] = int(row["size"] or 0)
 
         crawl_process_pids: dict[str, int] = {}
         snapshot_process_pids: dict[str, int] = {}
@@ -1568,7 +1595,6 @@ def live_progress_view(request):
             modified_at__gte=recently_cancelled_after,
         )
         crawls_by_id = {str(crawl["id"]): crawl for crawl in active_crawls_list}
-        active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
         snapshots = list(
             active_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED)
             .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField()))
@@ -1616,12 +1642,6 @@ def live_progress_view(request):
         snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots}
         displayed_snapshots_by_crawl: dict[str, list[Snapshot]] = {str(crawl_id): [] for crawl_id in active_crawl_ids}
         for snapshot in snapshots:
-            crawl_snapshot_counts = snapshot_counts_by_crawl.setdefault(str(snapshot["crawl_id"]), {})
-            crawl_snapshot_counts[snapshot["status"]] = crawl_snapshot_counts.get(snapshot["status"], 0) + 1
-            if snapshot["status"] == Snapshot.StatusChoices.SEALED and not snapshot.get("downloaded_at"):
-                cancelled_snapshot_counts_by_crawl[str(snapshot["crawl_id"])] = (
-                    cancelled_snapshot_counts_by_crawl.get(str(snapshot["crawl_id"]), 0) + 1
-                )
             crawl_snapshots = displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), [])
             crawl_snapshots.append(snapshot)
         displayed_snapshot_ids = [
@@ -1933,19 +1953,18 @@ def live_progress_view(request):
                     worker_state = "stalled" if orchestrator_running else "crashed"
 
                 if snapshot["status"] == Snapshot.StatusChoices.QUEUED and not snapshot_process_pids.get(str(snapshot["id"])):
-                    active_snapshots_for_crawl.append(
-                        [
-                            str(snapshot["id"]),
-                            snapshot["url"],
-                            snapshot_title,
-                            snapshot["status"],
-                        ],
-                    )
+                    compact_snapshot = [
+                        str(snapshot["id"]),
+                        snapshot_display_url(snapshot["url"]),
+                    ]
+                    if snapshot_title:
+                        compact_snapshot.append(snapshot_title)
+                    active_snapshots_for_crawl.append(compact_snapshot)
                     continue
 
                 snapshot_payload = {
                     "id": str(snapshot["id"]),
-                    "url": snapshot["url"],
+                    "url": snapshot_display_url(snapshot["url"]),
                     "title": snapshot_title,
                     "status": snapshot["status"],
                     "worker_state": worker_state,
@@ -1985,7 +2004,7 @@ def live_progress_view(request):
             persona_name = persona_details["name"] if persona_details else str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default")
             persona_details = persona_details or persona_details_by_name.get(persona_name)
             crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
-            avg_snapshot_size = int(crawl_output_size / total_snapshots) if total_snapshots else 0
+            avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
 
             # Check if retry_at is in the future (would prevent worker from claiming)
             retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False

archivebox/templates/admin/progress_monitor.html

@@ -903,7 +903,7 @@
                 id: snapshot[0],
                 url: snapshot[1],
                 title: snapshot[2],
-                status: snapshot[3],
+                status: snapshot[3] || 'queued',
             };
         }
         const statusIcon = snapshot.status === 'started' ? '▤' : '▢';
@@ -1080,7 +1080,7 @@
             `;
         }
 
-        // Show crawl-scale limits and approximate output sizes from DB metadata.
+        // Show crawl-scale limits and sealed output sizes from DB metadata.
         const currentUrlCount = Math.max(crawl.total_snapshots || 0, crawl.urls_count || 0);
         const maxUrlsText = (crawl.max_urls || 0) > 0 ? crawl.max_urls : 'unlimited';
         const urlLimitText = `${currentUrlCount} / ${maxUrlsText}`;
@@ -1088,7 +1088,7 @@
         const snapshotSizeLimitText = `${crawl.avg_snapshot_size_display || '0 B'} / ${crawl.max_snapshot_size_display || 'unlimited'}`;
         const crawlBadges = [
             crawlBadge('persona', 'Persona Config', crawl.persona || 'Default', crawl.persona_admin_url, 'Edit persona config'),
-            crawlBadge('limit', 'depth', crawl.max_depth || 0, null, ''),
+            crawlBadge('limit', 'depth', crawl.max_depth || 0, adminFieldUrl('max_depth'), 'Edit crawl depth'),
             crawlBadge('limit', 'urls', urlLimitText, adminFieldUrl('max_urls'), 'Edit max URLs'),
             crawlBadge('size', 'crawl size', crawlSizeLimitText, adminFieldUrl('crawl_max_size'), 'Edit max crawl size'),
             crawlBadge('size', 'avg snap', snapshotSizeLimitText, adminFieldUrl('snapshot_max_size'), 'Edit max snapshot size'),

bin/build_docker.sh

@@ -96,6 +96,7 @@ check_platforms || (recreate_builder && check_platforms) || exit 1
 
 
 echo "[+] Building archivebox:$VERSION docker image..."
+mkdir -p "$HOME/.cache/docker/archivebox"
 # docker builder prune
 # docker build . --no-cache -t archivebox-dev \
 # replace --load with --push to deploy
@@ -104,4 +105,8 @@ if [[ "$SELECTED_PLATFORMS" == *,* ]]; then
     echo "[X] --load only supports a single platform. Use bin/release_docker.sh or set DOCKER_PLATFORMS to one platform." >&2
     exit 1
 fi
-docker buildx build --platform "$SELECTED_PLATFORMS" --load . ${FULL_TAG_NAMES[@]}
+docker buildx build \
+    --platform "$SELECTED_PLATFORMS" \
+    --cache-from type=local,src="$HOME/.cache/docker/archivebox" \
+    --cache-to type=local,compression=zstd,mode=min,oci-mediatypes=true,dest="$HOME/.cache/docker/archivebox" \
+    --load . ${FULL_TAG_NAMES[@]}

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.32rc35",
+  "version": "0.9.32rc36",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.32rc35"
+version = "0.9.32rc36"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -80,9 +80,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus==2.5.7",          # EventBus API
-    "abxpkg>=1.11.19",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.11.22",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.11.22",       # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.11.22",        # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.11.25",  # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.11.25",       # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]