From 3f31223ba82264f834ce7a001323a865b2011199 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 27 May 2026 16:24:09 -0700 Subject: [PATCH] release: archivebox 0.9.32rc36 --- Dockerfile | 18 +++++- archivebox/core/views.py | 55 +++++++++++++------ .../templates/admin/progress_monitor.html | 6 +- bin/build_docker.sh | 7 ++- etc/package.json | 2 +- pyproject.toml | 8 +-- 6 files changed, 66 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c8b1ae3..5c7cd6ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -251,6 +251,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T # installed into LIB_DIR below by archivebox init --install and resolved from # LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH. ENV PERSONAS_DIR=/data/personas \ + CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \ CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ CHROME_HEADLESS=true \ CHROME_SANDBOX=false \ @@ -309,17 +310,28 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \ --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \ + --mount=type=cache,target=/opt/archivebox/lib,sharing=locked,id=archivebox-lib-$TARGETARCH$TARGETVARIANT \ echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \ + && export PERSONAS_DIR="$LIB_DIR/personas" \ + && export CHROME_EXTENSIONS_DIR="$LIB_DIR/chrome_extensions" \ + && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \ + && mkdir -p "$LIB_DIR" "$LIB_DIR/chrome_extensions" \ && apt-get update -qq \ && if [ "$TARGETARCH" = "arm64" ]; then \ abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \ abxpkg install --no-cache --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \ fi \ && PUID=0 PGID=0 abx-dl plugins --install \ - && find "$LIB_DIR" "$DATA_DIR"/personas -type d -name __pycache__ -prune -exec rm -rf {} + \ - && find "$LIB_DIR" "$DATA_DIR"/personas -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \ + && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ + && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \ + && mkdir -p /opt/archivebox/lib-layer \ + && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \ && rm -rf /var/lib/apt/lists/* \ - && (chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR"/personas 2>/dev/null || true) \ + && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer + +RUN rm -rf "$LIB_DIR" \ + && mv /opt/archivebox/lib-layer "$LIB_DIR" \ && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" # Install ArchiveBox Python package from the checked-out source. diff --git a/archivebox/core/views.py b/archivebox/core/views.py index cbe30201..9654e6aa 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -18,11 +18,12 @@ from django.utils.safestring import mark_safe from django.views import View from django.views.generic.list import ListView from django.views.generic import FormView -from django.db.models import CharField, Count, Q, Prefetch +from django.db.models import CharField, Count, Q, Prefetch, Sum from django.db.models.functions import Cast from django.contrib import messages from django.contrib.auth.mixins import UserPassesTestMixin from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.gzip import gzip_page from django.utils.decorators import method_decorator from admin_data_views.typing import TableContext, ItemContext, SectionData @@ -1294,6 +1295,7 @@ class HealthCheckView(View): return HttpResponse("OK", content_type="text/plain", status=200) +@gzip_page def live_progress_view(request): """Simple JSON endpoint for live progress status - used by admin progress monitor.""" try: @@ -1436,6 +1438,10 @@ def live_progress_view(request): config=request_config, ) + def snapshot_display_url(url: str) -> str: + url = str(url or "") + return url if len(url) <= 96 else f"{url[:93]}..." + machine_id = Machine.current().id orchestrator_proc = ( Process.objects.filter( @@ -1555,6 +1561,27 @@ def live_progress_view(request): snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids} cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids} crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids} + active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids) + if active_crawl_ids: + for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")): + snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"] + + for row in ( + active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True) + .values("crawl_id") + .annotate(count=Count("id")) + ): + cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"] + + for row in ( + archiveresult_scope.filter( + snapshot__crawl_id__in=active_crawl_ids, + snapshot__status=Snapshot.StatusChoices.SEALED, + ) + .values("snapshot__crawl_id") + .annotate(size=Sum("output_size")) + ): + crawl_output_sizes_by_crawl[str(row["snapshot__crawl_id"])] = int(row["size"] or 0) crawl_process_pids: dict[str, int] = {} snapshot_process_pids: dict[str, int] = {} @@ -1568,7 +1595,6 @@ def live_progress_view(request): modified_at__gte=recently_cancelled_after, ) crawls_by_id = {str(crawl["id"]): crawl for crawl in active_crawls_list} - active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids) snapshots = list( active_snapshot_scope.filter(status=Snapshot.StatusChoices.QUEUED) .annotate(id_str=Cast("id", CharField()), crawl_id_str=Cast("crawl_id", CharField())) @@ -1616,12 +1642,6 @@ def live_progress_view(request): snapshots_by_id = {str(snapshot["id"]): snapshot for snapshot in snapshots} displayed_snapshots_by_crawl: dict[str, list[Snapshot]] = {str(crawl_id): [] for crawl_id in active_crawl_ids} for snapshot in snapshots: - crawl_snapshot_counts = snapshot_counts_by_crawl.setdefault(str(snapshot["crawl_id"]), {}) - crawl_snapshot_counts[snapshot["status"]] = crawl_snapshot_counts.get(snapshot["status"], 0) + 1 - if snapshot["status"] == Snapshot.StatusChoices.SEALED and not snapshot.get("downloaded_at"): - cancelled_snapshot_counts_by_crawl[str(snapshot["crawl_id"])] = ( - cancelled_snapshot_counts_by_crawl.get(str(snapshot["crawl_id"]), 0) + 1 - ) crawl_snapshots = displayed_snapshots_by_crawl.setdefault(str(snapshot["crawl_id"]), []) crawl_snapshots.append(snapshot) displayed_snapshot_ids = [ @@ -1933,19 +1953,18 @@ def live_progress_view(request): worker_state = "stalled" if orchestrator_running else "crashed" if snapshot["status"] == Snapshot.StatusChoices.QUEUED and not snapshot_process_pids.get(str(snapshot["id"])): - active_snapshots_for_crawl.append( - [ - str(snapshot["id"]), - snapshot["url"], - snapshot_title, - snapshot["status"], - ], - ) + compact_snapshot = [ + str(snapshot["id"]), + snapshot_display_url(snapshot["url"]), + ] + if snapshot_title: + compact_snapshot.append(snapshot_title) + active_snapshots_for_crawl.append(compact_snapshot) continue snapshot_payload = { "id": str(snapshot["id"]), - "url": snapshot["url"], + "url": snapshot_display_url(snapshot["url"]), "title": snapshot_title, "status": snapshot["status"], "worker_state": worker_state, @@ -1985,7 +2004,7 @@ def live_progress_view(request): persona_name = persona_details["name"] if persona_details else str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default") persona_details = persona_details or persona_details_by_name.get(persona_name) crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0) - avg_snapshot_size = int(crawl_output_size / total_snapshots) if total_snapshots else 0 + avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0 # Check if retry_at is in the future (would prevent worker from claiming) retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html index b295abd6..c703bc21 100644 --- a/archivebox/templates/admin/progress_monitor.html +++ b/archivebox/templates/admin/progress_monitor.html @@ -903,7 +903,7 @@ id: snapshot[0], url: snapshot[1], title: snapshot[2], - status: snapshot[3], + status: snapshot[3] || 'queued', }; } const statusIcon = snapshot.status === 'started' ? '▤' : '▢'; @@ -1080,7 +1080,7 @@ `; } - // Show crawl-scale limits and approximate output sizes from DB metadata. + // Show crawl-scale limits and sealed output sizes from DB metadata. const currentUrlCount = Math.max(crawl.total_snapshots || 0, crawl.urls_count || 0); const maxUrlsText = (crawl.max_urls || 0) > 0 ? crawl.max_urls : 'unlimited'; const urlLimitText = `${currentUrlCount} / ${maxUrlsText}`; @@ -1088,7 +1088,7 @@ const snapshotSizeLimitText = `${crawl.avg_snapshot_size_display || '0 B'} / ${crawl.max_snapshot_size_display || 'unlimited'}`; const crawlBadges = [ crawlBadge('persona', 'Persona Config', crawl.persona || 'Default', crawl.persona_admin_url, 'Edit persona config'), - crawlBadge('limit', 'depth', crawl.max_depth || 0, null, ''), + crawlBadge('limit', 'depth', crawl.max_depth || 0, adminFieldUrl('max_depth'), 'Edit crawl depth'), crawlBadge('limit', 'urls', urlLimitText, adminFieldUrl('max_urls'), 'Edit max URLs'), crawlBadge('size', 'crawl size', crawlSizeLimitText, adminFieldUrl('crawl_max_size'), 'Edit max crawl size'), crawlBadge('size', 'avg snap', snapshotSizeLimitText, adminFieldUrl('snapshot_max_size'), 'Edit max snapshot size'), diff --git a/bin/build_docker.sh b/bin/build_docker.sh index b39a4ff8..cbc4606f 100755 --- a/bin/build_docker.sh +++ b/bin/build_docker.sh @@ -96,6 +96,7 @@ check_platforms || (recreate_builder && check_platforms) || exit 1 echo "[+] Building archivebox:$VERSION docker image..." +mkdir -p "$HOME/.cache/docker/archivebox" # docker builder prune # docker build . --no-cache -t archivebox-dev \ # replace --load with --push to deploy @@ -104,4 +105,8 @@ if [[ "$SELECTED_PLATFORMS" == *,* ]]; then echo "[X] --load only supports a single platform. Use bin/release_docker.sh or set DOCKER_PLATFORMS to one platform." >&2 exit 1 fi -docker buildx build --platform "$SELECTED_PLATFORMS" --load . ${FULL_TAG_NAMES[@]} +docker buildx build \ + --platform "$SELECTED_PLATFORMS" \ + --cache-from type=local,src="$HOME/.cache/docker/archivebox" \ + --cache-to type=local,compression=zstd,mode=min,oci-mediatypes=true,dest="$HOME/.cache/docker/archivebox" \ + --load . ${FULL_TAG_NAMES[@]} diff --git a/etc/package.json b/etc/package.json index 4f5d9e31..116cee8b 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.32rc35", + "version": "0.9.32rc36", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index ba135e8b..1e16267c 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.32rc35" +version = "0.9.32rc36" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -80,9 +80,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.7", # EventBus API - "abxpkg>=1.11.19", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.22", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.22", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.22", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.25", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.25", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]