From 1105a18163c6997ee03c4c6b3aaf88feb8aac563 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 29 Aug 2026 14:04:54 -0700 Subject: [PATCH 01/52] fix: pack compact snapshot output cards --- archivebox/templates/core/snapshot.html | 7 +++---- archivebox/tests/test_ui_admin_snapshot.py | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index 8fb3cbb7..98119eaa 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -1054,10 +1054,9 @@ object-position: top center; } .thumb-grid { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(clamp(180px, 14vw, 250px), 1fr)); - gap: 6px; - align-content: start; + display: block; + column-width: clamp(180px, 14vw, 250px); + column-gap: 6px; width: 100%; max-width: 100%; margin-left: 0; diff --git a/archivebox/tests/test_ui_admin_snapshot.py b/archivebox/tests/test_ui_admin_snapshot.py index ac91a82f..5a4cdeb5 100644 --- a/archivebox/tests/test_ui_admin_snapshot.py +++ b/archivebox/tests/test_ui_admin_snapshot.py @@ -745,6 +745,15 @@ class TestSnapshotProgressStats: assert "event.preventDefault()" in rendered assert rendered.count(".on('click', handleSnapshotHeaderToggle)") == 1 + def test_compact_output_cards_pack_into_columns(self): + template = (REPO_ROOT / "archivebox" / "templates" / "core" / "snapshot.html").read_text() + thumb_grid_css = template.split(".thumb-grid {", 1)[1].split("}", 1)[0] + + assert "display: block;" in thumb_grid_css + assert "column-width: clamp(180px, 14vw, 250px);" in thumb_grid_css + assert "column-gap: 6px;" in thumb_grid_css + assert "grid-template-columns" not in thumb_grid_css + class TestSnapshotOutputDeletion: @staticmethod From e1434ab0a96e81446e18b38f2d8e77d3e8ab96bb Mon Sep 17 00:00:00 2001 From: ArchiveBox Release Bot Date: Sat, 29 Aug 2026 21:05:15 +0000 Subject: [PATCH 02/52] Bump release version to 0.9.35rc345 --- etc/package.json | 2 +- pyproject.toml | 4 ++-- uv.lock | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/package.json b/etc/package.json index 2566aae6..b5239c80 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc344", + "version": "0.9.35rc345", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index fab64fba..6a7ff704 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc344" +version = "0.9.35rc345" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc344" +current_version = "v0.9.35rc345" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index b053fe20..1129b953 100644 --- a/uv.lock +++ b/uv.lock @@ -120,7 +120,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc344" +version = "0.9.35rc345" source = { editable = "." } dependencies = [ { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, From 6ed11bd04f717bd22dd129a0b0a72d70061b5949 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 29 Aug 2026 14:10:25 -0700 Subject: [PATCH 03/52] fix: pack compact output cards in dense grid --- archivebox/templates/core/snapshot.html | 22 +++++++++++++--------- archivebox/tests/test_ui_admin_snapshot.py | 14 +++++++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index 98119eaa..04539175 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -1054,9 +1054,11 @@ object-position: top center; } .thumb-grid { - display: block; - column-width: clamp(180px, 14vw, 250px); - column-gap: 6px; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(clamp(180px, 14vw, 250px), 1fr)); + grid-auto-flow: row dense; + grid-auto-rows: 42px; + gap: 6px; width: 100%; max-width: 100%; margin-left: 0; @@ -1076,17 +1078,19 @@ display: inline-flex; flex-direction: column; align-items: stretch; + grid-row: span 3; break-inside: avoid; - margin-bottom: 6px; + margin-bottom: 0; vertical-align: top; content-visibility: auto; contain-intrinsic-size: 138px; } .thumb-card:has([data-compact]) { - height: 46px; - min-height: 46px; - max-height: 46px; - contain-intrinsic-size: 46px; + height: 42px; + min-height: 42px; + max-height: 42px; + grid-row: span 1; + contain-intrinsic-size: 42px; } .thumb-card .thumb-body { display: grid; @@ -1246,7 +1250,7 @@ } .thumb-card:has([data-compact]) .thumbnail-wrapper, .thumb-card:has([data-compact]) .thumbnail-wrapper.compact { - height: 24px; + height: 20px; flex: 0 0 auto; } .thumb-card:has([data-compact]) .thumb-body { diff --git a/archivebox/tests/test_ui_admin_snapshot.py b/archivebox/tests/test_ui_admin_snapshot.py index 5a4cdeb5..3970dd02 100644 --- a/archivebox/tests/test_ui_admin_snapshot.py +++ b/archivebox/tests/test_ui_admin_snapshot.py @@ -745,14 +745,18 @@ class TestSnapshotProgressStats: assert "event.preventDefault()" in rendered assert rendered.count(".on('click', handleSnapshotHeaderToggle)") == 1 - def test_compact_output_cards_pack_into_columns(self): + def test_compact_output_cards_pack_into_dense_grid_rows(self): template = (REPO_ROOT / "archivebox" / "templates" / "core" / "snapshot.html").read_text() thumb_grid_css = template.split(".thumb-grid {", 1)[1].split("}", 1)[0] + thumb_card_css = template.split(".thumb-card {", 1)[1].split("}", 1)[0] + compact_card_css = template.split(".thumb-card:has([data-compact]) {", 1)[1].split("}", 1)[0] - assert "display: block;" in thumb_grid_css - assert "column-width: clamp(180px, 14vw, 250px);" in thumb_grid_css - assert "column-gap: 6px;" in thumb_grid_css - assert "grid-template-columns" not in thumb_grid_css + assert "display: grid;" in thumb_grid_css + assert "grid-template-columns: repeat(auto-fit, minmax(clamp(180px, 14vw, 250px), 1fr));" in thumb_grid_css + assert "grid-auto-flow: row dense;" in thumb_grid_css + assert "grid-auto-rows: 42px;" in thumb_grid_css + assert "grid-row: span 3;" in thumb_card_css + assert "grid-row: span 1;" in compact_card_css class TestSnapshotOutputDeletion: From 0a1058097523c6e1e821aa70df437a29c2940f23 Mon Sep 17 00:00:00 2001 From: ArchiveBox Release Bot Date: Sat, 29 Aug 2026 21:14:33 +0000 Subject: [PATCH 04/52] Bump release version to 0.9.35rc346 --- etc/package.json | 2 +- pyproject.toml | 4 ++-- uv.lock | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/package.json b/etc/package.json index b5239c80..34ea2aa9 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc345", + "version": "0.9.35rc346", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 6a7ff704..5284c71b 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc345" +version = "0.9.35rc346" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc345" +current_version = "v0.9.35rc346" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index 1129b953..2abde30d 100644 --- a/uv.lock +++ b/uv.lock @@ -120,7 +120,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc345" +version = "0.9.35rc346" source = { editable = "." } dependencies = [ { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, From 3d64d14abf4058c8107150ea77c0121838dd89c4 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 29 Aug 2026 14:15:18 -0700 Subject: [PATCH 05/52] ci: keep public screenshots updating --- .github/workflows/deploy-publicsite.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy-publicsite.yml b/.github/workflows/deploy-publicsite.yml index 5fcc78ca..f46d1fe6 100644 --- a/.github/workflows/deploy-publicsite.yml +++ b/.github/workflows/deploy-publicsite.yml @@ -27,7 +27,6 @@ concurrency: jobs: deploy: - if: github.event_name != 'push' || github.event.head_commit.author.email != 'release-bot@archivebox.io' environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} From 1e2b9bd1aadffc5ba21466060939368ab36625f7 Mon Sep 17 00:00:00 2001 From: ArchiveBox Release Bot Date: Sat, 29 Aug 2026 21:15:42 +0000 Subject: [PATCH 06/52] Bump release version to 0.9.35rc347 --- etc/package.json | 2 +- pyproject.toml | 4 ++-- uv.lock | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/package.json b/etc/package.json index 34ea2aa9..bde61aed 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc346", + "version": "0.9.35rc347", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 5284c71b..cc03c7de 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc346" +version = "0.9.35rc347" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc346" +current_version = "v0.9.35rc347" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index 2abde30d..ca1fea96 100644 --- a/uv.lock +++ b/uv.lock @@ -120,7 +120,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc346" +version = "0.9.35rc347" source = { editable = "." } dependencies = [ { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, From 3fcb5700edcf1b7f1d7647946c55bfbd1a9059a8 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 29 Aug 2026 15:06:49 -0700 Subject: [PATCH 07/52] fix: place auxiliary snapshot cards before compact cards --- archivebox/templates/core/snapshot.html | 4 ++++ archivebox/tests/test_ui_admin_snapshot.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index 04539175..eded76bd 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -1085,11 +1085,15 @@ content-visibility: auto; contain-intrinsic-size: 138px; } + .thumb-card:not([data-plugin-name]) { + order: 1; + } .thumb-card:has([data-compact]) { height: 42px; min-height: 42px; max-height: 42px; grid-row: span 1; + order: 2; contain-intrinsic-size: 42px; } .thumb-card .thumb-body { diff --git a/archivebox/tests/test_ui_admin_snapshot.py b/archivebox/tests/test_ui_admin_snapshot.py index 3970dd02..15d6a05f 100644 --- a/archivebox/tests/test_ui_admin_snapshot.py +++ b/archivebox/tests/test_ui_admin_snapshot.py @@ -749,6 +749,7 @@ class TestSnapshotProgressStats: template = (REPO_ROOT / "archivebox" / "templates" / "core" / "snapshot.html").read_text() thumb_grid_css = template.split(".thumb-grid {", 1)[1].split("}", 1)[0] thumb_card_css = template.split(".thumb-card {", 1)[1].split("}", 1)[0] + auxiliary_card_css = template.split(".thumb-card:not([data-plugin-name]) {", 1)[1].split("}", 1)[0] compact_card_css = template.split(".thumb-card:has([data-compact]) {", 1)[1].split("}", 1)[0] assert "display: grid;" in thumb_grid_css @@ -756,7 +757,9 @@ class TestSnapshotProgressStats: assert "grid-auto-flow: row dense;" in thumb_grid_css assert "grid-auto-rows: 42px;" in thumb_grid_css assert "grid-row: span 3;" in thumb_card_css + assert "order: 1;" in auxiliary_card_css assert "grid-row: span 1;" in compact_card_css + assert "order: 2;" in compact_card_css class TestSnapshotOutputDeletion: From 2d37ab29e47065c2e52d6c594e1cf7a38a883dd8 Mon Sep 17 00:00:00 2001 From: ArchiveBox Release Bot Date: Sat, 29 Aug 2026 22:07:04 +0000 Subject: [PATCH 08/52] Bump release version to 0.9.35rc348 --- etc/package.json | 2 +- pyproject.toml | 4 ++-- uv.lock | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/package.json b/etc/package.json index bde61aed..f9f59b1b 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc347", + "version": "0.9.35rc348", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index cc03c7de..76ef9161 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc347" +version = "0.9.35rc348" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc347" +current_version = "v0.9.35rc348" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index ca1fea96..24f32851 100644 --- a/uv.lock +++ b/uv.lock @@ -120,7 +120,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc347" +version = "0.9.35rc348" source = { editable = "." } dependencies = [ { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, From 6906edc07658e7b58815b6543a771c329c595a10 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 29 Aug 2026 16:03:21 -0700 Subject: [PATCH 09/52] fix: make portable exports and screenshot deploys reliable --- .github/workflows/deploy-publicsite.yml | 16 + README.md | 4 +- archivebox/cli/archivebox_update.py | 26 +- archivebox/core/models.py | 274 ++++++++----- archivebox/core/templatetags/core_tags.py | 140 +++++-- archivebox/core/views.py | 166 +------- archivebox/templates/core/snapshot.html | 56 +-- archivebox/templates/core/static_index.html | 379 ++++++------------ archivebox/tests/migrations_helpers.py | 19 + archivebox/tests/test_cli_list.py | 35 ++ archivebox/tests/test_cli_run.py | 4 +- .../test_cli_update_reindex_snapshots.py | 12 +- archivebox/tests/test_migrations_04_to_09.py | 5 +- archivebox/tests/test_migrations_08_to_09.py | 16 +- archivebox/tests/test_ui_admin_snapshot.py | 68 +++- archivebox/tests/test_urls.py | 6 +- bin/generate_ui_screenshot_gallery.py | 51 ++- docs/Merging-Collections.md | 2 +- docs/Publishing-Your-Archive.md | 8 +- docs/Upgrading.md | 2 +- .../archivebox/archivebox.core.models.md | 8 - 21 files changed, 680 insertions(+), 617 deletions(-) diff --git a/.github/workflows/deploy-publicsite.yml b/.github/workflows/deploy-publicsite.yml index f46d1fe6..5316ec46 100644 --- a/.github/workflows/deploy-publicsite.yml +++ b/.github/workflows/deploy-publicsite.yml @@ -50,6 +50,22 @@ jobs: env: BASE_URL: http://archivebox.localhost:8000 + - name: Verify gallery was generated from this exact revision + run: | + uv run --no-cache --no-sync python - <<'PY' + import json + import os + import tomllib + from pathlib import Path + + provenance = json.loads(Path("publicsite/screenshots/build.json").read_text()) + version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"] + assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance + assert provenance["version"] == version, provenance + assert provenance["capture_count"] >= 3 * 35, provenance + assert len(provenance["files"]) == provenance["capture_count"], provenance + PY + - name: Setup Pages uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 diff --git a/README.md b/README.md index b5e86667..11caef69 100644 --- a/README.md +++ b/README.md @@ -682,7 +682,7 @@ It uses all available methods out-of-the-box, but you can disable extractors and Expand to see the full list of ways it saves each page... -data/archive/{Snapshot.id}/
+data/archive/users/{username}/snapshots/{YYYYMMDD}/{domain}/{Snapshot.id}/
  • Index: index.html & index.json HTML and JSON index files containing metadata and details
  • Title, Favicon, Headers Response headers, site favicon, and parsed site title
    • @@ -849,7 +849,7 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te ... -Each snapshot subfolder includes static metadata and plain extractor output files. ArchiveBox also maintains a backwards-compatible data/archive/TIMESTAMP symlink for each snapshot. +Each snapshot subfolder includes static metadata and plain extractor output files. Current releases do not create top-level data/archive/TIMESTAMP projections; legacy timestamp directories are migrated into the user-scoped tree by archivebox update --migrate-only.

    Learn More

      diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index 9cc9f1fa..d1c3b310 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -561,21 +561,20 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500 """ Drain old archive/ directories (0.8.x โ†’ 0.9.x migration). - Only processes real directories (skips symlinks - those are already migrated). + Removes obsolete timestamp symlinks and processes real legacy directories. For each old dir found in archive/: 1. Load or create DB snapshot 2. Trigger fs migration on save() to move to data/archive/users/{user}/... - 3. Leave symlink in archive/ pointing to new location + 3. Remove the old timestamp path after the verified migration commits - After this drains, archive/ should only contain symlinks and we can trust - 1:1 mapping between DB and filesystem. + After this drains, current snapshot data exists only under archive/users/. """ from archivebox.core.models import Snapshot from archivebox.config import CONSTANTS from archivebox.crawls.models import Crawl from django.utils import timezone - stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0} + stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0, "removed_symlinks": 0} crawl_url_lines: dict[str, list[str]] = {} crawl_url_sets: dict[str, set[str]] = {} dirty_crawl_ids: set[str] = set() @@ -606,8 +605,23 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500 if changed: Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now()) - # Scan for real directories only (skip symlinks - they're already migrated) + # Compatibility timestamp projections are harmful in portable exports and + # duplicate the obsolete 0.7.x-looking namespace without containing data. all_entries = list(os.scandir(archive_dir)) + for entry in all_entries: + entry_path = Path(entry.path) + if entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry_path): + try: + target_path = entry_path.resolve(strict=True) + points_into_current_layout = target_path.is_relative_to(CONSTANTS.USERS_DIR.resolve(strict=True)) + except OSError: + points_into_current_layout = False + + if points_into_current_layout: + entry_path.unlink(missing_ok=True) + stats["removed_symlinks"] += 1 + + # Scan real legacy directories only; these still contain data to migrate. entries = [ (e.stat().st_mtime, e.path) for e in all_entries diff --git a/archivebox/core/models.py b/archivebox/core/models.py index c0fd9042..b03cc527 100644 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -46,8 +46,6 @@ from archivebox.misc.util import ( sanitize_html_text, to_json, ts_to_date_str, - urldecode, - urlencode, validate_url, ) from archivebox.plugins.discovery import ( @@ -427,7 +425,7 @@ class SnapshotQuerySet(models.QuerySet): else {} ) - snapshot_dicts = [s.to_dict(extended=True) for s in self.iterator(chunk_size=500)] + snapshot_dicts = [s.to_dict(extended=True, static_export=True) for s in self.iterator(chunk_size=500)] if with_headers: output = { @@ -461,6 +459,13 @@ class SnapshotQuerySet(models.QuerySet): template = "static_index.html" if with_headers else "minimal_index.html" snapshot_list = list(self.iterator(chunk_size=500)) + for snapshot in snapshot_list: + outputs = snapshot.discover_outputs(include_filesystem_fallback=True) + output_paths = [str(output.get("path") or "") for output in outputs] + snapshot._public_preview_paths = [ + path for preferred in ("screenshot/screenshot.png", "screenshot.png") for path in output_paths if path == preferred + ] + snapshot._public_favicon_paths = [path for path in output_paths if path in ("favicon/favicon.ico", "favicon.ico")] return render_to_string( template, @@ -472,6 +477,8 @@ class SnapshotQuerySet(models.QuerySet): "time_updated": datetime.now(UTC).strftime("%Y-%m-%d %H:%M"), "links": snapshot_list, "FOOTER_INFO": config.FOOTER_INFO, + "STATIC_EXPORT": True, + "STATIC_EXPORT_DIR": CONSTANTS.DATA_DIR, }, ) @@ -984,7 +991,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW from django.db import transaction def finish_snapshot_save(): - self.ensure_legacy_archive_symlink() + self.remove_legacy_archive_symlink() self.ensure_crawl_symlink() crawl = Crawl.objects.filter(pk=self.crawl_id).first() if crawl is None: @@ -1261,9 +1268,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return (old_dir, new_dir) def _cleanup_old_migration_dir(self, old_dir: Path, new_dir: Path): - """ - Delete old directory and create symlink after successful migration. - """ + """Delete the old directory after its contents are verified at the new path.""" import logging import shutil @@ -1280,20 +1285,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW logging.getLogger("archivebox.migration").warning( f"Could not remove old migration directory {old_dir}: {e}", ) - return # Don't create symlink if cleanup failed + return - # Create backwards-compat symlink (after old dir is deleted) - symlink_path = old_dir # Same path as old_dir - if symlink_path.is_symlink(): - symlink_path.unlink() - - if not symlink_path.exists(): - try: - symlink_path.symlink_to(new_dir, target_is_directory=True) - except OSError as e: - logging.getLogger("archivebox.migration").warning( - f"Could not create symlink from {symlink_path} to {new_dir}: {e}", - ) + # Older migration runs may already have left a timestamp projection. + if old_dir.is_symlink(): + old_dir.unlink(missing_ok=True) # ========================================================================= # Path Calculation and Migration Helpers @@ -2273,7 +2269,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return calc_tags_str() return calc_tags_str() - def icons(self, path: str | None = None) -> str: + def icons(self, path: str | None = None, prefix: str = "/") -> str: """Generate HTML icons showing which extractor plugins have succeeded for this snapshot""" from django.utils.html import format_html @@ -2360,7 +2356,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW archive_path = path or self.archive_path output = "" - output_template = '{}' + output_template = '{}' # Get all plugins from hooks system (sorted by numeric prefix) all_plugins = self.__dict__.get("_icons_plugin_names") @@ -2388,6 +2384,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW embed_path = f"{plugin}/" if compact_icons else result.embed_path() output += format_html( output_template, + prefix, archive_path, embed_path, str(bool(existing)), @@ -2538,35 +2535,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW return current_path - def ensure_legacy_archive_symlink(self) -> None: - """Ensure the legacy archive/ path resolves to this snapshot.""" - import os - - legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp - target = Path(self.get_storage_path_for_version(self._fs_current_version())) - - if target == legacy_path: - return - - legacy_path.parent.mkdir(parents=True, exist_ok=True) - - if legacy_path.exists() or legacy_path.is_symlink(): - if legacy_path.is_symlink(): - try: - if legacy_path.resolve() == target.resolve(): - return - except OSError: - pass - legacy_path.unlink(missing_ok=True) - else: - return - - rel_target = os.path.relpath(target, legacy_path.parent) - try: - legacy_path.symlink_to(rel_target, target_is_directory=True) - except OSError: - return - def ensure_crawl_symlink(self, *, crawl_dir: Path | None = None, snapshot_dir: Path | None = None) -> None: """Ensure snapshot is symlinked under its crawl output directory.""" import os @@ -2606,6 +2574,21 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW except OSError: return + def remove_legacy_archive_symlink(self) -> None: + """Remove a stale archive/ compatibility projection.""" + legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp + current_path = self.get_storage_path_for_version(self._fs_current_version()) + if not legacy_path.is_symlink() or not current_path.exists(): + return + + try: + points_to_current_path = legacy_path.resolve(strict=True) == current_path.resolve(strict=True) + except OSError: + points_to_current_path = False + + if points_to_current_path: + legacy_path.unlink(missing_ok=True) + @cached_property def legacy_archive_path(self) -> str: return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{self.timestamp}" @@ -3437,7 +3420,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW # Serialization Methods # ========================================================================= - def to_dict(self, extended: bool = False) -> dict[str, Any]: + @property + def static_archive_path(self) -> str: + """Snapshot output path relative to the data root, for portable exports.""" + try: + return Path(self.output_dir).relative_to(CONSTANTS.DATA_DIR).as_posix() + except ValueError: + return Path(self.output_dir).as_posix() + + def to_dict(self, extended: bool = False, static_export: bool = False) -> dict[str, Any]: """Convert Snapshot to a dictionary (replacement for Link._asdict())""" from archivebox.core.routes_util import build_snapshot_url @@ -3468,8 +3459,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW "extension": self.extension, "is_static": self.is_static, "is_archived": self.is_archived, - "archive_path": self.archive_path, - "archive_url": build_snapshot_url(str(self.id), "index.html"), + "archive_path": self.static_archive_path if static_export else self.archive_path, + "archive_url": f"./{self.static_archive_path}/index.html" if static_export else build_snapshot_url(str(self.id), "index.html"), "output_dir": self.output_dir, "link_dir": self.output_dir, # backwards compatibility alias "archive_size": archive_size, @@ -3499,67 +3490,154 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW """Write JSON index file for this snapshot to its output directory""" output_dir = Path(out_dir) if out_dir is not None else self.output_dir path = output_dir / CONSTANTS.JSON_INDEX_FILENAME - atomic_write(str(path), self.to_dict(extended=True)) - - def write_html_details(self, out_dir: Path | str | None = None) -> None: - """Write HTML detail page for this snapshot to its output directory""" - from django.template.loader import render_to_string + atomic_write(str(path), self.to_dict(extended=True, static_export=True)) + def get_html_details_context(self, request=None, *, static_export_dir: Path | None = None) -> dict[str, Any]: + """Build the one context used by both served and on-disk snapshot pages.""" + from archivebox.config.common import get_request_config + from archivebox.core.permissions import get_snapshot_permissions from archivebox.core.widgets import TagEditorWidget from archivebox.misc.logging_util import printable_filesize + from archivebox.progressmonitor.views import progress_endpoint - output_dir = Path(out_dir) if out_dir is not None else self.output_dir - TITLE_LOADING_MSG = "Not yet archived..." + runtime_config = get_request_config(request) if request is not None else get_config() + self._runtime_config = runtime_config + snapshot_permissions = get_snapshot_permissions(self) + archive_results = list(self.archiveresult_set.all().order_by("start_ts")) + tags = list(self.tags.all()) + self.__dict__["_admin_archiveresults"] = archive_results + self.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags)) + self.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results) + self.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results) - preview_priority = [ - "singlefile", - "screenshot", - "wget", - "dom", - "pdf", - "readability", + hidden_card_plugins = {"archivedotorg", "favicon", "title"} + outputs = [ + output + for output in self.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results) + if (output.get("size") or 0) > 0 and output.get("name") not in hidden_card_plugins ] + outputs_by_name: dict[str, dict[str, Any]] = {} + result_ids_by_name: dict[str, list[str]] = {} + for output in outputs: + if output.get("result"): + result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id)) + current = outputs_by_name.get(output["name"]) + if current is None or (output.get("size") or 0) > (current.get("size") or 0): + outputs_by_name[output["name"]] = output + for name, output in outputs_by_name.items(): + output["result_ids"] = ",".join(result_ids_by_name.get(name, ())) - outputs = self.discover_outputs(include_filesystem_fallback=True) - loose_items, failed_items = self.get_detail_page_auxiliary_items(outputs) - outputs_by_plugin = {out["name"]: out for out in outputs} - output_size = sum(int(out.get("size") or 0) for out in outputs) - is_archived = bool(outputs or self.downloaded_at or self.status == self.StatusChoices.SEALED) - - best_preview_path = "about:blank" + hash_index = self.hashes_index + loose_items, failed_items = self.get_detail_page_auxiliary_items( + outputs, + hidden_card_plugins=hidden_card_plugins, + archive_results=archive_results, + ) + preview_priority = ("singlefile", "screenshot", "wget", "dom", "pdf", "readability") + output_order = {result_type: index for index, result_type in enumerate(outputs_by_name)} + ordered_outputs = sorted( + outputs_by_name.values(), + key=lambda output: ( + preview_priority.index(output["name"]) if output["name"] in preview_priority else len(preview_priority), + output_order.get(output["name"], len(output_order)), + ), + ) best_result = {"path": "about:blank", "result": None} - for plugin in preview_priority: - out = outputs_by_plugin.get(plugin) - if out and out.get("path"): - best_preview_path = str(out["path"]) - best_result = out + for result_type in preview_priority: + if result_type in outputs_by_name: + best_result = outputs_by_name[result_type] break + if best_result["path"] == "about:blank" and ordered_outputs: + best_result = ordered_outputs[0] - if best_preview_path == "about:blank" and outputs: - best_preview_path = str(outputs[0].get("path") or "about:blank") - best_result = outputs[0] + non_compact_outputs = [output for output in ordered_outputs if not output.get("is_compact") and not output.get("is_metadata")] + compact_outputs = [output for output in ordered_outputs if output.get("is_compact") or output.get("is_metadata")] + archive_dates = [result.start_ts for result in archive_results if result.start_ts] + output_size = sum(int(output.get("size") or 0) for output in ordered_outputs) + has_outputs = bool(ordered_outputs) + is_archived = has_outputs or self.status == self.StatusChoices.SEALED + snapshot_status = str(self.status or "").lower() + status_label_by_state = { + "queued": ("queued", "info"), + "started": ("running", "warning"), + "paused": ("paused", "default"), + "sealed": ("archived", "success"), + } + if has_outputs: + status_label, status_color = ("archived", "success") if is_archived else ("partial", "warning") + else: + status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger")) + + related_snapshots = list( + type(self) + .objects.filter(url=self.url) + .exclude(id=self.id) + .only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size") + .order_by("-bookmarked_at", "-created_at", "-timestamp")[:25], + ) + related_years_map: dict[int, list[Snapshot]] = {} + for snapshot in [self, *related_snapshots]: + snapshot_date = snapshot.bookmarked_at or snapshot.created_at or snapshot.downloaded_at + if snapshot_date: + related_years_map.setdefault(snapshot_date.year, []).append(snapshot) + related_years = [] + for year, snapshots in related_years_map.items(): + snapshots.sort( + key=lambda snapshot: snapshot.bookmarked_at or snapshot.created_at or snapshot.downloaded_at or timezone.now(), + reverse=True, + ) + related_years.append({"year": year, "latest": snapshots[0], "snapshots": snapshots}) + related_years.sort(key=lambda item: item["year"], reverse=True) + + warc_path = next( + (rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name), + "warc/", + ) + user = getattr(request, "user", None) tag_widget = TagEditorWidget() - context = { - **self.to_dict(extended=True), - "snapshot": self, - "title": htmldecode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)), - "url_str": htmldecode(urldecode(self.base_url)), - "archive_url": urlencode(f"warc/{self.timestamp}") or "about:blank", + return { + "id": str(self.id), + "snapshot_id": str(self.id), + "progress_endpoint": progress_endpoint("snapshot", self.id) if request is not None else "", + "progress_auto_expand": snapshot_status in {"queued", "started", "paused"}, + "url": self.url, + "archive_path": self.archive_path_from_db, + "title": htmldecode(self.resolved_title or (self.base_url if is_archived else "Not yet archived...")), "extension": self.extension or "html", "tags": self.tags_str() or "untagged", - "size": printable_filesize(output_size) if output_size else "pending", - "status": "archived" if is_archived else "not yet archived", - "status_color": "success" if is_archived else "danger", - "oldest_archive_date": ts_to_date_str(self.oldest_archive_date), - "best_preview_path": best_preview_path, + "size": printable_filesize(output_size) if output_size else "โ€”", + "status": status_label, + "status_color": status_color, + "snapshot_state": snapshot_status, + "has_outputs": has_outputs, + "snapshot_permissions": snapshot_permissions, + "snapshot_permissions_icon": {"public": "๐Ÿ‘ฅ", "unlisted": "๐Ÿ”—", "private": "๐Ÿ”’"}.get(snapshot_permissions, "๐Ÿ‘ฅ"), + "bookmarked_date": self.bookmarked_date, + "downloaded_datestr": self.downloaded_datestr, + "num_outputs": self.num_outputs, + "num_failures": self.num_failures, + "oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None), + "warc_path": warc_path, + "archiveresults": [*non_compact_outputs, *compact_outputs], "best_result": best_result, - "archiveresults": outputs, + "snapshot": self, + "CONFIG": runtime_config, + "related_snapshots": related_snapshots, + "related_years": related_years, "loose_items": loose_items, "failed_items": failed_items, - "related_snapshots": [], - "related_years": [], - "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in self.tags.all().order_by("name")], + "can_delete_outputs": bool(user and user.is_authenticated and user.is_active and user.is_superuser), + "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)], + "STATIC_EXPORT": static_export_dir is not None, + "STATIC_EXPORT_DIR": static_export_dir, } + + def write_html_details(self, out_dir: Path | str | None = None) -> None: + """Write the unified snapshot detail page with portable filesystem URLs.""" + from django.template.loader import render_to_string + + output_dir = Path(out_dir) if out_dir is not None else self.output_dir + context = self.get_html_details_context(static_export_dir=output_dir) rendered_html = render_to_string("core/snapshot.html", context) atomic_write(str(output_dir / CONSTANTS.HTML_INDEX_FILENAME), rendered_html) diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py index efcb9f34..c45e1d50 100644 --- a/archivebox/core/templatetags/core_tags.py +++ b/archivebox/core/templatetags/core_tags.py @@ -2,6 +2,7 @@ import os from html import unescape from pathlib import Path from typing import Any +from urllib.parse import quote from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target from django import template @@ -11,6 +12,7 @@ from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.text import Truncator +from archivebox.config import CONSTANTS from archivebox.core.routes_util import ( build_snapshot_url, get_admin_base_url, @@ -28,6 +30,7 @@ register = template.Library() _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log") _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif") +_STATIC_URL_SAFE = "/@-._~!$&'()*+,;=" _MEDIA_FILE_EXTS = { ".mp4", @@ -202,16 +205,70 @@ def _is_root_snapshot_output_path(raw_output_path: str | None) -> bool: return normalized in ("", ".", "./", "/", "index.html", "index.json") -def _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> str: - return build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config) +def _static_snapshot_base_url(context, snapshot) -> str: + start_dir = Path(context["STATIC_EXPORT_DIR"]) + relative_path = os.path.relpath(Path(snapshot.output_dir), start=start_dir).replace(os.sep, "/") + if relative_path == ".": + return "." + return f"./{quote(relative_path, safe=_STATIC_URL_SAFE)}" -def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None, plugin: str = "") -> str: +def _snapshot_base_url_for_context(context, snapshot) -> str: + if context.get("STATIC_EXPORT"): + return _static_snapshot_base_url(context, snapshot) + return get_snapshot_base_url( + str(_snapshot_id(snapshot)), + request=context.get("request"), + config=context.get("CONFIG"), + ) + + +def _snapshot_url_for_context(context, snapshot, path: str = "") -> str: + if not context.get("STATIC_EXPORT"): + return build_snapshot_url( + str(_snapshot_id(snapshot)), + path, + request=context.get("request"), + config=context.get("CONFIG"), + ) + + base_url = _static_snapshot_base_url(context, snapshot) + raw_path = str(path or "") + if not raw_path: + return base_url + path_part, separator, query = raw_path.lstrip("/").partition("?") + quoted_path = quote(path_part, safe=_STATIC_URL_SAFE) + suffix = f"?{query}" if separator else "" + return f"{base_url.rstrip('/')}/{quoted_path}{suffix}" + + +def _build_snapshot_files_url(snapshot_id: str, request=None, config=None, base_url: str | None = None) -> str: + return ( + f"{base_url.rstrip('/')}/?files=1" + if base_url + else build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config) + ) + + +def _build_snapshot_preview_url( + snapshot_id: str, + path: str = "", + request=None, + config=None, + plugin: str = "", + base_url: str | None = None, +) -> str: if path == "about:blank": return path if _is_root_snapshot_output_path(path): - return _build_snapshot_files_url(snapshot_id, request=request, config=config) - url = build_snapshot_url(str(snapshot_id), path, request=request, config=config) + return _build_snapshot_files_url(snapshot_id, request=request, config=config, base_url=base_url) + if base_url: + path_part, separator, query = str(path).lstrip("/").partition("?") + url = f"{base_url.rstrip('/')}/{quote(path_part, safe=_STATIC_URL_SAFE)}" + if separator: + url = f"{url}?{query}" + else: + url = build_snapshot_url(str(snapshot_id), path, request=request, config=config) path_parts = Path(path).parts plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "") has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False)) @@ -441,23 +498,35 @@ def web_base_url(context) -> str: return get_web_base_url(request=context.get("request"), config=context.get("CONFIG")) +@register.simple_tag(takes_context=True) +def static_export_root_url(context) -> str: + if not context.get("STATIC_EXPORT"): + return "" + relative_path = os.path.relpath(CONSTANTS.DATA_DIR, start=Path(context["STATIC_EXPORT_DIR"])) + relative_path = relative_path.replace(os.sep, "/") + return "./" if relative_path == "." else f"{quote(relative_path, safe=_STATIC_URL_SAFE)}/" + + @register.simple_tag(takes_context=True) def snapshot_base_url(context, snapshot) -> str: - snapshot_id = _snapshot_id(snapshot) - return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG")) + return _snapshot_base_url_for_context(context, snapshot) @register.simple_tag(takes_context=True) def snapshot_url(context, snapshot, path: str = "") -> str: - snapshot_id = _snapshot_id(snapshot) - return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG")) + return _snapshot_url_for_context(context, snapshot, path) @register.simple_tag(takes_context=True) def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str: snapshot_id = str(_snapshot_id(snapshot)) url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {}) - cache_key = (plugin, filename) + cache_key = ( + plugin, + filename, + bool(context.get("STATIC_EXPORT")), + str(context.get("STATIC_EXPORT_DIR") or ""), + ) if cache_key in url_cache: return url_cache[cache_key] @@ -493,7 +562,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0: continue output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}" - url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG")) + url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path) return url_cache[cache_key] url_cache[cache_key] = "" @@ -502,10 +571,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> @register.simple_tag(takes_context=True) def snapshot_index_row(context, link) -> str: - snapshot_id = str(_snapshot_id(link)) - request = context.get("request") - config = context.get("CONFIG") - snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config) + snapshot_base = _snapshot_base_url_for_context(context, link) status = getattr(link, "status", None) or "unknown" bookmarked_at = getattr(link, "bookmarked_at", None) @@ -534,22 +600,28 @@ def snapshot_index_row(context, link) -> str: tag_cell = '...' num_outputs = int(getattr(link, "num_outputs", 0) or 0) - icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "") + if context.get("STATIC_EXPORT") and callable(getattr(link, "icons", None)): + icons = link.icons(path=quote(link.static_archive_path, safe=_STATIC_URL_SAFE), prefix="./") + else: + icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "") icons_cell = str(icons) if icons else '...' archive_size = int(getattr(link, "archive_size", 0) or 0) size_cell = file_size(archive_size) if archive_size else '...' output_plural = "" if num_outputs == 1 else "s" + files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1" if is_pending: - preview_html = ( - '" - ) + preview_html = '' + if not context.get("STATIC_EXPORT"): + preview_html = ( + '" + ) elif "_public_preview_paths" in link.__dict__: preview_paths = list(getattr(link, "_public_preview_paths", []) or []) if preview_paths: - preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths] + preview_urls = [_snapshot_url_for_context(context, link, path) for path in preview_paths] preview_html = ( f' str: if "_public_favicon_paths" in link.__dict__: favicon_paths = list(getattr(link, "_public_favicon_paths", []) or []) if favicon_paths: - favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths] + favicon_urls = [_snapshot_url_for_context(context, link, path) for path in favicon_paths] favicon_html = ( f' str: - + {size_cell} {num_outputs} output{output_plural} @@ -660,6 +732,7 @@ def snapshot_preview_url(context, snapshot, path: str = "", result=None) -> str: request=context.get("request"), config=context.get("CONFIG"), plugin=plugin, + base_url=_snapshot_base_url_for_context(context, snapshot) if context.get("STATIC_EXPORT") else None, ) @@ -699,24 +772,16 @@ def plugin_card(context, result) -> str: # Use embed_path() for the display path raw_output_path = result.embed_path() or "" - output_url = build_snapshot_url( - str(result.snapshot_id), - raw_output_path or "", - request=context.get("request"), - config=context.get("CONFIG"), - ) + output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path or "") icon_html = get_plugin_icon(plugin) plugin_lower = (plugin or "").lower() media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0 media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else [] if media_files: - snapshot_id = str(result.snapshot_id) - request = context.get("request") - config = context.get("CONFIG") for item in media_files: path = item.get("path") or "" - item["url"] = build_snapshot_url(snapshot_id, path, request=request, config=config) if path else "" + item["url"] = _snapshot_url_for_context(context, result.snapshot, path) if path else "" output_lower = (raw_output_path or "").lower() force_text_preview = output_lower.endswith(_TEXT_PREVIEW_EXTS) @@ -794,12 +859,7 @@ def plugin_full(context, result) -> str: raw_output_path = result.embed_path() or "" if _is_root_snapshot_output_path(raw_output_path): return "" - output_url = build_snapshot_url( - str(result.snapshot_id), - raw_output_path, - request=context.get("request"), - config=context.get("CONFIG"), - ) + output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path) try: tpl = template.Template(template_str) diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 52907de3..c6849e91 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -21,7 +21,6 @@ from django.core.paginator import InvalidPage from django.db.models import Case, IntegerField, Q, Value, When from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict from django.shortcuts import redirect, render -from django.utils import timezone from django.utils.decorators import method_decorator from django.utils.html import format_html, format_html_join from django.utils.safestring import mark_safe @@ -54,7 +53,6 @@ from archivebox.core.permissions import ( can_view_snapshot, direct_snapshots_queryset, filter_personas_by_permissions, - get_snapshot_permissions, is_admin_user, public_snapshots_queryset, ) @@ -69,15 +67,12 @@ from archivebox.core.routes_util import ( host_matches, ) from archivebox.crawls.models import Crawl -from archivebox.misc.logging_util import printable_filesize from archivebox.misc.paginators import AcceleratedPaginator from archivebox.misc.serve_static import serve_static_with_byterange_support from archivebox.misc.util import ( base_url, filter_queryset_by_uuid_substring, - htmldecode, sanitize_html_text, - ts_to_date_str, urldecode, validate_url, without_fragment, @@ -85,7 +80,7 @@ from archivebox.misc.util import ( from archivebox.plugins.discovery import get_plugin_name, get_plugin_template from archivebox.plugins.forms import get_plugin_config_binary_urls from archivebox.plugins.views import get_config_definition_link -from archivebox.progressmonitor.views import live_progress_view, progress_endpoint +from archivebox.progressmonitor.views import live_progress_view from archivebox.search.config import ( get_search_mode, get_search_mode_backend, @@ -332,162 +327,11 @@ class SnapshotView(View): @staticmethod def render_live_index(request, snapshot): - TITLE_LOADING_MSG = "Not yet archived..." - from archivebox.core.widgets import TagEditorWidget - - # Reuse the middleware-attached config; never re-bootstrap from env + plugin - # schemas just to render a snapshot page (that pays ~30ms for no reason). - runtime_config = get_request_config(request) - snapshot._runtime_config = runtime_config - snapshot_permissions = get_snapshot_permissions(snapshot) - archive_results = list(snapshot.archiveresult_set.all().order_by("start_ts")) - tags = list(snapshot.tags.all()) - snapshot.__dict__["_admin_archiveresults"] = archive_results - snapshot.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags)) - snapshot.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results) - snapshot.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results) - hidden_card_plugins = {"archivedotorg", "favicon", "title"} - outputs = [ - out - for out in snapshot.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results) - if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins - ] - archiveresults = {} - result_ids_by_name = {} - for output in outputs: - if output.get("result"): - result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id)) - current = archiveresults.get(output["name"]) - if current is None or (output.get("size") or 0) > (current.get("size") or 0): - archiveresults[output["name"]] = output - for name, output in archiveresults.items(): - output["result_ids"] = ",".join(result_ids_by_name.get(name, ())) - hash_index = snapshot.hashes_index - loose_items, failed_items = snapshot.get_detail_page_auxiliary_items( - outputs, - hidden_card_plugins=hidden_card_plugins, - archive_results=archive_results, + return render( + template_name="core/snapshot.html", + request=request, + context=snapshot.get_html_details_context(request=request), ) - preview_priority = [ - "singlefile", - "screenshot", - "wget", - "dom", - "pdf", - "readability", - ] - preferred_types = tuple(preview_priority) - output_order = {result_type: index for index, result_type in enumerate(archiveresults.keys())} - - best_result = {"path": "about:blank", "result": None} - for result_type in preferred_types: - if result_type in archiveresults: - best_result = archiveresults[result_type] - break - - related_snapshots_qs = SnapshotView.find_snapshots_for_url( - snapshot.url, - allow_fallback=False, - ).only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size") - related_snapshots = list( - related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25], - ) - related_years_map: dict[int, list[Snapshot]] = {} - for snap in [snapshot, *related_snapshots]: - snap_dt = snap.bookmarked_at or snap.created_at or snap.downloaded_at - if not snap_dt: - continue - related_years_map.setdefault(snap_dt.year, []).append(snap) - related_years = [] - for year, snaps in related_years_map.items(): - snaps_sorted = sorted( - snaps, - key=lambda s: s.bookmarked_at or s.created_at or s.downloaded_at or timezone.now(), - reverse=True, - ) - related_years.append( - { - "year": year, - "latest": snaps_sorted[0], - "snapshots": snaps_sorted, - }, - ) - related_years.sort(key=lambda item: item["year"], reverse=True) - - warc_path = next( - (rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name), - "warc/", - ) - - ordered_outputs = sorted( - archiveresults.values(), - key=lambda r: ( - preferred_types.index(r["name"]) if r["name"] in preferred_types else len(preferred_types), - output_order.get(r["name"], len(output_order)), - ), - ) - if best_result["path"] == "about:blank" and ordered_outputs: - best_result = ordered_outputs[0] - non_compact_outputs = [out for out in ordered_outputs if not out.get("is_compact") and not out.get("is_metadata")] - compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")] - tag_widget = TagEditorWidget() - output_size = sum(int(out.get("size") or 0) for out in ordered_outputs) - archive_dates = [result.start_ts for result in archive_results if result.start_ts] - has_outputs = bool(ordered_outputs) - is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED - snapshot_status = str(snapshot.status or "").lower() - status_label_by_state = { - "queued": ("queued", "info"), - "started": ("running", "warning"), - "paused": ("paused", "default"), - "sealed": ("archived", "success"), - } - if has_outputs and not is_archived: - status_label, status_color = ("partial", "warning") - elif has_outputs: - status_label, status_color = ("archived", "success") - else: - status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger")) - - context = { - "id": str(snapshot.id), - "snapshot_id": str(snapshot.id), - "progress_endpoint": progress_endpoint("snapshot", snapshot.id), - "progress_auto_expand": snapshot_status in {"queued", "started", "paused"}, - "url": snapshot.url, - "archive_path": snapshot.archive_path_from_db, - "title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)), - "extension": snapshot.extension or "html", - "tags": snapshot.tags_str() or "untagged", - "size": printable_filesize(output_size) if output_size else "โ€”", - "status": status_label, - "status_color": status_color, - "snapshot_state": snapshot_status, - "has_outputs": has_outputs, - "snapshot_permissions": snapshot_permissions, - "snapshot_permissions_icon": { - "public": "๐Ÿ‘ฅ", - "unlisted": "๐Ÿ”—", - "private": "๐Ÿ”’", - }.get(snapshot_permissions, "๐Ÿ‘ฅ"), - "bookmarked_date": snapshot.bookmarked_date, - "downloaded_datestr": snapshot.downloaded_datestr, - "num_outputs": snapshot.num_outputs, - "num_failures": snapshot.num_failures, - "oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None), - "warc_path": warc_path, - "archiveresults": [*non_compact_outputs, *compact_outputs], - "best_result": best_result, - "snapshot": snapshot, # Pass the snapshot object for template tags - "CONFIG": runtime_config, - "related_snapshots": related_snapshots, - "related_years": related_years, - "loose_items": loose_items, - "failed_items": failed_items, - "can_delete_outputs": bool(request.user.is_authenticated and request.user.is_active and request.user.is_superuser), - "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)], - } - return render(template_name="core/snapshot.html", request=request, context=context) def get(self, request, path): snapshot = None diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html index eded76bd..1bc558b0 100644 --- a/archivebox/templates/core/snapshot.html +++ b/archivebox/templates/core/snapshot.html @@ -319,9 +319,9 @@ user-select: all; } .header-toggle { - line-height: 12px; - font-size: 70px; - margin-top: -13px; + line-height: 1; + font-size: 24px; + margin-top: 0; margin-left: 4px; } @container snapshot-header (max-width: 900px) { @@ -1475,11 +1475,13 @@
      + {% if not STATIC_EXPORT %} {% if snapshot_state == 'queued' or snapshot_state == 'started' or snapshot_state == 'paused' %}
      {% include "progressmonitor/progress_monitor.html" with progress_endpoint=progress_endpoint progress_scope="snapshot" %}
      {% endif %} + {% endif %} {% if has_outputs %} @@ -1539,7 +1539,7 @@ const frame = document.getElementById('main-frame') if (!frame) return const snapshotBaseUrlEarly = "{% snapshot_base_url snapshot %}" - const snapshotFilesUrlEarly = `${snapshotBaseUrlEarly}/?files=1` + const snapshotFilesUrlEarly = "{% if STATIC_EXPORT %}{% snapshot_url snapshot 'index.jsonl' %}{% else %}{% snapshot_base_url snapshot %}/?files=1{% endif %}" const defaultSrc = frame.dataset.defaultSrc || snapshotFilesUrlEarly const rawHash = window.location.hash ? window.location.hash.slice(1) : '' @@ -1783,13 +1783,13 @@
      - ๐Ÿ“ + ๐Ÿ“

      ๐Ÿ“ฆ Other files

      {% for item in loose_items %} {% if item.is_dir %} - ๐Ÿ“ {{item.name}} + ๐Ÿ“ {{item.name}} {% else %} ๐Ÿ“„ {{item.name}} {% endif %} @@ -1802,13 +1802,13 @@
      - ๐Ÿ“ + ๐Ÿ“

      โš ๏ธ Failed

      {% for item in failed_items %} {% if item.is_dir %} - ๐Ÿ“ {{item.name}} + ๐Ÿ“ {{item.name}} {% else %} ๐Ÿ“„ {{item.name}} {% endif %} @@ -1825,7 +1825,7 @@