compatibility projection."""
+ legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
+ current_path = self.get_storage_path_for_version(self._fs_current_version())
+ if not legacy_path.is_symlink() or not current_path.exists():
+ return
+
+ try:
+ points_to_current_path = legacy_path.resolve(strict=True) == current_path.resolve(strict=True)
+ except OSError:
+ points_to_current_path = False
+
+ if points_to_current_path:
+ legacy_path.unlink(missing_ok=True)
+
@cached_property
def legacy_archive_path(self) -> str:
return f"{CONSTANTS.ARCHIVE_DIR_NAME}/{self.timestamp}"
@@ -3437,7 +3420,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
# Serialization Methods
# =========================================================================
- def to_dict(self, extended: bool = False) -> dict[str, Any]:
+ @property
+ def static_archive_path(self) -> str:
+ """Snapshot output path relative to the data root, for portable exports."""
+ try:
+ return Path(self.output_dir).relative_to(CONSTANTS.DATA_DIR).as_posix()
+ except ValueError:
+ return Path(self.output_dir).as_posix()
+
+ def to_dict(self, extended: bool = False, static_export: bool = False) -> dict[str, Any]:
"""Convert Snapshot to a dictionary (replacement for Link._asdict())"""
from archivebox.core.routes_util import build_snapshot_url
@@ -3468,8 +3459,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"extension": self.extension,
"is_static": self.is_static,
"is_archived": self.is_archived,
- "archive_path": self.archive_path,
- "archive_url": build_snapshot_url(str(self.id), "index.html"),
+ "archive_path": self.static_archive_path if static_export else self.archive_path,
+ "archive_url": f"./{self.static_archive_path}/index.html" if static_export else build_snapshot_url(str(self.id), "index.html"),
"output_dir": self.output_dir,
"link_dir": self.output_dir, # backwards compatibility alias
"archive_size": archive_size,
@@ -3499,67 +3490,154 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""Write JSON index file for this snapshot to its output directory"""
output_dir = Path(out_dir) if out_dir is not None else self.output_dir
path = output_dir / CONSTANTS.JSON_INDEX_FILENAME
- atomic_write(str(path), self.to_dict(extended=True))
-
- def write_html_details(self, out_dir: Path | str | None = None) -> None:
- """Write HTML detail page for this snapshot to its output directory"""
- from django.template.loader import render_to_string
+ atomic_write(str(path), self.to_dict(extended=True, static_export=True))
+ def get_html_details_context(self, request=None, *, static_export_dir: Path | None = None) -> dict[str, Any]:
+ """Build the one context used by both served and on-disk snapshot pages."""
+ from archivebox.config.common import get_request_config
+ from archivebox.core.permissions import get_snapshot_permissions
from archivebox.core.widgets import TagEditorWidget
from archivebox.misc.logging_util import printable_filesize
+ from archivebox.progressmonitor.views import progress_endpoint
- output_dir = Path(out_dir) if out_dir is not None else self.output_dir
- TITLE_LOADING_MSG = "Not yet archived..."
+ runtime_config = get_request_config(request) if request is not None else get_config()
+ self._runtime_config = runtime_config
+ snapshot_permissions = get_snapshot_permissions(self)
+ archive_results = list(self.archiveresult_set.all().order_by("start_ts"))
+ tags = list(self.tags.all())
+ self.__dict__["_admin_archiveresults"] = archive_results
+ self.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags))
+ self.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results)
+ self.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results)
- preview_priority = [
- "singlefile",
- "screenshot",
- "wget",
- "dom",
- "pdf",
- "readability",
+ hidden_card_plugins = {"archivedotorg", "favicon", "title"}
+ outputs = [
+ output
+ for output in self.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results)
+ if (output.get("size") or 0) > 0 and output.get("name") not in hidden_card_plugins
]
+ outputs_by_name: dict[str, dict[str, Any]] = {}
+ result_ids_by_name: dict[str, list[str]] = {}
+ for output in outputs:
+ if output.get("result"):
+ result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id))
+ current = outputs_by_name.get(output["name"])
+ if current is None or (output.get("size") or 0) > (current.get("size") or 0):
+ outputs_by_name[output["name"]] = output
+ for name, output in outputs_by_name.items():
+ output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
- outputs = self.discover_outputs(include_filesystem_fallback=True)
- loose_items, failed_items = self.get_detail_page_auxiliary_items(outputs)
- outputs_by_plugin = {out["name"]: out for out in outputs}
- output_size = sum(int(out.get("size") or 0) for out in outputs)
- is_archived = bool(outputs or self.downloaded_at or self.status == self.StatusChoices.SEALED)
-
- best_preview_path = "about:blank"
+ hash_index = self.hashes_index
+ loose_items, failed_items = self.get_detail_page_auxiliary_items(
+ outputs,
+ hidden_card_plugins=hidden_card_plugins,
+ archive_results=archive_results,
+ )
+ preview_priority = ("singlefile", "screenshot", "wget", "dom", "pdf", "readability")
+ output_order = {result_type: index for index, result_type in enumerate(outputs_by_name)}
+ ordered_outputs = sorted(
+ outputs_by_name.values(),
+ key=lambda output: (
+ preview_priority.index(output["name"]) if output["name"] in preview_priority else len(preview_priority),
+ output_order.get(output["name"], len(output_order)),
+ ),
+ )
best_result = {"path": "about:blank", "result": None}
- for plugin in preview_priority:
- out = outputs_by_plugin.get(plugin)
- if out and out.get("path"):
- best_preview_path = str(out["path"])
- best_result = out
+ for result_type in preview_priority:
+ if result_type in outputs_by_name:
+ best_result = outputs_by_name[result_type]
break
+ if best_result["path"] == "about:blank" and ordered_outputs:
+ best_result = ordered_outputs[0]
- if best_preview_path == "about:blank" and outputs:
- best_preview_path = str(outputs[0].get("path") or "about:blank")
- best_result = outputs[0]
+ non_compact_outputs = [output for output in ordered_outputs if not output.get("is_compact") and not output.get("is_metadata")]
+ compact_outputs = [output for output in ordered_outputs if output.get("is_compact") or output.get("is_metadata")]
+ archive_dates = [result.start_ts for result in archive_results if result.start_ts]
+ output_size = sum(int(output.get("size") or 0) for output in ordered_outputs)
+ has_outputs = bool(ordered_outputs)
+ is_archived = has_outputs or self.status == self.StatusChoices.SEALED
+ snapshot_status = str(self.status or "").lower()
+ status_label_by_state = {
+ "queued": ("queued", "info"),
+ "started": ("running", "warning"),
+ "paused": ("paused", "default"),
+ "sealed": ("archived", "success"),
+ }
+ if has_outputs:
+ status_label, status_color = ("archived", "success") if is_archived else ("partial", "warning")
+ else:
+ status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger"))
+
+ related_snapshots = list(
+ type(self)
+ .objects.filter(url=self.url)
+ .exclude(id=self.id)
+ .only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size")
+ .order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
+ )
+ related_years_map: dict[int, list[Snapshot]] = {}
+ for snapshot in [self, *related_snapshots]:
+ snapshot_date = snapshot.bookmarked_at or snapshot.created_at or snapshot.downloaded_at
+ if snapshot_date:
+ related_years_map.setdefault(snapshot_date.year, []).append(snapshot)
+ related_years = []
+ for year, snapshots in related_years_map.items():
+ snapshots.sort(
+ key=lambda snapshot: snapshot.bookmarked_at or snapshot.created_at or snapshot.downloaded_at or timezone.now(),
+ reverse=True,
+ )
+ related_years.append({"year": year, "latest": snapshots[0], "snapshots": snapshots})
+ related_years.sort(key=lambda item: item["year"], reverse=True)
+
+ warc_path = next(
+ (rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name),
+ "warc/",
+ )
+ user = getattr(request, "user", None)
tag_widget = TagEditorWidget()
- context = {
- **self.to_dict(extended=True),
- "snapshot": self,
- "title": htmldecode(self.resolved_title or (self.base_url if is_archived else TITLE_LOADING_MSG)),
- "url_str": htmldecode(urldecode(self.base_url)),
- "archive_url": urlencode(f"warc/{self.timestamp}") or "about:blank",
+ return {
+ "id": str(self.id),
+ "snapshot_id": str(self.id),
+ "progress_endpoint": progress_endpoint("snapshot", self.id) if request is not None else "",
+ "progress_auto_expand": snapshot_status in {"queued", "started", "paused"},
+ "url": self.url,
+ "archive_path": self.archive_path_from_db,
+ "title": htmldecode(self.resolved_title or (self.base_url if is_archived else "Not yet archived...")),
"extension": self.extension or "html",
"tags": self.tags_str() or "untagged",
- "size": printable_filesize(output_size) if output_size else "pending",
- "status": "archived" if is_archived else "not yet archived",
- "status_color": "success" if is_archived else "danger",
- "oldest_archive_date": ts_to_date_str(self.oldest_archive_date),
- "best_preview_path": best_preview_path,
+ "size": printable_filesize(output_size) if output_size else "β",
+ "status": status_label,
+ "status_color": status_color,
+ "snapshot_state": snapshot_status,
+ "has_outputs": has_outputs,
+ "snapshot_permissions": snapshot_permissions,
+ "snapshot_permissions_icon": {"public": "π₯", "unlisted": "π", "private": "π"}.get(snapshot_permissions, "π₯"),
+ "bookmarked_date": self.bookmarked_date,
+ "downloaded_datestr": self.downloaded_datestr,
+ "num_outputs": self.num_outputs,
+ "num_failures": self.num_failures,
+ "oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None),
+ "warc_path": warc_path,
+ "archiveresults": [*non_compact_outputs, *compact_outputs],
"best_result": best_result,
- "archiveresults": outputs,
+ "snapshot": self,
+ "CONFIG": runtime_config,
+ "related_snapshots": related_snapshots,
+ "related_years": related_years,
"loose_items": loose_items,
"failed_items": failed_items,
- "related_snapshots": [],
- "related_years": [],
- "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in self.tags.all().order_by("name")],
+ "can_delete_outputs": bool(user and user.is_authenticated and user.is_active and user.is_superuser),
+ "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)],
+ "STATIC_EXPORT": static_export_dir is not None,
+ "STATIC_EXPORT_DIR": static_export_dir,
}
+
+ def write_html_details(self, out_dir: Path | str | None = None) -> None:
+ """Write the unified snapshot detail page with portable filesystem URLs."""
+ from django.template.loader import render_to_string
+
+ output_dir = Path(out_dir) if out_dir is not None else self.output_dir
+ context = self.get_html_details_context(static_export_dir=output_dir)
rendered_html = render_to_string("core/snapshot.html", context)
atomic_write(str(output_dir / CONSTANTS.HTML_INDEX_FILENAME), rendered_html)
diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py
index efcb9f34..c45e1d50 100644
--- a/archivebox/core/templatetags/core_tags.py
+++ b/archivebox/core/templatetags/core_tags.py
@@ -2,6 +2,7 @@ import os
from html import unescape
from pathlib import Path
from typing import Any
+from urllib.parse import quote
from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
from django import template
@@ -11,6 +12,7 @@ from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.text import Truncator
+from archivebox.config import CONSTANTS
from archivebox.core.routes_util import (
build_snapshot_url,
get_admin_base_url,
@@ -28,6 +30,7 @@ register = template.Library()
_TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
_IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
+_STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
_MEDIA_FILE_EXTS = {
".mp4",
@@ -202,16 +205,70 @@ def _is_root_snapshot_output_path(raw_output_path: str | None) -> bool:
return normalized in ("", ".", "./", "/", "index.html", "index.json")
-def _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> str:
- return build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
+def _static_snapshot_base_url(context, snapshot) -> str:
+ start_dir = Path(context["STATIC_EXPORT_DIR"])
+ relative_path = os.path.relpath(Path(snapshot.output_dir), start=start_dir).replace(os.sep, "/")
+ if relative_path == ".":
+ return "."
+ return f"./{quote(relative_path, safe=_STATIC_URL_SAFE)}"
-def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None, plugin: str = "") -> str:
+def _snapshot_base_url_for_context(context, snapshot) -> str:
+ if context.get("STATIC_EXPORT"):
+ return _static_snapshot_base_url(context, snapshot)
+ return get_snapshot_base_url(
+ str(_snapshot_id(snapshot)),
+ request=context.get("request"),
+ config=context.get("CONFIG"),
+ )
+
+
+def _snapshot_url_for_context(context, snapshot, path: str = "") -> str:
+ if not context.get("STATIC_EXPORT"):
+ return build_snapshot_url(
+ str(_snapshot_id(snapshot)),
+ path,
+ request=context.get("request"),
+ config=context.get("CONFIG"),
+ )
+
+ base_url = _static_snapshot_base_url(context, snapshot)
+ raw_path = str(path or "")
+ if not raw_path:
+ return base_url
+ path_part, separator, query = raw_path.lstrip("/").partition("?")
+ quoted_path = quote(path_part, safe=_STATIC_URL_SAFE)
+ suffix = f"?{query}" if separator else ""
+ return f"{base_url.rstrip('/')}/{quoted_path}{suffix}"
+
+
+def _build_snapshot_files_url(snapshot_id: str, request=None, config=None, base_url: str | None = None) -> str:
+ return (
+ f"{base_url.rstrip('/')}/?files=1"
+ if base_url
+ else build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
+ )
+
+
+def _build_snapshot_preview_url(
+ snapshot_id: str,
+ path: str = "",
+ request=None,
+ config=None,
+ plugin: str = "",
+ base_url: str | None = None,
+) -> str:
if path == "about:blank":
return path
if _is_root_snapshot_output_path(path):
- return _build_snapshot_files_url(snapshot_id, request=request, config=config)
- url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
+ return _build_snapshot_files_url(snapshot_id, request=request, config=config, base_url=base_url)
+ if base_url:
+ path_part, separator, query = str(path).lstrip("/").partition("?")
+ url = f"{base_url.rstrip('/')}/{quote(path_part, safe=_STATIC_URL_SAFE)}"
+ if separator:
+ url = f"{url}?{query}"
+ else:
+ url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
path_parts = Path(path).parts
plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "")
has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False))
@@ -441,23 +498,35 @@ def web_base_url(context) -> str:
return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
+@register.simple_tag(takes_context=True)
+def static_export_root_url(context) -> str:
+ if not context.get("STATIC_EXPORT"):
+ return ""
+ relative_path = os.path.relpath(CONSTANTS.DATA_DIR, start=Path(context["STATIC_EXPORT_DIR"]))
+ relative_path = relative_path.replace(os.sep, "/")
+ return "./" if relative_path == "." else f"{quote(relative_path, safe=_STATIC_URL_SAFE)}/"
+
+
@register.simple_tag(takes_context=True)
def snapshot_base_url(context, snapshot) -> str:
- snapshot_id = _snapshot_id(snapshot)
- return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
+ return _snapshot_base_url_for_context(context, snapshot)
@register.simple_tag(takes_context=True)
def snapshot_url(context, snapshot, path: str = "") -> str:
- snapshot_id = _snapshot_id(snapshot)
- return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
+ return _snapshot_url_for_context(context, snapshot, path)
@register.simple_tag(takes_context=True)
def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
snapshot_id = str(_snapshot_id(snapshot))
url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
- cache_key = (plugin, filename)
+ cache_key = (
+ plugin,
+ filename,
+ bool(context.get("STATIC_EXPORT")),
+ str(context.get("STATIC_EXPORT_DIR") or ""),
+ )
if cache_key in url_cache:
return url_cache[cache_key]
@@ -493,7 +562,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
continue
output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
- url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
+ url_cache[cache_key] = _snapshot_url_for_context(context, snapshot, output_path)
return url_cache[cache_key]
url_cache[cache_key] = ""
@@ -502,10 +571,7 @@ def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) ->
@register.simple_tag(takes_context=True)
def snapshot_index_row(context, link) -> str:
- snapshot_id = str(_snapshot_id(link))
- request = context.get("request")
- config = context.get("CONFIG")
- snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
+ snapshot_base = _snapshot_base_url_for_context(context, link)
status = getattr(link, "status", None) or "unknown"
bookmarked_at = getattr(link, "bookmarked_at", None)
@@ -534,22 +600,28 @@ def snapshot_index_row(context, link) -> str:
tag_cell = '...'
num_outputs = int(getattr(link, "num_outputs", 0) or 0)
- icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
+ if context.get("STATIC_EXPORT") and callable(getattr(link, "icons", None)):
+ icons = link.icons(path=quote(link.static_archive_path, safe=_STATIC_URL_SAFE), prefix="./")
+ else:
+ icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
icons_cell = str(icons) if icons else '...'
archive_size = int(getattr(link, "archive_size", 0) or 0)
size_cell = file_size(archive_size) if archive_size else '...'
output_plural = "" if num_outputs == 1 else "s"
+ files_url = _snapshot_url_for_context(context, link, "index.jsonl") if context.get("STATIC_EXPORT") else f"{snapshot_base}/?files=1"
if is_pending:
- preview_html = (
- ''
- f'
'
- ""
- )
+ preview_html = ''
+ if not context.get("STATIC_EXPORT"):
+ preview_html = (
+ ''
+ f''
+ ""
+ )
elif "_public_preview_paths" in link.__dict__:
preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
if preview_paths:
- preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
+ preview_urls = [_snapshot_url_for_context(context, link, path) for path in preview_paths]
preview_html = (
f'})
str:
if "_public_favicon_paths" in link.__dict__:
favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
if favicon_paths:
- favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
+ favicon_urls = [_snapshot_url_for_context(context, link, path) for path in favicon_paths]
favicon_html = (
f'})
str:
-
+
{size_cell}
{num_outputs} output{output_plural}
@@ -660,6 +732,7 @@ def snapshot_preview_url(context, snapshot, path: str = "", result=None) -> str:
request=context.get("request"),
config=context.get("CONFIG"),
plugin=plugin,
+ base_url=_snapshot_base_url_for_context(context, snapshot) if context.get("STATIC_EXPORT") else None,
)
@@ -699,24 +772,16 @@ def plugin_card(context, result) -> str:
# Use embed_path() for the display path
raw_output_path = result.embed_path() or ""
- output_url = build_snapshot_url(
- str(result.snapshot_id),
- raw_output_path or "",
- request=context.get("request"),
- config=context.get("CONFIG"),
- )
+ output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path or "")
icon_html = get_plugin_icon(plugin)
plugin_lower = (plugin or "").lower()
media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
if media_files:
- snapshot_id = str(result.snapshot_id)
- request = context.get("request")
- config = context.get("CONFIG")
for item in media_files:
path = item.get("path") or ""
- item["url"] = build_snapshot_url(snapshot_id, path, request=request, config=config) if path else ""
+ item["url"] = _snapshot_url_for_context(context, result.snapshot, path) if path else ""
output_lower = (raw_output_path or "").lower()
force_text_preview = output_lower.endswith(_TEXT_PREVIEW_EXTS)
@@ -794,12 +859,7 @@ def plugin_full(context, result) -> str:
raw_output_path = result.embed_path() or ""
if _is_root_snapshot_output_path(raw_output_path):
return ""
- output_url = build_snapshot_url(
- str(result.snapshot_id),
- raw_output_path,
- request=context.get("request"),
- config=context.get("CONFIG"),
- )
+ output_url = _snapshot_url_for_context(context, result.snapshot, raw_output_path)
try:
tpl = template.Template(template_str)
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index 52907de3..c6849e91 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -21,7 +21,6 @@ from django.core.paginator import InvalidPage
from django.db.models import Case, IntegerField, Q, Value, When
from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
from django.shortcuts import redirect, render
-from django.utils import timezone
from django.utils.decorators import method_decorator
from django.utils.html import format_html, format_html_join
from django.utils.safestring import mark_safe
@@ -54,7 +53,6 @@ from archivebox.core.permissions import (
can_view_snapshot,
direct_snapshots_queryset,
filter_personas_by_permissions,
- get_snapshot_permissions,
is_admin_user,
public_snapshots_queryset,
)
@@ -69,15 +67,12 @@ from archivebox.core.routes_util import (
host_matches,
)
from archivebox.crawls.models import Crawl
-from archivebox.misc.logging_util import printable_filesize
from archivebox.misc.paginators import AcceleratedPaginator
from archivebox.misc.serve_static import serve_static_with_byterange_support
from archivebox.misc.util import (
base_url,
filter_queryset_by_uuid_substring,
- htmldecode,
sanitize_html_text,
- ts_to_date_str,
urldecode,
validate_url,
without_fragment,
@@ -85,7 +80,7 @@ from archivebox.misc.util import (
from archivebox.plugins.discovery import get_plugin_name, get_plugin_template
from archivebox.plugins.forms import get_plugin_config_binary_urls
from archivebox.plugins.views import get_config_definition_link
-from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
+from archivebox.progressmonitor.views import live_progress_view
from archivebox.search.config import (
get_search_mode,
get_search_mode_backend,
@@ -332,162 +327,11 @@ class SnapshotView(View):
@staticmethod
def render_live_index(request, snapshot):
- TITLE_LOADING_MSG = "Not yet archived..."
- from archivebox.core.widgets import TagEditorWidget
-
- # Reuse the middleware-attached config; never re-bootstrap from env + plugin
- # schemas just to render a snapshot page (that pays ~30ms for no reason).
- runtime_config = get_request_config(request)
- snapshot._runtime_config = runtime_config
- snapshot_permissions = get_snapshot_permissions(snapshot)
- archive_results = list(snapshot.archiveresult_set.all().order_by("start_ts"))
- tags = list(snapshot.tags.all())
- snapshot.__dict__["_admin_archiveresults"] = archive_results
- snapshot.__dict__["_tags_str_cached"] = ",".join(sorted(tag.name for tag in tags))
- snapshot.__dict__["num_outputs_cached"] = sum(result.status == ArchiveResult.StatusChoices.SUCCEEDED for result in archive_results)
- snapshot.__dict__["num_failures_cached"] = sum(result.status == ArchiveResult.StatusChoices.FAILED for result in archive_results)
- hidden_card_plugins = {"archivedotorg", "favicon", "title"}
- outputs = [
- out
- for out in snapshot.discover_outputs(include_filesystem_fallback=True, archive_results=archive_results)
- if (out.get("size") or 0) > 0 and out.get("name") not in hidden_card_plugins
- ]
- archiveresults = {}
- result_ids_by_name = {}
- for output in outputs:
- if output.get("result"):
- result_ids_by_name.setdefault(output["name"], []).append(str(output["result"].id))
- current = archiveresults.get(output["name"])
- if current is None or (output.get("size") or 0) > (current.get("size") or 0):
- archiveresults[output["name"]] = output
- for name, output in archiveresults.items():
- output["result_ids"] = ",".join(result_ids_by_name.get(name, ()))
- hash_index = snapshot.hashes_index
- loose_items, failed_items = snapshot.get_detail_page_auxiliary_items(
- outputs,
- hidden_card_plugins=hidden_card_plugins,
- archive_results=archive_results,
+ return render(
+ template_name="core/snapshot.html",
+ request=request,
+ context=snapshot.get_html_details_context(request=request),
)
- preview_priority = [
- "singlefile",
- "screenshot",
- "wget",
- "dom",
- "pdf",
- "readability",
- ]
- preferred_types = tuple(preview_priority)
- output_order = {result_type: index for index, result_type in enumerate(archiveresults.keys())}
-
- best_result = {"path": "about:blank", "result": None}
- for result_type in preferred_types:
- if result_type in archiveresults:
- best_result = archiveresults[result_type]
- break
-
- related_snapshots_qs = SnapshotView.find_snapshots_for_url(
- snapshot.url,
- allow_fallback=False,
- ).only("id", "url", "bookmarked_at", "created_at", "downloaded_at", "output_size")
- related_snapshots = list(
- related_snapshots_qs.exclude(id=snapshot.id).order_by("-bookmarked_at", "-created_at", "-timestamp")[:25],
- )
- related_years_map: dict[int, list[Snapshot]] = {}
- for snap in [snapshot, *related_snapshots]:
- snap_dt = snap.bookmarked_at or snap.created_at or snap.downloaded_at
- if not snap_dt:
- continue
- related_years_map.setdefault(snap_dt.year, []).append(snap)
- related_years = []
- for year, snaps in related_years_map.items():
- snaps_sorted = sorted(
- snaps,
- key=lambda s: s.bookmarked_at or s.created_at or s.downloaded_at or timezone.now(),
- reverse=True,
- )
- related_years.append(
- {
- "year": year,
- "latest": snaps_sorted[0],
- "snapshots": snaps_sorted,
- },
- )
- related_years.sort(key=lambda item: item["year"], reverse=True)
-
- warc_path = next(
- (rel_path for rel_path in hash_index if rel_path.startswith("warc/") and ".warc" in Path(rel_path).name),
- "warc/",
- )
-
- ordered_outputs = sorted(
- archiveresults.values(),
- key=lambda r: (
- preferred_types.index(r["name"]) if r["name"] in preferred_types else len(preferred_types),
- output_order.get(r["name"], len(output_order)),
- ),
- )
- if best_result["path"] == "about:blank" and ordered_outputs:
- best_result = ordered_outputs[0]
- non_compact_outputs = [out for out in ordered_outputs if not out.get("is_compact") and not out.get("is_metadata")]
- compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")]
- tag_widget = TagEditorWidget()
- output_size = sum(int(out.get("size") or 0) for out in ordered_outputs)
- archive_dates = [result.start_ts for result in archive_results if result.start_ts]
- has_outputs = bool(ordered_outputs)
- is_archived = has_outputs or snapshot.status == Snapshot.StatusChoices.SEALED
- snapshot_status = str(snapshot.status or "").lower()
- status_label_by_state = {
- "queued": ("queued", "info"),
- "started": ("running", "warning"),
- "paused": ("paused", "default"),
- "sealed": ("archived", "success"),
- }
- if has_outputs and not is_archived:
- status_label, status_color = ("partial", "warning")
- elif has_outputs:
- status_label, status_color = ("archived", "success")
- else:
- status_label, status_color = status_label_by_state.get(snapshot_status, ("not yet archived", "danger"))
-
- context = {
- "id": str(snapshot.id),
- "snapshot_id": str(snapshot.id),
- "progress_endpoint": progress_endpoint("snapshot", snapshot.id),
- "progress_auto_expand": snapshot_status in {"queued", "started", "paused"},
- "url": snapshot.url,
- "archive_path": snapshot.archive_path_from_db,
- "title": htmldecode(snapshot.resolved_title or (snapshot.base_url if is_archived else TITLE_LOADING_MSG)),
- "extension": snapshot.extension or "html",
- "tags": snapshot.tags_str() or "untagged",
- "size": printable_filesize(output_size) if output_size else "β",
- "status": status_label,
- "status_color": status_color,
- "snapshot_state": snapshot_status,
- "has_outputs": has_outputs,
- "snapshot_permissions": snapshot_permissions,
- "snapshot_permissions_icon": {
- "public": "π₯",
- "unlisted": "π",
- "private": "π",
- }.get(snapshot_permissions, "π₯"),
- "bookmarked_date": snapshot.bookmarked_date,
- "downloaded_datestr": snapshot.downloaded_datestr,
- "num_outputs": snapshot.num_outputs,
- "num_failures": snapshot.num_failures,
- "oldest_archive_date": ts_to_date_str(min(archive_dates) if archive_dates else None),
- "warc_path": warc_path,
- "archiveresults": [*non_compact_outputs, *compact_outputs],
- "best_result": best_result,
- "snapshot": snapshot, # Pass the snapshot object for template tags
- "CONFIG": runtime_config,
- "related_snapshots": related_snapshots,
- "related_years": related_years,
- "loose_items": loose_items,
- "failed_items": failed_items,
- "can_delete_outputs": bool(request.user.is_authenticated and request.user.is_active and request.user.is_superuser),
- "title_tags": [{"name": tag.name, "style": tag_widget._tag_style(tag.name)} for tag in sorted(tags, key=lambda tag: tag.name)],
- }
- return render(template_name="core/snapshot.html", request=request, context=context)
def get(self, request, path):
snapshot = None
diff --git a/archivebox/templates/core/snapshot.html b/archivebox/templates/core/snapshot.html
index eded76bd..1bc558b0 100644
--- a/archivebox/templates/core/snapshot.html
+++ b/archivebox/templates/core/snapshot.html
@@ -319,9 +319,9 @@
user-select: all;
}
.header-toggle {
- line-height: 12px;
- font-size: 70px;
- margin-top: -13px;
+ line-height: 1;
+ font-size: 24px;
+ margin-top: 0;
margin-left: 4px;
}
@container snapshot-header (max-width: 900px) {
@@ -1475,11 +1475,13 @@
+ {% if not STATIC_EXPORT %}
{% if snapshot_state == 'queued' or snapshot_state == 'started' or snapshot_state == 'paused' %}
{% include "progressmonitor/progress_monitor.html" with progress_endpoint=progress_endpoint progress_scope="snapshot" %}
{% endif %}
+ {% endif %}
{% if has_outputs %}
|