diff --git a/archivebox/core/urls.py b/archivebox/core/urls.py index a7bc4cfe..23117cf1 100644 --- a/archivebox/core/urls.py +++ b/archivebox/core/urls.py @@ -71,7 +71,7 @@ urlpatterns = [ path("accounts/logout/", RedirectView.as_view(url="/admin/logout/")), path("accounts/", include("django.contrib.auth.urls")), re_path( - r"^admin/live-progress/screencast/(?P[0-9a-fA-F-]{8,36})\.jpg$", + r"^admin/live-progress/screencast/(?P[0-9a-fA-F-]{8,36})\.jpg$", archivebox_admin.admin_view(live_progress_screencast_frame_view), name="live_progress_screencast_frame", ), diff --git a/archivebox/core/views.py b/archivebox/core/views.py index e36a1ab0..fd56ce75 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1386,24 +1386,20 @@ class HealthCheckView(View): return HttpResponse("OK", content_type="text/plain", status=200) -def live_progress_screencast_frame_view(request, snapshot_id: str): +def live_progress_screencast_frame_view(request, object_id: str): """Serve cache-only Chrome screencast frames through the admin app.""" if not is_admin_user(request): return HttpResponseForbidden("Permission denied") token = request.GET.get("token", "") try: - if SCREENCAST_SIGNER.unsign(token, max_age=60) != str(snapshot_id): + if SCREENCAST_SIGNER.unsign(token, max_age=60) != str(object_id): return HttpResponseForbidden("Permission denied") except (BadSignature, SignatureExpired): return HttpResponseForbidden("Permission denied") - snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first() - if not snapshot: - raise Http404 - live_root = (CONSTANTS.CACHE_DIR / "chrome_screencast").resolve() - frame_path = live_root / str(snapshot.id) / "latest.jpg" + frame_path = live_root / str(object_id) / "latest.jpg" try: resolved_frame_path = frame_path.resolve(strict=True) except FileNotFoundError: @@ -1975,6 +1971,19 @@ def live_progress_view(request): crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded") crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed") crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued") + crawl_screencast_url = "" + crawl_screencast_link = "" + live_crawl_preview_path = CONSTANTS.CACHE_DIR / "chrome_screencast" / crawl_id / "latest.jpg" + try: + live_crawl_preview_stat = live_crawl_preview_path.stat() + except OSError: + live_crawl_preview_stat = None + if live_crawl_preview_stat and live_crawl_preview_stat.st_size > 0: + token = SCREENCAST_SIGNER.sign(crawl_id) + crawl_screencast_url = ( + f"/admin/live-progress/screencast/{crawl_id}.jpg?v={live_crawl_preview_stat.st_mtime_ns}&token={quote(token)}" + ) + crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id}/change/" # Get active snapshots for this crawl (already prefetched) active_snapshots_for_crawl = [] @@ -2251,6 +2260,8 @@ def live_progress_view(request): "setup_completed_plugins": crawl_setup_completed, "setup_failed_plugins": crawl_setup_failed, "setup_pending_plugins": crawl_setup_pending, + "screencast_url": crawl_screencast_url, + "screencast_link": crawl_screencast_link, "active_snapshots": active_snapshots_for_crawl, "can_start": can_start, "urls_preview": urls_preview, diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index f920136c..e59fe137 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -80,7 +80,11 @@ def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_ print(f" ... and {len(missing_from_code) - 10} more", file=sys.stderr) print(file=sys.stderr) print( - " If you intentionally downgraded and need to roll the DB back, run this with a build that contains those migrations:", + " If you are intentionally trying to downgrade, switch back to the newer version temporarily", + file=sys.stderr, + ) + print( + " and run this to downgrade the DB version (back up your DB first!):", file=sys.stderr, ) for app, target in sorted(rollback_targets.items()): diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index f7096281..80838115 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -1208,11 +1208,6 @@ def run_snapshot_maintenance(snapshot_id: str) -> bool: snapshot.write_index_jsonl() snapshot.write_json_details() snapshot.write_html_details() - print( - f"[runner] Snapshot {str(snapshot.id)[-12:]} maintenance complete " - f"status={snapshot.status} fs_version={snapshot.fs_version} queued_results={'yes' if has_queued_results else 'no'}", - flush=True, - ) return True diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html index 05f29496..cf8fc2de 100644 --- a/archivebox/templates/admin/progress_monitor.html +++ b/archivebox/templates/admin/progress_monitor.html @@ -219,6 +219,39 @@ object-fit: cover; object-position: top center; } + #progress-monitor .screencast-placeholder { + display: flex; + width: 100%; + height: 100%; + align-items: center; + justify-content: center; + flex-direction: column; + gap: 6px; + text-align: center; + color: #8b949e; + background: #010409; + } + #progress-monitor .screencast-placeholder-icon { + color: #c9d1d9; + font-size: 32px; + font-weight: 700; + line-height: 1; + } + #progress-monitor .screencast-placeholder.launching .screencast-placeholder-icon { + color: #d29922; + } + #progress-monitor .screencast-placeholder-title { + color: #f0f6fc; + font-size: 13px; + font-weight: 700; + line-height: 1.25; + } + #progress-monitor .screencast-placeholder-subtitle { + max-width: 260px; + color: #8b949e; + font-size: 11px; + line-height: 1.35; + } #progress-monitor .screencast-caption { display: flex; align-items: center; @@ -237,6 +270,14 @@ background: #3fb950; box-shadow: 0 0 8px rgba(63, 185, 80, 0.9); } + #progress-monitor .screencast-dot.not-launched { + background: #6e7681; + box-shadow: none; + } + #progress-monitor .screencast-dot.launching { + background: #d29922; + box-shadow: 0 0 8px rgba(210, 153, 34, 0.8); + } #progress-monitor .screencast-text { min-width: 0; } @@ -1149,24 +1190,106 @@ }); } - function activeScreencastSnapshot(data) { + function setupHookMatches(item, pattern) { + const text = `${item.plugin || ''} ${item.hook_name || ''} ${item.label || ''}`.toLowerCase(); + return text.includes(pattern); + } + + function activeScreencastTarget(data) { for (const crawl of data.active_crawls || []) { for (const snapshot of crawl.active_snapshots || []) { if (!Array.isArray(snapshot) && snapshot.screencast_url) { - return snapshot; + return {type: 'frame', crawl, snapshot}; } } } + for (const crawl of data.active_crawls || []) { + if (crawl.screencast_url) { + return { + type: 'frame', + crawl, + snapshot: { + id: crawl.id, + title: 'Live browser setup', + url: 'Crawl setup', + admin_url: `/admin/crawls/crawl/${crawl.id}/change/`, + screencast_url: crawl.screencast_url, + screencast_link: crawl.screencast_link, + }, + }; + } + } + for (const crawl of data.active_crawls || []) { + const setupPlugins = crawl.setup_plugins || []; + const startedSetup = setupPlugins.filter((item) => item.status === 'started'); + if (startedSetup.some((item) => setupHookMatches(item, 'chrome launch'))) { + return { + type: 'placeholder', + crawl, + state: 'launching', + icon: '▣', + title: 'Launching browser', + subtitle: 'Chrome is starting and publishing its CDP session.', + }; + } + for (const snapshot of crawl.active_snapshots || []) { + if (Array.isArray(snapshot)) continue; + const startedSnapshotPlugins = (snapshot.all_plugins || []).filter((item) => item.status === 'started'); + if (startedSnapshotPlugins.some((item) => setupHookMatches(item, 'chrome tab'))) { + return { + type: 'placeholder', + crawl, + snapshot, + state: 'launching', + icon: '▣', + title: 'Opening browser tab', + subtitle: 'The snapshot tab is being attached for capture.', + }; + } + } + if ((crawl.status === 'queued' || crawl.status === 'started') && !setupPlugins.some((item) => setupHookMatches(item, 'chrome launch'))) { + return { + type: 'placeholder', + crawl, + state: 'not-launched', + icon: '▯', + title: 'Browser not launched yet', + subtitle: 'Waiting for the crawl setup hooks to start Chrome.', + }; + } + } return null; } function updateScreencastPanel(data) { - const snapshot = activeScreencastSnapshot(data); - if (!snapshot) { + const target = activeScreencastTarget(data); + if (!target) { screencastPanel.classList.remove('visible'); screencastPanel.innerHTML = ''; return; } + if (target.type === 'placeholder') { + const crawlUrl = target.crawl?.id ? `/admin/crawls/crawl/${target.crawl.id}/change/` : '/admin/'; + screencastPanel.classList.add('visible'); + screencastPanel.innerHTML = ` + + + ${escapeHtml(target.icon)} + ${escapeHtml(target.title)} + ${escapeHtml(target.subtitle)} + + + + + + ${escapeHtml(target.title)} + Crawl setup + + + `; + return; + } + const snapshot = target.snapshot; const adminUrl = snapshot.admin_url || `/admin/core/snapshot/${snapshot.id || 'unknown'}/change/`; const linkUrl = snapshot.screencast_link || snapshot.view_url || adminUrl; const titleText = snapshot.title || formatUrl(snapshot.full_url || snapshot.url); diff --git a/etc/package.json b/etc/package.json index 90ab3e9f..f01f2861 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.33rc24", + "version": "0.9.33rc25", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index ed545d77..107c8ee6 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.33rc24" +version = "0.9.33rc25" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.7", # EventBus API - "abxpkg>=1.11.50", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.57", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.57", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.51", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.58", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.58", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]