diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 338d1b8f..3a3c39a1 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1521,7 +1521,13 @@ def live_progress_view(request): "created_by_id", "modified_at", "urls", + "config", "max_depth", + "max_urls", + "crawl_max_size", + "snapshot_max_size", + "tags_str", + "persona_id", "status", "retry_at", "label", @@ -1541,6 +1547,14 @@ def live_progress_view(request): key=lambda crawl: crawl.modified_at, reverse=True, )[:10] + persona_names_by_id: dict[str, str] = {} + persona_ids = {crawl.persona_id for crawl in active_crawls_list if crawl.persona_id} + if persona_ids: + from archivebox.personas.models import Persona + + persona_names_by_id = { + str(persona_id): name for persona_id, name in Persona.objects.filter(id__in=persona_ids).values_list("id", "name") + } active_crawl_ids = [crawl.id for crawl in active_crawls_list] snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids} cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids} @@ -1926,6 +1940,9 @@ def live_progress_view(request): # Check if crawl can start (for debugging stuck crawls) can_start = bool(crawl.urls) urls_preview = crawl.urls[:60] if crawl.urls else None + crawl_tags = [tag.strip() for tag in (crawl.tags_str or "").replace("\n", ",").split(",") if tag.strip()] + persona_name = persona_names_by_id.get(str(crawl.persona_id)) if crawl.persona_id else None + persona_name = persona_name or str((crawl.config or {}).get("DEFAULT_PERSONA") or "Default") # Check if retry_at is in the future (would prevent worker from claiming) retry_at_future = crawl.retry_at > now if crawl.retry_at else False @@ -1949,7 +1966,15 @@ def live_progress_view(request): "status": crawl.status, "started": crawl.created_at.isoformat() if crawl.created_at else None, "progress": crawl_progress, + "created_by": crawl.created_by.username, + "persona": persona_name, "max_depth": crawl.max_depth, + "max_urls": crawl.max_urls, + "max_crawl_size": crawl.crawl_max_size, + "max_snapshot_size": crawl.snapshot_max_size, + "max_crawl_size_display": printable_filesize(crawl.crawl_max_size) if crawl.crawl_max_size else "unlimited", + "max_snapshot_size_display": printable_filesize(crawl.snapshot_max_size) if crawl.snapshot_max_size else "unlimited", + "tags": crawl_tags, "urls_count": urls_count, "total_snapshots": total_snapshots, "completed_snapshots": completed_snapshots, diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index f5334770..025b8017 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -1122,18 +1122,25 @@ def recover_orphaned_snapshots() -> int: from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.machine.models import Process - from django.db.models import Q active_snapshot_ids: set[str] = set() orphaned_snapshots = list( - Snapshot.objects.filter( - Q(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True) - | Q(status=Snapshot.StatusChoices.SEALED, archiveresult__status=ArchiveResult.StatusChoices.QUEUED), - ) + Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED, retry_at__isnull=True) .select_related("crawl") - .prefetch_related("archiveresult_set") - .distinct(), + .prefetch_related("archiveresult_set"), ) + + queued_result_snapshot_ids = list( + ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED).values_list("snapshot_id", flat=True).distinct(), + ) + if queued_result_snapshot_ids: + orphaned_snapshots.extend( + snapshot + for snapshot in Snapshot.objects.filter(id__in=queued_result_snapshot_ids) + .select_related("crawl") + .prefetch_related("archiveresult_set") + if snapshot.status == Snapshot.StatusChoices.SEALED + ) running_processes = Process.objects.filter( status=Process.StatusChoices.RUNNING, process_type__in=[ diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html index b1bc35cb..ddbe5723 100644 --- a/archivebox/templates/admin/progress_monitor.html +++ b/archivebox/templates/admin/progress_monitor.html @@ -222,6 +222,27 @@ color: #8b949e; margin-top: 2px; } + #progress-monitor .crawl-tags { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin-top: 5px; + } + #progress-monitor .crawl-tag { + display: inline-flex; + align-items: center; + max-width: 120px; + padding: 1px 6px; + border: 1px solid #30363d; + border-radius: 10px; + color: #c9d1d9; + background: rgba(139, 148, 158, 0.12); + font-size: 10px; + line-height: 16px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } #progress-monitor .crawl-stats { display: flex; gap: 12px; @@ -944,7 +965,10 @@ } // Show snapshot info or URL count if no snapshots yet - let metaText = `depth: ${crawl.max_depth || 0}`; + const maxUrlsText = (crawl.max_urls || 0) > 0 ? `${crawl.max_urls} max URLs` : 'all URLs'; + const crawlSizeText = crawl.max_crawl_size_display || 'unlimited'; + const snapshotSizeText = crawl.max_snapshot_size_display || 'unlimited'; + let metaText = `by ${escapeHtml(crawl.created_by || 'unknown')} | persona: ${escapeHtml(crawl.persona || 'Default')} | depth: ${crawl.max_depth || 0} | ${maxUrlsText} | crawl: ${escapeHtml(crawlSizeText)} | snapshot: ${escapeHtml(snapshotSizeText)}`; if ((crawl.total_snapshots || 0) > 0) { metaText += ` | ${crawl.total_snapshots} snapshots`; } else if ((crawl.urls_count || 0) > 0) { @@ -952,6 +976,9 @@ } else if (crawl.urls_preview) { metaText += ` | ${escapeHtml(crawl.urls_preview.substring(0, 40))}${crawl.urls_preview.length > 40 ? '...' : ''}`; } + const tagsHtml = (crawl.tags || []).length + ? `
` + : ''; return `