From b46d142cc62fc4a9f8c78574cb888bf6eb8ae0b5 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 2 Jun 2026 12:13:47 -0700 Subject: [PATCH] test cleanup --- archivebox/api/v1_cli.py | 5 +- archivebox/cli/archivebox_snapshot.py | 7 ++- archivebox/cli/archivebox_status.py | 2 +- .../progressmonitor/progress_monitor.html | 58 ++++++++++++------- archivebox/tests/test_binary_service.py | 4 +- archivebox/tests/test_cli_extract_input.py | 2 +- archivebox/tests/test_crawl_service.py | 2 +- 7 files changed, 52 insertions(+), 28 deletions(-) diff --git a/archivebox/api/v1_cli.py b/archivebox/api/v1_cli.py index 09af4319..ea175f25 100644 --- a/archivebox/api/v1_cli.py +++ b/archivebox/api/v1_cli.py @@ -291,7 +291,10 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema): remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact) timeout_arg = remove_kwargs.pop("timeout") timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0) - snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs) + try: + snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs) + except ValueError as err: + raise HttpError(400, str(err)) from err result = remove( yes=True, # no way to interactively ask for confirmation via API, so we force yes diff --git a/archivebox/cli/archivebox_snapshot.py b/archivebox/cli/archivebox_snapshot.py index 5aaefcb0..9f5417f6 100644 --- a/archivebox/cli/archivebox_snapshot.py +++ b/archivebox/cli/archivebox_snapshot.py @@ -38,6 +38,7 @@ from rich import print as rprint from django.db.models import QuerySet SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp") +SNAPSHOT_LIST_CHUNK_SIZE = 100 # ============================================================================= @@ -287,7 +288,7 @@ def list_snapshots( if with_headers: sys.stdout.write(",".join(cols)) sys.stdout.write("\n") - for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500): + for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE): sys.stdout.write(snapshot.to_csv(cols=cols, separator=",")) sys.stdout.write("\n") count += 1 @@ -295,13 +296,13 @@ def list_snapshots( return 0 if not is_tty: - for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500): + for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE): write_record(snapshot.to_json()) count += 1 rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr) return 0 - for snapshot in queryset.iterator(chunk_size=500): + for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE): status_color = { "queued": "yellow", "started": "blue", diff --git a/archivebox/cli/archivebox_status.py b/archivebox/cli/archivebox_status.py index 8431fbef..96f8c035 100644 --- a/archivebox/cli/archivebox_status.py +++ b/archivebox/cli/archivebox_status.py @@ -105,7 +105,7 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None: if num_indexed: print(" [violet]Hint:[/violet] You can list snapshots by status like so:") - print(" [green]archivebox list --status= (e.g. archived, queued, etc.)[/green]") + print(" [green]archivebox list --status= (e.g. sealed, queued, etc.)[/green]") if orphaned_dirs: print(" [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:") diff --git a/archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html b/archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html index d8bb74bf..2220364c 100644 --- a/archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html +++ b/archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html @@ -71,7 +71,12 @@ animation: idle-pulse 5s infinite; } #progress-monitor .status-dot.stopped { - background: #6e7681; + background: #d29922; + box-shadow: 0 0 4px #d29922; + } + #progress-monitor .status-dot.error { + background: #f85149; + box-shadow: 0 0 8px #f85149; } #progress-monitor .status-dot.flash { animation: flash 0.3s ease-out; @@ -967,19 +972,19 @@
Crawls - 0 active · 0 queued + 0 active · 0 queued
Snapshots - 0 active · 0 queued + 0 active · 0 queued
Downloads - 0 active · 0 queued + 0 active · 0 queued
Indexing - 0 active · 0 queued + 0 active · 0 queued
@@ -1682,17 +1687,24 @@ `; } + function setOrchestratorState(state, label) { + const dot = document.getElementById('orchestrator-dot'); + dot.classList.remove('stopped', 'idle', 'running', 'error'); + dot.classList.add(state); + document.getElementById('orchestrator-text').textContent = label; + return dot; + } + function updateProgress(data) { + idleMessage.style.color = ''; function setCount(id, value) { const el = document.getElementById(id); if (el) el.textContent = Number(value || 0).toLocaleString(); } - function setOrchestratorState(state, label) { - const dot = document.getElementById('orchestrator-dot'); - dot.classList.remove('stopped', 'idle', 'running'); - dot.classList.add(state); - document.getElementById('orchestrator-text').textContent = label; - return dot; + function setActiveCount(id, value) { + setCount(id, value); + const segment = document.getElementById(`${id}-segment`); + if (segment) segment.style.display = Number(value || 0) > 0 ? '' : 'none'; } // Calculate if there's activity @@ -1715,17 +1727,15 @@ // Update orchestrator status - show "Running" only when there are active workers. const pidEl = document.getElementById('orchestrator-pid'); const hasWorkers = data.total_workers > 0; - const hasBlockedCrawl = (data.active_crawls || []).some(c => c.worker_state === 'crashed'); let dot = null; if (hasWorkers) { dot = setOrchestratorState('running', 'Running'); - } else if (hasActivity && hasBlockedCrawl && !data.orchestrator_running) { + } else if (!data.orchestrator_running) { dot = setOrchestratorState('stopped', 'Runner stopped'); } else if (hasActivity) { - dot = setOrchestratorState('idle', data.orchestrator_running ? 'Idle' : 'Waiting'); + dot = setOrchestratorState('idle', 'Idle'); } else { - // No activity - show as idle (whether orchestrator process exists or not) dot = setOrchestratorState('idle', 'Idle'); } @@ -1742,15 +1752,17 @@ setTimeout(() => dot.classList.remove('flash'), 300); [ - ['crawls-active', data.crawls_active], ['crawls-queued', data.crawls_queued], - ['snapshots-active', data.snapshots_active], ['snapshots-queued', data.snapshots_queued], - ['downloads-active', data.downloads_active], ['downloads-queued', data.downloads_queued], - ['indexing-active', data.indexing_active], ['indexing-queued', data.indexing_queued], ].forEach(([id, value]) => setCount(id, value)); + [ + ['crawls-active', data.crawls_active], + ['snapshots-active', data.snapshots_active], + ['downloads-active', data.downloads_active], + ['indexing-active', data.indexing_active], + ].forEach(([id, value]) => setActiveCount(id, value)); updateScreencastPanel(data); // Render crawl tree @@ -1784,18 +1796,24 @@ function fetchProgress() { fetch(progressEndpoint, { credentials: 'same-origin' }) - .then(response => response.json()) + .then(response => { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return response.json(); + }) .then(data => { if (data.error) { console.error('Progress API error:', data.error, data.traceback); + setOrchestratorState('error', 'Backend not responding'); idleMessage.textContent = 'API Error: ' + data.error; idleMessage.style.color = '#f85149'; + return; } monitor.classList.toggle('is-guest', data.is_admin === false); updateProgress(data); }) .catch(error => { console.error('Progress fetch error:', error); + setOrchestratorState('error', 'Backend not responding'); idleMessage.textContent = 'Fetch Error: ' + error.message; idleMessage.style.color = '#f85149'; }); diff --git a/archivebox/tests/test_binary_service.py b/archivebox/tests/test_binary_service.py index aa73f654..443d9d9a 100644 --- a/archivebox/tests/test_binary_service.py +++ b/archivebox/tests/test_binary_service.py @@ -25,11 +25,13 @@ def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) -> def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]: + archivebox_bin = shutil.which("archivebox") + assert archivebox_bin, "archivebox console script must be available for CLI tests" return { "LIB_DIR": str(data_dir / "lib"), "LIB_BIN_DIR": str(data_dir / "lib" / "bin"), "ABXPKG_LIB_DIR": str(data_dir / "lib"), - "PATH": os.pathsep.join([str(bin_dir), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]), + "PATH": os.pathsep.join([str(bin_dir), str(Path(archivebox_bin).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]), } diff --git a/archivebox/tests/test_cli_extract_input.py b/archivebox/tests/test_cli_extract_input.py index 19427cdb..8fb3875e 100644 --- a/archivebox/tests/test_cli_extract_input.py +++ b/archivebox/tests/test_cli_extract_input.py @@ -145,7 +145,7 @@ def test_extract_pipeline_from_snapshot(initialized_archive): # Create snapshot and pipe to extract snapshot_proc = run_archivebox_cmd( - ["snapshot", "https://example.com"], + ["snapshot", "create", "https://example.com"], cwd=initialized_archive, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/archivebox/tests/test_crawl_service.py b/archivebox/tests/test_crawl_service.py index 0783fa8e..ada4801d 100644 --- a/archivebox/tests/test_crawl_service.py +++ b/archivebox/tests/test_crawl_service.py @@ -41,7 +41,7 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p port = get_free_port() env = cli_env( - port, + port=port, PLUGINS="wget,parse_html_urls", SAVE_WGET="True", SAVE_FAVICON="False",