The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

test cleanup

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-02 12:13:47 -07:00
parent 96437e1ffd
commit b46d142cc6
No known key found for this signature in database
7 changed files with 52 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
archivebox/api/v1_cli.py
View File

@ -291,7 +291,10 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
timeout_arg = remove_kwargs.pop("timeout")
timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
try:
snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
except ValueError as err:
raise HttpError(400, str(err)) from err
result = remove(
yes=True, # no way to interactively ask for confirmation via API, so we force yes

7
archivebox/cli/archivebox_snapshot.py
View File

@ -38,6 +38,7 @@ from rich import print as rprint
from django.db.models import QuerySet
SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
SNAPSHOT_LIST_CHUNK_SIZE = 100
# =============================================================================
@ -287,7 +288,7 @@ def list_snapshots(
if with_headers:
sys.stdout.write(",".join(cols))
sys.stdout.write("\n")
for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500):
for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
sys.stdout.write("\n")
count += 1
@ -295,13 +296,13 @@ def list_snapshots(
return 0
if not is_tty:
for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=500):
for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
write_record(snapshot.to_json())
count += 1
rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
return 0
for snapshot in queryset.iterator(chunk_size=500):
for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
status_color = {
"queued": "yellow",
"started": "blue",

2
archivebox/cli/archivebox_status.py
View File

@ -105,7 +105,7 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
if num_indexed:
print(" [violet]Hint:[/violet] You can list snapshots by status like so:")
print(" [green]archivebox list --status=<status> (e.g. archived, queued, etc.)[/green]")
print(" [green]archivebox list --status=<status> (e.g. sealed, queued, etc.)[/green]")
if orphaned_dirs:
print(" [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")

58
archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html
View File

@ -71,7 +71,12 @@
animation: idle-pulse 5s infinite;
}
#progress-monitor .status-dot.stopped {
background: #6e7681;
background: #d29922;
box-shadow: 0 0 4px #d29922;
}
#progress-monitor .status-dot.error {
background: #f85149;
box-shadow: 0 0 8px #f85149;
}
#progress-monitor .status-dot.flash {
animation: flash 0.3s ease-out;
@ -967,19 +972,19 @@
<div class="stats">
<div class="stat">
<span class="stat-label">Crawls</span>
<span class="stat-value compact info"><span id="crawls-active">0</span> active · <span id="crawls-queued">0</span> queued</span>
<span class="stat-value compact info"><span id="crawls-active-segment"><span id="crawls-active">0</span> active · </span><span id="crawls-queued">0</span> queued</span>
</div>
<div class="stat">
<span class="stat-label">Snapshots</span>
<span class="stat-value compact info"><span id="snapshots-active">0</span> active · <span id="snapshots-queued">0</span> queued</span>
<span class="stat-value compact info"><span id="snapshots-active-segment"><span id="snapshots-active">0</span> active · </span><span id="snapshots-queued">0</span> queued</span>
</div>
<div class="stat">
<span class="stat-label">Downloads</span>
<span class="stat-value compact warning"><span id="downloads-active">0</span> active · <span id="downloads-queued">0</span> queued</span>
<span class="stat-value compact warning"><span id="downloads-active-segment"><span id="downloads-active">0</span> active · </span><span id="downloads-queued">0</span> queued</span>
</div>
<div class="stat">
<span class="stat-label">Indexing</span>
<span class="stat-value compact success"><span id="indexing-active">0</span> active · <span id="indexing-queued">0</span> queued</span>
<span class="stat-value compact success"><span id="indexing-active-segment"><span id="indexing-active">0</span> active · </span><span id="indexing-queued">0</span> queued</span>
</div>
</div>
</div>
@ -1682,17 +1687,24 @@
`;
}
function setOrchestratorState(state, label) {
const dot = document.getElementById('orchestrator-dot');
dot.classList.remove('stopped', 'idle', 'running', 'error');
dot.classList.add(state);
document.getElementById('orchestrator-text').textContent = label;
return dot;
}
function updateProgress(data) {
idleMessage.style.color = '';
function setCount(id, value) {
const el = document.getElementById(id);
if (el) el.textContent = Number(value || 0).toLocaleString();
}
function setOrchestratorState(state, label) {
const dot = document.getElementById('orchestrator-dot');
dot.classList.remove('stopped', 'idle', 'running');
dot.classList.add(state);
document.getElementById('orchestrator-text').textContent = label;
return dot;
function setActiveCount(id, value) {
setCount(id, value);
const segment = document.getElementById(`${id}-segment`);
if (segment) segment.style.display = Number(value || 0) > 0 ? '' : 'none';
}
// Calculate if there's activity
@ -1715,17 +1727,15 @@
// Update orchestrator status - show "Running" only when there are active workers.
const pidEl = document.getElementById('orchestrator-pid');
const hasWorkers = data.total_workers > 0;
const hasBlockedCrawl = (data.active_crawls || []).some(c => c.worker_state === 'crashed');
let dot = null;
if (hasWorkers) {
dot = setOrchestratorState('running', 'Running');
} else if (hasActivity && hasBlockedCrawl && !data.orchestrator_running) {
} else if (!data.orchestrator_running) {
dot = setOrchestratorState('stopped', 'Runner stopped');
} else if (hasActivity) {
dot = setOrchestratorState('idle', data.orchestrator_running ? 'Idle' : 'Waiting');
dot = setOrchestratorState('idle', 'Idle');
} else {
// No activity - show as idle (whether orchestrator process exists or not)
dot = setOrchestratorState('idle', 'Idle');
}
@ -1742,15 +1752,17 @@
setTimeout(() => dot.classList.remove('flash'), 300);
[
['crawls-active', data.crawls_active],
['crawls-queued', data.crawls_queued],
['snapshots-active', data.snapshots_active],
['snapshots-queued', data.snapshots_queued],
['downloads-active', data.downloads_active],
['downloads-queued', data.downloads_queued],
['indexing-active', data.indexing_active],
['indexing-queued', data.indexing_queued],
].forEach(([id, value]) => setCount(id, value));
[
['crawls-active', data.crawls_active],
['snapshots-active', data.snapshots_active],
['downloads-active', data.downloads_active],
['indexing-active', data.indexing_active],
].forEach(([id, value]) => setActiveCount(id, value));
updateScreencastPanel(data);
// Render crawl tree
@ -1784,18 +1796,24 @@
function fetchProgress() {
fetch(progressEndpoint, { credentials: 'same-origin' })
.then(response => response.json())
.then(response => {
if (!response.ok) throw new Error(`HTTP ${response.status}`);
return response.json();
})
.then(data => {
if (data.error) {
console.error('Progress API error:', data.error, data.traceback);
setOrchestratorState('error', 'Backend not responding');
idleMessage.textContent = 'API Error: ' + data.error;
idleMessage.style.color = '#f85149';
return;
}
monitor.classList.toggle('is-guest', data.is_admin === false);
updateProgress(data);
})
.catch(error => {
console.error('Progress fetch error:', error);
setOrchestratorState('error', 'Backend not responding');
idleMessage.textContent = 'Fetch Error: ' + error.message;
idleMessage.style.color = '#f85149';
});

4
archivebox/tests/test_binary_service.py
View File

@ -25,11 +25,13 @@ def _link_real_binary(bin_dir: Path, name: str, *, source: str | None = None) ->
def _runtime_env(data_dir: Path, bin_dir: Path) -> dict[str, str]:
archivebox_bin = shutil.which("archivebox")
assert archivebox_bin, "archivebox console script must be available for CLI tests"
return {
"LIB_DIR": str(data_dir / "lib"),
"LIB_BIN_DIR": str(data_dir / "lib" / "bin"),
"ABXPKG_LIB_DIR": str(data_dir / "lib"),
"PATH": os.pathsep.join([str(bin_dir), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]),
"PATH": os.pathsep.join([str(bin_dir), str(Path(archivebox_bin).parent), "/usr/bin", "/bin", "/usr/sbin", "/sbin"]),
}

2
archivebox/tests/test_cli_extract_input.py
View File

@ -145,7 +145,7 @@ def test_extract_pipeline_from_snapshot(initialized_archive):
# Create snapshot and pipe to extract
snapshot_proc = run_archivebox_cmd(
["snapshot", "https://example.com"],
["snapshot", "create", "https://example.com"],
cwd=initialized_archive,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,

2
archivebox/tests/test_crawl_service.py
View File

@ -41,7 +41,7 @@ def test_crawl_service_run_processes_queued_crawl_and_applies_crawl_config(tmp_p
port = get_free_port()
env = cli_env(
port,
port=port,
PLUGINS="wget,parse_html_urls",
SAVE_WGET="True",
SAVE_FAVICON="False",