From ce6dbdf1a0540554eed96eb74f7d77e05149bb71 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 2 Jun 2026 13:45:05 -0700 Subject: [PATCH] test: keep cli update and extract shards focused --- archivebox/tests/test_api_v1_cli_update.py | 2 +- archivebox/tests/test_cli_extract_input.py | 63 ++++++---------------- 2 files changed, 18 insertions(+), 47 deletions(-) diff --git a/archivebox/tests/test_api_v1_cli_update.py b/archivebox/tests/test_api_v1_cli_update.py index 0892e51a..cadf4c72 100644 --- a/archivebox/tests/test_api_v1_cli_update.py +++ b/archivebox/tests/test_api_v1_cli_update.py @@ -86,7 +86,7 @@ def test_cli_update_api_supports_all_snapshot_list_filters_with_real_rows(tmp_pa "post", "/api/v1/cli/update", api_token=api_token, - json={**body, "batch_size": 100}, + json={**body, "batch_size": 100, "index_only": True}, timeout=30, ) assert response.status_code == 200, f"{label}: {response.text}" diff --git a/archivebox/tests/test_cli_extract_input.py b/archivebox/tests/test_cli_extract_input.py index 8fb3875e..6c5d7009 100644 --- a/archivebox/tests/test_cli_extract_input.py +++ b/archivebox/tests/test_cli_extract_input.py @@ -6,23 +6,26 @@ import json import pytest from archivebox.core.models import ArchiveResult, Snapshot -from archivebox.tests.conftest import run_archivebox_cmd, run_queued_crawls, cli_env +from archivebox.tests.conftest import run_archivebox_cmd, cli_env from archivebox.tests.test_orm_helpers import use_archivebox_db pytestmark = pytest.mark.django_db(transaction=True) +def create_extract_snapshot(initialized_archive, env, url="https://example.com"): + run_archivebox_cmd( + ["snapshot", "create", url], + cwd=initialized_archive, + env=env, + check=True, + ) + + def test_extract_runs_on_snapshot_id(initialized_archive): """Test that extract command accepts a snapshot ID.""" env = cli_env(disable_extractors=True) - - # First create a snapshot - run_archivebox_cmd( - ["add", "--index-only", "https://example.com"], - env=env, - ) - run_queued_crawls(initialized_archive, env) + create_extract_snapshot(initialized_archive, env) with use_archivebox_db(initialized_archive): snapshot_id = Snapshot.objects.values_list("id", flat=True).first() @@ -40,13 +43,7 @@ def test_extract_runs_on_snapshot_id(initialized_archive): def test_extract_with_enabled_extractor_creates_archiveresult(initialized_archive): """Test that extract creates ArchiveResult when extractor is enabled.""" env = cli_env(disable_extractors=True) - - # First create a snapshot - run_archivebox_cmd( - ["add", "--index-only", "https://example.com"], - env=env, - ) - run_queued_crawls(initialized_archive, env) + create_extract_snapshot(initialized_archive, env) with use_archivebox_db(initialized_archive): snapshot_id = Snapshot.objects.values_list("id", flat=True).first() @@ -70,13 +67,7 @@ def test_extract_with_enabled_extractor_creates_archiveresult(initialized_archiv def test_extract_plugin_option_accepted(initialized_archive): """Test that --plugin option is accepted.""" env = cli_env(disable_extractors=True) - - # First create a snapshot - run_archivebox_cmd( - ["add", "--index-only", "https://example.com"], - env=env, - ) - run_queued_crawls(initialized_archive, env) + create_extract_snapshot(initialized_archive, env) with use_archivebox_db(initialized_archive): snapshot_id = Snapshot.objects.values_list("id", flat=True).first() @@ -92,13 +83,7 @@ def test_extract_plugin_option_accepted(initialized_archive): def test_extract_stdin_snapshot_id(initialized_archive): """Test that extract reads snapshot IDs from stdin.""" env = cli_env(disable_extractors=True) - - # First create a snapshot - run_archivebox_cmd( - ["add", "--index-only", "https://example.com"], - env=env, - ) - run_queued_crawls(initialized_archive, env) + create_extract_snapshot(initialized_archive, env) with use_archivebox_db(initialized_archive): snapshot_id = Snapshot.objects.values_list("id", flat=True).first() @@ -116,13 +101,7 @@ def test_extract_stdin_snapshot_id(initialized_archive): def test_extract_stdin_jsonl_input(initialized_archive): """Test that extract reads JSONL records from stdin.""" env = cli_env(disable_extractors=True) - - # First create a snapshot - run_archivebox_cmd( - ["add", "--index-only", "https://example.com"], - env=env, - ) - run_queued_crawls(initialized_archive, env) + create_extract_snapshot(initialized_archive, env) with use_archivebox_db(initialized_archive): snapshot_id = Snapshot.objects.values_list("id", flat=True).first() @@ -179,16 +158,8 @@ def test_extract_multiple_snapshots(initialized_archive): """Test extracting from multiple snapshots.""" env = cli_env(disable_extractors=True) - # Create multiple snapshots one at a time to avoid deduplication issues - run_archivebox_cmd( - ["add", "--index-only", "https://example.com"], - env=env, - ) - run_archivebox_cmd( - ["add", "--index-only", "https://iana.org"], - env=env, - ) - run_queued_crawls(initialized_archive, env) + create_extract_snapshot(initialized_archive, env, "https://example.com") + create_extract_snapshot(initialized_archive, env, "https://iana.org") with use_archivebox_db(initialized_archive): snapshot_ids = list(Snapshot.objects.values_list("id", flat=True))