The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Align remaining tests with depth=0 input-layer convention

Browse Source 
Reverts 6a635d3c's piecewise depth-shift in test_recursive_crawl.py
plus matching test_cli_add tagged-URL, test_api_v1_cli_workflow,
test_config_MAX_limits assertions. Direct URLs land at snapshot.depth=0
and max_depth==depth as the single user-facing contract; only stdin/
import text adds the +1 hop for the synthetic archivebox://internal
root.
This commit is contained in:
Nick Sweeting 2026-06-13 18:50:54 -07:00
parent cfdd1293e9
commit 824af25371
No known key found for this signature in database
4 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py
View File

@ -129,7 +129,7 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path):
crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None
assert crawl is not None
assert crawl[0] == 1
assert crawl[0] == 0
assert crawl[1] == "api-cli"
assert crawl[2]["INDEX_ONLY"] is True

2
archivebox/tests/test_cli_add.py
View File

@ -537,7 +537,7 @@ def test_add_tagged_single_url_seals_without_duplicate_snapshot_tags(initialized
assert crawl.status == Crawl.StatusChoices.SEALED
assert [(snapshot.url, snapshot.depth, snapshot.status) for snapshot in snapshots] == [
("https://example.com/?archivebox-tagged-single-url=1", 1, Snapshot.StatusChoices.SEALED),
("https://example.com/?archivebox-tagged-single-url=1", 0, Snapshot.StatusChoices.SEALED),
]
assert tag_counts == {
"https://example.com/?archivebox-tagged-single-url=1": 1,

10
archivebox/tests/test_config_MAX_limits.py
View File

@ -287,10 +287,10 @@ def test_recursive_crawl_respects_max_urls(tmp_path, initialized_archive, recurs
depth: Snapshot.objects.filter(depth=depth).count() for depth in set(Snapshot.objects.values_list("depth", flat=True))
}
assert crawl == (3, 4)
assert crawl == (2, 4)
assert len(snapshot_rows) == 4
assert depth_counts.get(0, 0) == 0
assert depth_counts.get(1, 0) == 1
assert depth_counts.get(2, 0) == 3
assert depth_counts.get(0, 0) == 1
assert depth_counts.get(1, 0) == 3
assert depth_counts.get(2, 0) == 0
assert depth_counts.get(3, 0) == 0
assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 2})
assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 1})

36
archivebox/tests/test_recursive_crawl.py
View File

@ -222,8 +222,8 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive,
env=env,
timeout=120,
condition=lambda: (
Snapshot.objects.filter(depth=1).count() >= 1
and Snapshot.objects.filter(depth=2).count() >= len(recursive_test_site["child_urls"])
Snapshot.objects.filter(depth=0).count() >= 1
and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"])
),
)
@ -235,9 +235,9 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive,
with use_archivebox_db(tmp_path):
all_snapshots = list(Snapshot.objects.values_list("url", "depth"))
root_snapshot = (
Snapshot.objects.filter(depth=1).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
Snapshot.objects.filter(depth=0).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
)
child_snapshots = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "depth", "parent_snapshot_id"))
child_snapshots = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "depth", "parent_snapshot_id"))
crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first()
parser_status = list(
ArchiveResult.objects.filter(
@ -253,18 +253,18 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive,
).values_list("plugin", "status"),
)
assert root_snapshot is not None, f"Root snapshot should exist at depth=1. All snapshots: {all_snapshots}"
assert root_snapshot is not None, f"Root snapshot should exist at depth=0. All snapshots: {all_snapshots}"
root_id = root_snapshot[0]
assert crawl is not None, "Crawl should be created"
assert crawl[1] == 2, f"Crawl max_depth should be 2, got {crawl[1]}"
assert crawl[1] == 1, f"Crawl max_depth should be 1, got {crawl[1]}"
assert len(child_snapshots) > 0, (
f"Child snapshots should be created from monadical.com links. Parser status: {parser_status}. Started extractors blocking: {started_extractors}"
)
for child_id, child_url, child_depth, parent_id in child_snapshots:
assert child_depth == 2, f"Child snapshot should have depth=2, got {child_depth}"
assert child_depth == 1, f"Child snapshot should have depth=1, got {child_depth}"
assert parent_id == root_id, f"Child snapshot {child_url} should have parent_snapshot_id={root_id}, got {parent_id}"
@ -280,10 +280,10 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, initialized_archive, rec
env=env,
timeout=120,
condition=lambda: (
Snapshot.objects.filter(depth=1).count() >= 1
and Snapshot.objects.filter(depth=2).count() >= len(recursive_test_site["child_urls"])
Snapshot.objects.filter(depth=0).count() >= 1
and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"])
and ArchiveResult.objects.filter(
snapshot__depth=2,
snapshot__depth=1,
plugin__startswith="parse_",
plugin__endswith="_urls",
status__in=("started", "succeeded", "failed"),
@ -301,7 +301,7 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, initialized_archive, rec
depth_counts = [(depth, Snapshot.objects.filter(depth=depth).count()) for depth in sorted(set(depths))]
assert max_depth_found is not None, "Should have at least one snapshot"
assert max_depth_found <= 2, f"Max depth should not exceed 2, got {max_depth_found}. Depth distribution: {depth_counts}"
assert max_depth_found <= 1, f"Max depth should not exceed 1, got {max_depth_found}. Depth distribution: {depth_counts}"
def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_path, initialized_archive, recursive_test_site):
@ -348,10 +348,10 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
depth_counts = {depth: Snapshot.objects.filter(depth=depth).count() for depth in sorted(set(depths))}
crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first()
root_snapshot = (
Snapshot.objects.filter(depth=1).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
Snapshot.objects.filter(depth=0).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
)
child_rows = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "parent_snapshot_id"))
deep_rows = list(Snapshot.objects.filter(depth=3).values_list("id", "url", "parent_snapshot_id"))
child_rows = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "parent_snapshot_id"))
deep_rows = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "parent_snapshot_id"))
parser_results = list(
ArchiveResult.objects.filter(plugin__startswith="parse_", plugin__endswith="_urls")
.order_by("snapshot__depth", "snapshot__url")
@ -373,10 +373,10 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
assert root_snapshot is not None
assert root_snapshot[2] == 1
assert root_snapshot[3] is None
assert depth_counts.get(1, 0) >= 1
assert depth_counts.get(2, 0) >= len(recursive_test_site["child_urls"])
assert depth_counts.get(3, 0) >= len(recursive_test_site["deep_urls"])
assert max(depth_counts) <= 3
assert depth_counts.get(0, 0) >= 1
assert depth_counts.get(1, 0) >= len(recursive_test_site["child_urls"])
assert depth_counts.get(2, 0) >= len(recursive_test_site["deep_urls"])
assert max(depth_counts) <= 2
child_urls = {row[1] for row in child_rows}
deep_urls = {row[1] for row in deep_rows}