diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 92cda6d3..c24381db 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -1197,9 +1197,11 @@ class CrawlRunner: snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first() if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED: return - if snapshot.status == Snapshot.StatusChoices.STARTED: - snapshot.sm.seal() - return + # Limit stops are runner-owned cancellation decisions, not normal + # "all ArchiveResults finished" lifecycle seals. Updating the row + # directly avoids racing the state machine's in-memory state while + # concurrent snapshot tasks are stopping because the crawl-wide limit + # has already been reached. snapshot.update_and_requeue( status=Snapshot.StatusChoices.SEALED, retry_at=None, diff --git a/archivebox/tests/test_config_DELETE_AFTER.py b/archivebox/tests/test_config_DELETE_AFTER.py index d01997da..6a4c0f0b 100644 --- a/archivebox/tests/test_config_DELETE_AFTER.py +++ b/archivebox/tests/test_config_DELETE_AFTER.py @@ -231,8 +231,9 @@ def test_delete_after_real_add_page_and_rest_create_paths(client): from archivebox.services.runner import run_due_crawl assert run_due_crawl(ui_crawl, lock_seconds=10) - ui_snapshot = ui_crawl.snapshot_set.get(url="https://example.com/delete-after-ui") + ui_snapshot = ui_crawl.snapshot_set.get(url="archivebox://internal") assert ui_snapshot.delete_at is not None + assert ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").read_text() == "https://example.com/delete-after-ui" from archivebox.api.auth import get_or_create_api_token diff --git a/archivebox/tests/test_config_MAX_limits.py b/archivebox/tests/test_config_MAX_limits.py index b3e44163..99da0254 100644 --- a/archivebox/tests/test_config_MAX_limits.py +++ b/archivebox/tests/test_config_MAX_limits.py @@ -287,9 +287,10 @@ def test_recursive_crawl_respects_max_urls(tmp_path, initialized_archive, recurs depth: Snapshot.objects.filter(depth=depth).count() for depth in set(Snapshot.objects.values_list("depth", flat=True)) } - assert crawl == (2, 4) + assert crawl == (3, 4) assert len(snapshot_rows) == 4 - assert depth_counts.get(0, 0) == 1 - assert depth_counts.get(1, 0) == 3 - assert depth_counts.get(2, 0) == 0 - assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 1}) + assert depth_counts.get(0, 0) == 0 + assert depth_counts.get(1, 0) == 1 + assert depth_counts.get(2, 0) == 3 + assert depth_counts.get(3, 0) == 0 + assert set(recursive_test_site["child_urls"]).issubset({url for url, depth, _parent in snapshot_rows if depth == 2})