From 9149ee12bfbc1fefc15bc8e686b595fa1d509192 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 10 Jun 2026 10:43:52 -0700 Subject: [PATCH] Reopen crawls before queuing discovered snapshots --- archivebox/services/runner.py | 11 +++++++++++ ...st_api_v1_cli_workflow_add_search_update_remove.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 4b8218ad..1010582f 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -771,6 +771,17 @@ class CrawlRunner: if not discovered_urls: return + if self.crawl.status == self.crawl.StatusChoices.SEALED: + # Snapshot completion projectors can observe the root snapshot seal + # before the runner has consumed parser urls.jsonl output. A sealed + # crawl must not block those freshly discovered child snapshots; the + # runner is still inside the same crawl lifecycle and will seal it + # again after the discovered queue is empty. + await sync_to_async(self.crawl.update_and_requeue, thread_sensitive=True)( + status=self.crawl.StatusChoices.STARTED, + retry_at=timezone.now(), + ) + parent_snapshot = await sync_to_async( lambda: Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_payload["id"]).first(), thread_sensitive=True, diff --git a/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py b/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py index ca5c3b70..175e4dd9 100644 --- a/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py +++ b/archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py @@ -42,7 +42,7 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path): "tag": "api-cli", "depth": 0, "parser": "url_list", - "plugins": "wget", + "plugins": "parse_txt_urls,wget", "update": True, "overwrite": False, "index_only": True, @@ -129,7 +129,7 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path): crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None assert crawl is not None - assert crawl[0] == 0 + assert crawl[0] == 1 assert crawl[1] == "api-cli" assert crawl[2]["INDEX_ONLY"] is True