The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Reopen crawls before queuing discovered snapshots

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-10 10:43:52 -07:00
parent 1fd8c8b7c2
commit 9149ee12bf
No known key found for this signature in database
2 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
archivebox/services/runner.py
View File

@ -771,6 +771,17 @@ class CrawlRunner:
if not discovered_urls:
return
if self.crawl.status == self.crawl.StatusChoices.SEALED:
# Snapshot completion projectors can observe the root snapshot seal
# before the runner has consumed parser urls.jsonl output. A sealed
# crawl must not block those freshly discovered child snapshots; the
# runner is still inside the same crawl lifecycle and will seal it
# again after the discovered queue is empty.
await sync_to_async(self.crawl.update_and_requeue, thread_sensitive=True)(
status=self.crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
parent_snapshot = await sync_to_async(
lambda: Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_payload["id"]).first(),
thread_sensitive=True,

4
archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py
View File

@ -42,7 +42,7 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path):
"tag": "api-cli",
"depth": 0,
"parser": "url_list",
"plugins": "wget",
"plugins": "parse_txt_urls,wget",
"update": True,
"overwrite": False,
"index_only": True,
@ -129,7 +129,7 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path):
crawl = (crawl_obj.max_depth, crawl_obj.tags_str, crawl_obj.config) if crawl_obj else None
assert crawl is not None
assert crawl[0] == 0
assert crawl[0] == 1
assert crawl[1] == "api-cli"
assert crawl[2]["INDEX_ONLY"] is True