Avoid blocking browser artifact uploads during crawl

archivebox/api/v1_core.py

@@ -963,6 +963,20 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
+    # Browser clients sync metadata and upload capture artifacts immediately
+    # after queueing a URL. The runner holds the crawl lifecycle lock for the
+    # whole crawl, so idempotently posting an already-created snapshot must not
+    # wait behind that potentially long-running archive job.
+    existing_snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if existing_snapshot is not None and (status is None or existing_snapshot.status == status):
+        if data.title is not None and existing_snapshot.title != data.title:
+            Snapshot.objects.filter(pk=existing_snapshot.pk).update(title=data.title, modified_at=timezone.now())
+            existing_snapshot.title = data.title
+        if tags:
+            existing_snapshot.save_tags(tags)
+        setattr(request, "with_archiveresults", False)
+        return existing_snapshot
+
     with crawl_lifecycle_lock(str(crawl.id)):
         snapshot_defaults = {
             "depth": data.depth,

archivebox/tests/test_api_v1_core_snapshots.py

@@ -1,6 +1,10 @@
+import threading
+import time
+
 import pytest
 
 from archivebox.core.models import Snapshot
+from archivebox.crawls.locks import crawl_lifecycle_lock
 from archivebox.crawls.models import Crawl
 
 
@@ -36,3 +40,48 @@ def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers
     items = payload["items"] if isinstance(payload, dict) and "items" in payload else payload
     assert [item["id"] for item in items] == [str(sealed_snapshot.id)]
     assert [item["status"] for item in items] == ["sealed"]
+
+
+def test_existing_snapshot_metadata_sync_does_not_wait_for_active_crawl(client, api_admin_user, api_headers):
+    url = "https://example.com/browser-extension-upload"
+    crawl = Crawl.objects.create(
+        urls=url,
+        created_by=api_admin_user,
+        status=Crawl.StatusChoices.STARTED,
+    )
+    snapshot = Snapshot.objects.create(
+        url=url,
+        crawl=crawl,
+        title="Original title",
+        status=Snapshot.StatusChoices.STARTED,
+    )
+    lock_acquired = threading.Event()
+
+    def hold_active_crawl_lock():
+        with crawl_lifecycle_lock(str(crawl.id)):
+            lock_acquired.set()
+            time.sleep(2)
+
+    holder = threading.Thread(target=hold_active_crawl_lock)
+    holder.start()
+    assert lock_acquired.wait(timeout=1)
+
+    started_at = time.monotonic()
+    response = client.post(
+        "/api/v1/core/snapshots",
+        data={
+            "url": url,
+            "crawl_id": str(crawl.id),
+            "depth": 0,
+            "title": "Browser title",
+            "status": Snapshot.StatusChoices.STARTED,
+        },
+        content_type="application/json",
+        **api_headers,
+    )
+    elapsed = time.monotonic() - started_at
+    holder.join(timeout=3)
+
+    assert response.status_code == 200, response.content
+    assert response.json()["id"] == str(snapshot.id)
+    assert elapsed < 1