diff --git a/archivebox/api/v1_core.py b/archivebox/api/v1_core.py index 8dfe4677..af6d65ff 100644 --- a/archivebox/api/v1_core.py +++ b/archivebox/api/v1_core.py @@ -963,6 +963,20 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema): created_by=request.user if isinstance(request.user, User) else None, ) + # Browser clients sync metadata and upload capture artifacts immediately + # after queueing a URL. The runner holds the crawl lifecycle lock for the + # whole crawl, so idempotently posting an already-created snapshot must not + # wait behind that potentially long-running archive job. + existing_snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first() + if existing_snapshot is not None and (status is None or existing_snapshot.status == status): + if data.title is not None and existing_snapshot.title != data.title: + Snapshot.objects.filter(pk=existing_snapshot.pk).update(title=data.title, modified_at=timezone.now()) + existing_snapshot.title = data.title + if tags: + existing_snapshot.save_tags(tags) + setattr(request, "with_archiveresults", False) + return existing_snapshot + with crawl_lifecycle_lock(str(crawl.id)): snapshot_defaults = { "depth": data.depth, diff --git a/archivebox/tests/test_api_v1_core_snapshots.py b/archivebox/tests/test_api_v1_core_snapshots.py index 41e1518d..30e95a7e 100644 --- a/archivebox/tests/test_api_v1_core_snapshots.py +++ b/archivebox/tests/test_api_v1_core_snapshots.py @@ -1,6 +1,10 @@ +import threading +import time + import pytest from archivebox.core.models import Snapshot +from archivebox.crawls.locks import crawl_lifecycle_lock from archivebox.crawls.models import Crawl @@ -36,3 +40,48 @@ def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers items = payload["items"] if isinstance(payload, dict) and "items" in payload else payload assert [item["id"] for item in items] == [str(sealed_snapshot.id)] assert [item["status"] for item in items] == ["sealed"] + + +def test_existing_snapshot_metadata_sync_does_not_wait_for_active_crawl(client, api_admin_user, api_headers): + url = "https://example.com/browser-extension-upload" + crawl = Crawl.objects.create( + urls=url, + created_by=api_admin_user, + status=Crawl.StatusChoices.STARTED, + ) + snapshot = Snapshot.objects.create( + url=url, + crawl=crawl, + title="Original title", + status=Snapshot.StatusChoices.STARTED, + ) + lock_acquired = threading.Event() + + def hold_active_crawl_lock(): + with crawl_lifecycle_lock(str(crawl.id)): + lock_acquired.set() + time.sleep(2) + + holder = threading.Thread(target=hold_active_crawl_lock) + holder.start() + assert lock_acquired.wait(timeout=1) + + started_at = time.monotonic() + response = client.post( + "/api/v1/core/snapshots", + data={ + "url": url, + "crawl_id": str(crawl.id), + "depth": 0, + "title": "Browser title", + "status": Snapshot.StatusChoices.STARTED, + }, + content_type="application/json", + **api_headers, + ) + elapsed = time.monotonic() - started_at + holder.join(timeout=3) + + assert response.status_code == 200, response.content + assert response.json()["id"] == str(snapshot.id) + assert elapsed < 1