import asyncio import json import subprocess import sys from pathlib import Path from types import SimpleNamespace import pytest from asgiref.sync import sync_to_async from django.test import RequestFactory pytestmark = pytest.mark.django_db class _DummyBus: def __init__(self, name: str): self.name = name self.registrations = [] self.emitted = [] def on(self, event_pattern, handler): registration = SimpleNamespace(event_pattern=event_pattern, handler=handler) self.registrations.append(registration) return registration def off(self, event_pattern, registration): self.registrations = [existing for existing in self.registrations if existing is not registration] def emit(self, event): self.emitted.append(event) bus = self class _Pending: def __getattr__(self, name): return getattr(event, name) async def now(self, *args, **kwargs): from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent if isinstance(event, SnapshotEvent): completed = SnapshotCompletedEvent( url=event.url, snapshot_id=event.snapshot_id, output_dir=event.output_dir, event_parent_id=event.event_id, ) completed._mark_completed() bus.emitted.append(completed) return event async def wait(self, *args, **kwargs): return event async def event_results_list(self): return [] return _Pending() async def find(self, event_type, where=None, child_of=None, **kwargs): for event in reversed(self.emitted): if not isinstance(event, event_type): continue if child_of is not None and event.event_parent_id != child_of.event_id: continue if where is not None and not where(event): continue return event return None async def stop(self): return None async def wait_until_idle(self): return None class _NoIdleBus(_DummyBus): async def wait_until_idle(self): raise AssertionError("run_snapshot should not wait on the whole crawl bus") class _DummyService: def __init__(self, *args, **kwargs): pass @pytest.mark.django_db(transaction=True) def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://blog.sweeting.me\nhttps://sweeting.me", created_by_id=get_or_create_system_user_pk(), ) snapshot_a = Snapshot.objects.create( url="https://blog.sweeting.me", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, ) snapshot_b = Snapshot.objects.create( url="https://sweeting.me", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, ) created_buses: list[_DummyBus] = [] original_create_bus = runner_module.create_bus def fake_create_bus(*, name, total_timeout=3600.0, **kwargs): bus = original_create_bus(name=name, total_timeout=total_timeout, **kwargs) created_buses.append(bus) return bus monkeypatch.setattr(runner_module, "create_bus", fake_create_bus) monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) monkeypatch.setattr(runner_module, "TagService", _DummyService) monkeypatch.setattr(runner_module, "CrawlService", _DummyService) monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) crawl_runner = runner_module.CrawlRunner(crawl) snapshot_data = { str(snapshot_a.id): { "id": str(snapshot_a.id), "url": snapshot_a.url, "status": snapshot_a.status, "title": snapshot_a.title, "timestamp": snapshot_a.timestamp, "bookmarked_at": snapshot_a.bookmarked_at.isoformat() if snapshot_a.bookmarked_at else "", "created_at": snapshot_a.created_at.isoformat() if snapshot_a.created_at else "", "tags": snapshot_a.tags_str(), "depth": snapshot_a.depth, "output_dir": str(snapshot_a.output_dir), "config": crawl_runner.load_snapshot_payload(str(snapshot_a.id))["config"], }, str(snapshot_b.id): { "id": str(snapshot_b.id), "url": snapshot_b.url, "status": snapshot_b.status, "title": snapshot_b.title, "timestamp": snapshot_b.timestamp, "bookmarked_at": snapshot_b.bookmarked_at.isoformat() if snapshot_b.bookmarked_at else "", "created_at": snapshot_b.created_at.isoformat() if snapshot_b.created_at else "", "tags": snapshot_b.tags_str(), "depth": snapshot_b.depth, "output_dir": str(snapshot_b.output_dir), "config": crawl_runner.load_snapshot_payload(str(snapshot_b.id))["config"], }, } monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_data[snapshot_id]) monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0)) asyncio.run(crawl_runner.run_crawl(str(snapshot_a.id), [str(snapshot_a.id), str(snapshot_b.id)])) from abx_dl.events import SnapshotEvent snapshot_events = asyncio.run(crawl_runner.bus.filter(SnapshotEvent, past=True)) assert len(snapshot_events) == 2 assert {event.snapshot_id for event in snapshot_events} == {str(snapshot_a.id), str(snapshot_b.id)} assert {event.url for event in snapshot_events} == {snapshot_a.url, snapshot_b.url} assert crawl_runner.bus is created_buses[0] assert len(created_buses) == 1 @pytest.mark.django_db(transaction=True) def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, ) monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) monkeypatch.setattr(runner_module, "TagService", _DummyService) monkeypatch.setattr(runner_module, "CrawlService", _DummyService) monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) crawl_runner = runner_module.CrawlRunner(crawl) snapshot_payload = crawl_runner.load_snapshot_payload(str(snapshot.id)) monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_payload) monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0)) crawl_runner.bus.wait_until_idle = _NoIdleBus("unused").wait_until_idle asyncio.run(crawl_runner.run_crawl(str(snapshot.id), [str(snapshot.id)])) @pytest.mark.django_db(transaction=True) def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.runner import CrawlRunner from abx_dl.events import CrawlAbortEvent, CrawlEvent crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, ) runner = CrawlRunner(crawl) async def run() -> CrawlAbortEvent | None: abort_event_holder: dict[str, CrawlAbortEvent | None] = {"event": None} async def on_CrawlEvent(event: CrawlEvent) -> None: watcher = asyncio.create_task(runner.watch_for_cancelled_crawl(event, poll_interval=0.01)) await asyncio.sleep(0.02) await sync_to_async(Crawl.objects.filter(id=crawl.id).update, thread_sensitive=True)( status=Crawl.StatusChoices.SEALED, retry_at=None, ) abort_event = await runner.bus.find(CrawlAbortEvent, child_of=event, past=True, future=1.0) abort_event_holder["event"] = abort_event if isinstance(abort_event, CrawlAbortEvent) else None await watcher runner.bus.on(CrawlEvent, on_CrawlEvent) await runner.bus.emit( CrawlEvent( url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=str(crawl.output_dir), ), ).now() await runner.bus.wait_until_idle() return abort_event_holder["event"] abort_event = asyncio.run(run()) assert abort_event is not None @pytest.mark.django_db(transaction=True) def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.runner import CrawlRunner crawl = Crawl.objects.create( urls="https://example.com", max_depth=0, max_urls=5, created_by_id=get_or_create_system_user_pk(), ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.SEALED, depth=0, ) parser_dir = Path(snapshot.output_dir) / "parse_html_urls" parser_dir.mkdir(parents=True, exist_ok=True) (parser_dir / "urls.jsonl").write_text( "\n".join( [ json.dumps({"type": "Snapshot", "url": "https://example.com/child-a", "depth": 1}), json.dumps({"type": "Snapshot", "url": "https://example.com/child-b", "depth": 1}), "", ], ), ) runner = CrawlRunner(crawl) Crawl.objects.filter(id=crawl.id).update(max_depth=1) payload = runner.load_snapshot_payload(str(snapshot.id)) asyncio.run(runner.enqueue_discovered_snapshots_from_outputs(payload)) child_snapshots = list(crawl.snapshot_set.filter(depth=1).order_by("url").values_list("url", "status")) assert child_snapshots == [ ("https://example.com/child-a", Snapshot.StatusChoices.QUEUED), ("https://example.com/child-b", Snapshot.StatusChoices.QUEUED), ] def test_ensure_background_runner_starts_when_none_running(monkeypatch): import archivebox.machine.models as machine_models from archivebox.services import runner as runner_module popen_calls = [] class DummyPopen: def __init__(self, args, **kwargs): popen_calls.append((args, kwargs)) monkeypatch.setattr(machine_models.Process, "cleanup_stale_running", classmethod(lambda cls, machine=None: 0)) monkeypatch.setattr(machine_models.Process, "cleanup_orphaned_workers", classmethod(lambda cls: 0)) monkeypatch.setattr(machine_models.Machine, "current", classmethod(lambda cls: SimpleNamespace(id="machine-1"))) monkeypatch.setattr( machine_models.Process.objects, "filter", lambda **kwargs: SimpleNamespace(exists=lambda: False), ) monkeypatch.setattr(runner_module.subprocess, "Popen", DummyPopen) started = runner_module.ensure_background_runner(allow_under_pytest=True) assert started is True assert len(popen_calls) == 1 assert popen_calls[0][0] == [runner_module.sys.executable, "-m", "archivebox", "run", "--daemon"] assert popen_calls[0][1]["stdin"] is subprocess.DEVNULL def test_ensure_background_runner_skips_when_orchestrator_running(monkeypatch): import archivebox.machine.models as machine_models from archivebox.services import runner as runner_module monkeypatch.setattr(machine_models.Process, "cleanup_stale_running", classmethod(lambda cls, machine=None: 0)) monkeypatch.setattr(machine_models.Process, "cleanup_orphaned_workers", classmethod(lambda cls: 0)) monkeypatch.setattr(machine_models.Machine, "current", classmethod(lambda cls: SimpleNamespace(id="machine-1"))) monkeypatch.setattr( machine_models.Process.objects, "filter", lambda **kwargs: SimpleNamespace(exists=lambda: True), ) monkeypatch.setattr( runner_module.subprocess, "Popen", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("runner should not be spawned")), ) started = runner_module.ensure_background_runner(allow_under_pytest=True) assert started is False def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path): from abx_dl.events import CrawlEvent, MachineEvent from abx_dl.orchestrator import create_bus from abxbus.event_bus import in_handler_context from archivebox.services import runner as runner_module bus = create_bus(name="test_runner_task_context_clears_inherited_abxbus_handler_context") observations = [] async def emit_from_runner_task(): observations.append(("in_handler_context", in_handler_context())) machine_event = bus.emit(MachineEvent(config={"ABX_RUNTIME": "archivebox"}, config_type="user")) await machine_event.now() observations.append(("machine_event_path", bool(machine_event.event_path))) async def on_crawl(event): assert in_handler_context() is True task = asyncio.create_task(emit_from_runner_task(), context=runner_module._runner_task_context()) await task bus.on(CrawlEvent, on_crawl) async def run_test(): try: await bus.emit( CrawlEvent( url="https://example.com", snapshot_id="snapshot-1", output_dir=str(tmp_path), ), ).now() await bus.wait_until_idle() finally: await bus.destroy() asyncio.run(run_test()) assert observations == [ ("in_handler_context", False), ("machine_event_path", True), ] @pytest.mark.django_db(transaction=True) def test_machine_service_persists_only_derived_config_events(): from abx_dl.events import MachineEvent from abx_dl.orchestrator import create_bus from archivebox.machine.models import Machine from archivebox.services.machine_service import MachineService machine = Machine.current() machine.config = {} machine.save(update_fields=["config"]) async def run_test(): bus = create_bus(name="test_machine_service_persists_only_derived_config_events") try: MachineService(bus) user_event = bus.emit( MachineEvent( config={ "CHROME_ISOLATION": "snapshot", "CHROME_USER_DATA_DIR": "/tmp/stale-profile", "ABX_RUNTIME": "archivebox", }, config_type="user", ), ) await user_event.now() await user_event.event_results_list() derived_event = bus.emit( MachineEvent( config={ "WGET_BINARY": "/tmp/wget", "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"}, "CHROME_USER_DATA_DIR": "/tmp/stale-derived-profile", }, config_type="derived", ), ) await derived_event.now() await derived_event.event_results_list() await bus.wait_until_idle() finally: await bus.destroy() asyncio.run(run_test()) machine.refresh_from_db() assert machine.config == { "WGET_BINARY": "/tmp/wget", "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"}, } def test_runner_prepare_refreshes_network_interface_and_attaches_current_process(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) class _Iface: id = "iface-1" machine = SimpleNamespace(id="machine-1") machine_id = "machine-1" saved_updates = [] class _Proc: iface_id = None machine_id = "machine-1" iface = None machine = None def save(self, *, update_fields): saved_updates.append(tuple(update_fields)) proc = _Proc() monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) monkeypatch.setattr(runner_module, "TagService", _DummyService) monkeypatch.setattr(runner_module, "CrawlService", _DummyService) monkeypatch.setattr(runner_module, "SnapshotService", _DummyService) monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) from archivebox.machine.models import NetworkInterface, Process from archivebox.config import common as config_common refresh_calls = [] monkeypatch.setattr(NetworkInterface, "current", classmethod(lambda cls, refresh=False: refresh_calls.append(refresh) or _Iface())) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) original_get_config = config_common.get_config monkeypatch.setattr( config_common, "get_config", lambda **kwargs: original_get_config( overrides={"PLUGINS": "", "CHROME_BINARY": "", "CHROME_KEEPALIVE": False, "TIMEOUT": 60}, **kwargs, ), ) crawl_runner = runner_module.CrawlRunner(crawl) crawl_runner.load_run_state() assert refresh_calls == [True] assert proc.iface is not None assert proc.machine == proc.iface.machine assert saved_updates == [("iface", "machine", "modified_at")] def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch): from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services import runner as runner_module from archivebox.config import common as config_common from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl machine = Machine.objects.create( guid="test-guid-runner-overrides", hostname="runner-host", hw_in_docker=False, hw_in_vm=False, hw_manufacturer="Test", hw_product="Test Product", hw_uuid="test-hw-runner-overrides", os_arch="arm64", os_family="darwin", os_platform="macOS", os_release="14.0", os_kernel="Darwin", stats={}, config={ "WGET_BINARY": "/tmp/wget", "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"}, "CHROME_ISOLATION": "snapshot", "CHROME_USER_DATA_DIR": "/tmp/stale-profile", }, ) crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) proc = SimpleNamespace(iface_id=str(machine.id), machine_id=str(machine.id), iface=None, machine=machine, save=lambda **kwargs: None) monkeypatch.setattr( NetworkInterface, "current", classmethod(lambda cls, refresh=False: SimpleNamespace(id=machine.id, machine=machine)), ) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) original_get_config = config_common.get_config monkeypatch.setattr( config_common, "get_config", lambda **kwargs: original_get_config(overrides={"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs), ) crawl_runner = runner_module.CrawlRunner(crawl) crawl_runner.load_run_state() assert crawl_runner.derived_config == { "WGET_BINARY": "/tmp/wget", "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"}, } def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch): from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services import runner as runner_module from archivebox.config import common as config_common from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl machine = Machine.objects.create( guid="test-guid-runner-chrome-keepalive", hostname="runner-host-chrome-keepalive", hw_in_docker=False, hw_in_vm=False, hw_manufacturer="Test", hw_product="Test Product", hw_uuid="test-hw-runner-chrome-keepalive", os_arch="arm64", os_family="darwin", os_platform="macOS", os_release="14.0", os_kernel="Darwin", stats={}, config={}, ) crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) proc = SimpleNamespace(iface_id=str(machine.id), machine_id=str(machine.id), iface=None, machine=machine, save=lambda **kwargs: None) monkeypatch.setattr( NetworkInterface, "current", classmethod(lambda cls, refresh=False: SimpleNamespace(id=machine.id, machine=machine)), ) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) original_get_config = config_common.get_config monkeypatch.setattr( config_common, "get_config", lambda **kwargs: original_get_config(overrides={"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs), ) crawl_runner = runner_module.CrawlRunner(crawl) crawl_runner.load_run_state() assert crawl_runner.base_config["CHROME_KEEPALIVE"] is False def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatch): from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services import runner as runner_module from archivebox.config import common as config_common from archivebox import hooks as hooks_module from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from pathlib import Path machine = Machine.objects.create( guid="test-guid-runner-missing-plugins", hostname="runner-host-missing-plugins", hw_in_docker=False, hw_in_vm=False, hw_manufacturer="Test", hw_product="Test Product", hw_uuid="test-hw-runner-missing-plugins", os_arch="arm64", os_family="darwin", os_platform="macOS", os_release="14.0", os_kernel="Darwin", stats={}, config={}, ) crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) proc = SimpleNamespace(iface_id=str(machine.id), machine_id=str(machine.id), iface=None, machine=machine, save=lambda **kwargs: None) monkeypatch.setattr( NetworkInterface, "current", classmethod(lambda cls, refresh=False: SimpleNamespace(id=machine.id, machine=machine)), ) monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) original_get_config = config_common.get_config monkeypatch.setattr( config_common, "get_config", lambda **kwargs: original_get_config(overrides={"CHROME_BINARY": "", "TIMEOUT": 60}, **kwargs), ) monkeypatch.setattr( hooks_module, "discover_hooks", lambda event_name, config=None: ( [ Path(f"/tmp/{event_name.lower()}/wget/on_{event_name}__test.py"), Path(f"/tmp/{event_name.lower()}/favicon/on_{event_name}__test.py"), ] if event_name in {"CrawlSetup", "Snapshot"} else [] ), ) crawl_runner = runner_module.CrawlRunner(crawl) snapshot_ids = crawl_runner.load_run_state() assert crawl_runner.selected_plugins == ["favicon", "wget"] assert len(snapshot_ids) == 1 @pytest.mark.django_db(transaction=True) def test_run_snapshot_skips_descendant_when_max_size_already_reached(monkeypatch, tmp_path): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), max_size=16, ) monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) monkeypatch.setattr(runner_module, "TagService", _DummyService) monkeypatch.setattr(runner_module, "CrawlService", _DummyService) monkeypatch.setattr(runner_module, "SnapshotService", _DummyService) monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) crawl_runner = runner_module.CrawlRunner(crawl) state_dir = tmp_path / ".abx-dl" state_dir.mkdir(parents=True, exist_ok=True) (state_dir / "limits.json").write_text( json.dumps( { "admitted_snapshot_ids": ["child-1"], "counted_process_ids": ["proc-1"], "total_size": 32, "stop_reason": "max_size", }, ), encoding="utf-8", ) cancelled: list[str] = [] crawl_runner.load_snapshot_payload = lambda snapshot_id: { "id": snapshot_id, "url": "https://example.com/child", "title": "", "timestamp": "", "bookmarked_at": "", "created_at": "", "tags": "", "depth": 1, "status": "queued", "output_dir": "/tmp/child", "config": {"CRAWL_DIR": str(tmp_path), "MAX_SIZE": 16}, } crawl_runner.seal_snapshot_due_to_limit = lambda snapshot_id: cancelled.append(snapshot_id) async def run_in_crawl_start_context() -> None: from abx_dl.events import CrawlStartEvent async def run_child_snapshot(event: CrawlStartEvent) -> None: await crawl_runner.run_snapshot("child-1") crawl_runner.bus.on(CrawlStartEvent, run_child_snapshot) await crawl_runner.bus.emit( CrawlStartEvent( url="https://example.com", snapshot_id="child-1", output_dir=str(tmp_path), event_timeout=0, event_handler_timeout=0, ), ).now() asyncio.run(run_in_crawl_start_context()) assert cancelled == ["child-1"] @pytest.mark.django_db(transaction=True) def test_seal_snapshot_cancels_queued_descendants_after_max_size(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.snapshot_service import SnapshotService from abx_dl.events import SnapshotCompletedEvent from abx_dl.orchestrator import create_bus crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), max_size=16, ) root = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, ) child = Snapshot.objects.create( url="https://example.com/child", crawl=crawl, depth=1, parent_snapshot_id=root.id, status=Snapshot.StatusChoices.QUEUED, ) state_dir = Path(crawl.output_dir) / ".abx-dl" state_dir.mkdir(parents=True, exist_ok=True) (state_dir / "limits.json").write_text( json.dumps( { "admitted_snapshot_ids": [str(root.id), str(child.id)], "counted_process_ids": ["proc-1"], "total_size": 32, "stop_reason": "max_size", }, ), encoding="utf-8", ) bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}") service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: None) try: async def emit_event() -> None: await service.on_SnapshotCompletedEvent( SnapshotCompletedEvent( url=root.url, snapshot_id=str(root.id), output_dir=str(root.output_dir), ), ) asyncio.run(emit_event()) finally: asyncio.run(bus.wait_until_idle()) asyncio.run(bus.destroy()) root.refresh_from_db() child.refresh_from_db() assert root.status == Snapshot.StatusChoices.SEALED assert child.status == Snapshot.StatusChoices.SEALED assert child.retry_at is None def test_sealed_crawl_does_not_create_discovered_snapshots(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.SEALED, retry_at=None, max_depth=3, ) root = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.SEALED, retry_at=None, ) assert crawl.create_snapshots_from_urls() == [] assert crawl.create_discovered_snapshot(root, url="https://example.com/child", depth=1) is None assert crawl.snapshot_set.count() == 1 def test_create_crawl_api_queues_crawl_without_spawning_runner(monkeypatch): from django.contrib.auth import get_user_model from archivebox.api.v1_crawls import CrawlCreateSchema, create_crawl user = get_user_model().objects.create_superuser( username="runner-api-admin", email="runner-api-admin@example.com", password="testpassword", ) request = RequestFactory().post("/api/v1/crawls") request.user = user crawl = create_crawl( request, CrawlCreateSchema( urls=["https://example.com"], max_depth=0, tags=[], tags_str="", label="", notes="", config={}, ), ) assert str(crawl.id) assert crawl.status == "queued" assert crawl.retry_at is not None def test_crawl_runner_does_not_seal_unfinished_crawl(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, ) monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) monkeypatch.setattr(runner_module.CrawlRunner, "load_run_state", lambda self: [str(snapshot.id)]) monkeypatch.setattr( runner_module.CrawlRunner, "load_snapshot_payload", lambda self, _snapshot_id: { "id": str(snapshot.id), "url": snapshot.url, "depth": snapshot.depth, "output_dir": str(snapshot.output_dir), }, ) monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None) monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl", lambda self, root_snapshot_id, snapshot_ids: asyncio.sleep(0)) monkeypatch.setattr(runner_module.CrawlRunner, "finalize_run_state", lambda self: None) asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run()) crawl.refresh_from_db() assert crawl.status != Crawl.StatusChoices.SEALED assert crawl.retry_at is not None def test_crawl_runner_calls_load_and_finalize_run_state(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, ) monkeypatch.setattr(runner_module, "create_bus", lambda *args, **kwargs: _DummyBus("runner")) monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) monkeypatch.setattr(runner_module, "BinaryService", _DummyService) monkeypatch.setattr(runner_module, "TagService", _DummyService) monkeypatch.setattr(runner_module, "CrawlService", _DummyService) monkeypatch.setattr(runner_module, "SnapshotService", _DummyService) monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) monkeypatch.setattr(runner_module.CrawlRunner, "load_run_state", lambda self: [str(snapshot.id)]) monkeypatch.setattr( runner_module.CrawlRunner, "load_snapshot_payload", lambda self, _snapshot_id: { "id": str(snapshot.id), "url": snapshot.url, "depth": snapshot.depth, "output_dir": str(snapshot.output_dir), }, ) monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None) monkeypatch.setattr(runner_module.CrawlRunner, "run_crawl", lambda self, root_snapshot_id, snapshot_ids: asyncio.sleep(0)) monkeypatch.setenv("DJANGO_ALLOW_ASYNC_UNSAFE", "true") method_calls: list[str] = [] def wrapped_finalize(self): method_calls.append("finalize_run_state") return None def wrapped_load(self): method_calls.append("load_run_state") return [str(snapshot.id)] monkeypatch.setattr(runner_module.CrawlRunner, "finalize_run_state", wrapped_finalize) monkeypatch.setattr(runner_module.CrawlRunner, "load_run_state", wrapped_load) asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run()) crawl.refresh_from_db() assert crawl.status == Crawl.StatusChoices.STARTED assert crawl.retry_at is not None assert method_calls == ["load_run_state", "finalize_run_state"] def test_wait_for_snapshot_tasks_surfaces_already_failed_task(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) crawl_runner = runner_module.CrawlRunner(crawl) async def run_test(): task = asyncio.get_running_loop().create_future() task.set_exception(RuntimeError("snapshot failed")) crawl_runner.snapshot_tasks["snap-1"] = task with pytest.raises(RuntimeError, match="snapshot failed"): await crawl_runner.wait_for_snapshot_tasks() asyncio.run(run_test()) def test_wait_for_snapshot_tasks_returns_after_completed_tasks_are_pruned(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), ) crawl_runner = runner_module.CrawlRunner(crawl) async def finish_snapshot() -> None: await asyncio.sleep(0) async def run_test(): task = asyncio.create_task(finish_snapshot()) crawl_runner.snapshot_tasks["snap-1"] = task await asyncio.wait_for(crawl_runner.wait_for_snapshot_tasks(), timeout=0.5) assert crawl_runner.snapshot_tasks == {} asyncio.run(run_test()) def test_crawl_runner_calls_crawl_lifecycle(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, ) monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) monkeypatch.setattr(runner_module.CrawlRunner, "load_run_state", lambda self: [str(snapshot.id)]) monkeypatch.setattr( runner_module.CrawlRunner, "load_snapshot_payload", lambda self, _snapshot_id: { "id": str(snapshot.id), "url": snapshot.url, "depth": snapshot.depth, "output_dir": str(snapshot.output_dir), }, ) monkeypatch.setattr(runner_module.CrawlRunner, "_create_live_ui", lambda self: None) monkeypatch.setattr(runner_module.CrawlRunner, "finalize_run_state", lambda self: None) lifecycle_calls = [] monkeypatch.setattr( runner_module.CrawlRunner, "run_crawl", lambda self, root_snapshot_id, snapshot_ids: lifecycle_calls.append((root_snapshot_id, snapshot_ids)) or asyncio.sleep(0), ) asyncio.run(runner_module.CrawlRunner(crawl, snapshot_ids=[str(snapshot.id)]).run()) assert lifecycle_calls == [(str(snapshot.id), [str(snapshot.id)])] def test_abx_process_service_background_process_finishes_after_process_exit(monkeypatch, tmp_path): from abx_dl.events import ProcessCompletedEvent, ProcessEvent from abx_dl.orchestrator import create_bus from abx_dl.services.process_service import ProcessService bus = create_bus(name="test_abx_process_service_background_process_finishes_after_process_exit") service = ProcessService(bus, emit_jsonl=False, interactive_tty=False) emitted_events = [] async def collect_completed(event): emitted_events.append(event) bus.on(ProcessCompletedEvent, collect_completed) async def fake_stream_stdout(**kwargs): return ["daemon output\n"] monkeypatch.setattr(service, "_stream_stdout", fake_stream_stdout) plugin_output_dir = tmp_path / "chrome" plugin_output_dir.mkdir() # stdout_file = plugin_output_dir / "on_CrawlSetup__90_chrome_launch.daemon.bg.stdout.log" stderr_file = plugin_output_dir / "on_CrawlSetup__90_chrome_launch.daemon.bg.stderr.log" stderr_file.write_text("") pid_file = plugin_output_dir / "on_CrawlSetup__90_chrome_launch.daemon.bg.pid" pid_file.write_text("12345") async def run_test(): try: event = ProcessEvent( plugin_name="chrome", hook_name="on_CrawlSetup__90_chrome_launch.daemon.bg", hook_path=sys.executable, hook_args=["-c", "pass"], env={}, output_dir=str(plugin_output_dir), timeout=60, is_background=True, url="https://example.org/", process_type="hook", worker_type="hook", ) await asyncio.wait_for(bus.emit(event).now(), timeout=0.5) await bus.wait_until_idle() finally: await bus.destroy() asyncio.run(run_test()) assert pid_file.exists() is False assert any(isinstance(event, ProcessCompletedEvent) for event in emitted_events) def test_run_pending_crawls_runs_due_snapshot_in_place(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services import runner as runner_module crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.SEALED, ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=runner_module.timezone.now(), ) monkeypatch.setattr(type(snapshot), "claim_processing_lock", lambda self, lock_seconds=60: True) monkeypatch.setattr(type(crawl), "claim_processing_lock", lambda self, lock_seconds=60: True) run_calls: list[tuple[str, list[str] | None, bool]] = [] def fake_run_crawl(crawl_id, snapshot_ids=None, selected_plugins=None, process_discovered_snapshots_inline=True): run_calls.append((crawl_id, snapshot_ids, process_discovered_snapshots_inline)) snapshot.status = Snapshot.StatusChoices.SEALED snapshot.retry_at = None snapshot.save(update_fields=["status", "retry_at", "modified_at"]) monkeypatch.setattr(runner_module, "run_crawl", fake_run_crawl) result = runner_module.run_pending_crawls(daemon=False) assert result == 0 assert run_calls == [(str(crawl.id), [str(snapshot.id)], True)] def test_run_pending_crawls_prioritizes_new_queued_crawl_before_snapshot_backlog(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services import runner as runner_module older_crawl = Crawl.objects.create( urls="https://older.example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, ) older_snapshot = Snapshot.objects.create( url="https://older.example.com", crawl=older_crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=runner_module.timezone.now(), ) newer_crawl = Crawl.objects.create( urls="https://newer.example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.QUEUED, retry_at=runner_module.timezone.now(), ) monkeypatch.setattr(type(older_snapshot), "claim_processing_lock", lambda self, lock_seconds=60: True) monkeypatch.setattr(type(older_crawl), "claim_processing_lock", lambda self, lock_seconds=60: True) monkeypatch.setattr(type(newer_crawl), "claim_processing_lock", lambda self, lock_seconds=60: True) run_calls: list[tuple[str, list[str] | None, bool]] = [] class _StopScheduling(Exception): pass def fake_run_crawl(crawl_id, snapshot_ids=None, selected_plugins=None, process_discovered_snapshots_inline=True): run_calls.append((crawl_id, snapshot_ids, process_discovered_snapshots_inline)) raise _StopScheduling monkeypatch.setattr(runner_module, "run_crawl", fake_run_crawl) with pytest.raises(_StopScheduling): runner_module.run_pending_crawls(daemon=False) assert run_calls == [(str(newer_crawl.id), None, True)] def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_backlog(monkeypatch): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.machine.models import Binary, Machine from archivebox.services import runner as runner_module queued_crawl = Crawl.objects.create( urls="https://scheduled.example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.QUEUED, retry_at=runner_module.timezone.now(), ) unrelated_binary = Binary.objects.create( machine=Machine.current(), name="papers-dl", status=Binary.StatusChoices.QUEUED, retry_at=runner_module.timezone.now(), ) monkeypatch.setattr(type(queued_crawl), "claim_processing_lock", lambda self, lock_seconds=60: True) monkeypatch.setattr(type(unrelated_binary), "claim_processing_lock", lambda self, lock_seconds=60: True) run_calls: list[tuple[str, list[str] | None, bool]] = [] binary_calls: list[str] = [] class _StopScheduling(Exception): pass def fake_run_crawl(crawl_id, snapshot_ids=None, selected_plugins=None, process_discovered_snapshots_inline=True): run_calls.append((crawl_id, snapshot_ids, process_discovered_snapshots_inline)) raise _StopScheduling def fake_run_binary(binary_id): binary_calls.append(binary_id) monkeypatch.setattr(runner_module, "run_crawl", fake_run_crawl) monkeypatch.setattr(runner_module, "run_binary", fake_run_binary) with pytest.raises(_StopScheduling): runner_module.run_pending_crawls(daemon=False) assert run_calls == [(str(queued_crawl.id), None, True)] assert binary_calls == [] @pytest.mark.django_db(transaction=True) def test_crawl_completed_event_requeues_active_snapshots(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.crawl_service import CrawlService from abx_dl.events import CrawlCompletedEvent from abx_dl.orchestrator import create_bus crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, retry_at=None, ) Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, retry_at=None, ) bus = create_bus(name=f"test_crawl_completed_active_snapshots_{str(crawl.id).replace('-', '_')}") CrawlService(bus, crawl_id=str(crawl.id)) try: async def emit_completed() -> None: event = CrawlCompletedEvent( url="https://example.com", snapshot_id="", output_dir=str(crawl.output_dir), ) emitted = bus.emit(event) await emitted.now() await emitted.event_results_list() asyncio.run(emit_completed()) finally: asyncio.run(bus.wait_until_idle()) asyncio.run(bus.destroy()) crawl.refresh_from_db() assert crawl.status == Crawl.StatusChoices.STARTED assert crawl.retry_at is not None @pytest.mark.django_db(transaction=True) def test_crawl_cleanup_event_requeues_unfinished_crawl(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.crawl_service import CrawlService from abx_dl.events import CrawlCleanupEvent from abx_dl.orchestrator import create_bus crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, retry_at=None, ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=None, ) bus = create_bus(name=f"test_crawl_cleanup_requeues_unfinished_{str(crawl.id).replace('-', '_')}") CrawlService(bus, crawl_id=str(crawl.id)) try: async def emit_cleanup() -> None: event = CrawlCleanupEvent( url="https://example.com", snapshot_id=str(snapshot.id), output_dir=str(crawl.output_dir), ) emitted = bus.emit(event) await emitted.now() await emitted.event_results_list() asyncio.run(emit_cleanup()) finally: asyncio.run(bus.wait_until_idle()) asyncio.run(bus.destroy()) crawl.refresh_from_db() assert crawl.status == Crawl.StatusChoices.STARTED assert crawl.retry_at is not None @pytest.mark.django_db(transaction=True) def test_crawl_cleanup_event_seals_finished_crawl(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.crawl_service import CrawlService from abx_dl.events import CrawlCleanupEvent from abx_dl.orchestrator import create_bus from django.utils import timezone crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, retry_at=timezone.now(), ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.SEALED, retry_at=None, ) bus = create_bus(name=f"test_crawl_cleanup_finished_crawl_{str(crawl.id).replace('-', '_')}") CrawlService(bus, crawl_id=str(crawl.id)) try: async def emit_cleanup() -> None: event = CrawlCleanupEvent( url="https://example.com", snapshot_id=str(snapshot.id), output_dir=str(crawl.output_dir), ) emitted = bus.emit(event) await emitted.now() await emitted.event_results_list() asyncio.run(emit_cleanup()) finally: asyncio.run(bus.wait_until_idle()) asyncio.run(bus.destroy()) crawl.refresh_from_db() assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None @pytest.mark.django_db(transaction=True) def test_snapshot_completed_event_seals_finished_crawl(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.snapshot_service import SnapshotService from abx_dl.events import SnapshotCompletedEvent from abx_dl.orchestrator import create_bus from django.utils import timezone crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, retry_at=timezone.now(), ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, retry_at=None, ) bus = create_bus(name=f"test_snapshot_completed_finished_crawl_{str(crawl.id).replace('-', '_')}") service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0)) try: async def emit_completed() -> None: await service.on_SnapshotCompletedEvent( SnapshotCompletedEvent( url="https://example.com", snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir), ), ) asyncio.run(emit_completed()) finally: asyncio.run(bus.destroy()) snapshot.refresh_from_db() crawl.refresh_from_db() assert snapshot.status == Snapshot.StatusChoices.SEALED assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None @pytest.mark.django_db(transaction=True) def test_snapshot_completed_event_bus_seals_finished_crawl(): from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.services.snapshot_service import SnapshotService from abx_dl.events import SnapshotCompletedEvent from abx_dl.orchestrator import create_bus from django.utils import timezone crawl = Crawl.objects.create( urls="https://example.com", created_by_id=get_or_create_system_user_pk(), status=Crawl.StatusChoices.STARTED, retry_at=timezone.now(), ) snapshot = Snapshot.objects.create( url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.STARTED, retry_at=None, ) bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}") service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0)) assert service is not None try: async def emit_completed() -> None: emitted = bus.emit( SnapshotCompletedEvent( url="https://example.com", snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir), ), ) await emitted.wait() await emitted.event_results_list() asyncio.run(emit_completed()) finally: asyncio.run(bus.wait_until_idle()) asyncio.run(bus.destroy()) snapshot.refresh_from_db() crawl.refresh_from_db() assert snapshot.status == Snapshot.StatusChoices.SEALED assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None