mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
Find snapshot completion events through bus history
This commit is contained in:
parent
7e97dbf14c
commit
1c0d8b44a0
@ -583,14 +583,13 @@ class CrawlRunner:
|
||||
)
|
||||
emitted_snapshot_event = self.bus.emit(snapshot_event)
|
||||
await emitted_snapshot_event.now()
|
||||
await emitted_snapshot_event.event_results_list()
|
||||
snapshot_event_results = getattr(snapshot_event, "event_results", {})
|
||||
snapshot_event_children = getattr(snapshot_event, "event_children", [])
|
||||
completed_snapshot = next(
|
||||
(child for child in snapshot_event_children if isinstance(child, SnapshotCompletedEvent)),
|
||||
None,
|
||||
completed_snapshot = await self.bus.find(
|
||||
SnapshotCompletedEvent,
|
||||
child_of=emitted_snapshot_event,
|
||||
past=True,
|
||||
future=snapshot_phase_timeout,
|
||||
)
|
||||
if snapshot_event_results and completed_snapshot is None:
|
||||
if completed_snapshot is None:
|
||||
raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
|
||||
await self.enqueue_discovered_snapshots_from_outputs(snapshot)
|
||||
finally:
|
||||
|
||||
@ -28,9 +28,24 @@ class _DummyBus:
|
||||
|
||||
def emit(self, event):
|
||||
self.emitted.append(event)
|
||||
bus = self
|
||||
|
||||
class _Pending:
|
||||
def __getattr__(self, name):
|
||||
return getattr(event, name)
|
||||
|
||||
async def now(self, *args, **kwargs):
|
||||
from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
|
||||
|
||||
if isinstance(event, SnapshotEvent):
|
||||
bus.emitted.append(
|
||||
SnapshotCompletedEvent(
|
||||
url=event.url,
|
||||
snapshot_id=event.snapshot_id,
|
||||
output_dir=event.output_dir,
|
||||
event_parent_id=event.event_id,
|
||||
),
|
||||
)
|
||||
return event
|
||||
|
||||
async def wait(self, *args, **kwargs):
|
||||
@ -41,6 +56,17 @@ class _DummyBus:
|
||||
|
||||
return _Pending()
|
||||
|
||||
async def find(self, event_type, where=None, child_of=None, **kwargs):
|
||||
for event in reversed(self.emitted):
|
||||
if not isinstance(event, event_type):
|
||||
continue
|
||||
if child_of is not None and event.event_parent_id != child_of.event_id:
|
||||
continue
|
||||
if where is not None and not where(event):
|
||||
continue
|
||||
return event
|
||||
return None
|
||||
|
||||
async def stop(self):
|
||||
return None
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.30rc19"
|
||||
version = "0.9.30rc20"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user