From 32362ffbd9d7d4d0d2abc413e1dc8843f40c4ba7 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Fri, 28 Aug 2026 11:27:46 -0700 Subject: [PATCH] Ignore historical hooks during plugin recovery --- archivebox/core/recovery_util.py | 3 +- archivebox/tests/test_cli_run.py | 79 ++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/archivebox/core/recovery_util.py b/archivebox/core/recovery_util.py index ac702348..7837beaf 100644 --- a/archivebox/core/recovery_util.py +++ b/archivebox/core/recovery_util.py @@ -195,8 +195,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | "status": ArchiveResult.StatusChoices.QUEUED, }, ) - process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts)) - if result.status == ArchiveResult.StatusChoices.QUEUED or process_is_newer: + if created or result.status == ArchiveResult.StatusChoices.QUEUED: requeue_snapshot = False # A runner can die after the hook Process exits but before the # ProcessCompletedEvent projector links/finalizes ArchiveResult. diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index fc9a4325..79a17a86 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -2362,6 +2362,85 @@ class TestRecoverOrchestratorStateRedFailureModes: assert result.start_ts == newer_start assert result.process.started_at == newer_start + def test_recovery_does_not_replace_final_plugin_result_with_older_unlinked_hook(self): + import json + from datetime import timedelta + + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl + from archivebox.machine.models import Machine, NetworkInterface, Process + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl, status=Snapshot.StatusChoices.SEALED, retry_at=None) + machine = Machine.current(refresh=True) + iface = NetworkInterface.current(refresh=True) + start_ts = timezone.now() - timedelta(minutes=2) + stop_ts = timezone.now() - timedelta(minutes=1) + start_process = Process.objects.create( + machine=machine, + iface=iface, + process_type=Process.TypeChoices.HOOK, + worker_type="archiveresult", + pwd=str(snapshot.output_dir / "archivewebpage"), + cmd=["on_Snapshot__16_archivewebpage_start.js"], + status=Process.StatusChoices.EXITED, + retry_at=None, + exit_code=0, + started_at=start_ts, + ended_at=start_ts + timedelta(seconds=1), + stdout=json.dumps( + { + "type": "ArchiveResult", + "plugin": "archivewebpage", + "hook_name": "on_Snapshot__16_archivewebpage_start", + "status": "succeeded", + "output_str": "recording started", + }, + ), + ) + stop_process = Process.objects.create( + machine=machine, + iface=iface, + process_type=Process.TypeChoices.HOOK, + worker_type="archiveresult", + pwd=str(snapshot.output_dir / "archivewebpage"), + cmd=["on_Snapshot__65_archivewebpage_stop.js"], + status=Process.StatusChoices.EXITED, + retry_at=None, + exit_code=0, + started_at=stop_ts, + ended_at=stop_ts + timedelta(seconds=1), + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="archivewebpage", + hook_name="on_Snapshot__65_archivewebpage_stop", + status=ArchiveResult.StatusChoices.SUCCEEDED, + output_str="archivewebpage/archivewebpage.wacz", + start_ts=start_ts, + end_ts=stop_ts + timedelta(seconds=1), + process=stop_process, + ) + + recover_orchestrator_state() + + result.refresh_from_db() + assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="archivewebpage").count() == 1 + assert result.hook_name == "on_Snapshot__65_archivewebpage_stop" + assert result.status == ArchiveResult.StatusChoices.SUCCEEDED + assert result.output_str == "archivewebpage/archivewebpage.wacz" + assert result.process == stop_process + assert not ArchiveResult.objects.filter(process=start_process).exists() + def test_recovery_does_not_seal_queued_snapshot_waiting_for_future_retry_even_with_final_results(self): from datetime import timedelta