The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Recover emitted plugin results durably

Browse Source
This commit is contained in:
Nick Sweeting 2026-08-28 10:19:08 -07:00
parent 820d395dd6
commit 5caa61f694
No known key found for this signature in database
2 changed files with 36 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
archivebox/core/recovery_util.py
View File

@ -202,6 +202,12 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
# ProcessCompletedEvent projector links/finalizes ArchiveResult.
# Reconstruct the plugin row from its newest durable Process row.
output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
emitted_records = [
record
for record in Process.parse_records_from_text(process.stdout or "")
if record.get("type") == "ArchiveResult" and (record.get("plugin") or plugin_dir.name) == plugin_dir.name
]
emitted_result = emitted_records[-1] if emitted_records else {}
result.hook_name = Path(hook_script_name).stem
result.process = process
result.start_ts = process.started_at
@ -221,11 +227,21 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
result.output_files = output_files
result.output_size = output_size
result.output_mimetypes = output_mimetypes
result.output_str = process.stderr if process.exit_code not in (0, None) else ""
result.output_str = (
emitted_result.get("output_str")
or emitted_result.get("output")
or (process.stderr if process.exit_code not in (0, None) else "")
)
result.output_json = emitted_result.get("output_json") if isinstance(emitted_result.get("output_json"), dict) else None
emitted_status = emitted_result.get("status")
result.status = (
ArchiveResult.StatusChoices.FAILED
if process.exit_code not in (0, None)
else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
emitted_status
if emitted_status in ArchiveResult.StatusChoices.values
else (
ArchiveResult.StatusChoices.FAILED
if process.exit_code not in (0, None)
else (ArchiveResult.StatusChoices.SUCCEEDED if output_files else ArchiveResult.StatusChoices.NORESULTS)
)
)
result.save(
update_fields=[
@ -237,6 +253,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
"output_size",
"output_mimetypes",
"output_str",
"output_json",
"status",
"modified_at",
],

18
archivebox/tests/test_cli_run.py
View File

@ -2303,6 +2303,7 @@ class TestRunDueCrawlState:
@pytest.mark.django_db
class TestRecoverOrchestratorStateRedFailureModes:
def test_recovery_uses_newest_orphaned_hook_process_for_one_plugin_result(self):
import json
from datetime import timedelta
from django.utils import timezone
@ -2324,9 +2325,9 @@ class TestRecoverOrchestratorStateRedFailureModes:
iface = NetworkInterface.current(refresh=True)
older_start = timezone.now() - timedelta(minutes=2)
newer_start = timezone.now() - timedelta(minutes=1)
for hook_name, started_at in (
("on_Snapshot__01_title.py", older_start),
("on_Snapshot__02_title.py", newer_start),
for hook_name, started_at, output_str in (
("on_Snapshot__01_title.py", older_start, "older title"),
("on_Snapshot__02_title.py", newer_start, "newer title"),
):
Process.objects.create(
machine=machine,
@ -2340,6 +2341,15 @@ class TestRecoverOrchestratorStateRedFailureModes:
exit_code=0,
started_at=started_at,
ended_at=started_at + timedelta(seconds=1),
stdout=json.dumps(
{
"type": "ArchiveResult",
"plugin": "title",
"hook_name": hook_name.removesuffix(".py"),
"status": "succeeded",
"output_str": output_str,
},
),
)
recover_orchestrator_state()
@ -2347,6 +2357,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="title").count() == 1
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title")
assert result.hook_name == "on_Snapshot__02_title"
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
assert result.output_str == "newer title"
assert result.start_ts == newer_start
assert result.process.started_at == newer_start