The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Stabilize live chrome signal cleanup test

Browse Source
This commit is contained in:
Nick Sweeting 2026-07-25 22:54:05 -07:00
parent bdc821eebc
commit 9129e41c76
No known key found for this signature in database
1 changed files with 58 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
archivebox/tests/test_cli_run.py
View File

@ -11,16 +11,16 @@ import os
import signal
import subprocess
import pytest
import psutil
import pytest
from archivebox.tests.conftest import (
cleanup_process_group,
cli_env,
run_archivebox_cmd,
parse_jsonl_output,
create_test_url,
parse_jsonl_output,
pid_is_alive,
run_archivebox_cmd,
wait_for_log,
wait_for_pid_to_disappear,
)
@ -80,9 +80,8 @@ def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archiv
run_log_handle.close()
try:
wait_for_log(browser_state, '"ready": true', timeout=120)
child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True)]
child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True) if pid_is_alive(child.pid)]
assert child_pids
assert all(pid_is_alive(pid) for pid in child_pids)
run_process.send_signal(signal.SIGTERM)
run_process.wait(timeout=30)
@ -333,8 +332,8 @@ class TestRunRecovery:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.tests.test_orm_helpers import use_archivebox_db
old = timezone.now() - timedelta(hours=13)
@ -533,8 +532,8 @@ class TestRunDaemonMode:
db,
stdin_kind,
):
from archivebox.machine.models import Process
from archivebox.core.models import Snapshot
from archivebox.machine.models import Process
from archivebox.tests.test_orm_helpers import use_archivebox_db
snapshot_url = None
@ -606,8 +605,8 @@ class TestRunDaemonMode:
assert "No records to process" not in output
def test_run_daemon_takeover_has_single_active_runner_gate(self, initialized_archive, db):
from archivebox.machine.models import Process
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE
from archivebox.machine.models import Process
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(PLUGINS="__archivebox_test_no_plugins__")
@ -690,9 +689,9 @@ class TestRunDaemonMode:
class TestRecoverOrchestratorState:
def test_recover_orchestrator_state_unlocks_started_crawl_with_pending_snapshot(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -716,9 +715,9 @@ class TestRecoverOrchestratorState:
def test_recover_orchestrator_state_unlocks_started_crawl_with_finished_snapshots_for_runner(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl
crawl = Crawl.objects.create(
@ -752,9 +751,9 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
user_id = get_or_create_system_user_pk()
queued_crawl = Crawl.objects.create(
@ -802,9 +801,9 @@ class TestRecoverOrchestratorState:
def test_recover_orchestrator_state_requeues_backoff_archiveresults(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -839,9 +838,9 @@ class TestRecoverOrchestratorState:
def test_recover_orchestrator_state_leaves_due_queued_snapshot_for_runner_even_with_final_results(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -881,9 +880,9 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl, run_due_snapshot
old = timezone.now() - timedelta(hours=13)
@ -941,8 +940,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
@ -975,8 +974,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -1001,8 +1000,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults
crawl = Crawl.objects.create(
@ -1037,8 +1036,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
from archivebox.workers.models import RETRY_AT_MAX
@ -1074,8 +1073,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
from archivebox.workers.models import RETRY_AT_MAX
@ -1149,8 +1148,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
old = timezone.now() - timedelta(hours=13)
crawl = Crawl.objects.create(
@ -1174,9 +1173,9 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
future = timezone.now() + timedelta(seconds=45)
crawl = Crawl.objects.create(
@ -1211,9 +1210,9 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
future = timezone.now() + timedelta(seconds=45)
crawl = Crawl.objects.create(
@ -1249,10 +1248,10 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Machine, NetworkInterface, Process
worker = run_archivebox_cmd(
["manage", "shell"],
@ -1315,9 +1314,9 @@ class TestRecoverOrchestratorState:
def test_recover_orchestrator_state_does_not_resume_paused_rows_with_max_retry_at(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.workers.models import RETRY_AT_MAX
crawl = Crawl.objects.create(
@ -1349,9 +1348,9 @@ class TestRecoverOrchestratorState:
def test_recover_orchestrator_state_does_not_wake_sealed_snapshot_maintenance_rows(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -1393,8 +1392,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
@ -1431,8 +1430,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
@ -1475,8 +1474,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
@ -1511,8 +1510,8 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
@ -1677,9 +1676,9 @@ class TestRecoverOrchestratorState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -1708,9 +1707,9 @@ class TestRecoverOrchestratorState:
def test_recover_orchestrator_state_unlocks_started_snapshot_with_final_results_for_runner(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
crawl = Crawl.objects.create(
@ -1751,8 +1750,8 @@ class TestRecoverOrchestratorState:
class TestRunDueCrawlState:
def test_idle_maintenance_repairs_archive_result_delete_at(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_pending_crawls
crawl = Crawl.objects.create(
@ -1810,8 +1809,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -1837,8 +1836,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://example.com",
@ -1900,8 +1899,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.plugins.hooks import extract_records_from_process, run_hook
from archivebox.services.runner import run_due_snapshot
@ -1966,8 +1965,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.services.runner import run_due_snapshot
@ -2022,8 +2021,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl
crawl = Crawl.objects.create(
@ -2051,8 +2050,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl
future = timezone.now() + timedelta(hours=1)
@ -2081,8 +2080,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl
future = timezone.now() + timedelta(minutes=5)
@ -2109,8 +2108,8 @@ class TestRunDueCrawlState:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl
crawl = Crawl.objects.create(
@ -2143,9 +2142,9 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
future = timezone.now() + timedelta(days=1)
crawl = Crawl.objects.create(
@ -2179,9 +2178,9 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
future = timezone.now() + timedelta(days=1)
crawl = Crawl.objects.create(
@ -2204,9 +2203,9 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
future = timezone.now() + timedelta(days=1)
crawl = Crawl.objects.create(
@ -2225,9 +2224,9 @@ class TestRecoverOrchestratorStateRedFailureModes:
def test_recovery_requeues_started_archiveresult_without_process(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://www.mathjax.org/",
@ -2257,10 +2256,10 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Machine, NetworkInterface, Process
crawl = Crawl.objects.create(
urls="https://revealjs.com/",
@ -2298,10 +2297,10 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Machine, NetworkInterface, Process
crawl = Crawl.objects.create(
urls="https://pdfobject.com/pdf/sample-3pp.pdf",
@ -2345,9 +2344,9 @@ class TestRecoverOrchestratorStateRedFailureModes:
def test_recovery_requeues_started_snapshot_result_before_unlocking_snapshot(self):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(
urls="https://mermaid-js.github.io/mermaid/",
@ -2381,8 +2380,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import CrawlRunner
future = timezone.now() + timedelta(days=1)
@ -2404,8 +2403,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import CrawlRunner
future = timezone.now() + timedelta(days=1)
@ -2428,8 +2427,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl
now = timezone.now()
@ -2462,8 +2461,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl, run_due_snapshot
now = timezone.now()
@ -2542,8 +2541,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
future = timezone.now() + timedelta(days=1)
crawl = Crawl.objects.create(
@ -2584,8 +2583,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_crawl
now = timezone.now()
@ -2616,8 +2615,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
now = timezone.now()
crawl = Crawl.objects.create(
@ -2651,9 +2650,9 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.core.recovery_util import recover_orchestrator_state
old = timezone.now() - timedelta(hours=13)
crawl = Crawl.objects.create(
@ -2688,8 +2687,8 @@ class TestRecoverOrchestratorStateRedFailureModes:
from django.utils import timezone
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.machine.models import Machine, NetworkInterface, Process
runtime_dir = tmp_path / "https_example_com" / ".hooks" / "on_Snapshot__01_title.py"
runtime_dir.mkdir(parents=True)