diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 9534f15a..0d8fd2a7 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -11,16 +11,16 @@ import os import signal import subprocess -import pytest import psutil +import pytest from archivebox.tests.conftest import ( cleanup_process_group, cli_env, - run_archivebox_cmd, - parse_jsonl_output, create_test_url, + parse_jsonl_output, pid_is_alive, + run_archivebox_cmd, wait_for_log, wait_for_pid_to_disappear, ) @@ -80,9 +80,8 @@ def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archiv run_log_handle.close() try: wait_for_log(browser_state, '"ready": true', timeout=120) - child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True)] + child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True) if pid_is_alive(child.pid)] assert child_pids - assert all(pid_is_alive(pid) for pid in child_pids) run_process.send_signal(signal.SIGTERM) run_process.wait(timeout=30) @@ -333,8 +332,8 @@ class TestRunRecovery: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.tests.test_orm_helpers import use_archivebox_db old = timezone.now() - timedelta(hours=13) @@ -533,8 +532,8 @@ class TestRunDaemonMode: db, stdin_kind, ): - from archivebox.machine.models import Process from archivebox.core.models import Snapshot + from archivebox.machine.models import Process from archivebox.tests.test_orm_helpers import use_archivebox_db snapshot_url = None @@ -606,8 +605,8 @@ class TestRunDaemonMode: assert "No records to process" not in output def test_run_daemon_takeover_has_single_active_runner_gate(self, initialized_archive, db): - from archivebox.machine.models import Process from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE + from archivebox.machine.models import Process from archivebox.tests.test_orm_helpers import use_archivebox_db env = cli_env(PLUGINS="__archivebox_test_no_plugins__") @@ -690,9 +689,9 @@ class TestRunDaemonMode: class TestRecoverOrchestratorState: def test_recover_orchestrator_state_unlocks_started_crawl_with_pending_snapshot(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -716,9 +715,9 @@ class TestRecoverOrchestratorState: def test_recover_orchestrator_state_unlocks_started_crawl_with_finished_snapshots_for_runner(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl crawl = Crawl.objects.create( @@ -752,9 +751,9 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl user_id = get_or_create_system_user_pk() queued_crawl = Crawl.objects.create( @@ -802,9 +801,9 @@ class TestRecoverOrchestratorState: def test_recover_orchestrator_state_requeues_backoff_archiveresults(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -839,9 +838,9 @@ class TestRecoverOrchestratorState: def test_recover_orchestrator_state_leaves_due_queued_snapshot_for_runner_even_with_final_results(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -881,9 +880,9 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl, run_due_snapshot old = timezone.now() - timedelta(hours=13) @@ -941,8 +940,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( @@ -975,8 +974,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -1001,8 +1000,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import snapshot_hooks_for_pending_archiveresults crawl = Crawl.objects.create( @@ -1037,8 +1036,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot from archivebox.workers.models import RETRY_AT_MAX @@ -1074,8 +1073,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot from archivebox.workers.models import RETRY_AT_MAX @@ -1149,8 +1148,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl old = timezone.now() - timedelta(hours=13) crawl = Crawl.objects.create( @@ -1174,9 +1173,9 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl future = timezone.now() + timedelta(seconds=45) crawl = Crawl.objects.create( @@ -1211,9 +1210,9 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl future = timezone.now() + timedelta(seconds=45) crawl = Crawl.objects.create( @@ -1249,10 +1248,10 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl + from archivebox.machine.models import Machine, NetworkInterface, Process worker = run_archivebox_cmd( ["manage", "shell"], @@ -1315,9 +1314,9 @@ class TestRecoverOrchestratorState: def test_recover_orchestrator_state_does_not_resume_paused_rows_with_max_retry_at(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl from archivebox.workers.models import RETRY_AT_MAX crawl = Crawl.objects.create( @@ -1349,9 +1348,9 @@ class TestRecoverOrchestratorState: def test_recover_orchestrator_state_does_not_wake_sealed_snapshot_maintenance_rows(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -1393,8 +1392,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( @@ -1431,8 +1430,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( @@ -1475,8 +1474,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( @@ -1511,8 +1510,8 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( @@ -1677,9 +1676,9 @@ class TestRecoverOrchestratorState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -1708,9 +1707,9 @@ class TestRecoverOrchestratorState: def test_recover_orchestrator_state_unlocks_started_snapshot_with_final_results_for_runner(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_snapshot crawl = Crawl.objects.create( @@ -1751,8 +1750,8 @@ class TestRecoverOrchestratorState: class TestRunDueCrawlState: def test_idle_maintenance_repairs_archive_result_delete_at(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_pending_crawls crawl = Crawl.objects.create( @@ -1810,8 +1809,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -1837,8 +1836,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://example.com", @@ -1900,8 +1899,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.plugins.hooks import extract_records_from_process, run_hook from archivebox.services.runner import run_due_snapshot @@ -1966,8 +1965,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services.runner import run_due_snapshot @@ -2022,8 +2021,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl crawl = Crawl.objects.create( @@ -2051,8 +2050,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl future = timezone.now() + timedelta(hours=1) @@ -2081,8 +2080,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl future = timezone.now() + timedelta(minutes=5) @@ -2109,8 +2108,8 @@ class TestRunDueCrawlState: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl crawl = Crawl.objects.create( @@ -2143,9 +2142,9 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl future = timezone.now() + timedelta(days=1) crawl = Crawl.objects.create( @@ -2179,9 +2178,9 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl future = timezone.now() + timedelta(days=1) crawl = Crawl.objects.create( @@ -2204,9 +2203,9 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl future = timezone.now() + timedelta(days=1) crawl = Crawl.objects.create( @@ -2225,9 +2224,9 @@ class TestRecoverOrchestratorStateRedFailureModes: def test_recovery_requeues_started_archiveresult_without_process(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://www.mathjax.org/", @@ -2257,10 +2256,10 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl + from archivebox.machine.models import Machine, NetworkInterface, Process crawl = Crawl.objects.create( urls="https://revealjs.com/", @@ -2298,10 +2297,10 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot - from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl + from archivebox.machine.models import Machine, NetworkInterface, Process crawl = Crawl.objects.create( urls="https://pdfobject.com/pdf/sample-3pp.pdf", @@ -2345,9 +2344,9 @@ class TestRecoverOrchestratorStateRedFailureModes: def test_recovery_requeues_started_snapshot_result_before_unlocking_snapshot(self): from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.crawls.models import Crawl crawl = Crawl.objects.create( urls="https://mermaid-js.github.io/mermaid/", @@ -2381,8 +2380,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import CrawlRunner future = timezone.now() + timedelta(days=1) @@ -2404,8 +2403,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import CrawlRunner future = timezone.now() + timedelta(days=1) @@ -2428,8 +2427,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl now = timezone.now() @@ -2462,8 +2461,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl, run_due_snapshot now = timezone.now() @@ -2542,8 +2541,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl future = timezone.now() + timedelta(days=1) crawl = Crawl.objects.create( @@ -2584,8 +2583,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl from archivebox.services.runner import run_due_crawl now = timezone.now() @@ -2616,8 +2615,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk - from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot + from archivebox.crawls.models import Crawl now = timezone.now() crawl = Crawl.objects.create( @@ -2651,9 +2650,9 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.core.recovery_util import recover_orchestrator_state from archivebox.crawls.models import Crawl from archivebox.machine.models import Machine, NetworkInterface, Process - from archivebox.core.recovery_util import recover_orchestrator_state old = timezone.now() - timedelta(hours=13) crawl = Crawl.objects.create( @@ -2688,8 +2687,8 @@ class TestRecoverOrchestratorStateRedFailureModes: from django.utils import timezone - from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.core.recovery_util import recover_orchestrator_state + from archivebox.machine.models import Machine, NetworkInterface, Process runtime_dir = tmp_path / "https_example_com" / ".hooks" / "on_Snapshot__01_title.py" runtime_dir.mkdir(parents=True)