diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index f03cc033..d9e57a4f 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -817,7 +817,17 @@ class CrawlRunner: + CrawlCompletedEvent.model_fields["event_timeout"].default + 30.0 ) - await plan.seed_config(self.bus) + if plan.catalog: + await abx_install_plugins( + plan, + output_dir=output_dir, + emit_jsonl=False, + bus=self.bus, + BinaryService=None, + ProcessService=None, + ) + else: + await plan.seed_config(self.bus) plan.attach_services( self.bus, url=snapshot["url"], @@ -834,6 +844,7 @@ class CrawlRunner: emit_jsonl=False, abort_requested=self.crawl_is_cancelled, PluginBinariesService=None, + PluginBinaryEnvService=None, BinaryService=None, ProcessService=None, ArchiveResultService=None, diff --git a/archivebox/tests/test_crawl_runner.py b/archivebox/tests/test_crawl_runner.py index fa2a5ee8..c21e6fcf 100644 --- a/archivebox/tests/test_crawl_runner.py +++ b/archivebox/tests/test_crawl_runner.py @@ -657,6 +657,35 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp assert snapshot.archiveresult_set.count() == 0 +@pytest.mark.django_db(transaction=True) +def test_crawl_runner_preflights_plugins_before_crawl_lifecycle(tmp_path): + from abx_dl.events import CrawlEvent, CrawlSetupEvent, InstallEvent + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.services.runner import CrawlRunner + + crawl = Crawl.objects.create( + urls="https://example.com", + config={ + "ABXPKG_LIB_DIR": str(tmp_path / "lib"), + "PLUGINS": "base", + "CHROME_BINARY": "", + }, + created_by_id=get_or_create_system_user_pk(), + ) + runner = CrawlRunner(crawl) + event_order = [] + runner.bus.on(InstallEvent, lambda _event: event_order.append(InstallEvent)) + runner.bus.on(CrawlEvent, lambda _event: event_order.append(CrawlEvent)) + runner.bus.on(CrawlSetupEvent, lambda _event: event_order.append(CrawlSetupEvent)) + + asyncio.run(runner.run()) + + assert event_order.count(InstallEvent) == 1 + assert event_order.index(InstallEvent) < event_order.index(CrawlEvent) + assert event_order.index(InstallEvent) < event_order.index(CrawlSetupEvent) + + @pytest.mark.django_db(transaction=True) def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot(): from abx_dl.events import SnapshotCompletedEvent diff --git a/archivebox/tests/test_migrations_07_to_09.py b/archivebox/tests/test_migrations_07_to_09.py index 5f4048c5..a5794267 100644 --- a/archivebox/tests/test_migrations_07_to_09.py +++ b/archivebox/tests/test_migrations_07_to_09.py @@ -403,7 +403,7 @@ def test_update_saves_migrated_snapshots_without_foreign_key_errors(archive_07): result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45) assert result.returncode == 0, f"Init failed: {result.stderr}" - result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=60) + result = run_archivebox_migration_cmd(work_dir, ["update", "--migrate-only"], timeout=60) output = result.stdout + result.stderr assert result.returncode == 0, f"Update failed after migration: {result.stderr}" assert "FOREIGN KEY constraint failed" not in output