From 6e0f2a41a3cac542cfb2c298b57ab4e9ec1d437e Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 11 Jun 2026 07:06:41 -0700 Subject: [PATCH] Fix add input routing and plugin dependencies --- archivebox/api/v1_cli.py | 9 +++- archivebox/crawls/models.py | 9 +++- archivebox/services/runner.py | 15 ++++-- archivebox/tests/test_api_v1_cli_add.py | 8 +-- archivebox/tests/test_cli_add.py | 69 +++++++++++-------------- 5 files changed, 62 insertions(+), 48 deletions(-) diff --git a/archivebox/api/v1_cli.py b/archivebox/api/v1_cli.py index ea175f25..675393cf 100644 --- a/archivebox/api/v1_cli.py +++ b/archivebox/api/v1_cli.py @@ -121,14 +121,21 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_ @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]") def cli_add(request: HttpRequest, args: AddCommandSchema): from archivebox.cli.archivebox_add import add + from archivebox.misc.util import validate_url config_overrides: dict[str, object] = {} if args.only_new is not None: config_overrides["ONLY_NEW"] = bool(args.only_new) if args.update or args.overwrite: config_overrides["ONLY_NEW"] = False + submitted_urls: str | list[str] = args.urls + if len(args.urls) == 1: + try: + validate_url(args.urls[0]) + except ValueError: + submitted_urls = args.urls[0] crawl, snapshots = add( - urls=args.urls, + urls=submitted_urls, snapshot_ids=args.snapshot_ids, tag=args.tag, depth=args.depth, diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index d018fa7d..a8909374 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -701,7 +701,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith continue try: entry = json.loads(stripped) - entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip())) + # Crawl.urls accepts plain lines and JSONL URL records. Other + # valid JSON values, e.g. a quoted string from hostile input, + # are not records and must stay inert text instead of raising + # during later Snapshot.save() bookkeeping. + if isinstance(entry, dict): + entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip())) + else: + entries.append((raw_line.rstrip(), stripped)) except json.JSONDecodeError: entries.append((raw_line.rstrip(), stripped)) return entries diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 715d0a28..48ab36d8 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -1038,6 +1038,15 @@ class CrawlRunner: snapshot_selected_plugins = ( self.selected_plugins if self.selected_plugins_from_args else (snapshot_config_plugins or self.selected_plugins) ) + + def queued_plugins_selected_by_config(queued_plugins: list[str]) -> list[str]: + if not snapshot_selected_plugins: + return queued_plugins + expanded_selected_plugins = set( + filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True).keys(), + ) + return [plugin for plugin in queued_plugins if plugin in expanded_selected_plugins] + selected_hooks_by_plugin = None if snapshot["status"] == "started": _reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)() @@ -1062,7 +1071,7 @@ class CrawlRunner: )(snapshot["id"]) if queued_plugins: if snapshot_selected_plugins: - queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins] + queued_plugins = queued_plugins_selected_by_config(queued_plugins) selected_hooks_by_plugin = { plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins } @@ -1074,7 +1083,7 @@ class CrawlRunner: )(snapshot["id"]) if queued_plugins: if snapshot_selected_plugins: - queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins] + queued_plugins = queued_plugins_selected_by_config(queued_plugins) selected_hooks_by_plugin = { plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins } @@ -1103,7 +1112,7 @@ class CrawlRunner: thread_sensitive=True, )(snapshot["id"]) if snapshot_selected_plugins and remaining_queued_plugins: - remaining_queued_plugins = [plugin for plugin in remaining_queued_plugins if plugin in snapshot_selected_plugins] + remaining_queued_plugins = queued_plugins_selected_by_config(remaining_queued_plugins) if not remaining_queued_plugins: await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir) return diff --git a/archivebox/tests/test_api_v1_cli_add.py b/archivebox/tests/test_api_v1_cli_add.py index 1eb8933a..9544f84c 100644 --- a/archivebox/tests/test_api_v1_cli_add.py +++ b/archivebox/tests/test_api_v1_cli_add.py @@ -261,10 +261,10 @@ def test_basic_success_case_request(client, tmp_path, api_headers): assert response.status_code == 200, response.content assert response.json()["success"] is True crawl = Crawl.objects.get() - root_snapshot = Snapshot.objects.get() + snapshot = Snapshot.objects.get() assert crawl.urls == submitted_url - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL - assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == submitted_url + assert snapshot.url == submitted_url + assert snapshot.depth == 1 @pytest.mark.timeout(360) @@ -387,7 +387,7 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path): with use_archivebox_db(tmp_path): snapshot = Snapshot.objects.get(url=safe_url) crawl = Crawl.objects.get() - assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED} + assert crawl.status in {Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED} assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED} with use_archivebox_db(tmp_path): tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True)) diff --git a/archivebox/tests/test_cli_add.py b/archivebox/tests/test_cli_add.py index 8201564f..88f32acb 100644 --- a/archivebox/tests/test_cli_add.py +++ b/archivebox/tests/test_cli_add.py @@ -247,14 +247,12 @@ def test_add_single_url_records_url_in_crawl(initialized_archive): with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() - root_snapshot = Snapshot.objects.get() - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + snapshot = Snapshot.objects.get() assert crawl.urls == "https://example.com" assert crawl.get_urls_list() == ["https://example.com"] - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL - assert root_snapshot.depth == 0 - assert root_input == "https://example.com" + assert snapshot.url == "https://example.com" + assert snapshot.depth == 1 @pytest.mark.timeout(360) @@ -462,8 +460,8 @@ def test_run_rejects_depth_two_file_url_snapshot_injected_directly_with_sql(init assert file_results == [] -def test_add_bg_queues_internal_input_root_snapshot(initialized_archive): - """Background add stores submitted input on an internal root snapshot for the runner.""" +def test_add_bg_queues_direct_url_snapshot(initialized_archive): + """Background add queues explicit URL arguments as real URL snapshots.""" env = cli_env(disable_extractors=True) result = run_archivebox_cmd( ["add", "--bg", "--depth=0", "https://example.com"], @@ -475,15 +473,13 @@ def test_add_bg_queues_internal_input_root_snapshot(initialized_archive): with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() - root_snapshot = Snapshot.objects.get() - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + snapshot = Snapshot.objects.get() assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is not None assert crawl.urls == "https://example.com" - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL - assert root_snapshot.depth == 0 - assert root_input == "https://example.com" + assert snapshot.url == "https://example.com" + assert snapshot.depth == 1 @pytest.mark.timeout(180) @@ -528,15 +524,12 @@ def test_add_tagged_single_url_seals_without_duplicate_snapshot_tags(initialized assert crawl.status == Crawl.StatusChoices.SEALED assert [(snapshot.url, snapshot.depth, snapshot.status) for snapshot in snapshots] == [ - (Snapshot.INTERNAL_INPUT_URL, 0, Snapshot.StatusChoices.SEALED), ("https://example.com/?archivebox-tagged-single-url=1", 1, Snapshot.StatusChoices.SEALED), ] assert tag_counts == { - Snapshot.INTERNAL_INPUT_URL: 1, "https://example.com/?archivebox-tagged-single-url=1": 1, } by_url_plugin = {(url, plugin): status for url, plugin, status, _output in results} - assert by_url_plugin[(Snapshot.INTERNAL_INPUT_URL, "parse_txt_urls")] == "succeeded" assert by_url_plugin[("https://example.com/?archivebox-tagged-single-url=1", "title")] == "succeeded" unexpected_failures = [(url, plugin, status, output) for url, plugin, status, output in results if status == "failed"] assert not unexpected_failures @@ -561,12 +554,12 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() - root_snapshot = Snapshot.objects.get() + snapshot_urls = set(Snapshot.objects.values_list("url", flat=True)) assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is None assert crawl.urls == "https://example.com" - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL + assert snapshot_urls == set() run_queued_crawls(initialized_archive, env) @@ -577,7 +570,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi assert crawl.status == Crawl.StatusChoices.SEALED assert crawl.retry_at is None assert crawl.urls == "https://example.com" - assert snapshot_urls == {Snapshot.INTERNAL_INPUT_URL} + assert snapshot_urls == set() def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive): @@ -604,7 +597,7 @@ def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive): assert crawl.urls == "\n".join(internal_urls) assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is None - assert snapshot_urls == {Snapshot.INTERNAL_INPUT_URL} + assert snapshot_urls == set() def test_add_creates_crawl_record(initialized_archive): @@ -622,8 +615,8 @@ def test_add_creates_crawl_record(initialized_archive): assert crawl_count == 1 -def test_add_creates_internal_input_file(initialized_archive): - """Test that add stores submitted text under the root snapshot staticfile output.""" +def test_add_direct_url_creates_snapshot_without_internal_input_file(initialized_archive): + """Test that explicit URL args queue real snapshots without stdin import files.""" env = cli_env(disable_extractors=True) run_archivebox_cmd( ["add", "--index-only", "--depth=0", "https://example.com"], @@ -632,9 +625,10 @@ def test_add_creates_internal_input_file(initialized_archive): ) with use_archivebox_db(initialized_archive): - root_snapshot = Snapshot.objects.get() - source_content = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") - assert source_content == "https://example.com" + snapshot = Snapshot.objects.get() + assert snapshot.url == "https://example.com" + assert snapshot.depth == 1 + assert not (snapshot.output_dir / "staticfile" / "stdin.txt").exists() def test_add_multiple_urls_single_command(initialized_archive): @@ -650,11 +644,10 @@ def test_add_multiple_urls_single_command(initialized_archive): with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() - root_snapshot = Snapshot.objects.get() - root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") + snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth")) assert crawl.urls == "https://example.com\nhttps://example.org" - assert root_input == "https://example.com\nhttps://example.org" + assert snapshots == [("https://example.com", 1), ("https://example.org", 1)] def test_add_rejects_file_path_argument(initialized_archive): @@ -807,14 +800,11 @@ def test_add_duplicate_url_creates_separate_crawls(initialized_archive): with use_archivebox_db(initialized_archive): crawl_count = Crawl.objects.count() - root_inputs = [ - snapshot.output_dir.joinpath("staticfile", "stdin.txt").read_text(encoding="utf-8") - for snapshot in Snapshot.objects.order_by("created_at") - ] + snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "depth")) # Each add creates a new crawl with its own queued work. assert crawl_count == 2 - assert root_inputs == ["https://example.com", "https://example.com"] + assert snapshots == [("https://example.com", 1), ("https://example.com", 1)] def test_add_with_overwrite_flag(initialized_archive): @@ -929,16 +919,17 @@ def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() - root_snapshot = Snapshot.objects.get() + snapshot = Snapshot.objects.get() assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.retry_at is None assert crawl.urls == "https://example.com" - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL + assert snapshot.url == "https://example.com" + assert snapshot.depth == 1 -def test_add_index_only_creates_only_internal_root_snapshot(initialized_archive): - """Test that index-only add creates the input root but not parsed child snapshots.""" +def test_add_index_only_creates_direct_url_snapshot(initialized_archive): + """Test that index-only add queues explicit URL args as real URL snapshots.""" env = cli_env(disable_extractors=True) run_archivebox_cmd( ["add", "--index-only", "--depth=0", "https://example.com"], @@ -948,10 +939,11 @@ def test_add_index_only_creates_only_internal_root_snapshot(initialized_archive) with use_archivebox_db(initialized_archive): crawl = Crawl.objects.get() - root_snapshot = Snapshot.objects.get() + snapshot = Snapshot.objects.get() assert crawl.urls == "https://example.com" - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL + assert snapshot.url == "https://example.com" + assert snapshot.depth == 1 def test_snapshot_create_sets_snapshot_timestamp(initialized_archive): @@ -1188,7 +1180,6 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive, assert crawl_config["CRAWL_MAX_URLS"] == 2 assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024 assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0 - assert (Snapshot.INTERNAL_INPUT_URL, 0, "sealed") in snapshots assert (root_url, 1, "sealed") in snapshots assert any(url == child_url and depth == 2 and status == "sealed" for url, depth, status in snapshots)