The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Fix add input routing and plugin dependencies

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-11 07:06:41 -07:00
parent b6921d5e03
commit 6e0f2a41a3
No known key found for this signature in database
5 changed files with 62 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
archivebox/api/v1_cli.py
View File

@ -121,14 +121,21 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
def cli_add(request: HttpRequest, args: AddCommandSchema):
from archivebox.cli.archivebox_add import add
from archivebox.misc.util import validate_url
config_overrides: dict[str, object] = {}
if args.only_new is not None:
config_overrides["ONLY_NEW"] = bool(args.only_new)
if args.update or args.overwrite:
config_overrides["ONLY_NEW"] = False
submitted_urls: str | list[str] = args.urls
if len(args.urls) == 1:
try:
validate_url(args.urls[0])
except ValueError:
submitted_urls = args.urls[0]
crawl, snapshots = add(
urls=args.urls,
urls=submitted_urls,
snapshot_ids=args.snapshot_ids,
tag=args.tag,
depth=args.depth,

9
archivebox/crawls/models.py
View File

@ -701,7 +701,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
continue
try:
entry = json.loads(stripped)
entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
# Crawl.urls accepts plain lines and JSONL URL records. Other
# valid JSON values, e.g. a quoted string from hostile input,
# are not records and must stay inert text instead of raising
# during later Snapshot.save() bookkeeping.
if isinstance(entry, dict):
entries.append((raw_line.rstrip(), str(entry.get("url", "") or "").strip()))
else:
entries.append((raw_line.rstrip(), stripped))
except json.JSONDecodeError:
entries.append((raw_line.rstrip(), stripped))
return entries

15
archivebox/services/runner.py
View File

@ -1038,6 +1038,15 @@ class CrawlRunner:
snapshot_selected_plugins = (
self.selected_plugins if self.selected_plugins_from_args else (snapshot_config_plugins or self.selected_plugins)
)
def queued_plugins_selected_by_config(queued_plugins: list[str]) -> list[str]:
if not snapshot_selected_plugins:
return queued_plugins
expanded_selected_plugins = set(
filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True).keys(),
)
return [plugin for plugin in queued_plugins if plugin in expanded_selected_plugins]
selected_hooks_by_plugin = None
if snapshot["status"] == "started":
_reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)()
@ -1062,7 +1071,7 @@ class CrawlRunner:
)(snapshot["id"])
if queued_plugins:
if snapshot_selected_plugins:
queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins]
queued_plugins = queued_plugins_selected_by_config(queued_plugins)
selected_hooks_by_plugin = {
plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
}
@ -1074,7 +1083,7 @@ class CrawlRunner:
)(snapshot["id"])
if queued_plugins:
if snapshot_selected_plugins:
queued_plugins = [plugin for plugin in queued_plugins if plugin in snapshot_selected_plugins]
queued_plugins = queued_plugins_selected_by_config(queued_plugins)
selected_hooks_by_plugin = {
plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
}
@ -1103,7 +1112,7 @@ class CrawlRunner:
thread_sensitive=True,
)(snapshot["id"])
if snapshot_selected_plugins and remaining_queued_plugins:
remaining_queued_plugins = [plugin for plugin in remaining_queued_plugins if plugin in snapshot_selected_plugins]
remaining_queued_plugins = queued_plugins_selected_by_config(remaining_queued_plugins)
if not remaining_queued_plugins:
await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
return

8
archivebox/tests/test_api_v1_cli_add.py
View File

@ -261,10 +261,10 @@ def test_basic_success_case_request(client, tmp_path, api_headers):
assert response.status_code == 200, response.content
assert response.json()["success"] is True
crawl = Crawl.objects.get()
root_snapshot = Snapshot.objects.get()
snapshot = Snapshot.objects.get()
assert crawl.urls == submitted_url
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == submitted_url
assert snapshot.url == submitted_url
assert snapshot.depth == 1
@pytest.mark.timeout(360)
@ -387,7 +387,7 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
with use_archivebox_db(tmp_path):
snapshot = Snapshot.objects.get(url=safe_url)
crawl = Crawl.objects.get()
assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
assert crawl.status in {Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
with use_archivebox_db(tmp_path):
tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True))

69
archivebox/tests/test_cli_add.py
View File

@ -247,14 +247,12 @@ def test_add_single_url_records_url_in_crawl(initialized_archive):
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
root_snapshot = Snapshot.objects.get()
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
snapshot = Snapshot.objects.get()
assert crawl.urls == "https://example.com"
assert crawl.get_urls_list() == ["https://example.com"]
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert root_snapshot.depth == 0
assert root_input == "https://example.com"
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
@pytest.mark.timeout(360)
@ -462,8 +460,8 @@ def test_run_rejects_depth_two_file_url_snapshot_injected_directly_with_sql(init
assert file_results == []
def test_add_bg_queues_internal_input_root_snapshot(initialized_archive):
"""Background add stores submitted input on an internal root snapshot for the runner."""
def test_add_bg_queues_direct_url_snapshot(initialized_archive):
"""Background add queues explicit URL arguments as real URL snapshots."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--bg", "--depth=0", "https://example.com"],
@ -475,15 +473,13 @@ def test_add_bg_queues_internal_input_root_snapshot(initialized_archive):
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
root_snapshot = Snapshot.objects.get()
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
snapshot = Snapshot.objects.get()
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
assert crawl.urls == "https://example.com"
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert root_snapshot.depth == 0
assert root_input == "https://example.com"
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
@pytest.mark.timeout(180)
@ -528,15 +524,12 @@ def test_add_tagged_single_url_seals_without_duplicate_snapshot_tags(initialized
assert crawl.status == Crawl.StatusChoices.SEALED
assert [(snapshot.url, snapshot.depth, snapshot.status) for snapshot in snapshots] == [
(Snapshot.INTERNAL_INPUT_URL, 0, Snapshot.StatusChoices.SEALED),
("https://example.com/?archivebox-tagged-single-url=1", 1, Snapshot.StatusChoices.SEALED),
]
assert tag_counts == {
Snapshot.INTERNAL_INPUT_URL: 1,
"https://example.com/?archivebox-tagged-single-url=1": 1,
}
by_url_plugin = {(url, plugin): status for url, plugin, status, _output in results}
assert by_url_plugin[(Snapshot.INTERNAL_INPUT_URL, "parse_txt_urls")] == "succeeded"
assert by_url_plugin[("https://example.com/?archivebox-tagged-single-url=1", "title")] == "succeeded"
unexpected_failures = [(url, plugin, status, output) for url, plugin, status, output in results if status == "failed"]
assert not unexpected_failures
@ -561,12 +554,12 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
root_snapshot = Snapshot.objects.get()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert snapshot_urls == set()
run_queued_crawls(initialized_archive, env)
@ -577,7 +570,7 @@ def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initi
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert snapshot_urls == {Snapshot.INTERNAL_INPUT_URL}
assert snapshot_urls == set()
def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
@ -604,7 +597,7 @@ def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
assert crawl.urls == "\n".join(internal_urls)
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert snapshot_urls == {Snapshot.INTERNAL_INPUT_URL}
assert snapshot_urls == set()
def test_add_creates_crawl_record(initialized_archive):
@ -622,8 +615,8 @@ def test_add_creates_crawl_record(initialized_archive):
assert crawl_count == 1
def test_add_creates_internal_input_file(initialized_archive):
"""Test that add stores submitted text under the root snapshot staticfile output."""
def test_add_direct_url_creates_snapshot_without_internal_input_file(initialized_archive):
"""Test that explicit URL args queue real snapshots without stdin import files."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
@ -632,9 +625,10 @@ def test_add_creates_internal_input_file(initialized_archive):
)
with use_archivebox_db(initialized_archive):
root_snapshot = Snapshot.objects.get()
source_content = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert source_content == "https://example.com"
snapshot = Snapshot.objects.get()
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
assert not (snapshot.output_dir / "staticfile" / "stdin.txt").exists()
def test_add_multiple_urls_single_command(initialized_archive):
@ -650,11 +644,10 @@ def test_add_multiple_urls_single_command(initialized_archive):
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
root_snapshot = Snapshot.objects.get()
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth"))
assert crawl.urls == "https://example.com\nhttps://example.org"
assert root_input == "https://example.com\nhttps://example.org"
assert snapshots == [("https://example.com", 1), ("https://example.org", 1)]
def test_add_rejects_file_path_argument(initialized_archive):
@ -807,14 +800,11 @@ def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
with use_archivebox_db(initialized_archive):
crawl_count = Crawl.objects.count()
root_inputs = [
snapshot.output_dir.joinpath("staticfile", "stdin.txt").read_text(encoding="utf-8")
for snapshot in Snapshot.objects.order_by("created_at")
]
snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "depth"))
# Each add creates a new crawl with its own queued work.
assert crawl_count == 2
assert root_inputs == ["https://example.com", "https://example.com"]
assert snapshots == [("https://example.com", 1), ("https://example.com", 1)]
def test_add_with_overwrite_flag(initialized_archive):
@ -929,16 +919,17 @@ def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
root_snapshot = Snapshot.objects.get()
snapshot = Snapshot.objects.get()
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
def test_add_index_only_creates_only_internal_root_snapshot(initialized_archive):
"""Test that index-only add creates the input root but not parsed child snapshots."""
def test_add_index_only_creates_direct_url_snapshot(initialized_archive):
"""Test that index-only add queues explicit URL args as real URL snapshots."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
@ -948,10 +939,11 @@ def test_add_index_only_creates_only_internal_root_snapshot(initialized_archive)
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
root_snapshot = Snapshot.objects.get()
snapshot = Snapshot.objects.get()
assert crawl.urls == "https://example.com"
assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
assert snapshot.url == "https://example.com"
assert snapshot.depth == 1
def test_snapshot_create_sets_snapshot_timestamp(initialized_archive):
@ -1188,7 +1180,6 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive,
assert crawl_config["CRAWL_MAX_URLS"] == 2
assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024
assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0
assert (Snapshot.INTERNAL_INPUT_URL, 0, "sealed") in snapshots
assert (root_url, 1, "sealed") in snapshots
assert any(url == child_url and depth == 2 and status == "sealed" for url, depth, status in snapshots)