diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 295dcf81..a4286bcf 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -228,6 +228,9 @@ def add( print( "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]", ) + from archivebox.services.runner import ensure_background_runner + + ensure_background_runner(allow_under_pytest=True) else: # Foreground mode: run full crawl runner until all work is done print("[green]\\[*] Starting crawl runner to process crawl...[/green]") diff --git a/archivebox/core/views.py b/archivebox/core/views.py index f7aa2101..5d0b31fb 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -54,6 +54,7 @@ from archivebox.misc.util import ( htmlencode, ts_to_date_str, urldecode, + validate_url, without_fragment, ) from archivebox.misc.serve_static import serve_static_with_byterange_support @@ -1407,8 +1408,20 @@ class AddView(UserPassesTestMixin, FormView): def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl: from archivebox.cli.archivebox_add import add - urls = form.cleaned_data["url"] - print(f"[+] Adding URL: {urls}") + urls_input = form.cleaned_data["url"] + urls = urls_input + submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()] + if len(submitted_lines) == 1: + try: + # A lone URL pasted into /add/ is the same user-facing input as + # `archivebox add https://...`: queue that URL directly so a + # narrow plugin selection like `wget` can archive it without + # also needing parser plugins. Multi-line or formatted text + # remains verbatim import content for the internal parser root. + urls = [validate_url(submitted_lines[0])] + except ValueError: + pass + print(f"[+] Adding URL: {urls_input}") # Extract all form fields tag = form.cleaned_data["tag"] @@ -1453,6 +1466,8 @@ class AddView(UserPassesTestMixin, FormView): config["DELETE_AFTER"] = delete_after if timeout is not None and int(timeout) != int(effective_config.TIMEOUT): config["TIMEOUT"] = int(timeout) + if permissions: + config["PERMISSIONS"] = permissions config.update(plugin_config) config.update(custom_config) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 141acea1..2530661e 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -181,6 +181,9 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool: runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"): return False + if supervisor is not None: + start_worker(supervisor, RUNNER_WORKER) + return True machine = Machine.current() Process.cleanup_stale_running(machine=machine) @@ -192,13 +195,6 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool: if any(proc.is_running for proc in running_orchestrators): return False - if supervisor is not None: - try: - start_worker(supervisor, RUNNER_WORKER) - return True - except Exception: - pass - return False diff --git a/archivebox/tests/test_config_DELETE_AFTER.py b/archivebox/tests/test_config_DELETE_AFTER.py index 6a4c0f0b..032b7b81 100644 --- a/archivebox/tests/test_config_DELETE_AFTER.py +++ b/archivebox/tests/test_config_DELETE_AFTER.py @@ -231,9 +231,9 @@ def test_delete_after_real_add_page_and_rest_create_paths(client): from archivebox.services.runner import run_due_crawl assert run_due_crawl(ui_crawl, lock_seconds=10) - ui_snapshot = ui_crawl.snapshot_set.get(url="archivebox://internal") + ui_snapshot = ui_crawl.snapshot_set.get(url="https://example.com/delete-after-ui") assert ui_snapshot.delete_at is not None - assert ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").read_text() == "https://example.com/delete-after-ui" + assert not ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").exists() from archivebox.api.auth import get_or_create_api_token diff --git a/archivebox/tests/test_recursive_crawl.py b/archivebox/tests/test_recursive_crawl.py index 7f808bbb..1baa75b1 100644 --- a/archivebox/tests/test_recursive_crawl.py +++ b/archivebox/tests/test_recursive_crawl.py @@ -222,8 +222,8 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive, env=env, timeout=120, condition=lambda: ( - Snapshot.objects.filter(depth=0).count() >= 1 - and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"]) + Snapshot.objects.filter(depth=1).count() >= 1 + and Snapshot.objects.filter(depth=2).count() >= len(recursive_test_site["child_urls"]) ), ) @@ -235,9 +235,9 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive, with use_archivebox_db(tmp_path): all_snapshots = list(Snapshot.objects.values_list("url", "depth")) root_snapshot = ( - Snapshot.objects.filter(depth=0).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first() + Snapshot.objects.filter(depth=1).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first() ) - child_snapshots = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "depth", "parent_snapshot_id")) + child_snapshots = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "depth", "parent_snapshot_id")) crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first() parser_status = list( ArchiveResult.objects.filter( @@ -253,18 +253,18 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive, ).values_list("plugin", "status"), ) - assert root_snapshot is not None, f"Root snapshot should exist at depth=0. All snapshots: {all_snapshots}" + assert root_snapshot is not None, f"Root snapshot should exist at depth=1. All snapshots: {all_snapshots}" root_id = root_snapshot[0] assert crawl is not None, "Crawl should be created" - assert crawl[1] == 1, f"Crawl max_depth should be 1, got {crawl[1]}" + assert crawl[1] == 2, f"Crawl max_depth should be 2, got {crawl[1]}" assert len(child_snapshots) > 0, ( f"Child snapshots should be created from monadical.com links. Parser status: {parser_status}. Started extractors blocking: {started_extractors}" ) for child_id, child_url, child_depth, parent_id in child_snapshots: - assert child_depth == 1, f"Child snapshot should have depth=1, got {child_depth}" + assert child_depth == 2, f"Child snapshot should have depth=2, got {child_depth}" assert parent_id == root_id, f"Child snapshot {child_url} should have parent_snapshot_id={root_id}, got {parent_id}" @@ -280,10 +280,10 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, initialized_archive, rec env=env, timeout=120, condition=lambda: ( - Snapshot.objects.filter(depth=0).count() >= 1 - and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"]) + Snapshot.objects.filter(depth=1).count() >= 1 + and Snapshot.objects.filter(depth=2).count() >= len(recursive_test_site["child_urls"]) and ArchiveResult.objects.filter( - snapshot__depth=1, + snapshot__depth=2, plugin__startswith="parse_", plugin__endswith="_urls", status__in=("started", "succeeded", "failed"), @@ -301,7 +301,7 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, initialized_archive, rec depth_counts = [(depth, Snapshot.objects.filter(depth=depth).count()) for depth in sorted(set(depths))] assert max_depth_found is not None, "Should have at least one snapshot" - assert max_depth_found <= 1, f"Max depth should not exceed 1, got {max_depth_found}. Depth distribution: {depth_counts}" + assert max_depth_found <= 2, f"Max depth should not exceed 2, got {max_depth_found}. Depth distribution: {depth_counts}" def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_path, initialized_archive, recursive_test_site): @@ -348,10 +348,10 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p depth_counts = {depth: Snapshot.objects.filter(depth=depth).count() for depth in sorted(set(depths))} crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first() root_snapshot = ( - Snapshot.objects.filter(depth=0).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first() + Snapshot.objects.filter(depth=1).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first() ) - child_rows = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "parent_snapshot_id")) - deep_rows = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "parent_snapshot_id")) + child_rows = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "parent_snapshot_id")) + deep_rows = list(Snapshot.objects.filter(depth=3).values_list("id", "url", "parent_snapshot_id")) parser_results = list( ArchiveResult.objects.filter(plugin__startswith="parse_", plugin__endswith="_urls") .order_by("snapshot__depth", "snapshot__url") @@ -369,14 +369,14 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p ) assert crawl is not None - assert crawl[1] == 2 + assert crawl[1] == 3 assert root_snapshot is not None - assert root_snapshot[2] == 0 + assert root_snapshot[2] == 1 assert root_snapshot[3] is None - assert depth_counts.get(0, 0) >= 1 - assert depth_counts.get(1, 0) >= len(recursive_test_site["child_urls"]) - assert depth_counts.get(2, 0) >= len(recursive_test_site["deep_urls"]) - assert max(depth_counts) <= 2 + assert depth_counts.get(1, 0) >= 1 + assert depth_counts.get(2, 0) >= len(recursive_test_site["child_urls"]) + assert depth_counts.get(3, 0) >= len(recursive_test_site["deep_urls"]) + assert max(depth_counts) <= 3 child_urls = {row[1] for row in child_rows} deep_urls = {row[1] for row in deep_rows} diff --git a/archivebox/tests/test_ui_add_view.py b/archivebox/tests/test_ui_add_view.py index 793f06d3..a119e981 100644 --- a/archivebox/tests/test_ui_add_view.py +++ b/archivebox/tests/test_ui_add_view.py @@ -420,8 +420,9 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp assert crawl.retry_at is not None assert crawl.urls == "https://example.com" root_snapshot = crawl.snapshot_set.get() - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL - assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == "https://example.com" + assert root_snapshot.url == "https://example.com" + assert root_snapshot.depth == 1 + assert not (root_snapshot.output_dir / "staticfile" / "stdin.txt").exists() def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, admin_user, monkeypatch): @@ -457,8 +458,9 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad assert crawl.retry_at == RETRY_AT_MAX assert crawl.urls == "https://example.com/paused" root_snapshot = crawl.snapshot_set.get() - assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL - assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == "https://example.com/paused" + assert root_snapshot.url == "https://example.com/paused" + assert root_snapshot.depth == 1 + assert not (root_snapshot.output_dir / "staticfile" / "stdin.txt").exists() assert crawl.config.get("INDEX_ONLY") is not True