Fix UI add direct URL runner flow

archivebox/cli/archivebox_add.py

@@ -228,6 +228,9 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
+        from archivebox.services.runner import ensure_background_runner
+
+        ensure_background_runner(allow_under_pytest=True)
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")

archivebox/core/views.py

@@ -54,6 +54,7 @@ from archivebox.misc.util import (
     htmlencode,
     ts_to_date_str,
     urldecode,
+    validate_url,
     without_fragment,
 )
 from archivebox.misc.serve_static import serve_static_with_byterange_support
@@ -1407,8 +1408,20 @@ class AddView(UserPassesTestMixin, FormView):
     def _create_crawl_from_form(self, form, *, created_by_id=None) -> Crawl:
         from archivebox.cli.archivebox_add import add
 
-        urls = form.cleaned_data["url"]
-        print(f"[+] Adding URL: {urls}")
+        urls_input = form.cleaned_data["url"]
+        urls = urls_input
+        submitted_lines = [line.strip() for line in urls_input.splitlines() if line.strip()]
+        if len(submitted_lines) == 1:
+            try:
+                # A lone URL pasted into /add/ is the same user-facing input as
+                # `archivebox add https://...`: queue that URL directly so a
+                # narrow plugin selection like `wget` can archive it without
+                # also needing parser plugins. Multi-line or formatted text
+                # remains verbatim import content for the internal parser root.
+                urls = [validate_url(submitted_lines[0])]
+            except ValueError:
+                pass
+        print(f"[+] Adding URL: {urls_input}")
 
         # Extract all form fields
         tag = form.cleaned_data["tag"]
@@ -1453,6 +1466,8 @@ class AddView(UserPassesTestMixin, FormView):
             config["DELETE_AFTER"] = delete_after
         if timeout is not None and int(timeout) != int(effective_config.TIMEOUT):
             config["TIMEOUT"] = int(timeout)
+        if permissions:
+            config["PERMISSIONS"] = permissions
 
         config.update(plugin_config)
         config.update(custom_config)

archivebox/services/runner.py

@@ -181,6 +181,9 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
     runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
     if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
         return False
+    if supervisor is not None:
+        start_worker(supervisor, RUNNER_WORKER)
+        return True
 
     machine = Machine.current()
     Process.cleanup_stale_running(machine=machine)
@@ -192,13 +195,6 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
     if any(proc.is_running for proc in running_orchestrators):
         return False
 
-    if supervisor is not None:
-        try:
-            start_worker(supervisor, RUNNER_WORKER)
-            return True
-        except Exception:
-            pass
-
     return False
 
 

archivebox/tests/test_config_DELETE_AFTER.py

@@ -231,9 +231,9 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
     from archivebox.services.runner import run_due_crawl
 
     assert run_due_crawl(ui_crawl, lock_seconds=10)
-    ui_snapshot = ui_crawl.snapshot_set.get(url="archivebox://internal")
+    ui_snapshot = ui_crawl.snapshot_set.get(url="https://example.com/delete-after-ui")
     assert ui_snapshot.delete_at is not None
-    assert ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").read_text() == "https://example.com/delete-after-ui"
+    assert not ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").exists()
 
     from archivebox.api.auth import get_or_create_api_token
 

archivebox/tests/test_recursive_crawl.py

@@ -222,8 +222,8 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive,
         env=env,
         timeout=120,
         condition=lambda: (
-            Snapshot.objects.filter(depth=0).count() >= 1
-            and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"])
+            Snapshot.objects.filter(depth=1).count() >= 1
+            and Snapshot.objects.filter(depth=2).count() >= len(recursive_test_site["child_urls"])
         ),
     )
 
@@ -235,9 +235,9 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive,
     with use_archivebox_db(tmp_path):
         all_snapshots = list(Snapshot.objects.values_list("url", "depth"))
         root_snapshot = (
-            Snapshot.objects.filter(depth=0).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
+            Snapshot.objects.filter(depth=1).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
         )
-        child_snapshots = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "depth", "parent_snapshot_id"))
+        child_snapshots = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "depth", "parent_snapshot_id"))
         crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first()
         parser_status = list(
             ArchiveResult.objects.filter(
@@ -253,18 +253,18 @@ def test_recursive_crawl_creates_child_snapshots(tmp_path, initialized_archive,
             ).values_list("plugin", "status"),
         )
 
-    assert root_snapshot is not None, f"Root snapshot should exist at depth=0. All snapshots: {all_snapshots}"
+    assert root_snapshot is not None, f"Root snapshot should exist at depth=1. All snapshots: {all_snapshots}"
     root_id = root_snapshot[0]
 
     assert crawl is not None, "Crawl should be created"
-    assert crawl[1] == 1, f"Crawl max_depth should be 1, got {crawl[1]}"
+    assert crawl[1] == 2, f"Crawl max_depth should be 2, got {crawl[1]}"
 
     assert len(child_snapshots) > 0, (
         f"Child snapshots should be created from monadical.com links. Parser status: {parser_status}. Started extractors blocking: {started_extractors}"
     )
 
     for child_id, child_url, child_depth, parent_id in child_snapshots:
-        assert child_depth == 1, f"Child snapshot should have depth=1, got {child_depth}"
+        assert child_depth == 2, f"Child snapshot should have depth=2, got {child_depth}"
         assert parent_id == root_id, f"Child snapshot {child_url} should have parent_snapshot_id={root_id}, got {parent_id}"
 
 
@@ -280,10 +280,10 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, initialized_archive, rec
         env=env,
         timeout=120,
         condition=lambda: (
-            Snapshot.objects.filter(depth=0).count() >= 1
-            and Snapshot.objects.filter(depth=1).count() >= len(recursive_test_site["child_urls"])
+            Snapshot.objects.filter(depth=1).count() >= 1
+            and Snapshot.objects.filter(depth=2).count() >= len(recursive_test_site["child_urls"])
             and ArchiveResult.objects.filter(
-                snapshot__depth=1,
+                snapshot__depth=2,
                 plugin__startswith="parse_",
                 plugin__endswith="_urls",
                 status__in=("started", "succeeded", "failed"),
@@ -301,7 +301,7 @@ def test_recursive_crawl_respects_depth_limit(tmp_path, initialized_archive, rec
         depth_counts = [(depth, Snapshot.objects.filter(depth=depth).count()) for depth in sorted(set(depths))]
 
     assert max_depth_found is not None, "Should have at least one snapshot"
-    assert max_depth_found <= 1, f"Max depth should not exceed 1, got {max_depth_found}. Depth distribution: {depth_counts}"
+    assert max_depth_found <= 2, f"Max depth should not exceed 2, got {max_depth_found}. Depth distribution: {depth_counts}"
 
 
 def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_path, initialized_archive, recursive_test_site):
@@ -348,10 +348,10 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
         depth_counts = {depth: Snapshot.objects.filter(depth=depth).count() for depth in sorted(set(depths))}
         crawl = Crawl.objects.order_by("-created_at").values_list("id", "max_depth").first()
         root_snapshot = (
-            Snapshot.objects.filter(depth=0).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
+            Snapshot.objects.filter(depth=1).order_by("created_at").values_list("id", "url", "depth", "parent_snapshot_id").first()
         )
-        child_rows = list(Snapshot.objects.filter(depth=1).values_list("id", "url", "parent_snapshot_id"))
-        deep_rows = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "parent_snapshot_id"))
+        child_rows = list(Snapshot.objects.filter(depth=2).values_list("id", "url", "parent_snapshot_id"))
+        deep_rows = list(Snapshot.objects.filter(depth=3).values_list("id", "url", "parent_snapshot_id"))
         parser_results = list(
             ArchiveResult.objects.filter(plugin__startswith="parse_", plugin__endswith="_urls")
             .order_by("snapshot__depth", "snapshot__url")
@@ -369,14 +369,14 @@ def test_recursive_crawl_depth_two_writes_real_outputs_and_process_records(tmp_p
         )
 
     assert crawl is not None
-    assert crawl[1] == 2
+    assert crawl[1] == 3
     assert root_snapshot is not None
-    assert root_snapshot[2] == 0
+    assert root_snapshot[2] == 1
     assert root_snapshot[3] is None
-    assert depth_counts.get(0, 0) >= 1
-    assert depth_counts.get(1, 0) >= len(recursive_test_site["child_urls"])
-    assert depth_counts.get(2, 0) >= len(recursive_test_site["deep_urls"])
-    assert max(depth_counts) <= 2
+    assert depth_counts.get(1, 0) >= 1
+    assert depth_counts.get(2, 0) >= len(recursive_test_site["child_urls"])
+    assert depth_counts.get(3, 0) >= len(recursive_test_site["deep_urls"])
+    assert max(depth_counts) <= 3
 
     child_urls = {row[1] for row in child_rows}
     deep_urls = {row[1] for row in deep_rows}

archivebox/tests/test_ui_add_view.py

@@ -420,8 +420,9 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp
     assert crawl.retry_at is not None
     assert crawl.urls == "https://example.com"
     root_snapshot = crawl.snapshot_set.get()
-    assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
-    assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == "https://example.com"
+    assert root_snapshot.url == "https://example.com"
+    assert root_snapshot.depth == 1
+    assert not (root_snapshot.output_dir / "staticfile" / "stdin.txt").exists()
 
 
 def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, admin_user, monkeypatch):
@@ -457,8 +458,9 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad
     assert crawl.retry_at == RETRY_AT_MAX
     assert crawl.urls == "https://example.com/paused"
     root_snapshot = crawl.snapshot_set.get()
-    assert root_snapshot.url == Snapshot.INTERNAL_INPUT_URL
-    assert (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8") == "https://example.com/paused"
+    assert root_snapshot.url == "https://example.com/paused"
+    assert root_snapshot.depth == 1
+    assert not (root_snapshot.output_dir / "staticfile" / "stdin.txt").exists()
     assert crawl.config.get("INDEX_ONLY") is not True