chore: checkpoint dev stack changes

archivebox/cli/archivebox_add.py

@@ -180,8 +180,8 @@ def add(
         persona_id=persona_obj.id,
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
         created_by_id=created_by_id,
-        status=Crawl.StatusChoices.QUEUED if bg or index_only else Crawl.StatusChoices.STARTED,
-        retry_at=timezone.now() if bg else None,
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=None if index_only else timezone.now(),
         config=crawl_config,
     )
 
@@ -258,12 +258,9 @@ def add(
             except Exception:
                 rel_output_str = str(crawl.output_dir)
 
-            bind_addr = config.BIND_ADDR or "127.0.0.1:8000"
-            if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
-                base_url = bind_addr
-            else:
-                base_url = f"http://{bind_addr}"
-            admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/"
+            from archivebox.core.host_utils import build_admin_url
+
+            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
             print("\n[bold]crawl output saved to:[/bold]")
             print(f"  {rel_output_str}")

archivebox/cli/archivebox_run.py

@@ -238,7 +238,7 @@ def process_stdin_records() -> int:
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
+def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
     """
     Run the background runner loop.
 
@@ -264,7 +264,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int:
     current.mark_running(process_type=Process.TypeChoices.ORCHESTRATOR, pwd=str(CONSTANTS.DATA_DIR), timeout=0)
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
-            run_pending_crawls(daemon=daemon, crawl_id=crawl_id)
+            run_pending_crawls(daemon=daemon, crawl_id=crawl_id, maintenance_only=maintenance_only)
         return 0
     except KeyboardInterrupt:
         return 0

archivebox/cli/archivebox_server.py

@@ -25,7 +25,7 @@ def server(
     from archivebox.config.common import get_config
 
     config = get_config()
-    runserver_args = list(runserver_args or (config.BIND_ADDR,))
+    runserver_args = list(runserver_args or (config.LISTEN_HOST,))
 
     if init:
         from archivebox.cli.archivebox_init import init as archivebox_init
@@ -62,6 +62,11 @@ def server(
     except IndexError:
         pass
 
+    os.environ["LISTEN_HOST"] = f"{host}:{port}"
+    from archivebox.core.host_utils import build_admin_url
+
+    admin_url = build_admin_url("/admin/")
+
     from archivebox.workers.supervisord_util import (
         start_server_workers,
         stop_existing_supervisord_process,
@@ -83,7 +88,7 @@ def server(
         f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
     )
     print(
-        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]",
+        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
     )
     print("    > Writing ArchiveBox error log to ./logs/errors.log")
     print()

archivebox/cli/archivebox_update.py

@@ -341,7 +341,7 @@ def update(
                             if exit_code != 0:
                                 raise SystemExit(exit_code)
                     else:
-                        exit_code = run_runner(daemon=False)
+                        exit_code = run_runner(daemon=False, maintenance_only=index_only or migrate_only)
                         if exit_code != 0:
                             raise SystemExit(exit_code)
 

archivebox/config/common.py

@@ -123,12 +123,10 @@ class ServerConfig(BaseConfigSet):
     )
 
     SECRET_KEY: str = Field(default_factory=lambda: "".join(secrets.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(50)))
-    BIND_ADDR: str = Field(default="127.0.0.1:8000")
-    LISTEN_HOST: str = Field(default="archivebox.localhost:8000")
-    ADMIN_BASE_URL: str = Field(default="")
-    ARCHIVE_BASE_URL: str = Field(default="")
+    LISTEN_HOST: str = Field(default="127.0.0.1:8000")
+    BASE_URL: str = Field(default="")
     ALLOWED_HOSTS: str = Field(default="*")
-    CSRF_TRUSTED_ORIGINS: str = Field(default="http://admin.archivebox.localhost:8000")
+    CSRF_TRUSTED_ORIGINS: str = Field(default="")
     SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay")
 
     SNAPSHOTS_PER_PAGE: int = Field(default=40)

archivebox/core/host_utils.py

@@ -46,49 +46,78 @@ def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any)
     return split_host_port(get_listen_host(config=config))
 
 
-def _build_listen_host(subdomain: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+def _with_port(host: str, port: str | None) -> str:
+    return f"{host}:{port}" if port else host
+
+
+def _is_local_bind_host(host: str) -> bool:
+    return host in {"", "0.0.0.0", "::", "127.0.0.1", "::1", "localhost"}
+
+
+def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
-    host, port = get_listen_parts(config=config)
-    if not host:
+    listen_host, listen_port = get_listen_parts(config=config)
+    root_host = "archivebox.localhost" if _is_local_bind_host(listen_host) else listen_host
+    return _with_port(root_host, listen_port) if root_host else ""
+
+
+def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    config = config or get_config(**config_kwargs)
+    override = _normalize_base_url(config.BASE_URL)
+    if override:
+        return override
+
+    scheme = request.scheme if request else "http"
+    if request:
+        req_host, req_port = split_host_port(request.get_host())
+        if req_host.endswith(".archivebox.localhost"):
+            return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
+        if _is_local_bind_host(req_host):
+            return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
+
+    root_host = _root_host_from_listen(config=config)
+    return f"{scheme}://{root_host}" if root_host else ""
+
+
+def get_base_host(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    return urlparse(get_base_url(request=request, config=config, **config_kwargs)).netloc.lower()
+
+
+def _build_base_host(subdomain: str | None, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
+    base_host = get_base_host(request=request, config=config, **config_kwargs)
+    if not base_host:
         return ""
+    host, port = split_host_port(base_host)
     full_host = f"{subdomain}.{host}" if subdomain else host
-    if port:
-        return f"{full_host}:{port}"
-    return full_host
+    return _with_port(full_host, port)
 
 
 def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    override = _normalize_base_url(config.ADMIN_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("admin", config=config)
+        return get_base_host(config=config)
+    return _build_base_host("admin", config=config)
 
 
 def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    override = _normalize_base_url(config.ARCHIVE_BASE_URL)
-    if override:
-        return urlparse(override).netloc.lower()
-    return _build_listen_host("web", config=config)
+        return get_base_host(config=config)
+    return _build_base_host("web", config=config)
 
 
 def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host("api", config=config)
+        return get_base_host(config=config)
+    return _build_base_host("api", config=config)
 
 
 def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host("public", config=config)
+        return get_base_host(config=config)
+    return _build_base_host("public", config=config)
 
 
 def get_snapshot_subdomain(snapshot_id: str) -> str:
@@ -100,15 +129,15 @@ def get_snapshot_subdomain(snapshot_id: str) -> str:
 def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host(get_snapshot_subdomain(snapshot_id), config=config)
+        return get_base_host(config=config)
+    return _build_base_host(get_snapshot_subdomain(snapshot_id), config=config)
 
 
 def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return get_listen_host(config=config).lower()
-    return _build_listen_host(domain, config=config)
+        return get_base_host(config=config)
+    return _build_base_host(domain, config=config)
 
 
 def is_snapshot_subdomain(subdomain: str) -> bool:
@@ -129,14 +158,14 @@ def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None
     if not config.USES_SUBDOMAIN_ROUTING:
         return ""
     req_host, req_port = split_host_port(request_host)
-    listen_host, listen_port = get_listen_parts(config=config)
-    if not listen_host:
+    base_host, base_port = split_host_port(get_base_host(config=config))
+    if not base_host:
         return ""
-    if listen_port and req_port and listen_port != req_port:
+    if base_port and req_port and base_port != req_port:
         return ""
-    if req_host == listen_host:
+    if req_host == base_host:
         return ""
-    suffix = f".{listen_host}"
+    suffix = f".{base_host}"
     if req_host.endswith(suffix):
         return req_host[: -len(suffix)]
     return ""
@@ -155,13 +184,10 @@ def host_matches(request_host: str, target_host: str) -> bool:
 
 
 def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
-    if request and request.scheme != "http":
-        return request.scheme
     config = config or get_config()
-    for base_url in (config.ARCHIVE_BASE_URL, config.ADMIN_BASE_URL):
-        override = _normalize_base_url(base_url)
-        if override:
-            return urlparse(override).scheme
+    override = _normalize_base_url(config.BASE_URL)
+    if override:
+        return urlparse(override).scheme
     if request:
         return request.scheme
     return "http"
@@ -176,48 +202,48 @@ def _build_base_url_for_host(host: str, request=None, config: dict[str, Any] | N
 
 def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
-    override = _normalize_base_url(config.ADMIN_BASE_URL)
-    if override:
-        return override
     if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_admin_host(config=config), request=request, config=config)
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("admin", request=request, config=config), request=request, config=config)
 
 
 def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
-    override = _normalize_base_url(config.ARCHIVE_BASE_URL)
-    if override:
-        return override
     if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_web_host(config=config), request=request, config=config)
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("web", request=request, config=config), request=request, config=config)
 
 
 def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config)
-    return _build_base_url_for_host(get_api_host(config=config), request=request, config=config)
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
 
 
 def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
-    return _build_base_url_for_host(get_public_host(config=config), request=request, config=config)
+    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_base_url(request=request, config=config)
+    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
 
 
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
         return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{snapshot_id}")
-    return _build_base_url_for_host(get_snapshot_host(snapshot_id, config=config), request=request, config=config)
+    return _build_base_url_for_host(
+        _build_base_host(get_snapshot_subdomain(snapshot_id), request=request, config=config),
+        request=request,
+        config=config,
+    )
 
 
 def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
     config = config or get_config(**config_kwargs)
     if not config.USES_SUBDOMAIN_ROUTING:
         return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
-    return _build_base_url_for_host(get_original_host(domain, config=config), request=request, config=config)
+    return _build_base_url_for_host(_build_base_host(domain, request=request, config=config), request=request, config=config)
 
 
 def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:

archivebox/core/migrations/0023_upgrade_to_0_9_0.py

@@ -301,7 +301,20 @@ def upgrade_core_tables(apps, schema_editor):
                         COALESCE(added, CURRENT_TIMESTAMP) as created_at,
                         COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
                         updated as downloaded_at,
-                        'queued' as status
+                        CASE
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN 'queued'
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE 'queued'
+                        END as status
                     FROM core_snapshot;
                 """)
             elif has_bookmarked_at and not has_added:
@@ -320,7 +333,25 @@ def upgrade_core_tables(apps, schema_editor):
                     select_cols.append("REPLACE(crawl_id, '-', '')")
                 if has_status:
                     insert_cols.append("status")
-                    select_cols.append("status")
+                    select_cols.append(
+                        """
+                        CASE
+                            WHEN status IN ('sealed', 'started', 'paused') THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                            )
+                            THEN status
+                            WHEN EXISTS (
+                                SELECT 1 FROM core_archiveresult
+                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                            )
+                            THEN 'sealed'
+                            ELSE status
+                        END
+                        """,
+                    )
                 if has_retry_at:
                     insert_cols.append("retry_at")
                     select_cols.append("retry_at")

archivebox/core/settings.py

@@ -360,8 +360,8 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}
 
 SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_")
 
-ALLOWED_HOSTS = CONFIG.ALLOWED_HOSTS.split(",")
-CSRF_TRUSTED_ORIGINS = list(set(CONFIG.CSRF_TRUSTED_ORIGINS.split(",")))
+ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()]
+CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()})
 
 admin_base_url = normalize_base_url(get_admin_base_url())
 if admin_base_url and admin_base_url not in CSRF_TRUSTED_ORIGINS:

archivebox/core/views.py

@@ -191,8 +191,7 @@ class SnapshotView(View):
             "PREVIEW_ORIGINALS",
             "LISTEN_HOST",
             "USES_SUBDOMAIN_ROUTING",
-            "ADMIN_BASE_URL",
-            "ARCHIVE_BASE_URL",
+            "BASE_URL",
             "PERMISSIONS",
             "SERVER_SECURITY_MODE",
         }

archivebox/services/runner.py

@@ -1285,6 +1285,8 @@ def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool:
     if snapshot.is_paused:
         selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
         if not selected_plugins:
+            if snapshot.fs_migration_needed and Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
+                run_snapshot_maintenance(str(snapshot.id))
             # Paused is a real lifecycle state; retry_at=MAX is only the
             # orchestrator selection marker. If a direct maintenance/update
             # command bumps retry_at on a paused snapshot but there are no
@@ -1465,7 +1467,7 @@ def run_install(*, plugin_names: list[str] | None = None) -> None:
     asyncio.run(_run_install(plugin_names=plugin_names))
 
 
-def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> int:
+def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.machine.models import Binary, Process
@@ -1486,16 +1488,19 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
                 if schedule.is_due(now):
                     schedule.enqueue(queued_at=now)
 
-        due_crawls = Crawl.objects.filter(retry_at__lte=timezone.now())
-        if crawl_id:
-            due_crawls = due_crawls.filter(id=crawl_id)
-        due_crawl = due_crawls.order_by("retry_at", "created_at").first()
-        if due_crawl is not None:
-            if not run_due_crawl(due_crawl, lock_seconds=crawl_claim_lock_seconds):
+        if not maintenance_only:
+            due_crawls = Crawl.objects.filter(retry_at__lte=timezone.now())
+            if crawl_id:
+                due_crawls = due_crawls.filter(id=crawl_id)
+            due_crawl = due_crawls.order_by("retry_at", "created_at").first()
+            if due_crawl is not None:
+                if not run_due_crawl(due_crawl, lock_seconds=crawl_claim_lock_seconds):
+                    continue
                 continue
-            continue
 
         due_snapshots = Snapshot.objects.filter(retry_at__lte=timezone.now()).select_related("crawl")
+        if maintenance_only:
+            due_snapshots = due_snapshots.filter(status__in=[Snapshot.StatusChoices.PAUSED, Snapshot.StatusChoices.SEALED])
         if crawl_id:
             due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
         due_snapshot = due_snapshots.order_by("retry_at", "created_at").first()
@@ -1504,7 +1509,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) ->
                 continue
             continue
 
-        if crawl_id is None:
+        if crawl_id is None and not maintenance_only:
             due_binary = (
                 Binary.objects.filter(retry_at__lte=timezone.now())
                 .exclude(status=Binary.StatusChoices.INSTALLED)

archivebox/tests/conftest.py

@@ -419,9 +419,9 @@ def build_test_env(port: int, **extra: str) -> dict[str, str]:
     env.update(
         {
             "PLUGINS": "wget",
-            "LISTEN_HOST": f"archivebox.localhost:{port}",
+            "LISTEN_HOST": f"127.0.0.1:{port}",
+            "BASE_URL": f"http://archivebox.localhost:{port}",
             "ALLOWED_HOSTS": "*",
-            "CSRF_TRUSTED_ORIGINS": f"http://admin.archivebox.localhost:{port}",
             "PUBLIC_ADD_VIEW": "True",
             "USE_COLOR": "False",
             "SHOW_PROGRESS": "False",
@@ -830,7 +830,8 @@ def real_archive_with_example(tmp_path_factory, request):
         [
             "config",
             "--set",
-            "LISTEN_HOST=archivebox.localhost:8000",
+            "LISTEN_HOST=127.0.0.1:8000",
+            "BASE_URL=http://archivebox.localhost:8000",
             "PUBLIC_INDEX=True",
             "PUBLIC_ADD_VIEW=True",
         ],

archivebox/tests/test_cli_run.py

@@ -278,10 +278,19 @@ class TestRunEmpty:
 
 
 class TestRunDaemonMode:
-    def test_run_daemon_ignores_piped_stdin_and_starts_real_runner(self, initialized_archive, db):
+    @pytest.mark.parametrize("stdin_kind", ["malformed", "valid-snapshot"])
+    def test_run_daemon_ignores_piped_stdin_and_starts_real_runner(self, initialized_archive, db, stdin_kind):
         from archivebox.machine.models import Process
+        from archivebox.core.models import Snapshot
         from archivebox.tests.test_orm_helpers import use_archivebox_db
 
+        snapshot_url = None
+        if stdin_kind == "valid-snapshot":
+            snapshot_url = create_test_url()
+            piped_stdin = json.dumps(create_test_snapshot_json(url=snapshot_url)) + "\n"
+        else:
+            piped_stdin = "{this is not jsonl}\n"
+
         env = os.environ.copy()
         env.update(
             {
@@ -306,7 +315,7 @@ class TestRunDaemonMode:
         assert proc.stderr is not None
 
         try:
-            proc.stdin.write("{this is not jsonl}\n")
+            proc.stdin.write(piped_stdin)
             proc.stdin.close()
 
             deadline = time.monotonic() + 20
@@ -327,6 +336,9 @@ class TestRunDaemonMode:
                 time.sleep(0.25)
 
             assert started is True
+            if snapshot_url is not None:
+                with use_archivebox_db(initialized_archive):
+                    assert not Snapshot.objects.filter(url=snapshot_url).exists()
         finally:
             if proc.poll() is None:
                 os.killpg(proc.pid, signal.SIGTERM)
@@ -725,6 +737,28 @@ class TestRecoverOrchestratorState:
 
 @pytest.mark.django_db
 class TestRunDueCrawlState:
+    def test_maintenance_only_runner_does_not_start_regular_queued_crawls(self):
+        from django.utils import timezone
+
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.services.runner import run_pending_crawls
+
+        now = timezone.now()
+        crawl = Crawl.objects.create(
+            urls="https://example.com",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.QUEUED,
+            retry_at=now,
+        )
+
+        assert run_pending_crawls(daemon=False, maintenance_only=True) == 0
+
+        crawl.refresh_from_db()
+        assert crawl.status == Crawl.StatusChoices.QUEUED
+        assert crawl.retry_at == now
+        assert crawl.snapshot_set.count() == 0
+
     def test_snapshot_start_writes_short_future_lease(self):
         from django.utils import timezone
 

archivebox/tests/test_server_security_browser.py

@@ -345,9 +345,9 @@ def _start_server(data_dir: Path, *, mode: str, port: int) -> subprocess.Popen[s
     env.update(
         {
             "PYTHONPATH": str(Path(__file__).resolve().parents[2]),
-            "LISTEN_HOST": f"archivebox.localhost:{port}",
+            "LISTEN_HOST": f"127.0.0.1:{port}",
+            "BASE_URL": f"http://archivebox.localhost:{port}",
             "ALLOWED_HOSTS": "*",
-            "CSRF_TRUSTED_ORIGINS": f"http://archivebox.localhost:{port},http://admin.archivebox.localhost:{port}",
             "SERVER_SECURITY_MODE": mode,
             "USE_COLOR": "False",
             "SHOW_PROGRESS": "False",

archivebox/tests/test_urls.py

@@ -54,6 +54,7 @@ def _build_script(body: str) -> str:
     from archivebox.core.host_utils import (
         get_admin_host,
         get_admin_base_url,
+        get_base_host,
         get_api_host,
         get_web_host,
         get_web_base_url,
@@ -201,8 +202,12 @@ class TestUrlRouting:
             snapshot_subdomain = get_snapshot_subdomain(snapshot_id)
             snapshot_host = get_snapshot_host(snapshot_id)
             original_host = get_original_host(domain)
-            base_host = SERVER_CONFIG.LISTEN_HOST
+            listen_host = SERVER_CONFIG.LISTEN_HOST
+            base_host = get_base_host()
 
+            listen_host_only, listen_port = split_host_port(listen_host)
+            assert listen_host_only == "127.0.0.1"
+            assert listen_port == "8000"
             host_only, port = split_host_port(base_host)
             assert host_only == "archivebox.localhost"
             assert port == "8000"
@@ -262,7 +267,7 @@ class TestUrlRouting:
             """,
             mode="safe-subdomains-fullreplay",
             env_overrides={
-                "LISTEN_HOST": "archivebox.io",
+                "BASE_URL": "https://archivebox.io",
             },
         )
 
@@ -330,7 +335,7 @@ class TestUrlRouting:
             snapshot_host = get_snapshot_host(snapshot_id)
             original_host = get_original_host(snapshot.domain)
             web_host = get_web_host()
-            host_only, port = split_host_port(SERVER_CONFIG.LISTEN_HOST)
+            host_only, port = split_host_port(get_base_host())
             legacy_snapshot_host = f"{snapshot_id}.{host_only}"
             if port:
                 legacy_snapshot_host = f"{legacy_snapshot_host}:{port}"
@@ -574,7 +579,7 @@ class TestUrlRouting:
             snapshot_id = str(snapshot.id)
 
             client = Client()
-            base_host = SERVER_CONFIG.LISTEN_HOST
+            base_host = get_base_host()
             web_host = get_web_host()
             admin_host = get_admin_host()
             api_host = get_api_host()
@@ -635,7 +640,7 @@ class TestUrlRouting:
             snapshot_id = str(snapshot.id)
 
             client = Client()
-            base_host = SERVER_CONFIG.LISTEN_HOST
+            base_host = get_base_host()
 
             assert SERVER_CONFIG.SERVER_SECURITY_MODE == "unsafe-onedomain-noadmin"
             assert SERVER_CONFIG.CONTROL_PLANE_ENABLED is False
@@ -670,7 +675,7 @@ class TestUrlRouting:
             snapshot_id = str(snapshot.id)
 
             client = Client()
-            base_host = SERVER_CONFIG.LISTEN_HOST
+            base_host = get_base_host()
 
             assert SERVER_CONFIG.SERVER_SECURITY_MODE == "danger-onedomain-fullreplay"
             assert SERVER_CONFIG.CONTROL_PLANE_ENABLED is True
@@ -710,15 +715,15 @@ class TestUrlRouting:
             """
             snapshot = get_snapshot()
             snapshot_id = str(snapshot.id)
-            base_host = SERVER_CONFIG.LISTEN_HOST
+            base_host = get_base_host()
 
             assert SERVER_CONFIG.SERVER_SECURITY_MODE == "safe-onedomain-nojsreplay"
             assert get_admin_host() == base_host
             assert get_web_host() == base_host
 
-            assert get_admin_base_url() == "https://admin.archivebox.example"
+            assert get_admin_base_url() == "https://archivebox.example"
             assert get_web_base_url() == "https://archivebox.example"
-            assert build_admin_url("/admin/login/") == "https://admin.archivebox.example/admin/login/"
+            assert build_admin_url("/admin/login/") == "https://archivebox.example/admin/login/"
             assert build_snapshot_url(snapshot_id, "index.jsonl") == (
                 f"https://archivebox.example/snapshot/{snapshot_id}/index.jsonl"
             )
@@ -727,8 +732,7 @@ class TestUrlRouting:
             """,
             mode="safe-onedomain-nojsreplay",
             env_overrides={
-                "ADMIN_BASE_URL": "https://admin.archivebox.example",
-                "ARCHIVE_BASE_URL": "https://archivebox.example",
+                "BASE_URL": "https://archivebox.example",
             },
         )
 
@@ -748,7 +752,7 @@ class TestUrlRouting:
             """,
             mode="safe-subdomains-fullreplay",
             env_overrides={
-                "ARCHIVE_BASE_URL": "https://web.archivebox.example",
+                "BASE_URL": "https://archivebox.example",
             },
         )
 
@@ -769,7 +773,7 @@ class TestUrlRouting:
             resp = client.get("/public/", HTTP_HOST=web_host)
             assert resp.status_code == 200
             public_html = response_body(resp).decode("utf-8", "ignore")
-            assert "http://web.archivebox.localhost:8000" in public_html
+            assert f"http://{snapshot_host}/" in public_html
 
             ensure_admin_user()
             assert client.login(username="testadmin", password="testpassword")
@@ -839,7 +843,7 @@ class TestUrlRouting:
             resp = client.get(f"/admin/core/snapshot/{snapshot_id}/change/", HTTP_HOST=admin_host)
             assert resp.status_code == 200
             admin_html = response_body(resp).decode("utf-8", "ignore")
-            assert f"http://web.archivebox.localhost:8000/{snapshot.archive_path}" in admin_html
+            assert f"http://{web_host}/{snapshot.archive_path}" in admin_html
             assert f"http://{snapshot_host}/" in admin_html
 
             result = ArchiveResult.objects.filter(snapshot=snapshot).first()
@@ -866,6 +870,7 @@ class TestUrlRouting:
                 '{"level":"warn","text":"second line"}\\n',
                 encoding="utf-8",
             )
+            snapshot.write_html_details()
 
             client = Client()
             resp = client.get(f"/{snapshot.url_path}/index.html", HTTP_HOST=web_host)

docker-compose.yml

@@ -20,9 +20,9 @@ services:
         environment:
             # - ADMIN_USERNAME=admin            # creates an admin user on first run with the given user/pass combo
             # - ADMIN_PASSWORD=SomeSecretPassword
-            - LISTEN_HOST=archivebox.localhost:8000
+            - LISTEN_HOST=0.0.0.0:8000
+            - BASE_URL=http://archivebox.localhost:8000  # public URL used to build admin/web/api/snapshot links
             - ALLOWED_HOSTS=*                   # set this to the hostname(s) you're going to serve the site from!
-            - CSRF_TRUSTED_ORIGINS=http://admin.archivebox.localhost:8000  # MUST match the admin UI URL for login/API to work
             - PUBLIC_INDEX=True                 # set to False to prevent anonymous users from viewing snapshot list
             - PUBLIC_SNAPSHOTS=True             # set to False to prevent anonymous users from viewing snapshot content
             - PUBLIC_ADD_VIEW=False             # set to True to allow anonymous users to submit new URLs to archive