diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 2902a441..39b65ae9 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -180,8 +180,8 @@ def add( persona_id=persona_obj.id, label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]", created_by_id=created_by_id, - status=Crawl.StatusChoices.QUEUED if bg or index_only else Crawl.StatusChoices.STARTED, - retry_at=timezone.now() if bg else None, + status=Crawl.StatusChoices.QUEUED, + retry_at=None if index_only else timezone.now(), config=crawl_config, ) @@ -258,12 +258,9 @@ def add( except Exception: rel_output_str = str(crawl.output_dir) - bind_addr = config.BIND_ADDR or "127.0.0.1:8000" - if bind_addr.startswith("http://") or bind_addr.startswith("https://"): - base_url = bind_addr - else: - base_url = f"http://{bind_addr}" - admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/" + from archivebox.core.host_utils import build_admin_url + + admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config) print("\n[bold]crawl output saved to:[/bold]") print(f" {rel_output_str}") diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py index c770947e..22161f14 100644 --- a/archivebox/cli/archivebox_run.py +++ b/archivebox/cli/archivebox_run.py @@ -238,7 +238,7 @@ def process_stdin_records() -> int: return 0 -def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int: +def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int: """ Run the background runner loop. @@ -264,7 +264,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None) -> int: current.mark_running(process_type=Process.TypeChoices.ORCHESTRATOR, pwd=str(CONSTANTS.DATA_DIR), timeout=0) try: with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon): - run_pending_crawls(daemon=daemon, crawl_id=crawl_id) + run_pending_crawls(daemon=daemon, crawl_id=crawl_id, maintenance_only=maintenance_only) return 0 except KeyboardInterrupt: return 0 diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 5b4dbed1..ae4925c7 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -25,7 +25,7 @@ def server( from archivebox.config.common import get_config config = get_config() - runserver_args = list(runserver_args or (config.BIND_ADDR,)) + runserver_args = list(runserver_args or (config.LISTEN_HOST,)) if init: from archivebox.cli.archivebox_init import init as archivebox_init @@ -62,6 +62,11 @@ def server( except IndexError: pass + os.environ["LISTEN_HOST"] = f"{host}:{port}" + from archivebox.core.host_utils import build_admin_url + + admin_url = build_admin_url("/admin/") + from archivebox.workers.supervisord_util import ( start_server_workers, stop_existing_supervisord_process, @@ -83,7 +88,7 @@ def server( f" [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]", ) print( - f" [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]", + f" [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]", ) print(" > Writing ArchiveBox error log to ./logs/errors.log") print() diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index eda1d1c8..0d574261 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -341,7 +341,7 @@ def update( if exit_code != 0: raise SystemExit(exit_code) else: - exit_code = run_runner(daemon=False) + exit_code = run_runner(daemon=False, maintenance_only=index_only or migrate_only) if exit_code != 0: raise SystemExit(exit_code) diff --git a/archivebox/config/common.py b/archivebox/config/common.py index 958f1700..941ea1f5 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -123,12 +123,10 @@ class ServerConfig(BaseConfigSet): ) SECRET_KEY: str = Field(default_factory=lambda: "".join(secrets.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(50))) - BIND_ADDR: str = Field(default="127.0.0.1:8000") - LISTEN_HOST: str = Field(default="archivebox.localhost:8000") - ADMIN_BASE_URL: str = Field(default="") - ARCHIVE_BASE_URL: str = Field(default="") + LISTEN_HOST: str = Field(default="127.0.0.1:8000") + BASE_URL: str = Field(default="") ALLOWED_HOSTS: str = Field(default="*") - CSRF_TRUSTED_ORIGINS: str = Field(default="http://admin.archivebox.localhost:8000") + CSRF_TRUSTED_ORIGINS: str = Field(default="") SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay") SNAPSHOTS_PER_PAGE: int = Field(default=40) diff --git a/archivebox/core/host_utils.py b/archivebox/core/host_utils.py index c5688a56..98fdc6c1 100644 --- a/archivebox/core/host_utils.py +++ b/archivebox/core/host_utils.py @@ -46,49 +46,78 @@ def get_listen_parts(config: dict[str, Any] | None = None, **config_kwargs: Any) return split_host_port(get_listen_host(config=config)) -def _build_listen_host(subdomain: str | None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: +def _with_port(host: str, port: str | None) -> str: + return f"{host}:{port}" if port else host + + +def _is_local_bind_host(host: str) -> bool: + return host in {"", "0.0.0.0", "::", "127.0.0.1", "::1", "localhost"} + + +def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) - host, port = get_listen_parts(config=config) - if not host: + listen_host, listen_port = get_listen_parts(config=config) + root_host = "archivebox.localhost" if _is_local_bind_host(listen_host) else listen_host + return _with_port(root_host, listen_port) if root_host else "" + + +def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + config = config or get_config(**config_kwargs) + override = _normalize_base_url(config.BASE_URL) + if override: + return override + + scheme = request.scheme if request else "http" + if request: + req_host, req_port = split_host_port(request.get_host()) + if req_host.endswith(".archivebox.localhost"): + return f"{scheme}://{_with_port('archivebox.localhost', req_port)}" + if _is_local_bind_host(req_host): + return f"{scheme}://{_with_port('archivebox.localhost', req_port)}" + + root_host = _root_host_from_listen(config=config) + return f"{scheme}://{root_host}" if root_host else "" + + +def get_base_host(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + return urlparse(get_base_url(request=request, config=config, **config_kwargs)).netloc.lower() + + +def _build_base_host(subdomain: str | None, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: + base_host = get_base_host(request=request, config=config, **config_kwargs) + if not base_host: return "" + host, port = split_host_port(base_host) full_host = f"{subdomain}.{host}" if subdomain else host - if port: - return f"{full_host}:{port}" - return full_host + return _with_port(full_host, port) def get_admin_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: - return get_listen_host(config=config).lower() - override = _normalize_base_url(config.ADMIN_BASE_URL) - if override: - return urlparse(override).netloc.lower() - return _build_listen_host("admin", config=config) + return get_base_host(config=config) + return _build_base_host("admin", config=config) def get_web_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: - return get_listen_host(config=config).lower() - override = _normalize_base_url(config.ARCHIVE_BASE_URL) - if override: - return urlparse(override).netloc.lower() - return _build_listen_host("web", config=config) + return get_base_host(config=config) + return _build_base_host("web", config=config) def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: - return get_listen_host(config=config).lower() - return _build_listen_host("api", config=config) + return get_base_host(config=config) + return _build_base_host("api", config=config) def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: - return get_listen_host(config=config).lower() - return _build_listen_host("public", config=config) + return get_base_host(config=config) + return _build_base_host("public", config=config) def get_snapshot_subdomain(snapshot_id: str) -> str: @@ -100,15 +129,15 @@ def get_snapshot_subdomain(snapshot_id: str) -> str: def get_snapshot_host(snapshot_id: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: - return get_listen_host(config=config).lower() - return _build_listen_host(get_snapshot_subdomain(snapshot_id), config=config) + return get_base_host(config=config) + return _build_base_host(get_snapshot_subdomain(snapshot_id), config=config) def get_original_host(domain: str, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: - return get_listen_host(config=config).lower() - return _build_listen_host(domain, config=config) + return get_base_host(config=config) + return _build_base_host(domain, config=config) def is_snapshot_subdomain(subdomain: str) -> bool: @@ -129,14 +158,14 @@ def get_listen_subdomain(request_host: str, config: dict[str, Any] | None = None if not config.USES_SUBDOMAIN_ROUTING: return "" req_host, req_port = split_host_port(request_host) - listen_host, listen_port = get_listen_parts(config=config) - if not listen_host: + base_host, base_port = split_host_port(get_base_host(config=config)) + if not base_host: return "" - if listen_port and req_port and listen_port != req_port: + if base_port and req_port and base_port != req_port: return "" - if req_host == listen_host: + if req_host == base_host: return "" - suffix = f".{listen_host}" + suffix = f".{base_host}" if req_host.endswith(suffix): return req_host[: -len(suffix)] return "" @@ -155,13 +184,10 @@ def host_matches(request_host: str, target_host: str) -> bool: def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str: - if request and request.scheme != "http": - return request.scheme config = config or get_config() - for base_url in (config.ARCHIVE_BASE_URL, config.ADMIN_BASE_URL): - override = _normalize_base_url(base_url) - if override: - return urlparse(override).scheme + override = _normalize_base_url(config.BASE_URL) + if override: + return urlparse(override).scheme if request: return request.scheme return "http" @@ -176,48 +202,48 @@ def _build_base_url_for_host(host: str, request=None, config: dict[str, Any] | N def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) - override = _normalize_base_url(config.ADMIN_BASE_URL) - if override: - return override if not config.USES_SUBDOMAIN_ROUTING: - return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config) - return _build_base_url_for_host(get_admin_host(config=config), request=request, config=config) + return get_base_url(request=request, config=config) + return _build_base_url_for_host(_build_base_host("admin", request=request, config=config), request=request, config=config) def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) - override = _normalize_base_url(config.ARCHIVE_BASE_URL) - if override: - return override if not config.USES_SUBDOMAIN_ROUTING: - return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config) - return _build_base_url_for_host(get_web_host(config=config), request=request, config=config) + return get_base_url(request=request, config=config) + return _build_base_url_for_host(_build_base_host("web", request=request, config=config), request=request, config=config) def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: - return _build_base_url_for_host(get_listen_host(config=config), request=request, config=config) - return _build_base_url_for_host(get_api_host(config=config), request=request, config=config) + return get_base_url(request=request, config=config) + return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config) def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) - return _build_base_url_for_host(get_public_host(config=config), request=request, config=config) + if not config.USES_SUBDOMAIN_ROUTING: + return get_base_url(request=request, config=config) + return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config) def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{snapshot_id}") - return _build_base_url_for_host(get_snapshot_host(snapshot_id, config=config), request=request, config=config) + return _build_base_url_for_host( + _build_base_host(get_snapshot_subdomain(snapshot_id), request=request, config=config), + request=request, + config=config, + ) def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) if not config.USES_SUBDOMAIN_ROUTING: return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}") - return _build_base_url_for_host(get_original_host(domain, config=config), request=request, config=config) + return _build_base_url_for_host(_build_base_host(domain, request=request, config=config), request=request, config=config) def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: diff --git a/archivebox/core/migrations/0023_upgrade_to_0_9_0.py b/archivebox/core/migrations/0023_upgrade_to_0_9_0.py index 4203c49c..d823625b 100644 --- a/archivebox/core/migrations/0023_upgrade_to_0_9_0.py +++ b/archivebox/core/migrations/0023_upgrade_to_0_9_0.py @@ -301,7 +301,20 @@ def upgrade_core_tables(apps, schema_editor): COALESCE(added, CURRENT_TIMESTAMP) as created_at, COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at, updated as downloaded_at, - 'queued' as status + CASE + WHEN EXISTS ( + SELECT 1 FROM core_archiveresult + WHERE core_archiveresult.snapshot_id = core_snapshot.id + AND core_archiveresult.status IN ('queued', 'started', 'backoff') + ) + THEN 'queued' + WHEN EXISTS ( + SELECT 1 FROM core_archiveresult + WHERE core_archiveresult.snapshot_id = core_snapshot.id + ) + THEN 'sealed' + ELSE 'queued' + END as status FROM core_snapshot; """) elif has_bookmarked_at and not has_added: @@ -320,7 +333,25 @@ def upgrade_core_tables(apps, schema_editor): select_cols.append("REPLACE(crawl_id, '-', '')") if has_status: insert_cols.append("status") - select_cols.append("status") + select_cols.append( + """ + CASE + WHEN status IN ('sealed', 'started', 'paused') THEN status + WHEN EXISTS ( + SELECT 1 FROM core_archiveresult + WHERE core_archiveresult.snapshot_id = core_snapshot.id + AND core_archiveresult.status IN ('queued', 'started', 'backoff') + ) + THEN status + WHEN EXISTS ( + SELECT 1 FROM core_archiveresult + WHERE core_archiveresult.snapshot_id = core_snapshot.id + ) + THEN 'sealed' + ELSE status + END + """, + ) if has_retry_at: insert_cols.append("retry_at") select_cols.append("retry_at") diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 70e3c3e1..8aecbb26 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -360,8 +360,8 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"} SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_") -ALLOWED_HOSTS = CONFIG.ALLOWED_HOSTS.split(",") -CSRF_TRUSTED_ORIGINS = list(set(CONFIG.CSRF_TRUSTED_ORIGINS.split(","))) +ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()] +CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()}) admin_base_url = normalize_base_url(get_admin_base_url()) if admin_base_url and admin_base_url not in CSRF_TRUSTED_ORIGINS: diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 2ff05f28..5ab1de79 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -191,8 +191,7 @@ class SnapshotView(View): "PREVIEW_ORIGINALS", "LISTEN_HOST", "USES_SUBDOMAIN_ROUTING", - "ADMIN_BASE_URL", - "ARCHIVE_BASE_URL", + "BASE_URL", "PERMISSIONS", "SERVER_SECURITY_MODE", } diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 779c31a8..bd2f2a2c 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -1285,6 +1285,8 @@ def run_due_snapshot(snapshot, *, lock_seconds: int) -> bool: if snapshot.is_paused: selected_plugins = queued_plugins_for_snapshot(str(snapshot.id)) if not selected_plugins: + if snapshot.fs_migration_needed and Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds): + run_snapshot_maintenance(str(snapshot.id)) # Paused is a real lifecycle state; retry_at=MAX is only the # orchestrator selection marker. If a direct maintenance/update # command bumps retry_at on a paused snapshot but there are no @@ -1465,7 +1467,7 @@ def run_install(*, plugin_names: list[str] | None = None) -> None: asyncio.run(_run_install(plugin_names=plugin_names)) -def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> int: +def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int: from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.core.models import ArchiveResult, Snapshot from archivebox.machine.models import Binary, Process @@ -1486,16 +1488,19 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> if schedule.is_due(now): schedule.enqueue(queued_at=now) - due_crawls = Crawl.objects.filter(retry_at__lte=timezone.now()) - if crawl_id: - due_crawls = due_crawls.filter(id=crawl_id) - due_crawl = due_crawls.order_by("retry_at", "created_at").first() - if due_crawl is not None: - if not run_due_crawl(due_crawl, lock_seconds=crawl_claim_lock_seconds): + if not maintenance_only: + due_crawls = Crawl.objects.filter(retry_at__lte=timezone.now()) + if crawl_id: + due_crawls = due_crawls.filter(id=crawl_id) + due_crawl = due_crawls.order_by("retry_at", "created_at").first() + if due_crawl is not None: + if not run_due_crawl(due_crawl, lock_seconds=crawl_claim_lock_seconds): + continue continue - continue due_snapshots = Snapshot.objects.filter(retry_at__lte=timezone.now()).select_related("crawl") + if maintenance_only: + due_snapshots = due_snapshots.filter(status__in=[Snapshot.StatusChoices.PAUSED, Snapshot.StatusChoices.SEALED]) if crawl_id: due_snapshots = due_snapshots.filter(crawl_id=crawl_id) due_snapshot = due_snapshots.order_by("retry_at", "created_at").first() @@ -1504,7 +1509,7 @@ def run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) -> continue continue - if crawl_id is None: + if crawl_id is None and not maintenance_only: due_binary = ( Binary.objects.filter(retry_at__lte=timezone.now()) .exclude(status=Binary.StatusChoices.INSTALLED) diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index ff3b3146..f409a071 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -419,9 +419,9 @@ def build_test_env(port: int, **extra: str) -> dict[str, str]: env.update( { "PLUGINS": "wget", - "LISTEN_HOST": f"archivebox.localhost:{port}", + "LISTEN_HOST": f"127.0.0.1:{port}", + "BASE_URL": f"http://archivebox.localhost:{port}", "ALLOWED_HOSTS": "*", - "CSRF_TRUSTED_ORIGINS": f"http://admin.archivebox.localhost:{port}", "PUBLIC_ADD_VIEW": "True", "USE_COLOR": "False", "SHOW_PROGRESS": "False", @@ -830,7 +830,8 @@ def real_archive_with_example(tmp_path_factory, request): [ "config", "--set", - "LISTEN_HOST=archivebox.localhost:8000", + "LISTEN_HOST=127.0.0.1:8000", + "BASE_URL=http://archivebox.localhost:8000", "PUBLIC_INDEX=True", "PUBLIC_ADD_VIEW=True", ], diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index 7b785aa0..9aee4dfc 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -278,10 +278,19 @@ class TestRunEmpty: class TestRunDaemonMode: - def test_run_daemon_ignores_piped_stdin_and_starts_real_runner(self, initialized_archive, db): + @pytest.mark.parametrize("stdin_kind", ["malformed", "valid-snapshot"]) + def test_run_daemon_ignores_piped_stdin_and_starts_real_runner(self, initialized_archive, db, stdin_kind): from archivebox.machine.models import Process + from archivebox.core.models import Snapshot from archivebox.tests.test_orm_helpers import use_archivebox_db + snapshot_url = None + if stdin_kind == "valid-snapshot": + snapshot_url = create_test_url() + piped_stdin = json.dumps(create_test_snapshot_json(url=snapshot_url)) + "\n" + else: + piped_stdin = "{this is not jsonl}\n" + env = os.environ.copy() env.update( { @@ -306,7 +315,7 @@ class TestRunDaemonMode: assert proc.stderr is not None try: - proc.stdin.write("{this is not jsonl}\n") + proc.stdin.write(piped_stdin) proc.stdin.close() deadline = time.monotonic() + 20 @@ -327,6 +336,9 @@ class TestRunDaemonMode: time.sleep(0.25) assert started is True + if snapshot_url is not None: + with use_archivebox_db(initialized_archive): + assert not Snapshot.objects.filter(url=snapshot_url).exists() finally: if proc.poll() is None: os.killpg(proc.pid, signal.SIGTERM) @@ -725,6 +737,28 @@ class TestRecoverOrchestratorState: @pytest.mark.django_db class TestRunDueCrawlState: + def test_maintenance_only_runner_does_not_start_regular_queued_crawls(self): + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.services.runner import run_pending_crawls + + now = timezone.now() + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.QUEUED, + retry_at=now, + ) + + assert run_pending_crawls(daemon=False, maintenance_only=True) == 0 + + crawl.refresh_from_db() + assert crawl.status == Crawl.StatusChoices.QUEUED + assert crawl.retry_at == now + assert crawl.snapshot_set.count() == 0 + def test_snapshot_start_writes_short_future_lease(self): from django.utils import timezone diff --git a/archivebox/tests/test_server_security_browser.py b/archivebox/tests/test_server_security_browser.py index ee9a375e..1b509d2e 100644 --- a/archivebox/tests/test_server_security_browser.py +++ b/archivebox/tests/test_server_security_browser.py @@ -345,9 +345,9 @@ def _start_server(data_dir: Path, *, mode: str, port: int) -> subprocess.Popen[s env.update( { "PYTHONPATH": str(Path(__file__).resolve().parents[2]), - "LISTEN_HOST": f"archivebox.localhost:{port}", + "LISTEN_HOST": f"127.0.0.1:{port}", + "BASE_URL": f"http://archivebox.localhost:{port}", "ALLOWED_HOSTS": "*", - "CSRF_TRUSTED_ORIGINS": f"http://archivebox.localhost:{port},http://admin.archivebox.localhost:{port}", "SERVER_SECURITY_MODE": mode, "USE_COLOR": "False", "SHOW_PROGRESS": "False", diff --git a/archivebox/tests/test_urls.py b/archivebox/tests/test_urls.py index e8c0b602..0e718c36 100644 --- a/archivebox/tests/test_urls.py +++ b/archivebox/tests/test_urls.py @@ -54,6 +54,7 @@ def _build_script(body: str) -> str: from archivebox.core.host_utils import ( get_admin_host, get_admin_base_url, + get_base_host, get_api_host, get_web_host, get_web_base_url, @@ -201,8 +202,12 @@ class TestUrlRouting: snapshot_subdomain = get_snapshot_subdomain(snapshot_id) snapshot_host = get_snapshot_host(snapshot_id) original_host = get_original_host(domain) - base_host = SERVER_CONFIG.LISTEN_HOST + listen_host = SERVER_CONFIG.LISTEN_HOST + base_host = get_base_host() + listen_host_only, listen_port = split_host_port(listen_host) + assert listen_host_only == "127.0.0.1" + assert listen_port == "8000" host_only, port = split_host_port(base_host) assert host_only == "archivebox.localhost" assert port == "8000" @@ -262,7 +267,7 @@ class TestUrlRouting: """, mode="safe-subdomains-fullreplay", env_overrides={ - "LISTEN_HOST": "archivebox.io", + "BASE_URL": "https://archivebox.io", }, ) @@ -330,7 +335,7 @@ class TestUrlRouting: snapshot_host = get_snapshot_host(snapshot_id) original_host = get_original_host(snapshot.domain) web_host = get_web_host() - host_only, port = split_host_port(SERVER_CONFIG.LISTEN_HOST) + host_only, port = split_host_port(get_base_host()) legacy_snapshot_host = f"{snapshot_id}.{host_only}" if port: legacy_snapshot_host = f"{legacy_snapshot_host}:{port}" @@ -574,7 +579,7 @@ class TestUrlRouting: snapshot_id = str(snapshot.id) client = Client() - base_host = SERVER_CONFIG.LISTEN_HOST + base_host = get_base_host() web_host = get_web_host() admin_host = get_admin_host() api_host = get_api_host() @@ -635,7 +640,7 @@ class TestUrlRouting: snapshot_id = str(snapshot.id) client = Client() - base_host = SERVER_CONFIG.LISTEN_HOST + base_host = get_base_host() assert SERVER_CONFIG.SERVER_SECURITY_MODE == "unsafe-onedomain-noadmin" assert SERVER_CONFIG.CONTROL_PLANE_ENABLED is False @@ -670,7 +675,7 @@ class TestUrlRouting: snapshot_id = str(snapshot.id) client = Client() - base_host = SERVER_CONFIG.LISTEN_HOST + base_host = get_base_host() assert SERVER_CONFIG.SERVER_SECURITY_MODE == "danger-onedomain-fullreplay" assert SERVER_CONFIG.CONTROL_PLANE_ENABLED is True @@ -710,15 +715,15 @@ class TestUrlRouting: """ snapshot = get_snapshot() snapshot_id = str(snapshot.id) - base_host = SERVER_CONFIG.LISTEN_HOST + base_host = get_base_host() assert SERVER_CONFIG.SERVER_SECURITY_MODE == "safe-onedomain-nojsreplay" assert get_admin_host() == base_host assert get_web_host() == base_host - assert get_admin_base_url() == "https://admin.archivebox.example" + assert get_admin_base_url() == "https://archivebox.example" assert get_web_base_url() == "https://archivebox.example" - assert build_admin_url("/admin/login/") == "https://admin.archivebox.example/admin/login/" + assert build_admin_url("/admin/login/") == "https://archivebox.example/admin/login/" assert build_snapshot_url(snapshot_id, "index.jsonl") == ( f"https://archivebox.example/snapshot/{snapshot_id}/index.jsonl" ) @@ -727,8 +732,7 @@ class TestUrlRouting: """, mode="safe-onedomain-nojsreplay", env_overrides={ - "ADMIN_BASE_URL": "https://admin.archivebox.example", - "ARCHIVE_BASE_URL": "https://archivebox.example", + "BASE_URL": "https://archivebox.example", }, ) @@ -748,7 +752,7 @@ class TestUrlRouting: """, mode="safe-subdomains-fullreplay", env_overrides={ - "ARCHIVE_BASE_URL": "https://web.archivebox.example", + "BASE_URL": "https://archivebox.example", }, ) @@ -769,7 +773,7 @@ class TestUrlRouting: resp = client.get("/public/", HTTP_HOST=web_host) assert resp.status_code == 200 public_html = response_body(resp).decode("utf-8", "ignore") - assert "http://web.archivebox.localhost:8000" in public_html + assert f"http://{snapshot_host}/" in public_html ensure_admin_user() assert client.login(username="testadmin", password="testpassword") @@ -839,7 +843,7 @@ class TestUrlRouting: resp = client.get(f"/admin/core/snapshot/{snapshot_id}/change/", HTTP_HOST=admin_host) assert resp.status_code == 200 admin_html = response_body(resp).decode("utf-8", "ignore") - assert f"http://web.archivebox.localhost:8000/{snapshot.archive_path}" in admin_html + assert f"http://{web_host}/{snapshot.archive_path}" in admin_html assert f"http://{snapshot_host}/" in admin_html result = ArchiveResult.objects.filter(snapshot=snapshot).first() @@ -866,6 +870,7 @@ class TestUrlRouting: '{"level":"warn","text":"second line"}\\n', encoding="utf-8", ) + snapshot.write_html_details() client = Client() resp = client.get(f"/{snapshot.url_path}/index.html", HTTP_HOST=web_host) diff --git a/docker-compose.yml b/docker-compose.yml index 5fcdc688..85c58b34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,9 +20,9 @@ services: environment: # - ADMIN_USERNAME=admin # creates an admin user on first run with the given user/pass combo # - ADMIN_PASSWORD=SomeSecretPassword - - LISTEN_HOST=archivebox.localhost:8000 + - LISTEN_HOST=0.0.0.0:8000 + - BASE_URL=http://archivebox.localhost:8000 # public URL used to build admin/web/api/snapshot links - ALLOWED_HOSTS=* # set this to the hostname(s) you're going to serve the site from! - - CSRF_TRUSTED_ORIGINS=http://admin.archivebox.localhost:8000 # MUST match the admin UI URL for login/API to work - PUBLIC_INDEX=True # set to False to prevent anonymous users from viewing snapshot list - PUBLIC_SNAPSHOTS=True # set to False to prevent anonymous users from viewing snapshot content - PUBLIC_ADD_VIEW=False # set to True to allow anonymous users to submit new URLs to archive