From 1eadff04f1ad58edecb05ca0679206bfd3f13463 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 8 Jun 2026 04:45:13 -0700 Subject: [PATCH] root dir cleanup --- Dockerfile.multistage | 225 -------------------------------- archivebox/services/runner.py | 10 +- archivebox/tests/conftest.py | 21 ++- archivebox/tests/test_search.py | 18 +++ conftest.py | 34 ----- 5 files changed, 45 insertions(+), 263 deletions(-) delete mode 100644 Dockerfile.multistage delete mode 100644 conftest.py diff --git a/Dockerfile.multistage b/Dockerfile.multistage deleted file mode 100644 index 22e1cc35..00000000 --- a/Dockerfile.multistage +++ /dev/null @@ -1,225 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image. -# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes. -# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime. -# Build abx-dl first, then point this file at it: -# docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \ -# --build-context abxbus=../abxbus \ -# --build-context abxpkg=../abxpkg \ -# --build-context abx-plugins=../abx-plugins \ -# -t archivebox/abx-dl:dev -# docker buildx build . -f Dockerfile \ -# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \ -# -t archivebox:multistage - -ARG ABX_DL_IMAGE=archivebox/abx-dl:latest - -FROM archivebox/sonic:1.4.9 AS sonic -FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base - -ARG TARGETPLATFORM -ARG TARGETOS -ARG TARGETARCH -ARG TARGETVARIANT - -ENV TZ=UTC \ - LANGUAGE=en_US:en \ - LC_ALL=C.UTF-8 \ - LANG=C.UTF-8 \ - DEBIAN_FRONTEND=noninteractive \ - APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \ - PYTHONIOENCODING=UTF-8 \ - PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PIP_NO_COMPILE=1 \ - PIP_ONLY_BINARY=aiohttp \ - npm_config_loglevel=error - -ENV PYTHON_VERSION=3.13 \ - NODE_VERSION=22.22.3 - -ENV ARCHIVEBOX_USER=archivebox \ - DEFAULT_PUID=911 \ - DEFAULT_PGID=911 \ - IN_DOCKER=True - -ENV CODE_DIR=/app \ - DATA_DIR=/data \ - CONFIG_DIR=/home/archivebox/.config/abx \ - LIB_DIR=/home/archivebox/.config/abx/lib \ - ABXPKG_LIB_DIR=/home/archivebox/.config/abx/lib \ - PLAYWRIGHT_BROWSERS_PATH=/home/archivebox/.config/abx/lib/playwright/cache \ - PERSONAS_DIR=/data/personas \ - CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ - CHROME_HEADLESS=true \ - CHROME_SANDBOX=false \ - CHROME_ISOLATION=crawl \ - CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]' - -ENV TMP_DIR=/tmp/archivebox \ - PIP_VENV_PYTHON=/venv/bin/python3 \ - GOOGLE_API_KEY=no \ - GOOGLE_DEFAULT_CLIENT_ID=no \ - GOOGLE_DEFAULT_CLIENT_SECRET=no - -ENV HOME=/home/archivebox \ - XDG_CONFIG_HOME=/home/archivebox/.config \ - XDG_CACHE_HOME=/home/archivebox/.cache \ - ABXPKG_INSTALL_TIMEOUT=600 \ - ABXPKG_POSTINSTALL_SCRIPTS=True \ - ABXPKG_MIN_RELEASE_AGE=0 \ - TIMEOUT=600 - -ENV UV_COMPILE_BYTECODE=false \ - UV_PYTHON_PREFERENCE=managed \ - UV_PYTHON_INSTALL_DIR=/opt/uv/python \ - ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \ - UV_LINK_MODE=copy \ - UV_PROJECT_ENVIRONMENT=/venv \ - VIRTUAL_ENV=/venv \ - PATH="/venv/bin:/opt/node/bin:$PATH" - -SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] -WORKDIR "$CODE_DIR" - -RUN cp /VERSION.txt /ABX-DL-VERSION.txt \ - && (echo "[i] Docker build for ArchiveBox multistage starting..." \ - && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ - && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \ - && uname -a \ - && sed -n '1,7p' /etc/os-release \ - && which node && node --version \ - && which uv && uv self version \ - ) | tee -a /VERSION.txt - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends ripgrep \ - && rm -rf /var/lib/apt/lists/* - -FROM archivebox-runtime-base AS archivebox-builder - -WORKDIR "$CODE_DIR" -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \ - echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \ - && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \ - && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \ - && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \ - && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \ - && rm -f /etc/apt/apt.conf.d/docker-clean \ - && apt-get update -qq \ - && apt-get install -qq -y --no-install-recommends \ - build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ - && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \ - && /usr/bin/uv pip install setuptools pip wheel \ - && /usr/bin/uv sync \ - --refresh \ - --no-dev \ - --inexact \ - --all-extras \ - --no-install-project \ - --no-install-workspace \ - --no-sources \ - && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* - -COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" -RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \ - && COMMIT_HASH="$( \ - if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \ - HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \ - if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \ - echo "$HEAD_REF"; \ - elif [[ "$HEAD_REF" == ref:\ * ]]; then \ - REF_PATH="${HEAD_REF#ref: }"; \ - cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \ - fi; \ - fi)" \ - && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \ - && /usr/bin/uv pip install --no-deps "$CODE_DIR" \ - && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \ - && rm -rf "$CODE_DIR/.git" - -FROM archivebox-runtime-base - -LABEL name="archivebox" \ - maintainer="Nick Sweeting " \ - description="All-in-one self-hosted internet archiving solution" \ - homepage="https://github.com/ArchiveBox/ArchiveBox" \ - documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \ - org.opencontainers.image.title="ArchiveBox" \ - org.opencontainers.image.vendor="ArchiveBox" \ - org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \ - org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \ - com.docker.image.source.entrypoint="Dockerfile" - -COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic -COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg - -COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python -COPY --from=archivebox-builder /venv /venv -COPY --from=archivebox-builder /app /app -COPY --from=archivebox-builder /VERSION.txt /VERSION.txt - -RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \ - && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \ - && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \ - && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \ - && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \ - && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \ - && (which sonic && sonic --version) | tee -a /VERSION.txt \ - && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \ - && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \ - && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && openssl rand -hex 16 > /etc/machine-id \ - && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \ - && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt - -WORKDIR "$DATA_DIR" -RUN echo "[+] Initializing image collection..." \ - && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \ - && PUID=0 PGID=0 archivebox init \ - && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \ - "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \ - "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \ - "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \ - "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \ - "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - 2>/dev/null || true) - -RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" - -RUN "$LIB_DIR/playwright/bin/chromium" --version | tee -a /VERSION.txt \ - && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \ - && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \ - && /usr/local/bin/sonic --version | tee -a /VERSION.txt \ - && /venv/bin/supervisord --version | tee -a /VERSION.txt \ - && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \ - && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \ - && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \ - && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \ - && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/* - -RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \ - && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \ - && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \ - ) | tee -a /VERSION.txt - -WORKDIR "$DATA_DIR" -VOLUME "$DATA_DIR" -EXPOSE 8000 - -HEALTHCHECK --interval=30s --timeout=20s --retries=15 \ - CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK' - -ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"] -CMD ["archivebox", "server", "--init", "0.0.0.0:8000"] diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 9a9a51a5..3ad3a679 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -726,14 +726,20 @@ class CrawlRunner: }, ) snapshot_output_dir = str(snapshot.output_dir) + tags = snapshot.tags_str() config = self.base_config.for_crawl_runtime( crawl=snapshot.crawl, snapshot=snapshot, persona=self.persona, runtime_overrides=runtime_chrome_overrides, - extra_context={"snapshot_id": str(snapshot.id), "snapshot_depth": snapshot.depth}, + extra_context={ + "snapshot_id": str(snapshot.id), + "snapshot_depth": snapshot.depth, + "snapshot_url": snapshot.url, + "snapshot_title": snapshot.title or "", + "snapshot_tags": tags, + }, ) - tags = snapshot.tags_str() normalized_config = normalize_runtime_config(config) return { "id": str(snapshot.id), diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index 8164592e..599f6814 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -23,15 +23,23 @@ from collections.abc import Callable import psutil import pytest import requests -from django.utils import timezone REPO_ROOT = Path(__file__).resolve().parents[2] WORKSPACE_ROOT = REPO_ROOT.parent PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve() -SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve() +SESSION_DATA_DIR = Path( + os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"), +).resolve() + +os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR) +os.environ["DATA_DIR"] = str(SESSION_DATA_DIR) (SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True) os.chdir(SESSION_DATA_DIR) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings") +os.environ.pop("ARCHIVE_DIR", None) +os.environ.pop("USERS_DIR", None) +os.environ.pop("CRAWL_DIR", None) +os.environ.pop("SNAP_DIR", None) def _is_repo_path(path: Path) -> bool: @@ -286,6 +294,14 @@ def find_snapshot_dir(data_dir: Path, snapshot_id: str) -> Path | None: # ============================================================================= +def pytest_configure(): + import django + from django.apps import apps + + if not apps.ready: + django.setup() + + @pytest.fixture(autouse=True) def isolate_test_runtime(tmp_path, monkeypatch): """ @@ -1145,6 +1161,7 @@ def wait_for_http( def make_latest_schedule_due(cwd: Path) -> None: from archivebox.crawls.models import Crawl, CrawlSchedule from archivebox.tests.test_orm_helpers import use_archivebox_db + from django.utils import timezone with use_archivebox_db(cwd): schedule = CrawlSchedule.objects.order_by("-created_at").select_related("template").first() diff --git a/archivebox/tests/test_search.py b/archivebox/tests/test_search.py index df5ce043..22d4f556 100644 --- a/archivebox/tests/test_search.py +++ b/archivebox/tests/test_search.py @@ -938,6 +938,21 @@ class TestSearchBackendsE2E: backend_urls = [line.strip().strip('"') for line in backend_result.stdout.splitlines() if line.strip()] assert set(backend_urls) == set(expected_urls), (backend_name, query, backend_result.stdout) + for query, expected_urls in ( + (url_only_needle, [url_only_url]), + (title_only_needle, [title_only_url]), + (tag_only_needle, [tag_only_url]), + ): + sonic_metadata_result = run_archivebox_cmd( + ["list", "--search=contents", "--csv=url", query], + cwd=initialized_archive, + env={**env, "SEARCH_BACKEND_ENGINE": "sonic"}, + timeout=60, + ) + assert sonic_metadata_result.returncode == 0, sonic_metadata_result.stderr or sonic_metadata_result.stdout + sonic_metadata_urls = [line.strip().strip('"') for line in sonic_metadata_result.stdout.splitlines() if line.strip()] + assert set(sonic_metadata_urls) == set(expected_urls), (query, sonic_metadata_result.stdout) + session = requests.Session() login_page = session.get( f"http://127.0.0.1:{archivebox_port}/admin/login/", @@ -1015,6 +1030,9 @@ class TestSearchBackendsE2E: ("deep:sonic", second_batch_content_needle, second_mercury_urls), ("deep:sonic", shared_content_needle, matrix_urls), ("deep:sonic", overlapping_content_needle, [*first_mercury_urls[:3], *second_mercury_urls[:2]]), + ("deep:sonic", url_only_needle, [url_only_url]), + ("deep:sonic", title_only_needle, [title_only_url]), + ("deep:sonic", tag_only_needle, [tag_only_url]), ): params = {"q": query, "search_mode": search_mode} search_url = f"{list_path}?{urlencode(params)}" diff --git a/conftest.py b/conftest.py deleted file mode 100644 index a1038570..00000000 --- a/conftest.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Root pytest bootstrap. - -This file is intentionally outside the ``archivebox`` package so pytest can -load it before importing ``archivebox/__init__.py``. ArchiveBox constants are -computed at import time, so DATA_DIR must already point at a temp collection. -""" - -import os -import tempfile -from pathlib import Path - - -REPO_ROOT = Path(__file__).resolve().parent -SESSION_DATA_DIR = Path( - os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"), -).resolve() - -os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR) -os.environ["DATA_DIR"] = str(SESSION_DATA_DIR) -(SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True) -os.chdir(SESSION_DATA_DIR) -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings") -os.environ.pop("ARCHIVE_DIR", None) -os.environ.pop("USERS_DIR", None) -os.environ.pop("CRAWL_DIR", None) -os.environ.pop("SNAP_DIR", None) - - -def pytest_configure(): - import django - from django.apps import apps - - if not apps.ready: - django.setup()