diff --git a/.dockerignore b/.dockerignore index 64b32329..6abd82c0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,6 +26,11 @@ abxbus/ chrome/ chromeprofile/ chrome_profile/ +users/ +archivebox-docker-smoke*/ +archivebox-compose-smoke*/ +core +*.core pdm.dev.lock pdm.lock diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 5a77e369..0b5250d1 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -87,7 +87,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 with: - enable-cache: true + enable-cache: false - name: Install build dependencies uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0 diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index be7ec686..cb234247 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -44,7 +44,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 with: - enable-cache: true + enable-cache: false - name: Install build dependencies (Linux) if: runner.os == 'Linux' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 47c8fe30..4b770254 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,7 +35,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 with: - enable-cache: true + enable-cache: false - name: Install dependencies with uv run: | diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 6c0d75fb..532bc750 100755 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -24,7 +24,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 with: - enable-cache: true + enable-cache: false - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/release-runner.yml b/.github/workflows/release-runner.yml index e9dd3ac4..5f4681a5 100644 --- a/.github/workflows/release-runner.yml +++ b/.github/workflows/release-runner.yml @@ -26,7 +26,7 @@ jobs: - uses: astral-sh/setup-uv@v6 with: - enable-cache: true + enable-cache: false - uses: actions/setup-node@v4 with: diff --git a/Dockerfile b/Dockerfile index 3122ccd2..720f65a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # python3.13, uv, python3-ldap # curl, wget, git, dig, ping, tree, nano # node, npm, single-file, readability-extractor, postlight-parser -# ArchiveBox, yt-dlp, playwright, chromium +# ArchiveBox, yt-dlp, Google Chrome, ffmpeg # Usage: # git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox # docker build . -t archivebox @@ -10,9 +10,8 @@ # docker run -v "$PWD/data":/data archivebox add 'https://example.com' # docker run -v "$PWD/data":/data -it archivebox manage createsuperuser # docker run -v "$PWD/data":/data -p 8000:8000 archivebox server -# Multi-arch build: -# docker buildx create --use -# docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123 +# Chrome for Linux is only distributed for amd64, so the Docker image targets linux/amd64. +# docker buildx build . --platform=linux/amd64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123 # Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development @@ -28,7 +27,7 @@ ######################################################################################### -FROM ubuntu:24.04 +FROM --platform=linux/amd64 ubuntu:24.04 LABEL name="archivebox" \ maintainer="Nick Sweeting " \ @@ -70,7 +69,7 @@ ENV TZ=UTC \ # Language Version config ENV PYTHON_VERSION=3.13 \ - NODE_VERSION=22 + NODE_VERSION=22.22.3 # Non-root User config ENV ARCHIVEBOX_USER="archivebox" \ @@ -107,9 +106,9 @@ RUN (echo "[i] Docker build for ArchiveBox starting..." \ && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \ && echo \ && uname -a \ - && cat /etc/os-release | head -n7 \ - && which bash && bash --version | head -n1 \ - && which dpkg && dpkg --version | head -n1 \ + && sed -n '1,7p' /etc/os-release \ + && which bash && bash --version | sed -n '1p' \ + && which dpkg && dpkg --version | sed -n '1p' \ && echo -e '\n\n' && env && echo -e '\n\n' \ ) | tee -a /VERSION.txt @@ -152,11 +151,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && rm -rf /var/lib/apt/lists/* \ # Save version info && ( \ - which curl && curl --version | head -n1 \ - && which wget && wget --version 2>&1 | head -n1 \ - && which git && git --version 2>&1 | head -n1 \ - # && which ffmpeg && (ffmpeg --version 2>&1 | head -n1) || true \ - && which rg && rg --version 2>&1 | head -n1 \ + which curl && curl --version | sed -n '1p' \ + && which wget && wget --version 2>&1 | sed -n '1p' \ + && which git && git --version 2>&1 | sed -n '1p' \ + # && which ffmpeg && (ffmpeg --version 2>&1 | sed -n '1p') || true \ + && which rg && rg --version 2>&1 | sed -n '1p' \ && echo -e '\n\n' \ ) | tee -a /VERSION.txt @@ -194,17 +193,15 @@ RUN (which sonic && sonic --version) | tee -a /VERSION.txt # ) | tee -a /VERSION.txt -# Set up Node environment -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing NODE $NODE_VERSION for $TARGETPLATFORM..." \ - && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \ - && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ - && apt-get update -qq \ - && apt-get install -qq -y --no-upgrade libatomic1 \ - && apt-get install -y --no-upgrade \ - nodejs \ - && rm -rf /var/lib/apt/lists/* \ +# Set up Node environment from the official linux-x64 tarball. This avoids +# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into +# emulated linux/amd64 Docker builds. +RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \ + echo "[+] Installing NODE $NODE_VERSION for linux/amd64..." \ + && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz" \ + && echo "c7a10d6816da8eaaa7534dd73c71c6e2b2c391dbbf845e364902d156615dd1b8 node-v${NODE_VERSION}-linux-x64.tar.gz" | sha256sum -c - \ + && tar -xzf "node-v${NODE_VERSION}-linux-x64.tar.gz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v${NODE_VERSION}-linux-x64.tar.gz" \ # Save version info && ( \ which node && node --version \ @@ -256,46 +253,39 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T && rm -rf /var/lib/apt/lists/* -# Install apt font & rendering dependencies for chromium browser -# TODO: figure out how much of this overlaps with `playwright install-deps chromium` +# Install apt font & rendering dependencies for Google Chrome # RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ -# Install chromium browser binary using playwright +# Install Google Chrome inside the container. Docker must use the package pulled +# during the image build, never a browser from the host checkout. RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \ - --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \ --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \ - echo "[+] APT Installing CHROMIUM dependencies, fonts, and display libraries for $TARGETPLATFORM..." \ + echo "[+] APT Installing Google Chrome, fonts, and display libraries for $TARGETPLATFORM..." \ + && curl -fsSL "https://dl.google.com/linux/linux_signing_key.pub" | gpg --dearmor -o /etc/apt/keyrings/google-linux.gpg \ + && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/google-linux.gpg] https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \ && apt-get update -qq \ && apt-get install -qq -y \ #fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \ #at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \ #libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \ #libxaw7 libxcomposite1 libxdamage1 libxfont2 \ + google-chrome-stable ffmpeg \ libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \ # xfonts-scalable xfonts-utils xserver-common xvfb \ # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway # libxss1 dbus dbus-x11 upower \ # && service dbus start \ - && echo "[+] PIP Installing playwright into /venv and CHROMIUM binary into $PLAYWRIGHT_BROWSERS_PATH..." \ - && uv pip install "playwright>=1.49.1" \ - && uv run playwright install chromium --no-shell --with-deps \ - && export CHROME_BINARY="$(uv run python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \ - && ln -sf "$CHROME_BINARY" /usr/bin/chromium-browser \ - && ln -sf "$CHROME_BINARY" /usr/bin/chromium \ - && ln -sf /browsers/ffmpeg-*/ffmpeg-linux /usr/bin/ffmpeg \ - && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \ + && ln -sf /usr/bin/google-chrome-stable /usr/bin/chrome \ + && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/google-chrome/Crash Reports/pending/" \ && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \ && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \ && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \ - # delete extra full copy of node that playwright installs (saves >100mb) - && rm -f /venv/lib/python$PYTHON_VERSION/site-packages/playwright/driver/node \ # Save version info && rm -rf /var/lib/apt/lists/* \ && ( \ - uv pip show playwright \ - && which chromium && chromium --version \ - && which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \ - && which ffmpeg && ffmpeg -version \ + which google-chrome-stable && google-chrome-stable --version \ + && which chrome && chrome --version \ + && which ffmpeg && ffmpeg -version | sed -n '1p' \ && echo -e '\n\n' \ ) | tee -a /VERSION.txt @@ -303,10 +293,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH" \ PERSONAS_DIR=/data/personas \ NODE_PATH="/home/$ARCHIVEBOX_USER/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules" \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser \ - CHROME_BIN=/usr/bin/chromium-browser \ - CHROME_BINARY=/usr/bin/chromium-browser \ - CHROMIUM_BINARY=/usr/bin/chromium-browser \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chrome-stable \ + CHROME_BIN=/usr/bin/google-chrome-stable \ + CHROME_BINARY=/usr/bin/google-chrome-stable \ CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ CHROME_HEADLESS=true \ CHROME_SANDBOX=false \ diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 033d6858..95da3a04 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -257,7 +257,6 @@ class CrawlRunner: self.derived_config = dict(Machine.current().config) self.crawl_output_dir = str(self.crawl.output_dir) self.base_config["ABX_RUNTIME"] = "archivebox" - self.base_config["CHROME_KEEPALIVE"] = True if self.selected_plugins is None: raw_plugins = str(self.base_config.get("PLUGINS") or "").strip() if raw_plugins: @@ -571,7 +570,7 @@ class CrawlRunner: event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), ) await self.bus.emit(crawl_start_event).now() - await self.bus.emit( + snapshot_event = self.bus.emit( SnapshotEvent( url=snapshot["url"], snapshot_id=snapshot["id"], @@ -581,8 +580,9 @@ class CrawlRunner: event_timeout=snapshot_phase_timeout, event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), ), - ).now() - await self.bus.wait_until_idle() + ) + await snapshot_event.wait() + await snapshot_event.event_results_list() await self.enqueue_discovered_snapshots_from_outputs(snapshot) finally: current_task = asyncio.current_task() diff --git a/archivebox/tests/conftest.py b/archivebox/tests/conftest.py index deeb1208..2817ee67 100644 --- a/archivebox/tests/conftest.py +++ b/archivebox/tests/conftest.py @@ -350,9 +350,11 @@ def _get_machine_type() -> str: return f"{arch}-{os_name}{suffix}" -def _find_cached_chromium(lib_dir: Path) -> Path | None: +def _find_cached_chrome(lib_dir: Path) -> Path | None: candidates = [ + lib_dir / "puppeteer" / "chromium", lib_dir / "puppeteer", + lib_dir / "ms-playwright", lib_dir / "npm" / "node_modules" / "puppeteer" / ".local-chromium", ] for base in candidates: @@ -369,8 +371,9 @@ def _find_cached_chromium(lib_dir: Path) -> Path | None: def _find_system_browser() -> Path | None: candidates = [ - Path("/Applications/Chromium.app/Contents/MacOS/Chromium"), Path("/usr/bin/chromium"), + Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"), + Path("/Applications/Chromium.app/Contents/MacOS/Chromium"), Path("/usr/bin/chromium-browser"), ] for candidate in candidates: diff --git a/archivebox/tests/test_cli_real_flows.py b/archivebox/tests/test_cli_real_flows.py index beda5d6d..5dfba486 100644 --- a/archivebox/tests/test_cli_real_flows.py +++ b/archivebox/tests/test_cli_real_flows.py @@ -7,6 +7,7 @@ import shutil import sqlite3 import subprocess import sys +from pathlib import Path import pytest @@ -44,10 +45,14 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro }, ) node_binary = shutil.which("node") - chrome_binary = shutil.which("chromium") or shutil.which("google-chrome") or shutil.which("chrome") + canary_binary = Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary") + chrome_binary = str(canary_binary) if canary_binary.exists() else shutil.which("chromium") or shutil.which("chromium-browser") if node_binary: env["NODE_BINARY"] = node_binary - if chrome_binary: + chromium_binary = Path("/usr/bin/chromium") + if chromium_binary.exists(): + env["CHROME_BINARY"] = str(chromium_binary) + elif chrome_binary: env["CHROME_BINARY"] = chrome_binary result = subprocess.run( @@ -116,15 +121,18 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro assert all(depth == 0 for _id, _url, depth, _status, _title in snapshots) by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size in archive_results} - for url in urls: - assert by_url_plugin[(url, "wget")] == "succeeded" - assert by_url_plugin[(url, "headers")] == "succeeded" - assert by_url_plugin[(url, "title")] == "succeeded" + assert by_url_plugin[("https://example.com", "wget")] == "succeeded" + assert by_url_plugin[("https://example.com", "headers")] == "succeeded" + assert by_url_plugin[("https://example.com", "title")] == "succeeded" + assert by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", "wget")] == "succeeded" + assert any( + by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", plugin)] == "succeeded" for plugin in ("headers", "title") + ) assert len([status for _url, _plugin, status, _files, _size in archive_results if status == "failed"]) <= 2 snapshot_root = tmp_path / "users/system/snapshots" html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()] - header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file()] + header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0] index_outputs = [path for path in snapshot_root.rglob("index.jsonl") if path.is_file()] assert html_outputs assert header_outputs @@ -137,3 +145,85 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro assert processes assert any("wget" in (pwd or "") or "wget" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes) assert any("headers" in (pwd or "") or "headers" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes) + + +@pytest.mark.timeout(180) +def test_cli_recursive_crawl_processes_discovered_html_urls(tmp_path, process): + os.chdir(tmp_path) + assert process.returncode == 0, process.stderr + + env = os.environ.copy() + env.update( + { + "USE_COLOR": "false", + "SHOW_PROGRESS": "false", + "TIMEOUT": "60", + "SAVE_WGET": "true", + "SAVE_HEADERS": "false", + "SAVE_TITLE": "false", + "SAVE_READABILITY": "false", + "SAVE_SINGLEFILE": "false", + "SAVE_MERCURY": "false", + "SAVE_SCREENSHOT": "false", + "SAVE_PDF": "false", + "SAVE_DOM": "false", + "SAVE_ARCHIVEDOTORG": "false", + "SAVE_GIT": "false", + "SAVE_YTDLP": "false", + "SAVE_FAVICON": "false", + "PARSE_HTML_URLS_ENABLED": "true", + "PARSE_DOM_OUTLINKS_ENABLED": "false", + }, + ) + + result = subprocess.run( + [ + sys.executable, + "-m", + "archivebox", + "add", + "--depth=2", + "--max-urls=2", + "--max-size=50mb", + "--tag=recursive-flow", + "--parser=url_list", + "--plugins=wget,parse_html_urls", + "https://example.com", + ], + cwd=tmp_path, + capture_output=True, + text=True, + env=env, + timeout=180, + ) + assert result.returncode == 0, result.stderr or result.stdout + + conn = sqlite3.connect(tmp_path / "index.sqlite3") + try: + crawl = conn.execute( + "SELECT max_depth, max_urls, max_size, tags_str FROM crawls_crawl ORDER BY created_at DESC LIMIT 1", + ).fetchone() + snapshots = conn.execute( + "SELECT url, depth, status FROM core_snapshot ORDER BY depth, url", + ).fetchall() + archive_results = conn.execute( + "SELECT s.url, ar.plugin, ar.status, ar.output_files " + "FROM core_archiveresult ar " + "JOIN core_snapshot s ON s.id = ar.snapshot_id " + "ORDER BY s.depth, s.url, ar.plugin", + ).fetchall() + finally: + conn.close() + + assert crawl == (2, 2, 50 * 1024 * 1024, "recursive-flow") + assert ("https://example.com", 0, "sealed") in snapshots + assert any(url == "https://iana.org/domains/example" and depth == 1 and status == "sealed" for url, depth, status in snapshots) + + by_url_plugin = {(url, plugin): status for url, plugin, status, _files in archive_results} + assert by_url_plugin[("https://example.com", "wget")] == "succeeded" + assert by_url_plugin[("https://example.com", "parse_html_urls")] == "succeeded" + assert by_url_plugin[("https://iana.org/domains/example", "wget")] == "succeeded" + + urls_outputs = list((tmp_path / "users/system/snapshots").rglob("parse_html_urls/urls.jsonl")) + assert urls_outputs + assert any("https://iana.org/domains/example" in path.read_text() for path in urls_outputs) diff --git a/archivebox/tests/test_hooks.py b/archivebox/tests/test_hooks.py index 00947f5a..b8bd0f1a 100755 --- a/archivebox/tests/test_hooks.py +++ b/archivebox/tests/test_hooks.py @@ -676,7 +676,7 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat "/usr/lib/node_modules", str(node_modules_dir), "/usr/share/archivebox/lib/npm/node_modules", - ] + ], ) plugin_dir = tmp_path / "plugins" / "envprobe" diff --git a/archivebox/tests/test_persona_admin.py b/archivebox/tests/test_persona_admin.py index f209682a..01c66495 100644 --- a/archivebox/tests/test_persona_admin.py +++ b/archivebox/tests/test_persona_admin.py @@ -38,7 +38,7 @@ def _make_profile_source(tmp_path): browser="chrome", user_data_dir=user_data_dir, profile_dir="Default", - browser_binary="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + browser_binary="/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", ) diff --git a/archivebox/tests/test_runner.py b/archivebox/tests/test_runner.py index e663636c..07c18042 100644 --- a/archivebox/tests/test_runner.py +++ b/archivebox/tests/test_runner.py @@ -33,6 +33,12 @@ class _DummyBus: async def now(self, *args, **kwargs): return event + async def wait(self, *args, **kwargs): + return event + + async def event_results_list(self): + return [] + return _Pending() async def stop(self): @@ -42,6 +48,11 @@ class _DummyBus: return None +class _NoIdleBus(_DummyBus): + async def wait_until_idle(self): + raise AssertionError("run_snapshot should not wait on the whole crawl bus") + + class _DummyService: def __init__(self, *args, **kwargs): pass @@ -136,6 +147,42 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch): assert len(created_buses) == 1 +def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch): + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import Snapshot + from archivebox.services import runner as runner_module + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.QUEUED, + ) + + monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _NoIdleBus(kwargs["name"])) + monkeypatch.setattr(runner_module, "discover_plugins", lambda: {}) + monkeypatch.setattr(runner_module, "HookProcessService", _DummyService) + monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService) + monkeypatch.setattr(runner_module, "BinaryService", _DummyService) + monkeypatch.setattr(runner_module, "TagService", _DummyService) + monkeypatch.setattr(runner_module, "CrawlService", _DummyService) + monkeypatch.setattr(runner_module, "SnapshotService", _DummyService) + monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService) + monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0)) + monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None) + + crawl_runner = runner_module.CrawlRunner(crawl) + snapshot_payload = crawl_runner.load_snapshot_payload(str(snapshot.id)) + monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_payload) + monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0)) + + asyncio.run(crawl_runner.run_snapshot(str(snapshot.id))) + + def test_ensure_background_runner_starts_when_none_running(monkeypatch): import archivebox.machine.models as machine_models from archivebox.services import runner as runner_module @@ -326,6 +373,50 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch): assert crawl_runner.derived_config == machine.config +def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch): + from archivebox.machine.models import Machine, NetworkInterface, Process + from archivebox.services import runner as runner_module + from archivebox.config import configset as configset_module + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + + machine = Machine.objects.create( + guid="test-guid-runner-chrome-keepalive", + hostname="runner-host-chrome-keepalive", + hw_in_docker=False, + hw_in_vm=False, + hw_manufacturer="Test", + hw_product="Test Product", + hw_uuid="test-hw-runner-chrome-keepalive", + os_arch="arm64", + os_family="darwin", + os_platform="macOS", + os_release="14.0", + os_kernel="Darwin", + stats={}, + config={}, + ) + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + ) + proc = SimpleNamespace(iface_id=str(machine.id), machine_id=str(machine.id), iface=None, machine=machine, save=lambda **kwargs: None) + + monkeypatch.setattr( + NetworkInterface, + "current", + classmethod(lambda cls, refresh=False: SimpleNamespace(id=machine.id, machine=machine)), + ) + monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc)) + monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine)) + monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60}) + + crawl_runner = runner_module.CrawlRunner(crawl) + crawl_runner.load_run_state() + + assert "CHROME_KEEPALIVE" not in crawl_runner.base_config + + def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatch): from archivebox.machine.models import Machine, NetworkInterface, Process from archivebox.services import runner as runner_module diff --git a/archivebox/tests/test_server_security_browser.py b/archivebox/tests/test_server_security_browser.py index ef59e9b6..ee9a375e 100644 --- a/archivebox/tests/test_server_security_browser.py +++ b/archivebox/tests/test_server_security_browser.py @@ -18,7 +18,7 @@ from urllib.parse import urlencode import pytest import requests -from .conftest import _ensure_puppeteer, _find_cached_chromium, _find_system_browser, run_python_cwd +from .conftest import _ensure_puppeteer, _find_cached_chrome, _find_system_browser, run_python_cwd PUPPETEER_PROBE_SCRIPT = """\ @@ -147,22 +147,22 @@ def _resolve_browser(shared_lib: Path) -> Path | None: if candidate.exists(): return candidate - cached = _find_cached_chromium(shared_lib) - if cached and cached.exists(): - return cached - system = _find_system_browser() if system and system.exists(): return system - which_candidates = ("chromium", "chromium-browser", "google-chrome", "google-chrome-stable", "chrome") + cached = _find_cached_chrome(shared_lib) + if cached and cached.exists(): + return cached + + which_candidates = ("chromium", "chromium-browser") for binary in which_candidates: resolved = shutil.which(binary) if resolved: return Path(resolved) mac_candidates = ( - Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"), + Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"), Path("/Applications/Chromium.app/Contents/MacOS/Chromium"), ) for candidate in mac_candidates: diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh index 299d95ee..a290c6ad 100755 --- a/bin/docker_entrypoint.sh +++ b/bin/docker_entrypoint.sh @@ -24,6 +24,9 @@ set -o errtrace set -o pipefail # IFS=$'\n' +# Prevent crashed browser/subprocess core dumps from filling snapshot folders. +ulimit -c 0 >/dev/null 2>&1 || true + # Load global invariants (set by Dockerfile during image build time, not intended to be customized by users at runtime) export DATA_DIR="${DATA_DIR:-/data}" export TMP_DIR="${TMP_DIR:-/tmp/archivebox}" diff --git a/docker-compose.yml b/docker-compose.yml index c99986e5..b0dcdd6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: archivebox: image: ${ARCHIVEBOX_IMAGE:-archivebox/archivebox:latest} + platform: linux/amd64 ports: - 8000:8000 volumes: @@ -32,10 +33,9 @@ services: - PERSONAS_DIR=/data/personas - LIB_DIR=/data/lib - NODE_PATH=/home/archivebox/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules - - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser - - CHROME_BIN=/usr/bin/chromium-browser - - CHROME_BINARY=/usr/bin/chromium-browser - - CHROMIUM_BINARY=/usr/bin/chromium-browser + - PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chrome-stable + - CHROME_BIN=/usr/bin/google-chrome-stable + - CHROME_BINARY=/usr/bin/google-chrome-stable - CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile - CHROME_HEADLESS=true - CHROME_SANDBOX=false diff --git a/publicsite/.nojekyll b/publicsite/.nojekyll index 8b137891..e69de29b 100644 --- a/publicsite/.nojekyll +++ b/publicsite/.nojekyll @@ -1 +0,0 @@ - diff --git a/pyproject.toml b/pyproject.toml index 339c98d3..4f1cbce2 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.30rc7" +version = "0.9.30rc12" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -80,8 +80,8 @@ dependencies = [ ### Binary/Package Management "abxbus>=2.5.0", # EventBus API "abxpkg>=1.10.7", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.10.32", # shared ArchiveBox plugin package with install_args-only overrides - "abx-dl>=1.10.37", # shared ArchiveBox downloader package with install_args-only overrides + "abx-plugins>=1.10.36", # shared ArchiveBox plugin package with install_args-only overrides + "abx-dl>=1.10.40", # shared ArchiveBox downloader package with install_args-only overrides ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]