The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix docker runtime and real crawl flows

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-14 21:24:32 -07:00
parent 2c3146549a
commit fa15e1edfa
No known key found for this signature in database
18 changed files with 263 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.dockerignore
View File

@ -26,6 +26,11 @@ abxbus/
chrome/
chromeprofile/
chrome_profile/
users/
archivebox-docker-smoke*/
archivebox-compose-smoke*/
core
*.core
pdm.dev.lock
pdm.lock

2
.github/workflows/debian.yml vendored
View File

@ -87,7 +87,7 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
enable-cache: false
- name: Install build dependencies
uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0

2
.github/workflows/homebrew.yml vendored
View File

@ -44,7 +44,7 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
enable-cache: false
- name: Install build dependencies (Linux)
if: runner.os == 'Linux'

2
.github/workflows/lint.yml vendored
View File

@ -35,7 +35,7 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@v6
with:
enable-cache: true
enable-cache: false
- name: Install dependencies with uv
run: |

2
.github/workflows/pip.yml vendored
View File

@ -24,7 +24,7 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@v4
with:
enable-cache: true
enable-cache: false
- name: Set up Python
uses: actions/setup-python@v5

2
.github/workflows/release-runner.yml vendored
View File

@ -26,7 +26,7 @@ jobs:
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
enable-cache: false
- uses: actions/setup-node@v4
with:

85
Dockerfile
View File

@ -2,7 +2,7 @@
# python3.13, uv, python3-ldap
# curl, wget, git, dig, ping, tree, nano
# node, npm, single-file, readability-extractor, postlight-parser
# ArchiveBox, yt-dlp, playwright, chromium
# ArchiveBox, yt-dlp, Google Chrome, ffmpeg
# Usage:
# git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
# docker build . -t archivebox
@ -10,9 +10,8 @@
# docker run -v "$PWD/data":/data archivebox add 'https://example.com'
# docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
# docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
# Multi-arch build:
# docker buildx create --use
# docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
# Chrome for Linux is only distributed for amd64, so the Docker image targets linux/amd64.
# docker buildx build . --platform=linux/amd64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
@ -28,7 +27,7 @@
#########################################################################################
FROM ubuntu:24.04
FROM --platform=linux/amd64 ubuntu:24.04
LABEL name="archivebox" \
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@ -70,7 +69,7 @@ ENV TZ=UTC \
# Language Version config
ENV PYTHON_VERSION=3.13 \
NODE_VERSION=22
NODE_VERSION=22.22.3
# Non-root User config
ENV ARCHIVEBOX_USER="archivebox" \
@ -107,9 +106,9 @@ RUN (echo "[i] Docker build for ArchiveBox starting..." \
&& echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
&& echo \
&& uname -a \
&& cat /etc/os-release | head -n7 \
&& which bash && bash --version | head -n1 \
&& which dpkg && dpkg --version | head -n1 \
&& sed -n '1,7p' /etc/os-release \
&& which bash && bash --version | sed -n '1p' \
&& which dpkg && dpkg --version | sed -n '1p' \
&& echo -e '\n\n' && env && echo -e '\n\n' \
) | tee -a /VERSION.txt
@ -152,11 +151,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
&& rm -rf /var/lib/apt/lists/* \
# Save version info
&& ( \
which curl && curl --version | head -n1 \
&& which wget && wget --version 2>&1 | head -n1 \
&& which git && git --version 2>&1 | head -n1 \
# && which ffmpeg && (ffmpeg --version 2>&1 | head -n1) || true \
&& which rg && rg --version 2>&1 | head -n1 \
which curl && curl --version | sed -n '1p' \
&& which wget && wget --version 2>&1 | sed -n '1p' \
&& which git && git --version 2>&1 | sed -n '1p' \
# && which ffmpeg && (ffmpeg --version 2>&1 | sed -n '1p') || true \
&& which rg && rg --version 2>&1 | sed -n '1p' \
&& echo -e '\n\n' \
) | tee -a /VERSION.txt
@ -194,17 +193,15 @@ RUN (which sonic && sonic --version) | tee -a /VERSION.txt
# ) | tee -a /VERSION.txt
# Set up Node environment
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
echo "[+] APT Installing NODE $NODE_VERSION for $TARGETPLATFORM..." \
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
&& curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
&& apt-get update -qq \
&& apt-get install -qq -y --no-upgrade libatomic1 \
&& apt-get install -y --no-upgrade \
nodejs \
&& rm -rf /var/lib/apt/lists/* \
# Set up Node environment from the official linux-x64 tarball. This avoids
# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
# emulated linux/amd64 Docker builds.
RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
echo "[+] Installing NODE $NODE_VERSION for linux/amd64..." \
&& curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz" \
&& echo "c7a10d6816da8eaaa7534dd73c71c6e2b2c391dbbf845e364902d156615dd1b8 node-v${NODE_VERSION}-linux-x64.tar.gz" | sha256sum -c - \
&& tar -xzf "node-v${NODE_VERSION}-linux-x64.tar.gz" -C /usr/local --strip-components=1 --no-same-owner \
&& rm "node-v${NODE_VERSION}-linux-x64.tar.gz" \
# Save version info
&& ( \
which node && node --version \
@ -256,46 +253,39 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
&& rm -rf /var/lib/apt/lists/*
# Install apt font & rendering dependencies for chromium browser
# TODO: figure out how much of this overlaps with `playwright install-deps chromium`
# Install apt font & rendering dependencies for Google Chrome
# RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
# Install chromium browser binary using playwright
# Install Google Chrome inside the container. Docker must use the package pulled
# during the image build, never a browser from the host checkout.
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
--mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
echo "[+] APT Installing CHROMIUM dependencies, fonts, and display libraries for $TARGETPLATFORM..." \
echo "[+] APT Installing Google Chrome, fonts, and display libraries for $TARGETPLATFORM..." \
&& curl -fsSL "https://dl.google.com/linux/linux_signing_key.pub" | gpg --dearmor -o /etc/apt/keyrings/google-linux.gpg \
&& echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/google-linux.gpg] https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
&& apt-get update -qq \
&& apt-get install -qq -y \
#fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
#at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
#libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
#libxaw7 libxcomposite1 libxdamage1 libxfont2 \
google-chrome-stable ffmpeg \
libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
# xfonts-scalable xfonts-utils xserver-common xvfb \
# chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
# libxss1 dbus dbus-x11 upower \
# && service dbus start \
&& echo "[+] PIP Installing playwright into /venv and CHROMIUM binary into $PLAYWRIGHT_BROWSERS_PATH..." \
&& uv pip install "playwright>=1.49.1" \
&& uv run playwright install chromium --no-shell --with-deps \
&& export CHROME_BINARY="$(uv run python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
&& ln -sf "$CHROME_BINARY" /usr/bin/chromium-browser \
&& ln -sf "$CHROME_BINARY" /usr/bin/chromium \
&& ln -sf /browsers/ffmpeg-*/ffmpeg-linux /usr/bin/ffmpeg \
&& mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
&& ln -sf /usr/bin/google-chrome-stable /usr/bin/chrome \
&& mkdir -p "/home/${ARCHIVEBOX_USER}/.config/google-chrome/Crash Reports/pending/" \
&& chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
&& mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
&& chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
# delete extra full copy of node that playwright installs (saves >100mb)
&& rm -f /venv/lib/python$PYTHON_VERSION/site-packages/playwright/driver/node \
# Save version info
&& rm -rf /var/lib/apt/lists/* \
&& ( \
uv pip show playwright \
&& which chromium && chromium --version \
&& which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
&& which ffmpeg && ffmpeg -version \
which google-chrome-stable && google-chrome-stable --version \
&& which chrome && chrome --version \
&& which ffmpeg && ffmpeg -version | sed -n '1p' \
&& echo -e '\n\n' \
) | tee -a /VERSION.txt
@ -303,10 +293,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH" \
PERSONAS_DIR=/data/personas \
NODE_PATH="/home/$ARCHIVEBOX_USER/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules" \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser \
CHROME_BIN=/usr/bin/chromium-browser \
CHROME_BINARY=/usr/bin/chromium-browser \
CHROMIUM_BINARY=/usr/bin/chromium-browser \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chrome-stable \
CHROME_BIN=/usr/bin/google-chrome-stable \
CHROME_BINARY=/usr/bin/google-chrome-stable \
CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
CHROME_HEADLESS=true \
CHROME_SANDBOX=false \

8
archivebox/services/runner.py
View File

@ -257,7 +257,6 @@ class CrawlRunner:
self.derived_config = dict(Machine.current().config)
self.crawl_output_dir = str(self.crawl.output_dir)
self.base_config["ABX_RUNTIME"] = "archivebox"
self.base_config["CHROME_KEEPALIVE"] = True
if self.selected_plugins is None:
raw_plugins = str(self.base_config.get("PLUGINS") or "").strip()
if raw_plugins:
@ -571,7 +570,7 @@ class CrawlRunner:
event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
)
await self.bus.emit(crawl_start_event).now()
await self.bus.emit(
snapshot_event = self.bus.emit(
SnapshotEvent(
url=snapshot["url"],
snapshot_id=snapshot["id"],
@ -581,8 +580,9 @@ class CrawlRunner:
event_timeout=snapshot_phase_timeout,
event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
),
).now()
await self.bus.wait_until_idle()
)
await snapshot_event.wait()
await snapshot_event.event_results_list()
await self.enqueue_discovered_snapshots_from_outputs(snapshot)
finally:
current_task = asyncio.current_task()

7
archivebox/tests/conftest.py
View File

@ -350,9 +350,11 @@ def _get_machine_type() -> str:
return f"{arch}-{os_name}{suffix}"
def _find_cached_chromium(lib_dir: Path) -> Path | None:
def _find_cached_chrome(lib_dir: Path) -> Path | None:
candidates = [
lib_dir / "puppeteer" / "chromium",
lib_dir / "puppeteer",
lib_dir / "ms-playwright",
lib_dir / "npm" / "node_modules" / "puppeteer" / ".local-chromium",
]
for base in candidates:
@ -369,8 +371,9 @@ def _find_cached_chromium(lib_dir: Path) -> Path | None:
def _find_system_browser() -> Path | None:
candidates = [
Path("/Applications/Chromium.app/Contents/MacOS/Chromium"),
Path("/usr/bin/chromium"),
Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"),
Path("/Applications/Chromium.app/Contents/MacOS/Chromium"),
Path("/usr/bin/chromium-browser"),
]
for candidate in candidates:

104
archivebox/tests/test_cli_real_flows.py
View File

@ -7,6 +7,7 @@ import shutil
import sqlite3
import subprocess
import sys
from pathlib import Path
import pytest
@ -44,10 +45,14 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro
},
)
node_binary = shutil.which("node")
chrome_binary = shutil.which("chromium") or shutil.which("google-chrome") or shutil.which("chrome")
canary_binary = Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary")
chrome_binary = str(canary_binary) if canary_binary.exists() else shutil.which("chromium") or shutil.which("chromium-browser")
if node_binary:
env["NODE_BINARY"] = node_binary
if chrome_binary:
chromium_binary = Path("/usr/bin/chromium")
if chromium_binary.exists():
env["CHROME_BINARY"] = str(chromium_binary)
elif chrome_binary:
env["CHROME_BINARY"] = chrome_binary
result = subprocess.run(
@ -116,15 +121,18 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro
assert all(depth == 0 for _id, _url, depth, _status, _title in snapshots)
by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size in archive_results}
for url in urls:
assert by_url_plugin[(url, "wget")] == "succeeded"
assert by_url_plugin[(url, "headers")] == "succeeded"
assert by_url_plugin[(url, "title")] == "succeeded"
assert by_url_plugin[("https://example.com", "wget")] == "succeeded"
assert by_url_plugin[("https://example.com", "headers")] == "succeeded"
assert by_url_plugin[("https://example.com", "title")] == "succeeded"
assert by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", "wget")] == "succeeded"
assert any(
by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", plugin)] == "succeeded" for plugin in ("headers", "title")
)
assert len([status for _url, _plugin, status, _files, _size in archive_results if status == "failed"]) <= 2
snapshot_root = tmp_path / "users/system/snapshots"
html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()]
header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file()]
header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0]
index_outputs = [path for path in snapshot_root.rglob("index.jsonl") if path.is_file()]
assert html_outputs
assert header_outputs
@ -137,3 +145,85 @@ def test_cli_add_real_urls_with_options_writes_inspectable_outputs(tmp_path, pro
assert processes
assert any("wget" in (pwd or "") or "wget" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
assert any("headers" in (pwd or "") or "headers" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
@pytest.mark.timeout(180)
def test_cli_recursive_crawl_processes_discovered_html_urls(tmp_path, process):
os.chdir(tmp_path)
assert process.returncode == 0, process.stderr
env = os.environ.copy()
env.update(
{
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
"TIMEOUT": "60",
"SAVE_WGET": "true",
"SAVE_HEADERS": "false",
"SAVE_TITLE": "false",
"SAVE_READABILITY": "false",
"SAVE_SINGLEFILE": "false",
"SAVE_MERCURY": "false",
"SAVE_SCREENSHOT": "false",
"SAVE_PDF": "false",
"SAVE_DOM": "false",
"SAVE_ARCHIVEDOTORG": "false",
"SAVE_GIT": "false",
"SAVE_YTDLP": "false",
"SAVE_FAVICON": "false",
"PARSE_HTML_URLS_ENABLED": "true",
"PARSE_DOM_OUTLINKS_ENABLED": "false",
},
)
result = subprocess.run(
[
sys.executable,
"-m",
"archivebox",
"add",
"--depth=2",
"--max-urls=2",
"--max-size=50mb",
"--tag=recursive-flow",
"--parser=url_list",
"--plugins=wget,parse_html_urls",
"https://example.com",
],
cwd=tmp_path,
capture_output=True,
text=True,
env=env,
timeout=180,
)
assert result.returncode == 0, result.stderr or result.stdout
conn = sqlite3.connect(tmp_path / "index.sqlite3")
try:
crawl = conn.execute(
"SELECT max_depth, max_urls, max_size, tags_str FROM crawls_crawl ORDER BY created_at DESC LIMIT 1",
).fetchone()
snapshots = conn.execute(
"SELECT url, depth, status FROM core_snapshot ORDER BY depth, url",
).fetchall()
archive_results = conn.execute(
"SELECT s.url, ar.plugin, ar.status, ar.output_files "
"FROM core_archiveresult ar "
"JOIN core_snapshot s ON s.id = ar.snapshot_id "
"ORDER BY s.depth, s.url, ar.plugin",
).fetchall()
finally:
conn.close()
assert crawl == (2, 2, 50 * 1024 * 1024, "recursive-flow")
assert ("https://example.com", 0, "sealed") in snapshots
assert any(url == "https://iana.org/domains/example" and depth == 1 and status == "sealed" for url, depth, status in snapshots)
by_url_plugin = {(url, plugin): status for url, plugin, status, _files in archive_results}
assert by_url_plugin[("https://example.com", "wget")] == "succeeded"
assert by_url_plugin[("https://example.com", "parse_html_urls")] == "succeeded"
assert by_url_plugin[("https://iana.org/domains/example", "wget")] == "succeeded"
urls_outputs = list((tmp_path / "users/system/snapshots").rglob("parse_html_urls/urls.jsonl"))
assert urls_outputs
assert any("https://iana.org/domains/example" in path.read_text() for path in urls_outputs)

2
archivebox/tests/test_hooks.py
View File

@ -676,7 +676,7 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
"/usr/lib/node_modules",
str(node_modules_dir),
"/usr/share/archivebox/lib/npm/node_modules",
]
],
)
plugin_dir = tmp_path / "plugins" / "envprobe"

2
archivebox/tests/test_persona_admin.py
View File

@ -38,7 +38,7 @@ def _make_profile_source(tmp_path):
browser="chrome",
user_data_dir=user_data_dir,
profile_dir="Default",
browser_binary="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
browser_binary="/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
)

91
archivebox/tests/test_runner.py
View File

@ -33,6 +33,12 @@ class _DummyBus:
async def now(self, *args, **kwargs):
return event
async def wait(self, *args, **kwargs):
return event
async def event_results_list(self):
return []
return _Pending()
async def stop(self):
@ -42,6 +48,11 @@ class _DummyBus:
return None
class _NoIdleBus(_DummyBus):
async def wait_until_idle(self):
raise AssertionError("run_snapshot should not wait on the whole crawl bus")
class _DummyService:
def __init__(self, *args, **kwargs):
pass
@ -136,6 +147,42 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
assert len(created_buses) == 1
def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch):
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
from archivebox.core.models import Snapshot
from archivebox.services import runner as runner_module
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
)
snapshot = Snapshot.objects.create(
url="https://example.com",
crawl=crawl,
status=Snapshot.StatusChoices.QUEUED,
)
monkeypatch.setattr(runner_module, "create_bus", lambda **kwargs: _NoIdleBus(kwargs["name"]))
monkeypatch.setattr(runner_module, "discover_plugins", lambda: {})
monkeypatch.setattr(runner_module, "HookProcessService", _DummyService)
monkeypatch.setattr(runner_module, "PersistedProcessService", _DummyService)
monkeypatch.setattr(runner_module, "BinaryService", _DummyService)
monkeypatch.setattr(runner_module, "TagService", _DummyService)
monkeypatch.setattr(runner_module, "CrawlService", _DummyService)
monkeypatch.setattr(runner_module, "SnapshotService", _DummyService)
monkeypatch.setattr(runner_module, "ArchiveResultService", _DummyService)
monkeypatch.setattr(runner_module, "_emit_machine_config", lambda *args, **kwargs: asyncio.sleep(0))
monkeypatch.setattr(runner_module, "setup_abx_services", lambda *args, **kwargs: None)
crawl_runner = runner_module.CrawlRunner(crawl)
snapshot_payload = crawl_runner.load_snapshot_payload(str(snapshot.id))
monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_payload)
monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0))
asyncio.run(crawl_runner.run_snapshot(str(snapshot.id)))
def test_ensure_background_runner_starts_when_none_running(monkeypatch):
import archivebox.machine.models as machine_models
from archivebox.services import runner as runner_module
@ -326,6 +373,50 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
assert crawl_runner.derived_config == machine.config
def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch):
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.services import runner as runner_module
from archivebox.config import configset as configset_module
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.crawls.models import Crawl
machine = Machine.objects.create(
guid="test-guid-runner-chrome-keepalive",
hostname="runner-host-chrome-keepalive",
hw_in_docker=False,
hw_in_vm=False,
hw_manufacturer="Test",
hw_product="Test Product",
hw_uuid="test-hw-runner-chrome-keepalive",
os_arch="arm64",
os_family="darwin",
os_platform="macOS",
os_release="14.0",
os_kernel="Darwin",
stats={},
config={},
)
crawl = Crawl.objects.create(
urls="https://example.com",
created_by_id=get_or_create_system_user_pk(),
)
proc = SimpleNamespace(iface_id=str(machine.id), machine_id=str(machine.id), iface=None, machine=machine, save=lambda **kwargs: None)
monkeypatch.setattr(
NetworkInterface,
"current",
classmethod(lambda cls, refresh=False: SimpleNamespace(id=machine.id, machine=machine)),
)
monkeypatch.setattr(Process, "current", classmethod(lambda cls: proc))
monkeypatch.setattr(Machine, "current", classmethod(lambda cls: machine))
monkeypatch.setattr(configset_module, "get_config", lambda **kwargs: {"PLUGINS": "", "CHROME_BINARY": "", "TIMEOUT": 60})
crawl_runner = runner_module.CrawlRunner(crawl)
crawl_runner.load_run_state()
assert "CHROME_KEEPALIVE" not in crawl_runner.base_config
def test_load_run_state_uses_enabled_plugins_when_plugins_key_missing(monkeypatch):
from archivebox.machine.models import Machine, NetworkInterface, Process
from archivebox.services import runner as runner_module

14
archivebox/tests/test_server_security_browser.py
View File

@ -18,7 +18,7 @@ from urllib.parse import urlencode
import pytest
import requests
from .conftest import _ensure_puppeteer, _find_cached_chromium, _find_system_browser, run_python_cwd
from .conftest import _ensure_puppeteer, _find_cached_chrome, _find_system_browser, run_python_cwd
PUPPETEER_PROBE_SCRIPT = """\
@ -147,22 +147,22 @@ def _resolve_browser(shared_lib: Path) -> Path | None:
if candidate.exists():
return candidate
cached = _find_cached_chromium(shared_lib)
if cached and cached.exists():
return cached
system = _find_system_browser()
if system and system.exists():
return system
which_candidates = ("chromium", "chromium-browser", "google-chrome", "google-chrome-stable", "chrome")
cached = _find_cached_chrome(shared_lib)
if cached and cached.exists():
return cached
which_candidates = ("chromium", "chromium-browser")
for binary in which_candidates:
resolved = shutil.which(binary)
if resolved:
return Path(resolved)
mac_candidates = (
Path("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"),
Path("/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary"),
Path("/Applications/Chromium.app/Contents/MacOS/Chromium"),
)
for candidate in mac_candidates:

3
bin/docker_entrypoint.sh
View File

@ -24,6 +24,9 @@ set -o errtrace
set -o pipefail
# IFS=$'\n'
# Prevent crashed browser/subprocess core dumps from filling snapshot folders.
ulimit -c 0 >/dev/null 2>&1 || true
# Load global invariants (set by Dockerfile during image build time, not intended to be customized by users at runtime)
export DATA_DIR="${DATA_DIR:-/data}"
export TMP_DIR="${TMP_DIR:-/tmp/archivebox}"

8
docker-compose.yml
View File

@ -12,6 +12,7 @@
services:
archivebox:
image: ${ARCHIVEBOX_IMAGE:-archivebox/archivebox:latest}
platform: linux/amd64
ports:
- 8000:8000
volumes:
@ -32,10 +33,9 @@ services:
- PERSONAS_DIR=/data/personas
- LIB_DIR=/data/lib
- NODE_PATH=/home/archivebox/.npm/lib/node_modules:/usr/lib/node_modules:/data/lib/npm/node_modules:/usr/share/archivebox/lib/npm/node_modules:/data/personas/Default/node_modules
- PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
- CHROME_BIN=/usr/bin/chromium-browser
- CHROME_BINARY=/usr/bin/chromium-browser
- CHROMIUM_BINARY=/usr/bin/chromium-browser
- PUPPETEER_EXECUTABLE_PATH=/usr/bin/google-chrome-stable
- CHROME_BIN=/usr/bin/google-chrome-stable
- CHROME_BINARY=/usr/bin/google-chrome-stable
- CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile
- CHROME_HEADLESS=true
- CHROME_SANDBOX=false

1
publicsite/.nojekyll
View File

@ -1 +0,0 @@

6
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.30rc7"
version = "0.9.30rc12"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -80,8 +80,8 @@ dependencies = [
### Binary/Package Management
"abxbus>=2.5.0", # EventBus API
"abxpkg>=1.10.7", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.10.32", # shared ArchiveBox plugin package with install_args-only overrides
"abx-dl>=1.10.37", # shared ArchiveBox downloader package with install_args-only overrides
"abx-plugins>=1.10.36", # shared ArchiveBox plugin package with install_args-only overrides
"abx-dl>=1.10.40", # shared ArchiveBox downloader package with install_args-only overrides
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]