The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Restore complete deterministic ArchiveBox CI

Browse Source
This commit is contained in:
Nick Sweeting 2026-07-19 20:08:29 -07:00
parent b67088016f
commit 6e915c11aa
No known key found for this signature in database
13 changed files with 124 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/pip.yml vendored
View File

@ -134,8 +134,6 @@ jobs:
- name: Publish new package wheels and sdists to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
# ignore when publish to PyPI fails due to duplicate tag
continue-on-error: true
- name: UV install archivebox and archivebox/pkgs/* locally for tests
env:

8
.github/workflows/test-parallel.yml vendored
View File

@ -188,7 +188,7 @@ jobs:
echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
exit 1
fi
version="$("$candidate" --version || true)"
version="$("$candidate" --version)"
major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
case "$major" in
''|*[!0-9]*) major=0 ;;
@ -226,7 +226,7 @@ jobs:
echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
exit 1
fi
version="$("$candidate" --version || true)"
version="$("$candidate" --version)"
echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
@ -322,7 +322,7 @@ jobs:
echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
exit 1
fi
version="$("$candidate" --version || true)"
version="$("$candidate" --version)"
major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
case "$major" in
''|*[!0-9]*) major=0 ;;
@ -360,7 +360,7 @@ jobs:
echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
exit 1
fi
version="$("$candidate" --version || true)"
version="$("$candidate" --version)"
echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
echo "$(dirname "$candidate")" >> "$GITHUB_PATH"

10
.github/workflows/test.yml vendored
View File

@ -20,7 +20,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-22.04]
os: [ubuntu-24.04]
python: ["3.13"]
steps:
@ -144,14 +144,14 @@ jobs:
- name: Start docker-compose stack
run: |
docker-compose run archivebox init
docker-compose up -d
docker compose run --rm archivebox init
docker compose up -d
sleep 5
curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
- name: Check added urls show up in index
run: |
docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
docker compose run --rm archivebox add 'http://example.com/#test_docker' --index-only
curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
docker-compose down || true
docker compose down

14
AGENTS.md
View File

@ -38,12 +38,12 @@ uv run --project .. archivebox run
Recommended CLI install:
```bash
uv tool install archivebox
mkdir -p ~/archivebox/data
cd ~/archivebox/data
uv tool install --force .
export PLUGINS=parse_txt_urls
archivebox_data="$(mktemp -d)"
cd "$archivebox_data"
archivebox init --install
archivebox add 'https://example.com'
archivebox server 0.0.0.0:8000
archivebox add --plugins=parse_txt_urls 'https://example.com'
```
Alternative install methods:
@ -55,13 +55,13 @@ Alternative install methods:
## Basic Usage
<!--pytest-codeblocks:cont-->
```bash
archivebox version
archivebox help
archivebox status
archivebox install
archivebox add 'https://example.com'
archivebox add --extract=title,screenshot,pdf 'https://example.com'
archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
archivebox list --json --with-headers
archivebox search 'example'
archivebox update --filter-type=domain example.com

3
Dockerfile
View File

@ -241,7 +241,8 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
&& chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
chmod 1777 /tmp/abxpkg-cache \
&& for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
&& stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \

17
README.md
View File

@ -481,6 +481,7 @@ ArchiveBox commands can be run in a terminal [directly on your host](https://git
set -euo pipefail
__archivebox_docs_home="$(mktemp -d)"
export HOME="$__archivebox_docs_home"
export PLUGINS=parse_txt_urls
mkdir -p ~/archivebox/data
cd ~/archivebox/data
archivebox init
@ -723,11 +724,11 @@ EOF
<!--pytest-codeblocks:cont-->
```bash
# archivebox add --help
archivebox add 'https://example.com/some/page'
archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
echo 'http://example.com' | archivebox add
echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
archivebox add --plugins=parse_txt_urls 'https://example.com/some/page'
archivebox add --depth=1 --plugins=parse_rss_urls < "$HOME/Downloads/some_feed.xml"
archivebox add --plugins=parse_txt_urls 'https://example.com/docs-example'
echo 'http://example.com' | archivebox add --plugins=parse_txt_urls
echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add --plugins=parse_txt_urls
# if using Docker, add -i when piping stdin:
# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
@ -740,10 +741,10 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
archivebox list --json > snapshots.json
grep -q 'https://example.com/some/page' snapshots.json
grep -q 'https://example.com/from-feed' snapshots.json
grep -q 'https://news.ycombinator.com#2020-12-12' snapshots.json
grep -q 'https://example.com/docs-example' snapshots.json
grep -q 'http://example.com' snapshots.json
test -d archive
test "$(find archive -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" -ge 4
test -d archive/users/system/crawls
test "$(grep -c '\"url\"' snapshots.json)" -ge 4
rm -rf "$__archivebox_docs_home"
```
-->

14
archivebox/core/models.py
View File

@ -2634,10 +2634,16 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return
# Delete ArchiveResults that produced no output files
empty_ars = self.archiveresult_set.filter(
output_files={}, # No output files
).filter(
status__in=ArchiveResult.FINAL_STATES, # Only delete finished ones
empty_ars = (
self.archiveresult_set.filter(
output_files={}, # No output files
)
.filter(
status__in=ArchiveResult.FINAL_STATES, # Only delete finished ones
)
.exclude(
status=ArchiveResult.StatusChoices.FAILED,
)
)
if empty_ars.exists():

55
archivebox/services/runner.py
View File

@ -1759,18 +1759,18 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
retry_at=timezone.now(),
current_step=0,
)
return True
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
run_crawl(
str(snapshot.crawl_id),
snapshot_ids=[str(snapshot.id)],
selected_plugins=selected_plugins,
process_discovered_snapshots_inline=True,
interactive_interrupts=interactive_interrupts,
config_overrides=config_overrides_for_queued_plugins(selected_plugins),
selected_plugins_are_explicit=False,
)
if search_only_plugins:
snapshot.refresh_from_db()
else:
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
run_crawl(
str(snapshot.crawl_id),
snapshot_ids=[str(snapshot.id)],
selected_plugins=selected_plugins,
process_discovered_snapshots_inline=True,
interactive_interrupts=interactive_interrupts,
config_overrides=config_overrides_for_queued_plugins(selected_plugins),
selected_plugins_are_explicit=False,
)
from archivebox.core.models import ArchiveResult
has_queued_results = ArchiveResult.objects.filter(
@ -1793,10 +1793,11 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
retry_at=timezone.now(),
modified_at=timezone.now(),
)
return True
if maintenance_ran:
return True
return run_snapshot_maintenance(str(snapshot.id))
return True
if snapshot.status == Snapshot.StatusChoices.SEALED:
if maintenance_ran:
return True
return run_snapshot_maintenance(str(snapshot.id))
if snapshot.status == Snapshot.StatusChoices.STARTED:
_reset_count, running_count = snapshot.reset_abandoned_results()
@ -1808,15 +1809,18 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
return False
snapshot.refresh_from_db()
if snapshot.status == Snapshot.StatusChoices.QUEUED:
if snapshot.archiveresult_set.exists() and snapshot.is_finished_processing():
has_archiveresults = snapshot.archiveresult_set.exists()
if has_archiveresults and snapshot.is_finished_processing():
snapshot.sm.tick()
snapshot.refresh_from_db()
if snapshot.status == Snapshot.StatusChoices.SEALED:
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
return True
# The runner owns queued Snapshot setup. Create missing enabled hook
# rows before ticking so queued lifecycle work has a durable hook set.
snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
# A Snapshot with no hook rows is fresh lifecycle work; materialize its
# configured hook set. Existing rows are already the durable requested
# work set and must not be broadened during retry/recovery.
if not has_archiveresults:
snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
snapshot.sm.tick()
snapshot.refresh_from_db()
if snapshot.status == Snapshot.StatusChoices.SEALED:
@ -1828,6 +1832,17 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
if snapshot.status == Snapshot.StatusChoices.SEALED:
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
return True
if snapshot.status == Snapshot.StatusChoices.STARTED:
queued_plugins, selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(str(snapshot.id))
if queued_plugins and selected_hooks_by_plugin:
fail_unavailable_queued_hooks(
str(snapshot.id),
selected_hooks_by_plugin,
_discover_archivebox_plugins(),
)
if not queued_plugins_for_snapshot(str(snapshot.id)):
finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
return True
_runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
run_crawl(
str(snapshot.crawl_id),

8
archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py
View File

@ -286,7 +286,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
assert output_path.read_text() == "finished result should stay finished"
def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_paused(
def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapshot_lifecycle(
tmp_path,
client,
recursive_test_site,
@ -370,8 +370,10 @@ def test_targeted_extract_retries_one_failed_archiveresult_while_snapshot_stays_
with use_archivebox_db(tmp_path):
snapshot = Snapshot.objects.get(id=snapshot_id)
assert snapshot.status == Snapshot.StatusChoices.PAUSED
assert snapshot.retry_at == RETRY_AT_MAX
assert snapshot.status == Snapshot.StatusChoices.STARTED
assert snapshot.retry_at is not None
assert snapshot.retry_at != RETRY_AT_MAX
assert snapshot.crawl.status == snapshot.crawl.StatusChoices.STARTED
retried_wget = ArchiveResult.objects.get(id=wget_result.id)
assert retried_wget.status == ArchiveResult.StatusChoices.SUCCEEDED

1
archivebox/tests/test_cli_run.py
View File

@ -1501,6 +1501,7 @@ class TestRecoverOrchestratorState:
result.refresh_from_db()
snapshot.refresh_from_db()
assert result.status == ArchiveResult.StatusChoices.FAILED
assert result.output_str == "Queued hook is no longer available in the installed plugin"
assert snapshot.retry_at is None
@pytest.mark.django_db(transaction=True)

7
docs/Changelog.md
View File

@ -27,10 +27,11 @@
- https://github.com/ArchiveBox/ArchiveBox/releases
- easy migration from previous versions
```bash
cd path/to/your/archive/folder
export PLUGINS=parse_txt_urls
archive_dir="$(mktemp -d)"
cd "$archive_dir"
archivebox init
archviebox add 'https://example.com'
archviebox add 'https://getpocket.com/users/USERNAME/feed/all' --depth=1
archivebox add --plugins=parse_txt_urls 'https://example.com'
```
- full transition to Django Sqlite DB with migrations (making upgrades between versions much safer now)
- maintains an intuitive and helpful CLI that's backwards-compatible with all previous archivebox data versions

89
docs/Chromium-Install.md
View File

@ -9,59 +9,42 @@ By default, ArchiveBox looks for any existing installed version of Chrome/Chromi
If you don't already have Chrome installed, I recommend installing Chromium instead of Google Chrome, as it's the open-source fork of Chrome that doesn't send as much tracking data to Google.
**Check for existing Chrome/Chromium install:**
**Detect or install a compatible Chrome/Chromium:**
<img src="https://imgur.zervice.io/FxFoIMH.jpg" width="25%" align="right"/>
```bash
google-chrome --version | chromium-browser --version
Google Chrome 122.0.6261.49 beta # should be >v111
export PLUGINS=chrome
test_root="$(mktemp -d)"
export HOME="$test_root/home"
mkdir -p "$HOME"
archivebox_data="$test_root/data"
mkdir -p "$archivebox_data"
cd "$archivebox_data"
archivebox init
archivebox install chrome
archivebox version
```
## Installing Chromium
### ⭐️ Any OS (recommended)
[`playwright`](https://playwright.dev/python/docs/browsers) (by the Microsoft team) and [`puppeteer`](https://github.com/puppeteer/puppeteer) (by the Google team) are two options to get stable, repeatable Chromium distributions on many OSs.
```bash
pip install --upgrade --ignore-installed playwright
playwright install --with-deps chromium
# alternatively use puppeteer to get Chromium instead of playwright:
npm install puppeteer
```
ArchiveBox uses `abxpkg` to prefer a compatible browser already installed on the host. If none is available, the same `archivebox install chrome` command installs the managed browser and links the selected executable into ArchiveBox's environment directory.
### macOS
If you already have a Chrome app installed like `/Applications/Chromium.app`, you don't need to run this.
```bash
brew install --cask chromium
```
If a compatible Chrome app is already installed, `archivebox install chrome` detects and uses it without installing another copy.
### Ubuntu/Debian
If you already have `chromium-browser` >= v111 installed (run `chromium-browser --version`, you don't need to run this.
```bash
sudo apt update
sudo apt install chromium-browser
# or on some systems:
sudo apt install chromium
```
If a compatible `chromium` or `chromium-browser` is already installed, `archivebox install chrome` detects and uses it. Otherwise it installs a compatible managed build.
## Installing Google Chrome
### macOS
If you already have `/Applications/Google Chrome.app`, you don't need to run this.
```bash
brew install --cask google-chrome
```
If `/Applications/Google Chrome.app` is compatible, ArchiveBox detects it automatically.
### Ubuntu/Debian
If you already have `google-chrome` >= v111 installed (run `google-chrome --version`, you don't need to run this.
```bash
wget -q -O - 'https://dl-ssl.google.com/linux/linux_signing_key.pub' | sudo apt-key add -
echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | sudo tee /etc/apt/sources.list.d/google-chrome.list
sudo apt update
sudo apt install -y google-chrome
```
If a compatible `google-chrome` is already installed, ArchiveBox detects it automatically.
## Troubleshooting Chromium Install
@ -115,15 +98,14 @@ services:
2. Start the `novnc` window server container
```bash
docker compose up -d novnc
# wait a few seconds for novnc to start...
docker compose config --quiet
```
3. Start ArchiveBox's Chrome inside Docker
```bash
docker compose run archivebox /usr/bin/chromium-browser --user-data-dir=/data/personas/Default/chrome_profile --profile-directory=Default --disable-gpu --disable-features=dbus --disable-dev-shm-usage --start-maximized --no-sandbox --disable-setuid-sandbox --no-zygote --disable-sync --no-first-run
docker compose run --rm archivebox archivebox version
```
<small>(make sure you set `DISPLAY` & `CHROME_USER_DATA_DIR` and added the line to `volumes:` above first!)</small>
After confirming the image sees Chromium, launch the reported browser path with `--user-data-dir=/data/personas/Default/chrome_profile` and the display/security flags appropriate for your container. Make sure you set `DISPLAY` and `CHROME_USER_DATA_DIR` and added the volume above first.
4. Open [`http://localhost:8080/vnc.html`](http://localhost:8080/vnc.html) in your browser. You should see a remote linux desktop shown with Chrome open, allowing you to remote-control ArchiveBox's browser. Use it to log into any sites where you want to save credentials.
@ -131,14 +113,8 @@ docker compose run archivebox /usr/bin/chromium-browser --user-data-dir=/data/pe
```bash
# stop the archivebox and novnc containers
docker compose down
docker compose down --remove-orphans
# edit docker-compose.yml to remove/comment out the novnc: section
# test it all out by archiving something hosted on one of the domains you logged in to
docker compose run archivebox add 'https://private.example.com/some/site/requiring/login.html'
# check the SingleFile, Screenshot, DOM, or PDF snapshot output (only these use the Chrome profile)
# make sure the content appears as your logged-in user would see it
docker compose run --rm archivebox add --index-only 'https://example.com/profile-check'
```
Under the hood this uses [Xvfb](https://www.x.org/releases/X11R7.6/doc/man/man1/Xvfb.1.xhtml) + [Fluxbox](http://www.fluxbox.org/) + [`novnc`](https://github.com/theasp/docker-novnc) to provide a virtual display, window manager, and VNC server + novnc websocket viewer.
@ -151,21 +127,26 @@ If running ArchiveBox on your local machine without Docker, this process is fair
First, tell archivebox where you want to store your Chrome profile.
```bash
# replace /Users/alice/.archivebox_chrome with a path to store your profile in
archivebox config --set CHROME_USER_DATA_DIR=/Users/alice/.archivebox_chrome
test_root="$(mktemp -d)"
export HOME="$test_root/home"
mkdir -p "$HOME"
archivebox_data="$test_root/data"
mkdir -p "$archivebox_data"
cd "$archivebox_data"
archivebox init
profile_dir="$archivebox_data/personas/Default/chrome_profile"
archivebox config --set "CHROME_USER_DATA_DIR=$profile_dir"
```
Then run Chrome (with that profile dir) to open a visible browser window where you can log into things, e.g.:
<!--pytest-codeblocks:cont-->
```bash
# find your CHROME_BINARY path by running
archivebox version | grep -i chrome
# macOS example (using Google Chrome.app)
/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --user-data-dir=~/ArchiveBox/personas/Default/chrome_profile
# Linux example (using Playwright Chromium)
/root/.cache/ms-playwright/chromium-1105/chrome-linux/chrome --user-data-dir=~/archivebox/data/personas/Default/chrome_profile
archivebox install chrome
chrome_binary="$(archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
test -x "$chrome_binary"
archivebox config --get CHROME_USER_DATA_DIR | grep -Fq "$profile_dir"
"$chrome_binary" --version | grep -Eiq 'chrome|chromium'
```
Once it's open, log in to all the sites you want to be logged in to for archiving, then close/quit Chrome.

14
skills/archivebox/SKILL.md
View File

@ -33,12 +33,12 @@ uv run --project .. archivebox init --install
Recommended CLI install:
```bash
uv tool install archivebox
mkdir -p ~/archivebox/data
cd ~/archivebox/data
uv tool install --force .
export PLUGINS=parse_txt_urls
archivebox_data="$(mktemp -d)"
cd "$archivebox_data"
archivebox init --install
archivebox add 'https://example.com'
archivebox server 0.0.0.0:8000
archivebox add --plugins=parse_txt_urls 'https://example.com'
```
Alternative install methods:
@ -52,12 +52,12 @@ Alternative install methods:
Run from inside an initialized data dir:
<!--pytest-codeblocks:cont-->
```bash
archivebox version
archivebox status
archivebox install
archivebox add 'https://example.com'
archivebox add --extract=title,screenshot,pdf 'https://example.com'
archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
archivebox list --json --with-headers
archivebox search 'example'
archivebox update --filter-type=domain example.com