mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
release: archivebox 0.9.31rc18
This commit is contained in:
Nick Sweeting
parent
f9512d5ea2
commit
3b8ce3f467
@ -331,7 +331,7 @@ RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary
|
||||
|
||||
# Verify ArchiveBox is installed and write full version/dependency info.
|
||||
RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& gosu "$DEFAULT_PUID" archivebox version 2>&1 | tee -a /VERSION.txt
|
||||
&& gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt
|
||||
|
||||
####################################################
|
||||
|
||||
|
||||
@ -232,7 +232,7 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
|
||||
CANDIDATES = [
|
||||
config.LIB_DIR, # <user-specified>
|
||||
CONSTANTS.DEFAULT_LIB_DIR, # ./data/lib/arm64-linux-docker
|
||||
Path("/usr/local/share/archivebox") / get_collection_id(), # /usr/local/share/archivebox/abc5
|
||||
Path("/opt/archivebox/lib") if IN_DOCKER else Path("/opt/archivebox/lib") / get_collection_id(),
|
||||
*(
|
||||
[Path("/opt/homebrew/share/archivebox") / get_collection_id()] if os.path.isfile("/opt/homebrew/bin/archivebox") else []
|
||||
), # /opt/homebrew/share/archivebox/abc5
|
||||
@ -408,8 +408,8 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
|
||||
# with SudoPermission(uid=ARCHIVEBOX_USER, fallback=True):
|
||||
# lib_dir = HOST_DIRS.site_data_path
|
||||
|
||||
# # Docker: /usr/local/share/archivebox/0.8.5
|
||||
# # Ubuntu: /usr/local/share/archivebox/0.8.5
|
||||
# # Docker: /opt/archivebox/lib
|
||||
# # Ubuntu: /opt/archivebox/lib/0.8.5
|
||||
# # macOS: /Library/Application Support/archivebox
|
||||
# try:
|
||||
# with SudoPermission(uid=0, fallback=True):
|
||||
|
||||
@ -288,7 +288,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
|
||||
" - It [yellow]should[/yellow] be able to hold at least 1GB of data (some dependencies like Chrome can be large).[/grey53]",
|
||||
"",
|
||||
"[violet]Hint:[/violet] Fix it by setting LIB_DIR to a path that meets these requirements, e.g.:",
|
||||
f" [green]archivebox config --set LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or '/usr/local/share/archivebox'}[/green]",
|
||||
f" [green]archivebox config --set LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or '/opt/archivebox/lib'}[/green]",
|
||||
"",
|
||||
),
|
||||
)
|
||||
|
||||
@ -144,7 +144,7 @@ ensure_runtime_tree() {
|
||||
|
||||
run_as_archivebox() {
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
gosu "$PUID:$PGID" "$@"
|
||||
gosu "$ARCHIVEBOX_USER" "$@"
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
@ -169,6 +169,8 @@ ensure_dir "$DATA_DIR/sources"
|
||||
ensure_dir "$DATA_DIR/archive"
|
||||
ensure_dir "$DATA_DIR/archive/users"
|
||||
ensure_dir "$PERSONAS_DIR"
|
||||
ensure_dir "$PERSONAS_DIR/Default"
|
||||
ensure_dir "$PERSONAS_DIR/Default/chrome_profile"
|
||||
[[ -e "$DATA_DIR/users" ]] && ensure_dir "$DATA_DIR/users"
|
||||
ensure_file_owner "$DATA_DIR/index.sqlite3"
|
||||
ensure_file_owner "$DATA_DIR/ArchiveBox.conf"
|
||||
@ -177,6 +179,8 @@ run_as_archivebox touch "$DATA_DIR/logs/.permissions_test_safe_to_delete" 2>/dev
|
||||
rm -f "$DATA_DIR/logs/.permissions_test_safe_to_delete"
|
||||
run_as_archivebox touch "$DATA_DIR/archive/.permissions_test_safe_to_delete" 2>/dev/null || permission_error "$DATA_DIR/archive"
|
||||
rm -f "$DATA_DIR/archive/.permissions_test_safe_to_delete"
|
||||
run_as_archivebox touch "$PERSONAS_DIR/Default/chrome_profile/.permissions_test_safe_to_delete" 2>/dev/null || permission_error "$PERSONAS_DIR/Default/chrome_profile"
|
||||
rm -f "$PERSONAS_DIR/Default/chrome_profile/.permissions_test_safe_to_delete"
|
||||
|
||||
# check if novnc x11 $DISPLAY is available
|
||||
export DISPLAY="${DISPLAY:-"novnc:0.0"}"
|
||||
@ -279,7 +283,7 @@ if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" ==
|
||||
# "docker run archivebox /bin/bash -c '...'"
|
||||
# "docker run archivebox cat /VERSION.txt"
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
exec gosu "$PUID:$PGID" /bin/bash -c "exec $(printf ' %q' "$@")"
|
||||
exec gosu "$ARCHIVEBOX_USER" /bin/bash -c "exec $(printf ' %q' "$@")"
|
||||
else
|
||||
exec /bin/bash -c "exec $(printf ' %q' "$@")"
|
||||
fi
|
||||
@ -293,7 +297,7 @@ else
|
||||
# "docker run archivebox manage createsupseruser"
|
||||
# "docker run archivebox server 0.0.0.0:8000"
|
||||
if [[ "$(id -u)" == "0" ]]; then
|
||||
exec gosu "$PUID:$PGID" "$ARCHIVEBOX_BIN_PATH" "$@"
|
||||
exec gosu "$ARCHIVEBOX_USER" "$ARCHIVEBOX_BIN_PATH" "$@"
|
||||
else
|
||||
exec "$ARCHIVEBOX_BIN_PATH" "$@"
|
||||
fi
|
||||
|
||||
@ -121,10 +121,10 @@ build_and_prek() {
|
||||
(
|
||||
cd "$repo"
|
||||
rm -rf dist
|
||||
uv build --out-dir dist
|
||||
uv run prek run --all-files
|
||||
uv --no-cache build --out-dir dist
|
||||
uv --no-cache run prek run --all-files
|
||||
rm -rf dist
|
||||
uv build --out-dir dist
|
||||
uv --no-cache build --out-dir dist
|
||||
)
|
||||
}
|
||||
|
||||
@ -146,7 +146,7 @@ commit_push_publish() {
|
||||
echo "[*] No staged changes in ${package}; reusing existing commit."
|
||||
fi
|
||||
git push origin "$branch"
|
||||
uv publish --username="${PYPI_USERNAME}" dist/*
|
||||
uv --no-cache publish --username="${PYPI_USERNAME}" dist/*
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@ -87,8 +87,56 @@ docker_setup() {
|
||||
$setup_script"
|
||||
}
|
||||
|
||||
default_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)"; touch /data/logs/probe /data/archive/probe "$LIB_DIR/probe"; stat -c "ABX_STAT %u:%g:%a %n" /data /data/logs /data/archive "$LIB_DIR"; echo ABX_OK'
|
||||
version_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)"; archivebox version >/tmp/archivebox-version.out; tail -n 12 /tmp/archivebox-version.out; echo ABX_OK'
|
||||
default_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\nABX_GROUPS=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)" "$(id -Gn)"; touch /data/logs/probe /data/archive/probe "$LIB_DIR/probe" "$PERSONAS_DIR/Default/chrome_profile/probe"; stat -c "ABX_STAT %u:%g:%a %n" /data /data/logs /data/archive "$LIB_DIR" "$PERSONAS_DIR" "$PERSONAS_DIR/Default" "$PERSONAS_DIR/Default/chrome_profile"; echo ABX_PERSONA_PROFILE_OK; echo ABX_OK'
|
||||
version_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\nABX_GROUPS=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)" "$(id -Gn)"; archivebox version >/tmp/archivebox-version.out; tail -n 12 /tmp/archivebox-version.out; echo ABX_OK'
|
||||
full_flow_cmd='set -Eeuo pipefail
|
||||
printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\nABX_GROUPS=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)" "$(id -Gn)"
|
||||
id -Gn | grep -qw audio
|
||||
id -Gn | grep -qw video
|
||||
mkdir -p "$PERSONAS_DIR/Default/chrome_profile"
|
||||
touch "$PERSONAS_DIR/Default/chrome_profile/probe"
|
||||
rm -f "$PERSONAS_DIR/Default/chrome_profile/probe"
|
||||
archivebox init
|
||||
archivebox install 2>&1 | tee /tmp/archivebox-install.log
|
||||
if grep -E "(/[[:alnum:]_.-]+/)?pip install|npm install|uv pip install" /tmp/archivebox-install.log; then
|
||||
echo "ABX_UNEXPECTED_RUNTIME_INSTALL"
|
||||
exit 1
|
||||
fi
|
||||
archivebox version 2>&1 | tee /tmp/archivebox-version.log
|
||||
grep -Eq "/opt/archivebox/lib/(pip/venv/bin|env/bin)/trafilatura" /tmp/archivebox-version.log
|
||||
grep -Eq "/opt/archivebox/lib/(npm/node_modules/.bin|env/bin)/defuddle" /tmp/archivebox-version.log
|
||||
grep -Eq "/opt/archivebox/lib/env/bin/sonic" /tmp/archivebox-version.log
|
||||
archivebox add --depth=0 https://example.com/ 2>&1 | tee /tmp/archivebox-add.log
|
||||
archivebox update --index-only 2>&1 | tee /tmp/archivebox-update.log
|
||||
snapshot_dir="$(find /data/archive/users/system/snapshots -mindepth 3 -maxdepth 3 -type d | head -n 1)"
|
||||
test -n "$snapshot_dir"
|
||||
test -s "$snapshot_dir/index.html"
|
||||
test -s "$snapshot_dir/wget/example.com/index.html"
|
||||
test -s "$snapshot_dir/dom/output.html"
|
||||
test -s "$snapshot_dir/screenshot/screenshot.png"
|
||||
test -s "$snapshot_dir/pdf/output.pdf"
|
||||
test -s "$snapshot_dir/singlefile/singlefile.html"
|
||||
test -s "$snapshot_dir/headers/headers.json"
|
||||
test -s "$snapshot_dir/readability/content.txt"
|
||||
test -s "$snapshot_dir/trafilatura/content.txt"
|
||||
test -s "$snapshot_dir/defuddle/content.txt"
|
||||
test -s "$snapshot_dir/liteparse/content.txt"
|
||||
test -s "$snapshot_dir/responses/index.jsonl"
|
||||
test -s "$snapshot_dir/search_backend_sonic/on_Snapshot__91_index_sonic."*.sh
|
||||
test -s "$snapshot_dir/search_backend_sqlite/on_Snapshot__90_index_sqlite."*.sh
|
||||
grep -R "Example Domain" \
|
||||
"$snapshot_dir/wget/example.com/index.html" \
|
||||
"$snapshot_dir/dom/output.html" \
|
||||
"$snapshot_dir/readability/content.txt" \
|
||||
"$snapshot_dir/trafilatura/content.txt" \
|
||||
"$snapshot_dir/defuddle/content.txt" \
|
||||
"$snapshot_dir/liteparse/content.txt"
|
||||
if grep -R "Permission denied\\|Operation not permitted" /tmp/archivebox-add.log /tmp/archivebox-update.log /data/logs 2>/dev/null; then
|
||||
echo "ABX_PERMISSION_ERROR_IN_FULL_FLOW"
|
||||
exit 1
|
||||
fi
|
||||
find "$snapshot_dir" -maxdepth 2 -type f | sort | sed "s#^#ABX_OUTPUT #"
|
||||
echo ABX_OK'
|
||||
|
||||
run_case() {
|
||||
local name="$1"
|
||||
@ -161,6 +209,11 @@ run_case() {
|
||||
if ! grep -q '^ABX_OK$' "$log_file"; then
|
||||
ok=0
|
||||
fi
|
||||
if [[ "$user_spec" == "-" ]]; then
|
||||
grep '^ABX_GROUPS=' "$log_file" | grep -qw audio || ok=0
|
||||
grep '^ABX_GROUPS=' "$log_file" | grep -qw video || ok=0
|
||||
grep -q '^ABX_PERSONA_PROFILE_OK$' "$log_file" || ok=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$post_assert" == "nested-root-stays" ]]; then
|
||||
@ -188,6 +241,132 @@ run_case() {
|
||||
fi
|
||||
}
|
||||
|
||||
run_readonly_case() {
|
||||
local name="$1"
|
||||
local setup_script="$2"
|
||||
local env_string="$3"
|
||||
local user_spec="$4"
|
||||
|
||||
total=$((total + 1))
|
||||
local slug case_dir log_file status ok
|
||||
slug="$(safe_name "$name")"
|
||||
case_dir="$VALIDATION_ROOT/$slug"
|
||||
log_file="$case_dir/output.log"
|
||||
docker_setup "$case_dir" "$setup_script"
|
||||
|
||||
local run_args=("${docker_base[@]}")
|
||||
if [[ "$user_spec" != "-" ]]; then
|
||||
run_args+=(--user "$user_spec")
|
||||
fi
|
||||
run_args+=(
|
||||
-e DATA_DIR=/data
|
||||
-e LIB_DIR=/libdir
|
||||
-e ABXPKG_LIB_DIR=/libdir
|
||||
-e PLAYWRIGHT_BROWSERS_PATH=/browsers
|
||||
)
|
||||
if [[ -n "$env_string" && "$env_string" != "-" ]]; then
|
||||
local env_parts=()
|
||||
read -r -a env_parts <<< "$env_string"
|
||||
local env_pair
|
||||
for env_pair in "${env_parts[@]}"; do
|
||||
run_args+=(-e "$env_pair")
|
||||
done
|
||||
fi
|
||||
run_args+=(
|
||||
-v "$ENTRYPOINT_PATH:/app/bin/docker_entrypoint.sh:ro"
|
||||
-v "$case_dir/data:/data:ro"
|
||||
-v "$case_dir/lib:/libdir:ro"
|
||||
-v "$case_dir/browsers:/browsers:ro"
|
||||
--entrypoint /app/bin/docker_entrypoint.sh
|
||||
"$IMAGE"
|
||||
sh -c "$default_cmd"
|
||||
)
|
||||
|
||||
set +e
|
||||
"${run_args[@]}" >"$log_file" 2>&1
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
ok=0
|
||||
if [[ "$status" != "0" ]] && grep -q "cannot write to /data" "$log_file"; then
|
||||
ok=1
|
||||
fi
|
||||
if [[ "$ok" == "1" ]]; then
|
||||
passed=$((passed + 1))
|
||||
log "PASS $name"
|
||||
else
|
||||
failed=$((failed + 1))
|
||||
log "FAIL $name (status=$status expected=readonly failure log=$log_file)"
|
||||
sed -n '1,160p' "$log_file"
|
||||
fi
|
||||
}
|
||||
|
||||
run_full_flow_case() {
|
||||
local name="$1"
|
||||
local setup_script="$2"
|
||||
local env_string="$3"
|
||||
local user_spec="$4"
|
||||
local expected_uid="$5"
|
||||
local expected_gid="$6"
|
||||
|
||||
total=$((total + 1))
|
||||
local slug case_dir log_file status ok
|
||||
slug="$(safe_name "$name")"
|
||||
case_dir="$VALIDATION_ROOT/$slug"
|
||||
log_file="$case_dir/output.log"
|
||||
docker_setup "$case_dir" "$setup_script"
|
||||
|
||||
local run_args=("${docker_base[@]}")
|
||||
if [[ "$user_spec" != "-" ]]; then
|
||||
run_args+=(--user "$user_spec")
|
||||
fi
|
||||
run_args+=(
|
||||
-e DATA_DIR=/data
|
||||
)
|
||||
if [[ -n "$env_string" && "$env_string" != "-" ]]; then
|
||||
local env_parts=()
|
||||
read -r -a env_parts <<< "$env_string"
|
||||
local env_pair
|
||||
for env_pair in "${env_parts[@]}"; do
|
||||
run_args+=(-e "$env_pair")
|
||||
done
|
||||
fi
|
||||
run_args+=(
|
||||
-v "$ENTRYPOINT_PATH:/app/bin/docker_entrypoint.sh:ro"
|
||||
-v "$case_dir/data:/data"
|
||||
--entrypoint /app/bin/docker_entrypoint.sh
|
||||
"$IMAGE"
|
||||
bash -lc "$full_flow_cmd"
|
||||
)
|
||||
|
||||
set +e
|
||||
"${run_args[@]}" >"$log_file" 2>&1
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
ok=1
|
||||
[[ "$status" == "0" ]] || ok=0
|
||||
grep -q "^ABX_UID=$expected_uid$" "$log_file" || ok=0
|
||||
grep -q "^ABX_GID=$expected_gid$" "$log_file" || ok=0
|
||||
grep '^ABX_GROUPS=' "$log_file" | grep -qw audio || ok=0
|
||||
grep '^ABX_GROUPS=' "$log_file" | grep -qw video || ok=0
|
||||
grep -q '^ABX_OK$' "$log_file" || ok=0
|
||||
grep -q 'total urls snapshotted: 1' "$log_file" || ok=0
|
||||
grep -q 'Search Reindex Complete' "$log_file" || ok=0
|
||||
grep -q 'ABX_OUTPUT .*/wget/example.com/index.html' "$log_file" || ok=0
|
||||
grep -q 'ABX_OUTPUT .*/screenshot/screenshot.png' "$log_file" || ok=0
|
||||
grep -q 'ABX_OUTPUT .*/trafilatura/content.txt' "$log_file" || ok=0
|
||||
|
||||
if [[ "$ok" == "1" ]]; then
|
||||
passed=$((passed + 1))
|
||||
log "PASS $name"
|
||||
else
|
||||
failed=$((failed + 1))
|
||||
log "FAIL $name (status=$status expected=full-flow success log=$log_file)"
|
||||
sed -n '1,220p' "$log_file"
|
||||
fi
|
||||
}
|
||||
|
||||
run_mount_case() {
|
||||
local fs_name="$1"
|
||||
local mount_dir="$2"
|
||||
@ -264,9 +443,9 @@ run_case "non-root start writable root-owned data succeeds" \
|
||||
"chown 0:0 /case/data /case/lib && chmod 777 /case/data /case/lib" \
|
||||
"-" "501:911" pass 501 911
|
||||
|
||||
run_case "non-root start unwritable root-owned data hard-errors" \
|
||||
run_case "non-root start root-owned data with mapped write access succeeds" \
|
||||
"chown 0:0 /case/data /case/lib && chmod 700 /case/data /case/lib" \
|
||||
"-" "501:911" fail "" ""
|
||||
"-" "501:911" pass 501 911
|
||||
|
||||
run_case "root start fixes read-only top-level data when chmod works" \
|
||||
"chown 0:0 /case/data && chmod 555 /case/data" \
|
||||
@ -288,9 +467,9 @@ run_case "non-root user 501 can run archivebox version with root-owned LIB_DIR"
|
||||
"chown 0:0 /case/data /case/lib && chmod 777 /case/data && chmod 755 /case/lib" \
|
||||
"-" "501:911" pass 501 911 "$version_cmd"
|
||||
|
||||
run_case "non-root user 501 cannot write root-owned LIB_DIR" \
|
||||
run_case "non-root user 501 can write root-owned mapped LIB_DIR when access is granted" \
|
||||
"chown 0:0 /case/data /case/lib && chmod 777 /case/data && chmod 755 /case/lib" \
|
||||
"-" "501:911" fail "" ""
|
||||
"-" "501:911" pass 501 911
|
||||
|
||||
run_case "non-root user 501 can write forced-owner style LIB_DIR when permissions allow" \
|
||||
"chown 0:0 /case/data /case/lib && chmod 777 /case/data /case/lib" \
|
||||
@ -308,6 +487,14 @@ run_case "root-owned ArchiveBox.conf only is repaired" \
|
||||
"chown 911:911 /case/data && chmod 755 /case/data && touch /case/data/ArchiveBox.conf /case/data/index.sqlite3 && chown 0:0 /case/data/ArchiveBox.conf /case/data/index.sqlite3" \
|
||||
"PUID=911 PGID=911" "-" pass 911 911 "$default_cmd" config-files-repaired
|
||||
|
||||
run_readonly_case "readonly bind mount hard-errors only when truly unwritable" \
|
||||
"chown 911:911 /case/data /case/lib /case/browsers && chmod 755 /case/data /case/lib /case/browsers" \
|
||||
"PUID=911 PGID=911" "-"
|
||||
|
||||
run_full_flow_case "prebuilt image init install version add update example.com" \
|
||||
"chown 0:0 /case/data && chmod 755 /case/data" \
|
||||
"PUID=911 PGID=911" "-" 911 911
|
||||
|
||||
run_mount_case "NFS" "${NFS_TEST_DIR:-}"
|
||||
run_mount_case "SMB" "${SMB_TEST_DIR:-}"
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.31rc17",
|
||||
"version": "0.9.31rc18",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.31rc17"
|
||||
version = "0.9.31rc18"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@ -79,9 +79,9 @@ dependencies = [
|
||||
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
|
||||
### Binary/Package Management
|
||||
"abxbus>=2.5.4", # EventBus API
|
||||
"abxpkg>=1.10.9", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.10.76", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.10.76", # shared ArchiveBox downloader package with blocking install preflight
|
||||
"abxpkg>=1.10.10", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.10.77", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.10.77", # shared ArchiveBox downloader package with blocking install preflight
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user