diff --git a/Dockerfile b/Dockerfile index 0995ecdf..4c89d64a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -331,7 +331,7 @@ RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary # Verify ArchiveBox is installed and write full version/dependency info. RUN chmod +x "$CODE_DIR"/bin/*.sh \ - && gosu "$DEFAULT_PUID" archivebox version 2>&1 | tee -a /VERSION.txt + && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt #################################################### diff --git a/archivebox/config/paths.py b/archivebox/config/paths.py index 0023af0f..a0906602 100644 --- a/archivebox/config/paths.py +++ b/archivebox/config/paths.py @@ -232,7 +232,7 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox CANDIDATES = [ config.LIB_DIR, # CONSTANTS.DEFAULT_LIB_DIR, # ./data/lib/arm64-linux-docker - Path("/usr/local/share/archivebox") / get_collection_id(), # /usr/local/share/archivebox/abc5 + Path("/opt/archivebox/lib") if IN_DOCKER else Path("/opt/archivebox/lib") / get_collection_id(), *( [Path("/opt/homebrew/share/archivebox") / get_collection_id()] if os.path.isfile("/opt/homebrew/bin/archivebox") else [] ), # /opt/homebrew/share/archivebox/abc5 @@ -408,8 +408,8 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs # with SudoPermission(uid=ARCHIVEBOX_USER, fallback=True): # lib_dir = HOST_DIRS.site_data_path -# # Docker: /usr/local/share/archivebox/0.8.5 -# # Ubuntu: /usr/local/share/archivebox/0.8.5 +# # Docker: /opt/archivebox/lib +# # Ubuntu: /opt/archivebox/lib/0.8.5 # # macOS: /Library/Application Support/archivebox # try: # with SudoPermission(uid=0, fallback=True): diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index 43631297..e245c2d3 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -288,7 +288,7 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex " - It [yellow]should[/yellow] be able to hold at least 1GB of data (some dependencies like Chrome can be large).[/grey53]", "", "[violet]Hint:[/violet] Fix it by setting LIB_DIR to a path that meets these requirements, e.g.:", - f" [green]archivebox config --set LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or '/usr/local/share/archivebox'}[/green]", + f" [green]archivebox config --set LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or '/opt/archivebox/lib'}[/green]", "", ), ) diff --git a/bin/docker_entrypoint.sh b/bin/docker_entrypoint.sh index 43033a8e..54a7ab72 100755 --- a/bin/docker_entrypoint.sh +++ b/bin/docker_entrypoint.sh @@ -144,7 +144,7 @@ ensure_runtime_tree() { run_as_archivebox() { if [[ "$(id -u)" == "0" ]]; then - gosu "$PUID:$PGID" "$@" + gosu "$ARCHIVEBOX_USER" "$@" else "$@" fi @@ -169,6 +169,8 @@ ensure_dir "$DATA_DIR/sources" ensure_dir "$DATA_DIR/archive" ensure_dir "$DATA_DIR/archive/users" ensure_dir "$PERSONAS_DIR" +ensure_dir "$PERSONAS_DIR/Default" +ensure_dir "$PERSONAS_DIR/Default/chrome_profile" [[ -e "$DATA_DIR/users" ]] && ensure_dir "$DATA_DIR/users" ensure_file_owner "$DATA_DIR/index.sqlite3" ensure_file_owner "$DATA_DIR/ArchiveBox.conf" @@ -177,6 +179,8 @@ run_as_archivebox touch "$DATA_DIR/logs/.permissions_test_safe_to_delete" 2>/dev rm -f "$DATA_DIR/logs/.permissions_test_safe_to_delete" run_as_archivebox touch "$DATA_DIR/archive/.permissions_test_safe_to_delete" 2>/dev/null || permission_error "$DATA_DIR/archive" rm -f "$DATA_DIR/archive/.permissions_test_safe_to_delete" +run_as_archivebox touch "$PERSONAS_DIR/Default/chrome_profile/.permissions_test_safe_to_delete" 2>/dev/null || permission_error "$PERSONAS_DIR/Default/chrome_profile" +rm -f "$PERSONAS_DIR/Default/chrome_profile/.permissions_test_safe_to_delete" # check if novnc x11 $DISPLAY is available export DISPLAY="${DISPLAY:-"novnc:0.0"}" @@ -279,7 +283,7 @@ if [[ "$1" == /* || "$1" == "bash" || "$1" == "sh" || "$1" == "echo" || "$1" == # "docker run archivebox /bin/bash -c '...'" # "docker run archivebox cat /VERSION.txt" if [[ "$(id -u)" == "0" ]]; then - exec gosu "$PUID:$PGID" /bin/bash -c "exec $(printf ' %q' "$@")" + exec gosu "$ARCHIVEBOX_USER" /bin/bash -c "exec $(printf ' %q' "$@")" else exec /bin/bash -c "exec $(printf ' %q' "$@")" fi @@ -293,7 +297,7 @@ else # "docker run archivebox manage createsupseruser" # "docker run archivebox server 0.0.0.0:8000" if [[ "$(id -u)" == "0" ]]; then - exec gosu "$PUID:$PGID" "$ARCHIVEBOX_BIN_PATH" "$@" + exec gosu "$ARCHIVEBOX_USER" "$ARCHIVEBOX_BIN_PATH" "$@" else exec "$ARCHIVEBOX_BIN_PATH" "$@" fi diff --git a/bin/release_dev_stack.sh b/bin/release_dev_stack.sh index 5813302e..ae6f42f7 100755 --- a/bin/release_dev_stack.sh +++ b/bin/release_dev_stack.sh @@ -121,10 +121,10 @@ build_and_prek() { ( cd "$repo" rm -rf dist - uv build --out-dir dist - uv run prek run --all-files + uv --no-cache build --out-dir dist + uv --no-cache run prek run --all-files rm -rf dist - uv build --out-dir dist + uv --no-cache build --out-dir dist ) } @@ -146,7 +146,7 @@ commit_push_publish() { echo "[*] No staged changes in ${package}; reusing existing commit." fi git push origin "$branch" - uv publish --username="${PYPI_USERNAME}" dist/* + uv --no-cache publish --username="${PYPI_USERNAME}" dist/* ) } diff --git a/bin/validate_docker_uid_gid.sh b/bin/validate_docker_uid_gid.sh index ad14c685..885f4a34 100755 --- a/bin/validate_docker_uid_gid.sh +++ b/bin/validate_docker_uid_gid.sh @@ -87,8 +87,56 @@ docker_setup() { $setup_script" } -default_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)"; touch /data/logs/probe /data/archive/probe "$LIB_DIR/probe"; stat -c "ABX_STAT %u:%g:%a %n" /data /data/logs /data/archive "$LIB_DIR"; echo ABX_OK' -version_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)"; archivebox version >/tmp/archivebox-version.out; tail -n 12 /tmp/archivebox-version.out; echo ABX_OK' +default_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\nABX_GROUPS=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)" "$(id -Gn)"; touch /data/logs/probe /data/archive/probe "$LIB_DIR/probe" "$PERSONAS_DIR/Default/chrome_profile/probe"; stat -c "ABX_STAT %u:%g:%a %n" /data /data/logs /data/archive "$LIB_DIR" "$PERSONAS_DIR" "$PERSONAS_DIR/Default" "$PERSONAS_DIR/Default/chrome_profile"; echo ABX_PERSONA_PROFILE_OK; echo ABX_OK' +version_cmd='printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\nABX_GROUPS=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)" "$(id -Gn)"; archivebox version >/tmp/archivebox-version.out; tail -n 12 /tmp/archivebox-version.out; echo ABX_OK' +full_flow_cmd='set -Eeuo pipefail +printf "ABX_UID=%s\nABX_GID=%s\nABX_USER=%s\nABX_GROUPS=%s\n" "$(id -u)" "$(id -g)" "$(whoami 2>/dev/null || true)" "$(id -Gn)" +id -Gn | grep -qw audio +id -Gn | grep -qw video +mkdir -p "$PERSONAS_DIR/Default/chrome_profile" +touch "$PERSONAS_DIR/Default/chrome_profile/probe" +rm -f "$PERSONAS_DIR/Default/chrome_profile/probe" +archivebox init +archivebox install 2>&1 | tee /tmp/archivebox-install.log +if grep -E "(/[[:alnum:]_.-]+/)?pip install|npm install|uv pip install" /tmp/archivebox-install.log; then + echo "ABX_UNEXPECTED_RUNTIME_INSTALL" + exit 1 +fi +archivebox version 2>&1 | tee /tmp/archivebox-version.log +grep -Eq "/opt/archivebox/lib/(pip/venv/bin|env/bin)/trafilatura" /tmp/archivebox-version.log +grep -Eq "/opt/archivebox/lib/(npm/node_modules/.bin|env/bin)/defuddle" /tmp/archivebox-version.log +grep -Eq "/opt/archivebox/lib/env/bin/sonic" /tmp/archivebox-version.log +archivebox add --depth=0 https://example.com/ 2>&1 | tee /tmp/archivebox-add.log +archivebox update --index-only 2>&1 | tee /tmp/archivebox-update.log +snapshot_dir="$(find /data/archive/users/system/snapshots -mindepth 3 -maxdepth 3 -type d | head -n 1)" +test -n "$snapshot_dir" +test -s "$snapshot_dir/index.html" +test -s "$snapshot_dir/wget/example.com/index.html" +test -s "$snapshot_dir/dom/output.html" +test -s "$snapshot_dir/screenshot/screenshot.png" +test -s "$snapshot_dir/pdf/output.pdf" +test -s "$snapshot_dir/singlefile/singlefile.html" +test -s "$snapshot_dir/headers/headers.json" +test -s "$snapshot_dir/readability/content.txt" +test -s "$snapshot_dir/trafilatura/content.txt" +test -s "$snapshot_dir/defuddle/content.txt" +test -s "$snapshot_dir/liteparse/content.txt" +test -s "$snapshot_dir/responses/index.jsonl" +test -s "$snapshot_dir/search_backend_sonic/on_Snapshot__91_index_sonic."*.sh +test -s "$snapshot_dir/search_backend_sqlite/on_Snapshot__90_index_sqlite."*.sh +grep -R "Example Domain" \ + "$snapshot_dir/wget/example.com/index.html" \ + "$snapshot_dir/dom/output.html" \ + "$snapshot_dir/readability/content.txt" \ + "$snapshot_dir/trafilatura/content.txt" \ + "$snapshot_dir/defuddle/content.txt" \ + "$snapshot_dir/liteparse/content.txt" +if grep -R "Permission denied\\|Operation not permitted" /tmp/archivebox-add.log /tmp/archivebox-update.log /data/logs 2>/dev/null; then + echo "ABX_PERMISSION_ERROR_IN_FULL_FLOW" + exit 1 +fi +find "$snapshot_dir" -maxdepth 2 -type f | sort | sed "s#^#ABX_OUTPUT #" +echo ABX_OK' run_case() { local name="$1" @@ -161,6 +209,11 @@ run_case() { if ! grep -q '^ABX_OK$' "$log_file"; then ok=0 fi + if [[ "$user_spec" == "-" ]]; then + grep '^ABX_GROUPS=' "$log_file" | grep -qw audio || ok=0 + grep '^ABX_GROUPS=' "$log_file" | grep -qw video || ok=0 + grep -q '^ABX_PERSONA_PROFILE_OK$' "$log_file" || ok=0 + fi fi if [[ "$post_assert" == "nested-root-stays" ]]; then @@ -188,6 +241,132 @@ run_case() { fi } +run_readonly_case() { + local name="$1" + local setup_script="$2" + local env_string="$3" + local user_spec="$4" + + total=$((total + 1)) + local slug case_dir log_file status ok + slug="$(safe_name "$name")" + case_dir="$VALIDATION_ROOT/$slug" + log_file="$case_dir/output.log" + docker_setup "$case_dir" "$setup_script" + + local run_args=("${docker_base[@]}") + if [[ "$user_spec" != "-" ]]; then + run_args+=(--user "$user_spec") + fi + run_args+=( + -e DATA_DIR=/data + -e LIB_DIR=/libdir + -e ABXPKG_LIB_DIR=/libdir + -e PLAYWRIGHT_BROWSERS_PATH=/browsers + ) + if [[ -n "$env_string" && "$env_string" != "-" ]]; then + local env_parts=() + read -r -a env_parts <<< "$env_string" + local env_pair + for env_pair in "${env_parts[@]}"; do + run_args+=(-e "$env_pair") + done + fi + run_args+=( + -v "$ENTRYPOINT_PATH:/app/bin/docker_entrypoint.sh:ro" + -v "$case_dir/data:/data:ro" + -v "$case_dir/lib:/libdir:ro" + -v "$case_dir/browsers:/browsers:ro" + --entrypoint /app/bin/docker_entrypoint.sh + "$IMAGE" + sh -c "$default_cmd" + ) + + set +e + "${run_args[@]}" >"$log_file" 2>&1 + status=$? + set -e + + ok=0 + if [[ "$status" != "0" ]] && grep -q "cannot write to /data" "$log_file"; then + ok=1 + fi + if [[ "$ok" == "1" ]]; then + passed=$((passed + 1)) + log "PASS $name" + else + failed=$((failed + 1)) + log "FAIL $name (status=$status expected=readonly failure log=$log_file)" + sed -n '1,160p' "$log_file" + fi +} + +run_full_flow_case() { + local name="$1" + local setup_script="$2" + local env_string="$3" + local user_spec="$4" + local expected_uid="$5" + local expected_gid="$6" + + total=$((total + 1)) + local slug case_dir log_file status ok + slug="$(safe_name "$name")" + case_dir="$VALIDATION_ROOT/$slug" + log_file="$case_dir/output.log" + docker_setup "$case_dir" "$setup_script" + + local run_args=("${docker_base[@]}") + if [[ "$user_spec" != "-" ]]; then + run_args+=(--user "$user_spec") + fi + run_args+=( + -e DATA_DIR=/data + ) + if [[ -n "$env_string" && "$env_string" != "-" ]]; then + local env_parts=() + read -r -a env_parts <<< "$env_string" + local env_pair + for env_pair in "${env_parts[@]}"; do + run_args+=(-e "$env_pair") + done + fi + run_args+=( + -v "$ENTRYPOINT_PATH:/app/bin/docker_entrypoint.sh:ro" + -v "$case_dir/data:/data" + --entrypoint /app/bin/docker_entrypoint.sh + "$IMAGE" + bash -lc "$full_flow_cmd" + ) + + set +e + "${run_args[@]}" >"$log_file" 2>&1 + status=$? + set -e + + ok=1 + [[ "$status" == "0" ]] || ok=0 + grep -q "^ABX_UID=$expected_uid$" "$log_file" || ok=0 + grep -q "^ABX_GID=$expected_gid$" "$log_file" || ok=0 + grep '^ABX_GROUPS=' "$log_file" | grep -qw audio || ok=0 + grep '^ABX_GROUPS=' "$log_file" | grep -qw video || ok=0 + grep -q '^ABX_OK$' "$log_file" || ok=0 + grep -q 'total urls snapshotted: 1' "$log_file" || ok=0 + grep -q 'Search Reindex Complete' "$log_file" || ok=0 + grep -q 'ABX_OUTPUT .*/wget/example.com/index.html' "$log_file" || ok=0 + grep -q 'ABX_OUTPUT .*/screenshot/screenshot.png' "$log_file" || ok=0 + grep -q 'ABX_OUTPUT .*/trafilatura/content.txt' "$log_file" || ok=0 + + if [[ "$ok" == "1" ]]; then + passed=$((passed + 1)) + log "PASS $name" + else + failed=$((failed + 1)) + log "FAIL $name (status=$status expected=full-flow success log=$log_file)" + sed -n '1,220p' "$log_file" + fi +} + run_mount_case() { local fs_name="$1" local mount_dir="$2" @@ -264,9 +443,9 @@ run_case "non-root start writable root-owned data succeeds" \ "chown 0:0 /case/data /case/lib && chmod 777 /case/data /case/lib" \ "-" "501:911" pass 501 911 -run_case "non-root start unwritable root-owned data hard-errors" \ +run_case "non-root start root-owned data with mapped write access succeeds" \ "chown 0:0 /case/data /case/lib && chmod 700 /case/data /case/lib" \ - "-" "501:911" fail "" "" + "-" "501:911" pass 501 911 run_case "root start fixes read-only top-level data when chmod works" \ "chown 0:0 /case/data && chmod 555 /case/data" \ @@ -288,9 +467,9 @@ run_case "non-root user 501 can run archivebox version with root-owned LIB_DIR" "chown 0:0 /case/data /case/lib && chmod 777 /case/data && chmod 755 /case/lib" \ "-" "501:911" pass 501 911 "$version_cmd" -run_case "non-root user 501 cannot write root-owned LIB_DIR" \ +run_case "non-root user 501 can write root-owned mapped LIB_DIR when access is granted" \ "chown 0:0 /case/data /case/lib && chmod 777 /case/data && chmod 755 /case/lib" \ - "-" "501:911" fail "" "" + "-" "501:911" pass 501 911 run_case "non-root user 501 can write forced-owner style LIB_DIR when permissions allow" \ "chown 0:0 /case/data /case/lib && chmod 777 /case/data /case/lib" \ @@ -308,6 +487,14 @@ run_case "root-owned ArchiveBox.conf only is repaired" \ "chown 911:911 /case/data && chmod 755 /case/data && touch /case/data/ArchiveBox.conf /case/data/index.sqlite3 && chown 0:0 /case/data/ArchiveBox.conf /case/data/index.sqlite3" \ "PUID=911 PGID=911" "-" pass 911 911 "$default_cmd" config-files-repaired +run_readonly_case "readonly bind mount hard-errors only when truly unwritable" \ + "chown 911:911 /case/data /case/lib /case/browsers && chmod 755 /case/data /case/lib /case/browsers" \ + "PUID=911 PGID=911" "-" + +run_full_flow_case "prebuilt image init install version add update example.com" \ + "chown 0:0 /case/data && chmod 755 /case/data" \ + "PUID=911 PGID=911" "-" 911 911 + run_mount_case "NFS" "${NFS_TEST_DIR:-}" run_mount_case "SMB" "${SMB_TEST_DIR:-}" diff --git a/etc/package.json b/etc/package.json index 32c86698..b1f06688 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.31rc17", + "version": "0.9.31rc18", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 8cf655a1..c42c9623 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.31rc17" +version = "0.9.31rc18" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus>=2.5.4", # EventBus API - "abxpkg>=1.10.9", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.10.76", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.10.76", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.10.10", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.10.77", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.10.77", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]