mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
Exercise ArchiveBox through unified runtime lifecycles
This commit is contained in:
Nick Sweeting
parent
e89388b264
commit
6592af0d19
51
.github/configs/ci-linux-build.json
vendored
Normal file
51
.github/configs/ci-linux-build.json
vendored
Normal file
@ -0,0 +1,51 @@
|
||||
{
|
||||
"properties": {
|
||||
"CC_BINARY": {
|
||||
"default": "cc"
|
||||
},
|
||||
"LDAPSEARCH_BINARY": {
|
||||
"default": "ldapsearch"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{CC_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"build-essential"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{LDAPSEARCH_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapsearch",
|
||||
"-VV"
|
||||
]
|
||||
},
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"ldap-utils",
|
||||
"python3-dev",
|
||||
"python3-setuptools",
|
||||
"libssl-dev",
|
||||
"libldap2-dev",
|
||||
"libsasl2-dev",
|
||||
"zlib1g-dev",
|
||||
"libatomic1"
|
||||
],
|
||||
"version": [
|
||||
"ldapsearch",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
28
.github/configs/ci-linux-env.json
vendored
Normal file
28
.github/configs/ci-linux-env.json
vendored
Normal file
@ -0,0 +1,28 @@
|
||||
{
|
||||
"properties": {
|
||||
"CC_BINARY": {
|
||||
"default": "cc"
|
||||
},
|
||||
"LDAPSEARCH_BINARY": {
|
||||
"default": "ldapsearch"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{CC_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{LDAPSEARCH_BINARY}",
|
||||
"binproviders": "env",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapsearch",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
13
.github/configs/ci-macos-brew.json
vendored
Normal file
13
.github/configs/ci-macos-brew.json
vendored
Normal file
@ -0,0 +1,13 @@
|
||||
{
|
||||
"properties": {
|
||||
"BREW_BINARY": {
|
||||
"default": "brew"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{BREW_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
]
|
||||
}
|
||||
44
.github/configs/ci-macos-build.json
vendored
Normal file
44
.github/configs/ci-macos-build.json
vendored
Normal file
@ -0,0 +1,44 @@
|
||||
{
|
||||
"properties": {
|
||||
"CLANG_BINARY": {
|
||||
"default": "clang"
|
||||
},
|
||||
"LDAPVC_BINARY": {
|
||||
"default": "ldapvc"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{CLANG_BINARY}",
|
||||
"binproviders": "env,brew",
|
||||
"overrides": {
|
||||
"brew": {
|
||||
"install_args": [
|
||||
"llvm"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{LDAPVC_BINARY}",
|
||||
"binproviders": "env,brew",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
},
|
||||
"brew": {
|
||||
"install_args": [
|
||||
"openldap"
|
||||
],
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
21
.github/configs/ci-macos-openldap-env.json
vendored
Normal file
21
.github/configs/ci-macos-openldap-env.json
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"properties": {
|
||||
"LDAPVC_BINARY": {
|
||||
"default": "ldapvc"
|
||||
}
|
||||
},
|
||||
"required_binaries": [
|
||||
{
|
||||
"name": "{LDAPVC_BINARY}",
|
||||
"binproviders": "env",
|
||||
"overrides": {
|
||||
"env": {
|
||||
"version": [
|
||||
"ldapvc",
|
||||
"-VV"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
321
.github/configs/ci-tooling.json
vendored
Normal file
321
.github/configs/ci-tooling.json
vendored
Normal file
@ -0,0 +1,321 @@
|
||||
{
|
||||
"properties": {
|
||||
"CURL_BINARY": {
|
||||
"default": "curl"
|
||||
},
|
||||
"BASH_BINARY": {
|
||||
"default": "bash"
|
||||
},
|
||||
"CHMOD_BINARY": {
|
||||
"default": "chmod"
|
||||
},
|
||||
"DOCKER_BINARY": {
|
||||
"default": "docker"
|
||||
},
|
||||
"GIT_BINARY": {
|
||||
"default": "git"
|
||||
},
|
||||
"GREP_BINARY": {
|
||||
"default": "grep"
|
||||
},
|
||||
"GH_BINARY": {
|
||||
"default": "gh"
|
||||
},
|
||||
"EXPORTFS_BINARY": {
|
||||
"default": "exportfs"
|
||||
},
|
||||
"JQ_BINARY": {
|
||||
"default": "jq"
|
||||
},
|
||||
"MKDIR_BINARY": {
|
||||
"default": "mkdir"
|
||||
},
|
||||
"MOUNT_CIFS_BINARY": {
|
||||
"default": "mount.cifs"
|
||||
},
|
||||
"MOUNT_NFS_BINARY": {
|
||||
"default": "mount.nfs"
|
||||
},
|
||||
"NODE_BINARY": {
|
||||
"default": "node"
|
||||
},
|
||||
"PS_BINARY": {
|
||||
"default": "ps"
|
||||
},
|
||||
"PYTHON_BINARY": {
|
||||
"default": "python"
|
||||
},
|
||||
"PV_BINARY": {
|
||||
"default": "pv"
|
||||
},
|
||||
"SSH_BINARY": {
|
||||
"default": "ssh"
|
||||
},
|
||||
"SED_BINARY": {
|
||||
"default": "sed"
|
||||
},
|
||||
"SMBD_BINARY": {
|
||||
"default": "smbd"
|
||||
},
|
||||
"SUDO_BINARY": {
|
||||
"default": "sudo"
|
||||
},
|
||||
"SYSTEMCTL_BINARY": {
|
||||
"default": "systemctl"
|
||||
},
|
||||
"TAR_BINARY": {
|
||||
"default": "tar"
|
||||
},
|
||||
"TREE_BINARY": {
|
||||
"default": "tree"
|
||||
},
|
||||
"TR_BINARY": {
|
||||
"default": "tr"
|
||||
},
|
||||
"RM_BINARY": {
|
||||
"default": "rm"
|
||||
},
|
||||
"UMOUNT_BINARY": {
|
||||
"default": "umount"
|
||||
},
|
||||
"UV_BINARY": {
|
||||
"default": "uv"
|
||||
},
|
||||
"UNAME_BINARY": {
|
||||
"default": "uname"
|
||||
}
|
||||
},
|
||||
"ci_binaries": [
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{JQ_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{UV_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"docker_binaries": [
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UNAME_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"deploy_binaries": [
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SSH_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"docker_debug_binaries": [
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{PV_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{TAR_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{TREE_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
}
|
||||
],
|
||||
"docker_validation_binaries": [
|
||||
{
|
||||
"name": "{DOCKER_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GREP_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{MKDIR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{RM_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SED_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SSH_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{TR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UNAME_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"mount_validation_binaries": [
|
||||
{
|
||||
"name": "{BASH_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{CHMOD_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{EXPORTFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"nfs-kernel-server"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{MKDIR_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{MOUNT_CIFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"cifs-utils"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{MOUNT_NFS_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"nfs-common"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{SMBD_BINARY}",
|
||||
"binproviders": "env,apt",
|
||||
"overrides": {
|
||||
"apt": {
|
||||
"install_args": [
|
||||
"samba"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "{SUDO_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{SYSTEMCTL_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{UMOUNT_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"process_binaries": [
|
||||
{
|
||||
"name": "{PS_BINARY}",
|
||||
"binproviders": "env"
|
||||
}
|
||||
],
|
||||
"docs_binaries": [
|
||||
{
|
||||
"name": "{CURL_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
}
|
||||
],
|
||||
"git_binaries": [
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
}
|
||||
],
|
||||
"github_binaries": [
|
||||
{
|
||||
"name": "{GH_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
}
|
||||
],
|
||||
"node_binaries": [
|
||||
{
|
||||
"name": "{NODE_BINARY}",
|
||||
"binproviders": "env,npm",
|
||||
"min_version": "22.23.1",
|
||||
"overrides": {
|
||||
"npm": {
|
||||
"install_args": [
|
||||
"node@22.23.1"
|
||||
],
|
||||
"postinstall_scripts": true
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"release_binaries": [
|
||||
{
|
||||
"name": "{UV_BINARY}",
|
||||
"binproviders": "env,pip,brew"
|
||||
},
|
||||
{
|
||||
"name": "{PYTHON_BINARY}",
|
||||
"binproviders": "env"
|
||||
},
|
||||
{
|
||||
"name": "{GH_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{GIT_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{JQ_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
},
|
||||
{
|
||||
"name": "{CURL_BINARY}",
|
||||
"binproviders": "env,apt,brew"
|
||||
}
|
||||
]
|
||||
}
|
||||
55
.github/scripts/clone_abx_repo.sh
vendored
55
.github/scripts/clone_abx_repo.sh
vendored
@ -3,25 +3,44 @@ set -Eeuo pipefail
|
||||
|
||||
repo_name="$1"
|
||||
target_dir="${2:-$repo_name}"
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
lock_file="$repo_root/uv.lock"
|
||||
tooling_config="$repo_root/.github/configs/ci-tooling.json"
|
||||
|
||||
version="$(
|
||||
python3 - "$repo_name" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
locked_version() {
|
||||
local wanted="$1" line package=""
|
||||
while IFS= read -r line; do
|
||||
case "$line" in
|
||||
'[[package]]') package="" ;;
|
||||
"name = \"${wanted}\"") package="$wanted" ;;
|
||||
'version = "'*'"')
|
||||
if [[ "$package" == "$wanted" ]]; then
|
||||
line="${line#version = \"}"
|
||||
printf '%s\n' "${line%\"}"
|
||||
return 0
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done < "$lock_file"
|
||||
return 1
|
||||
}
|
||||
|
||||
repo_name = sys.argv[1]
|
||||
lock_text = Path("uv.lock").read_text()
|
||||
match = re.search(
|
||||
rf'^\[\[package\]\]\s*\nname = "{re.escape(repo_name)}"\s*\nversion = "([^"]+)"',
|
||||
lock_text,
|
||||
re.MULTILINE,
|
||||
)
|
||||
if not match:
|
||||
raise SystemExit(f"Could not find {repo_name} in uv.lock")
|
||||
print(match.group(1))
|
||||
PY
|
||||
)"
|
||||
version="$(locked_version "$repo_name")"
|
||||
[[ -n "$version" ]] || { echo "Could not find ${repo_name} in uv.lock" >&2; exit 1; }
|
||||
|
||||
abxpkg_version="$(locked_version abxpkg)"
|
||||
[[ -n "$abxpkg_version" ]] || { echo "Could not find abxpkg in uv.lock" >&2; exit 1; }
|
||||
|
||||
ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-${RUNNER_TEMP:-/tmp}/archivebox-clone-abxpkg}"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-project --with "abxpkg==$abxpkg_version" abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$tooling_config:git_binaries" \
|
||||
>/dev/null
|
||||
git_binary="$ABXPKG_LIB_DIR/env/bin/git"
|
||||
[[ -L "$git_binary" ]]
|
||||
[[ -x "$git_binary" ]]
|
||||
|
||||
echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
|
||||
git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"
|
||||
"$git_binary" clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"
|
||||
|
||||
109
.github/scripts/discover_test_matrix.py
vendored
Executable file
109
.github/scripts/discover_test_matrix.py
vendored
Executable file
@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build deterministic GitHub Actions matrices from every discovered test file."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CHROMIUM_PATTERN = re.compile(
|
||||
rb"chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
SONIC_PATTERN = re.compile(
|
||||
rb"""shutil\.which\(["']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic""",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def contains(pattern: re.Pattern[bytes], paths: list[Path]) -> bool:
|
||||
return any(pattern.search(path.read_bytes()) for path in paths)
|
||||
|
||||
|
||||
def archivebox_matrix(root: Path) -> list[dict[str, object]]:
|
||||
tests = sorted((root / "archivebox/tests").glob("test_*.py"))
|
||||
if not tests:
|
||||
raise SystemExit("No ArchiveBox tests discovered")
|
||||
|
||||
shard_count = min(16, len(tests))
|
||||
matrix = []
|
||||
assigned: list[Path] = []
|
||||
for shard in range(shard_count):
|
||||
shard_tests = tests[shard::shard_count]
|
||||
assigned.extend(shard_tests)
|
||||
matrix.append(
|
||||
{
|
||||
"name": f"main/shard-{shard + 1}",
|
||||
"paths": [path.relative_to(root).as_posix() for path in shard_tests],
|
||||
"needs_chromium": contains(CHROMIUM_PATTERN, shard_tests),
|
||||
"needs_sonic": contains(SONIC_PATTERN, shard_tests),
|
||||
},
|
||||
)
|
||||
|
||||
if sorted(assigned) != tests or len(assigned) != len(set(assigned)):
|
||||
raise SystemExit("ArchiveBox tests were not assigned exactly once")
|
||||
print(f"Assigned {len(tests)} test files exactly once across {shard_count} shards")
|
||||
return matrix
|
||||
|
||||
|
||||
def plugin_matrix(root: Path) -> list[dict[str, object]]:
|
||||
plugins_root = root / "abx-plugins/abx_plugins/plugins"
|
||||
suite_dirs = sorted(path for path in plugins_root.glob("*/tests") if path.is_dir())
|
||||
root_tests = sorted((root / "abx-plugins/tests").glob("test_*.py"))
|
||||
if not suite_dirs or not root_tests:
|
||||
raise SystemExit("Plugin suites or root tests were not discovered")
|
||||
|
||||
matrix: list[dict[str, object]] = []
|
||||
expected = list(root_tests)
|
||||
for suite_dir in suite_dirs:
|
||||
suite_tests = sorted(suite_dir.rglob("test_*.py"))
|
||||
if not suite_tests:
|
||||
raise SystemExit(f"No tests found in {suite_dir}")
|
||||
expected.extend(suite_tests)
|
||||
plugin = suite_dir.parent.name
|
||||
matrix.append(
|
||||
{
|
||||
"plugin": plugin,
|
||||
"name": f"plugin/{plugin}",
|
||||
"test_path": suite_dir.relative_to(root).as_posix(),
|
||||
"config_path": (suite_dir.parent / "config.json").relative_to(root).as_posix(),
|
||||
"needs_chromium": contains(re.compile(rb"chrom", re.IGNORECASE), suite_tests),
|
||||
"needs_sonic": plugin == "search_backend_sonic",
|
||||
},
|
||||
)
|
||||
|
||||
matrix.append(
|
||||
{
|
||||
"plugin": "root",
|
||||
"name": "plugin/root",
|
||||
"test_path": "abx-plugins/tests",
|
||||
"config_path": "abx-plugins/abx_plugins/plugins/base/config.json",
|
||||
"needs_chromium": contains(re.compile(rb"chrom|archivewebpage", re.IGNORECASE), root_tests),
|
||||
"needs_sonic": False,
|
||||
},
|
||||
)
|
||||
|
||||
assigned = []
|
||||
for entry in matrix:
|
||||
assigned.extend(sorted((root / str(entry["test_path"])).rglob("test_*.py")))
|
||||
if sorted(assigned) != sorted(expected) or len(assigned) != len(set(assigned)):
|
||||
raise SystemExit("Plugin tests were not assigned exactly once")
|
||||
print(
|
||||
f"Assigned {len(suite_dirs)} plugin suites and {len(root_tests)} root test files exactly once",
|
||||
)
|
||||
return matrix
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("scope", choices=("archivebox", "plugins"))
|
||||
parser.add_argument("--workspace", type=Path, default=Path.cwd())
|
||||
args = parser.parse_args()
|
||||
root = args.workspace.resolve()
|
||||
matrix = archivebox_matrix(root) if args.scope == "archivebox" else plugin_matrix(root)
|
||||
print(json.dumps(matrix, separators=(",", ":")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
34
.github/scripts/docs_http_server.py
vendored
Normal file
34
.github/scripts/docs_http_server.py
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class DocsRequestHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None:
|
||||
body = f"<!doctype html><title>ArchiveBox docs fixture</title><p>{self.path}</p>\n".encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ready-fifo", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
with ThreadingHTTPServer(("127.0.0.1", 0), DocsRequestHandler) as server:
|
||||
host, port = server.server_address
|
||||
with args.ready_fifo.open("w") as ready_fifo:
|
||||
ready_fifo.write(f"http://{host}:{port}\n")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
70
.github/workflows/ci.yml
vendored
Normal file
70
.github/workflows/ci.yml
vendored
Normal file
@ -0,0 +1,70 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [dev]
|
||||
push:
|
||||
branches: [dev]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
security-events: write
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Linters
|
||||
uses: ./.github/workflows/lint.yml
|
||||
secrets: inherit
|
||||
|
||||
complete-tests:
|
||||
name: Complete test suite
|
||||
uses: ./.github/workflows/test.yml
|
||||
secrets: inherit
|
||||
|
||||
sharded-plugin-tests:
|
||||
name: Sharded and plugin tests
|
||||
uses: ./.github/workflows/test-parallel.yml
|
||||
secrets: inherit
|
||||
|
||||
documentation:
|
||||
name: Documentation and root tests
|
||||
uses: ./.github/workflows/docs.yml
|
||||
secrets: inherit
|
||||
|
||||
codeql:
|
||||
name: CodeQL
|
||||
uses: ./.github/workflows/codeql.yml
|
||||
secrets: inherit
|
||||
|
||||
python-artifacts:
|
||||
name: Tested Python artifacts
|
||||
uses: ./.github/workflows/pip.yml
|
||||
secrets: inherit
|
||||
|
||||
docker-digests:
|
||||
name: Tested Docker images
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
push_digests: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' }}
|
||||
secrets: inherit
|
||||
|
||||
required:
|
||||
name: All required CI lanes
|
||||
needs:
|
||||
- lint
|
||||
- complete-tests
|
||||
- sharded-plugin-tests
|
||||
- documentation
|
||||
- codeql
|
||||
- python-artifacts
|
||||
- docker-digests
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- run: echo 'All required CI lanes and tested release artifacts succeeded.'
|
||||
17
.github/workflows/claude.yml
vendored
17
.github/workflows/claude.yml
vendored
@ -17,33 +17,24 @@ jobs:
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
id-token: write
|
||||
actions: read # Required for Claude to read CI results on PRs
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@v1
|
||||
uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
# This is an optional setting that allows Claude to read CI results on PRs
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
|
||||
# Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
|
||||
# prompt: 'Update the pull request description to include a summary of changes.'
|
||||
|
||||
# Optional: Add claude_args to customize behavior and configuration
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
claude_args: '--allowed-tools Bash(gh pr:*)'
|
||||
|
||||
72
.github/workflows/codeql.yml
vendored
72
.github/workflows/codeql.yml
vendored
@ -1,46 +1,17 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "dev" ]
|
||||
pull_request:
|
||||
branches: [ "dev" ]
|
||||
workflow_call:
|
||||
schedule:
|
||||
- cron: '33 17 * * 6'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
||||
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners (GitHub.com only)
|
||||
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
# required for all workflows
|
||||
security-events: write
|
||||
|
||||
# required to fetch internal or private CodeQL packs
|
||||
packages: read
|
||||
|
||||
# only required for workflows in private repositories
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
@ -50,47 +21,18 @@ jobs:
|
||||
include:
|
||||
- language: python
|
||||
build-mode: none
|
||||
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
|
||||
# Use `c-cpp` to analyze code written in C, C++ or both
|
||||
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
|
||||
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
|
||||
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
|
||||
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
|
||||
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
|
||||
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
||||
- language: javascript-typescript
|
||||
build-mode: none
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
# If the analyze step fails for one of the languages you are analyzing with
|
||||
# "We were unable to automatically build your code", modify the matrix above
|
||||
# to set the build mode to "manual" for that language. Then modify this step
|
||||
# to build your code.
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
- if: matrix.build-mode == 'manual'
|
||||
run: |
|
||||
echo 'If you are using a "manual" build mode for one or more of the' \
|
||||
'languages you are analyzing, replace this with the commands to build' \
|
||||
'your code, for example:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
|
||||
14
.github/workflows/deploy-publicsite.yml
vendored
14
.github/workflows/deploy-publicsite.yml
vendored
@ -15,29 +15,29 @@ permissions:
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: github-pages-publicsite
|
||||
cancel-in-progress: true
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }}
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v5
|
||||
uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||
with:
|
||||
path: ./publicsite
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
|
||||
360
.github/workflows/docker.yml
vendored
360
.github/workflows/docker.yml
vendored
@ -1,24 +1,23 @@
|
||||
name: Build Docker image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- '**'
|
||||
# pull_request:
|
||||
workflow_call:
|
||||
inputs:
|
||||
push_digests:
|
||||
description: Push validated per-platform images by digest for the release workflow
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
DOCKERHUB_IMAGE: archivebox/archivebox
|
||||
GHCR_IMAGE: ghcr.io/archivebox/archivebox
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build ${{ matrix.platform }}
|
||||
@ -31,22 +30,78 @@ jobs:
|
||||
runner: ubuntu-24.04
|
||||
cache_scope: docker-amd64
|
||||
artifact_name: digest-linux-amd64
|
||||
local_tag: archivebox-ci:amd64
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
cache_scope: docker-arm64
|
||||
artifact_name: digest-linux-arm64
|
||||
local_tag: archivebox-ci:arm64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
# with:
|
||||
# submodules: true
|
||||
# fetch-depth: 1
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
|
||||
- name: Resolve Docker through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_VERSION="$(uv run --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
matches = [package["version"] for package in packages if package["name"] == "abxpkg"]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
|
||||
print(matches[0])
|
||||
PY
|
||||
)"
|
||||
test -n "$ABXPKG_VERSION"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
docker_env="$(
|
||||
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
|
||||
)"
|
||||
JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$docker_env" >> "$GITHUB_ENV"
|
||||
|
||||
docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
{
|
||||
echo "DOCKER_BINARY=$docker_binary"
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "UV_BINARY=$UV_BINARY"
|
||||
} >> "$GITHUB_ENV"
|
||||
for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
"$docker_binary" --version
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
with:
|
||||
version: latest
|
||||
install: true
|
||||
|
||||
- name: Builder instance name
|
||||
@ -60,35 +115,45 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABX_DL_VERSION="$(python3 - <<'PY'
|
||||
import re
|
||||
ABX_DL_VERSION="$("$UV_BINARY" run --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
with open("pyproject.toml", "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
for dep in deps:
|
||||
match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep)
|
||||
if match:
|
||||
print(match.group(1))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("Missing abx-dl dependency in pyproject.toml")
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
|
||||
print(matches[0])
|
||||
PY
|
||||
)"
|
||||
ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
|
||||
test -n "$ABX_DL_VERSION"
|
||||
ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
|
||||
INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
|
||||
printf '%s\n' "$INSPECT_OUTPUT"
|
||||
ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-project python - <<'PY'
|
||||
import os
|
||||
|
||||
digest = next(
|
||||
(fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
|
||||
"",
|
||||
)
|
||||
print(digest)
|
||||
PY
|
||||
)"
|
||||
[[ "$ABX_DL_DIGEST" == sha256:* ]]
|
||||
ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
|
||||
echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
|
||||
docker buildx imagetools inspect "${ABX_DL_IMAGE}"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
if: inputs.push_digests
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
if: inputs.push_digests
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@ -99,13 +164,14 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$(python3 - <<'PY'
|
||||
VERSION="$("$UV_BINARY" run --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
with open("pyproject.toml", "rb") as f:
|
||||
print(tomllib.load(f)["project"]["version"])
|
||||
print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
test -n "$VERSION"
|
||||
|
||||
{
|
||||
echo 'labels<<EOF'
|
||||
@ -121,12 +187,12 @@ jobs:
|
||||
|
||||
- name: Build and push digest
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v6
|
||||
if: inputs.push_digests
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: |
|
||||
${{ env.DOCKERHUB_IMAGE }}
|
||||
${{ env.GHCR_IMAGE }}
|
||||
@ -140,40 +206,77 @@ jobs:
|
||||
platforms: ${{ matrix.platform }}
|
||||
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Build pull request image
|
||||
if: ${{ !inputs.push_digests }}
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
load: true
|
||||
tags: ${{ matrix.local_tag }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
build-args: |
|
||||
ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
|
||||
ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
|
||||
cache-from: type=gha,scope=${{ matrix.cache_scope }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
|
||||
pull: true
|
||||
platforms: ${{ matrix.platform }}
|
||||
|
||||
- name: Image digest
|
||||
if: inputs.push_digests
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Validate pushed image version, commit, and size
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
|
||||
if [[ "${{ inputs.push_digests }}" != "true" ]]; then
|
||||
IMAGE="${{ matrix.local_tag }}"
|
||||
else
|
||||
IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
|
||||
fi
|
||||
SHORT_SHA="${GITHUB_SHA::7}"
|
||||
|
||||
DATA_DIR="$(mktemp -d)"
|
||||
docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
|
||||
docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version | tee /tmp/archivebox-version.out
|
||||
docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
|
||||
docker run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt > /tmp/archivebox-image-version.txt
|
||||
grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out
|
||||
grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out
|
||||
grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt
|
||||
|
||||
python3 - "$IMAGE" <<'PY'
|
||||
import json
|
||||
import subprocess
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
|
||||
ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
|
||||
printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
|
||||
IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
|
||||
ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
|
||||
"$UV_BINARY" run --no-project python - \
|
||||
"ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
|
||||
"COMMIT_HASH=${SHORT_SHA}" \
|
||||
"COMMIT_HASH=${GITHUB_SHA}" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
image = sys.argv[1]
|
||||
limit = 780 * 1024 * 1024
|
||||
manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image]))
|
||||
total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", []))
|
||||
print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB")
|
||||
if total > limit:
|
||||
raise SystemExit(f"{image} is over the compressed size limit")
|
||||
version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
|
||||
image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
|
||||
if sys.argv[1] not in version_output:
|
||||
raise SystemExit(f"Missing version marker: {sys.argv[1]}")
|
||||
if sys.argv[2] not in version_output:
|
||||
raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
|
||||
if sys.argv[3] not in image_version_lines:
|
||||
raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
|
||||
PY
|
||||
|
||||
- name: Validate compressed image size
|
||||
if: inputs.push_digests
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
|
||||
LIMIT=$((780 * 1024 * 1024))
|
||||
TOTAL="$("$DOCKER_BINARY" manifest inspect "$IMAGE" | "$JQ_BINARY" '[.config.size, (.layers[]?.size)] | add')"
|
||||
printf '%s compressed_size=%s MiB limit=%s MiB\n' \
|
||||
"$IMAGE" "$((TOTAL / 1024 / 1024))" "$((LIMIT / 1024 / 1024))"
|
||||
[[ "$TOTAL" -le "$LIMIT" ]] || { echo "$IMAGE is over the compressed size limit" >&2; exit 1; }
|
||||
|
||||
- name: Export digest
|
||||
if: inputs.push_digests
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
@ -182,151 +285,10 @@ jobs:
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
if: inputs.push_digests
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: ${{ matrix.artifact_name }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
publish:
|
||||
name: publish multiarch tags
|
||||
runs-on: ubuntu-24.04
|
||||
needs:
|
||||
- build
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
version: latest
|
||||
install: true
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
if: github.event_name != 'pull_request'
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Collect Docker tags
|
||||
id: docker_meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$(python3 - <<'PY'
|
||||
import tomllib
|
||||
|
||||
with open("pyproject.toml", "rb") as f:
|
||||
print(tomllib.load(f)["project"]["version"])
|
||||
PY
|
||||
)"
|
||||
BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
|
||||
SHORT_SHA="${GITHUB_SHA::12}"
|
||||
test -n "$BRANCH_TAG"
|
||||
test -n "$SHORT_SHA"
|
||||
|
||||
{
|
||||
echo 'dockerhub_tags<<EOF'
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
echo "${DOCKERHUB_IMAGE}:latest"
|
||||
fi
|
||||
echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
|
||||
echo "${DOCKERHUB_IMAGE}:${VERSION}"
|
||||
echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
echo 'EOF'
|
||||
echo 'ghcr_tags<<EOF'
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
echo "${GHCR_IMAGE}:latest"
|
||||
fi
|
||||
echo "${GHCR_IMAGE}:${BRANCH_TAG}"
|
||||
echo "${GHCR_IMAGE}:${VERSION}"
|
||||
echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
echo 'EOF'
|
||||
echo "version=${VERSION}"
|
||||
echo "branch_tag=${BRANCH_TAG}"
|
||||
echo "short_sha=${SHORT_SHA}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "[+] Publishing Docker Hub tags:"
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
|
||||
fi
|
||||
printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
echo "[+] Publishing GHCR tags:"
|
||||
if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
|
||||
printf '%s\n' "${GHCR_IMAGE}:latest"
|
||||
fi
|
||||
printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
|
||||
- name: Create Docker Hub manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
|
||||
[[ "${#DIGESTS[@]}" -gt 0 ]]
|
||||
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do
|
||||
[[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
|
||||
done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
|
||||
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do
|
||||
REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
|
||||
done
|
||||
|
||||
docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Create GHCR manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
|
||||
[[ "${#DIGESTS[@]}" -gt 0 ]]
|
||||
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do
|
||||
[[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
|
||||
done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
|
||||
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do
|
||||
REFS+=("${GHCR_IMAGE}@sha256:${digest}")
|
||||
done
|
||||
|
||||
docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Inspect published images
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
while IFS= read -r tag; do
|
||||
[[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
|
||||
done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
|
||||
while IFS= read -r tag; do
|
||||
[[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
|
||||
done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
|
||||
|
||||
- name: Update README
|
||||
uses: peter-evans/dockerhub-description@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
repository: archivebox/archivebox
|
||||
|
||||
289
.github/workflows/docs.yml
vendored
Normal file
289
.github/workflows/docs.yml
vendored
Normal file
@ -0,0 +1,289 @@
|
||||
name: Documentation tests
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
USE_COLOR: "False"
|
||||
SHOW_PROGRESS: "False"
|
||||
ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD: "archivebox-docs-ci-only"
|
||||
|
||||
jobs:
|
||||
docs-matrix:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
matrix: ${{ steps.matrix.outputs.matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
- id: matrix
|
||||
run: |
|
||||
uv run --no-project python - <<'PY' >> "$GITHUB_OUTPUT"
|
||||
import json
|
||||
import tomllib
|
||||
|
||||
with open("docs/codeblocks.toml", "rb") as manifest_file:
|
||||
ci = tomllib.load(manifest_file)["ci"]
|
||||
|
||||
include = []
|
||||
for environment, runner in ci["standard"].items():
|
||||
if environment == "core":
|
||||
include.extend(
|
||||
{
|
||||
"environment": environment,
|
||||
"runner": runner,
|
||||
"core_shard": shard,
|
||||
"job_name": f"core/{shard}",
|
||||
"validate_manifest": shard == "metadata",
|
||||
}
|
||||
for shard in ci["core_shards"]
|
||||
)
|
||||
else:
|
||||
include.append(
|
||||
{
|
||||
"environment": environment,
|
||||
"runner": runner,
|
||||
"core_shard": "",
|
||||
"job_name": environment,
|
||||
"validate_manifest": False,
|
||||
},
|
||||
)
|
||||
|
||||
print("matrix=" + json.dumps({"include": include}, separators=(",", ":")))
|
||||
PY
|
||||
|
||||
docs-standard:
|
||||
name: docs/${{ matrix.job_name }}
|
||||
needs: docs-matrix
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
DOCS_CORE_SHARD: ${{ matrix.core_shard }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.docs-matrix.outputs.matrix) }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
- name: Install ArchiveBox
|
||||
run: uv sync --dev --all-extras
|
||||
- name: Prepare abxpkg environment
|
||||
shell: bash
|
||||
run: |
|
||||
{
|
||||
echo "ABXPKG_LIB_DIR=${{ runner.temp }}/archivebox-docs-lib"
|
||||
echo "ARCHIVEBOX_PROJECT_DIR=$GITHUB_WORKSPACE"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "${{ runner.temp }}/archivebox-docs-lib/env/bin" >> "$GITHUB_PATH"
|
||||
- name: Resolve Node.js through abxpkg
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
tooling_env="$(
|
||||
uv run --no-sync abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
|
||||
)"
|
||||
JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$tooling_env" >> "$GITHUB_ENV"
|
||||
|
||||
node_binary="$ABXPKG_LIB_DIR/env/bin/node"
|
||||
{
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "NODE_BINARY=$node_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
test -L "$node_binary"
|
||||
test -x "$node_binary"
|
||||
"$node_binary" --version
|
||||
- name: Resolve documentation shell tools through abxpkg
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
shell_env="$(
|
||||
uv run --no-sync abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries"
|
||||
)"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$shell_env" >> "$GITHUB_ENV"
|
||||
|
||||
curl_binary="$ABXPKG_LIB_DIR/env/bin/curl"
|
||||
echo "CURL_BINARY=$curl_binary" >> "$GITHUB_ENV"
|
||||
test -L "$curl_binary"
|
||||
test -x "$curl_binary"
|
||||
- name: Resolve Docker through abxpkg
|
||||
if: matrix.environment == 'docker'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
docker_env="$(
|
||||
uv run --no-sync abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
|
||||
)"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$docker_env" >> "$GITHUB_ENV"
|
||||
|
||||
docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
|
||||
test -L "$docker_binary"
|
||||
test -x "$docker_binary"
|
||||
"$docker_binary" --version
|
||||
- name: Start bound local documentation site
|
||||
shell: bash
|
||||
run: |
|
||||
ready_fifo="${{ runner.temp }}/archivebox-docs-http-ready"
|
||||
mkfifo "$ready_fifo"
|
||||
uv run --no-sync python .github/scripts/docs_http_server.py --ready-fifo "$ready_fifo" &
|
||||
server_pid=$!
|
||||
IFS= read -r docs_url < "$ready_fifo"
|
||||
{
|
||||
echo "ARCHIVEBOX_DOCS_SERVER_PID=$server_pid"
|
||||
echo "ARCHIVEBOX_DOCS_URL_ONE=$docs_url/collection-one"
|
||||
echo "ARCHIVEBOX_DOCS_URL_TWO=$docs_url/collection-two"
|
||||
echo "ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT=18000"
|
||||
echo "ARCHIVEBOX_DOCS_STATIC_PORT=18001"
|
||||
} >> "$GITHUB_ENV"
|
||||
- name: Resolve documentation validators through abxpkg
|
||||
if: matrix.validate_manifest
|
||||
run: |
|
||||
validators_env="$(
|
||||
uv run abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
|
||||
)"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$validators_env" >> "$GITHUB_ENV"
|
||||
- name: Resolve merge tools through abxpkg
|
||||
if: matrix.environment == 'merge'
|
||||
run: |
|
||||
uv run abxpkg install rsync --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew
|
||||
uv run abxpkg install sqlite3 --lib "$ABXPKG_LIB_DIR" --binproviders env,apt,brew
|
||||
- name: Validate documentation inventory and structured fences
|
||||
if: matrix.validate_manifest
|
||||
run: uv run --no-sync pytest -q docs/test_codeblocks_manifest.py
|
||||
- name: Initialize documentation collection
|
||||
if: contains(fromJSON('["core", "macos", "root"]'), matrix.environment)
|
||||
shell: bash
|
||||
run: |
|
||||
docs_data_dir="${{ runner.temp }}/archivebox-docs-data"
|
||||
mkdir -p "$docs_data_dir"
|
||||
(cd "$docs_data_dir" && uv run --project "$GITHUB_WORKSPACE" --no-sync archivebox init)
|
||||
echo "ARCHIVEBOX_DOCS_DATA_DIR=$docs_data_dir" >> "$GITHUB_ENV"
|
||||
- name: Build local documentation image
|
||||
if: matrix.environment == 'docker'
|
||||
run: '"$DOCKER_BINARY" build --tag archivebox-docs-ci .'
|
||||
- name: Run documentation code blocks
|
||||
if: matrix.environment != 'root' && !contains(fromJSON('["core", "macos"]'), matrix.environment)
|
||||
run: |
|
||||
mapfile -t docs_paths < <(
|
||||
uv run --no-sync python - <<'PY'
|
||||
from pathlib import Path
|
||||
|
||||
print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()), sep="\n")
|
||||
PY
|
||||
)
|
||||
uv run --no-sync pytest -vv --tb=long README.md AGENTS.md skills "${docs_paths[@]}" --docs-environment=${{ matrix.environment }}
|
||||
- name: Run collection documentation code blocks
|
||||
if: contains(fromJSON('["core", "macos"]'), matrix.environment)
|
||||
working-directory: ${{ env.ARCHIVEBOX_DOCS_DATA_DIR }}
|
||||
run: |
|
||||
docs_paths=()
|
||||
if [[ -n "$DOCS_CORE_SHARD" ]]; then
|
||||
mapfile -t docs_paths < <(
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync python - "$DOCS_CORE_SHARD" <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
workspace = Path(os.environ["GITHUB_WORKSPACE"])
|
||||
with (workspace / "docs" / "codeblocks.toml").open("rb") as manifest_file:
|
||||
shard_paths = tomllib.load(manifest_file)["ci"]["core_shards"][sys.argv[1]]
|
||||
print(*(workspace / path for path in shard_paths), sep="\n")
|
||||
PY
|
||||
)
|
||||
else
|
||||
while IFS= read -r docs_path; do
|
||||
docs_paths+=("$docs_path")
|
||||
done < <(
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync python - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
docs_dir = Path(os.environ["GITHUB_WORKSPACE"]) / "docs"
|
||||
print(*(str(path) for path in sorted(docs_dir.rglob("*.md")) if not path.is_symlink()), sep="\n")
|
||||
PY
|
||||
)
|
||||
docs_paths=("$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}")
|
||||
fi
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "${docs_paths[@]}" --docs-environment=${{ matrix.environment }}
|
||||
- name: Run root documentation code blocks
|
||||
if: matrix.environment == 'root'
|
||||
working-directory: ${{ env.ARCHIVEBOX_DOCS_DATA_DIR }}
|
||||
run: |
|
||||
uv_bin="$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
test -L "$uv_bin"
|
||||
test -x "$uv_bin"
|
||||
mapfile -t docs_paths < <(
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync python - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
docs_dir = Path(os.environ["GITHUB_WORKSPACE"]) / "docs"
|
||||
print(*(str(path) for path in sorted(docs_dir.rglob("*.md")) if not path.is_symlink()), sep="\n")
|
||||
PY
|
||||
)
|
||||
sudo --preserve-env=PATH,ABXPKG_LIB_DIR,ARCHIVEBOX_DOCS_URL_ONE,ARCHIVEBOX_DOCS_URL_TWO,ARCHIVEBOX_DOCS_ARCHIVEBOX_PORT,ARCHIVEBOX_DOCS_STATIC_PORT,ARCHIVEBOX_PUBLISH_ADMIN_PASSWORD "$uv_bin" run --project "$GITHUB_WORKSPACE" --no-sync pytest -vv --tb=long "$GITHUB_WORKSPACE/README.md" "$GITHUB_WORKSPACE/AGENTS.md" "$GITHUB_WORKSPACE/skills" "${docs_paths[@]}" --docs-environment=${{ matrix.environment }}
|
||||
- name: Stop local documentation site
|
||||
if: always()
|
||||
run: kill "$ARCHIVEBOX_DOCS_SERVER_PID"
|
||||
|
||||
docs-freebsd:
|
||||
name: docs/freebsd
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: vmactions/freebsd-vm@77ed28d336d03fe19a3f4f7266c1d2c4714dd79d # v1.5.2
|
||||
with:
|
||||
usesh: true
|
||||
prepare: pkg install -y py313-uv
|
||||
run: uv run --no-project --with pytest --with pytest-codeblocks pytest -o addopts=--codeblocks -vv --tb=long README.md AGENTS.md skills $(uv run --no-project python -c 'from pathlib import Path; print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()))') --docs-environment=freebsd
|
||||
|
||||
docs-openbsd:
|
||||
name: docs/openbsd
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
- uses: vmactions/openbsd-vm@c941015845c0f0c429676840963dc63b226d4f69 # v1.4.5
|
||||
with:
|
||||
usesh: true
|
||||
prepare: pkg_add uv
|
||||
run: uv run --no-project --with pytest --with pytest-codeblocks pytest -o addopts=--codeblocks -vv --tb=long README.md AGENTS.md skills $(uv run --no-project python -c 'from pathlib import Path; print(*(str(path) for path in sorted(Path("docs").rglob("*.md")) if not path.is_symlink()))') --docs-environment=openbsd
|
||||
47
.github/workflows/duplicate-issue-detection.yml
vendored
47
.github/workflows/duplicate-issue-detection.yml
vendored
@ -6,18 +6,55 @@ on:
|
||||
|
||||
jobs:
|
||||
check-duplicates:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Install opencode
|
||||
run: curl -fsSL https://opencode.ai/install | bash
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Resolve OpenCode and workflow tools through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv sync --locked --no-sources --no-install-project
|
||||
OPENCODE_CONFIG="$(uv run --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "opencode" / "config.json")')"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
opencode_env="$(
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:github_binaries" \
|
||||
--deps-from="$OPENCODE_CONFIG:required_binaries"
|
||||
)"
|
||||
jq_binary="$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
"$jq_binary" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$opencode_env" >> "$GITHUB_ENV"
|
||||
|
||||
opencode_binary="$ABXPKG_LIB_DIR/env/bin/opencode"
|
||||
echo "OPENCODE_BINARY=$opencode_binary" >> "$GITHUB_ENV"
|
||||
test -L "$opencode_binary"
|
||||
test -x "$opencode_binary"
|
||||
|
||||
- name: Check for duplicate issues
|
||||
env:
|
||||
@ -33,7 +70,7 @@ jobs:
|
||||
"webfetch": "allow"
|
||||
}
|
||||
run: |
|
||||
opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
|
||||
"$OPENCODE_BINARY" run -m anthropic/claude-haiku-4-5 "A new issue has been created:
|
||||
|
||||
Issue number:
|
||||
${{ github.event.issue.number }}
|
||||
|
||||
34
.github/workflows/lint.yml
vendored
34
.github/workflows/lint.yml
vendored
@ -1,13 +1,7 @@
|
||||
name: Run linters
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: ['**']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
UV_NO_SOURCES: "1"
|
||||
@ -17,11 +11,23 @@ jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Clone abxpkg
|
||||
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
||||
|
||||
@ -31,18 +37,6 @@ jobs:
|
||||
- name: Clone abx-dl
|
||||
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: "0.10.6"
|
||||
enable-cache: false
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: |
|
||||
uv venv
|
||||
|
||||
164
.github/workflows/pip.yml
vendored
164
.github/workflows/pip.yml
vendored
@ -1,66 +1,134 @@
|
||||
name: Build Pip package
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- '**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.13"
|
||||
UV_VERSION: "0.11.3"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build distributions
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
version: "0.10.6"
|
||||
enable-cache: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
python-version: "3.13"
|
||||
architecture: x64
|
||||
|
||||
- name: APT install archivebox dev + run dependencies
|
||||
uses: awalsh128/cache-apt-pkgs-action@latest
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
|
||||
version: 1.0
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: false
|
||||
|
||||
- name: UV install archivebox dev + run sub-dependencies
|
||||
env:
|
||||
UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
|
||||
run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
|
||||
|
||||
- name: UV build archivebox and archivebox/pkgs/* packages
|
||||
run: |
|
||||
uv build --all
|
||||
|
||||
- name: UV install archivebox and archivebox/pkgs/* locally for tests
|
||||
env:
|
||||
UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
|
||||
run: |
|
||||
uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
|
||||
uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
|
||||
|
||||
- name: Verify built package full install
|
||||
env:
|
||||
UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
|
||||
- name: Build every distribution
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
DATA_DIR="$(mktemp -d)"
|
||||
cd "$DATA_DIR"
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox init
|
||||
timeout 30m uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox install
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox version
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox status
|
||||
uv build --all-packages --no-sources --out-dir dist --clear
|
||||
|
||||
shopt -s nullglob
|
||||
artifacts=(dist/*)
|
||||
wheels=(dist/archivebox-*.whl)
|
||||
sdists=(dist/archivebox-*.tar.gz)
|
||||
[[ "${#artifacts[@]}" -gt 0 ]]
|
||||
[[ "${#wheels[@]}" -eq 1 ]]
|
||||
[[ "${#sdists[@]}" -eq 1 ]]
|
||||
printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
|
||||
uv run --no-project python - <<'PY'
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
artifacts = sorted(Path("dist").iterdir())
|
||||
checksum_lines = []
|
||||
for artifact in artifacts:
|
||||
if artifact.name == "SHA256SUMS":
|
||||
continue
|
||||
digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
line = f"{digest} {artifact.name}"
|
||||
checksum_lines.append(line)
|
||||
print(line)
|
||||
Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
|
||||
PY
|
||||
|
||||
- name: Upload every distribution
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: dist/*
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
install-smoke:
|
||||
name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
|
||||
needs: build
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-24.04, macos-15]
|
||||
python: ["3.13", "3.14"]
|
||||
artifact: [wheel, sdist]
|
||||
|
||||
steps:
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
enable-cache: false
|
||||
|
||||
- name: Download distributions
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: ${{ runner.temp }}/python-distributions
|
||||
|
||||
- name: Install and smoke-test the built ${{ matrix.artifact }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
shopt -s nullglob
|
||||
|
||||
case "${{ matrix.artifact }}" in
|
||||
wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
|
||||
sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
[[ "${#candidates[@]}" -eq 1 ]]
|
||||
artifact="${candidates[0]}"
|
||||
|
||||
smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
|
||||
trap 'rm -rf "$smoke_root"' EXIT
|
||||
smoke_env="$smoke_root/env"
|
||||
data_dir="$smoke_root/data"
|
||||
mkdir -p "$data_dir"
|
||||
|
||||
uv venv --python "${{ matrix.python }}" "$smoke_env"
|
||||
uv pip install --python "$smoke_env" --no-cache "$artifact"
|
||||
|
||||
unset PYTHONPATH
|
||||
cd "$data_dir"
|
||||
VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync python - <<'PY'
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import archivebox
|
||||
|
||||
installed_path = Path(archivebox.__file__).resolve()
|
||||
environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
|
||||
workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
|
||||
assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
|
||||
assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
|
||||
print(installed_path)
|
||||
PY
|
||||
VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync archivebox version
|
||||
VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync archivebox init
|
||||
VIRTUAL_ENV="$smoke_env" uv run --active --no-project --no-sync archivebox status
|
||||
|
||||
52
.github/workflows/release-runner.yml
vendored
52
.github/workflows/release-runner.yml
vendored
@ -1,52 +0,0 @@
|
||||
name: Release State
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- dev
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release-state:
|
||||
if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:')
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
ref: ${{ github.ref_name }}
|
||||
token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: "0.10.6"
|
||||
enable-cache: false
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Configure git identity
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Run release script
|
||||
env:
|
||||
DEFAULT_BRANCH: dev
|
||||
GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
run: ./bin/release.sh
|
||||
243
.github/workflows/release.yml
vendored
Normal file
243
.github/workflows/release.yml
vendored
Normal file
@ -0,0 +1,243 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI]
|
||||
types: [completed]
|
||||
branches: [dev]
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
id-token: write
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: release-dev
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
DOCKERHUB_IMAGE: archivebox/archivebox
|
||||
GHCR_IMAGE: ghcr.io/archivebox/archivebox
|
||||
RELEASE_BRANCH: dev
|
||||
RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
CI_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
|
||||
jobs:
|
||||
python-release:
|
||||
name: Publish tested Python artifacts
|
||||
if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push'
|
||||
runs-on: ubuntu-24.04
|
||||
environment: pypi
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
steps:
|
||||
- name: Checkout the tested commit
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Resolve release binaries through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
|
||||
for name in uv gh git jq curl; do
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
done
|
||||
{
|
||||
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
|
||||
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download exact tested distributions
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
name: python-distributions
|
||||
path: ${{ runner.temp }}/python-distributions
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ env.CI_RUN_ID }}
|
||||
|
||||
- name: Record release version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION="$($UV_BINARY run --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Publish the exact tested distributions
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
|
||||
run: ./bin/release.sh
|
||||
|
||||
docker-release:
|
||||
name: Publish tested multiarch image
|
||||
if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push'
|
||||
needs: python-release
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout the tested commit
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
ref: ${{ env.RELEASE_SHA }}
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Resolve release and Docker binaries through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
|
||||
for name in uv git jq docker; do
|
||||
test -L "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
test -x "$ABXPKG_LIB_DIR/env/bin/$name"
|
||||
done
|
||||
{
|
||||
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
with:
|
||||
install: true
|
||||
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Download exact tested image digests
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
github-token: ${{ github.token }}
|
||||
repository: ${{ github.repository }}
|
||||
run-id: ${{ env.CI_RUN_ID }}
|
||||
|
||||
- name: Verify digest set and collect tags
|
||||
id: docker_meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
|
||||
[[ "${#DIGESTS[@]}" -eq 2 ]]
|
||||
for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
|
||||
|
||||
VERSION='${{ needs.python-release.outputs.version }}'
|
||||
SHORT_SHA="${RELEASE_SHA::12}"
|
||||
TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
|
||||
TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
|
||||
if [[ -z "$TAG_TARGET" ]]; then
|
||||
TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
|
||||
TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
|
||||
fi
|
||||
{
|
||||
echo 'dockerhub_tags<<EOF'
|
||||
echo "${DOCKERHUB_IMAGE}:dev"
|
||||
echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
|
||||
echo 'EOF'
|
||||
echo 'ghcr_tags<<EOF'
|
||||
echo "${GHCR_IMAGE}:dev"
|
||||
echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create Docker Hub manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Create GHCR manifest
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
|
||||
TAG_ARGS=()
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Inspect published images
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
|
||||
while IFS= read -r tag; do [[ -n "$tag" ]] && $DOCKER_BINARY buildx imagetools inspect "$tag"; done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
|
||||
|
||||
- name: Update Docker Hub README
|
||||
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
repository: archivebox/archivebox
|
||||
|
||||
downstream-packages:
|
||||
name: Dispatch downstream packages
|
||||
needs:
|
||||
- python-release
|
||||
- docker-release
|
||||
uses: ./.github/workflows/update-homebrew-tap.yml
|
||||
with:
|
||||
release_tag: ${{ needs.python-release.outputs.tag }}
|
||||
release_sha: ${{ github.event.workflow_run.head_sha }}
|
||||
secrets: inherit
|
||||
443
.github/workflows/test-parallel.yml
vendored
443
.github/workflows/test-parallel.yml
vendored
@ -1,22 +1,15 @@
|
||||
name: Parallel Tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [dev, main, master]
|
||||
push:
|
||||
branches: [dev]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
PYTHONLEGACYWINDOWSSTDIO: utf-8
|
||||
USE_COLOR: False
|
||||
UV_NO_SOURCES: "1"
|
||||
ABXPKG_MIN_RELEASE_AGE: "0"
|
||||
CI_PYTHON_VERSION: "3.13.14"
|
||||
UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache
|
||||
|
||||
jobs:
|
||||
discover-tests:
|
||||
@ -27,84 +20,150 @@ jobs:
|
||||
plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Resolve matrix tools through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
|
||||
test -n "$ABXPKG_VERSION"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
>/dev/null
|
||||
JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
PYTHON_BINARY="$ABXPKG_LIB_DIR/env/bin/python"
|
||||
test -L "$JQ_BINARY"
|
||||
test -x "$JQ_BINARY"
|
||||
test -L "$PYTHON_BINARY"
|
||||
test -x "$PYTHON_BINARY"
|
||||
{
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "PYTHON_BINARY=$PYTHON_BINARY"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Discover test files
|
||||
id: set-matrix
|
||||
shell: bash
|
||||
run: |
|
||||
# Find all main test files
|
||||
all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
|
||||
|
||||
# Create JSON array with test file info
|
||||
json_array="["
|
||||
first=true
|
||||
for test_file in $all_tests; do
|
||||
if [ "$first" = true ]; then
|
||||
first=false
|
||||
else
|
||||
json_array+=","
|
||||
fi
|
||||
|
||||
# Extract a display name for the test
|
||||
name="main/$(basename $test_file .py | sed 's/^test_//')"
|
||||
needs_chromium=false
|
||||
if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
|
||||
needs_chromium=true
|
||||
fi
|
||||
needs_sonic=false
|
||||
if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
|
||||
needs_sonic=true
|
||||
fi
|
||||
|
||||
json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
|
||||
done
|
||||
json_array+="]"
|
||||
|
||||
echo "test-files=$json_array" >> $GITHUB_OUTPUT
|
||||
echo "Found $(echo $all_tests | wc -w) test files"
|
||||
echo "$json_array" | jq '.'
|
||||
set -Eeuo pipefail
|
||||
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py archivebox)
|
||||
test "${#output[@]}" -eq 2
|
||||
json_array="${output[1]}"
|
||||
echo "test-files=$json_array" >> "$GITHUB_OUTPUT"
|
||||
echo "${output[0]}"
|
||||
echo "$json_array" | "$JQ_BINARY" '.'
|
||||
|
||||
- name: Clone abx-plugins
|
||||
run: bash .github/scripts/clone_abx_repo.sh abx-plugins
|
||||
|
||||
- name: Discover plugin tests
|
||||
id: set-plugin-matrix
|
||||
shell: bash
|
||||
run: |
|
||||
plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
|
||||
set -Eeuo pipefail
|
||||
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py plugins)
|
||||
test "${#output[@]}" -eq 2
|
||||
json_array="${output[1]}"
|
||||
echo "plugin-tests=$json_array" >> "$GITHUB_OUTPUT"
|
||||
echo "${output[0]}"
|
||||
echo "$json_array" | "$JQ_BINARY" '.'
|
||||
|
||||
json_array="["
|
||||
first=true
|
||||
for plugin_name in $plugin_tests; do
|
||||
if [ "$first" = true ]; then
|
||||
first=false
|
||||
else
|
||||
json_array+=","
|
||||
fi
|
||||
prepare-python-dependencies:
|
||||
name: Prepare Python 3.13.14 dependency cache
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
|
||||
LIB_DIR: /tmp/abx-lib
|
||||
ABXPKG_LIB_DIR: /tmp/abx-lib
|
||||
|
||||
needs_chromium=false
|
||||
if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
|
||||
needs_chromium=true
|
||||
fi
|
||||
needs_sonic=false
|
||||
if [ "$plugin_name" = "search_backend_sonic" ]; then
|
||||
needs_sonic=true
|
||||
fi
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
|
||||
done
|
||||
json_array+="]"
|
||||
- name: Set up Python ${{ env.CI_PYTHON_VERSION }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
architecture: x64
|
||||
|
||||
echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
|
||||
echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
|
||||
echo "$json_array" | jq '.'
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Clone abxpkg
|
||||
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
||||
|
||||
- name: Clone abx-plugins
|
||||
run: bash .github/scripts/clone_abx_repo.sh abx-plugins
|
||||
|
||||
- name: Clone abx-dl
|
||||
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
||||
|
||||
- name: Restore or create the SHA-specific uv cache
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: ${{ env.UV_CACHE_DIR }}
|
||||
key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
|
||||
|
||||
- name: Bootstrap local abxpkg
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv venv
|
||||
uv pip install -e ./abxpkg
|
||||
|
||||
- name: Resolve Linux build dependencies through abxpkg
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mkdir -p "$LIB_DIR/env/bin"
|
||||
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--no-cache \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries"
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-env.json:required_binaries"
|
||||
|
||||
- name: Populate the full editable dependency cache
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
||||
uv run --no-sync --no-sources python -c 'import ldap; print(ldap.__version__)'
|
||||
|
||||
run-tests:
|
||||
name: ${{ matrix.test.name }}
|
||||
runs-on: ubuntu-24.04
|
||||
needs: discover-tests
|
||||
needs: [discover-tests, prepare-python-dependencies]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
|
||||
CHROME_HEADLESS: "true"
|
||||
@ -118,11 +177,23 @@ jobs:
|
||||
python: ["3.13"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Clone abxpkg
|
||||
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
||||
|
||||
@ -132,56 +203,76 @@ jobs:
|
||||
- name: Clone abx-dl
|
||||
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
||||
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: "0.10.6"
|
||||
|
||||
- name: Set up Node JS
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Cache uv
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.python }}-uv-
|
||||
|
||||
- uses: awalsh128/cache-apt-pkgs-action@latest
|
||||
with:
|
||||
packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
|
||||
version: 1.1
|
||||
path: ${{ env.UV_CACHE_DIR }}
|
||||
key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv venv
|
||||
uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
||||
uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
||||
|
||||
- name: Resolve Node.js through abxpkg
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mkdir -p "$LIB_DIR/env/bin"
|
||||
export PATH="$LIB_DIR/env/bin:$PATH"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
tooling_env="$(
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
|
||||
)"
|
||||
JQ_BINARY="$LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$tooling_env" >> "$GITHUB_ENV"
|
||||
|
||||
node_binary="$LIB_DIR/env/bin/node"
|
||||
{
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "NODE_BINARY=$node_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
test -L "$JQ_BINARY"
|
||||
test -x "$JQ_BINARY"
|
||||
test -L "$node_binary"
|
||||
test -x "$node_binary"
|
||||
"$node_binary" --version
|
||||
|
||||
- name: Resolve core test binaries through abxpkg
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_INSTALL_TIMEOUT: "900"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$LIB_DIR"
|
||||
echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
mkdir -p "$LIB_DIR/env/bin"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/wget/config.json:required_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/git/config.json:required_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_ripgrep/config.json:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve Chrome through abxpkg
|
||||
@ -189,17 +280,20 @@ jobs:
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_INSTALL_TIMEOUT: "900"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$LIB_DIR"
|
||||
echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve Sonic through abxpkg
|
||||
@ -207,28 +301,36 @@ jobs:
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_INSTALL_TIMEOUT: "900"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$LIB_DIR"
|
||||
echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- name: Run test - ${{ matrix.test.name }}
|
||||
env:
|
||||
TEST_PATHS_JSON: ${{ toJson(matrix.test.paths) }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t test_paths < <("$JQ_BINARY" -r '.[]' <<<"$TEST_PATHS_JSON")
|
||||
[[ "${#test_paths[@]}" -gt 0 ]]
|
||||
mkdir -p tests/out
|
||||
uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out
|
||||
uv run --no-sync --no-sources pytest -vs "${test_paths[@]}" --basetemp=tests/out
|
||||
|
||||
plugin-tests:
|
||||
name: ${{ matrix.plugin.name }}
|
||||
runs-on: ubuntu-24.04
|
||||
needs: discover-tests
|
||||
needs: [discover-tests, prepare-python-dependencies]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
|
||||
CHROME_HEADLESS: "true"
|
||||
@ -241,11 +343,23 @@ jobs:
|
||||
plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python ${{ env.CI_PYTHON_VERSION }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
enable-cache: false
|
||||
|
||||
- name: Clone abxpkg
|
||||
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
||||
|
||||
@ -255,54 +369,74 @@ jobs:
|
||||
- name: Clone abx-dl
|
||||
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
||||
|
||||
- name: Set up Python 3.13
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: "0.10.6"
|
||||
|
||||
- name: Set up Node JS
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Cache uv
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-3.13-uv-
|
||||
|
||||
- uses: awalsh128/cache-apt-pkgs-action@latest
|
||||
with:
|
||||
packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
|
||||
version: 1.1
|
||||
path: ${{ env.UV_CACHE_DIR }}
|
||||
key: ${{ runner.os }}-python-${{ env.CI_PYTHON_VERSION }}-uv-${{ github.sha }}
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv venv
|
||||
uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
||||
uv pip install --offline --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
||||
|
||||
- name: Resolve Node.js through abxpkg
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mkdir -p "$LIB_DIR/env/bin"
|
||||
export PATH="$LIB_DIR/env/bin:$PATH"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
tooling_env="$(
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
|
||||
)"
|
||||
JQ_BINARY="$LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$tooling_env" >> "$GITHUB_ENV"
|
||||
|
||||
node_binary="$LIB_DIR/env/bin/node"
|
||||
{
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "NODE_BINARY=$node_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
test -L "$JQ_BINARY"
|
||||
test -x "$JQ_BINARY"
|
||||
test -L "$node_binary"
|
||||
test -x "$node_binary"
|
||||
"$node_binary" --version
|
||||
|
||||
- name: Resolve plugin dependencies through abxpkg
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_INSTALL_TIMEOUT: "900"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$LIB_DIR"
|
||||
echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
mkdir -p "$LIB_DIR/env/bin"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/${{ matrix.plugin.plugin }}/config.json:required_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/${{ matrix.plugin.config_path }}:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve Chrome through abxpkg
|
||||
@ -310,17 +444,20 @@ jobs:
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_INSTALL_TIMEOUT: "900"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$LIB_DIR"
|
||||
echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve Sonic through abxpkg
|
||||
@ -328,23 +465,37 @@ jobs:
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_INSTALL_TIMEOUT: "900"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$LIB_DIR"
|
||||
echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- name: Run plugin tests - ${{ matrix.plugin.name }}
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
|
||||
API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
|
||||
run: |
|
||||
uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage
|
||||
set -Eeuo pipefail
|
||||
test_path="$GITHUB_WORKSPACE/${{ matrix.plugin.test_path }}"
|
||||
test -d "$test_path"
|
||||
test_count="$("$LIB_DIR/env/bin/python" -c 'import pathlib, sys; print(sum(1 for _ in pathlib.Path(sys.argv[1]).rglob("test_*.py")))' "$test_path")"
|
||||
test "$test_count" -gt 0
|
||||
DATA_DIR="$(mktemp -d -t archivebox_plugin_tests.XXXXXX)"
|
||||
export DATA_DIR
|
||||
plugin_tmpdir="$(mktemp -d -t archivebox_plugin_run.XXXXXX)"
|
||||
cd "$plugin_tmpdir"
|
||||
uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources python -m pytest \
|
||||
"$test_path" -p no:django -v --tb=short
|
||||
|
||||
371
.github/workflows/test.yml
vendored
371
.github/workflows/test.yml
vendored
@ -1,12 +1,7 @@
|
||||
name: Run tests
|
||||
name: Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ['**']
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
DOCKER_IMAGE: archivebox-ci
|
||||
@ -23,16 +18,36 @@ jobs:
|
||||
PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-24.04]
|
||||
python: ["3.13"]
|
||||
include:
|
||||
- os: ubuntu-24.04
|
||||
os_name: linux
|
||||
python: "3.14"
|
||||
- os: macos-15
|
||||
os_name: macOS
|
||||
python: "3.13"
|
||||
- os: macos-15
|
||||
os_name: macOS
|
||||
python: "3.14"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
### Set up Python
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
version: "0.11.3"
|
||||
|
||||
- name: Clone abxpkg
|
||||
run: bash .github/scripts/clone_abx_repo.sh abxpkg
|
||||
|
||||
@ -42,121 +57,329 @@ jobs:
|
||||
- name: Clone abx-dl
|
||||
run: bash .github/scripts/clone_abx_repo.sh abx-dl
|
||||
|
||||
### Setup Python & JS Languages
|
||||
- name: Set up Python ${{ matrix.python }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python }}
|
||||
architecture: x64
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
version: "0.10.6"
|
||||
|
||||
- name: Set up Node JS
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
### Install Python & JS Dependencies
|
||||
### Install dependencies
|
||||
- name: Cache uv
|
||||
uses: actions/cache@v3
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: ~/.cache/uv
|
||||
key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ matrix.python }}-uv-
|
||||
|
||||
- uses: awalsh128/cache-apt-pkgs-action@latest
|
||||
with:
|
||||
packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
|
||||
version: 1.0
|
||||
- name: Bootstrap local abxpkg
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv venv
|
||||
uv pip install -e ./abxpkg
|
||||
|
||||
- name: Prepare abxpkg environment
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mkdir -p "$LIB_DIR/env/bin"
|
||||
{
|
||||
echo "LIB_DIR=$LIB_DIR"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "$LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Resolve Node.js through abxpkg
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
export PATH="$LIB_DIR/env/bin:$PATH"
|
||||
tooling_env="$(
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:node_binaries"
|
||||
)"
|
||||
JQ_BINARY="$LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$tooling_env" >> "$GITHUB_ENV"
|
||||
|
||||
node_binary="$LIB_DIR/env/bin/node"
|
||||
{
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
echo "NODE_BINARY=$node_binary"
|
||||
} >> "$GITHUB_ENV"
|
||||
test -L "$node_binary"
|
||||
test -x "$node_binary"
|
||||
"$node_binary" --version
|
||||
|
||||
- name: Resolve Linux build dependencies through abxpkg
|
||||
if: runner.os == 'Linux'
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--no-cache \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries"
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-env.json:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve host Homebrew through abxpkg
|
||||
if: runner.os == 'macOS'
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
brew_env="$(
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries"
|
||||
)"
|
||||
brew_binary="$("$JQ_BINARY" -r '.BREW_BINARY' <<< "$brew_env")"
|
||||
test -L "$brew_binary"
|
||||
echo "BREW_BINARY=$brew_binary" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Resolve macOS build dependencies through abxpkg
|
||||
if: runner.os == 'macOS'
|
||||
env:
|
||||
LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
brew_target="$(readlink "$BREW_BINARY")"
|
||||
test -x "$brew_target"
|
||||
brew_root="$(dirname "$(dirname "$brew_target")")"
|
||||
export ABXPKG_BREW_ROOT="$brew_root"
|
||||
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-build.json:required_binaries"
|
||||
|
||||
PATH="$brew_root/opt/openldap/bin:$PATH" \
|
||||
uv run --no-sync --no-sources abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-openldap-env.json:required_binaries" \
|
||||
| "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
>> "$GITHUB_ENV"
|
||||
|
||||
ldapvc_target="$(readlink "$LIB_DIR/env/bin/ldapvc")"
|
||||
test -x "$ldapvc_target"
|
||||
openldap_prefix="$(dirname "$(dirname "$ldapvc_target")")"
|
||||
test -f "$openldap_prefix/include/ldap.h"
|
||||
test -f "$openldap_prefix/lib/libldap.dylib"
|
||||
{
|
||||
echo "ABXPKG_BREW_ROOT=$brew_root"
|
||||
echo "CPPFLAGS=-I$openldap_prefix/include${CPPFLAGS:+ $CPPFLAGS}"
|
||||
echo "LDFLAGS=-L$openldap_prefix/lib${LDFLAGS:+ $LDFLAGS}"
|
||||
echo "PKG_CONFIG_PATH=$openldap_prefix/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: |
|
||||
uv venv
|
||||
uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
|
||||
|
||||
### Run the tests
|
||||
- name: Directory listing for debugging
|
||||
run: |
|
||||
pwd
|
||||
ls
|
||||
|
||||
- name: ArchiveBox full install check
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
DATA_DIR="$PWD/tests/out/data"
|
||||
mkdir -p "$DATA_DIR"
|
||||
uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init
|
||||
timeout 30m uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
|
||||
uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
|
||||
uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
|
||||
uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
|
||||
|
||||
- name: Test built package with pytest
|
||||
if: github.ref != 'refs/heads/dev'
|
||||
- name: Run consolidated core suite
|
||||
if: matrix.os_name == 'macOS' || matrix.python == '3.14.6'
|
||||
run: |
|
||||
mkdir -p tests/out
|
||||
uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out
|
||||
|
||||
- name: Run documentation code blocks
|
||||
run: uv run --no-sync --no-sources pytest -vv --tb=long README.md AGENTS.md skills docs
|
||||
|
||||
- name: Run plugin tests
|
||||
if: github.ref != 'refs/heads/dev'
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
|
||||
API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
|
||||
run: |
|
||||
uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
|
||||
uv run --no-sync --no-sources pytest -q archivebox/tests --basetemp="tests/out/${{ matrix.os_name }}-python-${{ matrix.python }}"
|
||||
|
||||
docker_tests:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
|
||||
- name: Resolve Docker and curl through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
|
||||
test -n "$ABXPKG_VERSION"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
echo "ABXPKG_VERSION=$ABXPKG_VERSION" >> "$GITHUB_ENV"
|
||||
echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
|
||||
|
||||
tooling_env="$(
|
||||
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries"
|
||||
)"
|
||||
JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$tooling_env" >> "$GITHUB_ENV"
|
||||
|
||||
for binary in "$ABXPKG_LIB_DIR/env/bin/docker" "$ABXPKG_LIB_DIR/env/bin/curl"; do
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
{
|
||||
echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
|
||||
echo "JQ_BINARY=$JQ_BINARY"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build image
|
||||
run: |
|
||||
docker build . -t "$DOCKER_IMAGE"
|
||||
"$DOCKER_BINARY" build . -t "$DOCKER_IMAGE"
|
||||
|
||||
- name: Resolve real mount test dependencies through abxpkg
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mount_env="$(
|
||||
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
|
||||
--install \
|
||||
--json \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
|
||||
)"
|
||||
"$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
|
||||
<<< "$mount_env" >> "$GITHUB_ENV"
|
||||
|
||||
for binary_name in bash chmod exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
|
||||
binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
|
||||
- name: Validate Docker UID and GID behavior on real NFS and SMB mounts
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
NFS_SOURCE="/srv/archivebox-ci-nfs"
|
||||
SMB_SOURCE="/srv/archivebox-ci-smb"
|
||||
NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
|
||||
SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
|
||||
mounted_nfs=0
|
||||
mounted_smb=0
|
||||
|
||||
cleanup() {
|
||||
local original_status=$? cleanup_status=0
|
||||
trap - EXIT
|
||||
if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$original_status" != "0" ]]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
exit "$cleanup_status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '%s *(rw,sync,no_subtree_check,no_root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
|
||||
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet smbd.service
|
||||
|
||||
"$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
|
||||
mounted_nfs=1
|
||||
"$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
|
||||
-o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
|
||||
mounted_smb=1
|
||||
|
||||
NFS_TEST_DIR="$NFS_MOUNT" \
|
||||
SMB_TEST_DIR="$SMB_MOUNT" \
|
||||
IMAGE="$DOCKER_IMAGE" \
|
||||
"$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
|
||||
|
||||
- name: Init data dir
|
||||
run: |
|
||||
mkdir "${{ github.workspace }}/data"
|
||||
docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
|
||||
"$MKDIR_BINARY" "${{ github.workspace }}/data"
|
||||
"$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
|
||||
|
||||
- name: Run test server
|
||||
run: |
|
||||
sudo bash -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
|
||||
docker run --name www-nginx -p 80:80 -d nginx
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
|
||||
"$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
|
||||
|
||||
- name: Add link
|
||||
run: |
|
||||
docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
|
||||
"$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
|
||||
|
||||
- name: Add stdin link
|
||||
run: |
|
||||
echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
|
||||
echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
|
||||
|
||||
- name: List links
|
||||
run: |
|
||||
docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
|
||||
archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$DOCKER_IMAGE" list)"
|
||||
[[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
[[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
|
||||
|
||||
- name: Start docker-compose stack
|
||||
run: |
|
||||
docker compose run --rm archivebox init
|
||||
docker compose up -d --wait
|
||||
curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
|
||||
curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
|
||||
"$DOCKER_BINARY" compose run --rm archivebox init
|
||||
"$DOCKER_BINARY" compose up -d --wait
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
|
||||
admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
|
||||
[[ "$index_html" == *ArchiveBox* ]]
|
||||
[[ "$admin_js" == *window.django* ]]
|
||||
|
||||
- name: Check added urls show up in index
|
||||
run: |
|
||||
docker compose run --rm archivebox add --plugins=parse_txt_urls 'http://example.com/#test_docker'
|
||||
curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
|
||||
docker compose down
|
||||
"$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://example.com/#test_docker'
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
|
||||
[[ "$index_html" == *'http://example.com/#test_docker'* ]]
|
||||
"$DOCKER_BINARY" compose down
|
||||
|
||||
119
.github/workflows/update-homebrew-tap.yml
vendored
119
.github/workflows/update-homebrew-tap.yml
vendored
@ -1,37 +1,130 @@
|
||||
name: Update downstream packages
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
workflow_call:
|
||||
inputs:
|
||||
release_tag:
|
||||
required: true
|
||||
type: string
|
||||
release_sha:
|
||||
required: true
|
||||
type: string
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: Exact ArchiveBox release tag
|
||||
required: true
|
||||
type: string
|
||||
release_sha:
|
||||
description: Full commit SHA owned by the release tag
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: update-downstream-packages-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
group: update-downstream-packages-${{ inputs.release_tag || github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
dispatch:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Dispatch downstream package updates
|
||||
- name: Checkout release source
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
ref: ${{ inputs.release_sha || github.sha }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
version: "0.11.3"
|
||||
|
||||
- name: Resolve release and Docker binaries through abxpkg
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
ABXPKG_VERSION="$(uv run --no-project python - <<'PY'
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
matches = [package["version"] for package in packages if package["name"] == "abxpkg"]
|
||||
if len(matches) != 1:
|
||||
raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
|
||||
print(matches[0])
|
||||
PY
|
||||
)"
|
||||
test -n "$ABXPKG_VERSION"
|
||||
mkdir -p "$ABXPKG_LIB_DIR/env/bin"
|
||||
echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
|
||||
|
||||
uv run --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
|
||||
--install \
|
||||
--lib="$ABXPKG_LIB_DIR" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
|
||||
--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" \
|
||||
>/dev/null
|
||||
|
||||
for binary_name in uv gh git jq curl docker; do
|
||||
binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
|
||||
test -L "$binary"
|
||||
test -x "$binary"
|
||||
done
|
||||
{
|
||||
echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
|
||||
echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
|
||||
echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
|
||||
echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
|
||||
echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
|
||||
echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify the exact release and dispatch downstream builds
|
||||
env:
|
||||
DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
INPUT_RELEASE_TAG: ${{ inputs.release_tag }}
|
||||
INPUT_RELEASE_SHA: ${{ inputs.release_sha }}
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -z "$DOWNSTREAM_TOKEN" ]; then
|
||||
echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured; downstream schedules will update packages."
|
||||
exit 0
|
||||
fi
|
||||
set -Eeuo pipefail
|
||||
[[ -n "$DOWNSTREAM_TOKEN" ]] || { echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured" >&2; exit 1; }
|
||||
|
||||
RELEASE_TAG="${INPUT_RELEASE_TAG:-${GITHUB_REF_NAME}}"
|
||||
[[ "$RELEASE_TAG" == v* ]]
|
||||
VERSION="${RELEASE_TAG#v}"
|
||||
|
||||
TAG_TARGET="$("$GIT_BINARY" ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${RELEASE_TAG}^{}" | "$UV_BINARY" run --no-project python -c 'import sys; print(next((line.split()[0] for line in sys.stdin if line.strip()), ""))')"
|
||||
if [[ -z "$TAG_TARGET" ]]; then
|
||||
TAG_TARGET="$("$GIT_BINARY" ls-remote "https://github.com/${GITHUB_REPOSITORY}.git" "refs/tags/${RELEASE_TAG}" | "$UV_BINARY" run --no-project python -c 'import sys; print(next((line.split()[0] for line in sys.stdin if line.strip()), ""))')"
|
||||
fi
|
||||
RELEASE_SHA="${INPUT_RELEASE_SHA:-${TAG_TARGET}}"
|
||||
[[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || { echo "${RELEASE_TAG} points to ${TAG_TARGET}, not ${RELEASE_SHA}" >&2; exit 1; }
|
||||
|
||||
"$CURL_BINARY" -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" >/dev/null
|
||||
"$DOCKER_BINARY" manifest inspect "archivebox/archivebox:${VERSION}" >/dev/null
|
||||
"$DOCKER_BINARY" manifest inspect "ghcr.io/archivebox/archivebox:${VERSION}" >/dev/null
|
||||
|
||||
PAYLOAD_FILTER="{event_type: \"archivebox-dev-updated\", client_payload: {ref: \$ref, sha: \$sha, version: \$version}}"
|
||||
PAYLOAD="$("$JQ_BINARY" -nc --arg ref "$RELEASE_TAG" --arg sha "$RELEASE_SHA" --arg version "$VERSION" "$PAYLOAD_FILTER")"
|
||||
for repo in homebrew-archivebox debian-archivebox; do
|
||||
curl --fail-with-body \
|
||||
"$CURL_BINARY" --fail-with-body \
|
||||
-X POST \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "Authorization: Bearer ${DOWNSTREAM_TOKEN}" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
"https://api.github.com/repos/ArchiveBox/${repo}/dispatches" \
|
||||
-d '{"event_type":"archivebox-dev-updated","client_payload":{"ref":"dev"}}'
|
||||
--data "$PAYLOAD"
|
||||
done
|
||||
|
||||
49
AGENTS.md
49
AGENTS.md
@ -18,19 +18,19 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev
|
||||
## Development Setup
|
||||
|
||||
```bash
|
||||
uv sync --dev --all-extras
|
||||
mkdir -p data
|
||||
cd data
|
||||
uv run --project .. archivebox init --install
|
||||
project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
|
||||
archivebox_data="$(mktemp -d)"
|
||||
uv sync --project "$project_dir" --dev --all-extras
|
||||
cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install
|
||||
```
|
||||
|
||||
Run collection commands from inside an initialized data directory:
|
||||
|
||||
```bash
|
||||
cd data
|
||||
uv run --project .. archivebox status
|
||||
uv run --project .. archivebox add 'https://example.com'
|
||||
uv run --project .. archivebox run
|
||||
project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
|
||||
archivebox_data="$(mktemp -d)"
|
||||
cd "$archivebox_data"
|
||||
uv run --project "$project_dir" --no-sync archivebox init --install && uv run --project "$project_dir" --no-sync archivebox status && uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}" && uv run --project "$project_dir" --no-sync archivebox run
|
||||
```
|
||||
|
||||
## User-Facing Setup
|
||||
@ -38,12 +38,12 @@ uv run --project .. archivebox run
|
||||
Recommended CLI install:
|
||||
|
||||
```bash
|
||||
uv tool install --force .
|
||||
project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
|
||||
tool_root="$(mktemp -d)"; export UV_TOOL_DIR="$tool_root/tools" UV_TOOL_BIN_DIR="$tool_root/bin"
|
||||
uv tool install --force "$project_dir"
|
||||
export PLUGINS=parse_txt_urls
|
||||
archivebox_data="$(mktemp -d)"
|
||||
cd "$archivebox_data"
|
||||
archivebox init --install
|
||||
archivebox add --plugins=parse_txt_urls 'https://example.com'
|
||||
cd "$archivebox_data" && "$UV_TOOL_BIN_DIR/archivebox" init --install && "$UV_TOOL_BIN_DIR/archivebox" add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/}"
|
||||
```
|
||||
|
||||
Alternative install methods:
|
||||
@ -57,16 +57,16 @@ Alternative install methods:
|
||||
|
||||
<!--pytest-codeblocks:cont-->
|
||||
```bash
|
||||
archivebox version
|
||||
archivebox help
|
||||
archivebox status
|
||||
archivebox install
|
||||
archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
|
||||
archivebox list --json --with-headers
|
||||
archivebox search 'example'
|
||||
archivebox update --filter-type=domain example.com
|
||||
archivebox remove --filter-type=exact 'https://example.com'
|
||||
archivebox run
|
||||
project_dir="${ARCHIVEBOX_PROJECT_DIR:-$PWD}"
|
||||
archivebox_data="$(mktemp -d)"
|
||||
cd "$archivebox_data" && uv run --project "$project_dir" --no-sync archivebox init --install
|
||||
uv run --project "$project_dir" --no-sync archivebox version && uv run --project "$project_dir" --no-sync archivebox help && uv run --project "$project_dir" --no-sync archivebox status
|
||||
uv run --project "$project_dir" --no-sync archivebox install
|
||||
uv run --project "$project_dir" --no-sync archivebox add --plugins=parse_txt_urls "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}"
|
||||
uv run --project "$project_dir" --no-sync archivebox list --json --with-headers
|
||||
uv run --project "$project_dir" --no-sync archivebox search 'example' && uv run --project "$project_dir" --no-sync archivebox update --filter-type=domain example.com
|
||||
uv run --project "$project_dir" --no-sync archivebox remove --yes --delete --filter-type=exact "${ARCHIVEBOX_DOCS_URL_ONE:-https://example.com/docs-basic-usage}"
|
||||
uv run --project "$project_dir" --no-sync archivebox run
|
||||
```
|
||||
|
||||
## Verification
|
||||
@ -74,8 +74,9 @@ archivebox run
|
||||
Use targeted tests for focused work:
|
||||
|
||||
```bash
|
||||
uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
|
||||
uv run prek run --all-files
|
||||
project_dir="${ARCHIVEBOX_PROJECT_DIR:-.}"
|
||||
uv run --project "$project_dir" --no-sync pytest "$project_dir/archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options" -q
|
||||
(cd "$project_dir" && uv run --no-sync prek run --all-files)
|
||||
```
|
||||
|
||||
Use the full release/deploy loop only when requested:
|
||||
|
||||
@ -10,10 +10,10 @@
|
||||
# --build-context abx-plugins=../abx-plugins \
|
||||
# -t archivebox/abx-dl:dev
|
||||
# docker buildx build . -f Dockerfile \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
|
||||
# --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.11.263 \
|
||||
# -t archivebox:multistage
|
||||
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
|
||||
ARG ABX_DL_IMAGE=archivebox/abx-dl:1.11.263
|
||||
|
||||
FROM archivebox/sonic:1.4.9 AS sonic
|
||||
FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
|
||||
@ -66,9 +66,6 @@ ENV TMP_DIR=/tmp/archivebox \
|
||||
ENV HOME=/home/archivebox \
|
||||
XDG_CONFIG_HOME=/home/archivebox/.config \
|
||||
XDG_CACHE_HOME=/opt/archivebox/lib/cache \
|
||||
ABXPKG_INSTALL_TIMEOUT=600 \
|
||||
ABXPKG_POSTINSTALL_SCRIPTS=True \
|
||||
ABXPKG_MIN_RELEASE_AGE=0 \
|
||||
TIMEOUT=600
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=false \
|
||||
|
||||
@ -8,14 +8,8 @@
|
||||
|
||||
<br/>
|
||||
|
||||
<!--<a href="http://webchat.freenode.net?channels=ArchiveBox&uio=d4"><img src="https://img.shields.io/badge/Community_chat-IRC-%2328A745.svg"/></a>-->
|
||||
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a> <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls"/></a> <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae"/></a> <a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8"/></a>
|
||||
|
||||
<!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
|
||||
curl -fsSL 'https://get.archivebox.io' | bash # (or see pip/brew/Docker instructions below)
|
||||
</code></pre>-->
|
||||
|
||||
</div>
|
||||
<hr/>
|
||||
<br/>
|
||||
@ -171,7 +165,7 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
|
||||
|
||||
# Quickstart
|
||||
|
||||
**🖥 [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker) **👾 CPUs:** `amd64` (`x86_64`), `arm64`, `arm7` <sup>(raspi>=3)</sup><br/>
|
||||
**🖥 [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker) **👾 CPUs:** `amd64` (`x86_64`), `arm64`, `arm7`<br/>
|
||||
|
||||
<br/>
|
||||
|
||||
@ -1291,7 +1285,6 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
|
||||
|
||||
<br/>
|
||||
|
||||
<!--<div align="center" style="text-align: center"><br/><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/04808ac2-3133-44fd-8703-3387e06dc851" width="100%" alt="dependencies graphic"></div>-->
|
||||
|
||||
## Internet Archiving Ecosystem
|
||||
|
||||
|
||||
@ -38,11 +38,6 @@ ASCII_LOGO = """
|
||||
|
||||
PACKAGE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
|
||||
# # Migrations reference models like 'machine.Binary' which need to be importable
|
||||
# if str(PACKAGE_DIR) not in sys.path:
|
||||
# sys.path.append(str(PACKAGE_DIR))
|
||||
|
||||
os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
|
||||
os.environ["TZ"] = "UTC"
|
||||
|
||||
|
||||
@ -23,7 +23,4 @@ urlpatterns = [
|
||||
path("v1/", RedirectView.as_view(url="/api/v1/docs")),
|
||||
path("v1/", v1_api_urls),
|
||||
path("v1", RedirectView.as_view(url="/api/v1/docs")),
|
||||
# ... v2 can be added here ...
|
||||
# path("v2/", v2_api_urls),
|
||||
# path("v2", RedirectView.as_view(url='/api/v2/docs')),
|
||||
]
|
||||
|
||||
@ -78,9 +78,6 @@ class NinjaAPIWithIOCapture(NinjaAPI):
|
||||
response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
|
||||
response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
|
||||
|
||||
# import ipdb; ipdb.set_trace()
|
||||
# print('RESPONDING NOW', response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@ -91,8 +88,6 @@ api = NinjaAPIWithIOCapture(
|
||||
auth=API_AUTH_METHODS,
|
||||
urls_namespace="api-1",
|
||||
docs=Swagger(settings={"persistAuthorization": True}),
|
||||
# docs_decorator=login_required,
|
||||
# renderer=ORJSONRenderer(),
|
||||
)
|
||||
api = register_urls(api)
|
||||
urls = api.urls
|
||||
@ -113,24 +108,7 @@ def generic_exception_handler(request, err):
|
||||
"message": f"{err.__class__.__name__}: {err}",
|
||||
"errors": [
|
||||
"".join(format_exception(err)),
|
||||
# or send simpler parent-only traceback:
|
||||
# *([str(err.__context__)] if getattr(err, '__context__', None) else []),
|
||||
],
|
||||
},
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
# import orjson
|
||||
# from ninja.renderers import BaseRenderer
|
||||
# class ORJSONRenderer(BaseRenderer):
|
||||
# media_type = "application/json"
|
||||
# def render(self, request, data, *, response_status):
|
||||
# return {
|
||||
# "success": True,
|
||||
# "errors": [],
|
||||
# "result": data,
|
||||
# "stdout": ansi_to_html(stdout.getvalue().strip()),
|
||||
# "stderr": ansi_to_html(stderr.getvalue().strip()),
|
||||
# }
|
||||
# return orjson.dumps(data)
|
||||
|
||||
@ -15,8 +15,6 @@ from archivebox.misc.util import ansi_to_html
|
||||
from archivebox.core.models import SnapshotQuerySet
|
||||
|
||||
|
||||
# from .auth import API_AUTH_METHODS
|
||||
|
||||
# router for API that exposes archivebox cli subcommands as REST endpoints
|
||||
router = Router(tags=["ArchiveBox CLI Sub-Commands"])
|
||||
|
||||
|
||||
@ -49,8 +49,6 @@ class CrawlSchema(Schema):
|
||||
tags_str: str
|
||||
config: dict
|
||||
|
||||
# snapshots: List[SnapshotSchema]
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by_id)
|
||||
|
||||
@ -1,7 +0,0 @@
|
||||
# from django.apps import AppConfig
|
||||
|
||||
|
||||
# class BaseModelsConfig(AppConfig):
|
||||
# default_auto_field = 'django.db.models.BigAutoField'
|
||||
|
||||
# name = 'base_models'
|
||||
@ -128,16 +128,12 @@ class ArchiveBoxGroup(click.Group):
|
||||
import_path = cmd_name_or_path
|
||||
modname, funcname = import_path.rsplit(".", 1)
|
||||
|
||||
# print(f'LAZY LOADING {import_path}')
|
||||
mod = import_module(modname)
|
||||
func = vars(mod)[funcname]
|
||||
|
||||
if func.__doc__ is None:
|
||||
raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
|
||||
|
||||
# if not isinstance(cmd, click.BaseCommand):
|
||||
# raise ValueError(f'lazy loading of {import_path} failed - not a click command')
|
||||
|
||||
return func
|
||||
|
||||
|
||||
@ -158,7 +154,6 @@ def cli(ctx, help=False):
|
||||
# then we need to set up the django environment and check that we're in a valid data folder
|
||||
wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
|
||||
if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
|
||||
# print('SETUP DJANGO AND CHECK DATA FOLDER')
|
||||
try:
|
||||
if subcommand == "server":
|
||||
run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
|
||||
|
||||
@ -216,7 +216,7 @@ def add(
|
||||
)
|
||||
from archivebox.services.runner import ensure_background_runner
|
||||
|
||||
ensure_background_runner(allow_under_pytest=True)
|
||||
ensure_background_runner()
|
||||
else:
|
||||
# Foreground mode: run full crawl runner until all work is done
|
||||
print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
|
||||
|
||||
@ -59,28 +59,20 @@ def config(
|
||||
matching_config = {}
|
||||
if search:
|
||||
if config_options:
|
||||
config_options = [
|
||||
core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
|
||||
]
|
||||
matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
|
||||
for config_section in CONFIGS.values():
|
||||
aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
|
||||
search_terms = [key.strip().lower() for key in config_options]
|
||||
|
||||
for search_key in config_options:
|
||||
# search all aliases in the section
|
||||
for alias_key, key in aliases.items():
|
||||
if key in readable_config and search_key.lower() in alias_key.lower():
|
||||
matching_config[key] = dict(config_section)[key]
|
||||
for existing_key, value in readable_config.items():
|
||||
if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
|
||||
matching_config[existing_key] = value
|
||||
|
||||
# search all keys and values in the section
|
||||
for existing_key, value in dict(config_section).items():
|
||||
if existing_key in readable_config and (
|
||||
search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
|
||||
):
|
||||
matching_config[existing_key] = value
|
||||
for existing_key, value in readonly_config.items():
|
||||
if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
|
||||
matching_config[existing_key] = value
|
||||
for alias, key in core_config_aliases.items():
|
||||
if key in readable_config and any(term in alias.lower() for term in search_terms):
|
||||
matching_config[key] = readable_config[key]
|
||||
|
||||
for schema in plugin_schemas.values():
|
||||
for key, metadata in schema.items():
|
||||
if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
|
||||
matching_config[key] = readable_config[key]
|
||||
|
||||
print(printable_config(matching_config))
|
||||
raise SystemExit(not matching_config)
|
||||
|
||||
@ -33,10 +33,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
|
||||
|
||||
config = get_config()
|
||||
|
||||
# if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
|
||||
# print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
|
||||
# print("[red] You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
|
||||
|
||||
is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
|
||||
existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
|
||||
if is_empty and not existing_index:
|
||||
@ -111,9 +107,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
|
||||
print()
|
||||
print(f" √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
|
||||
|
||||
# from django.contrib.auth.models import User
|
||||
# call_command("createsuperuser", interactive=True)
|
||||
|
||||
print()
|
||||
print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
|
||||
|
||||
|
||||
@ -4,11 +4,13 @@ __package__ = "archivebox.cli"
|
||||
__command__ = "archivebox oneshot"
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import rich_click as click
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
|
||||
@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
|
||||
@ -20,7 +22,23 @@ def main(args: tuple[str, ...] = ()) -> None:
|
||||
raise click.ClickException(
|
||||
"Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
|
||||
)
|
||||
raise SystemExit(subprocess.run(["abx-dl", *args]).returncode)
|
||||
abxpkg_binary = Path(sys.executable).with_name("abxpkg")
|
||||
if not abxpkg_binary.is_file():
|
||||
raise click.ClickException(f"abxpkg executable is missing from the ArchiveBox environment: {abxpkg_binary}")
|
||||
abxpkg_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
|
||||
raise SystemExit(
|
||||
subprocess.run(
|
||||
[
|
||||
str(abxpkg_binary),
|
||||
f"--lib={abxpkg_lib_dir}",
|
||||
"--binproviders=env",
|
||||
"--install",
|
||||
"run",
|
||||
"abx-dl",
|
||||
*args,
|
||||
],
|
||||
).returncode,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -9,7 +9,6 @@ from collections.abc import Iterable
|
||||
|
||||
import rich_click as click
|
||||
|
||||
from django.db import OperationalError
|
||||
from django.db.models import QuerySet
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
@ -92,12 +91,6 @@ def remove(
|
||||
# in ``base_models/models.py`` and runs AFTER its row's tx commits — so
|
||||
# rmtree doesn't hold the lock either.
|
||||
#
|
||||
# The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
|
||||
# errors when called inside an atomic block (because it can't safely
|
||||
# release+reacquire a transaction), so we wrap each row's delete in our
|
||||
# own retry loop at this outer (non-atomic) level. Each attempt is a
|
||||
# fresh atomic; an exception cleanly rolls it back before we sleep.
|
||||
retry_interval = 1.0
|
||||
deleted_snapshot_pks = []
|
||||
timed_out = False
|
||||
timeout_error = ""
|
||||
@ -106,23 +99,9 @@ def remove(
|
||||
timed_out = True
|
||||
timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
|
||||
break
|
||||
while True:
|
||||
try:
|
||||
deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
|
||||
if deleted_count:
|
||||
deleted_snapshot_pks.append(pk)
|
||||
break
|
||||
except OperationalError as err:
|
||||
if "database is locked" not in str(err):
|
||||
raise
|
||||
remaining_time = deadline - time.monotonic() if deadline is not None else None
|
||||
if remaining_time is not None and remaining_time <= 0:
|
||||
timed_out = True
|
||||
timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
|
||||
break
|
||||
time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
|
||||
if timed_out:
|
||||
break
|
||||
deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
|
||||
if deleted_count:
|
||||
deleted_snapshot_pks.append(pk)
|
||||
|
||||
all_snapshots = Snapshot.objects.all()
|
||||
remaining_count = all_snapshots.count()
|
||||
|
||||
@ -267,11 +267,13 @@ def server(
|
||||
host, port = _parse_and_validate_bind_spec(bind_spec)
|
||||
|
||||
if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
|
||||
from archivebox.workers.supervisord_util import resolve_env_binary
|
||||
|
||||
log_path = CONSTANTS.LOGS_DIR / "server.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
daemon_env = os.environ.copy()
|
||||
daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1"
|
||||
daemon_cmd = [sys.executable, "-m", "archivebox", "server"]
|
||||
daemon_cmd = [str(resolve_env_binary("archivebox")), "server"]
|
||||
if debug:
|
||||
daemon_cmd.append("--debug")
|
||||
if reload:
|
||||
|
||||
@ -31,14 +31,62 @@ __package__ = "archivebox.cli"
|
||||
__command__ = "archivebox snapshot"
|
||||
|
||||
import sys
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Iterator
|
||||
from itertools import islice
|
||||
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
from django.db.models import QuerySet
|
||||
|
||||
SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
|
||||
SNAPSHOT_LIST_CHUNK_SIZE = 100
|
||||
SNAPSHOT_LIST_CHUNK_SIZE = 5000
|
||||
|
||||
|
||||
def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.core.models import SnapshotTag
|
||||
|
||||
fields = (
|
||||
"id",
|
||||
"crawl_id",
|
||||
"url",
|
||||
"title",
|
||||
"bookmarked_at",
|
||||
"created_at",
|
||||
"timestamp",
|
||||
"depth",
|
||||
"status",
|
||||
"fs_version",
|
||||
"output_size",
|
||||
)
|
||||
rows = queryset.values(*fields).iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE)
|
||||
while batch := list(islice(rows, SNAPSHOT_LIST_CHUNK_SIZE)):
|
||||
tags_by_snapshot = {row["id"]: [] for row in batch}
|
||||
tag_rows = (
|
||||
SnapshotTag.objects.filter(snapshot_id__in=tags_by_snapshot).order_by("tag__name").values_list("snapshot_id", "tag__name")
|
||||
)
|
||||
for snapshot_id, tag_name in tag_rows:
|
||||
tags_by_snapshot[snapshot_id].append(tag_name)
|
||||
|
||||
for row in batch:
|
||||
archive_size = int(row["output_size"] or 0)
|
||||
yield {
|
||||
"type": "Snapshot",
|
||||
"schema_version": VERSION,
|
||||
"id": str(row["id"]),
|
||||
"crawl_id": str(row["crawl_id"]),
|
||||
"url": row["url"],
|
||||
"title": row["title"],
|
||||
"tags": ",".join(sorted(tags_by_snapshot[row["id"]])),
|
||||
"bookmarked_at": row["bookmarked_at"].isoformat() if row["bookmarked_at"] else None,
|
||||
"created_at": row["created_at"].isoformat() if row["created_at"] else None,
|
||||
"timestamp": row["timestamp"],
|
||||
"depth": row["depth"],
|
||||
"status": row["status"],
|
||||
"fs_version": row["fs_version"],
|
||||
"archive_size": archive_size,
|
||||
"output_size": archive_size,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@ -296,8 +344,8 @@ def list_snapshots(
|
||||
return 0
|
||||
|
||||
if not is_tty:
|
||||
for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
|
||||
write_record(snapshot.to_json())
|
||||
for snapshot_json in iter_snapshot_json(queryset):
|
||||
write_record(snapshot_json)
|
||||
count += 1
|
||||
rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
@ -293,5 +293,3 @@ CONSTANTS = ConstantsDict
|
||||
CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")})
|
||||
|
||||
# add all key: values to globals() for easier importing, e.g.:
|
||||
# from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...
|
||||
# globals().update(CONSTANTS)
|
||||
|
||||
@ -2,7 +2,6 @@ __package__ = "archivebox.config"
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@ -37,7 +36,6 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
|
||||
global DJANGO_SET_UP
|
||||
|
||||
if DJANGO_SET_UP:
|
||||
# raise Exception('django is already set up!')
|
||||
# TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
|
||||
return
|
||||
|
||||
@ -59,10 +57,10 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
|
||||
with SudoPermission(uid=0):
|
||||
# running as root is a special case where it's ok to be a bit slower
|
||||
# make sure data dir is always owned by the correct user
|
||||
subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(CONSTANTS.DATA_DIR)], stderr=subprocess.DEVNULL)
|
||||
os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP)
|
||||
if CONSTANTS.DATA_DIR.exists():
|
||||
for child in CONSTANTS.DATA_DIR.iterdir():
|
||||
subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(child)], stderr=subprocess.DEVNULL)
|
||||
os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP)
|
||||
|
||||
# Suppress the "database access during app initialization" warning
|
||||
# This warning can be triggered during django.setup() but is safe to ignore
|
||||
@ -156,17 +154,6 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
|
||||
f"No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)"
|
||||
)
|
||||
|
||||
# https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
|
||||
# if settings.DEBUG_LOGFIRE:
|
||||
# from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
|
||||
# SQLite3Instrumentor().instrument()
|
||||
|
||||
# import logfire
|
||||
|
||||
# logfire.configure()
|
||||
# logfire.instrument_django(is_sql_commentor_enabled=True)
|
||||
# logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
|
||||
|
||||
@ -5,7 +5,6 @@ import socket
|
||||
import hashlib
|
||||
import tempfile
|
||||
import platform
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from functools import cache
|
||||
from datetime import datetime
|
||||
@ -69,9 +68,9 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
|
||||
if IS_ROOT:
|
||||
with SudoPermission(uid=0):
|
||||
if ARCHIVEBOX_USER == 0:
|
||||
subprocess.run(["chmod", "777", str(collection_id_file)])
|
||||
collection_id_file.chmod(0o777)
|
||||
else:
|
||||
subprocess.run(["chown", str(ARCHIVEBOX_USER), str(collection_id_file)])
|
||||
os.chown(collection_id_file, ARCHIVEBOX_USER, -1)
|
||||
except (OSError, FileNotFoundError, PermissionError):
|
||||
pass
|
||||
return collection_id
|
||||
@ -129,7 +128,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
|
||||
if chown:
|
||||
# try fixing it using sudo permissions
|
||||
with SudoPermission(uid=uid, fallback=fallback):
|
||||
subprocess.run(["chown", f"{uid}:{gid}", str(dir_path)], stderr=subprocess.DEVNULL)
|
||||
os.chown(dir_path, uid, gid)
|
||||
return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False)
|
||||
return False
|
||||
|
||||
@ -348,11 +347,6 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
|
||||
and os.access(tmp_dir, os.W_OK)
|
||||
and tmp_dir_socket_path_is_short_enough(tmp_dir),
|
||||
},
|
||||
# "CACHE_DIR": {
|
||||
# "path": CACHE_DIR.resolve(),
|
||||
# "enabled": True,
|
||||
# "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK), # read + write
|
||||
# },
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@ -37,8 +37,6 @@ HOSTNAME: str = cast(str, max([socket.gethostname(), platform.node()], key=len))
|
||||
|
||||
IS_ROOT = RUNNING_AS_UID == 0
|
||||
IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
|
||||
# IN_DOCKER_COMPOSE = # TODO: figure out a way to detect if running in docker compose
|
||||
|
||||
|
||||
FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
|
||||
FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
|
||||
|
||||
@ -36,7 +36,6 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
|
||||
# building docs, pyproject.toml is not available
|
||||
pass
|
||||
|
||||
# raise Exception('Failed to detect installed archivebox version!')
|
||||
return "dev"
|
||||
|
||||
|
||||
@ -115,53 +114,4 @@ def get_BUILD_TIME() -> str:
|
||||
return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")
|
||||
|
||||
|
||||
# def get_versions_available_on_github(config):
|
||||
# """
|
||||
# returns a dictionary containing the ArchiveBox GitHub release info for
|
||||
# the recommended upgrade version and the currently installed version
|
||||
# """
|
||||
|
||||
# # we only want to perform the (relatively expensive) check for new versions
|
||||
# # when its most relevant, e.g. when the user runs a long-running command
|
||||
# subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
|
||||
# long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
|
||||
# if subcommand_run_by_user not in long_running_commands:
|
||||
# return None
|
||||
|
||||
# github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
|
||||
# response = requests.get(github_releases_api)
|
||||
# if response.status_code != 200:
|
||||
# stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
|
||||
# return None
|
||||
# all_releases = response.json()
|
||||
|
||||
# installed_version = parse_version_string(config['VERSION'])
|
||||
|
||||
# # find current version or nearest older version (to link to)
|
||||
# current_version = None
|
||||
# for idx, release in enumerate(all_releases):
|
||||
# release_version = parse_version_string(release['tag_name'])
|
||||
# if release_version <= installed_version:
|
||||
# current_version = release
|
||||
# break
|
||||
|
||||
# current_version = current_version or all_releases[-1]
|
||||
|
||||
# # recommended version is whatever comes after current_version in the release list
|
||||
# # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
|
||||
# try:
|
||||
# recommended_version = all_releases[idx+1]
|
||||
# except IndexError:
|
||||
# recommended_version = None
|
||||
|
||||
# return {'recommended_version': recommended_version, 'current_version': current_version}
|
||||
|
||||
# def can_upgrade(config):
|
||||
# if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
|
||||
# recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
|
||||
# current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
|
||||
# return recommended_version > current_version
|
||||
# return False
|
||||
|
||||
|
||||
VERSION: str = detect_installed_version()
|
||||
|
||||
@ -328,15 +328,12 @@ class ArchiveResultInline(admin.TabularInline):
|
||||
name = "Archive Results Log"
|
||||
model = ArchiveResult
|
||||
parent_model = Snapshot
|
||||
# fk_name = 'snapshot'
|
||||
extra = 0
|
||||
sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version")
|
||||
readonly_fields = ("id", "result_id", "completed", "command", "version")
|
||||
fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str")
|
||||
# exclude = ('id',)
|
||||
ordering = ("end_ts",)
|
||||
show_change_link = True
|
||||
# # classes = ['collapse']
|
||||
|
||||
def get_parent_object_from_request(self, request):
|
||||
resolved = resolve(request.path_info)
|
||||
@ -371,7 +368,6 @@ class ArchiveResultInline(admin.TabularInline):
|
||||
base_fields = formset.form.base_fields
|
||||
snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
|
||||
|
||||
# import ipdb; ipdb.set_trace()
|
||||
# formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
|
||||
|
||||
# default values for new entries
|
||||
@ -817,8 +813,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
|
||||
else:
|
||||
root_dir = str(snapshot_dir)
|
||||
|
||||
# print(root_dir, str(list(os.walk(root_dir))))
|
||||
|
||||
for root, dirs, files in os.walk(root_dir):
|
||||
depth = root.replace(root_dir, "").count(os.sep) + 1
|
||||
if depth > 2:
|
||||
|
||||
@ -43,7 +43,6 @@ from archivebox.core.permissions import (
|
||||
from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
|
||||
|
||||
|
||||
# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
|
||||
GLOBAL_CONTEXT = {}
|
||||
|
||||
SNAPSHOT_PERMISSION_META = PERMISSIONS_META
|
||||
@ -72,13 +71,6 @@ class SnapshotActionForm(ActionForm):
|
||||
|
||||
return [name.strip() for name in tags_str.split(",") if name.strip()]
|
||||
|
||||
# TODO: allow selecting actions for specific extractor plugins? is this useful?
|
||||
# plugin = forms.ChoiceField(
|
||||
# choices=ArchiveResult.PLUGIN_CHOICES,
|
||||
# required=False,
|
||||
# widget=forms.MultileChoiceField(attrs={'class': "form-control"})
|
||||
# )
|
||||
|
||||
|
||||
class TagNameListFilter(admin.SimpleListFilter):
|
||||
title = "By tag name"
|
||||
@ -748,31 +740,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
html = Template("""{{bookmarked_date}} (<code>{{timestamp}}</code>)""")
|
||||
return mark_safe(html.render(context))
|
||||
|
||||
# pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S')
|
||||
# return f'{pretty_time} ({obj.timestamp})'
|
||||
|
||||
# TODO: figure out a different way to do this, you cant nest forms so this doenst work
|
||||
# def action(self, obj):
|
||||
# # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
|
||||
# # action: update_snapshots
|
||||
# # select_across: 0
|
||||
# # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
|
||||
# return format_html(
|
||||
# '''
|
||||
# <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
|
||||
# <input type="hidden" name="csrfmiddlewaretoken" value="{}">
|
||||
# <input type="hidden" name="_selected_action" value="{}">
|
||||
# <button name="update_snapshots">Check</button>
|
||||
# <button name="update_titles">Pull title + favicon</button>
|
||||
# <button name="update_snapshots">Update</button>
|
||||
# <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
|
||||
# <button name="delete_snapshots">Permanently delete</button>
|
||||
# </form>
|
||||
# ''',
|
||||
# csrf.get_token(self.request),
|
||||
# obj.pk,
|
||||
# )
|
||||
|
||||
@admin.display(description="")
|
||||
def admin_actions(self, obj):
|
||||
summary_url = self.get_snapshot_view_url(obj)
|
||||
@ -1149,9 +1116,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
mark_safe("".join(output)),
|
||||
)
|
||||
|
||||
@admin.display(
|
||||
# ordering='archiveresult_count'
|
||||
)
|
||||
@admin.display()
|
||||
def size(self, obj):
|
||||
request = self.request
|
||||
config = request.archivebox_config
|
||||
@ -1401,11 +1366,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
extra_context["snapshot_is_grid_view"] = True
|
||||
return self.changelist_view(request, extra_context=extra_context)
|
||||
|
||||
# for debugging, uncomment this to print all requests:
|
||||
# def changelist_view(self, request, extra_context=None):
|
||||
# print('[*] Got request', request.method, request.POST)
|
||||
# return super().changelist_view(request, extra_context=None)
|
||||
|
||||
@admin.action(
|
||||
description="🔁 Redo Failed",
|
||||
)
|
||||
|
||||
@ -73,16 +73,3 @@ _patch_thread_sensitive_context_shutdown()
|
||||
|
||||
# Standard Django ASGI application (no websockets/channels needed)
|
||||
application = get_asgi_application()
|
||||
|
||||
# If websocket support is needed later, install channels and use:
|
||||
# from channels.routing import ProtocolTypeRouter, URLRouter
|
||||
# from channels.auth import AuthMiddlewareStack
|
||||
# from channels.security.websocket import AllowedHostsOriginValidator
|
||||
# from archivebox.core.routing import websocket_urlpatterns
|
||||
#
|
||||
# application = ProtocolTypeRouter({
|
||||
# "http": get_asgi_application(),
|
||||
# "websocket": AllowedHostsOriginValidator(
|
||||
# AuthMiddlewareStack(URLRouter(websocket_urlpatterns))
|
||||
# ),
|
||||
# })
|
||||
|
||||
@ -73,7 +73,6 @@ def detect_timezone(request, activate: bool = True):
|
||||
tz = timezone.get_fixed_timezone(int(gmt_offset))
|
||||
if activate:
|
||||
timezone.activate(tz)
|
||||
# print('GMT_OFFSET', gmt_offset, tz)
|
||||
return tz
|
||||
|
||||
|
||||
@ -153,7 +152,6 @@ def CacheControlMiddleware(get_response):
|
||||
request.archivebox_config = config
|
||||
policy = "private" if config.PERMISSIONS == "private" else "public"
|
||||
response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
|
||||
# print('Set Cache-Control header to', response['Cache-Control'])
|
||||
return response
|
||||
|
||||
return middleware
|
||||
|
||||
@ -1032,21 +1032,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir))
|
||||
delattr(self, "_pending_fs_migration_cleanup")
|
||||
|
||||
# if is_new:
|
||||
# from archivebox.misc.logging_util import log_worker_event
|
||||
# log_worker_event(
|
||||
# worker_type='DB',
|
||||
# event='Created Snapshot',
|
||||
# indent_level=2,
|
||||
# url=self.url,
|
||||
# metadata={
|
||||
# 'id': str(self.id),
|
||||
# 'crawl_id': str(self.crawl_id),
|
||||
# 'depth': self.depth,
|
||||
# 'status': self.status,
|
||||
# },
|
||||
# )
|
||||
|
||||
# =========================================================================
|
||||
# Filesystem Migration Methods
|
||||
# =========================================================================
|
||||
@ -4063,21 +4048,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields:
|
||||
transaction.on_commit(type(self).clear_majority_status_cache)
|
||||
|
||||
# if is_new:
|
||||
# from archivebox.misc.logging_util import log_worker_event
|
||||
# log_worker_event(
|
||||
# worker_type='DB',
|
||||
# event='Created ArchiveResult',
|
||||
# indent_level=3,
|
||||
# plugin=self.plugin,
|
||||
# metadata={
|
||||
# 'id': str(self.id),
|
||||
# 'snapshot_id': str(self.snapshot_id),
|
||||
# 'snapshot_url': str(self.snapshot.url)[:64],
|
||||
# 'status': self.status,
|
||||
# },
|
||||
# )
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
snapshot_id = self.snapshot_id
|
||||
deleted = super().delete(*args, **kwargs)
|
||||
|
||||
@ -18,7 +18,7 @@ from .settings_logging import SETTINGS_LOGGING
|
||||
|
||||
|
||||
IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3]
|
||||
IS_TESTING = "test" in sys.argv[:3] or "PYTEST_CURRENT_TEST" in os.environ
|
||||
IS_TESTING = "test" in sys.argv[:3]
|
||||
IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3]
|
||||
IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv
|
||||
CONFIG = get_config()
|
||||
@ -103,7 +103,6 @@ MIDDLEWARE = [
|
||||
### Authentication Settings
|
||||
################################################################################
|
||||
|
||||
# AUTH_USER_MODEL = 'auth.User' # cannot be easily changed unfortunately
|
||||
|
||||
AUTHENTICATION_BACKENDS = [
|
||||
"django.contrib.auth.backends.RemoteUserBackend",
|
||||
@ -181,23 +180,11 @@ TEMPLATES_DIR_NAME = "templates"
|
||||
CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
|
||||
STATICFILES_DIRS = [
|
||||
*([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
|
||||
# *[
|
||||
# str(plugin_dir / 'static')
|
||||
# for plugin_dir in PLUGIN_DIRS.values()
|
||||
# if (plugin_dir / 'static').is_dir()
|
||||
# ],
|
||||
# Additional static file dirs from plugins
|
||||
str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "static"),
|
||||
]
|
||||
|
||||
TEMPLATE_DIRS = [
|
||||
*([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
|
||||
# *[
|
||||
# str(plugin_dir / 'templates')
|
||||
# for plugin_dir in PLUGIN_DIRS.values()
|
||||
# if (plugin_dir / 'templates').is_dir()
|
||||
# ],
|
||||
# Additional template dirs from plugins
|
||||
str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "core"),
|
||||
str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "admin"),
|
||||
str(PACKAGE_DIR / TEMPLATES_DIR_NAME),
|
||||
@ -225,10 +212,6 @@ TEMPLATES = [
|
||||
### External Service Settings
|
||||
################################################################################
|
||||
|
||||
# CACHE_DB_FILENAME = 'cache.sqlite3'
|
||||
# CACHE_DB_PATH = CONSTANTS.CACHE_DIR / CACHE_DB_FILENAME
|
||||
# CACHE_DB_TABLE = 'django_cache'
|
||||
|
||||
DATABASE_NAME = CONFIG.DATABASE_NAME
|
||||
SQLITE_JOURNAL_MODE = CONFIG.SQLITE_JOURNAL_MODE
|
||||
SQLITE_MMAP_SIZE = CONFIG.SQLITE_MMAP_SIZE
|
||||
@ -267,14 +250,6 @@ DATABASES = {
|
||||
"NAME": DATABASE_NAME,
|
||||
**SQLITE_CONNECTION_OPTIONS,
|
||||
},
|
||||
# "filestore": {
|
||||
# "NAME": CONSTANTS.FILESTORE_DATABASE_FILE,
|
||||
# **SQLITE_CONNECTION_OPTIONS,
|
||||
# },
|
||||
# 'cache': {
|
||||
# 'NAME': CACHE_DB_PATH,
|
||||
# **SQLITE_CONNECTION_OPTIONS,
|
||||
# },
|
||||
}
|
||||
MIGRATION_MODULES = {"signal_webhooks": None}
|
||||
|
||||
@ -285,43 +260,10 @@ MIGRATION_MODULES = {"signal_webhooks": None}
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
||||
|
||||
|
||||
# class FilestoreDBRouter:
|
||||
# """
|
||||
# A router to store all the File models in the filestore.sqlite3 database.
|
||||
# This data just mirrors what is in the file system, so we want to keep it in a separate database
|
||||
# from the main index database to avoid contention.
|
||||
# """
|
||||
|
||||
# route_app_labels = {"filestore"}
|
||||
# db_name = "filestore"
|
||||
|
||||
# def db_for_read(self, model, **hints):
|
||||
# if model._meta.app_label in self.route_app_labels:
|
||||
# return self.db_name
|
||||
# return 'default'
|
||||
|
||||
# def db_for_write(self, model, **hints):
|
||||
# if model._meta.app_label in self.route_app_labels:
|
||||
# return self.db_name
|
||||
# return 'default'
|
||||
|
||||
# def allow_relation(self, obj1, obj2, **hints):
|
||||
# if obj1._meta.app_label in self.route_app_labels or obj2._meta.app_label in self.route_app_labels:
|
||||
# return obj1._meta.app_label == obj2._meta.app_label
|
||||
# return None
|
||||
|
||||
# def allow_migrate(self, db, app_label, model_name=None, **hints):
|
||||
# if app_label in self.route_app_labels:
|
||||
# return db == self.db_name
|
||||
# return db == "default"
|
||||
|
||||
DATABASE_ROUTERS = []
|
||||
|
||||
CACHES = {
|
||||
"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"},
|
||||
# 'sqlite': {'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'cache'},
|
||||
# 'dummy': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
|
||||
# 'filebased': {"BACKEND": "django.core.cache.backends.filebased.FileBasedCache", "LOCATION": CACHE_DIR / 'cache_filebased'},
|
||||
}
|
||||
|
||||
EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
|
||||
@ -666,16 +608,3 @@ if DEBUG_REQUESTS_TRACKER:
|
||||
r"^SELECT .* FROM django_migrations WHERE app = 'auth'",
|
||||
),
|
||||
}
|
||||
|
||||
# # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.)
|
||||
# DEBUG_LOGFIRE = False
|
||||
# DEBUG_LOGFIRE = DEBUG_LOGFIRE and os.access(DATA_DIR / '.logfire', os.W_OK) and (DATA_DIR / '.logfire').is_dir()
|
||||
|
||||
|
||||
# For usage with https://www.jetadmin.io/integrations/django
|
||||
# INSTALLED_APPS += ['jet_django']
|
||||
# JET_PROJECT = 'archivebox'
|
||||
# JET_TOKEN = 'some-api-token-here'
|
||||
|
||||
|
||||
# import ipdb; ipdb.set_trace()
|
||||
|
||||
@ -130,8 +130,6 @@ if os.access(LOGS_DIR, os.W_OK) and LOGS_DIR.is_dir():
|
||||
ERROR_LOG = LOGS_DIR / "errors.log"
|
||||
else:
|
||||
# historically too many edge cases here around creating log dir w/ correct permissions early on
|
||||
# if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr
|
||||
# print(f'[!] WARNING: data/logs dir does not exist. Logging to temp file: {ERROR_LOG}')
|
||||
pass
|
||||
|
||||
LOG_LEVEL_DATABASE = "WARNING" # change to DEBUG to log all SQL queries
|
||||
|
||||
@ -30,12 +30,6 @@ from archivebox.progressmonitor.views import live_progress_view
|
||||
from archivebox.search.views import public_snapshot_search_stream_view
|
||||
from abx_plugins.plugins.opencode.views import opencode_proxy_view
|
||||
|
||||
|
||||
# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
|
||||
# from archivebox.config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
|
||||
# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
|
||||
|
||||
|
||||
CONFIG = get_config()
|
||||
DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv)
|
||||
|
||||
@ -86,7 +80,6 @@ urlpatterns = [
|
||||
path("api/", include("archivebox.api.urls"), name="api"),
|
||||
path("health/", HealthCheckView.as_view(), name="healthcheck"),
|
||||
path("error/", lambda request: _raise_test_error(request)),
|
||||
# path('jet_api/', include('jet_django.urls')), Enable to use https://www.jetadmin.io/integrations/django
|
||||
path("index.html", RedirectView.as_view(url="/")),
|
||||
path("", HomepageView.as_view(), name="Home"),
|
||||
]
|
||||
@ -101,36 +94,3 @@ if getattr(settings, "DEBUG_TOOLBAR", False):
|
||||
|
||||
if getattr(settings, "DEBUG_REQUESTS_TRACKER", False) and find_spec("requests_tracker"):
|
||||
urlpatterns += [path("__requests_tracker__/", include("requests_tracker.urls"))]
|
||||
|
||||
|
||||
# # Proposed FUTURE URLs spec
|
||||
# path('', HomepageView)
|
||||
# path('/add', AddView)
|
||||
# path('/public', PublicIndexView)
|
||||
# path('/snapshot/:slug', SnapshotView)
|
||||
|
||||
# path('/admin', admin.site.urls)
|
||||
# path('/accounts', django.contrib.auth.urls)
|
||||
|
||||
# # Proposed REST API spec
|
||||
# # :slugs can be uuid, short_uuid, or any of the unique index_fields
|
||||
# path('api/v1/'),
|
||||
# path('api/v1/core/' [GET])
|
||||
# path('api/v1/core/snapshot/', [GET, POST, PUT]),
|
||||
# path('api/v1/core/snapshot/:slug', [GET, PATCH, DELETE]),
|
||||
# path('api/v1/core/archiveresult', [GET, POST, PUT]),
|
||||
# path('api/v1/core/archiveresult/:slug', [GET, PATCH, DELETE]),
|
||||
# path('api/v1/core/tag/', [GET, POST, PUT]),
|
||||
# path('api/v1/core/tag/:slug', [GET, PATCH, DELETE]),
|
||||
|
||||
# path('api/v1/cli/', [GET])
|
||||
# path('api/v1/cli/{add,list,config,...}', [POST]), # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
|
||||
|
||||
# path('api/v1/extractors/', [GET])
|
||||
# path('api/v1/extractors/:extractor/', [GET]),
|
||||
# path('api/v1/extractors/:extractor/:func', [GET, POST]), # pass query as args directly to chosen function
|
||||
|
||||
# future, just an idea:
|
||||
# path('api/v1/scheduler/', [GET])
|
||||
# path('api/v1/scheduler/task/', [GET, POST, PUT]),
|
||||
# path('api/v1/scheduler/task/:slug', [GET, PATCH, DELETE]),
|
||||
|
||||
@ -1746,8 +1746,6 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
find_config_default(key) or "See here...",
|
||||
),
|
||||
)
|
||||
# rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
|
||||
# rows['Aliases'].append(', '.join(find_config_aliases(key)))
|
||||
|
||||
section = "CONSTANT"
|
||||
for key in CONSTANTS_CONFIG.keys():
|
||||
@ -1763,8 +1761,6 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
find_config_default(key) or "See here...",
|
||||
),
|
||||
)
|
||||
# rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
|
||||
# rows['Aliases'].append('')
|
||||
|
||||
return TableContext(
|
||||
title="Computed Configuration Values",
|
||||
@ -1820,7 +1816,6 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
|
||||
((color, source, value) for source, value, color in sources_info),
|
||||
)
|
||||
|
||||
# aliases = USER_CONFIG.get(key, {}).get("aliases", [])
|
||||
aliases = []
|
||||
|
||||
if key in CONSTANTS_CONFIG:
|
||||
|
||||
@ -209,7 +209,13 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
return resolve_delete_after_config_value(self.config)
|
||||
|
||||
def pause(self, *, save: bool = True) -> bool:
|
||||
return super().pause(save=save)
|
||||
paused = super().pause(save=save)
|
||||
if paused and save and self.pk:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator():
|
||||
snapshot.pause()
|
||||
return paused
|
||||
|
||||
def resume(self, *, when=None, save: bool = True) -> bool:
|
||||
resumed = super().resume(when=when, save=save)
|
||||
@ -338,20 +344,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
removed_tag_names=removed_tag_names,
|
||||
),
|
||||
)
|
||||
# if is_new:
|
||||
# from archivebox.misc.logging_util import log_worker_event
|
||||
# first_url = self.get_urls_list()[0] if self.get_urls_list() else ''
|
||||
# log_worker_event(
|
||||
# worker_type='DB',
|
||||
# event='Created Crawl',
|
||||
# indent_level=1,
|
||||
# metadata={
|
||||
# 'id': str(self.id),
|
||||
# 'first_url': first_url[:64],
|
||||
# 'max_depth': self.max_depth,
|
||||
# 'status': self.status,
|
||||
# },
|
||||
# )
|
||||
|
||||
def update_child_snapshot_permissions(self, old_permissions: str | None, new_permissions: str | None) -> int:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
@ -18,6 +18,24 @@ PACKAGE_DIR = Path(__file__).parent
|
||||
DATA_DIR = Path(os.getcwd()).resolve()
|
||||
|
||||
|
||||
def _run_abxpkg_host_binary(name: str, *args: str) -> subprocess.CompletedProcess[str]:
|
||||
from abxpkg import EnvProvider
|
||||
from archivebox.config.constants import CONSTANTS
|
||||
|
||||
provider = EnvProvider(install_root=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR / "env", PATH=os.environ["PATH"])
|
||||
if name == "system_profiler":
|
||||
provider = provider.get_provider_with_overrides(
|
||||
overrides={name: {"version": platform.mac_ver()[0] or "0.0.0"}},
|
||||
)
|
||||
loaded = provider.load(name)
|
||||
if loaded is None or loaded.loaded_abspath is None:
|
||||
raise RuntimeError(f"abxpkg could not resolve {name}")
|
||||
projection = Path(loaded.loaded_abspath)
|
||||
if not projection.is_symlink() or not os.access(projection, os.X_OK):
|
||||
raise RuntimeError(f"abxpkg did not project {name} into {projection}")
|
||||
return subprocess.run([str(projection), *args], capture_output=True, text=True, check=True)
|
||||
|
||||
|
||||
def get_vm_info():
|
||||
hw_in_docker = bool(os.getenv("IN_DOCKER", False) in ("1", "true", "True", "TRUE"))
|
||||
hw_in_vm = False
|
||||
@ -49,7 +67,7 @@ def get_vm_info():
|
||||
# Serial Number (system): M230YYTD77
|
||||
# Hardware UUID: 39A12B50-1972-5910-8BEE-235AD20C8EE3
|
||||
# ...
|
||||
result = subprocess.run(["system_profiler", "SPHardwareDataType"], capture_output=True, text=True, check=True)
|
||||
result = _run_abxpkg_host_binary("system_profiler", "SPHardwareDataType")
|
||||
for line in result.stdout.split("\n"):
|
||||
if "Model Name:" in line:
|
||||
hw_product = line.split(":", 1)[-1].strip()
|
||||
@ -73,7 +91,7 @@ def get_vm_info():
|
||||
# UUID: fb65f41c-ec24-4539-beaf-f941903bdb2c
|
||||
# ...
|
||||
# Family: DigitalOcean_Droplet
|
||||
dmidecode = subprocess.run(["dmidecode", "-t", "system"], capture_output=True, text=True, check=True)
|
||||
dmidecode = _run_abxpkg_host_binary("dmidecode", "-t", "system")
|
||||
for line in dmidecode.stdout.split("\n"):
|
||||
if "Manufacturer:" in line:
|
||||
hw_manufacturer = line.split(":", 1)[-1].strip()
|
||||
@ -90,7 +108,7 @@ def get_vm_info():
|
||||
|
||||
# Check for QEMU explicitly in pmap output
|
||||
try:
|
||||
result = subprocess.run(["pmap", "1"], capture_output=True, text=True, check=True)
|
||||
result = _run_abxpkg_host_binary("pmap", "1")
|
||||
if "qemu" in result.stdout.lower():
|
||||
hw_in_vm = True
|
||||
except Exception:
|
||||
@ -174,7 +192,7 @@ def get_isp_info(ip=None):
|
||||
# Get system DNS resolver servers
|
||||
dns_server = None
|
||||
try:
|
||||
result = subprocess.run(["dig", "example.com", "A"], capture_output=True, text=True, check=True).stdout
|
||||
result = _run_abxpkg_host_binary("dig", "example.com", "A").stdout
|
||||
dns_server = result.split(";; SERVER: ", 1)[-1].split("\n")[0].split("#")[0].strip()
|
||||
except Exception:
|
||||
try:
|
||||
@ -183,10 +201,6 @@ def get_isp_info(ip=None):
|
||||
dns_server = "127.0.0.1"
|
||||
print(f"[red]:warning: WARNING: Could not determine DNS server, using {dns_server}[/red]")
|
||||
|
||||
# Get DNS resolver's ISP name
|
||||
# url = f'https://ipapi.co/{dns_server}/json/'
|
||||
# dns_isp = json.loads(urllib.request.urlopen(url).read().decode()).get('org', 'Unknown')
|
||||
|
||||
return {
|
||||
"isp": isp,
|
||||
"city": city,
|
||||
@ -225,7 +239,7 @@ def get_os_info() -> dict[str, Any]:
|
||||
os_release = "macOS " + platform.mac_ver()[0]
|
||||
else:
|
||||
try:
|
||||
os_release = subprocess.run(["lsb_release", "-ds"], capture_output=True, text=True, check=True).stdout.strip()
|
||||
os_release = _run_abxpkg_host_binary("lsb_release", "-ds").stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@ -400,9 +400,6 @@ class NetworkInterface(ModelWithHealthStats):
|
||||
city = models.CharField(max_length=63, default="", null=False)
|
||||
region = models.CharField(max_length=63, default="", null=False)
|
||||
country = models.CharField(max_length=63, default="", null=False)
|
||||
# num_uses_failed = models.PositiveIntegerField(default=0) # from ModelWithHealthStats
|
||||
# num_uses_succeeded = models.PositiveIntegerField(default=0) # from ModelWithHealthStats
|
||||
|
||||
objects = NetworkInterfaceManager() # pyright: ignore[reportIncompatibleVariableOverride]
|
||||
machine_id: uuid.UUID
|
||||
|
||||
@ -1493,15 +1490,11 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
ppid = os.getppid()
|
||||
machine = machine or Machine.current()
|
||||
|
||||
# Debug logging
|
||||
# print(f"DEBUG _find_parent_process: my_pid={os.getpid()}, ppid={ppid}", file=sys.stderr)
|
||||
|
||||
# Get parent process start time from OS
|
||||
try:
|
||||
os_parent = psutil.Process(ppid)
|
||||
os_parent_start = os_parent.create_time()
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
# print(f"DEBUG _find_parent_process: Parent process {ppid} not accessible", file=sys.stderr)
|
||||
return None # Parent process doesn't exist
|
||||
|
||||
# Find matching Process record
|
||||
@ -1512,18 +1505,13 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
started_at__gte=timezone.now() - PID_REUSE_WINDOW,
|
||||
).order_by("-started_at")
|
||||
|
||||
# print(f"DEBUG _find_parent_process: Found {candidates.count()} candidates for ppid={ppid}", file=sys.stderr)
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate.started_at:
|
||||
db_start_time = candidate.started_at.timestamp()
|
||||
time_diff = abs(db_start_time - os_parent_start)
|
||||
# print(f"DEBUG _find_parent_process: Checking candidate id={candidate.id} time_diff={time_diff:.2f}s tolerance={START_TIME_TOLERANCE}s", file=sys.stderr)
|
||||
if time_diff < START_TIME_TOLERANCE:
|
||||
# print(f"DEBUG _find_parent_process: MATCH! Returning parent id={candidate.id} pid={candidate.pid}", file=sys.stderr)
|
||||
return candidate
|
||||
|
||||
# print(f"DEBUG _find_parent_process: No matching parent found for ppid={ppid}", file=sys.stderr)
|
||||
return None # No matching ArchiveBox parent process
|
||||
|
||||
@classmethod
|
||||
@ -2142,12 +2130,8 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
# Process exited - read output and copy to DB
|
||||
if self.stdout_file and self.stdout_file.exists():
|
||||
self.stdout = self.stdout_file.read_text(errors="replace")
|
||||
# TODO: Uncomment to cleanup (keeping for debugging for now)
|
||||
# self.stdout_file.unlink(missing_ok=True)
|
||||
if self.stderr_file and self.stderr_file.exists():
|
||||
self.stderr = self.stderr_file.read_text(errors="replace")
|
||||
# TODO: Uncomment to cleanup (keeping for debugging for now)
|
||||
# self.stderr_file.unlink(missing_ok=True)
|
||||
|
||||
self.exit_code = self.exit_code if self.exit_code is not None else _default_exit_code_for_unowned_process(self.process_type)
|
||||
if self.exit_code == -1:
|
||||
@ -2462,6 +2446,15 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
if not chrome_utils.exists():
|
||||
return 0
|
||||
|
||||
node_binary = cast(BinaryManager, Binary.objects).get_valid_binary("node")
|
||||
if node_binary is None or not node_binary.is_valid:
|
||||
return 0
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
node_projection = get_config().ABXPKG_LIB_DIR / "env" / "bin" / "node"
|
||||
if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK):
|
||||
return 0
|
||||
|
||||
crawl_roots = [
|
||||
crawls_dir
|
||||
for user_dir in CONSTANTS.USERS_DIR.iterdir()
|
||||
@ -2476,7 +2469,7 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
try:
|
||||
for crawl_root in crawl_roots:
|
||||
result = subprocess.run(
|
||||
["node", str(chrome_utils), "killZombieChrome", str(crawl_root)],
|
||||
[str(node_projection), str(chrome_utils), "killZombieChrome", str(crawl_root)],
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
text=True,
|
||||
|
||||
@ -85,7 +85,6 @@ def check_data_folder(config=None, **config_kwargs) -> None:
|
||||
create_and_chown_dir(CONSTANTS.USERS_DIR)
|
||||
create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default")
|
||||
create_and_chown_dir(CONSTANTS.LOGS_DIR)
|
||||
# create_and_chown_dir(CONSTANTS.CACHE_DIR)
|
||||
|
||||
# Create /tmp and /lib dirs if they don't exist
|
||||
get_or_create_working_tmp_dir(autofix=True, quiet=False, config=config)
|
||||
@ -182,18 +181,6 @@ def check_io_encoding():
|
||||
print(' python3 -c "import sys; print(sys.stdout.encoding)" # should output UTF-8', file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
# # hard errors: check python version
|
||||
# if sys.version_info[:3] < (3, 10, 0):
|
||||
# print('[red][X] Python version is not new enough: {sys.version} (>3.10 is required)[/red]', file=sys.stderr)
|
||||
# print(' See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.', file=sys.stderr)
|
||||
# raise SystemExit(2)
|
||||
|
||||
# # hard errors: check django version
|
||||
# if int(django.VERSION[0]) < 5:
|
||||
# print('[red][X] Django version is not new enough: {django.VERSION[:3]} (>=5.0 is required)[/red]', file=sys.stderr)
|
||||
# print(' Upgrade django using pip or your system package manager: pip3 install --upgrade django', file=sys.stderr)
|
||||
# raise SystemExit(2)
|
||||
|
||||
|
||||
def check_not_root():
|
||||
from archivebox.config.permissions import IS_ROOT
|
||||
@ -227,7 +214,6 @@ def check_data_dir_permissions(config=None, **config_kwargs):
|
||||
data_dir_uid, data_dir_gid = data_dir_stat.st_uid, data_dir_stat.st_gid
|
||||
data_owned_by_root = data_dir_uid == 0
|
||||
|
||||
# data_owned_by_default_user = data_dir_uid == DEFAULT_UID or data_dir_gid == DEFAULT_GID
|
||||
data_owner_doesnt_match = (data_dir_uid != ARCHIVEBOX_USER and data_dir_gid != ARCHIVEBOX_GROUP) if not IS_ROOT else False
|
||||
data_not_writable = not (os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.W_OK))
|
||||
if data_not_writable:
|
||||
|
||||
@ -132,10 +132,6 @@ def progress_bar(seconds: int, prefix: str = "", ANSI: dict[str, str] = ANSI, co
|
||||
),
|
||||
)
|
||||
sys.stdout.flush()
|
||||
# uncomment to have it disappear when it hits 100% instead of staying full red:
|
||||
# time.sleep(0.5)
|
||||
# sys.stdout.write('\r{}{}\r'.format((' ' * get_config().TERM_WIDTH), ANSI['reset']))
|
||||
# sys.stdout.flush()
|
||||
except (KeyboardInterrupt, BrokenPipeError):
|
||||
print()
|
||||
|
||||
@ -148,9 +144,6 @@ def log_cli_command(subcommand: str, subcommand_args: Iterable[str] = (), stdin:
|
||||
subcommand=subcommand,
|
||||
args=args,
|
||||
)
|
||||
# stderr()
|
||||
# stderr('[bright_black] > {pwd}[/]'.format(pwd=pwd, **ANSI))
|
||||
# stderr()
|
||||
print(Panel(version_msg), file=sys.stderr)
|
||||
|
||||
|
||||
|
||||
@ -15,17 +15,6 @@ django_stubs_ext.monkeypatch()
|
||||
# monkey patch django timezone to add back utc (it was removed in Django 5.0)
|
||||
setattr(timezone, "utc", datetime.UTC)
|
||||
|
||||
# monkey patch django-signals-webhooks to change how it shows up in Admin UI
|
||||
# from signal_webhooks.apps import DjangoSignalWebhooksConfig
|
||||
# DjangoSignalWebhooksConfig.verbose_name = 'API'
|
||||
|
||||
|
||||
# Rich traceback handler disabled - it adds frames/boxes that wrap weirdly in log files
|
||||
# Standard Python tracebacks are used instead (full width, no frames)
|
||||
# from rich.traceback import install
|
||||
# install(show_locals=True, word_wrap=False, ...)
|
||||
|
||||
|
||||
# Hide site-packages/sonic/client.py:115: SyntaxWarning
|
||||
# https://github.com/xmonader/python-sonic-client/pull/18
|
||||
warnings.filterwarnings("ignore", category=SyntaxWarning, module="sonic")
|
||||
|
||||
@ -22,7 +22,6 @@ def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool
|
||||
mode = "wb+" if isinstance(contents, bytes) else "w"
|
||||
encoding = None if isinstance(contents, bytes) else "utf-8" # enforce utf-8 on all text writes
|
||||
|
||||
# print('\n> Atomic Write:', mode, path, len(contents), f'overwrite={overwrite}')
|
||||
try:
|
||||
with lib_atomic_write(path, mode=mode, overwrite=overwrite, encoding=encoding) as f:
|
||||
if isinstance(contents, dict):
|
||||
|
||||
@ -90,9 +90,6 @@ class JSONSchemaWithLambdas(GenerateJsonSchema):
|
||||
serialize_unknown=True,
|
||||
)
|
||||
|
||||
# for computed_field properties render them like this instead:
|
||||
# inspect.getsource(field.wrapped_property.fget).split('def ', 1)[-1].split('\n', 1)[-1].strip().strip('return '),
|
||||
|
||||
|
||||
def better_toml_dump_str(val: Any) -> str:
|
||||
try:
|
||||
|
||||
@ -634,7 +634,12 @@ def export_browser_state(
|
||||
return False, None, "abxpkg returned an invalid browser dependency environment."
|
||||
if not isinstance(resolved_env, dict):
|
||||
return False, None, "abxpkg returned an invalid browser dependency environment."
|
||||
abxpkg_lib_dir = get_config().ABXPKG_LIB_DIR
|
||||
node_projection = abxpkg_lib_dir / "env" / "bin" / "node"
|
||||
if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK):
|
||||
return False, None, f"abxpkg did not resolve Node.js into {node_projection}."
|
||||
env.update({str(key): str(value) for key, value in resolved_env.items()})
|
||||
env["NODE_BINARY"] = str(node_projection)
|
||||
env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(chrome_plugin_dir)
|
||||
|
||||
if user_data_dir:
|
||||
@ -683,7 +688,7 @@ def export_browser_state(
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["node", str(state_script)],
|
||||
[str(node_projection), str(state_script)],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
||||
@ -323,21 +323,8 @@ def run_hook(
|
||||
# If Process.current() fails (e.g., not in a worker context), leave parent as None
|
||||
pass
|
||||
|
||||
if not script.exists():
|
||||
# Create a failed Process record for hooks that don't exist
|
||||
process = Process.objects.create(
|
||||
machine=machine,
|
||||
iface=iface,
|
||||
parent=parent,
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
pwd=str(output_dir),
|
||||
cmd=["echo", f"Hook script not found: {script}"],
|
||||
timeout=timeout,
|
||||
status=Process.StatusChoices.EXITED,
|
||||
exit_code=1,
|
||||
stderr=f"Hook script not found: {script}",
|
||||
)
|
||||
return process
|
||||
if not script.is_file():
|
||||
raise FileNotFoundError(f"Hook script not found: {script}")
|
||||
|
||||
# Python hooks carry their runtime contract in the shebang
|
||||
# (usually `abxpkg run --script python3`), so execute them directly.
|
||||
@ -345,11 +332,18 @@ def run_hook(
|
||||
# interpreter because those hooks do not need per-script Python env setup.
|
||||
ext = script.suffix.lower()
|
||||
if ext == ".sh":
|
||||
cmd = ["bash", str(script)]
|
||||
bash_projection = Path(hook_config["ABXPKG_LIB_DIR"]).expanduser() / "env" / "bin" / "bash"
|
||||
if not bash_projection.is_symlink() or not os.access(bash_projection, os.X_OK):
|
||||
raise RuntimeError(f"Bash must be resolved by abxpkg into {bash_projection}")
|
||||
cmd = [str(bash_projection), str(script)]
|
||||
elif ext == ".py":
|
||||
cmd = [str(script)]
|
||||
elif ext == ".js":
|
||||
cmd = ["node", str(script)]
|
||||
node_projection = Path(hook_config["ABXPKG_LIB_DIR"]).expanduser() / "env" / "bin" / "node"
|
||||
if not node_projection.is_symlink() or not os.access(node_projection, os.X_OK):
|
||||
raise RuntimeError(f"Node.js must be resolved by abxpkg into {node_projection}")
|
||||
hook_config["NODE_BINARY"] = str(node_projection)
|
||||
cmd = [str(node_projection), str(script)]
|
||||
else:
|
||||
# Try to execute directly (assumes shebang)
|
||||
cmd = [str(script)]
|
||||
@ -510,10 +504,12 @@ def extract_records_from_process(process: "Process") -> list[dict[str, Any]]:
|
||||
if not records:
|
||||
return []
|
||||
|
||||
# Extract plugin metadata from process.pwd and process.cmd
|
||||
# Extract plugin metadata from process.pwd and the shipped hook path in cmd.
|
||||
# Python hooks execute directly through their shebang, while JS and shell
|
||||
# hooks have an interpreter in cmd[0].
|
||||
plugin_name = Path(process.pwd).name if process.pwd else "unknown"
|
||||
hook_name = Path(process.cmd[1]).name if len(process.cmd) > 1 else "unknown"
|
||||
plugin_hook = process.cmd[1] if len(process.cmd) > 1 else ""
|
||||
plugin_hook = next((str(arg) for arg in process.cmd if Path(str(arg)).name.startswith("on_")), "")
|
||||
hook_name = Path(plugin_hook).name if plugin_hook else "unknown"
|
||||
|
||||
for record in records:
|
||||
# Add plugin metadata to record
|
||||
|
||||
@ -133,9 +133,12 @@ def live_progress_view(request):
|
||||
def process_label(cmd: list[str] | None) -> tuple[str, str, str, str]:
|
||||
hook_path = ""
|
||||
if isinstance(cmd, list) and cmd:
|
||||
first = cmd[0]
|
||||
if isinstance(first, str):
|
||||
hook_path = first
|
||||
hook_path = next(
|
||||
(arg for arg in cmd if isinstance(arg, str) and Path(arg).name.startswith("on_") and "__" in Path(arg).name),
|
||||
"",
|
||||
)
|
||||
if not hook_path and isinstance(cmd[0], str):
|
||||
hook_path = cmd[0]
|
||||
|
||||
if not hook_path:
|
||||
return ("", "setup", "unknown", "")
|
||||
|
||||
@ -381,6 +381,40 @@ def _save_archiveresult_event_to_db(
|
||||
snapshot.title = next_title
|
||||
snapshot.save(update_fields=["title", "modified_at"])
|
||||
|
||||
# Parser output becomes durable when its ArchiveResult reaches a final
|
||||
# state. Project at that same lifecycle boundary so every completed parser
|
||||
# can enrich an already-discovered URL before Snapshot/Crawl completion.
|
||||
# create_discovered_snapshots() owns cross-parser dedupe and metadata merge.
|
||||
if (
|
||||
result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS)
|
||||
and (plugin_dir / "urls.jsonl").exists()
|
||||
):
|
||||
from .snapshot_service import project_discovered_snapshots
|
||||
|
||||
with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.project_discovered_snapshots"):
|
||||
project_discovered_snapshots(str(snapshot.id))
|
||||
|
||||
|
||||
def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
|
||||
"""Advance an existing queued hook row after its OS process is persisted."""
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
started_at = parse_event_datetime(event.start_ts)
|
||||
if started_at is None:
|
||||
raise ValueError("ProcessStartedEvent.start_ts is required")
|
||||
ArchiveResult.objects.filter(
|
||||
snapshot_id=snapshot_id,
|
||||
plugin=event.plugin_name,
|
||||
hook_name=event.hook_name,
|
||||
status=ArchiveResult.StatusChoices.QUEUED,
|
||||
).update(
|
||||
status=ArchiveResult.StatusChoices.STARTED,
|
||||
start_ts=started_at,
|
||||
end_ts=None,
|
||||
process_id=process_id,
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
|
||||
|
||||
class ArchiveResultService(BaseService):
|
||||
LISTENS_TO = [ArchiveResultEvent, ProcessCompletedEvent]
|
||||
|
||||
@ -8,7 +8,7 @@ from asgiref.sync import sync_to_async
|
||||
from django.utils import timezone
|
||||
|
||||
from abxbus import BaseEvent
|
||||
from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, ProcessCompletedEvent, ProcessStartedEvent
|
||||
from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent
|
||||
from abx_dl.services.base import BaseService
|
||||
|
||||
|
||||
@ -147,6 +147,21 @@ class ProcessService(BaseService):
|
||||
binary_id=process.binary_id,
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
if event.hook_name.startswith("on_Snapshot"):
|
||||
snapshot_event = await self.bus.find(
|
||||
SnapshotEvent,
|
||||
past=True,
|
||||
future=False,
|
||||
where=lambda candidate: self.bus.event_is_child_of(event, candidate),
|
||||
)
|
||||
if snapshot_event is not None:
|
||||
from .archive_result_service import mark_archiveresult_started
|
||||
|
||||
await sync_to_async(mark_archiveresult_started, thread_sensitive=True)(
|
||||
event,
|
||||
snapshot_id=snapshot_event.snapshot_id,
|
||||
process_id=str(process.id),
|
||||
)
|
||||
|
||||
async def _completed_worker_loop(self) -> None:
|
||||
while True:
|
||||
|
||||
@ -180,10 +180,7 @@ async def _run_event_now(event, timeout: float | None = None):
|
||||
return event
|
||||
|
||||
|
||||
def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
|
||||
if os.environ.get("PYTEST_CURRENT_TEST") and not allow_under_pytest:
|
||||
return False
|
||||
|
||||
def ensure_background_runner() -> bool:
|
||||
from archivebox.machine.models import Machine, Process
|
||||
from archivebox.workers.supervisord_util import RUNNER_WORKER, get_existing_supervisord_process, get_worker, start_worker
|
||||
|
||||
@ -192,7 +189,7 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
|
||||
if runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"):
|
||||
return False
|
||||
if supervisor is not None:
|
||||
start_worker(supervisor, RUNNER_WORKER)
|
||||
start_worker(supervisor, RUNNER_WORKER())
|
||||
return True
|
||||
|
||||
machine = Machine.current()
|
||||
@ -238,13 +235,9 @@ class CrawlRunner:
|
||||
self.interactive_interrupts = interactive_interrupts
|
||||
self.config_overrides = dict(config_overrides or {})
|
||||
|
||||
async def ignore_snapshot(_snapshot_id: str) -> None:
|
||||
return None
|
||||
|
||||
SnapshotService(
|
||||
self.bus,
|
||||
crawl_id=str(crawl.id),
|
||||
schedule_snapshot=self.enqueue_snapshot if process_discovered_snapshots_inline else ignore_snapshot,
|
||||
)
|
||||
ArchiveResultService(self.bus)
|
||||
self.selected_plugins = selected_plugins
|
||||
|
||||
@ -98,9 +98,8 @@ class SnapshotService(BaseService):
|
||||
LISTENS_TO = [SnapshotEvent, SnapshotCompletedEvent]
|
||||
EMITS = []
|
||||
|
||||
def __init__(self, bus, *, crawl_id: str, schedule_snapshot):
|
||||
def __init__(self, bus, *, crawl_id: str):
|
||||
self.crawl_id = crawl_id
|
||||
self.schedule_snapshot = schedule_snapshot
|
||||
super().__init__(bus)
|
||||
self.bus.on(SnapshotEvent, self.on_SnapshotEvent)
|
||||
self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent)
|
||||
|
||||
@ -5,6 +5,7 @@ import json
|
||||
import re
|
||||
import secrets
|
||||
import signal
|
||||
import select
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
@ -12,10 +13,11 @@ import tempfile
|
||||
import textwrap
|
||||
import time
|
||||
import shutil
|
||||
import ctypes
|
||||
from datetime import timedelta
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from threading import Event, Thread
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from collections.abc import Callable
|
||||
@ -41,6 +43,8 @@ os.environ.pop("USERS_DIR", None)
|
||||
os.environ.pop("CRAWL_DIR", None)
|
||||
os.environ.pop("SNAP_DIR", None)
|
||||
|
||||
_RUNTIME_GUARD_ACTIVE = False
|
||||
|
||||
|
||||
def _is_repo_path(path: Path) -> bool:
|
||||
resolved = path.expanduser().resolve(strict=False)
|
||||
@ -120,6 +124,25 @@ def _sync_archivebox_test_data_dir(data_dir: Path) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _archivebox_test_audit_hook(event: str, args: tuple[Any, ...]) -> None:
|
||||
"""Enforce runtime path isolation through Python's native audit surface."""
|
||||
if not _RUNTIME_GUARD_ACTIVE:
|
||||
return
|
||||
if event == "os.chdir":
|
||||
path = Path(args[0])
|
||||
_assert_not_repo_path(path, label="cwd")
|
||||
_sync_archivebox_test_data_dir(path)
|
||||
elif event == "subprocess.Popen":
|
||||
cwd = args[2]
|
||||
env = args[3]
|
||||
if cwd is not None:
|
||||
_assert_not_repo_path(Path(cwd), label="cwd")
|
||||
_assert_safe_runtime_paths(cwd=Path(cwd) if cwd is not None else None, env=env)
|
||||
|
||||
|
||||
sys.addaudithook(_archivebox_test_audit_hook)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Helpers (defined before fixtures that use them)
|
||||
# =============================================================================
|
||||
@ -171,9 +194,6 @@ class ArchiveBoxCmdResult:
|
||||
def terminate(self) -> None:
|
||||
self._process.terminate()
|
||||
|
||||
def kill(self) -> None:
|
||||
self._process.kill()
|
||||
|
||||
def send_signal(self, sig: int) -> None:
|
||||
self._process.send_signal(sig)
|
||||
|
||||
@ -263,8 +283,8 @@ def run_archivebox_cmd(
|
||||
try:
|
||||
result.communicate(input=input, timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
result.communicate()
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
raise
|
||||
if check and result.returncode:
|
||||
raise subprocess.CalledProcessError(
|
||||
@ -305,7 +325,7 @@ def pytest_configure():
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_test_runtime(tmp_path, monkeypatch):
|
||||
def isolate_test_runtime(tmp_path):
|
||||
"""
|
||||
Run each pytest test from an isolated temp cwd and restore env mutations.
|
||||
|
||||
@ -317,11 +337,11 @@ def isolate_test_runtime(tmp_path, monkeypatch):
|
||||
ArchiveBox derives DATA_DIR from cwd, so subprocess helpers pass the target
|
||||
collection as cwd instead of using DATA_DIR as an override.
|
||||
"""
|
||||
global _RUNTIME_GUARD_ACTIVE
|
||||
|
||||
_assert_not_repo_path(tmp_path, label="tmp_path")
|
||||
original_cwd = Path.cwd()
|
||||
original_env = os.environ.copy()
|
||||
original_chdir = os.chdir
|
||||
original_popen = subprocess.Popen
|
||||
os.chdir(tmp_path)
|
||||
_sync_archivebox_test_data_dir(tmp_path)
|
||||
os.environ.pop("DATA_DIR", None)
|
||||
@ -334,28 +354,15 @@ def isolate_test_runtime(tmp_path, monkeypatch):
|
||||
machine_models._CURRENT_PROCESS = None
|
||||
machine_models._CURRENT_BINARIES.clear()
|
||||
|
||||
def guarded_chdir(path: os.PathLike[str] | str) -> None:
|
||||
_assert_not_repo_path(Path(path), label="cwd")
|
||||
original_chdir(path)
|
||||
_sync_archivebox_test_data_dir(Path(path))
|
||||
|
||||
def guarded_popen(*args: Any, **kwargs: Any):
|
||||
cwd = kwargs.get("cwd")
|
||||
env = kwargs.get("env")
|
||||
if cwd is not None:
|
||||
_assert_not_repo_path(Path(cwd), label="cwd")
|
||||
_assert_safe_runtime_paths(cwd=Path(cwd) if cwd is not None else None, env=env)
|
||||
return original_popen(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(os, "chdir", guarded_chdir)
|
||||
monkeypatch.setattr(subprocess, "Popen", guarded_popen)
|
||||
reset_machine_model_caches()
|
||||
_RUNTIME_GUARD_ACTIVE = True
|
||||
try:
|
||||
_assert_safe_runtime_paths(cwd=Path.cwd(), env=os.environ)
|
||||
yield
|
||||
finally:
|
||||
_RUNTIME_GUARD_ACTIVE = False
|
||||
reset_machine_model_caches()
|
||||
original_chdir(original_cwd)
|
||||
os.chdir(original_cwd)
|
||||
_sync_archivebox_test_data_dir(original_cwd)
|
||||
os.environ.clear()
|
||||
os.environ.update(original_env)
|
||||
@ -378,7 +385,7 @@ def isolated_data_dir(tmp_path):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hermetic_lib_dir(tmp_path, monkeypatch):
|
||||
def hermetic_lib_dir(tmp_path):
|
||||
"""
|
||||
Point ABXPKG_LIB_DIR at a temporary directory for isolated abxpkg resolution.
|
||||
|
||||
@ -390,10 +397,27 @@ def hermetic_lib_dir(tmp_path, monkeypatch):
|
||||
|
||||
lib_dir = tmp_path / "lib"
|
||||
lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("ABXPKG_LIB_DIR", str(lib_dir))
|
||||
original_lib_dir = os.environ.get("ABXPKG_LIB_DIR")
|
||||
os.environ["ABXPKG_LIB_DIR"] = str(lib_dir)
|
||||
machine_models._CURRENT_MACHINE = None
|
||||
machine_models._CURRENT_PROCESS = None
|
||||
return lib_dir
|
||||
try:
|
||||
yield lib_dir
|
||||
finally:
|
||||
if original_lib_dir is None:
|
||||
os.environ.pop("ABXPKG_LIB_DIR", None)
|
||||
else:
|
||||
os.environ["ABXPKG_LIB_DIR"] = original_lib_dir
|
||||
machine_models._CURRENT_MACHINE = None
|
||||
machine_models._CURRENT_PROCESS = None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cached_abxpkg_lib_dir():
|
||||
"""Reuse the configured abxpkg cache when a test is not validating LIB_DIR isolation."""
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
return get_config().ABXPKG_LIB_DIR
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@ -509,8 +533,43 @@ def recursive_test_site():
|
||||
}
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blocking_http_server():
|
||||
"""Serve one real request behind explicit start/release synchronization."""
|
||||
|
||||
request_started = Event()
|
||||
release_response = Event()
|
||||
|
||||
class BlockingHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
request_started.set()
|
||||
release_response.wait()
|
||||
body = b"<html><head><title>Barrier</title></head><body>released</body></html>"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), BlockingHandler)
|
||||
thread = Thread(target=server.serve_forever, name="archivebox-test-http-barrier")
|
||||
thread.start()
|
||||
yield SimpleNamespace(
|
||||
url=f"http://127.0.0.1:{server.server_port}/",
|
||||
request_started=request_started,
|
||||
release_response=release_response,
|
||||
)
|
||||
release_response.set()
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@ -555,21 +614,18 @@ def archivebox_daemon_server(initialized_archive, free_tcp_port_factory):
|
||||
_stop_archivebox_supervisord(cwd, env)
|
||||
|
||||
|
||||
def wait_for_process(predicate: Callable[[psutil.Process, str], bool], *, timeout: float = 20.0) -> psutil.Process:
|
||||
deadline = time.time() + timeout
|
||||
def find_process(predicate: Callable[[psutil.Process, str], bool]) -> psutil.Process:
|
||||
"""Locate a process after its native readiness signal has fired."""
|
||||
last_seen: list[str] = []
|
||||
while time.time() < deadline:
|
||||
last_seen = []
|
||||
for proc in psutil.process_iter(["pid", "ppid", "cmdline"]):
|
||||
try:
|
||||
cmdline = proc.info.get("cmdline") or []
|
||||
command = " ".join(cmdline)
|
||||
last_seen.append(f"{proc.info.get('pid')} {proc.info.get('ppid')} {command}")
|
||||
if predicate(proc, command):
|
||||
return proc
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
continue
|
||||
time.sleep(0.2)
|
||||
for proc in psutil.process_iter(["pid", "ppid", "cmdline"]):
|
||||
try:
|
||||
cmdline = proc.info.get("cmdline") or []
|
||||
command = " ".join(cmdline)
|
||||
last_seen.append(f"{proc.info.get('pid')} {proc.info.get('ppid')} {command}")
|
||||
if predicate(proc, command):
|
||||
return proc
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
continue
|
||||
raise AssertionError("No matching live process found. Last seen:\n" + "\n".join(last_seen[-50:]))
|
||||
|
||||
|
||||
@ -583,31 +639,37 @@ def pid_is_alive(pid: int) -> bool:
|
||||
|
||||
|
||||
def wait_for_pid_to_disappear(pid: int, *, timeout: float = 20.0) -> None:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if not pid_is_alive(pid):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(f"PID {pid} is still running")
|
||||
try:
|
||||
psutil.Process(pid).wait(timeout=timeout)
|
||||
except psutil.NoSuchProcess:
|
||||
return
|
||||
except psutil.TimeoutExpired as exc:
|
||||
raise AssertionError(f"PID {pid} is still running") from exc
|
||||
|
||||
|
||||
def cleanup_process_group(group_pid: int | None, *child_pids: int | None) -> None:
|
||||
processes: list[psutil.Process] = []
|
||||
if group_pid and pid_is_alive(group_pid):
|
||||
try:
|
||||
os.killpg(group_pid, signal.SIGKILL)
|
||||
os.killpg(group_pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except OSError:
|
||||
try:
|
||||
os.kill(group_pid, signal.SIGKILL)
|
||||
os.kill(group_pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
processes.append(psutil.Process(group_pid))
|
||||
for pid in child_pids:
|
||||
if pid and pid_is_alive(pid):
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
else:
|
||||
processes.append(psutil.Process(pid))
|
||||
_gone, alive = psutil.wait_procs(processes, timeout=10)
|
||||
assert not alive, f"processes did not stop after SIGTERM: {[proc.pid for proc in alive]}"
|
||||
|
||||
|
||||
def cli_env(
|
||||
@ -685,52 +747,142 @@ def cli_env(
|
||||
return env
|
||||
|
||||
|
||||
def wait_for_port_open(host: str, port: int, *, timeout: float = 30.0) -> None:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=0.25):
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(f"server did not listen on {host}:{port}")
|
||||
def assert_port_open(host: str, port: int, *, timeout: float = 30.0) -> None:
|
||||
"""Verify a listening socket after the server emitted its readiness event."""
|
||||
with socket.create_connection((host, port), timeout=timeout) as connection:
|
||||
assert connection.getpeername() == (host, port)
|
||||
|
||||
|
||||
def _wait_for_log_match(log_path: Path, pattern: str, *, fixed: bool, count: int, timeout: float) -> str:
|
||||
"""Block on native filesystem notifications until the requested log event exists."""
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
if sys.platform == "darwin":
|
||||
parent_fd = os.open(log_path.parent, os.O_RDONLY)
|
||||
watched_fd: int | None = None
|
||||
event_queue = select.kqueue()
|
||||
event_queue.control(
|
||||
[
|
||||
select.kevent(
|
||||
parent_fd,
|
||||
filter=select.KQ_FILTER_VNODE,
|
||||
flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR,
|
||||
fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_RENAME | select.KQ_NOTE_DELETE,
|
||||
),
|
||||
],
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
def refresh_file_watcher() -> None:
|
||||
nonlocal watched_fd
|
||||
if watched_fd is not None:
|
||||
try:
|
||||
same_file = os.fstat(watched_fd).st_ino == log_path.stat().st_ino
|
||||
except (FileNotFoundError, OSError):
|
||||
same_file = False
|
||||
if same_file:
|
||||
return
|
||||
try:
|
||||
event_queue.control(
|
||||
[
|
||||
select.kevent(
|
||||
watched_fd,
|
||||
filter=select.KQ_FILTER_VNODE,
|
||||
flags=select.KQ_EV_DELETE,
|
||||
),
|
||||
],
|
||||
0,
|
||||
0,
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
os.close(watched_fd)
|
||||
watched_fd = None
|
||||
if log_path.exists():
|
||||
try:
|
||||
watched_fd = os.open(log_path, os.O_RDONLY)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
event_queue.control(
|
||||
[
|
||||
select.kevent(
|
||||
watched_fd,
|
||||
filter=select.KQ_FILTER_VNODE,
|
||||
flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR,
|
||||
fflags=(
|
||||
select.KQ_NOTE_WRITE
|
||||
| select.KQ_NOTE_EXTEND
|
||||
| select.KQ_NOTE_ATTRIB
|
||||
| select.KQ_NOTE_RENAME
|
||||
| select.KQ_NOTE_DELETE
|
||||
),
|
||||
),
|
||||
],
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
def wait_for_change(remaining: float) -> None:
|
||||
events = event_queue.control(None, 1, remaining)
|
||||
assert events, f"timed out waiting for filesystem event on {log_path}"
|
||||
refresh_file_watcher()
|
||||
|
||||
def close_watcher() -> None:
|
||||
event_queue.close()
|
||||
if watched_fd is not None:
|
||||
os.close(watched_fd)
|
||||
os.close(parent_fd)
|
||||
|
||||
refresh_file_watcher()
|
||||
|
||||
else:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
inotify_fd = libc.inotify_init1(os.O_CLOEXEC)
|
||||
assert inotify_fd >= 0, os.strerror(ctypes.get_errno())
|
||||
watch_mask = 0x00000002 | 0x00000008 | 0x00000080 | 0x00000100
|
||||
watch_descriptor = libc.inotify_add_watch(inotify_fd, os.fsencode(log_path.parent), watch_mask)
|
||||
assert watch_descriptor >= 0, os.strerror(ctypes.get_errno())
|
||||
|
||||
def wait_for_change(remaining: float) -> None:
|
||||
readable, _writable, _errors = select.select([inotify_fd], [], [], remaining)
|
||||
assert readable, f"timed out waiting for filesystem event on {log_path}"
|
||||
os.read(inotify_fd, 65536)
|
||||
|
||||
def close_watcher() -> None:
|
||||
os.close(inotify_fd)
|
||||
|
||||
try:
|
||||
while True:
|
||||
content = log_path.read_text(encoding="utf-8", errors="replace") if log_path.exists() else ""
|
||||
matches = content.count(pattern) if fixed else len(re.findall(pattern, content))
|
||||
if matches >= count:
|
||||
return content
|
||||
remaining = deadline - time.monotonic()
|
||||
assert remaining > 0, f"timed out waiting for {pattern!r} in {log_path}:\n{content}"
|
||||
wait_for_change(remaining)
|
||||
finally:
|
||||
close_watcher()
|
||||
|
||||
|
||||
def wait_for_log(log_path: Path, text: str, *, timeout: float = 30.0) -> str:
|
||||
deadline = time.time() + timeout
|
||||
content = ""
|
||||
while time.time() < deadline:
|
||||
if log_path.exists():
|
||||
content = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
if text in content:
|
||||
return content
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(f"timed out waiting for {text!r} in {log_path}:\n{content}")
|
||||
content = _wait_for_log_match(log_path, text, fixed=True, count=1, timeout=timeout)
|
||||
assert text in content, content
|
||||
return content
|
||||
|
||||
|
||||
def wait_for_log_count(log_path: Path, text: str, count: int, *, timeout: float = 30.0) -> str:
|
||||
deadline = time.time() + timeout
|
||||
content = ""
|
||||
while time.time() < deadline:
|
||||
if log_path.exists():
|
||||
content = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
if content.count(text) >= count:
|
||||
return content
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(f"timed out waiting for {count} occurrences of {text!r} in {log_path}:\n{content}")
|
||||
content = _wait_for_log_match(log_path, text, fixed=True, count=count, timeout=timeout)
|
||||
assert content.count(text) >= count, content
|
||||
return content
|
||||
|
||||
|
||||
def wait_for_log_pattern(log_path: Path, pattern: str, *, timeout: float = 30.0) -> re.Match[str]:
|
||||
deadline = time.time() + timeout
|
||||
content = ""
|
||||
while time.time() < deadline:
|
||||
if log_path.exists():
|
||||
content = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(f"timed out waiting for pattern {pattern!r} in {log_path}:\n{content}")
|
||||
content = _wait_for_log_match(log_path, pattern, fixed=False, count=1, timeout=timeout)
|
||||
match = re.search(pattern, content)
|
||||
assert match is not None, content
|
||||
return match
|
||||
|
||||
|
||||
def supervisor_pid_from_log(log_path: Path) -> int:
|
||||
@ -748,20 +900,27 @@ def worker_pid_from_log(log_path: Path, worker_name: str) -> int:
|
||||
|
||||
|
||||
def wait_for_worker_pid_from_log(log_path: Path, worker_name: str, *, timeout: float = 45.0) -> int:
|
||||
deadline = time.time() + timeout
|
||||
last_error = ""
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
return worker_pid_from_log(log_path, worker_name)
|
||||
except AssertionError as err:
|
||||
last_error = str(err)
|
||||
time.sleep(0.1)
|
||||
raise AssertionError(last_error or f"timed out waiting for worker {worker_name!r} in {log_path}")
|
||||
wait_for_log_pattern(
|
||||
log_path,
|
||||
rf"Worker {re.escape(worker_name)}: started RUNNING \(pid [0-9]+,",
|
||||
timeout=timeout,
|
||||
)
|
||||
return worker_pid_from_log(log_path, worker_name)
|
||||
|
||||
|
||||
def pgrep_data_dir(data_dir: Path) -> list[str]:
|
||||
result = subprocess.run(["pgrep", "-af", str(data_dir)], capture_output=True, text=True, timeout=5)
|
||||
lines = [line for line in result.stdout.splitlines() if "pgrep -af" not in line]
|
||||
lines: list[str] = []
|
||||
seen_pids: set[int] = set()
|
||||
for process in psutil.process_iter(["pid", "ppid", "cmdline"]):
|
||||
try:
|
||||
command = " ".join(process.info.get("cmdline") or [])
|
||||
if str(data_dir) not in command:
|
||||
continue
|
||||
pid = int(process.info["pid"])
|
||||
seen_pids.add(pid)
|
||||
lines.append(f"{pid} {process.info.get('ppid') or 0} {command}")
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
continue
|
||||
|
||||
for runtime_root in (Path("/tmp/archivebox"), data_dir / "tmp"):
|
||||
for config_path in runtime_root.glob("*/supervisord.conf"):
|
||||
@ -778,30 +937,34 @@ def pgrep_data_dir(data_dir: Path) -> list[str]:
|
||||
continue
|
||||
if not pid_is_alive(pid):
|
||||
continue
|
||||
ps_line = subprocess.run(
|
||||
["ps", "-p", str(pid), "-o", "pid=,ppid=,command="],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
).stdout.strip()
|
||||
if ps_line:
|
||||
lines.append(ps_line)
|
||||
if pid in seen_pids:
|
||||
continue
|
||||
try:
|
||||
process = psutil.Process(pid)
|
||||
lines.append(f"{pid} {process.ppid()} {' '.join(process.cmdline())}")
|
||||
seen_pids.add(pid)
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
continue
|
||||
|
||||
return sorted(set(lines))
|
||||
|
||||
|
||||
def assert_no_processes_for_data_dir(data_dir: Path, *, timeout: float = 10.0) -> None:
|
||||
deadline = time.time() + timeout
|
||||
remaining: list[str] = []
|
||||
while time.time() < deadline:
|
||||
remaining = pgrep_data_dir(data_dir)
|
||||
if not remaining:
|
||||
return
|
||||
time.sleep(0.25)
|
||||
raise AssertionError("processes still reference test DATA_DIR:\n" + "\n".join(remaining))
|
||||
remaining = pgrep_data_dir(data_dir)
|
||||
processes = []
|
||||
for line in remaining:
|
||||
pid = int(line.split(None, 1)[0])
|
||||
try:
|
||||
processes.append(psutil.Process(pid))
|
||||
except psutil.NoSuchProcess:
|
||||
continue
|
||||
_gone, alive = psutil.wait_procs(processes, timeout=timeout)
|
||||
final_remaining = pgrep_data_dir(data_dir)
|
||||
assert not alive and not final_remaining, "processes still reference test DATA_DIR:\n" + "\n".join(final_remaining)
|
||||
|
||||
|
||||
def kill_processes_for_data_dir(data_dir: Path) -> None:
|
||||
processes: list[psutil.Process] = []
|
||||
for line in pgrep_data_dir(data_dir):
|
||||
try:
|
||||
pid = int(line.split(None, 1)[0])
|
||||
@ -809,9 +972,13 @@ def kill_processes_for_data_dir(data_dir: Path) -> None:
|
||||
continue
|
||||
if pid != os.getpid():
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
else:
|
||||
processes.append(psutil.Process(pid))
|
||||
_gone, alive = psutil.wait_procs(processes, timeout=10)
|
||||
assert not alive, f"processes did not stop after SIGTERM: {[proc.pid for proc in alive]}"
|
||||
|
||||
|
||||
def start_archivebox_server(
|
||||
@ -821,7 +988,7 @@ def start_archivebox_server(
|
||||
env: dict[str, str] | None = None,
|
||||
daemonize: bool | None = None,
|
||||
log_name: str | None = None,
|
||||
wait_for_log_text: str | None = "Tailing worker logs",
|
||||
wait_for_log_text: str | None = "Listening on TCP",
|
||||
):
|
||||
if daemonize is None:
|
||||
daemonize = log_name is None
|
||||
@ -848,34 +1015,22 @@ def start_archivebox_server(
|
||||
if daemonize:
|
||||
assert proc.returncode == 0, proc.stderr or proc.stdout
|
||||
return proc
|
||||
wait_for_port_open("127.0.0.1", port)
|
||||
if log_path is not None and wait_for_log_text is not None:
|
||||
wait_for_log(log_path, wait_for_log_text, timeout=30.0)
|
||||
assert_port_open("127.0.0.1", port)
|
||||
return proc
|
||||
|
||||
|
||||
def stop_archivebox_process(proc: subprocess.Popen[str], sig=signal.SIGTERM, *, timeout: float = 15.0) -> str:
|
||||
if proc.poll() is None:
|
||||
try:
|
||||
os.killpg(proc.pid, sig)
|
||||
except (ProcessLookupError, OSError):
|
||||
try:
|
||||
os.kill(proc.pid, sig)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
stdout, _stderr = proc.communicate(timeout=timeout)
|
||||
return stdout or ""
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(proc.pid, sig)
|
||||
except (ProcessLookupError, OSError):
|
||||
try:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
except (ProcessLookupError, OSError):
|
||||
try:
|
||||
os.kill(proc.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
stdout, _stderr = proc.communicate(timeout=5)
|
||||
return stdout or ""
|
||||
os.kill(proc.pid, sig)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
stdout, _stderr = proc.communicate(timeout=timeout)
|
||||
return stdout or ""
|
||||
|
||||
|
||||
def run_queued_crawls(cwd: Path, env: dict[str, str] | None = None, timeout: int = 180) -> None:
|
||||
@ -933,24 +1088,24 @@ def _stop_archivebox_supervisord(cwd: Path, env: dict[str, str]) -> None:
|
||||
|
||||
|
||||
def _wait_for_archivebox_workers(cwd: Path, env: dict[str, str], names: tuple[str, ...] | list[str], timeout: int = 45) -> dict[str, Any]:
|
||||
deadline = time.time() + timeout
|
||||
state: dict[str, Any] = {}
|
||||
while time.time() < deadline:
|
||||
state = _archivebox_worker_state(cwd, env)
|
||||
if all(isinstance(state.get(name), dict) and state[name].get("statename") == "RUNNING" for name in names):
|
||||
return state
|
||||
time.sleep(1)
|
||||
supervisord_log = cwd / "logs" / "supervisord.log"
|
||||
deadline = time.monotonic() + timeout
|
||||
for name in names:
|
||||
remaining = deadline - time.monotonic()
|
||||
assert remaining > 0, f"timed out waiting for workers {names} in {supervisord_log}"
|
||||
wait_for_log_pattern(
|
||||
supervisord_log,
|
||||
rf"success: {re.escape(name)} entered RUNNING state,",
|
||||
timeout=remaining,
|
||||
)
|
||||
state = _archivebox_worker_state(cwd, env)
|
||||
assert all(isinstance(state.get(name), dict) and state[name].get("statename") == "RUNNING" for name in names), state
|
||||
return state
|
||||
|
||||
|
||||
def stop_process(proc: subprocess.Popen[str]) -> tuple[str, str]:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
return proc.communicate(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
return proc.communicate()
|
||||
proc.terminate()
|
||||
return proc.communicate(timeout=5)
|
||||
|
||||
|
||||
def run_python_cwd(
|
||||
@ -1082,7 +1237,7 @@ def api_auth_headers(api_token: str, *, django_client: bool = False, port: int |
|
||||
|
||||
|
||||
def wait_for_live_api(port: int, *, path: str = "/api/v1/docs"):
|
||||
return wait_for_http(port, host=f"api.archivebox.localhost:{port}", path=path)
|
||||
return get_http_response(port, host=f"api.archivebox.localhost:{port}", path=path)
|
||||
|
||||
|
||||
def live_api_request(port: int, method: str, path: str, *, api_token: str, timeout: int = 30, **kwargs):
|
||||
@ -1146,32 +1301,25 @@ def stop_server(cwd: Path) -> None:
|
||||
run_python_cwd(script, cwd=cwd, timeout=30)
|
||||
|
||||
|
||||
def wait_for_http(
|
||||
def get_http_response(
|
||||
port: int,
|
||||
host: str,
|
||||
path: str = "/",
|
||||
timeout: float = 30.0,
|
||||
process: subprocess.Popen[str] | None = None,
|
||||
) -> requests.Response:
|
||||
deadline = time.time() + timeout
|
||||
last_exc = None
|
||||
while time.time() < deadline:
|
||||
if process is not None and process.poll() is not None:
|
||||
raise AssertionError(f"Server exited before becoming ready with code {process.returncode}")
|
||||
try:
|
||||
response = requests.get(
|
||||
f"http://127.0.0.1:{port}{path}",
|
||||
headers={"Host": host},
|
||||
timeout=2,
|
||||
allow_redirects=False,
|
||||
)
|
||||
if response.status_code < 500:
|
||||
return response
|
||||
last_exc = f"HTTP {response.status_code}"
|
||||
except requests.RequestException as exc:
|
||||
last_exc = exc
|
||||
time.sleep(0.5)
|
||||
raise AssertionError(f"Timed out waiting for HTTP on {host}: {last_exc}")
|
||||
"""Perform one blocking HTTP exchange after native server readiness."""
|
||||
if process is not None:
|
||||
returncode = process.poll()
|
||||
assert returncode is None, f"Server exited before becoming ready with code {returncode}"
|
||||
response = requests.get(
|
||||
f"http://127.0.0.1:{port}{path}",
|
||||
headers={"Host": host},
|
||||
timeout=timeout,
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert response.status_code < 500, f"HTTP {response.status_code} from {host}{path}"
|
||||
return response
|
||||
|
||||
|
||||
def make_latest_schedule_due(cwd: Path) -> None:
|
||||
@ -1244,15 +1392,19 @@ def get_snapshot_file_text(cwd: Path, url: str) -> str:
|
||||
|
||||
|
||||
def wait_for_snapshot_capture(cwd: Path, url: str, timeout: int = 180) -> str:
|
||||
deadline = time.time() + timeout
|
||||
last_error = None
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
return get_snapshot_file_text(cwd, url)
|
||||
except AssertionError as err:
|
||||
last_error = err
|
||||
time.sleep(2)
|
||||
raise AssertionError(f"timed out waiting for captured content for {url}: {last_error}")
|
||||
script = textwrap.dedent(
|
||||
f"""
|
||||
from archivebox.core.models import Snapshot
|
||||
snapshot = Snapshot.objects.filter(url={url!r}).order_by('-created_at').first()
|
||||
assert snapshot is not None
|
||||
print(snapshot.output_dir / 'index.jsonl')
|
||||
""",
|
||||
)
|
||||
result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=30)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
index_path = Path(result.stdout.strip().splitlines()[-1])
|
||||
_wait_for_log_match(index_path, ".", fixed=False, count=1, timeout=timeout)
|
||||
return get_snapshot_file_text(cwd, url)
|
||||
|
||||
|
||||
def get_counts(cwd: Path, scheduled_url: str, one_shot_url: str) -> tuple[int, int, int]:
|
||||
@ -1358,8 +1510,8 @@ def wait_for_archive_outputs(
|
||||
cwd: Path,
|
||||
url: str,
|
||||
timeout: int = 120,
|
||||
interval: float = 1.0,
|
||||
) -> bool:
|
||||
wait_for_snapshot_capture(cwd, url, timeout=timeout)
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
from pathlib import Path
|
||||
@ -1412,13 +1564,8 @@ def wait_for_archive_outputs(
|
||||
""",
|
||||
)
|
||||
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
stdout, _stderr, returncode = run_python_cwd(script, cwd=cwd, timeout=30)
|
||||
if returncode == 0 and "READY" in stdout:
|
||||
return True
|
||||
time.sleep(interval)
|
||||
return False
|
||||
stdout, _stderr, returncode = run_python_cwd(script, cwd=cwd, timeout=30)
|
||||
return returncode == 0 and "READY" in stdout
|
||||
|
||||
|
||||
def _get_machine_type() -> str:
|
||||
@ -1439,8 +1586,7 @@ def resolve_abxpkg_binary_env(
|
||||
install: bool = True,
|
||||
) -> dict[str, str]:
|
||||
"""Resolve real test dependencies through abxpkg and return its exported env."""
|
||||
command_env = os.environ.copy()
|
||||
command_env.update(env or {})
|
||||
command_env = dict(env) if env is not None else os.environ.copy()
|
||||
command_env["ABXPKG_LIB_DIR"] = str(lib_dir)
|
||||
command = [
|
||||
str(Path(sys.executable).with_name("abxpkg")),
|
||||
@ -1481,6 +1627,34 @@ def resolve_abxpkg_chrome_env(lib_dir: Path, env: dict[str, str] | None = None)
|
||||
return payload
|
||||
|
||||
|
||||
def install_real_binary(
|
||||
name: str,
|
||||
*,
|
||||
machine=None,
|
||||
binproviders: str = "env",
|
||||
overrides: dict[str, dict[str, Any]] | None = None,
|
||||
):
|
||||
"""Install and persist a real binary through the normal Binary state machine."""
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
|
||||
binary = Binary.objects.create(
|
||||
machine=machine or Machine.current(refresh=True),
|
||||
name=name,
|
||||
binproviders=binproviders,
|
||||
overrides=overrides or {},
|
||||
status=Binary.StatusChoices.QUEUED,
|
||||
)
|
||||
assert binary.tick_claimed(lock_seconds=600)
|
||||
binary.refresh_from_db()
|
||||
assert binary.status == Binary.StatusChoices.INSTALLED
|
||||
assert binary.retry_at is None
|
||||
assert binary.binprovider in binary.binproviders.split(",")
|
||||
assert binary.version
|
||||
assert binary.abspath
|
||||
assert Path(binary.abspath).exists()
|
||||
return binary
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def real_archive_with_example(tmp_path_factory, request):
|
||||
"""
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from threading import Event
|
||||
@ -12,8 +12,13 @@ from .conftest import (
|
||||
get_free_port,
|
||||
init_archive,
|
||||
live_api_request,
|
||||
run_archivebox_cmd,
|
||||
run_queued_crawls,
|
||||
start_archivebox_server,
|
||||
stop_archivebox_process,
|
||||
stop_server,
|
||||
get_http_response,
|
||||
wait_for_log,
|
||||
)
|
||||
from archivebox.core.models import Snapshot, SnapshotTag
|
||||
from archivebox.crawls.models import Crawl
|
||||
@ -147,44 +152,54 @@ IMPORT_FORMAT_ENV = {
|
||||
}
|
||||
|
||||
|
||||
def wait_for_expected_import_snapshots(
|
||||
def start_api_server_without_runner(cwd: Path, env: dict[str, str], port: int):
|
||||
log_path = cwd / "api-server.log"
|
||||
log = log_path.open("w", encoding="utf-8")
|
||||
process = run_archivebox_cmd(
|
||||
["manage", "runserver", f"127.0.0.1:{port}", "--noreload"],
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
log.close()
|
||||
wait_for_log(log_path, "Listening on TCP", timeout=30)
|
||||
get_http_response(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
|
||||
return process
|
||||
|
||||
|
||||
def assert_expected_import_snapshots(
|
||||
cwd: Path,
|
||||
expected_urls: set[str],
|
||||
*,
|
||||
timeout: float = 180.0,
|
||||
expected_tags: set[str] | None = None,
|
||||
) -> None:
|
||||
import time
|
||||
|
||||
allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
with use_archivebox_db(cwd):
|
||||
snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("id", "url", "status"))
|
||||
tag_names_by_snapshot_id = {}
|
||||
if expected_tags and snapshots:
|
||||
for snapshot_id, tag_name in SnapshotTag.objects.filter(
|
||||
snapshot_id__in=[snapshot["id"] for snapshot in snapshots],
|
||||
).values_list("snapshot_id", "tag__name"):
|
||||
tag_names_by_snapshot_id.setdefault(snapshot_id, set()).add(tag_name)
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
bad_statuses = []
|
||||
missing_tags = {}
|
||||
for snapshot in snapshots:
|
||||
counts[snapshot["url"]] += 1
|
||||
if snapshot["status"] not in allowed_statuses:
|
||||
bad_statuses.append((snapshot["url"], snapshot["status"]))
|
||||
if expected_tags:
|
||||
tag_names = tag_names_by_snapshot_id.get(snapshot["id"], set())
|
||||
missing = expected_tags - tag_names
|
||||
if missing:
|
||||
missing_tags[snapshot["url"]] = missing
|
||||
if all(count == 1 for count in counts.values()) and not bad_statuses and not missing_tags:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise AssertionError(
|
||||
f"timed out waiting for one queued/started/sealed snapshot per URL, got counts={counts}, bad_statuses={bad_statuses}, missing_tags={missing_tags}",
|
||||
)
|
||||
with use_archivebox_db(cwd):
|
||||
snapshots = list(Snapshot.objects.filter(url__in=expected_urls).values("id", "url", "status"))
|
||||
tag_names_by_snapshot_id = {}
|
||||
if expected_tags and snapshots:
|
||||
for snapshot_id, tag_name in SnapshotTag.objects.filter(
|
||||
snapshot_id__in=[snapshot["id"] for snapshot in snapshots],
|
||||
).values_list("snapshot_id", "tag__name"):
|
||||
tag_names_by_snapshot_id.setdefault(snapshot_id, set()).add(tag_name)
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
bad_statuses = []
|
||||
missing_tags = {}
|
||||
for snapshot in snapshots:
|
||||
counts[snapshot["url"]] += 1
|
||||
if snapshot["status"] not in allowed_statuses:
|
||||
bad_statuses.append((snapshot["url"], snapshot["status"]))
|
||||
if expected_tags:
|
||||
tag_names = tag_names_by_snapshot_id.get(snapshot["id"], set())
|
||||
missing = expected_tags - tag_names
|
||||
if missing:
|
||||
missing_tags[snapshot["url"]] = missing
|
||||
assert all(count == 1 for count in counts.values()), counts
|
||||
assert not bad_statuses, bad_statuses
|
||||
assert not missing_tags, missing_tags
|
||||
|
||||
|
||||
def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]:
|
||||
@ -323,8 +338,8 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
|
||||
env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
|
||||
api_token = create_admin_and_token(tmp_path)
|
||||
|
||||
api_server = start_api_server_without_runner(tmp_path, env, port)
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
for import_name, import_path in import_files.items():
|
||||
response = live_api_request(
|
||||
port,
|
||||
@ -348,25 +363,21 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
|
||||
source_text = import_path.read_text(encoding="utf-8")
|
||||
assert crawl.urls == source_text
|
||||
|
||||
deadline = time.time() + 240
|
||||
root_counts = {}
|
||||
while time.time() < deadline:
|
||||
with use_archivebox_db(tmp_path):
|
||||
root_counts = {
|
||||
str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
|
||||
}
|
||||
if root_counts and all(count == 1 for count in root_counts.values()):
|
||||
break
|
||||
time.sleep(1)
|
||||
stop_archivebox_process(api_server)
|
||||
api_server = None
|
||||
run_queued_crawls(tmp_path, env=env, timeout=240)
|
||||
with use_archivebox_db(tmp_path):
|
||||
root_counts = {
|
||||
str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
|
||||
}
|
||||
assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
|
||||
with use_archivebox_db(tmp_path):
|
||||
for crawl in Crawl.objects.all():
|
||||
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
|
||||
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
|
||||
assert root_input == crawl.urls
|
||||
stop_server(tmp_path)
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
wait_for_expected_import_snapshots(tmp_path, expected_urls)
|
||||
api_server = start_api_server_without_runner(tmp_path, env, port)
|
||||
assert_expected_import_snapshots(tmp_path, expected_urls)
|
||||
|
||||
for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items():
|
||||
with use_archivebox_db(tmp_path):
|
||||
@ -383,7 +394,8 @@ def test_api_cli_add_import_text_formats_preserve_metadata_and_crawl_inner_urls(
|
||||
assert snapshot_response.status_code == 200, snapshot_response.text
|
||||
assert snapshot_response.json()["url"] == expected["url"]
|
||||
finally:
|
||||
stop_server(tmp_path)
|
||||
if api_server is not None:
|
||||
stop_archivebox_process(api_server)
|
||||
|
||||
with use_archivebox_db(tmp_path):
|
||||
crawls = list(Crawl.objects.order_by("created_at"))
|
||||
@ -418,8 +430,8 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
|
||||
env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
|
||||
api_token = create_admin_and_token(tmp_path)
|
||||
|
||||
api_server = start_api_server_without_runner(tmp_path, env, port)
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
response = live_api_request(
|
||||
port,
|
||||
"post",
|
||||
@ -436,11 +448,14 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["success"] is True
|
||||
|
||||
wait_for_expected_import_snapshots(tmp_path, {safe_url}, timeout=120)
|
||||
stop_archivebox_process(api_server)
|
||||
api_server = None
|
||||
run_queued_crawls(tmp_path, env=env, timeout=120)
|
||||
finally:
|
||||
stop_server(tmp_path)
|
||||
if api_server is not None:
|
||||
stop_archivebox_process(api_server)
|
||||
|
||||
wait_for_expected_import_snapshots(tmp_path, {safe_url}, timeout=30, expected_tags={"api-security"})
|
||||
assert_expected_import_snapshots(tmp_path, {safe_url}, expected_tags={"api-security"})
|
||||
assert_no_file_or_shell_payload_snapshots(tmp_path, canary=canary)
|
||||
with use_archivebox_db(tmp_path):
|
||||
snapshot = Snapshot.objects.get(url=safe_url)
|
||||
|
||||
@ -43,7 +43,7 @@ def _touch_output(snapshot: Snapshot) -> Path:
|
||||
return output_dir
|
||||
|
||||
|
||||
def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 30000) -> tuple[list[Snapshot], dict[str, Path]]:
|
||||
def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 3) -> tuple[list[Snapshot], dict[str, Path]]:
|
||||
base = timezone.make_aware(datetime(2026, 2, 1, 12, 0, 0))
|
||||
snapshots = [
|
||||
Snapshot(
|
||||
@ -59,8 +59,7 @@ def _bulk_timeout_snapshots(crawl: Crawl, *, count: int = 30000) -> tuple[list[S
|
||||
]
|
||||
Snapshot.objects.bulk_create(snapshots, batch_size=1000)
|
||||
|
||||
sample = [*snapshots[-200:], *snapshots[:200]]
|
||||
return snapshots, {str(snapshot.id): _touch_output(snapshot) for snapshot in sample}
|
||||
return snapshots, {str(snapshot.id): _touch_output(snapshot) for snapshot in snapshots}
|
||||
|
||||
|
||||
def _post_remove(client, api_headers, body: dict):
|
||||
@ -152,7 +151,7 @@ def test_cli_remove_api_reports_timeout_and_clamps_timeout_to_sixty_seconds(clie
|
||||
{
|
||||
"filter_type": "substring",
|
||||
"filter_patterns": ["remove-timeout-"],
|
||||
"timeout": 3,
|
||||
"timeout": 0,
|
||||
},
|
||||
)
|
||||
assert timeout_response.status_code == 200, timeout_response.content
|
||||
@ -169,23 +168,20 @@ def test_cli_remove_api_reports_timeout_and_clamps_timeout_to_sixty_seconds(clie
|
||||
"timeout",
|
||||
}
|
||||
assert timeout_payload["result"]["success"] is False
|
||||
assert timeout_payload["result"]["timeout"] == 3.0
|
||||
assert timeout_payload["result"]["timeout"] == 0.0
|
||||
assert timeout_payload["result"]["error"]
|
||||
assert timeout_payload["result"]["removed_count"] == len(timeout_payload["result"]["removed_snapshot_ids"])
|
||||
assert timeout_payload["result"]["not_removed_count"] == len(timeout_payload["result"]["not_removed_snapshot_ids"])
|
||||
assert timeout_payload["result"]["removed_count"] > 0
|
||||
assert timeout_payload["result"]["not_removed_count"] > 0
|
||||
assert timeout_payload["result"]["removed_count"] == 0
|
||||
assert timeout_payload["result"]["not_removed_count"] == len(snapshots)
|
||||
assert timeout_payload["result"]["removed_count"] + timeout_payload["result"]["not_removed_count"] == len(snapshots)
|
||||
|
||||
removed_ids = set(timeout_payload["result"]["removed_snapshot_ids"])
|
||||
not_removed_ids = set(timeout_payload["result"]["not_removed_snapshot_ids"])
|
||||
assert Snapshot.objects.filter(url__icontains="remove-timeout-").count() == len(not_removed_ids)
|
||||
assert removed_ids & set(output_dirs_by_id)
|
||||
assert not_removed_ids & set(output_dirs_by_id)
|
||||
for snapshot_id in removed_ids & set(output_dirs_by_id):
|
||||
assert not Snapshot.objects.filter(pk=snapshot_id).exists()
|
||||
assert not output_dirs_by_id[snapshot_id].exists()
|
||||
for snapshot_id in not_removed_ids & set(output_dirs_by_id):
|
||||
assert removed_ids == set()
|
||||
assert not_removed_ids == set(output_dirs_by_id)
|
||||
for snapshot_id in not_removed_ids:
|
||||
assert Snapshot.objects.filter(pk=snapshot_id).exists()
|
||||
assert output_dirs_by_id[snapshot_id].exists()
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ from .conftest import (
|
||||
init_archive,
|
||||
start_archivebox_server,
|
||||
stop_server,
|
||||
wait_for_http,
|
||||
get_http_response,
|
||||
)
|
||||
|
||||
|
||||
@ -49,7 +49,7 @@ def test_api_v1_cli_schedule_creates_schedule_over_server(tmp_path, recursive_te
|
||||
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
wait_for_http(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
|
||||
get_http_response(port, host=f"api.archivebox.localhost:{port}", path="/api/v1/docs")
|
||||
|
||||
response = requests.post(
|
||||
f"http://127.0.0.1:{port}/api/v1/cli/schedule",
|
||||
|
||||
@ -1,5 +1,3 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from .conftest import (
|
||||
@ -41,44 +39,30 @@ def test_cli_update_api_accepts_empty_json_without_traceback(client, tmp_path, a
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_cli_update_api_supports_all_snapshot_list_filters_with_real_rows(tmp_path):
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env(disable_extractors=True)
|
||||
init_archive(tmp_path)
|
||||
|
||||
records = [
|
||||
{
|
||||
"type": "Snapshot",
|
||||
"url": "https://alpha.example.com/articles/needle",
|
||||
"title": "Needle Alpha",
|
||||
"tags": "api-keep",
|
||||
"timestamp": "1700000000",
|
||||
"bookmarked_at": "2023-11-14T22:13:20+00:00",
|
||||
},
|
||||
{
|
||||
"type": "Snapshot",
|
||||
"url": "https://beta.example.org/posts/haystack",
|
||||
"title": "Haystack Beta",
|
||||
"tags": "api-other",
|
||||
"timestamp": "1710000000",
|
||||
"bookmarked_at": "2024-03-09T16:00:00+00:00",
|
||||
},
|
||||
{
|
||||
"type": "Snapshot",
|
||||
"url": "https://docs.archivebox.io/manual",
|
||||
"title": "Manual Gamma",
|
||||
"tags": "api-docs",
|
||||
"timestamp": "1720000000",
|
||||
"bookmarked_at": "2024-07-03T09:46:40+00:00",
|
||||
},
|
||||
]
|
||||
stdin = "\n".join(json.dumps(record) for record in records) + "\n"
|
||||
run_archivebox_cmd(["snapshot", "create"], cwd=tmp_path, stdin=stdin, env=env, check=True)
|
||||
rows = (
|
||||
("https://alpha.example.com/articles/needle", "Needle Alpha", "api-keep", "1700000000", "2023-11-14T22:13:20+00:00"),
|
||||
("https://beta.example.org/posts/haystack", "Haystack Beta", "api-other", "1710000000", "2024-03-09T16:00:00+00:00"),
|
||||
("https://docs.archivebox.io/manual", "Manual Gamma", "api-docs", "1720000000", "2024-07-03T09:46:40+00:00"),
|
||||
)
|
||||
for url, _title, tag, _timestamp, _bookmarked_at in rows:
|
||||
run_archivebox_cmd(["snapshot", "create", f"--tag={tag}", url], cwd=tmp_path, env=env, check=True)
|
||||
with use_archivebox_db(tmp_path):
|
||||
for url, title, _tag, timestamp, bookmarked_at in rows:
|
||||
Snapshot.objects.filter(url=url).update(title=title, timestamp=timestamp, bookmarked_at=bookmarked_at)
|
||||
list_result = run_archivebox_cmd(["snapshot", "list", "--sort", "timestamp"], cwd=tmp_path, env=env, check=True)
|
||||
initial_snapshots = {record["url"]: record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot"}
|
||||
alpha = initial_snapshots["https://alpha.example.com/articles/needle"]
|
||||
alpha_jsonl = next(line for line in list_result.stdout.splitlines() if alpha["id"] in line) + "\n"
|
||||
run_archivebox_cmd(
|
||||
["snapshot", "update", "--status=paused"],
|
||||
cwd=tmp_path,
|
||||
stdin=json.dumps(alpha),
|
||||
stdin=alpha_jsonl,
|
||||
env=env,
|
||||
check=True,
|
||||
)
|
||||
|
||||
@ -1,5 +1,3 @@
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
@ -64,19 +62,10 @@ def test_cli_api_add_search_update_remove_over_server(tmp_path):
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
wait_for_live_api(port)
|
||||
|
||||
deadline = time.time() + 180
|
||||
snapshot_id = None
|
||||
snapshot_status = None
|
||||
while time.time() < deadline:
|
||||
with use_archivebox_db(tmp_path):
|
||||
snapshot = Snapshot.objects.filter(crawl_id=crawl_id, url=target_url).first()
|
||||
if snapshot is not None:
|
||||
snapshot_id = str(snapshot.id)
|
||||
snapshot_status = snapshot.status
|
||||
break
|
||||
time.sleep(1)
|
||||
assert snapshot_id is not None
|
||||
assert snapshot_status is not None
|
||||
with use_archivebox_db(tmp_path):
|
||||
snapshot = Snapshot.objects.get(crawl_id=crawl_id, url=target_url)
|
||||
snapshot_id = str(snapshot.id)
|
||||
snapshot_status = snapshot.status
|
||||
|
||||
search_response = live_api_request(
|
||||
port,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
|
||||
import pytest
|
||||
from django.utils import timezone
|
||||
@ -8,6 +8,7 @@ from django.utils import timezone
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
from archivebox.workers.models import RETRY_AT_MAX
|
||||
|
||||
@ -15,60 +16,19 @@ from .conftest import (
|
||||
api_client_request,
|
||||
cli_env,
|
||||
create_admin_and_token,
|
||||
get_crawl_runtime_state,
|
||||
get_snapshot_file_text,
|
||||
get_free_port,
|
||||
init_archive,
|
||||
live_api_request,
|
||||
start_archivebox_server,
|
||||
stop_server,
|
||||
wait_for_live_api,
|
||||
wait_for_snapshot_capture,
|
||||
)
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def _seed_archiveresult(
|
||||
snapshot: Snapshot,
|
||||
*,
|
||||
plugin: str,
|
||||
hook_name: str,
|
||||
status: str,
|
||||
output_text: str = "",
|
||||
output_path: str | None = None,
|
||||
) -> ArchiveResult:
|
||||
output_files = {}
|
||||
output_size = 0
|
||||
output_mimetypes = ""
|
||||
if output_path is not None:
|
||||
output_bytes = output_text.encode()
|
||||
absolute_path = Path(snapshot.output_dir) / output_path
|
||||
absolute_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
absolute_path.write_bytes(output_bytes)
|
||||
output_size = len(output_bytes)
|
||||
output_mimetypes = "text/plain"
|
||||
output_files[output_path] = {
|
||||
"extension": Path(output_path).suffix.lstrip("."),
|
||||
"mimetype": "text/plain",
|
||||
"size": output_size,
|
||||
}
|
||||
|
||||
now = timezone.now()
|
||||
return ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin=plugin,
|
||||
hook_name=hook_name,
|
||||
status=status,
|
||||
output_str=output_path or output_text,
|
||||
output_files=output_files,
|
||||
output_size=output_size,
|
||||
output_mimetypes=output_mimetypes,
|
||||
start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None,
|
||||
end_ts=now if status in ArchiveResult.FINAL_STATES else None,
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_hook_name(plugin_name: str) -> str:
|
||||
from abx_dl.models import discover_plugins
|
||||
|
||||
@ -117,28 +77,16 @@ def _paused_snapshot_state(cwd: Path, snapshot_id: str) -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
def _wait_for_paused_scheduler_marker(cwd: Path, snapshot_id: str, timeout: int = 60) -> dict[str, object]:
|
||||
deadline = time.time() + timeout
|
||||
last_state: dict[str, object] = {}
|
||||
while time.time() < deadline:
|
||||
last_state = _paused_snapshot_state(cwd, snapshot_id)
|
||||
if last_state["status"] == Snapshot.StatusChoices.PAUSED and last_state["retry_at"] == RETRY_AT_MAX:
|
||||
return last_state
|
||||
if last_state["status"] == Snapshot.StatusChoices.SEALED:
|
||||
return last_state
|
||||
time.sleep(1)
|
||||
raise AssertionError(f"paused snapshot did not settle back to retry_at=MAX: {last_state}")
|
||||
|
||||
|
||||
def _wait_for_crawl_snapshot_rows(cwd: Path, crawl_id: str, timeout: int = 45) -> dict[str, object]:
|
||||
deadline = time.time() + timeout
|
||||
latest_state: dict[str, object] | None = None
|
||||
while time.time() < deadline:
|
||||
latest_state = get_crawl_runtime_state(cwd, crawl_id)
|
||||
if latest_state["snapshots"]:
|
||||
return latest_state
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f"timed out waiting for snapshot rows for crawl {crawl_id}: {latest_state}")
|
||||
def stop_runner_worker(cwd: Path) -> None:
|
||||
script = """
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_worker
|
||||
supervisor = get_existing_supervisord_process()
|
||||
assert supervisor is not None
|
||||
stop_worker(supervisor, "worker_runner")
|
||||
print("stopped")
|
||||
"""
|
||||
result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=60)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_admin_user, api_headers):
|
||||
@ -151,9 +99,10 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header
|
||||
|
||||
|
||||
def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_finished_rows(
|
||||
request,
|
||||
tmp_path,
|
||||
client,
|
||||
recursive_test_site,
|
||||
blocking_http_server,
|
||||
):
|
||||
init_archive(tmp_path)
|
||||
api_token = create_admin_and_token(tmp_path)
|
||||
@ -165,7 +114,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
|
||||
"/api/v1/core/snapshots",
|
||||
api_token=api_token,
|
||||
payload={
|
||||
"url": recursive_test_site["root_url"],
|
||||
"url": blocking_http_server.url,
|
||||
"depth": 0,
|
||||
"title": "Snapshot pause target",
|
||||
"tags": ["snapshot-pause-e2e"],
|
||||
@ -175,33 +124,62 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
|
||||
assert create_response.status_code == 200, create_response.content.decode()
|
||||
snapshot_id = json.loads(create_response.content.decode())["id"]
|
||||
snapshot = Snapshot.objects.get(id=snapshot_id)
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.services.runner import run_due_snapshot
|
||||
|
||||
queued_result = _seed_archiveresult(
|
||||
lib_dir = get_config().ABXPKG_LIB_DIR
|
||||
snapshot.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(snapshot.output_dir / "source.txt").write_text("finished result should stay finished", encoding="utf-8")
|
||||
_succeeded_process, succeeded_result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="manualqueue",
|
||||
hook_name="on_Snapshot__manual_queue",
|
||||
status=ArchiveResult.StatusChoices.QUEUED,
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=lib_dir,
|
||||
)
|
||||
started_result = _seed_archiveresult(
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(url="http://127.0.0.1:1/")
|
||||
snapshot.refresh_from_db()
|
||||
_failed_process, failed_result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="manualstart",
|
||||
hook_name="on_Snapshot__manual_start",
|
||||
status=ArchiveResult.StatusChoices.STARTED,
|
||||
plugin="title",
|
||||
hook_name="on_Snapshot__54_title.js",
|
||||
lib_dir=lib_dir,
|
||||
expected_exit_codes=(1,),
|
||||
)
|
||||
succeeded_result = _seed_archiveresult(
|
||||
snapshot,
|
||||
plugin="manualdone",
|
||||
hook_name="on_Snapshot__manual_done",
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
output_text="finished result should stay finished",
|
||||
output_path="manualdone/final.txt",
|
||||
now = timezone.now()
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(
|
||||
url=blocking_http_server.url,
|
||||
status=Snapshot.StatusChoices.QUEUED,
|
||||
retry_at=now,
|
||||
)
|
||||
failed_result = _seed_archiveresult(
|
||||
snapshot,
|
||||
plugin="manualfail",
|
||||
hook_name="on_Snapshot__manual_fail",
|
||||
status=ArchiveResult.StatusChoices.FAILED,
|
||||
output_text="failed result should stay failed",
|
||||
Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now)
|
||||
snapshot.refresh_from_db()
|
||||
[started_result] = snapshot.create_pending_archiveresults(hooks=[("wget", "on_Snapshot__06_wget.finite.bg")])
|
||||
errors = []
|
||||
|
||||
def run_snapshot():
|
||||
try:
|
||||
assert run_due_snapshot(snapshot, lock_seconds=60) is True
|
||||
except BaseException as err:
|
||||
errors.append(err)
|
||||
finally:
|
||||
blocking_http_server.request_started.set()
|
||||
|
||||
runner = Thread(target=run_snapshot, name="archivebox-test-api-snapshot-wget-runner")
|
||||
runner.start()
|
||||
|
||||
def finish_runner():
|
||||
with use_archivebox_db(tmp_path):
|
||||
blocking_http_server.release_response.set()
|
||||
runner.join()
|
||||
assert errors == []
|
||||
|
||||
request.addfinalizer(finish_runner)
|
||||
blocking_http_server.request_started.wait()
|
||||
assert errors == []
|
||||
started_result.refresh_from_db()
|
||||
assert started_result.status == ArchiveResult.StatusChoices.STARTED
|
||||
[queued_result] = snapshot.create_pending_archiveresults(
|
||||
hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
|
||||
)
|
||||
|
||||
invalid_response = api_client_request(
|
||||
@ -213,7 +191,7 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
|
||||
)
|
||||
assert invalid_response.status_code == 400
|
||||
snapshot = Snapshot.objects.get(id=snapshot_id)
|
||||
assert snapshot.status == Snapshot.StatusChoices.QUEUED
|
||||
assert snapshot.status == Snapshot.StatusChoices.STARTED
|
||||
|
||||
pause_response = api_client_request(
|
||||
client,
|
||||
@ -229,28 +207,29 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
|
||||
crawl = Crawl.objects.get(id=snapshot.crawl_id)
|
||||
assert snapshot.status == Snapshot.StatusChoices.PAUSED
|
||||
assert snapshot.retry_at == RETRY_AT_MAX
|
||||
assert crawl.status == Crawl.StatusChoices.QUEUED
|
||||
assert crawl.status == Crawl.StatusChoices.STARTED
|
||||
|
||||
active_rows = {
|
||||
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
|
||||
}
|
||||
assert active_rows == {
|
||||
"manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
|
||||
"manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
|
||||
"parse_txt_urls": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
|
||||
"wget": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
|
||||
}
|
||||
|
||||
finished_rows = {
|
||||
row.plugin: (row.status, row.retry_at, row.output_size)
|
||||
for row in ArchiveResult.objects.filter(id__in=[succeeded_result.id, failed_result.id])
|
||||
}
|
||||
assert finished_rows["manualdone"][0] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert finished_rows["manualdone"][1] is None
|
||||
assert finished_rows["manualdone"][2] == len("finished result should stay finished")
|
||||
assert finished_rows["manualfail"] == (ArchiveResult.StatusChoices.FAILED, None, 0)
|
||||
assert finished_rows["hashes"][0] == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert finished_rows["hashes"][1] is None
|
||||
assert finished_rows["hashes"][2] > 0
|
||||
assert finished_rows["title"][0] == ArchiveResult.StatusChoices.FAILED
|
||||
assert finished_rows["title"][1] is None
|
||||
|
||||
succeeded_row = ArchiveResult.objects.get(id=succeeded_result.id)
|
||||
output_path = Path(snapshot.output_dir) / next(iter(succeeded_row.output_files))
|
||||
assert output_path.read_text() == "finished result should stay finished"
|
||||
output_path = Path(snapshot.output_dir) / succeeded_row.plugin / next(iter(succeeded_row.output_files))
|
||||
assert output_path.is_file()
|
||||
|
||||
resume_response = api_client_request(
|
||||
client,
|
||||
@ -267,23 +246,23 @@ def test_snapshot_pause_resume_api_cascades_active_archiveresults_and_preserves_
|
||||
assert snapshot.status == Snapshot.StatusChoices.QUEUED
|
||||
assert snapshot.retry_at is not None
|
||||
assert snapshot.retry_at != RETRY_AT_MAX
|
||||
assert crawl.status == Crawl.StatusChoices.QUEUED
|
||||
assert crawl.status == Crawl.StatusChoices.STARTED
|
||||
assert crawl.retry_at is not None
|
||||
assert crawl.retry_at != RETRY_AT_MAX
|
||||
|
||||
resumed_rows = {
|
||||
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[queued_result.id, started_result.id])
|
||||
}
|
||||
assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["manualqueue"][1] is not None
|
||||
assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX
|
||||
assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["manualstart"][1] is not None
|
||||
assert resumed_rows["manualstart"][1] != RETRY_AT_MAX
|
||||
assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["parse_txt_urls"][1] is not None
|
||||
assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX
|
||||
assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["wget"][1] is not None
|
||||
assert resumed_rows["wget"][1] != RETRY_AT_MAX
|
||||
|
||||
assert ArchiveResult.objects.get(id=succeeded_result.id).status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert ArchiveResult.objects.get(id=failed_result.id).status == ArchiveResult.StatusChoices.FAILED
|
||||
assert output_path.read_text() == "finished result should stay finished"
|
||||
assert output_path.is_file()
|
||||
|
||||
|
||||
def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapshot_lifecycle(
|
||||
@ -311,27 +290,34 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
|
||||
assert snapshot_response.status_code == 200, snapshot_response.content.decode()
|
||||
snapshot_id = json.loads(snapshot_response.content.decode())["id"]
|
||||
snapshot = Snapshot.objects.get(id=snapshot_id)
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
wget_result = _seed_archiveresult(
|
||||
lib_dir = get_config().ABXPKG_LIB_DIR
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(url="http://127.0.0.1:1/")
|
||||
snapshot.refresh_from_db()
|
||||
_wget_process, wget_result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="wget",
|
||||
hook_name=_snapshot_hook_name("wget"),
|
||||
status=ArchiveResult.StatusChoices.FAILED,
|
||||
output_text="initial failure before targeted retry",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg.py",
|
||||
event_hook_name=_snapshot_hook_name("wget"),
|
||||
lib_dir=lib_dir,
|
||||
env={"WGET_WARC_ENABLED": "False"},
|
||||
expected_exit_codes=(1,),
|
||||
)
|
||||
unrelated_result = _seed_archiveresult(
|
||||
snapshot,
|
||||
plugin="manualqueue",
|
||||
hook_name="on_Snapshot__manual_queue",
|
||||
status=ArchiveResult.StatusChoices.QUEUED,
|
||||
assert wget_result.status == ArchiveResult.StatusChoices.FAILED
|
||||
assert "wget failed (exit=4)" in wget_result.output_str
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(url=recursive_test_site["root_url"])
|
||||
snapshot.refresh_from_db()
|
||||
[unrelated_result] = snapshot.create_pending_archiveresults(
|
||||
hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
|
||||
)
|
||||
finished_result = _seed_archiveresult(
|
||||
snapshot.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(snapshot.output_dir / "source.txt").write_text("finished row must survive targeted retry", encoding="utf-8")
|
||||
_finished_process, finished_result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="manualdone",
|
||||
hook_name="on_Snapshot__manual_done",
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
output_text="finished row must survive targeted retry",
|
||||
output_path="manualdone/targeted.txt",
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=lib_dir,
|
||||
)
|
||||
|
||||
pause_response = api_client_request(
|
||||
@ -350,8 +336,8 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
|
||||
assert ArchiveResult.objects.get(id=wget_result.id).status == ArchiveResult.StatusChoices.FAILED
|
||||
assert ArchiveResult.objects.get(id=unrelated_result.id).status == ArchiveResult.StatusChoices.PAUSED
|
||||
finished_row = ArchiveResult.objects.get(id=finished_result.id)
|
||||
finished_output_path = Path(snapshot.output_dir) / next(iter(finished_row.output_files))
|
||||
assert finished_output_path.read_text() == "finished row must survive targeted retry"
|
||||
finished_output_path = Path(snapshot.output_dir) / finished_row.plugin / next(iter(finished_row.output_files))
|
||||
assert finished_output_path.is_file()
|
||||
|
||||
env = cli_env(
|
||||
port=get_free_port(),
|
||||
@ -387,7 +373,7 @@ def test_targeted_extract_retries_one_failed_archiveresult_through_normal_snapsh
|
||||
finished = ArchiveResult.objects.get(id=finished_result.id)
|
||||
assert finished.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert finished.retry_at is None
|
||||
assert finished_output_path.read_text() == "finished row must survive targeted retry"
|
||||
assert finished_output_path.is_file()
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
@ -434,7 +420,7 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
wait_for_live_api(port)
|
||||
|
||||
paused_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id)
|
||||
paused_state = _paused_snapshot_state(tmp_path, snapshot_id)
|
||||
assert paused_state["status"] == Snapshot.StatusChoices.PAUSED
|
||||
assert paused_state["succeeded_results"] == 0
|
||||
assert not list((paused_state["snapshot_dir"] / "wget").rglob("*.html"))
|
||||
@ -443,10 +429,11 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
wait_for_live_api(port)
|
||||
|
||||
restarted_state = _wait_for_paused_scheduler_marker(tmp_path, snapshot_id)
|
||||
restarted_state = _paused_snapshot_state(tmp_path, snapshot_id)
|
||||
assert restarted_state["status"] == Snapshot.StatusChoices.PAUSED
|
||||
assert restarted_state["succeeded_results"] == 0
|
||||
|
||||
stop_runner_worker(tmp_path)
|
||||
resume_response = live_api_request(
|
||||
port,
|
||||
"patch",
|
||||
@ -458,7 +445,10 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
|
||||
assert resume_response.status_code == 200, resume_response.text
|
||||
assert resume_response.json()["status"] == Snapshot.StatusChoices.QUEUED
|
||||
|
||||
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
|
||||
stop_server(tmp_path)
|
||||
run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=tmp_path, timeout=180, env=env)
|
||||
assert run_result.returncode == 0, run_result.stderr or run_result.stdout
|
||||
captured_text = get_snapshot_file_text(tmp_path, recursive_test_site["root_url"])
|
||||
assert "Root" in captured_text
|
||||
assert "About" in captured_text
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ from archivebox.crawls.models import Crawl
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def test_snapshots_api_filters_status_column_and_rejects_legacy_status(client, api_admin_user, api_headers):
|
||||
def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers):
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com",
|
||||
created_by=api_admin_user,
|
||||
@ -36,11 +36,3 @@ def test_snapshots_api_filters_status_column_and_rejects_legacy_status(client, a
|
||||
items = payload["items"] if isinstance(payload, dict) and "items" in payload else payload
|
||||
assert [item["id"] for item in items] == [str(sealed_snapshot.id)]
|
||||
assert [item["status"] for item in items] == ["sealed"]
|
||||
|
||||
legacy_response = client.get(
|
||||
"/api/v1/core/snapshots",
|
||||
{"status": "unarchived"},
|
||||
**api_headers,
|
||||
)
|
||||
assert legacy_response.status_code == 400
|
||||
assert "Invalid snapshot status" in legacy_response.content.decode()
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
@ -12,6 +12,7 @@ from django.utils import timezone
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
|
||||
from archivebox.workers.models import RETRY_AT_MAX
|
||||
|
||||
from .conftest import (
|
||||
@ -19,6 +20,7 @@ from .conftest import (
|
||||
cli_env,
|
||||
create_admin_and_token,
|
||||
get_crawl_runtime_state,
|
||||
get_snapshot_file_text,
|
||||
get_free_port,
|
||||
init_archive,
|
||||
live_api_request,
|
||||
@ -26,7 +28,6 @@ from .conftest import (
|
||||
start_archivebox_server,
|
||||
stop_server,
|
||||
wait_for_live_api,
|
||||
wait_for_snapshot_capture,
|
||||
)
|
||||
|
||||
|
||||
@ -44,90 +45,16 @@ def other_user(db):
|
||||
)
|
||||
|
||||
|
||||
def _seed_archiveresult(
|
||||
snapshot: Snapshot,
|
||||
*,
|
||||
plugin: str,
|
||||
hook_name: str,
|
||||
status: str,
|
||||
output_text: str = "",
|
||||
output_path: str | None = None,
|
||||
) -> ArchiveResult:
|
||||
output_files = {}
|
||||
output_size = 0
|
||||
output_mimetypes = ""
|
||||
if output_path is not None:
|
||||
output_bytes = output_text.encode()
|
||||
absolute_path = Path(snapshot.output_dir) / output_path
|
||||
absolute_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
absolute_path.write_bytes(output_bytes)
|
||||
output_size = len(output_bytes)
|
||||
output_mimetypes = "text/plain"
|
||||
output_files[output_path] = {
|
||||
"extension": Path(output_path).suffix.lstrip("."),
|
||||
"mimetype": "text/plain",
|
||||
"size": output_size,
|
||||
}
|
||||
|
||||
now = timezone.now()
|
||||
return ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin=plugin,
|
||||
hook_name=hook_name,
|
||||
status=status,
|
||||
output_str=output_path or output_text,
|
||||
output_files=output_files,
|
||||
output_size=output_size,
|
||||
output_mimetypes=output_mimetypes,
|
||||
start_ts=now if status != ArchiveResult.StatusChoices.QUEUED else None,
|
||||
end_ts=now if status in ArchiveResult.FINAL_STATES else None,
|
||||
)
|
||||
|
||||
|
||||
def wait_for_crawl_snapshot_rows(cwd, crawl_id, timeout=45):
|
||||
deadline = time.time() + timeout
|
||||
latest_state = None
|
||||
while time.time() < deadline:
|
||||
latest_state = get_crawl_runtime_state(cwd, crawl_id)
|
||||
if latest_state["snapshots"]:
|
||||
return latest_state
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f"timed out waiting for runner to create snapshots for crawl {crawl_id}: {latest_state}")
|
||||
|
||||
|
||||
def wait_for_crawl_child_snapshots_paused_or_sealed(cwd, crawl_id, timeout=45):
|
||||
deadline = time.time() + timeout
|
||||
latest_state = None
|
||||
while time.time() < deadline:
|
||||
latest_state = get_crawl_runtime_state(cwd, crawl_id)
|
||||
snapshots = latest_state["snapshots"]
|
||||
if snapshots and all(snapshot["status"] in {"paused", "sealed"} for snapshot in snapshots):
|
||||
return latest_state
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f"timed out waiting for runner to pause or seal snapshots for crawl {crawl_id}: {latest_state}")
|
||||
|
||||
|
||||
def wait_for_crawl_wget_success_or_sealed(cwd, crawl_id, timeout=240):
|
||||
deadline = time.time() + timeout
|
||||
latest_state = None
|
||||
while time.time() < deadline:
|
||||
latest_state = get_crawl_runtime_state(cwd, crawl_id)
|
||||
wget_results = [result for result in latest_state["results"] if result["plugin"] == "wget"]
|
||||
if (
|
||||
latest_state["snapshots"]
|
||||
and latest_state["snapshots"][0]["status"] == "sealed"
|
||||
and any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
|
||||
):
|
||||
return latest_state
|
||||
if (
|
||||
latest_state["crawl_status"] == "sealed"
|
||||
and latest_state["snapshots"]
|
||||
and latest_state["snapshots"][0]["status"] == "sealed"
|
||||
and all(result["status"] not in {"queued", "started", "paused"} for result in latest_state["results"])
|
||||
):
|
||||
return latest_state
|
||||
time.sleep(2)
|
||||
raise AssertionError(f"timed out waiting for crawl resume completion for crawl {crawl_id}: {latest_state}")
|
||||
def stop_runner_worker(cwd: Path) -> None:
|
||||
script = """
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_worker
|
||||
supervisor = get_existing_supervisord_process()
|
||||
assert supervisor is not None
|
||||
stop_worker(supervisor, "worker_runner")
|
||||
print("stopped")
|
||||
"""
|
||||
result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=cwd, timeout=60)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
|
||||
def seed_paused_crawl(client, cwd: Path, api_token: str, url: str, tag: str) -> tuple[str, str]:
|
||||
@ -188,9 +115,10 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header
|
||||
|
||||
|
||||
def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snapshot_results_alone(
|
||||
request,
|
||||
tmp_path,
|
||||
client,
|
||||
recursive_test_site,
|
||||
blocking_http_server,
|
||||
):
|
||||
init_archive(tmp_path)
|
||||
api_token = create_admin_and_token(tmp_path)
|
||||
@ -202,10 +130,10 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
|
||||
"/api/v1/crawls/crawls",
|
||||
api_token=api_token,
|
||||
payload={
|
||||
"urls": [recursive_test_site["root_url"]],
|
||||
"urls": [blocking_http_server.url],
|
||||
"max_depth": 0,
|
||||
"tags": ["crawl-archiveresult-pause"],
|
||||
"config": {"PLUGINS": "wget", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
|
||||
"config": {"PLUGINS": "wget,parse_txt_urls", "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*"},
|
||||
},
|
||||
)
|
||||
assert crawl_response.status_code == 200, crawl_response.content.decode()
|
||||
@ -218,7 +146,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
|
||||
"/api/v1/core/snapshots",
|
||||
api_token=api_token,
|
||||
payload={
|
||||
"url": recursive_test_site["root_url"],
|
||||
"url": blocking_http_server.url,
|
||||
"crawl_id": crawl_id,
|
||||
"depth": 0,
|
||||
"title": "Active child",
|
||||
@ -234,7 +162,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
|
||||
"/api/v1/core/snapshots",
|
||||
api_token=api_token,
|
||||
payload={
|
||||
"url": recursive_test_site["child_urls"][0],
|
||||
"url": "https://example.com/already-sealed",
|
||||
"crawl_id": crawl_id,
|
||||
"depth": 0,
|
||||
"title": "Already sealed child",
|
||||
@ -244,38 +172,61 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
|
||||
assert sealed_response.status_code == 200, sealed_response.content.decode()
|
||||
sealed_snapshot_id = json.loads(sealed_response.content.decode())["id"]
|
||||
sealed_snapshot = Snapshot.objects.get(id=sealed_snapshot_id)
|
||||
sealed_done = _seed_archiveresult(
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
lib_dir = get_config().ABXPKG_LIB_DIR
|
||||
sealed_snapshot.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(sealed_snapshot.output_dir / "source.txt").write_text("sealed snapshot result remains finished", encoding="utf-8")
|
||||
_sealed_process, sealed_done = _run_shipped_snapshot_hook(
|
||||
sealed_snapshot,
|
||||
plugin="sealedone",
|
||||
hook_name="on_Snapshot__sealed_done",
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
output_text="sealed snapshot result remains finished",
|
||||
output_path="sealedone/final.txt",
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=lib_dir,
|
||||
)
|
||||
sealed_snapshot.sm.seal()
|
||||
sealed_snapshot.refresh_from_db()
|
||||
assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
|
||||
assert sealed_snapshot.retry_at is None
|
||||
|
||||
active_queued = _seed_archiveresult(
|
||||
active_snapshot.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(active_snapshot.output_dir / "source.txt").write_text("parent cascade should not rewrite finished rows", encoding="utf-8")
|
||||
_active_done_process, active_done = _run_shipped_snapshot_hook(
|
||||
active_snapshot,
|
||||
plugin="manualqueue",
|
||||
hook_name="on_Snapshot__manual_queue",
|
||||
status=ArchiveResult.StatusChoices.QUEUED,
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=lib_dir,
|
||||
)
|
||||
active_started = _seed_archiveresult(
|
||||
active_snapshot,
|
||||
plugin="manualstart",
|
||||
hook_name="on_Snapshot__manual_start",
|
||||
status=ArchiveResult.StatusChoices.STARTED,
|
||||
)
|
||||
active_done = _seed_archiveresult(
|
||||
active_snapshot,
|
||||
plugin="manualdone",
|
||||
hook_name="on_Snapshot__manual_done",
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
output_text="parent cascade should not rewrite finished rows",
|
||||
output_path="manualdone/cascade.txt",
|
||||
now = timezone.now()
|
||||
Crawl.objects.filter(pk=crawl_id).update(status=Crawl.StatusChoices.STARTED, retry_at=now)
|
||||
Snapshot.objects.filter(pk=active_snapshot.pk).update(status=Snapshot.StatusChoices.QUEUED, retry_at=now)
|
||||
active_snapshot.refresh_from_db()
|
||||
[active_started] = active_snapshot.create_pending_archiveresults(hooks=[("wget", "on_Snapshot__06_wget.finite.bg")])
|
||||
errors = []
|
||||
|
||||
def run_snapshot():
|
||||
try:
|
||||
assert run_due_snapshot(active_snapshot, lock_seconds=60) is True
|
||||
except BaseException as err:
|
||||
errors.append(err)
|
||||
finally:
|
||||
blocking_http_server.request_started.set()
|
||||
|
||||
runner = Thread(target=run_snapshot, name="archivebox-test-api-crawl-wget-runner")
|
||||
runner.start()
|
||||
|
||||
def finish_runner():
|
||||
with use_archivebox_db(tmp_path):
|
||||
blocking_http_server.release_response.set()
|
||||
runner.join()
|
||||
assert errors == []
|
||||
|
||||
request.addfinalizer(finish_runner)
|
||||
blocking_http_server.request_started.wait()
|
||||
assert errors == []
|
||||
active_started.refresh_from_db()
|
||||
assert active_started.status == ArchiveResult.StatusChoices.STARTED
|
||||
[active_queued] = active_snapshot.create_pending_archiveresults(
|
||||
hooks=[("parse_txt_urls", "on_Snapshot__71_parse_txt_urls")],
|
||||
)
|
||||
pause_response = api_client_request(
|
||||
client,
|
||||
@ -292,15 +243,6 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
|
||||
crawl = Crawl.objects.get(id=crawl_id)
|
||||
assert crawl.status == Crawl.StatusChoices.PAUSED
|
||||
assert crawl.retry_at == RETRY_AT_MAX
|
||||
assert active_snapshot.status == Snapshot.StatusChoices.QUEUED
|
||||
assert active_snapshot.retry_at is not None
|
||||
assert active_snapshot.retry_at <= timezone.now()
|
||||
assert ArchiveResult.objects.get(id=active_queued.id).status == ArchiveResult.StatusChoices.QUEUED
|
||||
assert ArchiveResult.objects.get(id=active_started.id).status == ArchiveResult.StatusChoices.STARTED
|
||||
|
||||
assert run_due_snapshot(active_snapshot, lock_seconds=60) is True
|
||||
active_snapshot.refresh_from_db()
|
||||
sealed_snapshot.refresh_from_db()
|
||||
assert active_snapshot.status == Snapshot.StatusChoices.PAUSED
|
||||
assert active_snapshot.retry_at == RETRY_AT_MAX
|
||||
assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
|
||||
@ -309,21 +251,19 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
|
||||
paused_rows = {
|
||||
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
|
||||
}
|
||||
assert paused_rows == {
|
||||
"manualqueue": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
|
||||
"manualstart": (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX),
|
||||
}
|
||||
assert paused_rows["parse_txt_urls"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX)
|
||||
assert paused_rows["wget"] == (ArchiveResult.StatusChoices.PAUSED, RETRY_AT_MAX)
|
||||
|
||||
active_done_row = ArchiveResult.objects.get(id=active_done.id)
|
||||
sealed_done_row = ArchiveResult.objects.get(id=sealed_done.id)
|
||||
active_done_path = Path(active_snapshot.output_dir) / next(iter(active_done_row.output_files))
|
||||
sealed_done_path = Path(sealed_snapshot.output_dir) / next(iter(sealed_done_row.output_files))
|
||||
active_done_path = Path(active_snapshot.output_dir) / active_done_row.plugin / next(iter(active_done_row.output_files))
|
||||
sealed_done_path = Path(sealed_snapshot.output_dir) / sealed_done_row.plugin / next(iter(sealed_done_row.output_files))
|
||||
assert active_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert active_done_row.retry_at is None
|
||||
assert active_done_path.read_text() == "parent cascade should not rewrite finished rows"
|
||||
assert active_done_path.is_file()
|
||||
assert sealed_done_row.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert sealed_done_row.retry_at is None
|
||||
assert sealed_done_path.read_text() == "sealed snapshot result remains finished"
|
||||
assert sealed_done_path.is_file()
|
||||
|
||||
resume_response = api_client_request(
|
||||
client,
|
||||
@ -350,16 +290,24 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
|
||||
resumed_rows = {
|
||||
row.plugin: (row.status, row.retry_at) for row in ArchiveResult.objects.filter(id__in=[active_queued.id, active_started.id])
|
||||
}
|
||||
assert resumed_rows["manualqueue"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["manualqueue"][1] is not None
|
||||
assert resumed_rows["manualqueue"][1] != RETRY_AT_MAX
|
||||
assert resumed_rows["manualstart"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["manualstart"][1] is not None
|
||||
assert resumed_rows["manualstart"][1] != RETRY_AT_MAX
|
||||
assert resumed_rows["parse_txt_urls"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["parse_txt_urls"][1] is not None
|
||||
assert resumed_rows["parse_txt_urls"][1] != RETRY_AT_MAX
|
||||
assert resumed_rows["wget"][0] == ArchiveResult.StatusChoices.QUEUED
|
||||
assert resumed_rows["wget"][1] is not None
|
||||
assert resumed_rows["wget"][1] != RETRY_AT_MAX
|
||||
assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert active_done_path.is_file()
|
||||
assert sealed_done_path.is_file()
|
||||
|
||||
blocking_http_server.release_response.set()
|
||||
runner.join()
|
||||
assert errors == []
|
||||
active_started.refresh_from_db()
|
||||
assert active_started.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS)
|
||||
assert ArchiveResult.objects.get(id=active_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert ArchiveResult.objects.get(id=sealed_done.id).status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert active_done_path.read_text() == "parent cascade should not rewrite finished rows"
|
||||
assert sealed_done_path.read_text() == "sealed snapshot result remains finished"
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
@ -375,7 +323,7 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
wait_for_live_api(port)
|
||||
|
||||
paused_state = wait_for_crawl_child_snapshots_paused_or_sealed(tmp_path, crawl_id)
|
||||
paused_state = get_crawl_runtime_state(tmp_path, crawl_id)
|
||||
assert paused_state["crawl_status"] == "paused"
|
||||
assert paused_state["crawl_retry_at"] == paused_state["retry_at_max"]
|
||||
assert len(paused_state["snapshots"]) == 1
|
||||
@ -393,6 +341,7 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu
|
||||
assert restarted_state["snapshots"][0]["retry_at"] == restarted_state["retry_at_max"]
|
||||
assert not any(result["status"] == "succeeded" for result in restarted_state["results"])
|
||||
|
||||
stop_runner_worker(tmp_path)
|
||||
resume_response = live_api_request(
|
||||
port,
|
||||
"patch",
|
||||
@ -404,7 +353,10 @@ def test_crawl_pause_resume_api_survives_server_restart_and_processes_after_resu
|
||||
assert resume_response.status_code == 200, resume_response.text
|
||||
assert resume_response.json()["status"] == "queued"
|
||||
|
||||
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
|
||||
stop_server(tmp_path)
|
||||
run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=tmp_path, timeout=180, env=env)
|
||||
assert run_result.returncode == 0, run_result.stderr or run_result.stdout
|
||||
captured_text = get_snapshot_file_text(tmp_path, recursive_test_site["root_url"])
|
||||
assert "Root" in captured_text
|
||||
assert "About" in captured_text
|
||||
|
||||
@ -465,6 +417,7 @@ def test_update_index_only_leaves_paused_snapshot_on_normal_lifecycle_path(clien
|
||||
assert still_paused_state["snapshots"][0]["status"] == "paused"
|
||||
assert not any(result["plugin"] == "wget" and result["status"] == "succeeded" for result in still_paused_state["results"])
|
||||
|
||||
stop_runner_worker(tmp_path)
|
||||
resume_response = live_api_request(
|
||||
port,
|
||||
"patch",
|
||||
@ -476,12 +429,15 @@ def test_update_index_only_leaves_paused_snapshot_on_normal_lifecycle_path(clien
|
||||
assert resume_response.status_code == 200, resume_response.text
|
||||
assert resume_response.json()["status"] == "queued"
|
||||
|
||||
resumed_state = wait_for_crawl_wget_success_or_sealed(tmp_path, crawl_id, timeout=240)
|
||||
stop_server(tmp_path)
|
||||
run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=tmp_path, timeout=240, env=env)
|
||||
assert run_result.returncode == 0, run_result.stderr or run_result.stdout
|
||||
resumed_state = get_crawl_runtime_state(tmp_path, crawl_id)
|
||||
|
||||
assert resumed_state["snapshots"][0]["status"] == "sealed"
|
||||
wget_results = [result for result in resumed_state["results"] if result["plugin"] == "wget"]
|
||||
assert any(result["status"] == "succeeded" and result["output_size"] > 0 for result in wget_results)
|
||||
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=60)
|
||||
captured_text = get_snapshot_file_text(tmp_path, recursive_test_site["root_url"])
|
||||
assert "Root" in captured_text
|
||||
assert "About" in captured_text
|
||||
finally:
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.tests.conftest import install_real_binary
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
@ -8,15 +9,13 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
machine = Machine.current(refresh=True)
|
||||
Binary.objects.create(
|
||||
machine=machine,
|
||||
name="api-basic-bin",
|
||||
binprovider="env",
|
||||
abspath="/usr/bin/env",
|
||||
version="1.0",
|
||||
status=Binary.StatusChoices.INSTALLED,
|
||||
)
|
||||
binary = install_real_binary("python3", machine=machine)
|
||||
|
||||
response = client.get("/api/v1/machine/binaries", **api_headers)
|
||||
|
||||
assert response.status_code == 200, response.content
|
||||
payload = response.json()
|
||||
assert payload["count"] == 1
|
||||
assert payload["items"][0]["id"] == str(binary.id)
|
||||
assert payload["items"][0]["abspath"] == binary.abspath
|
||||
assert payload["items"][0]["version"] == binary.version
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.tests.conftest import install_real_binary
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
@ -8,15 +9,11 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
machine = Machine.current(refresh=True)
|
||||
binary = Binary.objects.create(
|
||||
machine=machine,
|
||||
name="api-basic-bin",
|
||||
binprovider="env",
|
||||
abspath="/usr/bin/env",
|
||||
version="1.0",
|
||||
status=Binary.StatusChoices.INSTALLED,
|
||||
)
|
||||
binary = install_real_binary("python3", machine=machine)
|
||||
|
||||
response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers)
|
||||
|
||||
assert response.status_code == 200, response.content
|
||||
assert response.json()["id"] == str(binary.id)
|
||||
assert response.json()["abspath"] == binary.abspath
|
||||
assert response.json()["version"] == binary.version
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.tests.conftest import install_real_binary
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
@ -8,15 +9,13 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
machine = Machine.current(refresh=True)
|
||||
Binary.objects.create(
|
||||
machine=machine,
|
||||
name="api-basic-bin",
|
||||
binprovider="env",
|
||||
abspath="/usr/bin/env",
|
||||
version="1.0",
|
||||
status=Binary.StatusChoices.INSTALLED,
|
||||
)
|
||||
binary = install_real_binary("python3", machine=machine)
|
||||
|
||||
response = client.get("/api/v1/machine/binary/by-name/api-basic-bin", **api_headers)
|
||||
response = client.get("/api/v1/machine/binary/by-name/python3", **api_headers)
|
||||
|
||||
assert response.status_code == 200, response.content
|
||||
payload = response.json()
|
||||
assert len(payload) == 1
|
||||
assert payload[0]["id"] == str(binary.id)
|
||||
assert payload[0]["abspath"] == binary.abspath
|
||||
assert payload[0]["version"] == binary.version
|
||||
|
||||
@ -1,11 +1,17 @@
|
||||
from pathlib import Path
|
||||
from importlib.resources import files
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
from abxpkg.binary_service import BinaryRequestEvent
|
||||
from abx_dl.events import ArchiveResultEvent, ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, SnapshotEvent
|
||||
from abx_dl.events import ProcessCompletedEvent, ProcessEvent, ProcessStartedEvent, SnapshotEvent
|
||||
from abx_dl.orchestrator import create_bus
|
||||
from abx_dl.output_files import OutputFile
|
||||
from archivebox.tests.conftest import resolve_abxpkg_binary_env
|
||||
from archivebox.tests.conftest import install_real_binary
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
@ -17,6 +23,107 @@ def _cleanup_machine_process_rows() -> None:
|
||||
Process.objects.all().delete()
|
||||
|
||||
|
||||
def _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
*,
|
||||
plugin: str,
|
||||
hook_name: str,
|
||||
event_hook_name: str | None = None,
|
||||
lib_dir: Path,
|
||||
env: dict | None = None,
|
||||
expected_exit_codes: tuple[int, ...] = (0,),
|
||||
):
|
||||
"""Run one shipped hook through the production process/result bus services."""
|
||||
import asyncio
|
||||
|
||||
from abx_dl.services.process_service import ProcessService as HookProcessService
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
from archivebox.services.process_service import ProcessService as PersistedProcessService
|
||||
|
||||
hook_path = Path(str(files(f"abx_plugins.plugins.{plugin}").joinpath(hook_name)))
|
||||
projected_hook_name = event_hook_name or hook_name
|
||||
hook_config = hook_path.parent / "config.json"
|
||||
binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=hook_config)
|
||||
output_dir = Path(snapshot.output_dir) / plugin
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
bus = create_bus(name=f"test_real_{plugin}_{snapshot.id}")
|
||||
HookProcessService(bus, emit_jsonl=False, interactive_tty=False)
|
||||
PersistedProcessService(bus)
|
||||
ArchiveResultService(bus)
|
||||
|
||||
async def run() -> None:
|
||||
try:
|
||||
snapshot_event = SnapshotEvent(
|
||||
url=snapshot.url,
|
||||
snapshot_id=str(snapshot.id),
|
||||
output_dir=str(snapshot.output_dir),
|
||||
)
|
||||
await bus.emit(snapshot_event).now()
|
||||
process_event = bus.emit(
|
||||
ProcessEvent(
|
||||
plugin_name=plugin,
|
||||
hook_name=projected_hook_name,
|
||||
hook_path=str(hook_path),
|
||||
hook_args=[f"--url={snapshot.url}"],
|
||||
env={
|
||||
**binary_env,
|
||||
"ABXPKG_LIB_DIR": str(lib_dir),
|
||||
"SNAP_DIR": str(snapshot.output_dir),
|
||||
"PATH": f"{Path(os.sys.executable).parent}{os.pathsep}{os.environ['PATH']}",
|
||||
**(env or {}),
|
||||
},
|
||||
output_dir=str(output_dir),
|
||||
timeout=60,
|
||||
is_background=".bg." in hook_name,
|
||||
url=snapshot.url,
|
||||
process_type="hook",
|
||||
worker_type="hook",
|
||||
event_parent_id=snapshot_event.event_id,
|
||||
),
|
||||
)
|
||||
await process_event.now()
|
||||
if ".bg." in hook_name:
|
||||
completed_event = await bus.find(
|
||||
ProcessCompletedEvent,
|
||||
child_of=process_event,
|
||||
past=True,
|
||||
future=90,
|
||||
)
|
||||
assert completed_event is not None
|
||||
await completed_event.wait(timeout=90)
|
||||
await completed_event.event_results_list()
|
||||
await bus.wait_until_idle()
|
||||
finally:
|
||||
await bus.destroy(clear=False)
|
||||
|
||||
asyncio.run(run())
|
||||
process = Process.objects.filter(pwd=str(output_dir)).order_by("-created_at").first()
|
||||
assert process is not None
|
||||
process.refresh_from_db()
|
||||
assert process.exit_code in expected_exit_codes, (process.stdout, process.stderr)
|
||||
result = ArchiveResult.objects.get(snapshot=snapshot, plugin=plugin, hook_name=projected_hook_name)
|
||||
return process, result
|
||||
|
||||
|
||||
def _run_real_title_crawl(url: str, lib_dir: Path):
|
||||
import asyncio
|
||||
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.services.runner import CrawlRunner
|
||||
|
||||
crawl = Crawl.objects.create(
|
||||
urls=url,
|
||||
config={"ABXPKG_LIB_DIR": str(lib_dir), "PLUGINS": "title"},
|
||||
created_by_id=get_or_create_system_user_pk(),
|
||||
)
|
||||
asyncio.run(CrawlRunner(crawl, selected_plugins=["title"], show_progress=False).run())
|
||||
return Snapshot.objects.get(crawl=crawl, url=url)
|
||||
|
||||
|
||||
def _create_snapshot():
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.crawls.models import Crawl
|
||||
@ -37,95 +144,62 @@ def _create_snapshot():
|
||||
return snapshot
|
||||
|
||||
|
||||
def test_process_completed_projects_inline_archiveresult():
|
||||
def test_process_completed_projects_inline_archiveresult(tmp_path, hermetic_lib_dir):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "wget"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / "index.html").write_text("<html>ok</html>")
|
||||
|
||||
bus = create_bus(name="test_inline_archiveresult")
|
||||
service = ArchiveResultService(bus)
|
||||
|
||||
event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg",
|
||||
status="succeeded",
|
||||
output_str="wget/index.html",
|
||||
output_files=[OutputFile(path="index.html", extension="html", mimetype="text/html", size=15)],
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
snapshot_dir = Path(snapshot.output_dir)
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
(snapshot_dir / "source.txt").write_text("real hook input", encoding="utf-8")
|
||||
process, result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
)
|
||||
|
||||
async def emit_event() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(event)
|
||||
|
||||
asyncio.run(emit_event())
|
||||
|
||||
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget.finite.bg")
|
||||
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result.output_str == "wget/index.html"
|
||||
assert "index.html" in result.output_files
|
||||
assert result.output_files["index.html"] == {"extension": "html", "mimetype": "text/html", "size": 15}
|
||||
assert result.output_size == 15
|
||||
assert result.process_id == process.id
|
||||
assert result.output_str.endswith(json.loads((snapshot_dir / "hashes" / "hashes.json").read_text())["root_hash"][:12])
|
||||
assert result.output_files == {
|
||||
"hashes.json": {
|
||||
"extension": "json",
|
||||
"mimetype": "application/json",
|
||||
"size": (snapshot_dir / "hashes" / "hashes.json").stat().st_size,
|
||||
},
|
||||
}
|
||||
assert result.output_size == (snapshot_dir / "hashes" / "hashes.json").stat().st_size
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
def test_archiveresult_event_retry_updates_existing_hook_row():
|
||||
def test_archiveresult_event_retry_updates_existing_hook_row(tmp_path, hermetic_lib_dir):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "wget"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / "index.html").write_text("<html>ok</html>")
|
||||
|
||||
service = ArchiveResultService(create_bus(name="test_archiveresult_retry_updates_existing_hook_row"))
|
||||
first_event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg",
|
||||
status="failed",
|
||||
output_str="timed out",
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
snapshot_dir = Path(snapshot.output_dir)
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
(snapshot_dir / "source.txt").write_text("first input", encoding="utf-8")
|
||||
_, first_result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
env={"HASHES_ENABLED": "False"},
|
||||
)
|
||||
retry_event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg",
|
||||
status="succeeded",
|
||||
output_str="wget/index.html",
|
||||
output_files=[OutputFile(path="index.html", extension="html", mimetype="text/html", size=15)],
|
||||
start_ts="2026-03-22T12:01:00+00:00",
|
||||
end_ts="2026-03-22T12:01:01+00:00",
|
||||
first_result_id = first_result.id
|
||||
assert first_result.status == ArchiveResult.StatusChoices.SKIPPED
|
||||
|
||||
(snapshot_dir / "source.txt").write_text("retry input", encoding="utf-8")
|
||||
_, retry_result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
env={"HASHES_ENABLED": "True"},
|
||||
)
|
||||
|
||||
async def emit_events() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(first_event)
|
||||
first_result_id = await ArchiveResult.objects.values_list("id", flat=True).aget(
|
||||
snapshot=snapshot,
|
||||
plugin="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg",
|
||||
)
|
||||
await service.on_ArchiveResultEvent__save_to_db(retry_event)
|
||||
retry_result = await ArchiveResult.objects.aget(
|
||||
snapshot=snapshot,
|
||||
plugin="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg",
|
||||
)
|
||||
assert retry_result.id == first_result_id
|
||||
assert retry_result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert retry_result.output_str == "wget/index.html"
|
||||
|
||||
asyncio.run(emit_events())
|
||||
|
||||
assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget.finite.bg").count() == 1
|
||||
assert retry_result.id == first_result_id
|
||||
assert retry_result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="hashes", hook_name="on_Snapshot__93_hashes.py").count() == 1
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
@ -150,72 +224,37 @@ def test_archiveresult_duplicate_hook_rows_are_rejected():
|
||||
)
|
||||
|
||||
|
||||
def test_process_completed_projects_synthetic_failed_archiveresult():
|
||||
def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_path, hermetic_lib_dir):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "chrome"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bus = create_bus(name="test_synthetic_archiveresult")
|
||||
service = ArchiveResultService(bus)
|
||||
|
||||
event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="chrome",
|
||||
hook_name="on_Snapshot__11_chrome_wait",
|
||||
status="failed",
|
||||
output_str="Hook timed out after 60 seconds",
|
||||
error="Hook timed out after 60 seconds",
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:01:00+00:00",
|
||||
process, result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="title",
|
||||
hook_name="on_Snapshot__54_title.js",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
expected_exit_codes=(1,),
|
||||
)
|
||||
|
||||
async def emit_event() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(event)
|
||||
|
||||
asyncio.run(emit_event())
|
||||
|
||||
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="chrome", hook_name="on_Snapshot__11_chrome_wait")
|
||||
assert result.status == ArchiveResult.StatusChoices.FAILED
|
||||
assert result.output_str == "Hook timed out after 60 seconds"
|
||||
assert "Hook timed out" in result.notes
|
||||
assert result.process_id == process.id
|
||||
assert "Chrome session" in result.output_str
|
||||
assert result.output_str in result.notes
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
def test_failed_title_archiveresult_does_not_overwrite_snapshot_title():
|
||||
def test_failed_title_archiveresult_does_not_overwrite_snapshot_title(tmp_path, hermetic_lib_dir):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "title"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bus = create_bus(name="test_failed_title_does_not_update_snapshot")
|
||||
service = ArchiveResultService(bus)
|
||||
|
||||
event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
_, result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="title",
|
||||
hook_name="on_Snapshot__54_title.js",
|
||||
status="failed",
|
||||
output_str="No Chrome session found (chrome plugin must run first)",
|
||||
error="No Chrome session found (chrome plugin must run first)",
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
expected_exit_codes=(1,),
|
||||
)
|
||||
|
||||
async def emit_event() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(event)
|
||||
|
||||
asyncio.run(emit_event())
|
||||
|
||||
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js")
|
||||
assert result.status == ArchiveResult.StatusChoices.FAILED
|
||||
assert result.output_str == "No Chrome session found (chrome plugin must run first)"
|
||||
assert "Chrome session" in result.output_str
|
||||
snapshot.refresh_from_db()
|
||||
assert snapshot.title in (None, "")
|
||||
assert snapshot.resolved_title == ""
|
||||
@ -240,36 +279,21 @@ def test_snapshot_resolved_title_ignores_failed_title_output_str():
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
def test_snapshot_title_ignores_noresults_title_output_str():
|
||||
def test_snapshot_title_ignores_noresults_hook_output_str(tmp_path, hermetic_lib_dir):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "title"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bus = create_bus(name="test_noresults_title_does_not_update_snapshot")
|
||||
service = ArchiveResultService(bus)
|
||||
|
||||
event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="title",
|
||||
hook_name="on_Snapshot__54_title.js",
|
||||
status="noresults",
|
||||
output_str="TimeoutError: Navigation timeout of 54172 ms exceeded",
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
staticfile_dir = Path(snapshot.output_dir) / "staticfile"
|
||||
staticfile_dir.mkdir(parents=True, exist_ok=True)
|
||||
(staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8")
|
||||
_, result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="parse_txt_urls",
|
||||
hook_name="on_Snapshot__71_parse_txt_urls.py",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
)
|
||||
|
||||
async def emit_event() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(event)
|
||||
|
||||
asyncio.run(emit_event())
|
||||
|
||||
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js")
|
||||
assert result.status == ArchiveResult.StatusChoices.NORESULTS
|
||||
assert result.output_str == "TimeoutError: Navigation timeout of 54172 ms exceeded"
|
||||
assert result.output_str == "0 URLs parsed"
|
||||
snapshot.refresh_from_db()
|
||||
assert snapshot.title in (None, "")
|
||||
assert snapshot.resolved_title == ""
|
||||
@ -299,84 +323,41 @@ def test_snapshot_save_normalizes_url_title_to_none():
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
def test_process_completed_projects_noresults_archiveresult():
|
||||
def test_process_completed_projects_noresults_archiveresult(tmp_path, hermetic_lib_dir):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "title"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bus = create_bus(name="test_noresults_archiveresult")
|
||||
service = ArchiveResultService(bus)
|
||||
|
||||
event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="title",
|
||||
hook_name="on_Snapshot__54_title.js",
|
||||
status="noresults",
|
||||
output_str="No title found",
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
staticfile_dir = Path(snapshot.output_dir) / "staticfile"
|
||||
staticfile_dir.mkdir(parents=True, exist_ok=True)
|
||||
(staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8")
|
||||
process, result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="parse_txt_urls",
|
||||
hook_name="on_Snapshot__71_parse_txt_urls.py",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
)
|
||||
|
||||
async def emit_event() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(event)
|
||||
|
||||
asyncio.run(emit_event())
|
||||
|
||||
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js")
|
||||
assert result.status == ArchiveResult.StatusChoices.NORESULTS
|
||||
assert result.output_str == "No title found"
|
||||
assert result.output_str == "0 URLs parsed"
|
||||
assert result.process_id == process.id
|
||||
|
||||
|
||||
def test_process_completed_without_archive_result_does_not_infer_success_from_output_files(snapshot):
|
||||
def test_skipped_shipped_hook_does_not_infer_success_from_snapshot_files(snapshot, hermetic_lib_dir):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
plugin_dir = Path(snapshot.output_dir) / "wget"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / "index.html").write_text("<html>downloaded but not reported</html>")
|
||||
|
||||
bus = create_bus(name="test_process_completed_without_archive_result_output_files")
|
||||
ArchiveResultService(bus)
|
||||
|
||||
snapshot_event = SnapshotEvent(
|
||||
url=snapshot.url,
|
||||
snapshot_id=str(snapshot.id),
|
||||
output_dir=str(snapshot.output_dir),
|
||||
snapshot_dir = Path(snapshot.output_dir)
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
(snapshot_dir / "source.txt").write_text("real input remains present", encoding="utf-8")
|
||||
_, result = _run_shipped_snapshot_hook(
|
||||
snapshot,
|
||||
plugin="hashes",
|
||||
hook_name="on_Snapshot__93_hashes.py",
|
||||
lib_dir=hermetic_lib_dir,
|
||||
env={"HASHES_ENABLED": "False"},
|
||||
)
|
||||
completed_event = ProcessCompletedEvent(
|
||||
plugin_name="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg",
|
||||
hook_path="/usr/bin/env",
|
||||
hook_args=[],
|
||||
env={},
|
||||
timeout=60,
|
||||
stdout="",
|
||||
stderr="",
|
||||
exit_code=0,
|
||||
status="succeeded",
|
||||
output_dir=str(plugin_dir),
|
||||
output_files=[OutputFile(path="index.html", extension="html", mimetype="text/html", size=36)],
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
event_parent_id=snapshot_event.event_id,
|
||||
)
|
||||
|
||||
async def emit_events() -> None:
|
||||
await bus.emit(snapshot_event).now()
|
||||
await bus.emit(completed_event).now()
|
||||
await bus.wait_until_idle()
|
||||
|
||||
asyncio.run(emit_events())
|
||||
|
||||
result = ArchiveResult.objects.get(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget.finite.bg")
|
||||
assert result.status == ArchiveResult.StatusChoices.NORESULTS
|
||||
assert result.output_str == ""
|
||||
assert result.output_files == {"index.html": {"extension": "html", "mimetype": "text/html", "size": 36}}
|
||||
assert result.status == ArchiveResult.StatusChoices.SKIPPED
|
||||
assert result.output_str == "HASHES_ENABLED=False"
|
||||
assert "hashes.json" not in result.output_files
|
||||
assert not (snapshot_dir / "hashes" / "hashes.json").exists()
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
@ -416,115 +397,24 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
def test_retry_failed_archiveresults_preserves_legacy_plugin_rows_without_hook_name():
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
def test_process_completed_projects_snapshot_title_from_output_str(recursive_test_site, hermetic_lib_dir):
|
||||
snapshot = _run_real_title_crawl(recursive_test_site["root_url"], hermetic_lib_dir)
|
||||
result = snapshot.archiveresult_set.get(plugin="title")
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
legacy_result = ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin="wget",
|
||||
hook_name="",
|
||||
status=ArchiveResult.StatusChoices.FAILED,
|
||||
output_str="legacy failure",
|
||||
output_files={"index.html": {"size": 123}},
|
||||
output_size=123,
|
||||
output_mimetypes="text/html",
|
||||
)
|
||||
hook_result = ArchiveResult.objects.create(
|
||||
snapshot=snapshot,
|
||||
plugin="wget",
|
||||
hook_name="on_Snapshot__06_wget.finite.bg",
|
||||
status=ArchiveResult.StatusChoices.FAILED,
|
||||
output_str="hook failure",
|
||||
output_files={"stderr.log": {}},
|
||||
output_size=10,
|
||||
output_mimetypes="text/plain",
|
||||
)
|
||||
|
||||
reset_count = snapshot.retry_failed_archiveresults()
|
||||
|
||||
snapshot.refresh_from_db()
|
||||
snapshot.crawl.refresh_from_db()
|
||||
legacy_result.refresh_from_db()
|
||||
hook_result.refresh_from_db()
|
||||
|
||||
assert reset_count == 2
|
||||
assert snapshot.status == Snapshot.StatusChoices.QUEUED
|
||||
assert snapshot.retry_at is not None
|
||||
assert snapshot.crawl.status == snapshot.crawl.StatusChoices.QUEUED
|
||||
assert snapshot.crawl.retry_at is not None
|
||||
assert legacy_result.status == ArchiveResult.StatusChoices.FAILED
|
||||
assert legacy_result.output_str == "legacy failure"
|
||||
assert legacy_result.output_files == {"index.html": {"size": 123}}
|
||||
assert legacy_result.output_size == 123
|
||||
assert hook_result.status == ArchiveResult.StatusChoices.QUEUED
|
||||
assert hook_result.output_str == ""
|
||||
assert hook_result.output_files == {}
|
||||
assert hook_result.output_size == 0
|
||||
assert result.status == result.StatusChoices.SUCCEEDED
|
||||
assert result.output_str == "Root"
|
||||
assert snapshot.title == "Root"
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
def test_process_completed_projects_snapshot_title_from_output_str():
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
def test_process_completed_projects_snapshot_title_from_title_file(recursive_test_site, hermetic_lib_dir):
|
||||
snapshot = _run_real_title_crawl(recursive_test_site["root_url"], hermetic_lib_dir)
|
||||
title_file = Path(snapshot.output_dir) / "title" / "title.txt"
|
||||
result = snapshot.archiveresult_set.get(plugin="title")
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "title"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bus = create_bus(name="test_snapshot_title_output_str")
|
||||
service = ArchiveResultService(bus)
|
||||
|
||||
event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="title",
|
||||
hook_name="on_Snapshot__54_title.js",
|
||||
status="succeeded",
|
||||
output_str="Example Domain",
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
)
|
||||
|
||||
async def emit_event() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(event)
|
||||
|
||||
asyncio.run(emit_event())
|
||||
|
||||
snapshot.refresh_from_db()
|
||||
assert snapshot.title == "Example Domain"
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
def test_process_completed_projects_snapshot_title_from_title_file():
|
||||
from archivebox.services.archive_result_service import ArchiveResultService
|
||||
import asyncio
|
||||
|
||||
snapshot = _create_snapshot()
|
||||
plugin_dir = Path(snapshot.output_dir) / "title"
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / "title.txt").write_text("Example Domain")
|
||||
|
||||
bus = create_bus(name="test_snapshot_title_file")
|
||||
service = ArchiveResultService(bus)
|
||||
|
||||
event = ArchiveResultEvent(
|
||||
snapshot_id=str(snapshot.id),
|
||||
plugin="title",
|
||||
hook_name="on_Snapshot__54_title.js",
|
||||
status="noresults",
|
||||
output_str="No title found",
|
||||
output_files=[OutputFile(path="title.txt", extension="txt", mimetype="text/plain", size=14)],
|
||||
start_ts="2026-03-22T12:00:00+00:00",
|
||||
end_ts="2026-03-22T12:00:01+00:00",
|
||||
)
|
||||
|
||||
async def emit_event() -> None:
|
||||
await service.on_ArchiveResultEvent__save_to_db(event)
|
||||
|
||||
asyncio.run(emit_event())
|
||||
|
||||
snapshot.refresh_from_db()
|
||||
assert snapshot.title == "Example Domain"
|
||||
assert title_file.read_text() == "Root"
|
||||
assert result.output_files["title.txt"]["size"] == title_file.stat().st_size
|
||||
assert snapshot.resolved_title == title_file.read_text()
|
||||
_cleanup_machine_process_rows()
|
||||
|
||||
|
||||
@ -589,8 +479,13 @@ def test_collect_output_metadata_detects_warc_gz_mimetype(tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(tmp_path):
|
||||
from archivebox.machine.models import Binary, NetworkInterface
|
||||
def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(
|
||||
tmp_path,
|
||||
hermetic_lib_dir,
|
||||
recursive_test_site,
|
||||
):
|
||||
from abx_plugins.plugins.base.utils import get_hydrated_required_binary
|
||||
from archivebox.machine.models import NetworkInterface
|
||||
from archivebox.machine.models import Process as MachineProcess
|
||||
from archivebox.services.process_service import ProcessService as ArchiveBoxProcessService
|
||||
from abx_dl.services.process_service import ProcessService as DlProcessService
|
||||
@ -598,19 +493,28 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(
|
||||
iface = NetworkInterface.current()
|
||||
machine = iface.machine
|
||||
|
||||
binary = Binary.objects.create(
|
||||
machine=machine,
|
||||
name="postlight-parser",
|
||||
abspath="/tmp/postlight-parser",
|
||||
version="2.2.3",
|
||||
binprovider="npm",
|
||||
binproviders="npm",
|
||||
status=Binary.StatusChoices.INSTALLED,
|
||||
lib_dir = hermetic_lib_dir
|
||||
mercury_config = Path(str(files("abx_plugins.plugins.mercury").joinpath("config.json")))
|
||||
required_binary = get_hydrated_required_binary(
|
||||
"postlight-parser",
|
||||
mercury_config,
|
||||
environ=os.environ,
|
||||
)
|
||||
binary = install_real_binary(
|
||||
"postlight-parser",
|
||||
machine=machine,
|
||||
binproviders=required_binary["binproviders"],
|
||||
overrides=required_binary["overrides"],
|
||||
)
|
||||
mercury_env = resolve_abxpkg_binary_env(
|
||||
lib_dir,
|
||||
deps_from=mercury_config,
|
||||
install=False,
|
||||
)
|
||||
mercury_path = Path(mercury_env["MERCURY_BINARY"])
|
||||
assert Path(binary.abspath).resolve() == mercury_path.resolve()
|
||||
|
||||
hook_path = tmp_path / "on_Snapshot__57_mercury.py"
|
||||
hook_path.write_text("#!/bin/bash\nexit 0\n", encoding="utf-8")
|
||||
hook_path.chmod(0o755)
|
||||
hook_path = Path(str(files("abx_plugins.plugins.mercury").joinpath("on_Snapshot__57_mercury.py")))
|
||||
output_dir = tmp_path / "mercury"
|
||||
output_dir.mkdir()
|
||||
|
||||
@ -624,15 +528,17 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(
|
||||
plugin_name="mercury",
|
||||
hook_name="on_Snapshot__57_mercury.py",
|
||||
hook_path=str(hook_path),
|
||||
hook_args=["--url=https://example.com"],
|
||||
hook_args=[f"--url={recursive_test_site['root_url']}"],
|
||||
is_background=False,
|
||||
output_dir=str(output_dir),
|
||||
env={
|
||||
**mercury_env,
|
||||
"ABXPKG_LIB_DIR": str(lib_dir),
|
||||
"MERCURY_BINARY": binary.abspath,
|
||||
"NODE_BINARY": "/tmp/node",
|
||||
"SNAP_DIR": str(tmp_path),
|
||||
},
|
||||
timeout=60,
|
||||
url="https://example.com",
|
||||
url=recursive_test_site["root_url"],
|
||||
),
|
||||
).now()
|
||||
started = await bus.find(
|
||||
@ -652,15 +558,24 @@ def test_process_started_hydrates_binary_and_iface_from_existing_binary_records(
|
||||
|
||||
process = MachineProcess.objects.get(
|
||||
pwd=str(output_dir),
|
||||
cmd=[str(hook_path), "--url=https://example.com"],
|
||||
cmd=[str(hook_path), f"--url={recursive_test_site['root_url']}"],
|
||||
)
|
||||
assert process.binary_id == binary.id
|
||||
assert process.iface_id == iface.id
|
||||
assert process.exit_code == 0, process.stderr
|
||||
assert (output_dir / "content.html").read_text() == (
|
||||
'<body> <a href="/about">About</a> <a href="/blog">Blog</a> <a href="/contact">Contact</a> </body>'
|
||||
)
|
||||
assert (output_dir / "content.txt").read_text() == "About Blog Contact"
|
||||
article = json.loads((output_dir / "article.json").read_text())
|
||||
assert article["title"] == "Root"
|
||||
assert article["url"] == recursive_test_site["root_url"]
|
||||
assert article["word_count"] == 3
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp_path):
|
||||
from archivebox.machine.models import Binary, NetworkInterface
|
||||
def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp_path, hermetic_lib_dir):
|
||||
from archivebox.machine.models import NetworkInterface
|
||||
from archivebox.machine.models import Process as MachineProcess
|
||||
from archivebox.services.process_service import ProcessService as ArchiveBoxProcessService
|
||||
from abx_dl.services.process_service import ProcessService as DlProcessService
|
||||
@ -668,21 +583,17 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp
|
||||
iface = NetworkInterface.current()
|
||||
machine = iface.machine
|
||||
|
||||
node = Binary.objects.create(
|
||||
machine=machine,
|
||||
name="node",
|
||||
abspath="/tmp/node",
|
||||
version="22.0.0",
|
||||
binprovider="env",
|
||||
binproviders="env",
|
||||
status=Binary.StatusChoices.INSTALLED,
|
||||
)
|
||||
lib_dir = hermetic_lib_dir
|
||||
chrome_config = Path(str(files("abx_plugins.plugins.chrome").joinpath("config.json")))
|
||||
node_env = resolve_abxpkg_binary_env(lib_dir, deps_from=chrome_config)
|
||||
node_path = Path(node_env["NODE_BINARY"])
|
||||
node = install_real_binary("node", machine=machine)
|
||||
assert Path(node.abspath).resolve() == node_path.resolve()
|
||||
|
||||
hook_path = tmp_path / "on_Snapshot__75_parse_dom_outlinks.js"
|
||||
hook_path.write_text("#!/bin/bash\nexit 0\n", encoding="utf-8")
|
||||
hook_path.chmod(0o755)
|
||||
output_dir = tmp_path / "parse-dom-outlinks"
|
||||
output_dir.mkdir()
|
||||
hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
|
||||
crawl_dir = tmp_path / "crawl"
|
||||
output_dir = crawl_dir / "chrome"
|
||||
output_dir.mkdir(parents=True)
|
||||
|
||||
bus = create_bus(name="test_process_started_node_fallback")
|
||||
DlProcessService(bus, emit_jsonl=False, interactive_tty=False)
|
||||
@ -691,13 +602,20 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp
|
||||
async def run_test() -> None:
|
||||
await bus.emit(
|
||||
ProcessEvent(
|
||||
plugin_name="parse_dom_outlinks",
|
||||
hook_name="on_Snapshot__75_parse_dom_outlinks.js",
|
||||
plugin_name="chrome",
|
||||
hook_name="on_CrawlSetup__89_chrome_kill_zombies.js",
|
||||
hook_path=str(hook_path),
|
||||
hook_args=["--url=https://example.com"],
|
||||
hook_args=[],
|
||||
is_background=False,
|
||||
output_dir=str(output_dir),
|
||||
env={"NODE_BINARY": node.abspath},
|
||||
env={
|
||||
**node_env,
|
||||
"ABXPKG_LIB_DIR": str(lib_dir),
|
||||
"NODE_BINARY": node.abspath,
|
||||
"CRAWL_DIR": str(crawl_dir),
|
||||
"SNAP_DIR": str(crawl_dir / "snapshot"),
|
||||
"CHROME_USER_DATA_DIR": str(output_dir / "profile"),
|
||||
},
|
||||
timeout=60,
|
||||
url="https://example.com",
|
||||
),
|
||||
@ -706,7 +624,7 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp
|
||||
ProcessStartedEvent,
|
||||
past=True,
|
||||
future=False,
|
||||
hook_name="on_Snapshot__75_parse_dom_outlinks.js",
|
||||
hook_name="on_CrawlSetup__89_chrome_kill_zombies.js",
|
||||
output_dir=str(output_dir),
|
||||
)
|
||||
assert started is not None
|
||||
@ -719,32 +637,25 @@ def test_process_started_uses_node_binary_for_js_hooks_without_plugin_binary(tmp
|
||||
|
||||
process = MachineProcess.objects.get(
|
||||
pwd=str(output_dir),
|
||||
cmd=[str(hook_path), "--url=https://example.com"],
|
||||
cmd=[str(hook_path)],
|
||||
)
|
||||
assert process.binary_id == node.id
|
||||
assert process.iface_id == iface.id
|
||||
assert process.exit_code == 0, process.stderr
|
||||
assert "chrome zombies. cpu usage:" in process.stdout
|
||||
|
||||
|
||||
def test_binary_event_reuses_existing_installed_binary_row():
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend
|
||||
from abxpkg import PROVIDER_CLASS_BY_NAME
|
||||
from abxpkg.binary_service import BinaryCacheService, BinaryService
|
||||
import asyncio
|
||||
|
||||
machine = Machine.current()
|
||||
wget_path = PROVIDER_CLASS_BY_NAME["env"]().get_abspath("wget", quiet=True, no_cache=True)
|
||||
assert wget_path
|
||||
|
||||
binary = Binary.objects.create(
|
||||
machine=machine,
|
||||
name="wget",
|
||||
abspath=str(wget_path),
|
||||
version="9.9.9",
|
||||
binprovider="env",
|
||||
binproviders="env,apt,brew",
|
||||
status=Binary.StatusChoices.INSTALLED,
|
||||
)
|
||||
binary = install_real_binary("wget", machine=machine, binproviders="env,apt,brew")
|
||||
installed_abspath = binary.abspath
|
||||
installed_version = binary.version
|
||||
installed_provider = binary.binprovider
|
||||
|
||||
bus = create_bus(name="test_binary_event_reuses_existing_installed_binary_row")
|
||||
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
|
||||
@ -754,7 +665,7 @@ def test_binary_event_reuses_existing_installed_binary_row():
|
||||
binproviders=binary.binproviders,
|
||||
extra_context={
|
||||
"plugin_name": "wget",
|
||||
"output_dir": "/tmp/wget",
|
||||
"output_dir": str(binary.output_dir),
|
||||
},
|
||||
)
|
||||
|
||||
@ -767,7 +678,7 @@ def test_binary_event_reuses_existing_installed_binary_row():
|
||||
binary.refresh_from_db()
|
||||
assert Binary.objects.filter(machine=machine, name="wget").count() == 1
|
||||
assert binary.status == Binary.StatusChoices.INSTALLED
|
||||
assert binary.abspath == str(wget_path)
|
||||
assert binary.version == "9.9.9"
|
||||
assert binary.binprovider == "env"
|
||||
assert binary.abspath == installed_abspath
|
||||
assert binary.version == installed_version
|
||||
assert binary.binprovider == installed_provider
|
||||
assert binary.binproviders == "env,apt,brew"
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
import asyncio
|
||||
@ -8,7 +7,7 @@ import pytest
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.machine.models import Binary, Machine, Process
|
||||
from archivebox.tests.conftest import parse_jsonl_output, run_archivebox_cmd
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
@ -22,7 +21,25 @@ def _runtime_env(data_dir: Path, *, lib_dir: Path | None = None, **extra: str) -
|
||||
}
|
||||
|
||||
|
||||
def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event(monkeypatch):
|
||||
def _run_real_binary_state_machine(data_dir: Path, *, name: str, binproviders: str, env: dict[str, str]):
|
||||
"""Run a real Binary model through its abxpkg-backed state machine."""
|
||||
script = (
|
||||
"from archivebox.machine.models import Binary, Machine; "
|
||||
f"binary = Binary.objects.create(machine=Machine.current(), name={name!r}, binproviders={binproviders!r}, status=Binary.StatusChoices.QUEUED); "
|
||||
"assert binary.tick_claimed(lock_seconds=600); "
|
||||
"print('BINARY_STATE_MACHINE_E2E_DONE')"
|
||||
)
|
||||
return run_archivebox_cmd(
|
||||
["shell", "-c", script],
|
||||
cwd=data_dir,
|
||||
timeout=120,
|
||||
env=env,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
|
||||
def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event():
|
||||
from abxpkg.binary_service import BinaryCacheService, BinaryEvent, BinaryRequestEvent, BinaryService
|
||||
from abx_dl.orchestrator import create_bus
|
||||
from archivebox.services.binary_service import ArchiveBoxDBBinaryCacheBackend
|
||||
@ -34,7 +51,6 @@ def test_binary_request_preserves_raw_overrides_in_db_while_using_native_event(m
|
||||
"module_name": "imagesize",
|
||||
},
|
||||
}
|
||||
monkeypatch.setenv("PYTHON3_BINARY", sys.executable)
|
||||
binary = Binary.objects.create(
|
||||
machine=machine,
|
||||
name="python3",
|
||||
@ -85,26 +101,11 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
|
||||
name = "python"
|
||||
provider_bin_dir = initialized_archive / "lib" / "env" / "bin"
|
||||
runtime_env = _runtime_env(initialized_archive)
|
||||
request_record = {
|
||||
"type": "BinaryRequest",
|
||||
"name": name,
|
||||
"binproviders": "env",
|
||||
}
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
cwd=initialized_archive,
|
||||
stdin=json.dumps(request_record) + "\n",
|
||||
timeout=120,
|
||||
env=runtime_env,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env)
|
||||
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert returncode == 0, stderr
|
||||
output_records = parse_jsonl_output(stdout)
|
||||
assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in output_records)
|
||||
assert "BINARY_STATE_MACHINE_E2E_DONE" in stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
binary = Binary.objects.get(name=name)
|
||||
@ -173,10 +174,15 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
|
||||
lib_dir=changed_lib_dir,
|
||||
)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
Binary.objects.get(pk=first_binary_id).update_and_requeue(
|
||||
status=Binary.StatusChoices.QUEUED,
|
||||
retry_at=None,
|
||||
)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
["run", f"--binary-id={first_binary_id}"],
|
||||
cwd=initialized_archive,
|
||||
stdin=json.dumps(request_record) + "\n",
|
||||
timeout=120,
|
||||
env=changed_runtime_env,
|
||||
default_cli_env=True,
|
||||
@ -200,19 +206,11 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
|
||||
provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
|
||||
runtime_env = _runtime_env(initialized_archive)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
cwd=initialized_archive,
|
||||
stdin=json.dumps({"type": "BinaryRequest", "name": name, "binproviders": "env"}) + "\n",
|
||||
timeout=120,
|
||||
env=runtime_env,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env)
|
||||
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert returncode == 0, stderr
|
||||
assert any(record["type"] == "BinaryRequest" and record["name"] == name for record in parse_jsonl_output(stdout)), stdout + stderr
|
||||
assert returncode != 0, stdout + stderr
|
||||
assert "Binary http installation failed" in stderr
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
queued = Binary.objects.get(name=name)
|
||||
|
||||
@ -6,7 +6,6 @@ Verify add creates snapshots in DB, crawls, source files, and archive directorie
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@ -18,11 +17,8 @@ from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
cli_env,
|
||||
find_snapshot_dir,
|
||||
get_free_port,
|
||||
run_archivebox_cmd,
|
||||
run_queued_crawls,
|
||||
start_archivebox_server,
|
||||
stop_server,
|
||||
resolve_abxpkg_chrome_env,
|
||||
)
|
||||
|
||||
@ -158,43 +154,18 @@ IMPORT_FORMAT_ENV = {
|
||||
}
|
||||
|
||||
|
||||
def wait_for_import_processing(cwd: Path, expected_urls: set[str], *, timeout: float = 120.0) -> None:
|
||||
import time
|
||||
|
||||
deadline = time.time() + timeout
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
while time.time() < deadline:
|
||||
with use_archivebox_db(cwd):
|
||||
rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", flat=True))
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
for url in rows:
|
||||
counts[url] += 1
|
||||
if all(count >= 1 for count in counts.values()):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise AssertionError(f"timed out waiting for import crawl processing to start, got counts={counts}")
|
||||
|
||||
|
||||
def wait_for_expected_import_snapshots(cwd: Path, expected_urls: set[str], *, timeout: float = 180.0) -> None:
|
||||
import time
|
||||
|
||||
def assert_expected_import_snapshots(cwd: Path, expected_urls: set[str]) -> None:
|
||||
allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
with use_archivebox_db(cwd):
|
||||
rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status"))
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
bad_statuses = []
|
||||
for url, status in rows:
|
||||
counts[url] += 1
|
||||
if status not in allowed_statuses:
|
||||
bad_statuses.append((url, status))
|
||||
if all(count == 1 for count in counts.values()) and not bad_statuses:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise AssertionError(
|
||||
f"timed out waiting for one queued/started/sealed snapshot per URL, got counts={counts}, bad_statuses={bad_statuses}",
|
||||
)
|
||||
with use_archivebox_db(cwd):
|
||||
rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status"))
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
bad_statuses = []
|
||||
for url, status in rows:
|
||||
counts[url] += 1
|
||||
if status not in allowed_statuses:
|
||||
bad_statuses.append((url, status))
|
||||
assert all(count == 1 for count in counts.values()), counts
|
||||
assert not bad_statuses, bad_statuses
|
||||
|
||||
|
||||
def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]:
|
||||
@ -263,8 +234,7 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial
|
||||
"""`archivebox add < import-file` should normalize rich import formats before crawling URLs."""
|
||||
import_files = write_import_format_files(initialized_archive)
|
||||
expected_urls = {case["url"] for case in IMPORT_FORMAT_EXPECTATIONS.values()}
|
||||
port = get_free_port()
|
||||
env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
|
||||
env = cli_env(**IMPORT_FORMAT_ENV)
|
||||
|
||||
for import_path in import_files.values():
|
||||
source_text = import_path.read_text(encoding="utf-8")
|
||||
@ -282,40 +252,26 @@ def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initial
|
||||
assert crawl.snapshot_set.count() == 0
|
||||
assert crawl.urls == source_text
|
||||
|
||||
try:
|
||||
start_archivebox_server(initialized_archive, env=env, port=port)
|
||||
deadline = time.time() + 120
|
||||
root_counts = {}
|
||||
while time.time() < deadline:
|
||||
with use_archivebox_db(initialized_archive):
|
||||
root_counts = {
|
||||
str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()
|
||||
}
|
||||
if root_counts and all(count == 1 for count in root_counts.values()):
|
||||
break
|
||||
time.sleep(1)
|
||||
assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for crawl in Crawl.objects.all():
|
||||
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
|
||||
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
|
||||
assert root_input == crawl.urls
|
||||
wait_for_import_processing(initialized_archive, expected_urls)
|
||||
stop_server(initialized_archive)
|
||||
start_archivebox_server(initialized_archive, env=env, port=port)
|
||||
wait_for_expected_import_snapshots(initialized_archive, expected_urls)
|
||||
run_queued_crawls(initialized_archive, env=env, timeout=240)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
root_counts = {str(crawl.id): crawl.snapshot_set.filter(url=Snapshot.INTERNAL_INPUT_URL).count() for crawl in Crawl.objects.all()}
|
||||
assert root_counts and all(count == 1 for count in root_counts.values()), root_counts
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for crawl in Crawl.objects.all():
|
||||
root_snapshot = crawl.snapshot_set.get(url=Snapshot.INTERNAL_INPUT_URL)
|
||||
root_input = (root_snapshot.output_dir / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
|
||||
assert root_input == crawl.urls
|
||||
assert_expected_import_snapshots(initialized_archive, expected_urls)
|
||||
|
||||
list_result = run_archivebox_cmd(
|
||||
["list", "--json"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert list_result.returncode == 0, list_result.stderr or list_result.stdout
|
||||
for expected_url in expected_urls:
|
||||
assert expected_url in list_result.stdout
|
||||
finally:
|
||||
stop_server(initialized_archive)
|
||||
list_result = run_archivebox_cmd(
|
||||
["list", "--json"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert list_result.returncode == 0, list_result.stderr or list_result.stdout
|
||||
for expected_url in expected_urls:
|
||||
assert expected_url in list_result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawls = list(Crawl.objects.order_by("created_at"))
|
||||
@ -345,8 +301,7 @@ def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive)
|
||||
"""CLI add must not turn user-supplied local paths or shell payloads into snapshots."""
|
||||
safe_url = "https://example.com/?archivebox-cli-security=1"
|
||||
inputs, canary = malicious_add_inputs(initialized_archive, safe_url=safe_url)
|
||||
port = get_free_port()
|
||||
env = cli_env(port=port, server=True, **IMPORT_FORMAT_ENV)
|
||||
env = cli_env(**IMPORT_FORMAT_ENV)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--bg", "--depth=0", "--tag=cli-security"],
|
||||
@ -357,11 +312,8 @@ def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive)
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
try:
|
||||
start_archivebox_server(initialized_archive, env=env, port=port)
|
||||
wait_for_expected_import_snapshots(initialized_archive, {safe_url}, timeout=120)
|
||||
finally:
|
||||
stop_server(initialized_archive)
|
||||
run_queued_crawls(initialized_archive, env=env, timeout=120)
|
||||
assert_expected_import_snapshots(initialized_archive, {safe_url})
|
||||
|
||||
assert_no_file_or_shell_payload_snapshots(initialized_archive, canary=canary)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
|
||||
@ -126,21 +126,31 @@ class TestArchiveResultCreate:
|
||||
assert "Snapshot" in types
|
||||
assert "ArchiveResult" in types
|
||||
|
||||
def test_create_pass_through_only_when_no_snapshots(self, initialized_archive):
|
||||
"""Only pass-through records but no new snapshots returns success."""
|
||||
crawl_record = {"type": "Crawl", "id": "fake-id", "urls": "https://example.com"}
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["archiveresult", "create"],
|
||||
stdin=json.dumps(crawl_record),
|
||||
def test_create_passes_through_cli_crawl_when_no_snapshots(self, initialized_archive):
|
||||
"""A real Crawl with no Snapshot input passes through successfully."""
|
||||
crawl_result = run_archivebox_cmd(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
_stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert crawl_result.returncode == 0, crawl_result.stderr
|
||||
crawl_record = parse_jsonl_output(crawl_result.stdout)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["archiveresult", "create"],
|
||||
stdin=crawl_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert code == 0
|
||||
assert "Passed through" in stderr
|
||||
records = parse_jsonl_output(stdout)
|
||||
assert len(records) == 1
|
||||
assert records[0]["id"] == crawl_record["id"]
|
||||
|
||||
|
||||
class TestArchiveResultList:
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox binary command.
|
||||
|
||||
TODO: expand beyond command discovery into create/list/update/delete behavior.
|
||||
"""
|
||||
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
@ -158,6 +158,28 @@ def test_config_search_finds_keys(initialized_archive):
|
||||
assert "TIMEOUT" in result.stdout
|
||||
|
||||
|
||||
def test_config_search_finds_plugin_options(initialized_archive):
|
||||
"""Test that config --search finds plugin keys and descriptions."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--search", "wget"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "WGET_BINARY" in result.stdout
|
||||
|
||||
|
||||
def test_config_search_finds_core_aliases(initialized_archive):
|
||||
"""Test that config --search finds core options by partial alias."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--search", "URL_BLACK"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "URL_DENYLIST" in result.stdout
|
||||
|
||||
|
||||
def test_config_preserves_existing_values(initialized_archive):
|
||||
"""Test that setting new values preserves existing ones."""
|
||||
|
||||
|
||||
@ -106,11 +106,18 @@ class TestCrawlCreate:
|
||||
records = parse_jsonl_output(stdout)
|
||||
assert "test-tag" in records[0].get("tags_str", "")
|
||||
|
||||
def test_create_pass_through_other_types(self, initialized_archive):
|
||||
"""Pass-through records of other types unchanged."""
|
||||
tag_record = {"type": "Tag", "id": "fake-tag-id", "name": "test"}
|
||||
def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
|
||||
"""A real Tag emitted by the CLI remains available to the next stage."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
["tag", "create", "crawl-input-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
tag_record = parse_jsonl_output(tag_result.stdout)[0]
|
||||
url = create_test_url()
|
||||
stdin = json.dumps(tag_record) + "\n" + json.dumps({"url": url})
|
||||
stdin = tag_result.stdout + json.dumps({"url": url}) + "\n"
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["crawl", "create"],
|
||||
@ -124,10 +131,8 @@ class TestCrawlCreate:
|
||||
assert code == 0
|
||||
records = parse_jsonl_output(stdout)
|
||||
|
||||
# Should have both the passed-through Tag and the new Crawl
|
||||
types = [r.get("type") for r in records]
|
||||
assert "Tag" in types
|
||||
assert "Crawl" in types
|
||||
assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
|
||||
assert any(record.get("type") == "Crawl" and url in record["urls"] for record in records)
|
||||
|
||||
def test_create_pass_through_existing_crawl(self, initialized_archive):
|
||||
"""Existing Crawl records (with id) are passed through."""
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
"""Tests for archivebox extract input handling and pipelines."""
|
||||
|
||||
import subprocess
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
@ -219,15 +218,19 @@ def test_extract_stdin_jsonl_input(initialized_archive):
|
||||
env = cli_env(PLUGINS="wget,title")
|
||||
create_extract_snapshot(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot_id = Snapshot.objects.values_list("id", flat=True).first()
|
||||
|
||||
jsonl_input = json.dumps({"type": "Snapshot", "id": str(snapshot_id)}) + "\n"
|
||||
list_result = run_archivebox_cmd(
|
||||
["snapshot", "list", "--url__icontains=example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
check=True,
|
||||
)
|
||||
snapshot_record = next(record for record in parse_jsonl_output(list_result.stdout) if record.get("type") == "Snapshot")
|
||||
snapshot_id = snapshot_record["id"]
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["extract", "--plugins=wget,title"],
|
||||
cwd=initialized_archive,
|
||||
input=jsonl_input,
|
||||
input=list_result.stdout,
|
||||
env=env,
|
||||
timeout=90,
|
||||
)
|
||||
|
||||
@ -1,17 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox list command.
|
||||
Verify list emits snapshot JSONL and applies the documented filters.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db import connection
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.cli.archivebox_snapshot import iter_snapshot_json
|
||||
from archivebox.tests.conftest import create_test_url, parse_jsonl_output, run_archivebox_cmd, run_queued_crawls, cli_env
|
||||
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
@ -19,102 +19,128 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
class CountingStdout:
|
||||
encoding = "utf-8"
|
||||
|
||||
def __init__(self):
|
||||
self.rows = 0
|
||||
self._pending = ""
|
||||
|
||||
def isatty(self):
|
||||
return False
|
||||
|
||||
def write(self, text):
|
||||
self._pending += text
|
||||
lines = self._pending.split("\n")
|
||||
self._pending = lines.pop()
|
||||
self.rows += sum(1 for line in lines if line.startswith("{"))
|
||||
return len(text)
|
||||
|
||||
def flush(self):
|
||||
return None
|
||||
|
||||
|
||||
def test_list_limit_zero_streams_one_million_snapshots_without_materializing(admin_user, monkeypatch):
|
||||
"""Regression: archivebox list --limit=0 must stream unbounded result sets."""
|
||||
from archivebox.cli.archivebox_snapshot import list_snapshots
|
||||
def test_streaming_json_matches_snapshot_serializer(initialized_archive):
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com",
|
||||
created_by=admin_user,
|
||||
status=Crawl.StatusChoices.SEALED,
|
||||
retry_at=None,
|
||||
)
|
||||
now = timezone.now().isoformat()
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
WITH RECURSIVE seq(n) AS (
|
||||
SELECT 1
|
||||
UNION ALL
|
||||
SELECT n + 1 FROM seq WHERE n < 1000000
|
||||
with use_archivebox_db(initialized_archive):
|
||||
user = get_user_model().objects.create_user(username="streaming-json-parity")
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com/a\nhttps://example.com/b",
|
||||
created_by=user,
|
||||
status=Crawl.StatusChoices.SEALED,
|
||||
retry_at=None,
|
||||
)
|
||||
populated = Snapshot.objects.create(
|
||||
crawl=crawl,
|
||||
url="https://example.com/a",
|
||||
timestamp="20260721220000000000000000000001",
|
||||
title="Populated title",
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
retry_at=None,
|
||||
output_size=42,
|
||||
)
|
||||
populated.save_tags(["éclair", "Zulu", "alpha"])
|
||||
empty = Snapshot.objects.create(
|
||||
crawl=crawl,
|
||||
url="https://example.com/b",
|
||||
timestamp="20260721220000000000000000000002",
|
||||
title=None,
|
||||
status=Snapshot.StatusChoices.QUEUED,
|
||||
output_size=0,
|
||||
)
|
||||
queryset = Snapshot.objects.filter(id__in=(populated.id, empty.id)).order_by("url")
|
||||
|
||||
expected = [snapshot.to_json() for snapshot in queryset.prefetch_related("tags")]
|
||||
actual = list(iter_snapshot_json(queryset))
|
||||
|
||||
assert actual == expected
|
||||
assert [record["url"] for record in actual] == ["https://example.com/a", "https://example.com/b"]
|
||||
|
||||
|
||||
def test_list_limit_zero_streams_one_million_snapshots_without_materializing(initialized_archive, tmp_path):
|
||||
"""Regression: archivebox list --limit=0 must stream unbounded result sets."""
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
user = get_user_model().objects.create_user(username="million-snapshot-list")
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com",
|
||||
created_by=user,
|
||||
status=Crawl.StatusChoices.SEALED,
|
||||
retry_at=None,
|
||||
)
|
||||
now = timezone.now().isoformat()
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
WITH RECURSIVE seq(n) AS (
|
||||
SELECT 1
|
||||
UNION ALL
|
||||
SELECT n + 1 FROM seq WHERE n < 1000000
|
||||
)
|
||||
INSERT INTO core_snapshot (
|
||||
id,
|
||||
url,
|
||||
timestamp,
|
||||
title,
|
||||
bookmarked_at,
|
||||
created_at,
|
||||
modified_at,
|
||||
downloaded_at,
|
||||
fs_version,
|
||||
crawl_id,
|
||||
config,
|
||||
current_step,
|
||||
depth,
|
||||
notes,
|
||||
num_uses_failed,
|
||||
num_uses_succeeded,
|
||||
retry_at,
|
||||
status,
|
||||
delete_at,
|
||||
output_size,
|
||||
parent_snapshot_id
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(16))),
|
||||
'https://example.com/page-' || n,
|
||||
printf('9%031d', n),
|
||||
'',
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
NULL,
|
||||
'0.9.0',
|
||||
%s,
|
||||
'{}',
|
||||
0,
|
||||
0,
|
||||
'',
|
||||
0,
|
||||
0,
|
||||
NULL,
|
||||
'sealed',
|
||||
NULL,
|
||||
0,
|
||||
NULL
|
||||
FROM seq
|
||||
""",
|
||||
[now, now, now, str(crawl.id).replace("-", "")],
|
||||
)
|
||||
INSERT INTO core_snapshot (
|
||||
id,
|
||||
url,
|
||||
timestamp,
|
||||
title,
|
||||
bookmarked_at,
|
||||
created_at,
|
||||
modified_at,
|
||||
downloaded_at,
|
||||
fs_version,
|
||||
crawl_id,
|
||||
config,
|
||||
current_step,
|
||||
depth,
|
||||
notes,
|
||||
num_uses_failed,
|
||||
num_uses_succeeded,
|
||||
retry_at,
|
||||
status,
|
||||
delete_at,
|
||||
output_size,
|
||||
parent_snapshot_id
|
||||
)
|
||||
SELECT
|
||||
lower(hex(randomblob(16))),
|
||||
'https://example.com/page-' || n,
|
||||
printf('9%031d', n),
|
||||
'',
|
||||
%s,
|
||||
%s,
|
||||
%s,
|
||||
NULL,
|
||||
'0.9.0',
|
||||
%s,
|
||||
'{}',
|
||||
0,
|
||||
0,
|
||||
'',
|
||||
0,
|
||||
0,
|
||||
NULL,
|
||||
'sealed',
|
||||
NULL,
|
||||
0,
|
||||
NULL
|
||||
FROM seq
|
||||
""",
|
||||
[now, now, now, str(crawl.id).replace("-", "")],
|
||||
|
||||
output_path = tmp_path / "million-snapshots.jsonl"
|
||||
with output_path.open("w") as stdout:
|
||||
result = run_archivebox_cmd(
|
||||
["list", "--limit=0"],
|
||||
cwd=initialized_archive,
|
||||
stdout=stdout,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
stdout = CountingStdout()
|
||||
monkeypatch.setattr(sys, "stdout", stdout)
|
||||
|
||||
assert list_snapshots(limit=0) == 0
|
||||
assert stdout.rows == 1000000
|
||||
assert result.returncode == 0, result.stderr
|
||||
with output_path.open() as stdout:
|
||||
assert sum(1 for line in stdout if line.startswith("{")) == 1000000
|
||||
|
||||
|
||||
def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive):
|
||||
@ -127,6 +153,8 @@ def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive):
|
||||
check=True,
|
||||
)
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
Snapshot.objects.get(url="https://example.com").save_tags(["z-tag", "a-tag"])
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["list"],
|
||||
@ -135,10 +163,12 @@ def test_list_outputs_existing_snapshots_as_jsonl(initialized_archive):
|
||||
|
||||
rows = parse_jsonl_output(result.stdout)
|
||||
urls = {row["url"] for row in rows}
|
||||
rows_by_url = {row["url"]: row for row in rows}
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "https://example.com" in urls
|
||||
assert "https://iana.org" in urls
|
||||
assert rows_by_url["https://example.com"]["tags"] == "a-tag,z-tag"
|
||||
|
||||
|
||||
def test_list_filters_by_url_icontains(initialized_archive):
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox machine command.
|
||||
|
||||
TODO: expand beyond command discovery into list/filter behavior.
|
||||
"""
|
||||
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox mcp command.
|
||||
|
||||
TODO: expand beyond command discovery into JSON-RPC stdio behavior.
|
||||
"""
|
||||
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
19
archivebox/tests/test_cli_oneshot.py
Normal file
19
archivebox/tests/test_cli_oneshot.py
Normal file
@ -0,0 +1,19 @@
|
||||
from importlib.metadata import version
|
||||
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
|
||||
def test_oneshot_runs_abx_dl_through_abxpkg_env_projection(tmp_path):
|
||||
lib_dir = tmp_path / "lib"
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["oneshot", "--version"],
|
||||
cwd=tmp_path,
|
||||
env={"ABXPKG_LIB_DIR": str(lib_dir)},
|
||||
)
|
||||
|
||||
abx_dl_projection = lib_dir / "env" / "bin" / "abx-dl"
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert version("abx-dl") in result.stdout
|
||||
assert abx_dl_projection.is_symlink()
|
||||
assert abx_dl_projection.resolve().is_file()
|
||||
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox persona command.
|
||||
|
||||
TODO: expand beyond command discovery into create/list/update/delete behavior.
|
||||
"""
|
||||
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
@ -6,9 +6,12 @@ This file covers both:
|
||||
- subprocess integration for the supported records `archivebox run` consumes
|
||||
"""
|
||||
|
||||
import os
|
||||
import pty
|
||||
import sys
|
||||
import uuid
|
||||
from io import StringIO
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@ -33,18 +36,33 @@ PIPE_TEST_ENV = {
|
||||
}
|
||||
|
||||
|
||||
class MockTTYStringIO(StringIO):
|
||||
def __init__(self, initial_value: str = "", *, is_tty: bool):
|
||||
super().__init__(initial_value)
|
||||
self._is_tty = is_tty
|
||||
def run_real_txt_parser(tmp_path, text):
|
||||
"""Run the shipped text parser and return its real snapshot output directory."""
|
||||
from archivebox.plugins.hooks import run_hook
|
||||
|
||||
def isatty(self) -> bool:
|
||||
return self._is_tty
|
||||
snap_dir = tmp_path / "parser-snapshot"
|
||||
staticfile_dir = snap_dir / "staticfile"
|
||||
output_dir = snap_dir / "parse_txt_urls"
|
||||
staticfile_dir.mkdir(parents=True)
|
||||
output_dir.mkdir(parents=True)
|
||||
(staticfile_dir / "input.txt").write_text(text, encoding="utf-8")
|
||||
hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
|
||||
process = run_hook(
|
||||
hook_path,
|
||||
output_dir,
|
||||
config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
|
||||
timeout=30,
|
||||
url="file:///input.txt",
|
||||
depth=0,
|
||||
)
|
||||
process.refresh_from_db()
|
||||
assert process.exit_code == 0, process.stderr
|
||||
return snap_dir
|
||||
|
||||
|
||||
def test_parse_line_accepts_supported_piping_inputs():
|
||||
"""The JSONL parser should normalize the input forms CLI pipes accept."""
|
||||
from archivebox.misc.jsonl import TYPE_CRAWL, TYPE_SNAPSHOT, parse_line
|
||||
from archivebox.misc.jsonl import TYPE_SNAPSHOT, parse_line
|
||||
|
||||
assert parse_line("") is None
|
||||
assert parse_line(" ") is None
|
||||
@ -57,27 +75,8 @@ def test_parse_line_accepts_supported_piping_inputs():
|
||||
|
||||
assert parse_line("file:///tmp/example.txt") is None
|
||||
|
||||
snapshot_json = parse_line('{"type":"Snapshot","url":"https://example.com","tags":"tag1,tag2"}')
|
||||
assert snapshot_json is not None
|
||||
assert snapshot_json["type"] == TYPE_SNAPSHOT
|
||||
assert snapshot_json["tags"] == "tag1,tag2"
|
||||
|
||||
crawl_json = parse_line('{"type":"Crawl","id":"abc123","urls":"https://example.com","max_depth":1}')
|
||||
assert crawl_json is not None
|
||||
assert crawl_json["type"] == TYPE_CRAWL
|
||||
assert crawl_json["id"] == "abc123"
|
||||
assert crawl_json["max_depth"] == 1
|
||||
|
||||
snapshot_id = "01234567-89ab-cdef-0123-456789abcdef"
|
||||
parsed_id = parse_line(snapshot_id)
|
||||
assert parsed_id == {"type": TYPE_SNAPSHOT, "id": snapshot_id}
|
||||
|
||||
compact_snapshot_id = "0123456789abcdef0123456789abcdef"
|
||||
compact_parsed_id = parse_line(compact_snapshot_id)
|
||||
assert compact_parsed_id == {"type": TYPE_SNAPSHOT, "id": compact_snapshot_id}
|
||||
|
||||
|
||||
def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path):
|
||||
def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path, initialized_archive):
|
||||
"""Piping helpers should consume args, structured JSONL, and pass-through records."""
|
||||
from archivebox.misc.jsonl import TYPE_CRAWL, read_args_or_stdin
|
||||
|
||||
@ -88,80 +87,67 @@ def test_read_args_or_stdin_handles_args_stdin_and_mixed_jsonl(tmp_path):
|
||||
local_file.write_text("https://from-file-arg.example\n")
|
||||
assert list(read_args_or_stdin((str(local_file),))) == []
|
||||
|
||||
stdin_records = list(
|
||||
read_args_or_stdin(
|
||||
(),
|
||||
stream=MockTTYStringIO(
|
||||
"https://plain-url.com\n"
|
||||
'{"type":"Snapshot","url":"https://jsonl-url.com","tags":"test"}\n'
|
||||
'{"type":"Tag","id":"tag-1","name":"example"}\n'
|
||||
"01234567-89ab-cdef-0123-456789abcdef\n"
|
||||
"not valid json\n",
|
||||
is_tty=False,
|
||||
),
|
||||
),
|
||||
snapshot_result = run_archivebox_cmd(
|
||||
["snapshot", "create", "--tag=test", "https://jsonl-url.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
check=True,
|
||||
)
|
||||
assert len(stdin_records) == 4
|
||||
assert stdin_records[0]["url"] == "https://plain-url.com"
|
||||
assert stdin_records[1]["url"] == "https://jsonl-url.com"
|
||||
assert stdin_records[1]["tags"] == "test"
|
||||
assert stdin_records[2]["type"] == "Tag"
|
||||
assert stdin_records[2]["name"] == "example"
|
||||
assert stdin_records[3]["id"] == "01234567-89ab-cdef-0123-456789abcdef"
|
||||
|
||||
crawl_records = list(
|
||||
read_args_or_stdin(
|
||||
(),
|
||||
stream=MockTTYStringIO(
|
||||
'{"type":"Crawl","id":"crawl-1","urls":"https://example.com\\nhttps://foo.com"}\n',
|
||||
is_tty=False,
|
||||
),
|
||||
),
|
||||
crawl_result = run_archivebox_cmd(
|
||||
["crawl", "create", "https://crawl-url.com"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
check=True,
|
||||
)
|
||||
assert len(crawl_records) == 1
|
||||
assert crawl_records[0]["type"] == TYPE_CRAWL
|
||||
assert crawl_records[0]["id"] == "crawl-1"
|
||||
snapshot_record = next(record for record in parse_jsonl_output(snapshot_result.stdout) if record.get("type") == "Snapshot")
|
||||
|
||||
tty_records = list(read_args_or_stdin((), stream=MockTTYStringIO("https://example.com", is_tty=True)))
|
||||
assert tty_records == []
|
||||
read_fd, write_fd = os.pipe()
|
||||
os.write(
|
||||
write_fd,
|
||||
f"https://plain-url.com\n{snapshot_result.stdout}{crawl_result.stdout}{snapshot_record['id']}\nnot valid json\n".encode(),
|
||||
)
|
||||
os.close(write_fd)
|
||||
with os.fdopen(read_fd, encoding="utf-8") as pipe_stream:
|
||||
assert pipe_stream.isatty() is False
|
||||
stdin_records = list(read_args_or_stdin((), stream=pipe_stream))
|
||||
assert any(record.get("url") == "https://plain-url.com" for record in stdin_records)
|
||||
assert any(record.get("type") == "Snapshot" and record.get("id") == snapshot_record["id"] for record in stdin_records)
|
||||
assert any(record.get("type") == TYPE_CRAWL and record.get("urls") == "https://crawl-url.com" for record in stdin_records)
|
||||
assert any(
|
||||
record.get("type") == "Snapshot" and record.get("id") == snapshot_record["id"] and len(record) == 2 for record in stdin_records
|
||||
)
|
||||
|
||||
master_fd, slave_fd = pty.openpty()
|
||||
try:
|
||||
with os.fdopen(slave_fd, encoding="utf-8") as tty_stream:
|
||||
assert tty_stream.isatty() is True
|
||||
assert list(read_args_or_stdin((), stream=tty_stream)) == []
|
||||
finally:
|
||||
os.close(master_fd)
|
||||
|
||||
|
||||
def test_collect_urls_from_plugins_reads_only_parser_outputs(tmp_path):
|
||||
"""Parser extractor `urls.jsonl` outputs should be discoverable for recursive piping."""
|
||||
from archivebox.plugins.hooks import collect_urls_from_plugins
|
||||
|
||||
(tmp_path / "wget").mkdir()
|
||||
(tmp_path / "wget" / "urls.jsonl").write_text(
|
||||
'{"url":"https://wget-link-1.com"}\n{"url":"https://wget-link-2.com"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "parse_html_urls").mkdir()
|
||||
(tmp_path / "parse_html_urls" / "urls.jsonl").write_text(
|
||||
'{"url":"https://html-link-1.com"}\n{"url":"https://html-link-2.com","title":"HTML Link 2"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "screenshot").mkdir()
|
||||
snap_dir = run_real_txt_parser(tmp_path, "https://html-link-1.com https://html-link-2.com")
|
||||
(snap_dir / "screenshot").mkdir()
|
||||
|
||||
urls = collect_urls_from_plugins(tmp_path)
|
||||
assert len(urls) == 4
|
||||
assert {url["plugin"] for url in urls} == {"wget", "parse_html_urls"}
|
||||
titled = [url for url in urls if url.get("title") == "HTML Link 2"]
|
||||
assert len(titled) == 1
|
||||
assert titled[0]["url"] == "https://html-link-2.com"
|
||||
urls = collect_urls_from_plugins(snap_dir)
|
||||
assert {url["url"] for url in urls} == {"https://html-link-1.com", "https://html-link-2.com"}
|
||||
assert {url["plugin"] for url in urls} == {"parse_txt_urls"}
|
||||
|
||||
assert collect_urls_from_plugins(tmp_path / "nonexistent") == []
|
||||
assert collect_urls_from_plugins(snap_dir / "nonexistent") == []
|
||||
|
||||
|
||||
def test_collect_urls_from_plugins_trims_markdown_suffixes(tmp_path):
|
||||
from archivebox.plugins.hooks import collect_urls_from_plugins
|
||||
|
||||
(tmp_path / "parse_html_urls").mkdir()
|
||||
(tmp_path / "parse_html_urls" / "urls.jsonl").write_text(
|
||||
'{"url":"https://docs.sweeting.me/s/youtube-favorites)**"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
snap_dir = run_real_txt_parser(tmp_path, "[favorites](https://docs.sweeting.me/s/youtube-favorites)**")
|
||||
|
||||
urls = collect_urls_from_plugins(tmp_path)
|
||||
urls = collect_urls_from_plugins(snap_dir)
|
||||
assert len(urls) == 1
|
||||
assert urls[0]["url"] == "https://docs.sweeting.me/s/youtube-favorites"
|
||||
|
||||
@ -169,13 +155,9 @@ def test_collect_urls_from_plugins_trims_markdown_suffixes(tmp_path):
|
||||
def test_collect_urls_from_plugins_trims_trailing_punctuation(tmp_path):
|
||||
from archivebox.plugins.hooks import collect_urls_from_plugins
|
||||
|
||||
(tmp_path / "parse_html_urls").mkdir()
|
||||
(tmp_path / "parse_html_urls" / "urls.jsonl").write_text(
|
||||
('{"url":"https://github.com/ArchiveBox/ArchiveBox."}\n{"url":"https://github.com/abc?abc#234234?."}\n'),
|
||||
encoding="utf-8",
|
||||
)
|
||||
snap_dir = run_real_txt_parser(tmp_path, "https://github.com/ArchiveBox/ArchiveBox. https://github.com/abc?abc#234234?.")
|
||||
|
||||
urls = collect_urls_from_plugins(tmp_path)
|
||||
urls = collect_urls_from_plugins(snap_dir)
|
||||
assert [url["url"] for url in urls] == [
|
||||
"https://github.com/ArchiveBox/ArchiveBox",
|
||||
"https://github.com/abc?abc#234234",
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox pluginmap command.
|
||||
|
||||
TODO: expand beyond command discovery into quiet/event output behavior.
|
||||
"""
|
||||
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox process command.
|
||||
|
||||
TODO: expand beyond command discovery into list/filter behavior.
|
||||
"""
|
||||
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
|
||||
@ -7,14 +7,12 @@ Tests cover:
|
||||
- pass-through output (for chaining)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import psutil
|
||||
|
||||
from archivebox.tests.conftest import (
|
||||
cleanup_process_group,
|
||||
@ -22,9 +20,8 @@ from archivebox.tests.conftest import (
|
||||
run_archivebox_cmd,
|
||||
parse_jsonl_output,
|
||||
create_test_url,
|
||||
create_test_crawl_json,
|
||||
create_test_snapshot_json,
|
||||
pid_is_alive,
|
||||
wait_for_log,
|
||||
wait_for_pid_to_disappear,
|
||||
)
|
||||
|
||||
@ -48,133 +45,72 @@ def _install_real_chrome_for_test(data_dir, env, *, isolation):
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.timeout(90)
|
||||
def test_cli_run_signal_cleans_background_hook_process_group(initialized_archive):
|
||||
@pytest.mark.timeout(660)
|
||||
def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archive, recursive_test_site):
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
plugins_root = initialized_archive / "runtime_plugins"
|
||||
plugin_dir = plugins_root / "cancel_group"
|
||||
plugin_dir.mkdir(parents=True)
|
||||
daemon_hook = plugin_dir / "on_CrawlSetup__10_daemon.daemon.bg.sh"
|
||||
foreground_hook = plugin_dir / "on_CrawlSetup__20_foreground.sh"
|
||||
daemon_hook.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
"set -euo pipefail",
|
||||
'test_dir="${LEAK_TEST_DIR:?}"',
|
||||
"sleep 600 &",
|
||||
'echo $$ > "$test_dir/daemon.pid"',
|
||||
'echo $! > "$test_dir/daemon-child.pid"',
|
||||
'echo ready > "$test_dir/daemon.ready"',
|
||||
"trap 'echo cleaned > \"$test_dir/daemon.cleaned\"; exit 0' TERM INT",
|
||||
"wait",
|
||||
"",
|
||||
],
|
||||
),
|
||||
)
|
||||
foreground_hook.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
"set -euo pipefail",
|
||||
'test_dir="${LEAK_TEST_DIR:?}"',
|
||||
'echo $$ > "$test_dir/foreground.pid"',
|
||||
'echo ready > "$test_dir/foreground.ready"',
|
||||
"trap 'echo cleaned > \"$test_dir/foreground.cleaned\"; exit 0' TERM INT",
|
||||
"while true; do sleep 1; done",
|
||||
"",
|
||||
],
|
||||
),
|
||||
)
|
||||
daemon_hook.chmod(0o755)
|
||||
foreground_hook.chmod(0o755)
|
||||
|
||||
leak_test_dir = initialized_archive / "leak-check"
|
||||
leak_test_dir.mkdir()
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"ABX_PLUGINS_DIR": str(plugins_root),
|
||||
"LEAK_TEST_DIR": str(leak_test_dir),
|
||||
"PLUGINS": "cancel_group",
|
||||
"TIMEOUT": "30",
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
},
|
||||
)
|
||||
env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false")
|
||||
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["crawl", "create", "https://example.com"],
|
||||
["snapshot", "create", recursive_test_site["root_url"]],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert returncode == 0, stderr or stdout
|
||||
crawl_records = [json.loads(line) for line in stdout.splitlines() if line.strip().startswith("{")]
|
||||
crawl_id = next(record["id"] for record in crawl_records if record.get("type") == "Crawl")
|
||||
records = parse_jsonl_output(stdout)
|
||||
snapshot_id = next(record["id"] for record in records if record.get("type") == "Snapshot")
|
||||
with use_archivebox_db(initialized_archive):
|
||||
browser_state = Snapshot.objects.get(id=snapshot_id).output_dir / "chrome" / "browser.json"
|
||||
|
||||
daemon_pid: int | None = None
|
||||
daemon_child_pid: int | None = None
|
||||
foreground_pid: int | None = None
|
||||
run_log = initialized_archive / "run-signal-chrome.log"
|
||||
run_log_handle = run_log.open("w", encoding="utf-8")
|
||||
run_process = run_archivebox_cmd(
|
||||
["run", f"--crawl-id={crawl_id}"],
|
||||
["run", f"--snapshot-id={snapshot_id}"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stdout=run_log_handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
run_log_handle.close()
|
||||
try:
|
||||
deadline = time.time() + 20
|
||||
while time.time() < deadline:
|
||||
if (leak_test_dir / "daemon.ready").exists() and (leak_test_dir / "foreground.ready").exists():
|
||||
break
|
||||
if run_process.poll() is not None:
|
||||
output = run_process.communicate(timeout=1)[0]
|
||||
raise AssertionError(f"archivebox run exited before hooks were ready:\n{output}")
|
||||
time.sleep(0.05)
|
||||
assert (leak_test_dir / "daemon.ready").exists()
|
||||
assert (leak_test_dir / "foreground.ready").exists()
|
||||
|
||||
daemon_pid = int((leak_test_dir / "daemon.pid").read_text().strip())
|
||||
daemon_child_pid = int((leak_test_dir / "daemon-child.pid").read_text().strip())
|
||||
foreground_pid = int((leak_test_dir / "foreground.pid").read_text().strip())
|
||||
assert pid_is_alive(daemon_pid)
|
||||
assert pid_is_alive(daemon_child_pid)
|
||||
assert pid_is_alive(foreground_pid)
|
||||
wait_for_log(browser_state, '"ready": true', timeout=120)
|
||||
child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True)]
|
||||
assert child_pids
|
||||
assert all(pid_is_alive(pid) for pid in child_pids)
|
||||
|
||||
run_process.send_signal(signal.SIGTERM)
|
||||
output = run_process.communicate(timeout=20)[0]
|
||||
run_process.wait(timeout=30)
|
||||
output = run_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert "Runner error" not in output
|
||||
|
||||
wait_for_pid_to_disappear(daemon_pid, timeout=5)
|
||||
wait_for_pid_to_disappear(daemon_child_pid, timeout=5)
|
||||
wait_for_pid_to_disappear(foreground_pid, timeout=5)
|
||||
assert (leak_test_dir / "daemon.cleaned").read_text().strip() == "cleaned"
|
||||
assert (leak_test_dir / "foreground.cleaned").read_text().strip() == "cleaned"
|
||||
for pid in child_pids:
|
||||
wait_for_pid_to_disappear(pid, timeout=15)
|
||||
finally:
|
||||
if run_process.poll() is None:
|
||||
try:
|
||||
os.killpg(run_process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
run_process.communicate(timeout=5)
|
||||
cleanup_process_group(daemon_pid, daemon_child_pid)
|
||||
cleanup_process_group(foreground_pid)
|
||||
cleanup_process_group(run_process.pid)
|
||||
|
||||
|
||||
class TestRunWithCrawl:
|
||||
"""Tests for `archivebox run` with Crawl input."""
|
||||
|
||||
def test_run_with_new_crawl(self, initialized_archive):
|
||||
"""Run creates and processes a new Crawl (no id)."""
|
||||
crawl_record = create_test_crawl_json()
|
||||
"""Run processes a Crawl emitted by the public create command."""
|
||||
create_result = run_archivebox_cmd(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert create_result.returncode == 0, create_result.stderr
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(crawl_record),
|
||||
stdin=create_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -204,12 +140,10 @@ class TestRunWithCrawl:
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
crawl = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
# Run with the existing crawl
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(crawl),
|
||||
stdin=stdout1,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -227,12 +161,19 @@ class TestRunWithSnapshot:
|
||||
"""Tests for `archivebox run` with Snapshot input."""
|
||||
|
||||
def test_run_with_new_snapshot(self, initialized_archive):
|
||||
"""Run creates and processes a new Snapshot (no id, just url)."""
|
||||
snapshot_record = create_test_snapshot_json()
|
||||
"""Run processes a Snapshot emitted by the public create command."""
|
||||
create_result = run_archivebox_cmd(
|
||||
["snapshot", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert create_result.returncode == 0, create_result.stderr
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(snapshot_record),
|
||||
stdin=create_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -261,12 +202,10 @@ class TestRunWithSnapshot:
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
# Run with the existing snapshot
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(snapshot),
|
||||
stdin=stdout1,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -282,11 +221,9 @@ class TestRunWithSnapshot:
|
||||
def test_run_with_plain_url(self, initialized_archive):
|
||||
"""Run accepts plain URL records (no type field)."""
|
||||
url = create_test_url()
|
||||
url_record = {"url": url}
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(url_record),
|
||||
stdin=url + "\n",
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -316,34 +253,64 @@ class TestRunWithArchiveResult:
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
snapshot = parse_jsonl_output(stdout1)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["archiveresult", "create", "--plugin=favicon"],
|
||||
stdin=json.dumps(snapshot),
|
||||
stdin=stdout1,
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
ar = next(r for r in parse_jsonl_output(stdout2) if r.get("type") == "ArchiveResult")
|
||||
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(stdout2))
|
||||
|
||||
# Update to failed
|
||||
ar["status"] = "failed"
|
||||
run_archivebox_cmd(
|
||||
["archiveresult", "update", "--status=failed"],
|
||||
stdin=json.dumps(ar),
|
||||
initial_run = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=stdout2,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert initial_run.returncode == 0, initial_run.stderr
|
||||
persisted_result = run_archivebox_cmd(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert persisted_result.returncode == 0, persisted_result.stderr
|
||||
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(persisted_result.stdout))
|
||||
|
||||
# Update to failed
|
||||
update_result = run_archivebox_cmd(
|
||||
["archiveresult", "update", "--status=failed"],
|
||||
stdin=persisted_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert update_result.returncode == 0, update_result.stderr
|
||||
failed_result = run_archivebox_cmd(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert failed_result.returncode == 0, failed_result.stderr
|
||||
failed_records = [record for record in parse_jsonl_output(failed_result.stdout) if record.get("type") == "ArchiveResult"]
|
||||
assert len(failed_records) == 1
|
||||
assert failed_records[0]["status"] == "failed"
|
||||
failed_jsonl = next(line for line in failed_result.stdout.splitlines() if failed_records[0]["id"] in line) + "\n"
|
||||
|
||||
# Now run should re-queue it
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(ar),
|
||||
stdin=failed_jsonl,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -417,13 +384,20 @@ class TestRunRecovery:
|
||||
class TestRunPassThrough:
|
||||
"""Tests for pass-through behavior in `archivebox run`."""
|
||||
|
||||
def test_run_passes_through_unknown_types(self, initialized_archive):
|
||||
"""Run passes through records with unknown types."""
|
||||
unknown_record = {"type": "Unknown", "id": "fake-id", "data": "test"}
|
||||
def test_run_passes_through_tag_emitted_by_cli(self, initialized_archive):
|
||||
"""Run passes through a real non-runnable Tag record."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
["tag", "create", "run-input-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
tag_record = parse_jsonl_output(tag_result.stdout)[0]
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(unknown_record),
|
||||
stdin=tag_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
@ -432,18 +406,25 @@ class TestRunPassThrough:
|
||||
|
||||
assert code == 0
|
||||
records = parse_jsonl_output(stdout)
|
||||
unknown_records = [r for r in records if r.get("type") == "Unknown"]
|
||||
assert len(unknown_records) == 1
|
||||
assert unknown_records[0]["data"] == "test"
|
||||
tag_records = [record for record in records if record.get("type") == "Tag"]
|
||||
assert len(tag_records) == 1
|
||||
assert tag_records[0]["id"] == tag_record["id"]
|
||||
|
||||
def test_run_outputs_all_processed_records(self, initialized_archive):
|
||||
"""Run outputs all processed records for chaining."""
|
||||
url = create_test_url()
|
||||
crawl_record = create_test_crawl_json(urls=[url])
|
||||
create_result = run_archivebox_cmd(
|
||||
["crawl", "create", url],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert create_result.returncode == 0, create_result.stderr
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(crawl_record),
|
||||
stdin=create_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -461,23 +442,34 @@ class TestRunPassThrough:
|
||||
class TestRunMixedInput:
|
||||
"""Tests for `archivebox run` with mixed record types."""
|
||||
|
||||
def test_run_handles_mixed_types(self, initialized_archive):
|
||||
"""Run handles mixed Crawl/Snapshot/ArchiveResult input."""
|
||||
crawl = create_test_crawl_json()
|
||||
snapshot = create_test_snapshot_json()
|
||||
unknown = {"type": "Tag", "id": "fake", "name": "test"}
|
||||
|
||||
stdin = "\n".join(
|
||||
[
|
||||
json.dumps(crawl),
|
||||
json.dumps(snapshot),
|
||||
json.dumps(unknown),
|
||||
],
|
||||
def test_run_handles_mixed_records_emitted_by_cli(self, initialized_archive):
|
||||
"""Run handles real Crawl, Snapshot, and Tag records from CLI stages."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
["tag", "create", "mixed-run-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
crawl_result = run_archivebox_cmd(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert crawl_result.returncode == 0, crawl_result.stderr
|
||||
snapshot_result = run_archivebox_cmd(
|
||||
["snapshot", "create"],
|
||||
stdin=crawl_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert snapshot_result.returncode == 0, snapshot_result.stderr
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=stdin,
|
||||
stdin=tag_result.stdout + snapshot_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
@ -489,9 +481,8 @@ class TestRunMixedInput:
|
||||
assert code == 0
|
||||
records = parse_jsonl_output(stdout)
|
||||
|
||||
types = {r.get("type") for r in records}
|
||||
# Should have processed Crawl and Snapshot, passed through Tag
|
||||
assert "Crawl" in types or "Snapshot" in types or "Tag" in types
|
||||
types = {record.get("type") for record in records}
|
||||
assert {"Crawl", "Snapshot", "Tag"}.issubset(types)
|
||||
|
||||
|
||||
class TestRunEmpty:
|
||||
@ -510,13 +501,19 @@ class TestRunEmpty:
|
||||
|
||||
assert code == 0
|
||||
|
||||
def test_run_no_records_to_process(self, initialized_archive):
|
||||
"""Run with only pass-through records shows message."""
|
||||
unknown = {"type": "Unknown", "id": "fake"}
|
||||
def test_run_no_runnable_records_to_process(self, initialized_archive):
|
||||
"""Run with only a real non-runnable Tag reports no work."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
["tag", "create", "non-runnable-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=json.dumps(unknown),
|
||||
stdin=tag_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
@ -537,72 +534,69 @@ class TestRunDaemonMode:
|
||||
snapshot_url = None
|
||||
if stdin_kind == "valid-snapshot":
|
||||
snapshot_url = create_test_url()
|
||||
piped_stdin = json.dumps(create_test_snapshot_json(url=snapshot_url)) + "\n"
|
||||
snapshot_result = run_archivebox_cmd(
|
||||
["snapshot", "create", snapshot_url],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert snapshot_result.returncode == 0, snapshot_result.stderr
|
||||
piped_stdin = snapshot_result.stdout
|
||||
else:
|
||||
piped_stdin = "{this is not jsonl}\n"
|
||||
|
||||
env = cli_env()
|
||||
env = cli_env(PLUGINS="__archivebox_test_no_plugins__")
|
||||
queued = run_archivebox_cmd(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert queued.returncode == 0, queued.stderr or queued.stdout
|
||||
daemon_log = initialized_archive / f"run-daemon-{stdin_kind}.log"
|
||||
daemon_log_handle = daemon_log.open("w", encoding="utf-8")
|
||||
proc = run_archivebox_cmd(
|
||||
["run", "--daemon"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
stdout=daemon_log_handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
daemon_log_handle.close()
|
||||
assert proc.stdin is not None
|
||||
assert proc.stdout is not None
|
||||
assert proc.stderr is not None
|
||||
|
||||
try:
|
||||
proc.stdin.write(piped_stdin)
|
||||
proc.stdin.close()
|
||||
|
||||
deadline = time.monotonic() + 20
|
||||
started = False
|
||||
while time.monotonic() < deadline:
|
||||
if proc.poll() is not None:
|
||||
stdout = proc.stdout.read()
|
||||
stderr = proc.stderr.read()
|
||||
raise AssertionError(
|
||||
f"daemon exited before starting runner: code={proc.returncode}\nstdout={stdout}\nstderr={stderr}",
|
||||
)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
started = Process.objects.filter(
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pid=proc.pid,
|
||||
).exists()
|
||||
if started:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
|
||||
assert started is True
|
||||
wait_for_log(daemon_log, "[Crawl#", timeout=30)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
started = Process.objects.filter(
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pid=proc.pid,
|
||||
).exists()
|
||||
assert started
|
||||
if snapshot_url is not None:
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert not Snapshot.objects.filter(url=snapshot_url).exists()
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
os.killpg(proc.pid, signal.SIGTERM)
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
proc.wait(timeout=5)
|
||||
os.kill(proc.pid, signal.SIGTERM)
|
||||
proc.wait(timeout=15)
|
||||
|
||||
stdout = proc.stdout.read()
|
||||
stderr = proc.stderr.read()
|
||||
assert proc.returncode == 143, stdout + stderr
|
||||
assert "No records to process" not in stderr
|
||||
output = daemon_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert proc.returncode == 143, output
|
||||
assert "No records to process" not in output
|
||||
|
||||
def test_run_daemon_takeover_has_single_active_runner_gate(self, initialized_archive, db):
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env()
|
||||
env = cli_env(PLUGINS="__archivebox_test_no_plugins__")
|
||||
|
||||
def active_runners():
|
||||
with use_archivebox_db(initialized_archive):
|
||||
@ -617,67 +611,65 @@ class TestRunDaemonMode:
|
||||
if proc.is_running
|
||||
]
|
||||
|
||||
def wait_for_stable_single_active(*, timeout: float, stable_seconds: float = 1.0, exclude_pid: int | None = None):
|
||||
deadline = time.monotonic() + timeout
|
||||
stable_pid = None
|
||||
stable_since = None
|
||||
while time.monotonic() < deadline:
|
||||
active = active_runners()
|
||||
assert len(active) <= 1
|
||||
if len(active) == 1 and active[0].pid != exclude_pid:
|
||||
pid = active[0].pid
|
||||
if pid != stable_pid:
|
||||
stable_pid = pid
|
||||
stable_since = time.monotonic()
|
||||
elif stable_since is not None and time.monotonic() - stable_since >= stable_seconds:
|
||||
return pid
|
||||
else:
|
||||
stable_pid = None
|
||||
stable_since = None
|
||||
time.sleep(0.25)
|
||||
return None
|
||||
|
||||
procs = [
|
||||
run_archivebox_cmd(
|
||||
queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60)
|
||||
assert queued.returncode == 0, queued.stderr or queued.stdout
|
||||
procs = []
|
||||
logs = []
|
||||
for index in range(2):
|
||||
log_path = initialized_archive / f"run-daemon-takeover-{index}.log"
|
||||
log_handle = log_path.open("w", encoding="utf-8")
|
||||
proc = run_archivebox_cmd(
|
||||
["run", "--daemon"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
stdout=log_handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
log_handle.close()
|
||||
procs.append(proc)
|
||||
logs.append(log_path)
|
||||
if index == 0:
|
||||
wait_for_log(log_path, "[Crawl#", timeout=30)
|
||||
try:
|
||||
active_pid = wait_for_stable_single_active(timeout=30)
|
||||
assert active_pid is not None
|
||||
wait_for_log(logs[1], "Stopping older ArchiveBox runner process", timeout=30)
|
||||
queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60)
|
||||
assert queued.returncode == 0, queued.stderr or queued.stdout
|
||||
wait_for_log(logs[1], "[Crawl#", timeout=30)
|
||||
active = active_runners()
|
||||
assert len(active) == 1
|
||||
active_pid = active[0].pid
|
||||
assert active_pid == procs[1].pid
|
||||
|
||||
os.killpg(active_pid, signal.SIGKILL)
|
||||
os.kill(active_pid, signal.SIGTERM)
|
||||
wait_for_pid_to_disappear(active_pid, timeout=15)
|
||||
replacement_log = initialized_archive / "run-daemon-takeover-replacement.log"
|
||||
replacement_log_handle = replacement_log.open("w", encoding="utf-8")
|
||||
replacement = run_archivebox_cmd(
|
||||
["run", "--daemon"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
stdout=replacement_log_handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
replacement_log_handle.close()
|
||||
procs.append(replacement)
|
||||
recovered_pid = wait_for_stable_single_active(timeout=30, exclude_pid=active_pid)
|
||||
assert recovered_pid is not None
|
||||
queued = run_archivebox_cmd(["crawl", "create", create_test_url()], cwd=initialized_archive, env=env, timeout=60)
|
||||
assert queued.returncode == 0, queued.stderr or queued.stdout
|
||||
wait_for_log(replacement_log, "[Crawl#", timeout=30)
|
||||
recovered = active_runners()
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].pid == replacement.pid
|
||||
assert recovered[0].pid != active_pid
|
||||
finally:
|
||||
for proc in procs:
|
||||
if proc.poll() is None:
|
||||
os.killpg(proc.pid, signal.SIGTERM)
|
||||
for proc in procs:
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
proc.wait(timeout=5)
|
||||
cleanup_process_group(proc.pid)
|
||||
proc.wait(timeout=15)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@ -1114,11 +1106,12 @@ class TestRecoverOrchestratorState:
|
||||
sealed_crawl.cancel()
|
||||
|
||||
paused_child.refresh_from_db()
|
||||
paused_result.refresh_from_db()
|
||||
sealed_child.refresh_from_db()
|
||||
sealed_started_child.refresh_from_db()
|
||||
assert paused_child.status == Snapshot.StatusChoices.STARTED
|
||||
assert paused_child.retry_at is not None
|
||||
assert paused_child.retry_at <= timezone.now()
|
||||
assert paused_child.status == Snapshot.StatusChoices.PAUSED
|
||||
assert paused_child.retry_at == RETRY_AT_MAX
|
||||
assert paused_result.status == ArchiveResult.StatusChoices.PAUSED
|
||||
assert sealed_child.status == Snapshot.StatusChoices.PAUSED
|
||||
assert sealed_child.retry_at is not None
|
||||
assert sealed_child.retry_at <= timezone.now()
|
||||
@ -1126,13 +1119,6 @@ class TestRecoverOrchestratorState:
|
||||
assert sealed_started_child.retry_at is not None
|
||||
assert sealed_started_child.retry_at <= timezone.now()
|
||||
|
||||
assert run_due_snapshot(paused_child, lock_seconds=60) is True
|
||||
paused_child.refresh_from_db()
|
||||
paused_result.refresh_from_db()
|
||||
assert paused_child.status == Snapshot.StatusChoices.PAUSED
|
||||
assert paused_child.retry_at == RETRY_AT_MAX
|
||||
assert paused_result.status == ArchiveResult.StatusChoices.PAUSED
|
||||
|
||||
assert run_due_snapshot(sealed_child, lock_seconds=60) is True
|
||||
sealed_child.refresh_from_db()
|
||||
assert sealed_child.status == Snapshot.StatusChoices.SEALED
|
||||
@ -1243,7 +1229,7 @@ class TestRecoverOrchestratorState:
|
||||
assert crawl.retry_at < future
|
||||
assert snapshot.retry_at < future
|
||||
|
||||
def test_recover_orchestrator_state_preserves_future_started_snapshot_with_live_result_process(self):
|
||||
def test_recover_orchestrator_state_preserves_future_started_snapshot_with_live_result_process(self, initialized_archive):
|
||||
from datetime import timedelta
|
||||
|
||||
from django.utils import timezone
|
||||
@ -1254,13 +1240,19 @@ class TestRecoverOrchestratorState:
|
||||
from archivebox.machine.models import Machine, NetworkInterface, Process
|
||||
from archivebox.core.recovery_util import recover_orchestrator_state
|
||||
|
||||
worker = subprocess.Popen(
|
||||
[sys.executable, "-c", "import time; time.sleep(60)"],
|
||||
worker = run_archivebox_cmd(
|
||||
["manage", "shell"],
|
||||
cwd=initialized_archive,
|
||||
env=cli_env(live=True),
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
capture_output=False,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
assert worker.stdin is not None
|
||||
assert pid_is_alive(worker.pid)
|
||||
try:
|
||||
future = timezone.now() + timedelta(seconds=45)
|
||||
crawl = Crawl.objects.create(
|
||||
@ -1303,13 +1295,9 @@ class TestRecoverOrchestratorState:
|
||||
assert snapshot.status == Snapshot.StatusChoices.STARTED
|
||||
assert snapshot.retry_at == future
|
||||
finally:
|
||||
if worker.poll() is None:
|
||||
os.killpg(worker.pid, signal.SIGTERM)
|
||||
try:
|
||||
worker.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(worker.pid, signal.SIGKILL)
|
||||
worker.wait(timeout=5)
|
||||
worker.stdin.close()
|
||||
worker.wait(timeout=20)
|
||||
assert not pid_is_alive(worker.pid)
|
||||
|
||||
def test_recover_orchestrator_state_does_not_resume_paused_rows_with_max_retry_at(self):
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
@ -1628,7 +1616,7 @@ class TestRecoverOrchestratorState:
|
||||
|
||||
update_process = run_archivebox_cmd(
|
||||
["archiveresult", "update", "--status=queued"],
|
||||
stdin=json.dumps(wait_record) + "\n",
|
||||
stdin=next(line for line in list_process.stdout.splitlines() if wait_record["id"] in line) + "\n",
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
@ -1649,47 +1637,27 @@ class TestRecoverOrchestratorState:
|
||||
run_process.stdin.write(update_process.stdout)
|
||||
run_process.stdin.close()
|
||||
|
||||
resumed_full_plugin = False
|
||||
try:
|
||||
deadline = time.time() + 90
|
||||
last_wait_status = None
|
||||
last_tab_process_id = None
|
||||
while time.time() < deadline:
|
||||
with use_archivebox_db(initialized_archive):
|
||||
wait_result = ArchiveResult.objects.get(
|
||||
snapshot_id=snapshot_id,
|
||||
plugin="chrome",
|
||||
hook_name="on_Snapshot__11_chrome_wait",
|
||||
)
|
||||
tab_result = ArchiveResult.objects.get(
|
||||
snapshot_id=snapshot_id,
|
||||
plugin="chrome",
|
||||
hook_name="on_Snapshot__10_chrome_tab.daemon.bg",
|
||||
)
|
||||
last_wait_status = wait_result.status
|
||||
last_tab_process_id = tab_result.process_id
|
||||
if wait_result.status == ArchiveResult.StatusChoices.SUCCEEDED and tab_result.process_id != first_tab_process_id:
|
||||
resumed_full_plugin = True
|
||||
break
|
||||
if run_process.poll() is not None:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
|
||||
if resumed_full_plugin:
|
||||
try:
|
||||
run_process.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
cleanup_process_group(run_process.pid)
|
||||
run_process.wait(timeout=10)
|
||||
run_process.wait(timeout=120)
|
||||
finally:
|
||||
if run_process.poll() is None:
|
||||
cleanup_process_group(run_process.pid)
|
||||
run_process.wait(timeout=10)
|
||||
cleanup_process_group(run_process.pid)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
wait_result = ArchiveResult.objects.get(
|
||||
snapshot_id=snapshot_id,
|
||||
plugin="chrome",
|
||||
hook_name="on_Snapshot__11_chrome_wait",
|
||||
)
|
||||
tab_result = ArchiveResult.objects.get(
|
||||
snapshot_id=snapshot_id,
|
||||
plugin="chrome",
|
||||
hook_name="on_Snapshot__10_chrome_tab.daemon.bg",
|
||||
)
|
||||
|
||||
assert run_process.returncode == 0
|
||||
assert last_wait_status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert last_tab_process_id is not None
|
||||
assert last_tab_process_id != first_tab_process_id
|
||||
assert wait_result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert tab_result.process_id is not None
|
||||
assert tab_result.process_id != first_tab_process_id
|
||||
|
||||
def test_recover_orchestrator_state_ignores_sealed_downloaded_snapshot_without_results(self):
|
||||
from django.utils import timezone
|
||||
@ -1912,13 +1880,15 @@ class TestRunDueCrawlState:
|
||||
assert finished.output_files == {"favicon.ico": {"size": 1}}
|
||||
|
||||
def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self):
|
||||
import json
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.plugins.hooks import extract_records_from_process, run_hook
|
||||
from archivebox.services.runner import run_due_snapshot
|
||||
|
||||
crawl = Crawl.objects.create(
|
||||
@ -1935,17 +1905,34 @@ class TestRunDueCrawlState:
|
||||
status=Snapshot.StatusChoices.STARTED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
staticfile_dir = root.output_dir / "staticfile"
|
||||
parser_dir = root.output_dir / "parse_txt_urls"
|
||||
staticfile_dir.mkdir(parents=True, exist_ok=True)
|
||||
parser_dir.mkdir(parents=True, exist_ok=True)
|
||||
(parser_dir / "urls.jsonl").write_text(
|
||||
json.dumps({"type": "Snapshot", "url": "https://example.org/"}) + "\n",
|
||||
(staticfile_dir / "input.txt").write_text(
|
||||
"Plain text import containing https://example.org/\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
|
||||
process = run_hook(
|
||||
hook_path,
|
||||
parser_dir,
|
||||
config={"ABXPKG_LIB_DIR": str(root.output_dir.parent.parent / "lib"), "SNAP_DIR": str(root.output_dir)},
|
||||
timeout=30,
|
||||
url=root.url,
|
||||
depth=root.depth,
|
||||
snapshot_id=str(root.id),
|
||||
)
|
||||
process.refresh_from_db()
|
||||
assert process.exit_code == 0, process.stderr
|
||||
result_record = next(record for record in extract_records_from_process(process) if record.get("type") == "ArchiveResult")
|
||||
ArchiveResult.objects.create(
|
||||
snapshot=root,
|
||||
plugin="parse_txt_urls",
|
||||
hook_name="on_Snapshot__71_parse_txt_urls",
|
||||
status=ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
process=process,
|
||||
plugin=result_record["plugin"],
|
||||
hook_name=result_record["hook_name"],
|
||||
status=result_record["status"],
|
||||
output_str=result_record.get("output_str", ""),
|
||||
output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}},
|
||||
)
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@ from .conftest import (
|
||||
make_latest_schedule_due,
|
||||
start_archivebox_server,
|
||||
stop_server,
|
||||
wait_for_http,
|
||||
get_http_response,
|
||||
wait_for_snapshot_capture,
|
||||
)
|
||||
|
||||
@ -162,7 +162,7 @@ def test_schedule_due_crawl_runs_over_server_and_saves_real_content(tmp_path, re
|
||||
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
wait_for_http(port, host=f"web.archivebox.localhost:{port}")
|
||||
get_http_response(port, host=f"web.archivebox.localhost:{port}")
|
||||
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
|
||||
assert "Root" in captured_text
|
||||
assert "About" in captured_text
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for archivebox search command.
|
||||
|
||||
TODO: keep search-specific tests here instead of folding all coverage into test_cli_list.py.
|
||||
"""
|
||||
|
||||
import json
|
||||
@ -62,12 +60,3 @@ def test_cli_search_status_filters_snapshot_status_column(tmp_path, initialized_
|
||||
rows = [json.loads(line) for line in result.stdout.splitlines() if line.strip().startswith("{")]
|
||||
assert [row["status"] for row in rows] == ["sealed"]
|
||||
assert [row["url"] for row in rows] == ["https://example.com/search-status-sealed"]
|
||||
|
||||
legacy_result = run_archivebox_cmd(
|
||||
["search", "--status", "unarchived", "search-status"],
|
||||
env=env,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
assert legacy_result.returncode != 0
|
||||
assert "Invalid snapshot status" in legacy_result.stderr
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user