From a69e090ff7434c7301b0004e94aefe3b240c6d7a Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 24 Aug 2026 14:00:30 -0700 Subject: [PATCH] Fix custom plugin execution from data directories --- archivebox/cli/__init__.py | 4 ++ archivebox/tests/test_cli_extract.py | 66 ++++++++++++++++++++++++++++ etc/package.json | 2 +- pyproject.toml | 4 +- uv.lock | 2 +- 5 files changed, 74 insertions(+), 4 deletions(-) diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 12624d2e..6ad16dec 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -186,6 +186,10 @@ def cli(ctx, help=False): def main(args=None, prog_name=None): + from archivebox.config.constants import CONSTANTS + + os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR)) + # show `docker run archivebox xyz` in help messages if running in docker IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes") IS_TTY = sys.stdin.isatty() diff --git a/archivebox/tests/test_cli_extract.py b/archivebox/tests/test_cli_extract.py index 64e1c6b4..7c0fc2d2 100644 --- a/archivebox/tests/test_cli_extract.py +++ b/archivebox/tests/test_cli_extract.py @@ -1,7 +1,11 @@ #!/usr/bin/env python3 """Tests for archivebox extract command.""" +import shutil +from pathlib import Path + import pytest +from abx_plugins import get_plugins_dir from archivebox.core.models import ArchiveResult, Snapshot from archivebox.tests.conftest import cli_env, find_snapshot_dir, parse_jsonl_output, run_archivebox_cmd @@ -74,6 +78,68 @@ def test_extract_runs_on_existing_snapshots(archive_with_extractors): assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size +def test_extract_runs_custom_plugin_discovered_from_data_dir(initialized_archive): + """Custom plugins discovered by ArchiveBox must execute through abx-dl.""" + custom_plugin_name = "custom_parse_txt_urls" + custom_plugin_dir = initialized_archive / "custom_plugins" / custom_plugin_name + shutil.copytree(Path(get_plugins_dir()) / "parse_txt_urls", custom_plugin_dir) + input_path = initialized_archive / "custom-plugin-input.txt" + input_path.write_text("https://example.com\nhttps://example.org\n", encoding="utf-8") + env = cli_env(PLUGINS=f"{custom_plugin_name},hashes", HASHES_ENABLED="True") + + create_result = run_archivebox_cmd( + ["snapshot", "create", "https://archivebox.example/custom-plugin-input"], + cwd=initialized_archive, + env=env, + ) + assert create_result.returncode == 0, create_result.stderr or create_result.stdout + snapshot = next(record for record in parse_jsonl_output(create_result.stdout) if record.get("type") == "Snapshot") + + with use_archivebox_db(initialized_archive): + staticfile_dir = Snapshot.objects.get(id=snapshot["id"]).output_dir / "staticfile" + staticfile_dir.mkdir(parents=True) + shutil.copy2(input_path, staticfile_dir / input_path.name) + + result = run_archivebox_cmd( + ["extract", f"--plugins={custom_plugin_name},hashes", snapshot["id"]], + cwd=initialized_archive, + env=env, + timeout=90, + ) + + assert result.returncode == 0, result.stderr or result.stdout + records = parse_jsonl_output(result.stdout) + custom_result = next( + record for record in records if record.get("type") == "ArchiveResult" and record.get("plugin") == custom_plugin_name + ) + hashes_result = next(record for record in records if record.get("type") == "ArchiveResult" and record.get("plugin") == "hashes") + assert custom_result["status"] == ArchiveResult.StatusChoices.SUCCEEDED, custom_result + assert custom_result["output_str"] == "2 URLs parsed" + assert hashes_result["status"] == ArchiveResult.StatusChoices.SUCCEEDED, hashes_result + + snapshot_dir = find_snapshot_dir(initialized_archive, snapshot["id"]) + assert snapshot_dir is not None + urls_path = snapshot_dir / custom_plugin_name / "urls.jsonl" + hashes_path = snapshot_dir / "hashes" / "hashes.json" + assert urls_path.is_file() + assert hashes_path.is_file() + assert {record["url"] for record in parse_jsonl_output(urls_path.read_text(encoding="utf-8"))} == { + "https://example.com", + "https://example.org", + } + + with use_archivebox_db(initialized_archive): + archiveresults = { + row.plugin: row for row in ArchiveResult.objects.filter(snapshot_id=snapshot["id"], plugin__in=(custom_plugin_name, "hashes")) + } + + assert set(archiveresults) == {custom_plugin_name, "hashes"} + assert archiveresults[custom_plugin_name].status == ArchiveResult.StatusChoices.SUCCEEDED + assert archiveresults[custom_plugin_name].output_files["urls.jsonl"]["size"] == urls_path.stat().st_size + assert archiveresults["hashes"].status == ArchiveResult.StatusChoices.SUCCEEDED + assert archiveresults["hashes"].output_files["hashes.json"]["size"] == hashes_path.stat().st_size + + def test_extract_preserves_snapshot_count(archive_with_extractors): """Extract queues work without creating duplicate snapshots.""" initialized_archive = archive_with_extractors diff --git a/etc/package.json b/etc/package.json index e4287b81..3a4f1306 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc302", + "version": "0.9.35rc303", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 31d8fcda..93684cb8 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc302" +version = "0.9.35rc303" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc302" +current_version = "v0.9.35rc303" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index 64d25298..be76f343 100644 --- a/uv.lock +++ b/uv.lock @@ -120,7 +120,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc302" +version = "0.9.35rc303" source = { editable = "." } dependencies = [ { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },