Fix custom plugin execution from data directories

archivebox/cli/__init__.py

@@ -186,6 +186,10 @@ def cli(ctx, help=False):
 
 
 def main(args=None, prog_name=None):
+    from archivebox.config.constants import CONSTANTS
+
+    os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR))
+
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()

archivebox/tests/test_cli_extract.py

@@ -1,7 +1,11 @@
 #!/usr/bin/env python3
 """Tests for archivebox extract command."""
 
+import shutil
+from pathlib import Path
+
 import pytest
+from abx_plugins import get_plugins_dir
 
 from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.tests.conftest import cli_env, find_snapshot_dir, parse_jsonl_output, run_archivebox_cmd
@@ -74,6 +78,68 @@ def test_extract_runs_on_existing_snapshots(archive_with_extractors):
     assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
 
 
+def test_extract_runs_custom_plugin_discovered_from_data_dir(initialized_archive):
+    """Custom plugins discovered by ArchiveBox must execute through abx-dl."""
+    custom_plugin_name = "custom_parse_txt_urls"
+    custom_plugin_dir = initialized_archive / "custom_plugins" / custom_plugin_name
+    shutil.copytree(Path(get_plugins_dir()) / "parse_txt_urls", custom_plugin_dir)
+    input_path = initialized_archive / "custom-plugin-input.txt"
+    input_path.write_text("https://example.com\nhttps://example.org\n", encoding="utf-8")
+    env = cli_env(PLUGINS=f"{custom_plugin_name},hashes", HASHES_ENABLED="True")
+
+    create_result = run_archivebox_cmd(
+        ["snapshot", "create", "https://archivebox.example/custom-plugin-input"],
+        cwd=initialized_archive,
+        env=env,
+    )
+    assert create_result.returncode == 0, create_result.stderr or create_result.stdout
+    snapshot = next(record for record in parse_jsonl_output(create_result.stdout) if record.get("type") == "Snapshot")
+
+    with use_archivebox_db(initialized_archive):
+        staticfile_dir = Snapshot.objects.get(id=snapshot["id"]).output_dir / "staticfile"
+    staticfile_dir.mkdir(parents=True)
+    shutil.copy2(input_path, staticfile_dir / input_path.name)
+
+    result = run_archivebox_cmd(
+        ["extract", f"--plugins={custom_plugin_name},hashes", snapshot["id"]],
+        cwd=initialized_archive,
+        env=env,
+        timeout=90,
+    )
+
+    assert result.returncode == 0, result.stderr or result.stdout
+    records = parse_jsonl_output(result.stdout)
+    custom_result = next(
+        record for record in records if record.get("type") == "ArchiveResult" and record.get("plugin") == custom_plugin_name
+    )
+    hashes_result = next(record for record in records if record.get("type") == "ArchiveResult" and record.get("plugin") == "hashes")
+    assert custom_result["status"] == ArchiveResult.StatusChoices.SUCCEEDED, custom_result
+    assert custom_result["output_str"] == "2 URLs parsed"
+    assert hashes_result["status"] == ArchiveResult.StatusChoices.SUCCEEDED, hashes_result
+
+    snapshot_dir = find_snapshot_dir(initialized_archive, snapshot["id"])
+    assert snapshot_dir is not None
+    urls_path = snapshot_dir / custom_plugin_name / "urls.jsonl"
+    hashes_path = snapshot_dir / "hashes" / "hashes.json"
+    assert urls_path.is_file()
+    assert hashes_path.is_file()
+    assert {record["url"] for record in parse_jsonl_output(urls_path.read_text(encoding="utf-8"))} == {
+        "https://example.com",
+        "https://example.org",
+    }
+
+    with use_archivebox_db(initialized_archive):
+        archiveresults = {
+            row.plugin: row for row in ArchiveResult.objects.filter(snapshot_id=snapshot["id"], plugin__in=(custom_plugin_name, "hashes"))
+        }
+
+    assert set(archiveresults) == {custom_plugin_name, "hashes"}
+    assert archiveresults[custom_plugin_name].status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert archiveresults[custom_plugin_name].output_files["urls.jsonl"]["size"] == urls_path.stat().st_size
+    assert archiveresults["hashes"].status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert archiveresults["hashes"].output_files["hashes.json"]["size"] == hashes_path.stat().st_size
+
+
 def test_extract_preserves_snapshot_count(archive_with_extractors):
     """Extract queues work without creating duplicate snapshots."""
     initialized_archive = archive_with_extractors

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc302",
+  "version": "0.9.35rc303",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc302"
+version = "0.9.35rc303"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.35rc302"
+current_version = "v0.9.35rc303"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -120,7 +120,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.35rc302"
+version = "0.9.35rc303"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },