This agent can work directly with your ArchiveBox collection. It can inspect the crawl database, create and monitor crawls, run maintenance operations, answer research questions, query archived data, and help with complex collection workflows.
-
Example prompts:
-
-
Archive this list of URLs with depth 0, then report which ones failed and why.
-
Find snapshots from the last month that failed PDF or screenshot extraction and retry them.
-
Create a constrained crawl for this site, avoid login/privacy/sitemap URLs, and watch the logs as it runs.
-
Summarize what my collection contains about this topic and link to the best saved pages.
-
-
- The agent has unrestricted access to this collection and can make destructive edits. Review requests carefully. Permission settings can be changed from the OpenCode gear icon in the lower left.
-
-
-
-
-
-
-
-
- {% endif %}
-
-{% endblock %}
diff --git a/archivebox/opencode/views.py b/archivebox/opencode/views.py
index 6a728753..46ccdfe7 100644
--- a/archivebox/opencode/views.py
+++ b/archivebox/opencode/views.py
@@ -1,776 +1,76 @@
-from __future__ import annotations
+"""Django adapter for the optional plugin; never import its runtime at startup."""
-import atexit
-import base64
import logging
-import os
-import re
-import signal
-import shutil
-import subprocess
-import threading
-import time
-from pathlib import Path
-from typing import Any
-from urllib.parse import urljoin, urlsplit
-import httpx
-import requests
-from abx_plugins.plugins import opencode as opencode_plugin
+from django.contrib.auth.views import redirect_to_login
+from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
+from django.template import engines
+from django.views.decorators.csrf import csrf_exempt
+
from archivebox.config import CONSTANTS
from archivebox.config.common import get_config
from archivebox.core.routes_util import build_admin_url, get_api_base_url, get_base_url
-from asgiref.sync import sync_to_async
-from django.http import (
- Http404,
- HttpRequest,
- HttpResponse,
- HttpResponseForbidden,
- StreamingHttpResponse,
-)
-from django.shortcuts import redirect, render
-from django.views.decorators.csrf import csrf_exempt
+from archivebox.plugins.discovery import get_plugin_template
-
-_PROCESS: subprocess.Popen | None = None
-_PROCESS_READY: subprocess.Popen | None = None
-_PROCESS_LOCK = threading.Lock()
-_SESSION_LOCK = threading.Lock()
_LOGGER = logging.getLogger(__name__)
-_PROXY_PREFIX = "/admin/agent/opencode"
-_PROXY_PREFIX_REGEX = _PROXY_PREFIX.replace("/", r"\/")
-_PROXY_PREFIX_NO_SLASH_REGEX = _PROXY_PREFIX.lstrip("/").replace("/", r"\/")
-_CONFIG_PATH = Path(opencode_plugin.__file__).with_name("config.json")
-_DEFAULT_MODEL = "opencode/big-pickle"
-_DEFAULT_CONFIG = f'''{{
- "$schema": "https://opencode.ai/config.json",
- "model": "{_DEFAULT_MODEL}",
- "snapshot": false
-}}
-'''
-
-_TEXT_CONTENT_TYPES = (
- "text/",
- "application/javascript",
- "application/x-javascript",
-)
-_HOP_BY_HOP_HEADERS = {
- "connection",
- "keep-alive",
- "proxy-authenticate",
- "proxy-authorization",
- "te",
- "trailers",
- "transfer-encoding",
- "upgrade",
-}
-_ARCHIVEBOX_SKILL = """---
-name: archivebox
-description: Use ArchiveBox's CLI and local REST API from an ArchiveBox collection.
----
-
-You are running inside an ArchiveBox collection directory.
-
-- ArchiveBox collection directory: {archivebox_data_dir}
-- ArchiveBox BASE_URL: {archivebox_base_url}
-- ArchiveBox Admin URL: {archivebox_admin_url}
-- ArchiveBox REST API URL: {archivebox_api_url}
-- Prefer the `archivebox` CLI for authenticated changes, e.g. `archivebox add`, `archivebox schedule`, `archivebox update`, and `archivebox shell`.
-- Run ArchiveBox CLI commands from the ArchiveBox collection directory above.
-- Get command help with `archivebox list --help`, `archivebox add --help`, `archivebox schedule --help`, etc. Do not use `archivebox help `.
-- Use `--depth=0` by default. Only use recursive crawling when the user explicitly asks for it; use `--depth=1` when you need pages one hop out.
-- Before any recursive crawl, constrain scope with ArchiveBox config such as `CRAWL_MAX_URLS`, `CRAWL_MAX_SIZE`, `SNAPSHOT_MAX_*`, `URL_ALLOWLIST`, `URL_DENYLIST`, and related limits.
-- Respect the configured `archivebox config --get ONLY_NEW` behavior unless the user explicitly says otherwise. Remind users that expected crawl URLs can be skipped when the collection already contains snapshots with the same URL.
-- Always audit newly discovered crawl URLs before letting a crawl run broadly. Treat junk URLs such as privacy policies, legal pages, tag archives, sitemap files, feeds, login/logout URLs, and other low-value boilerplate as unwanted unless the user explicitly asked to archive them.
-- Always watch crawl output and logs as the crawl progresses, and correct errors early instead of waiting until the crawl finishes.
-- If a crawl contains bad URLs, pause it, edit the crawl's `urls` field to remove them, delete any unneeded snapshots already created under that crawl, then resume the crawl.
-- Use `archivebox shell -c '...'` or `archivebox shell <<'PY' ... PY` for Django ORM work. Shell Plus prints an import banner first; keep stderr visible while debugging.
-- Use full ArchiveBox module paths in shell code: `from archivebox.crawls.models import Crawl, CrawlSchedule` and `from archivebox.core.models import Snapshot, ArchiveResult`.
-- If a model/field/relation is unclear, inspect `_meta.fields` before guessing, e.g. `archivebox shell -c "from archivebox.crawls.models import Crawl; print([f.name for f in Crawl._meta.fields])"`.
-- Use `archivebox config --get BASE_URL` only to verify the configured base URL; prefer the seeded URLs above for API/admin requests.
-- Use `$ARCHIVEBOX_API_URL` for REST API inspection when helpful. Do not assume admin session cookies authenticate API subdomain requests; prefer CLI/shell for authenticated mutations unless the admin provides or asks you to create an API token.
-- Discover REST endpoints from `${{ARCHIVEBOX_API_URL}}v1/openapi.json`; crawl endpoints live under `/api/v1/crawls/`, snapshots under `/api/v1/core/`.
-- Do not bypass ArchiveBox auth, expose API keys, or modify config unless the admin explicitly asks.
-- After creating crawls or snapshots, report the crawl/snapshot IDs and the exact command or API request used.
-"""
-def _signal_owned_process(process: subprocess.Popen, sig: signal.Signals) -> None:
+def _dispatch(request, path=None):
try:
- os.killpg(process.pid, sig)
- except OSError:
- try:
- process.send_signal(sig)
- except ProcessLookupError:
- pass
+ config = dict(get_config().model_dump(mode="json"))
+ if not config.get("OPENCODE_ENABLED", False):
+ raise Http404
+ if not request.user.is_authenticated:
+ return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
+ if not request.user.is_active or not request.user.is_superuser:
+ return HttpResponseForbidden("Agent access requires a superuser account.")
+ from abx_plugins.plugins.opencode import runtime
-def _stop_owned_process(process: subprocess.Popen | None = None) -> None:
- global _PROCESS, _PROCESS_READY
- owned_process = process or _PROCESS
- if owned_process is None:
- return
- if owned_process.poll() is None:
- _signal_owned_process(owned_process, signal.SIGCONT)
- _signal_owned_process(owned_process, signal.SIGTERM)
- try:
- owned_process.wait(timeout=5)
- except subprocess.TimeoutExpired:
- _signal_owned_process(owned_process, signal.SIGKILL)
- owned_process.wait()
- if _PROCESS is owned_process:
- _PROCESS = None
- if _PROCESS_READY is owned_process:
- _PROCESS_READY = None
-
-
-atexit.register(_stop_owned_process)
-
-
-def _machine_config() -> dict[str, Any]:
- resolved = get_config()
- return dict(resolved.model_dump(mode="json"))
-
-
-def _archivebox_data_dir_default() -> Path:
- return Path(CONSTANTS.DATA_DIR)
-
-
-def _archivebox_route_urls(request: HttpRequest, route_config) -> tuple[str, str, str]:
- base_url = get_base_url(
- request=request,
- config=route_config,
- ).rstrip("/")
- admin_url = build_admin_url(
- "/admin/",
- request=request,
- config=route_config,
- ).rstrip("/")
- api_url = f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/"
- return base_url, admin_url, api_url
-
-
-def _config_value(config: dict, key: str, default):
- value = config.get(key, default)
- if value in (None, ""):
- return default
- return value
-
-
-def _opencode_enabled(config: dict) -> bool:
- value = config.get("OPENCODE_ENABLED", False)
- if isinstance(value, bool):
- return value
- return str(value).strip().lower() in {"1", "true", "yes", "on"}
-
-
-def _require_enabled(config: dict) -> None:
- if not _opencode_enabled(config):
- raise Http404
-
-
-def _require_superuser(request: HttpRequest):
- user = getattr(request, "user", None)
- if user is None:
- return redirect(f"/admin/login/?next={request.get_full_path()}")
- if (
- bool(getattr(user, "is_authenticated", False))
- and bool(getattr(user, "is_active", False))
- and bool(getattr(user, "is_superuser", False))
- ):
- return None
- if bool(getattr(user, "is_authenticated", False)):
- return HttpResponseForbidden(
- b"ArchiveBox agent access requires a superuser account.",
+ settings = runtime._settings(config, CONSTANTS.DATA_DIR)
+ route_config = request.__dict__.get("archivebox_config")
+ settings.update(
+ archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
+ archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
+ archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
)
- return redirect(f"/admin/login/?next={request.get_full_path()}")
+ if path is None:
+ from archivebox.core.admin_site import archivebox_admin
+ context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
+ source = get_plugin_template("opencode", "agent", fallback=False)
+ if source is None:
+ raise RuntimeError("Agent template unavailable")
+ return HttpResponse(engines["django"].from_string(source).render(context, request))
-def _origin_allowed(request: HttpRequest, path: str | None = None) -> bool:
- if request.method in {"GET", "HEAD", "OPTIONS", "TRACE"}:
- return True
-
- expected_host = request.get_host()
- pty_connect = bool(
- path and path.startswith("pty/") and path.endswith("/connect-token"),
- )
- if pty_connect:
- return True
-
- origin = _request_header(request, "Origin")
- if origin:
- return _same_host(origin, expected_host)
-
- referer = _request_header(request, "Referer")
- if referer:
- return _same_host(referer, expected_host)
-
- fetch_site = _request_header(request, "Sec-Fetch-Site")
- if fetch_site:
- return fetch_site in {"same-origin", "same-site", "none"}
-
- return False
-
-
-def _same_host(value: str, expected_host: str) -> bool:
- parsed = urlsplit(value)
- return parsed.scheme in {"http", "https"} and parsed.netloc == expected_host
-
-
-def _settings(config: dict) -> dict:
- host = str(_config_value(config, "OPENCODE_HOST", "127.0.0.1"))
- port = int(_config_value(config, "OPENCODE_PORT", 4096))
- default_data_dir = _archivebox_data_dir_default()
- opencode_dir = Path(
- str(_config_value(config, "OPENCODE_STATE_DIR", default_data_dir / "opencode")),
- ).expanduser()
- workdir = Path(
- str(_config_value(config, "OPENCODE_WORKDIR", default_data_dir)),
- ).expanduser()
- binary = str(_config_value(config, "OPENCODE_BINARY", "opencode"))
- timeout = int(_config_value(config, "OPENCODE_TIMEOUT", 120))
- return {
- "host": host,
- "port": port,
- "origin": f"http://{host}:{port}",
- "archivebox_data_dir": default_data_dir,
- "workdir": workdir,
- "opencode_dir": opencode_dir,
- "config_home": opencode_dir / "config",
- "data_home": opencode_dir / "data",
- "state_home": opencode_dir / "state",
- "cache_home": opencode_dir / "cache",
- "home": opencode_dir / "home",
- "binary": binary,
- "config": config,
- "timeout": timeout,
- }
-
-
-def _resolve_binary(binary: str, config: dict) -> tuple[Any, Any, dict[str, str]]:
- try:
- from abxpkg import BinProvider
- from abx_plugins.plugins.base.utils import load_required_binary_from_config
-
- binary_environ = os.environ.copy()
- lib_dir = config.get("ABXPKG_LIB_DIR")
- if lib_dir:
- binary_environ["ABXPKG_LIB_DIR"] = str(lib_dir)
- loaded_dependencies = [
- load_required_binary_from_config(
- required_binary,
- _CONFIG_PATH,
- global_config=config,
- environ=binary_environ,
- install=False,
- )
- for required_binary in (
- str(config.get("NODE_BINARY") or "node"),
- str(config.get("GIT_BINARY") or "git"),
- binary,
- )
- ]
- except Exception as err:
- raise RuntimeError(
- f"OpenCode dependency is not installed from required_binaries: {err}",
- ) from err
-
- if any(not loaded.loaded_abspath for loaded in loaded_dependencies):
- raise RuntimeError(
- "OpenCode dependency is not installed from required_binaries.",
+ if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
+ return HttpResponseForbidden("Cross-origin agent requests are blocked.")
+ status, headers, body = runtime.proxy(
+ settings,
+ request.method,
+ path,
+ tuple((key, value) for key, values in request.GET.lists() for value in values),
+ request.headers,
+ request.body,
)
-
- providers = [loaded.loaded_binprovider for loaded in loaded_dependencies if loaded.loaded_binprovider is not None]
- binary_env = BinProvider.build_exec_env(
- providers=providers,
- base_env=binary_environ,
- )
- return (
- loaded_dependencies[-1],
- loaded_dependencies[1],
- binary_env,
- )
+ response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
+ response = response_type(body, status=status, headers=headers)
+ response.xframe_options_exempt = True
+ response.headers["X-Frame-Options"] = "SAMEORIGIN"
+ response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
+ return response
+ except Http404:
+ raise
+ except Exception:
+ # Optional-service boundary, including imports and template rendering.
+ _LOGGER.exception("Optional AI service failed")
+ return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")
-def _project_route(workdir: Path, session_id: str = "") -> str:
- encoded = base64.b64encode(str(workdir.resolve()).encode()).decode()
- encoded = encoded.replace("+", "-").replace("/", "_").rstrip("=")
- route = f"{_PROXY_PREFIX}/{encoded}/session"
- return f"{route}/{session_id}" if session_id else route
-
-
-def _ensure_project_files(settings: dict) -> None:
- workdir = settings["workdir"].resolve()
- workdir.mkdir(parents=True, exist_ok=True)
- git_marker = workdir / ".git" / "not-a-git"
- if git_marker.exists():
- # Current OpenCode hangs on the legacy fake marker, so remove only
- # that invalid shape before initializing the real worktree.
- shutil.rmtree(git_marker.parent)
-
- editable_skill_path = settings["opencode_dir"] / "SKILL.md"
- editable_skill_path.parent.mkdir(parents=True, exist_ok=True)
- if not editable_skill_path.exists():
- editable_skill_path.write_text(
- _ARCHIVEBOX_SKILL.format(
- archivebox_data_dir=settings["archivebox_data_dir"],
- archivebox_base_url=settings.get("archivebox_base_url", ""),
- archivebox_admin_url=settings.get("archivebox_admin_url", ""),
- archivebox_api_url=settings.get("archivebox_api_url", ""),
- ),
- )
-
- opencode_skill_path = settings["config_home"] / "opencode" / "skills" / "archivebox" / "SKILL.md"
- opencode_skill_path.parent.mkdir(parents=True, exist_ok=True)
- if opencode_skill_path.resolve() != editable_skill_path.resolve():
- if opencode_skill_path.exists() or opencode_skill_path.is_symlink():
- opencode_skill_path.unlink()
- opencode_skill_path.symlink_to(editable_skill_path)
-
- opencode_config_path = settings["config_home"] / "opencode" / "opencode.jsonc"
- if not opencode_config_path.exists():
- opencode_config_path.write_text(_DEFAULT_CONFIG)
-
-
-def _ensure_default_session(settings: dict) -> str:
- workdir = settings["workdir"].resolve()
- params = {"directory": str(workdir)}
- timeout = settings["timeout"]
- project = requests.post(
- f"{settings['origin']}/project/git/init",
- params=params,
- timeout=timeout,
- )
- project.raise_for_status()
- project_data = project.json()
- if Path(str(project_data.get("worktree") or "/")).resolve() != workdir:
- raise RuntimeError(
- f"OpenCode initialized the wrong project worktree: {project_data.get('worktree')!r}",
- )
-
- sessions = requests.get(
- f"{settings['origin']}/session",
- params={**params, "roots": "true", "limit": 55},
- timeout=timeout,
- )
- sessions.raise_for_status()
- session_data = sessions.json()
- if not isinstance(session_data, list):
- raise RuntimeError("OpenCode returned an invalid project session list.")
- for session_data_item in session_data:
- if not isinstance(session_data_item, dict):
- continue
- session_id = str(session_data_item.get("id") or "")
- session_directory = session_data_item.get("directory")
- if session_id and session_directory and Path(str(session_directory)).resolve() == workdir:
- return session_id
-
- session = requests.post(
- f"{settings['origin']}/session",
- params=params,
- json={},
- timeout=timeout,
- )
- session.raise_for_status()
- session_data = session.json()
- session_id = str(session_data.get("id") or "")
- session_directory = session_data.get("directory")
- if not session_id or not session_directory or Path(str(session_directory)).resolve() != workdir:
- raise RuntimeError(
- "OpenCode did not create a session for the requested worktree.",
- )
- return session_id
-
-
-def _owned_process_running() -> bool:
- process = _PROCESS
- return process is not None and process.poll() is None
-
-
-def _owned_process_ready() -> bool:
- process = _PROCESS_READY
- return process is not None and process is _PROCESS and process.poll() is None
-
-
-def _health(settings: dict, timeout: float = 2) -> bool:
- try:
- response = requests.get(
- f"{settings['origin']}/global/health",
- timeout=timeout,
- )
- return response.status_code == 200
- except requests.RequestException:
- return False
-
-
-def _ensure_opencode(settings: dict) -> tuple[bool, str]:
- global _PROCESS, _PROCESS_READY
- started_process: subprocess.Popen | None = None
- workdir = settings["workdir"].resolve()
-
- with _PROCESS_LOCK:
- if _owned_process_ready():
- return True, ""
- if _health(settings):
- if _owned_process_running():
- _PROCESS_READY = _PROCESS
- return True, ""
- if _owned_process_running():
- _stop_owned_process(_PROCESS)
-
- try:
- binary, git_binary, binary_env = _resolve_binary(
- settings["binary"],
- settings["config"],
- )
- except RuntimeError as err:
- return False, str(err)
-
- env = {
- **os.environ,
- **binary_env,
- "ARCHIVEBOX_BASE_URL": str(settings.get("archivebox_base_url", "")),
- "ARCHIVEBOX_ADMIN_URL": str(settings.get("archivebox_admin_url", "")),
- "ARCHIVEBOX_API_URL": str(settings.get("archivebox_api_url", "")),
- "BROWSER": "false",
- "GIT_CEILING_DIRECTORIES": str(workdir),
- "HOME": str(settings["home"]),
- "OPENCODE_DISABLE_PROJECT_CONFIG": "true",
- "XDG_CONFIG_HOME": str(settings["config_home"]),
- "XDG_DATA_HOME": str(settings["data_home"]),
- "XDG_STATE_HOME": str(settings["state_home"]),
- "XDG_CACHE_HOME": str(settings["cache_home"]),
- }
-
- settings["workdir"].mkdir(parents=True, exist_ok=True)
- settings["config_home"].mkdir(parents=True, exist_ok=True)
- settings["data_home"].mkdir(parents=True, exist_ok=True)
- settings["state_home"].mkdir(parents=True, exist_ok=True)
- settings["cache_home"].mkdir(parents=True, exist_ok=True)
- settings["home"].mkdir(parents=True, exist_ok=True)
- _ensure_project_files(settings)
-
- if not (workdir / ".git").exists():
- try:
- git_init = git_binary.exec(
- cmd=("init", "--quiet"),
- cwd=workdir,
- env=env,
- timeout=settings["timeout"],
- )
- except (AssertionError, OSError, subprocess.SubprocessError) as err:
- return False, f"OpenCode project initialization failed: {err}"
- if git_init.returncode != 0:
- output = (git_init.stderr or git_init.stdout or "").strip()
- return (
- False,
- f"OpenCode project initialization failed: {output or f'git exited with {git_init.returncode}'}",
- )
-
- if _health(settings):
- return True, ""
-
- binary_abspath = binary.loaded_abspath
- if binary.loaded_binprovider is not None:
- binary_abspath = binary.loaded_binprovider._exec_bin_abspath(
- Path(binary.loaded_abspath),
- )
- cmd = [
- str(binary_abspath),
- "serve",
- "--hostname",
- settings["host"],
- "--port",
- str(settings["port"]),
- ]
- try:
- _PROCESS = subprocess.Popen(
- cmd,
- cwd=workdir,
- env=env,
- stdin=subprocess.DEVNULL,
- stdout=subprocess.DEVNULL,
- start_new_session=True,
- )
- _PROCESS_READY = None
- started_process = _PROCESS
- except FileNotFoundError:
- return False, f"OpenCode binary not found: {settings['binary']}"
-
- deadline = time.monotonic() + settings["timeout"]
- while True:
- remaining = deadline - time.monotonic()
- if remaining <= 0:
- break
- if _health(settings, timeout=min(2, remaining)):
- if time.monotonic() <= deadline:
- _PROCESS_READY = started_process
- return True, ""
- break
- if started_process and started_process.poll() is not None:
- if _PROCESS is started_process:
- _PROCESS = None
- if _PROCESS_READY is started_process:
- _PROCESS_READY = None
- return False, "OpenCode exited before the web server became ready."
- remaining = deadline - time.monotonic()
- if remaining > 0:
- time.sleep(min(0.25, remaining))
-
- _stop_owned_process(started_process)
- return False, "Timed out waiting for OpenCode to start."
-
-
-def agent_view(request: HttpRequest):
- config = _machine_config()
- _require_enabled(config)
- auth_response = _require_superuser(request)
- if auth_response:
- return auth_response
-
- settings = _settings(config)
- route_config = request.__dict__.get("archivebox_config")
- base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
- settings["archivebox_base_url"] = base_url
- settings["archivebox_admin_url"] = admin_url
- settings["archivebox_api_url"] = api_url
- ok, error = _ensure_opencode(settings)
- recent_session_id = ""
- if ok:
- try:
- with _SESSION_LOCK:
- recent_session_id = _ensure_default_session(settings)
- except (requests.RequestException, RuntimeError, ValueError) as err:
- if isinstance(err, requests.RequestException) and _owned_process_ready():
- _stop_owned_process()
- ok = False
- error = f"OpenCode project initialization failed: {err}"
- from archivebox.core.admin_site import archivebox_admin
-
- context = {
- **archivebox_admin.each_context(request),
- "title": "Agent",
- "error": "" if ok else error,
- "command": f"{settings['binary']} serve --hostname {settings['host']} --port {settings['port']}" if error else "",
- # OpenCode 1.17+ keeps durable sessions on the explicit
- # //session/ route. We still seed localStorage
- # below because the sidebar state uses it, but the iframe itself must
- # open the durable session URL so a fresh browser does not land on the
- # transient new-session route and appear to have lost prior sessions.
- "proxy_url": _project_route(settings["workdir"], recent_session_id),
- "proxy_prefix": _PROXY_PREFIX,
- "workdir": str(settings["workdir"].resolve()),
- "recent_session_id": recent_session_id,
- }
- return render(
- request,
- "opencode/agent.html",
- context,
- status=200 if ok else 502,
- )
-
-
-def _proxy_url(settings: dict, path: str | None) -> str:
- rel = "/" if not path else f"/{path}"
- return urljoin(settings["origin"], rel)
-
-
-def _request_header(request: HttpRequest, name: str) -> str | None:
- meta = getattr(request, "META", {})
- if name == "Content-Type":
- value = meta.get("CONTENT_TYPE")
- elif name == "Content-Length":
- value = meta.get("CONTENT_LENGTH")
- else:
- value = meta.get(f"HTTP_{name.upper().replace('-', '_')}")
- return str(value) if value else None
-
-
-def _request_headers(request: HttpRequest, settings: dict) -> dict[str, str]:
- forwarded = {}
- for key in ("Accept", "Accept-Language", "Content-Type", "Range", "User-Agent"):
- value = _request_header(request, key)
- if value:
- forwarded[key] = value
- return forwarded
-
-
-def _request_params(request: HttpRequest) -> tuple[tuple[str, str], ...]:
- if hasattr(request.GET, "lists"):
- return tuple((key, str(value)) for key, values in request.GET.lists() for value in values)
- return tuple((key, str(value)) for key, value in dict(request.GET).items())
-
-
-async def _event_chunks(
- settings: dict,
- path: str | None,
- method: str,
- params: tuple[tuple[str, str], ...],
- headers: dict[str, str],
-):
- if not _owned_process_ready():
- ok, error = await sync_to_async(_ensure_opencode, thread_sensitive=False)(settings)
- if not ok:
- _LOGGER.warning("OpenCode event stream unavailable: %s", error)
- yield b'event: error\ndata: {"error":"OpenCode upstream unavailable"}\n\n'
- return
-
- timeout = httpx.Timeout(settings["timeout"], read=None)
- url = _proxy_url(settings, path)
- try:
- async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
- async with client.stream(
- method,
- url,
- params=params,
- headers=headers,
- ) as upstream:
- async for chunk in upstream.aiter_raw(chunk_size=512):
- yield chunk
- except httpx.RequestError as err:
- _LOGGER.warning("OpenCode event stream ended: %s", err)
-
-
-def _rewrite_text(body: bytes, settings: dict) -> bytes:
- text = body.decode("utf-8", errors="replace")
- text = text.replace(settings["origin"], _PROXY_PREFIX)
- text = text.replace("location.origin", f'location.origin+"{_PROXY_PREFIX}"')
- text = text.replace(
- "k(k5,{get component(){return t.router??Az},",
- f'k(k5,{{base:"{_PROXY_PREFIX}",get component(){{return t.router??Az}},',
- )
- text = text.replace('"/assets/', f'"{_PROXY_PREFIX}/assets/')
- text = text.replace("'/assets/", f"'{_PROXY_PREFIX}/assets/")
- proxy_path = rf'(\1.replace(/^{_PROXY_PREFIX_REGEX}(?=\/|$)/,"")||"/")'
- text = re.sub(r"\b(window\.location\.pathname)\b", proxy_path, text)
- text = re.sub(r"(?\b(?:href|src|action)=["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
- rf"\g{_PROXY_PREFIX}/",
- text,
- )
- text = re.sub(
- rf"""(?P\b(?:fetch|EventSource)\(["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
- rf"\g{_PROXY_PREFIX}/",
- text,
- )
- text = re.sub(
- rf"""(?P\burl\(["']?)/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""",
- rf"\g{_PROXY_PREFIX}/",
- text,
- )
- return text.encode("utf-8")
-
-
-def _response_headers(upstream: requests.Response, settings: dict) -> dict[str, str]:
- headers = {}
- for key, value in upstream.headers.items():
- lower = key.lower()
- if lower in _HOP_BY_HOP_HEADERS or lower in {
- "content-length",
- "content-encoding",
- "x-frame-options",
- }:
- continue
- if lower == "location":
- if value.startswith(settings["origin"]):
- value = value.replace(settings["origin"], _PROXY_PREFIX, 1)
- elif value.startswith("/"):
- value = f"{_PROXY_PREFIX}{value}"
- headers[key] = value
- return headers
-
-
-def _proxy_error_response(error: Exception | str) -> HttpResponse:
- _LOGGER.warning("OpenCode upstream request failed: %s", error)
- return HttpResponse(
- b"OpenCode upstream request failed.",
- status=502,
- content_type="text/plain; charset=utf-8",
- )
+def agent_view(request):
+ return _dispatch(request)
@csrf_exempt
-def opencode_proxy_view(request: HttpRequest, path: str | None = None):
- config = _machine_config()
- _require_enabled(config)
- auth_response = _require_superuser(request)
- if auth_response:
- return auth_response
- if not _origin_allowed(request, path):
- return HttpResponseForbidden(
- b"Cross-origin OpenCode agent requests are blocked.",
- )
-
- settings = _settings(config)
- route_config = request.__dict__.get("archivebox_config")
- base_url, admin_url, api_url = _archivebox_route_urls(request, route_config)
- settings["archivebox_base_url"] = base_url
- settings["archivebox_admin_url"] = admin_url
- settings["archivebox_api_url"] = api_url
-
- if request.method == "GET" and (path or "").endswith("/event"):
- response = StreamingHttpResponse(
- _event_chunks(
- settings,
- path,
- request.method or "GET",
- _request_params(request),
- _request_headers(request, settings),
- ),
- content_type="text/event-stream",
- )
- response.headers["Cache-Control"] = "no-store"
- response.headers["X-Accel-Buffering"] = "no"
- return response
-
- if path == "global/health" or not _owned_process_ready():
- ok, error = _ensure_opencode(settings)
- if not ok:
- return _proxy_error_response(error)
-
- try:
- method = request.method or "GET"
- upstream = requests.request(
- method,
- _proxy_url(settings, path),
- params=_request_params(request),
- data=request.body if method not in {"GET", "HEAD"} else None,
- headers=_request_headers(request, settings),
- stream=True,
- timeout=settings["timeout"],
- allow_redirects=False,
- )
- except requests.RequestException as err:
- return _proxy_error_response(err)
-
- try:
- body = upstream.content
- except requests.RequestException as err:
- upstream.close()
- return _proxy_error_response(err)
-
- content_type = upstream.headers.get("Content-Type", "")
- headers = _response_headers(upstream, settings)
- if any(content_type.startswith(prefix) for prefix in _TEXT_CONTENT_TYPES):
- body = _rewrite_text(body, settings)
- response = HttpResponse(
- body,
- status=upstream.status_code,
- content_type=content_type or "application/octet-stream",
- )
- for key, value in headers.items():
- response.headers[key] = value
- response.headers["Cache-Control"] = "no-store"
- return response
+def opencode_proxy_view(request, path=None):
+ return _dispatch(request, path=path or "")
diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html
index 42aa2c37..8db954b5 100644
--- a/archivebox/templates/core/add.html
+++ b/archivebox/templates/core/add.html
@@ -1,6 +1,6 @@
{% extends "core/base.html" %}
-{% load static %}
+{% load static core_tags %}
{% load i18n %}
{% block breadcrumbs %}
@@ -48,9 +48,7 @@
- {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
- 💬 Crawl with AI |
- {% endif %}
+ {% plugin_ui "opencode" "add" %}
Get the extension
💡 Tip: Instantly save a single URL by visiting:
diff --git a/archivebox/templates/core/navigation.html b/archivebox/templates/core/navigation.html
index 96fd8ec0..cddb4439 100644
--- a/archivebox/templates/core/navigation.html
+++ b/archivebox/templates/core/navigation.html
@@ -1,4 +1,4 @@
-{% load i18n static %}
+{% load i18n static core_tags %}
Add âž•
@@ -11,10 +11,7 @@
|Tags
- {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %}
- 💬 AI
- |
- {% endif %}
+ {% plugin_ui "opencode" "navigation" %}
Docs|API
diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py
index af0ef60c..9e6ea78b 100644
--- a/archivebox/tests/test_opencode_agent.py
+++ b/archivebox/tests/test_opencode_agent.py
@@ -1,19 +1,18 @@
import asyncio
-import json
import os
-import signal
+import shutil
import socket
-import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
-from urllib.parse import quote
+from urllib.parse import parse_qs, quote, urlsplit
import pytest
import requests
from asgiref.testing import ApplicationCommunicator
from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
+from archivebox.config.common import get_config
pytestmark = pytest.mark.django_db(transaction=True)
@@ -78,8 +77,8 @@ def opencode_archive_config(initialized_archive):
@pytest.fixture
-def live_opencode(opencode_archive_config):
- from archivebox.opencode import views
+def installed_opencode(opencode_archive_config):
+ from abx_plugins.plugins.opencode import runtime
install = run_archivebox_cmd(
["install", "opencode", "--binproviders=env,pnpm"],
@@ -90,56 +89,61 @@ def live_opencode(opencode_archive_config):
assert install.returncode == 0, install.stderr or install.stdout
_reset_runtime_config()
- config = views._machine_config()
- settings = views._settings(config)
+ config = get_config().model_dump(mode="json")
+ settings = runtime._settings(config, opencode_archive_config.data_dir)
settings["archivebox_base_url"] = "http://admin.archivebox.localhost:8000"
settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:8000/admin"
settings["archivebox_api_url"] = "http://admin.archivebox.localhost:8000/api/"
- binary, _, binary_env = views._resolve_binary(settings["binary"], settings["config"])
+ binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"])
version = binary.exec(
cmd=("--version",),
env={**os.environ, **binary_env},
timeout=120,
)
assert version.returncode == 0, version.stderr or version.stdout
- ok, error = views._ensure_opencode(settings)
+ return SimpleNamespace(config=opencode_archive_config, settings=settings)
+
+
+@pytest.fixture
+def live_opencode(installed_opencode):
+ from abx_plugins.plugins.opencode import runtime
+
+ settings = installed_opencode.settings
+ ok, error = runtime._ensure_opencode(settings)
assert ok, error
- process = views._PROCESS
+ process = runtime._PROCESS
assert process is not None
try:
- yield SimpleNamespace(config=opencode_archive_config, settings=settings, process=process)
+ yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process)
finally:
- views._stop_owned_process()
+ runtime._stop_owned_process()
def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
from archivebox.machine.models import Machine
- from archivebox.opencode import views
+ from abx_plugins.plugins.opencode import runtime
os.chdir(initialized_archive)
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
_reset_runtime_config()
- assert views._machine_config()["OPENCODE_ENABLED"] is False
+ assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 404
- assert views._PROCESS is None or views._PROCESS.poll() is not None
+ assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None
-def test_stop_owned_process_falls_back_for_stopped_process_without_dedicated_group():
- from archivebox.opencode import views
+def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive):
+ _set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False")
- process = subprocess.Popen(["sleep", "60"])
- try:
- process.send_signal(signal.SIGSTOP)
- views._stop_owned_process(process)
- assert process.returncode == -signal.SIGTERM
- finally:
- if process.poll() is None:
- process.kill()
- process.wait()
+ assert get_config().OPENCODE_ENABLED is False
+ assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404
+ for path in ("/add/", "/admin/core/snapshot/"):
+ response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST)
+ assert response.status_code == 200
+ assert b'href="/admin/agent"' not in response.content
def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode):
@@ -147,6 +151,11 @@ def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_
assert response.status_code == 302
assert "/admin/login/" in response.headers["Location"]
+ next_path = "/admin/agent?x=1&next=https://example.com"
+ response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST)
+ assert response.status_code == 302
+ assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]}
+
user = django_user_model.objects.create_user(username="regular", password="testpassword")
client.force_login(user)
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
@@ -178,18 +187,22 @@ def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_
def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
- from archivebox.opencode import views
+ from abx_plugins.plugins.opencode import runtime
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
recent_session_id = response.context["recent_session_id"]
- session_path = views._project_route(live_opencode.config.data_dir, recent_session_id)
+ session_path = runtime._project_route(live_opencode.config.data_dir, recent_session_id)
assert response.status_code == 200
assert recent_session_id
assert f'