From d044f5ee0efeac31c55064b29d8c2f4cd3d6482d Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Thu, 3 Sep 2026 16:05:30 -0700 Subject: [PATCH] Isolate OpenCode in its plugin and stop forced Git initialization (#1869) * Stop initializing Git repositories for OpenCode sessions * Isolate optional AI failures behind a lazy plugin adapter * Keep explicit plugin enable flags boolean after config resolution * Use Django login redirect encoding at the optional agent boundary * Cover cold-start failures and real browser storage isolation * Verify persisted dismissal and native storage denial variants * Keep headless browser interaction independent of display timing * Install optional OpenCode clients through the plugin extra --- archivebox/config/common.py | 4 +- archivebox/core/middleware.py | 8 +- archivebox/core/settings.py | 1 - archivebox/core/templatetags/core_tags.py | 15 + archivebox/opencode/apps.py | 6 - .../opencode/templates/opencode/agent.html | 211 ----- archivebox/opencode/views.py | 808 ++---------------- archivebox/templates/core/add.html | 6 +- archivebox/templates/core/navigation.html | 7 +- archivebox/tests/test_opencode_agent.py | 308 ++++--- archivebox/tests/test_opencode_browser.py | 154 ++++ pyproject.toml | 3 +- uv.lock | 20 +- 13 files changed, 419 insertions(+), 1132 deletions(-) delete mode 100644 archivebox/opencode/apps.py delete mode 100644 archivebox/opencode/templates/opencode/agent.html create mode 100644 archivebox/tests/test_opencode_browser.py diff --git a/archivebox/config/common.py b/archivebox/config/common.py index 9d9a1f2e..d3f41d0b 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -18,7 +18,7 @@ from urllib.parse import quote, urlparse from abx_plugins.plugins.base.utils import build_config_model from django.db import DatabaseError -from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator +from pydantic import BaseModel, Field, PrivateAttr, TypeAdapter, create_model, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict from rich.console import Console @@ -1253,7 +1253,7 @@ def get_config( config = config.model_copy(update={"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay"}) for key in explicit_plugin_enabled_keys: if key in config_data: - setattr(config, key, config_data[key]) + setattr(config, key, TypeAdapter(bool).validate_python(config_data[key])) if config.PLUGINS: config._derive_plugin_enabled_config(respect_current_enabled=True) if redact_sensitive: diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py index 1267c8aa..fd04280f 100644 --- a/archivebox/core/middleware.py +++ b/archivebox/core/middleware.py @@ -89,13 +89,7 @@ def AdminCookieIsolationMiddleware(get_response): response = get_response(request) if request.path == "/admin" or request.path.startswith("/admin/"): - from archivebox.opencode.views import _PROXY_PREFIX - - is_opencode_proxy = request.path == _PROXY_PREFIX or request.path.startswith(f"{_PROXY_PREFIX}/") - if is_opencode_proxy: - response.headers["X-Frame-Options"] = "SAMEORIGIN" - response.headers["Content-Security-Policy"] = "frame-ancestors 'self'" - else: + if not getattr(response, "xframe_options_exempt", False): response.headers["X-Frame-Options"] = "DENY" response.headers["Content-Security-Policy"] = "frame-ancestors 'none'" diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 41ec3200..97f7fbf5 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -76,7 +76,6 @@ INSTALLED_APPS = [ "archivebox.crawls", # handles Crawl and CrawlSchedule models and management (depends on core) "archivebox.progressmonitor", # live progress endpoint and admin monitor template "archivebox.api", # Django-Ninja-based Rest API interfaces, config, APIToken model, etc. - "archivebox.opencode", # 3rd-party apps from PyPI that need to be loaded last "admin_data_views", # handles rendering some convenient automatic read-only views of data in Django admin "django_extensions", # provides Django Debug Toolbar (and other non-debug helpers) diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py index 2d15d582..77c50092 100644 --- a/archivebox/core/templatetags/core_tags.py +++ b/archivebox/core/templatetags/core_tags.py @@ -1,3 +1,4 @@ +import logging import os from html import unescape from pathlib import Path @@ -28,6 +29,20 @@ from archivebox.plugins.discovery import ( register = template.Library() + +@register.simple_tag(takes_context=True) +def plugin_ui(context, plugin: str, slot: str): + """Optional plugin UI must never prevent the host page from rendering.""" + try: + source = get_plugin_template(plugin, slot, fallback=False) + if not source: + return "" + return template.engines["django"].from_string(source).render(context.flatten()) + except Exception: + logging.getLogger(__name__).exception("Unable to render optional plugin UI: %s/%s", plugin, slot) + return "" + + _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log") _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif") _STATIC_URL_SAFE = "/@-._~!$&'()*+,;=" diff --git a/archivebox/opencode/apps.py b/archivebox/opencode/apps.py deleted file mode 100644 index c7f90faa..00000000 --- a/archivebox/opencode/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class OpencodeConfig(AppConfig): - name = "archivebox.opencode" - label = "opencode_plugin" diff --git a/archivebox/opencode/templates/opencode/agent.html b/archivebox/opencode/templates/opencode/agent.html deleted file mode 100644 index 8a252295..00000000 --- a/archivebox/opencode/templates/opencode/agent.html +++ /dev/null @@ -1,211 +0,0 @@ -{% extends "admin/base.html" %} -{% load i18n %} - -{% block title %}Agent{% endblock %} - -{% block breadcrumbs %} -
- {% trans 'Home' %} - › Agent -
-{% endblock %} - -{% block extrastyle %} - {{ block.super }} - -{% endblock %} - -{% block bodyclass %}{{ block.super }} opencode-agent{% endblock %} - -{% block content %} -
- {% if error %} -
- {{ error }} - {% if command %}

{{ command }}{% endif %} -
- {% else %} - -
- - -
- - {% endif %} -
-{% endblock %} diff --git a/archivebox/opencode/views.py b/archivebox/opencode/views.py index 6a728753..46ccdfe7 100644 --- a/archivebox/opencode/views.py +++ b/archivebox/opencode/views.py @@ -1,776 +1,76 @@ -from __future__ import annotations +"""Django adapter for the optional plugin; never import its runtime at startup.""" -import atexit -import base64 import logging -import os -import re -import signal -import shutil -import subprocess -import threading -import time -from pathlib import Path -from typing import Any -from urllib.parse import urljoin, urlsplit -import httpx -import requests -from abx_plugins.plugins import opencode as opencode_plugin +from django.contrib.auth.views import redirect_to_login +from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse +from django.template import engines +from django.views.decorators.csrf import csrf_exempt + from archivebox.config import CONSTANTS from archivebox.config.common import get_config from archivebox.core.routes_util import build_admin_url, get_api_base_url, get_base_url -from asgiref.sync import sync_to_async -from django.http import ( - Http404, - HttpRequest, - HttpResponse, - HttpResponseForbidden, - StreamingHttpResponse, -) -from django.shortcuts import redirect, render -from django.views.decorators.csrf import csrf_exempt +from archivebox.plugins.discovery import get_plugin_template - -_PROCESS: subprocess.Popen | None = None -_PROCESS_READY: subprocess.Popen | None = None -_PROCESS_LOCK = threading.Lock() -_SESSION_LOCK = threading.Lock() _LOGGER = logging.getLogger(__name__) -_PROXY_PREFIX = "/admin/agent/opencode" -_PROXY_PREFIX_REGEX = _PROXY_PREFIX.replace("/", r"\/") -_PROXY_PREFIX_NO_SLASH_REGEX = _PROXY_PREFIX.lstrip("/").replace("/", r"\/") -_CONFIG_PATH = Path(opencode_plugin.__file__).with_name("config.json") -_DEFAULT_MODEL = "opencode/big-pickle" -_DEFAULT_CONFIG = f'''{{ - "$schema": "https://opencode.ai/config.json", - "model": "{_DEFAULT_MODEL}", - "snapshot": false -}} -''' - -_TEXT_CONTENT_TYPES = ( - "text/", - "application/javascript", - "application/x-javascript", -) -_HOP_BY_HOP_HEADERS = { - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", -} -_ARCHIVEBOX_SKILL = """--- -name: archivebox -description: Use ArchiveBox's CLI and local REST API from an ArchiveBox collection. ---- - -You are running inside an ArchiveBox collection directory. - -- ArchiveBox collection directory: {archivebox_data_dir} -- ArchiveBox BASE_URL: {archivebox_base_url} -- ArchiveBox Admin URL: {archivebox_admin_url} -- ArchiveBox REST API URL: {archivebox_api_url} -- Prefer the `archivebox` CLI for authenticated changes, e.g. `archivebox add`, `archivebox schedule`, `archivebox update`, and `archivebox shell`. -- Run ArchiveBox CLI commands from the ArchiveBox collection directory above. -- Get command help with `archivebox list --help`, `archivebox add --help`, `archivebox schedule --help`, etc. Do not use `archivebox help `. -- Use `--depth=0` by default. Only use recursive crawling when the user explicitly asks for it; use `--depth=1` when you need pages one hop out. -- Before any recursive crawl, constrain scope with ArchiveBox config such as `CRAWL_MAX_URLS`, `CRAWL_MAX_SIZE`, `SNAPSHOT_MAX_*`, `URL_ALLOWLIST`, `URL_DENYLIST`, and related limits. -- Respect the configured `archivebox config --get ONLY_NEW` behavior unless the user explicitly says otherwise. Remind users that expected crawl URLs can be skipped when the collection already contains snapshots with the same URL. -- Always audit newly discovered crawl URLs before letting a crawl run broadly. Treat junk URLs such as privacy policies, legal pages, tag archives, sitemap files, feeds, login/logout URLs, and other low-value boilerplate as unwanted unless the user explicitly asked to archive them. -- Always watch crawl output and logs as the crawl progresses, and correct errors early instead of waiting until the crawl finishes. -- If a crawl contains bad URLs, pause it, edit the crawl's `urls` field to remove them, delete any unneeded snapshots already created under that crawl, then resume the crawl. -- Use `archivebox shell -c '...'` or `archivebox shell <<'PY' ... PY` for Django ORM work. Shell Plus prints an import banner first; keep stderr visible while debugging. -- Use full ArchiveBox module paths in shell code: `from archivebox.crawls.models import Crawl, CrawlSchedule` and `from archivebox.core.models import Snapshot, ArchiveResult`. -- If a model/field/relation is unclear, inspect `_meta.fields` before guessing, e.g. `archivebox shell -c "from archivebox.crawls.models import Crawl; print([f.name for f in Crawl._meta.fields])"`. -- Use `archivebox config --get BASE_URL` only to verify the configured base URL; prefer the seeded URLs above for API/admin requests. -- Use `$ARCHIVEBOX_API_URL` for REST API inspection when helpful. Do not assume admin session cookies authenticate API subdomain requests; prefer CLI/shell for authenticated mutations unless the admin provides or asks you to create an API token. -- Discover REST endpoints from `${{ARCHIVEBOX_API_URL}}v1/openapi.json`; crawl endpoints live under `/api/v1/crawls/`, snapshots under `/api/v1/core/`. -- Do not bypass ArchiveBox auth, expose API keys, or modify config unless the admin explicitly asks. -- After creating crawls or snapshots, report the crawl/snapshot IDs and the exact command or API request used. -""" -def _signal_owned_process(process: subprocess.Popen, sig: signal.Signals) -> None: +def _dispatch(request, path=None): try: - os.killpg(process.pid, sig) - except OSError: - try: - process.send_signal(sig) - except ProcessLookupError: - pass + config = dict(get_config().model_dump(mode="json")) + if not config.get("OPENCODE_ENABLED", False): + raise Http404 + if not request.user.is_authenticated: + return redirect_to_login(request.get_full_path(), login_url="/admin/login/") + if not request.user.is_active or not request.user.is_superuser: + return HttpResponseForbidden("Agent access requires a superuser account.") + from abx_plugins.plugins.opencode import runtime -def _stop_owned_process(process: subprocess.Popen | None = None) -> None: - global _PROCESS, _PROCESS_READY - owned_process = process or _PROCESS - if owned_process is None: - return - if owned_process.poll() is None: - _signal_owned_process(owned_process, signal.SIGCONT) - _signal_owned_process(owned_process, signal.SIGTERM) - try: - owned_process.wait(timeout=5) - except subprocess.TimeoutExpired: - _signal_owned_process(owned_process, signal.SIGKILL) - owned_process.wait() - if _PROCESS is owned_process: - _PROCESS = None - if _PROCESS_READY is owned_process: - _PROCESS_READY = None - - -atexit.register(_stop_owned_process) - - -def _machine_config() -> dict[str, Any]: - resolved = get_config() - return dict(resolved.model_dump(mode="json")) - - -def _archivebox_data_dir_default() -> Path: - return Path(CONSTANTS.DATA_DIR) - - -def _archivebox_route_urls(request: HttpRequest, route_config) -> tuple[str, str, str]: - base_url = get_base_url( - request=request, - config=route_config, - ).rstrip("/") - admin_url = build_admin_url( - "/admin/", - request=request, - config=route_config, - ).rstrip("/") - api_url = f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/" - return base_url, admin_url, api_url - - -def _config_value(config: dict, key: str, default): - value = config.get(key, default) - if value in (None, ""): - return default - return value - - -def _opencode_enabled(config: dict) -> bool: - value = config.get("OPENCODE_ENABLED", False) - if isinstance(value, bool): - return value - return str(value).strip().lower() in {"1", "true", "yes", "on"} - - -def _require_enabled(config: dict) -> None: - if not _opencode_enabled(config): - raise Http404 - - -def _require_superuser(request: HttpRequest): - user = getattr(request, "user", None) - if user is None: - return redirect(f"/admin/login/?next={request.get_full_path()}") - if ( - bool(getattr(user, "is_authenticated", False)) - and bool(getattr(user, "is_active", False)) - and bool(getattr(user, "is_superuser", False)) - ): - return None - if bool(getattr(user, "is_authenticated", False)): - return HttpResponseForbidden( - b"ArchiveBox agent access requires a superuser account.", + settings = runtime._settings(config, CONSTANTS.DATA_DIR) + route_config = request.__dict__.get("archivebox_config") + settings.update( + archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"), + archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"), + archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/", ) - return redirect(f"/admin/login/?next={request.get_full_path()}") + if path is None: + from archivebox.core.admin_site import archivebox_admin + context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)} + source = get_plugin_template("opencode", "agent", fallback=False) + if source is None: + raise RuntimeError("Agent template unavailable") + return HttpResponse(engines["django"].from_string(source).render(context, request)) -def _origin_allowed(request: HttpRequest, path: str | None = None) -> bool: - if request.method in {"GET", "HEAD", "OPTIONS", "TRACE"}: - return True - - expected_host = request.get_host() - pty_connect = bool( - path and path.startswith("pty/") and path.endswith("/connect-token"), - ) - if pty_connect: - return True - - origin = _request_header(request, "Origin") - if origin: - return _same_host(origin, expected_host) - - referer = _request_header(request, "Referer") - if referer: - return _same_host(referer, expected_host) - - fetch_site = _request_header(request, "Sec-Fetch-Site") - if fetch_site: - return fetch_site in {"same-origin", "same-site", "none"} - - return False - - -def _same_host(value: str, expected_host: str) -> bool: - parsed = urlsplit(value) - return parsed.scheme in {"http", "https"} and parsed.netloc == expected_host - - -def _settings(config: dict) -> dict: - host = str(_config_value(config, "OPENCODE_HOST", "127.0.0.1")) - port = int(_config_value(config, "OPENCODE_PORT", 4096)) - default_data_dir = _archivebox_data_dir_default() - opencode_dir = Path( - str(_config_value(config, "OPENCODE_STATE_DIR", default_data_dir / "opencode")), - ).expanduser() - workdir = Path( - str(_config_value(config, "OPENCODE_WORKDIR", default_data_dir)), - ).expanduser() - binary = str(_config_value(config, "OPENCODE_BINARY", "opencode")) - timeout = int(_config_value(config, "OPENCODE_TIMEOUT", 120)) - return { - "host": host, - "port": port, - "origin": f"http://{host}:{port}", - "archivebox_data_dir": default_data_dir, - "workdir": workdir, - "opencode_dir": opencode_dir, - "config_home": opencode_dir / "config", - "data_home": opencode_dir / "data", - "state_home": opencode_dir / "state", - "cache_home": opencode_dir / "cache", - "home": opencode_dir / "home", - "binary": binary, - "config": config, - "timeout": timeout, - } - - -def _resolve_binary(binary: str, config: dict) -> tuple[Any, Any, dict[str, str]]: - try: - from abxpkg import BinProvider - from abx_plugins.plugins.base.utils import load_required_binary_from_config - - binary_environ = os.environ.copy() - lib_dir = config.get("ABXPKG_LIB_DIR") - if lib_dir: - binary_environ["ABXPKG_LIB_DIR"] = str(lib_dir) - loaded_dependencies = [ - load_required_binary_from_config( - required_binary, - _CONFIG_PATH, - global_config=config, - environ=binary_environ, - install=False, - ) - for required_binary in ( - str(config.get("NODE_BINARY") or "node"), - str(config.get("GIT_BINARY") or "git"), - binary, - ) - ] - except Exception as err: - raise RuntimeError( - f"OpenCode dependency is not installed from required_binaries: {err}", - ) from err - - if any(not loaded.loaded_abspath for loaded in loaded_dependencies): - raise RuntimeError( - "OpenCode dependency is not installed from required_binaries.", + if not runtime._origin_allowed(request.method, request.get_host(), request.headers): + return HttpResponseForbidden("Cross-origin agent requests are blocked.") + status, headers, body = runtime.proxy( + settings, + request.method, + path, + tuple((key, value) for key, values in request.GET.lists() for value in values), + request.headers, + request.body, ) - - providers = [loaded.loaded_binprovider for loaded in loaded_dependencies if loaded.loaded_binprovider is not None] - binary_env = BinProvider.build_exec_env( - providers=providers, - base_env=binary_environ, - ) - return ( - loaded_dependencies[-1], - loaded_dependencies[1], - binary_env, - ) + response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse + response = response_type(body, status=status, headers=headers) + response.xframe_options_exempt = True + response.headers["X-Frame-Options"] = "SAMEORIGIN" + response.headers["Content-Security-Policy"] = "frame-ancestors 'self'" + return response + except Http404: + raise + except Exception: + # Optional-service boundary, including imports and template rendering. + _LOGGER.exception("Optional AI service failed") + return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain") -def _project_route(workdir: Path, session_id: str = "") -> str: - encoded = base64.b64encode(str(workdir.resolve()).encode()).decode() - encoded = encoded.replace("+", "-").replace("/", "_").rstrip("=") - route = f"{_PROXY_PREFIX}/{encoded}/session" - return f"{route}/{session_id}" if session_id else route - - -def _ensure_project_files(settings: dict) -> None: - workdir = settings["workdir"].resolve() - workdir.mkdir(parents=True, exist_ok=True) - git_marker = workdir / ".git" / "not-a-git" - if git_marker.exists(): - # Current OpenCode hangs on the legacy fake marker, so remove only - # that invalid shape before initializing the real worktree. - shutil.rmtree(git_marker.parent) - - editable_skill_path = settings["opencode_dir"] / "SKILL.md" - editable_skill_path.parent.mkdir(parents=True, exist_ok=True) - if not editable_skill_path.exists(): - editable_skill_path.write_text( - _ARCHIVEBOX_SKILL.format( - archivebox_data_dir=settings["archivebox_data_dir"], - archivebox_base_url=settings.get("archivebox_base_url", ""), - archivebox_admin_url=settings.get("archivebox_admin_url", ""), - archivebox_api_url=settings.get("archivebox_api_url", ""), - ), - ) - - opencode_skill_path = settings["config_home"] / "opencode" / "skills" / "archivebox" / "SKILL.md" - opencode_skill_path.parent.mkdir(parents=True, exist_ok=True) - if opencode_skill_path.resolve() != editable_skill_path.resolve(): - if opencode_skill_path.exists() or opencode_skill_path.is_symlink(): - opencode_skill_path.unlink() - opencode_skill_path.symlink_to(editable_skill_path) - - opencode_config_path = settings["config_home"] / "opencode" / "opencode.jsonc" - if not opencode_config_path.exists(): - opencode_config_path.write_text(_DEFAULT_CONFIG) - - -def _ensure_default_session(settings: dict) -> str: - workdir = settings["workdir"].resolve() - params = {"directory": str(workdir)} - timeout = settings["timeout"] - project = requests.post( - f"{settings['origin']}/project/git/init", - params=params, - timeout=timeout, - ) - project.raise_for_status() - project_data = project.json() - if Path(str(project_data.get("worktree") or "/")).resolve() != workdir: - raise RuntimeError( - f"OpenCode initialized the wrong project worktree: {project_data.get('worktree')!r}", - ) - - sessions = requests.get( - f"{settings['origin']}/session", - params={**params, "roots": "true", "limit": 55}, - timeout=timeout, - ) - sessions.raise_for_status() - session_data = sessions.json() - if not isinstance(session_data, list): - raise RuntimeError("OpenCode returned an invalid project session list.") - for session_data_item in session_data: - if not isinstance(session_data_item, dict): - continue - session_id = str(session_data_item.get("id") or "") - session_directory = session_data_item.get("directory") - if session_id and session_directory and Path(str(session_directory)).resolve() == workdir: - return session_id - - session = requests.post( - f"{settings['origin']}/session", - params=params, - json={}, - timeout=timeout, - ) - session.raise_for_status() - session_data = session.json() - session_id = str(session_data.get("id") or "") - session_directory = session_data.get("directory") - if not session_id or not session_directory or Path(str(session_directory)).resolve() != workdir: - raise RuntimeError( - "OpenCode did not create a session for the requested worktree.", - ) - return session_id - - -def _owned_process_running() -> bool: - process = _PROCESS - return process is not None and process.poll() is None - - -def _owned_process_ready() -> bool: - process = _PROCESS_READY - return process is not None and process is _PROCESS and process.poll() is None - - -def _health(settings: dict, timeout: float = 2) -> bool: - try: - response = requests.get( - f"{settings['origin']}/global/health", - timeout=timeout, - ) - return response.status_code == 200 - except requests.RequestException: - return False - - -def _ensure_opencode(settings: dict) -> tuple[bool, str]: - global _PROCESS, _PROCESS_READY - started_process: subprocess.Popen | None = None - workdir = settings["workdir"].resolve() - - with _PROCESS_LOCK: - if _owned_process_ready(): - return True, "" - if _health(settings): - if _owned_process_running(): - _PROCESS_READY = _PROCESS - return True, "" - if _owned_process_running(): - _stop_owned_process(_PROCESS) - - try: - binary, git_binary, binary_env = _resolve_binary( - settings["binary"], - settings["config"], - ) - except RuntimeError as err: - return False, str(err) - - env = { - **os.environ, - **binary_env, - "ARCHIVEBOX_BASE_URL": str(settings.get("archivebox_base_url", "")), - "ARCHIVEBOX_ADMIN_URL": str(settings.get("archivebox_admin_url", "")), - "ARCHIVEBOX_API_URL": str(settings.get("archivebox_api_url", "")), - "BROWSER": "false", - "GIT_CEILING_DIRECTORIES": str(workdir), - "HOME": str(settings["home"]), - "OPENCODE_DISABLE_PROJECT_CONFIG": "true", - "XDG_CONFIG_HOME": str(settings["config_home"]), - "XDG_DATA_HOME": str(settings["data_home"]), - "XDG_STATE_HOME": str(settings["state_home"]), - "XDG_CACHE_HOME": str(settings["cache_home"]), - } - - settings["workdir"].mkdir(parents=True, exist_ok=True) - settings["config_home"].mkdir(parents=True, exist_ok=True) - settings["data_home"].mkdir(parents=True, exist_ok=True) - settings["state_home"].mkdir(parents=True, exist_ok=True) - settings["cache_home"].mkdir(parents=True, exist_ok=True) - settings["home"].mkdir(parents=True, exist_ok=True) - _ensure_project_files(settings) - - if not (workdir / ".git").exists(): - try: - git_init = git_binary.exec( - cmd=("init", "--quiet"), - cwd=workdir, - env=env, - timeout=settings["timeout"], - ) - except (AssertionError, OSError, subprocess.SubprocessError) as err: - return False, f"OpenCode project initialization failed: {err}" - if git_init.returncode != 0: - output = (git_init.stderr or git_init.stdout or "").strip() - return ( - False, - f"OpenCode project initialization failed: {output or f'git exited with {git_init.returncode}'}", - ) - - if _health(settings): - return True, "" - - binary_abspath = binary.loaded_abspath - if binary.loaded_binprovider is not None: - binary_abspath = binary.loaded_binprovider._exec_bin_abspath( - Path(binary.loaded_abspath), - ) - cmd = [ - str(binary_abspath), - "serve", - "--hostname", - settings["host"], - "--port", - str(settings["port"]), - ] - try: - _PROCESS = subprocess.Popen( - cmd, - cwd=workdir, - env=env, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - start_new_session=True, - ) - _PROCESS_READY = None - started_process = _PROCESS - except FileNotFoundError: - return False, f"OpenCode binary not found: {settings['binary']}" - - deadline = time.monotonic() + settings["timeout"] - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - if _health(settings, timeout=min(2, remaining)): - if time.monotonic() <= deadline: - _PROCESS_READY = started_process - return True, "" - break - if started_process and started_process.poll() is not None: - if _PROCESS is started_process: - _PROCESS = None - if _PROCESS_READY is started_process: - _PROCESS_READY = None - return False, "OpenCode exited before the web server became ready." - remaining = deadline - time.monotonic() - if remaining > 0: - time.sleep(min(0.25, remaining)) - - _stop_owned_process(started_process) - return False, "Timed out waiting for OpenCode to start." - - -def agent_view(request: HttpRequest): - config = _machine_config() - _require_enabled(config) - auth_response = _require_superuser(request) - if auth_response: - return auth_response - - settings = _settings(config) - route_config = request.__dict__.get("archivebox_config") - base_url, admin_url, api_url = _archivebox_route_urls(request, route_config) - settings["archivebox_base_url"] = base_url - settings["archivebox_admin_url"] = admin_url - settings["archivebox_api_url"] = api_url - ok, error = _ensure_opencode(settings) - recent_session_id = "" - if ok: - try: - with _SESSION_LOCK: - recent_session_id = _ensure_default_session(settings) - except (requests.RequestException, RuntimeError, ValueError) as err: - if isinstance(err, requests.RequestException) and _owned_process_ready(): - _stop_owned_process() - ok = False - error = f"OpenCode project initialization failed: {err}" - from archivebox.core.admin_site import archivebox_admin - - context = { - **archivebox_admin.each_context(request), - "title": "Agent", - "error": "" if ok else error, - "command": f"{settings['binary']} serve --hostname {settings['host']} --port {settings['port']}" if error else "", - # OpenCode 1.17+ keeps durable sessions on the explicit - # //session/ route. We still seed localStorage - # below because the sidebar state uses it, but the iframe itself must - # open the durable session URL so a fresh browser does not land on the - # transient new-session route and appear to have lost prior sessions. - "proxy_url": _project_route(settings["workdir"], recent_session_id), - "proxy_prefix": _PROXY_PREFIX, - "workdir": str(settings["workdir"].resolve()), - "recent_session_id": recent_session_id, - } - return render( - request, - "opencode/agent.html", - context, - status=200 if ok else 502, - ) - - -def _proxy_url(settings: dict, path: str | None) -> str: - rel = "/" if not path else f"/{path}" - return urljoin(settings["origin"], rel) - - -def _request_header(request: HttpRequest, name: str) -> str | None: - meta = getattr(request, "META", {}) - if name == "Content-Type": - value = meta.get("CONTENT_TYPE") - elif name == "Content-Length": - value = meta.get("CONTENT_LENGTH") - else: - value = meta.get(f"HTTP_{name.upper().replace('-', '_')}") - return str(value) if value else None - - -def _request_headers(request: HttpRequest, settings: dict) -> dict[str, str]: - forwarded = {} - for key in ("Accept", "Accept-Language", "Content-Type", "Range", "User-Agent"): - value = _request_header(request, key) - if value: - forwarded[key] = value - return forwarded - - -def _request_params(request: HttpRequest) -> tuple[tuple[str, str], ...]: - if hasattr(request.GET, "lists"): - return tuple((key, str(value)) for key, values in request.GET.lists() for value in values) - return tuple((key, str(value)) for key, value in dict(request.GET).items()) - - -async def _event_chunks( - settings: dict, - path: str | None, - method: str, - params: tuple[tuple[str, str], ...], - headers: dict[str, str], -): - if not _owned_process_ready(): - ok, error = await sync_to_async(_ensure_opencode, thread_sensitive=False)(settings) - if not ok: - _LOGGER.warning("OpenCode event stream unavailable: %s", error) - yield b'event: error\ndata: {"error":"OpenCode upstream unavailable"}\n\n' - return - - timeout = httpx.Timeout(settings["timeout"], read=None) - url = _proxy_url(settings, path) - try: - async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client: - async with client.stream( - method, - url, - params=params, - headers=headers, - ) as upstream: - async for chunk in upstream.aiter_raw(chunk_size=512): - yield chunk - except httpx.RequestError as err: - _LOGGER.warning("OpenCode event stream ended: %s", err) - - -def _rewrite_text(body: bytes, settings: dict) -> bytes: - text = body.decode("utf-8", errors="replace") - text = text.replace(settings["origin"], _PROXY_PREFIX) - text = text.replace("location.origin", f'location.origin+"{_PROXY_PREFIX}"') - text = text.replace( - "k(k5,{get component(){return t.router??Az},", - f'k(k5,{{base:"{_PROXY_PREFIX}",get component(){{return t.router??Az}},', - ) - text = text.replace('"/assets/', f'"{_PROXY_PREFIX}/assets/') - text = text.replace("'/assets/", f"'{_PROXY_PREFIX}/assets/") - proxy_path = rf'(\1.replace(/^{_PROXY_PREFIX_REGEX}(?=\/|$)/,"")||"/")' - text = re.sub(r"\b(window\.location\.pathname)\b", proxy_path, text) - text = re.sub(r"(?\b(?:href|src|action)=["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""", - rf"\g{_PROXY_PREFIX}/", - text, - ) - text = re.sub( - rf"""(?P\b(?:fetch|EventSource)\(["'])/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""", - rf"\g{_PROXY_PREFIX}/", - text, - ) - text = re.sub( - rf"""(?P\burl\(["']?)/(?!{_PROXY_PREFIX_NO_SLASH_REGEX}(?:/|$))""", - rf"\g{_PROXY_PREFIX}/", - text, - ) - return text.encode("utf-8") - - -def _response_headers(upstream: requests.Response, settings: dict) -> dict[str, str]: - headers = {} - for key, value in upstream.headers.items(): - lower = key.lower() - if lower in _HOP_BY_HOP_HEADERS or lower in { - "content-length", - "content-encoding", - "x-frame-options", - }: - continue - if lower == "location": - if value.startswith(settings["origin"]): - value = value.replace(settings["origin"], _PROXY_PREFIX, 1) - elif value.startswith("/"): - value = f"{_PROXY_PREFIX}{value}" - headers[key] = value - return headers - - -def _proxy_error_response(error: Exception | str) -> HttpResponse: - _LOGGER.warning("OpenCode upstream request failed: %s", error) - return HttpResponse( - b"OpenCode upstream request failed.", - status=502, - content_type="text/plain; charset=utf-8", - ) +def agent_view(request): + return _dispatch(request) @csrf_exempt -def opencode_proxy_view(request: HttpRequest, path: str | None = None): - config = _machine_config() - _require_enabled(config) - auth_response = _require_superuser(request) - if auth_response: - return auth_response - if not _origin_allowed(request, path): - return HttpResponseForbidden( - b"Cross-origin OpenCode agent requests are blocked.", - ) - - settings = _settings(config) - route_config = request.__dict__.get("archivebox_config") - base_url, admin_url, api_url = _archivebox_route_urls(request, route_config) - settings["archivebox_base_url"] = base_url - settings["archivebox_admin_url"] = admin_url - settings["archivebox_api_url"] = api_url - - if request.method == "GET" and (path or "").endswith("/event"): - response = StreamingHttpResponse( - _event_chunks( - settings, - path, - request.method or "GET", - _request_params(request), - _request_headers(request, settings), - ), - content_type="text/event-stream", - ) - response.headers["Cache-Control"] = "no-store" - response.headers["X-Accel-Buffering"] = "no" - return response - - if path == "global/health" or not _owned_process_ready(): - ok, error = _ensure_opencode(settings) - if not ok: - return _proxy_error_response(error) - - try: - method = request.method or "GET" - upstream = requests.request( - method, - _proxy_url(settings, path), - params=_request_params(request), - data=request.body if method not in {"GET", "HEAD"} else None, - headers=_request_headers(request, settings), - stream=True, - timeout=settings["timeout"], - allow_redirects=False, - ) - except requests.RequestException as err: - return _proxy_error_response(err) - - try: - body = upstream.content - except requests.RequestException as err: - upstream.close() - return _proxy_error_response(err) - - content_type = upstream.headers.get("Content-Type", "") - headers = _response_headers(upstream, settings) - if any(content_type.startswith(prefix) for prefix in _TEXT_CONTENT_TYPES): - body = _rewrite_text(body, settings) - response = HttpResponse( - body, - status=upstream.status_code, - content_type=content_type or "application/octet-stream", - ) - for key, value in headers.items(): - response.headers[key] = value - response.headers["Cache-Control"] = "no-store" - return response +def opencode_proxy_view(request, path=None): + return _dispatch(request, path=path or "") diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html index 42aa2c37..8db954b5 100644 --- a/archivebox/templates/core/add.html +++ b/archivebox/templates/core/add.html @@ -1,6 +1,6 @@ {% extends "core/base.html" %} -{% load static %} +{% load static core_tags %} {% load i18n %} {% block breadcrumbs %} @@ -48,9 +48,7 @@

- {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %} - 💬 Crawl with AI   |   - {% endif %} + {% plugin_ui "opencode" "add" %} Chrome Extension  Get the extension 💡 Tip: Instantly save a single URL by visiting: diff --git a/archivebox/templates/core/navigation.html b/archivebox/templates/core/navigation.html index 96fd8ec0..cddb4439 100644 --- a/archivebox/templates/core/navigation.html +++ b/archivebox/templates/core/navigation.html @@ -1,4 +1,4 @@ -{% load i18n static %} +{% load i18n static core_tags %}

Add ➕ @@ -11,10 +11,7 @@ Tags - {% if user.is_authenticated and user.is_superuser and request.archivebox_config.OPENCODE_ENABLED %} - 💬 AI - - {% endif %} + {% plugin_ui "opencode" "navigation" %} Docs API diff --git a/archivebox/tests/test_opencode_agent.py b/archivebox/tests/test_opencode_agent.py index af0ef60c..9e6ea78b 100644 --- a/archivebox/tests/test_opencode_agent.py +++ b/archivebox/tests/test_opencode_agent.py @@ -1,19 +1,18 @@ import asyncio -import json import os -import signal +import shutil import socket -import subprocess from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace -from urllib.parse import quote +from urllib.parse import parse_qs, quote, urlsplit import pytest import requests from asgiref.testing import ApplicationCommunicator from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd +from archivebox.config.common import get_config pytestmark = pytest.mark.django_db(transaction=True) @@ -78,8 +77,8 @@ def opencode_archive_config(initialized_archive): @pytest.fixture -def live_opencode(opencode_archive_config): - from archivebox.opencode import views +def installed_opencode(opencode_archive_config): + from abx_plugins.plugins.opencode import runtime install = run_archivebox_cmd( ["install", "opencode", "--binproviders=env,pnpm"], @@ -90,56 +89,61 @@ def live_opencode(opencode_archive_config): assert install.returncode == 0, install.stderr or install.stdout _reset_runtime_config() - config = views._machine_config() - settings = views._settings(config) + config = get_config().model_dump(mode="json") + settings = runtime._settings(config, opencode_archive_config.data_dir) settings["archivebox_base_url"] = "http://admin.archivebox.localhost:8000" settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:8000/admin" settings["archivebox_api_url"] = "http://admin.archivebox.localhost:8000/api/" - binary, _, binary_env = views._resolve_binary(settings["binary"], settings["config"]) + binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"]) version = binary.exec( cmd=("--version",), env={**os.environ, **binary_env}, timeout=120, ) assert version.returncode == 0, version.stderr or version.stdout - ok, error = views._ensure_opencode(settings) + return SimpleNamespace(config=opencode_archive_config, settings=settings) + + +@pytest.fixture +def live_opencode(installed_opencode): + from abx_plugins.plugins.opencode import runtime + + settings = installed_opencode.settings + ok, error = runtime._ensure_opencode(settings) assert ok, error - process = views._PROCESS + process = runtime._PROCESS assert process is not None try: - yield SimpleNamespace(config=opencode_archive_config, settings=settings, process=process) + yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process) finally: - views._stop_owned_process() + runtime._stop_owned_process() def test_opencode_disabled_route_does_not_start_server(client, initialized_archive): from archivebox.machine.models import Machine - from archivebox.opencode import views + from abx_plugins.plugins.opencode import runtime os.chdir(initialized_archive) Machine.from_json({"config": {"OPENCODE_ENABLED": False}}) _reset_runtime_config() - assert views._machine_config()["OPENCODE_ENABLED"] is False + assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST) assert response.status_code == 404 - assert views._PROCESS is None or views._PROCESS.poll() is not None + assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None -def test_stop_owned_process_falls_back_for_stopped_process_without_dedicated_group(): - from archivebox.opencode import views +def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive): + _set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False") - process = subprocess.Popen(["sleep", "60"]) - try: - process.send_signal(signal.SIGSTOP) - views._stop_owned_process(process) - assert process.returncode == -signal.SIGTERM - finally: - if process.poll() is None: - process.kill() - process.wait() + assert get_config().OPENCODE_ENABLED is False + assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404 + for path in ("/add/", "/admin/core/snapshot/"): + response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST) + assert response.status_code == 200 + assert b'href="/admin/agent"' not in response.content def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode): @@ -147,6 +151,11 @@ def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_ assert response.status_code == 302 assert "/admin/login/" in response.headers["Location"] + next_path = "/admin/agent?x=1&next=https://example.com" + response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST) + assert response.status_code == 302 + assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]} + user = django_user_model.objects.create_user(username="regular", password="testpassword") client.force_login(user) response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST) @@ -178,18 +187,22 @@ def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_ def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode): - from archivebox.opencode import views + from abx_plugins.plugins.opencode import runtime response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST) recent_session_id = response.context["recent_session_id"] - session_path = views._project_route(live_opencode.config.data_dir, recent_session_id) + session_path = runtime._project_route(live_opencode.config.data_dir, recent_session_id) assert response.status_code == 200 assert recent_session_id assert f'