diff --git a/Dockerfile b/Dockerfile index 34935351..5882e841 100644 --- a/Dockerfile +++ b/Dockerfile @@ -309,6 +309,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \ echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \ && apt-get update -qq \ + && if [ "$TARGETARCH" = "arm64" ]; then \ + abxpkg install --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \ + fi \ && PUID=0 PGID=0 abx-dl plugins --install \ && find "$LIB_DIR" "$DATA_DIR"/personas -type d -name __pycache__ -prune -exec rm -rf {} + \ && find "$LIB_DIR" "$DATA_DIR"/personas -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \ diff --git a/archivebox/api/admin.py b/archivebox/api/admin.py index 1a71f88c..76586165 100644 --- a/archivebox/api/admin.py +++ b/archivebox/api/admin.py @@ -10,6 +10,11 @@ from archivebox.base_models.admin import BaseModelAdmin from archivebox.api.models import APIToken +def _webhook_fields(*names: str) -> tuple[str, ...]: + model_fields = {field.name for field in get_webhook_model()._meta.fields} + return tuple(name for name in names if name in model_fields) + + class APITokenAdmin(BaseModelAdmin): list_display = ("created_at", "id", "created_by", "token_redacted", "expires") sort_fields = ("id", "created_at", "created_by", "expires") @@ -47,42 +52,42 @@ class APITokenAdmin(BaseModelAdmin): class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin): list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display) - sort_fields = ("created_at", "created_by", "id", "referenced_model", "endpoint", "last_success", "last_error") - readonly_fields = ("created_at", "modified_at", *WebhookAdmin.readonly_fields) + sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure") + readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields) fieldsets = ( ( "Webhook", { - "fields": ("name", "signal", "referenced_model", "endpoint"), + "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"), "classes": ("card", "wide"), }, ), ( "Authentication", { - "fields": ("auth_token",), + "fields": _webhook_fields("auth_token"), "classes": ("card",), }, ), ( "Status", { - "fields": ("enabled", "last_success", "last_error"), + "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"), "classes": ("card",), }, ), ( "Owner", { - "fields": ("created_by",), + "fields": _webhook_fields("created_by"), "classes": ("card",), }, ), ( "Timestamps", { - "fields": ("created_at", "modified_at"), + "fields": _webhook_fields("created_at", "modified_at"), "classes": ("card",), }, ), diff --git a/archivebox/api/v1_core.py b/archivebox/api/v1_core.py index 731a9afa..ec512906 100644 --- a/archivebox/api/v1_core.py +++ b/archivebox/api/v1_core.py @@ -859,7 +859,6 @@ def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], wi @router.get("/snapshots.rss", url_name="get_snapshots_rss") -@router.get("/snapshot.rss", url_name="get_snapshot_rss") def get_snapshots_rss( request: HttpRequest, crawl_id: str = "", diff --git a/archivebox/api/webhooks.py b/archivebox/api/webhooks.py new file mode 100644 index 00000000..78e5ec33 --- /dev/null +++ b/archivebox/api/webhooks.py @@ -0,0 +1,32 @@ +__package__ = "archivebox.api" + +from typing import Any +from collections.abc import Callable +import logging + +from django.db import transaction +from signal_webhooks.handlers import sync_task_handler + + +logger = logging.getLogger(__name__) + + +def warning_error_handler(hook: Any, error: Exception | None) -> None: + if error is not None: + logger.warning("Outbound webhook %r failed: %s", hook.name, error) + return + + logger.warning("Outbound webhook %r returned a non-success response.", hook.name) + + +def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None: + def run_webhook() -> None: + try: + sync_task_handler(hook, **kwargs) + except Exception: + logger.warning("Outbound webhook failed after transaction commit.", exc_info=True) + + try: + transaction.on_commit(run_webhook) + except Exception: + logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True) diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index bb875ff7..cfeffa79 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -252,6 +252,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): save_on_top = True show_full_result_count = False + def change_view(self, request, object_id, form_url="", extra_context=None): + request.archivebox_config = get_config() + extra_context = extra_context or {} + extra_context["CONFIG"] = request.archivebox_config + return super().change_view(request, object_id, form_url, extra_context | GLOBAL_CONTEXT) + def changelist_view(self, request, extra_context=None): self.request = request request.archivebox_config = get_config() @@ -928,7 +934,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): def _get_expected_hook_total(self, obj) -> int: try: request = getattr(self, "request", None) - if getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_changelist": + if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}: return 0 crawl = getattr(obj, "crawl", None) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 1c9cb027..9a80f60b 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -4,7 +4,6 @@ import os import sys import inspect import importlib -from typing import Any, cast from pathlib import Path @@ -440,21 +439,23 @@ LOGGING = SETTINGS_LOGGING # Add default webhook configuration to the User model SIGNAL_WEBHOOKS_CUSTOM_MODEL = "archivebox.api.models.OutboundWebhook" SIGNAL_WEBHOOKS: dict[str, object] = { + "TASK_HANDLER": "archivebox.api.webhooks.transaction_on_commit_task_handler", + "ERROR_HANDLER": "archivebox.api.webhooks.warning_error_handler", "HOOKS": { # ... is a special sigil value that means "use the default autogenerated hooks" "django.contrib.auth.models.User": ..., + "archivebox.crawls.models.Crawl": ..., "archivebox.core.models.Snapshot": ..., "archivebox.core.models.ArchiveResult": ..., "archivebox.core.models.Tag": ..., "archivebox.api.models.APIToken": ..., + "archivebox.personas.models.Persona": ..., + "archivebox.machine.models.Machine": ..., + "archivebox.machine.models.Binary": ..., + "archivebox.machine.models.Process": ..., }, } -# Avoid background threads touching sqlite connections (especially during tests/migrations). -default_database = cast(dict[str, Any], DATABASES["default"]) -if str(default_database["ENGINE"]).endswith("sqlite3"): - SIGNAL_WEBHOOKS["TASK_HANDLER"] = "signal_webhooks.handlers.sync_task_handler" - ################################################################################ ### Admin Data View Settings ################################################################################ diff --git a/etc/package.json b/etc/package.json index c110ab8f..f6efe7b1 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.31rc40", + "version": "0.9.31rc41", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index c65264aa..5230e5b1 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.31rc40" +version = "0.9.31rc41" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus>=2.5.4", # EventBus API - "abxpkg>=1.10.30", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.10.95", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.10.95", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.10.31", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.10.96", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.10.96", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]