mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
release: archivebox 0.9.33rc45
This commit is contained in:
Nick Sweeting
parent
c3bd4a0637
commit
383e4b5c6e
@ -61,6 +61,14 @@ class CrawlSchema(Schema):
|
||||
username = getattr(user, "username", None)
|
||||
return username if isinstance(username, str) else str(user)
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses can never leak the raw
|
||||
# token/secret/api-key that the operator stored in Crawl.config.
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return redact_sensitive_config(obj.config)
|
||||
|
||||
@staticmethod
|
||||
def resolve_snapshots(obj, context):
|
||||
if bool(getattr(context["request"], "with_snapshots", False)):
|
||||
|
||||
@ -51,6 +51,14 @@ class PersonaSchema(Schema):
|
||||
def resolve_created_by_username(obj) -> str:
|
||||
return obj.created_by.username
|
||||
|
||||
@staticmethod
|
||||
def resolve_config(obj):
|
||||
# Redact credential values so REST responses don't leak the raw
|
||||
# token/secret/api-key the operator stored in Persona.config.
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return redact_sensitive_config(obj.config)
|
||||
|
||||
|
||||
class PersonaSyncResponseSchema(Schema):
|
||||
success: bool
|
||||
|
||||
@ -722,12 +722,34 @@ class KeyValueWidget(forms.Widget):
|
||||
return mark_safe(html)
|
||||
|
||||
def _render_row(self, widget_id: str, key: str, value: str) -> str:
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
# Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
|
||||
# rendered write-only: the input is a password field with a placeholder
|
||||
# showing the value is set, but the raw value is NEVER sent to the browser.
|
||||
# When the user submits the form with the field left blank, the
|
||||
# ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
|
||||
# value so leaving it untouched is a no-op rather than a destructive clear.
|
||||
is_sensitive = is_sensitive_config_key(key)
|
||||
has_value = bool(value)
|
||||
if is_sensitive:
|
||||
input_type = "password"
|
||||
rendered_value = ""
|
||||
placeholder = (
|
||||
"•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
|
||||
)
|
||||
extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
|
||||
else:
|
||||
input_type = "text"
|
||||
rendered_value = self._escape(value)
|
||||
placeholder = "value"
|
||||
extra_attrs = ""
|
||||
return f'''
|
||||
<div class="key-value-row" style="margin-bottom: 6px;">
|
||||
<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
|
||||
<input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
|
||||
style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
|
||||
<input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
|
||||
<input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
|
||||
style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
|
||||
<datalist class="kv-value-options"></datalist>
|
||||
<button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
|
||||
@ -771,6 +793,37 @@ class ConfigEditorMixin(admin.ModelAdmin):
|
||||
kwargs["widget"] = KeyValueWidget()
|
||||
return super().formfield_for_dbfield(db_field, request, **kwargs)
|
||||
|
||||
def save_model(self, request: HttpRequest, obj, form, change):
|
||||
"""Preserve write-only redacted credentials on save.
|
||||
|
||||
The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
|
||||
``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
|
||||
the real value never leaves the server. On submit, an empty value
|
||||
for a sensitive key that was previously set means "leave untouched",
|
||||
not "clear it." We honor that here by re-merging the stored value
|
||||
before the row is written. Explicitly removing the row in the UI
|
||||
still clears it (the key is gone from the submitted JSON, so there's
|
||||
nothing to merge over).
|
||||
"""
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
if change and obj.pk and getattr(obj, "config", None) is not None:
|
||||
try:
|
||||
stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
|
||||
except Exception:
|
||||
stored = {}
|
||||
if isinstance(stored, dict):
|
||||
new_config = dict(obj.config or {})
|
||||
for key, new_value in list(new_config.items()):
|
||||
if not is_sensitive_config_key(key):
|
||||
continue
|
||||
if new_value not in (None, "") and new_value != "********":
|
||||
continue
|
||||
if key in stored:
|
||||
new_config[key] = stored[key]
|
||||
obj.config = new_config
|
||||
super().save_model(request, obj, form, change)
|
||||
|
||||
|
||||
class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
|
||||
list_display = ("id", "created_at", "created_by")
|
||||
|
||||
@ -111,7 +111,7 @@ def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
|
||||
return host, port
|
||||
|
||||
|
||||
def _print_server_startup_warnings(config, host: str, *, base_url_explicit: bool) -> None:
|
||||
def _print_server_startup_warnings(config, host: str, port: str) -> None:
|
||||
"""Print startup-time security / routing warnings for the server command.
|
||||
|
||||
Runs only from ``archivebox server`` so other entry points (manage shell,
|
||||
@ -130,7 +130,32 @@ def _print_server_startup_warnings(config, host: str, *, base_url_explicit: bool
|
||||
)
|
||||
print()
|
||||
|
||||
if base_url_explicit:
|
||||
# ``config.BASE_URL`` is the merged value (env > Machine.config > file >
|
||||
# default), which is what the running server will actually use. Earlier we
|
||||
# gated the "BASE_URL not set" warning on ``os.environ["BASE_URL"]`` alone,
|
||||
# which fired noisily when the user pinned BASE_URL via Machine.config /
|
||||
# ArchiveBox.conf instead of via env.
|
||||
base_url = (config.BASE_URL or "").strip()
|
||||
if base_url:
|
||||
# BASE_URL is pinned. The only thing left to surface is a port
|
||||
# mismatch — bind port ≠ BASE_URL's explicit port usually means the
|
||||
# operator started the server with the wrong ``archivebox server PORT``
|
||||
# argument (or forgot to update one side after moving the listener).
|
||||
# A reverse-proxy setup typically omits the port in BASE_URL
|
||||
# (``https://archive.example.com``), so we only warn when BASE_URL
|
||||
# carries an explicit port — otherwise we'd nag every proxy deployment.
|
||||
from urllib.parse import urlparse
|
||||
|
||||
try:
|
||||
base_port = urlparse(base_url).port
|
||||
except (ValueError, TypeError):
|
||||
base_port = None
|
||||
if base_port is not None and str(base_port) != str(port):
|
||||
print(
|
||||
f"[yellow][!] BASE_URL ({base_url}) port {base_port} does not match the port the server is running on ({port}). "
|
||||
"Make sure this is intentional![/yellow]",
|
||||
)
|
||||
print()
|
||||
return
|
||||
|
||||
# If the user is upgrading from 0.7.3 and already had
|
||||
@ -273,9 +298,10 @@ def server(
|
||||
return
|
||||
|
||||
os.environ["BIND_ADDR"] = f"{host}:{port}"
|
||||
from archivebox.core.host_utils import build_admin_url
|
||||
from archivebox.core.host_utils import get_base_url
|
||||
|
||||
admin_url = build_admin_url("/admin/")
|
||||
base_url = get_base_url().rstrip("/")
|
||||
admin_url = f"{base_url}/admin/"
|
||||
|
||||
from archivebox.workers.supervisord_util import (
|
||||
active_supervisord_runtime_components,
|
||||
@ -298,10 +324,10 @@ def server(
|
||||
else:
|
||||
print("[green][+] Starting ArchiveBox webserver...[/green]")
|
||||
print(
|
||||
f" [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
|
||||
f" [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
|
||||
)
|
||||
print(
|
||||
f" [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
|
||||
f" [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
|
||||
)
|
||||
print(" > Writing ArchiveBox error log to ./logs/errors.log")
|
||||
print()
|
||||
@ -309,11 +335,7 @@ def server(
|
||||
# Reload config after we've set os.environ["BIND_ADDR"] above so the
|
||||
# security-mode + base-url warnings see the effective values.
|
||||
runtime_config = get_config()
|
||||
_print_server_startup_warnings(
|
||||
runtime_config,
|
||||
host,
|
||||
base_url_explicit=bool(os.environ.get("BASE_URL", "").strip()),
|
||||
)
|
||||
_print_server_startup_warnings(runtime_config, host, port)
|
||||
bind_url = f"http://{host}:{port}"
|
||||
command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
|
||||
|
||||
|
||||
@ -1,36 +1,235 @@
|
||||
__package__ = "archivebox.config"
|
||||
|
||||
import io
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from archivebox.config.constants import CONSTANTS
|
||||
from archivebox.config.configset import CaseConfigParser
|
||||
from archivebox.misc.logging import AttrDict
|
||||
|
||||
|
||||
CONFIG_FILE_HEADER = (
|
||||
"# This is the config file for your ArchiveBox collection.\n"
|
||||
"#\n"
|
||||
"# You can add options here manually in INI format, or automatically by running:\n"
|
||||
"# archivebox config --set KEY=VALUE\n"
|
||||
"#\n"
|
||||
"# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
|
||||
"# editing either side propagates to the other. ``archivebox init`` reads\n"
|
||||
"# this file on startup; the admin Machine.config editor writes both.\n"
|
||||
"#\n"
|
||||
"# Full reference:\n"
|
||||
"# https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration\n"
|
||||
"\n"
|
||||
)
|
||||
|
||||
|
||||
# Recursion guard for the bidirectional file<->DB mirror. Bidirectional sync
|
||||
# means a write on one side always triggers an update on the other; without
|
||||
# this flag, ``Machine.save -> mirror to file -> write_config_file -> mirror
|
||||
# back to Machine -> Machine.save -> ...`` would loop forever. Module-level
|
||||
# scalar is fine here — daphne handles each request on its own thread and the
|
||||
# admin/CLI write paths are inherently serialized through the DB.
|
||||
_MIRROR_IN_PROGRESS: bool = False
|
||||
# One-time-per-process startup sync, gated so subsequent ``Machine.current()``
|
||||
# calls collapse to a single boolean check.
|
||||
_INITIAL_SYNC_DONE: bool = False
|
||||
|
||||
|
||||
def _coerce_to_str_dict(config: Any) -> dict[str, str]:
|
||||
"""Project an arbitrary config payload to flat ``{UPPER_KEY: str}`` form.
|
||||
|
||||
INI files only round-trip strings, so we normalize Machine.config values
|
||||
to strings on the way out and accept the same shape on the way back.
|
||||
Pydantic re-coerces types at read time inside ``get_config``.
|
||||
"""
|
||||
if not config:
|
||||
return {}
|
||||
if hasattr(config, "items"):
|
||||
return {str(key).upper(): "" if value is None else str(value) for key, value in config.items()}
|
||||
return {}
|
||||
|
||||
|
||||
def _load_file_config_dict() -> tuple[dict[str, str], float | None]:
|
||||
"""Return ``(flat_dict, mtime)`` for ``ArchiveBox.conf`` (``({}, None)`` if missing)."""
|
||||
config_path = CONSTANTS.CONFIG_FILE
|
||||
try:
|
||||
mtime = config_path.stat().st_mtime
|
||||
except FileNotFoundError:
|
||||
return {}, None
|
||||
parser = CaseConfigParser()
|
||||
parser.read(config_path)
|
||||
flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
|
||||
return flat, mtime
|
||||
|
||||
|
||||
def _resolve_section_for_key(key: str, config_sections, plugin_configs) -> str:
|
||||
for section in config_sections.values():
|
||||
if key in type(section).model_fields:
|
||||
return section.toml_section_header
|
||||
for schema in plugin_configs.values():
|
||||
if "properties" in schema and key in schema["properties"]:
|
||||
return "PLUGINS"
|
||||
# Unknown / user-defined keys land in SERVER_CONFIG so we never lose them
|
||||
# (the previous code raised here, which was fine for the CLI-only path but
|
||||
# would break the mirror as soon as anyone added a plugin-tunable that
|
||||
# this process hasn't loaded a schema for).
|
||||
return "SERVER_CONFIG"
|
||||
|
||||
|
||||
def _render_config_file_content(config: dict[str, str]) -> str:
|
||||
"""Render a flat config dict to INI text, grouped by inferred section."""
|
||||
from archivebox.config.common import get_all_configs
|
||||
from archivebox.hooks import discover_plugin_configs
|
||||
|
||||
config_sections = get_all_configs()
|
||||
plugin_configs = discover_plugin_configs()
|
||||
|
||||
parser = CaseConfigParser()
|
||||
for key, val in sorted(config.items()):
|
||||
section = _resolve_section_for_key(key, config_sections, plugin_configs)
|
||||
if section not in parser:
|
||||
parser[section] = {}
|
||||
parser[section][key] = "" if val is None else str(val)
|
||||
|
||||
buf = io.StringIO()
|
||||
buf.write(CONFIG_FILE_HEADER)
|
||||
parser.write(buf)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _write_file_if_changed(content: str) -> bool:
|
||||
"""Atomic-write ``ArchiveBox.conf`` only when contents actually differ.
|
||||
|
||||
Skipping unchanged writes is the difference between every Machine.save in
|
||||
a hot autodetection loop costing one disk write vs. zero.
|
||||
"""
|
||||
from archivebox.misc.system import atomic_write
|
||||
|
||||
config_path = CONSTANTS.CONFIG_FILE
|
||||
try:
|
||||
existing = config_path.read_text(encoding="utf-8") if config_path.exists() else None
|
||||
except OSError:
|
||||
existing = None
|
||||
if existing == content:
|
||||
return False
|
||||
atomic_write(config_path, content)
|
||||
return True
|
||||
|
||||
|
||||
def mirror_machine_config_to_file(config: Any) -> None:
|
||||
"""Rewrite ``ArchiveBox.conf`` so it mirrors ``Machine.config`` exactly.
|
||||
|
||||
Called from ``Machine.save`` after the row is committed. Recursion-guarded
|
||||
so the matching write_config_file -> Machine.save bounce doesn't loop.
|
||||
"""
|
||||
global _MIRROR_IN_PROGRESS
|
||||
if _MIRROR_IN_PROGRESS:
|
||||
return
|
||||
_MIRROR_IN_PROGRESS = True
|
||||
try:
|
||||
flat = _coerce_to_str_dict(config)
|
||||
_write_file_if_changed(_render_config_file_content(flat))
|
||||
finally:
|
||||
_MIRROR_IN_PROGRESS = False
|
||||
|
||||
|
||||
def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None:
|
||||
"""Copy ``ArchiveBox.conf`` contents into ``Machine.config``.
|
||||
|
||||
Internal helper used by ``write_config_file`` and the startup sync —
|
||||
callers must hold the ``_MIRROR_IN_PROGRESS`` guard around it.
|
||||
"""
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
machine = Machine.current()
|
||||
if _coerce_to_str_dict(machine.config) == file_config:
|
||||
return
|
||||
machine.config = dict(file_config)
|
||||
machine.save(update_fields=["config", "modified_at"])
|
||||
|
||||
|
||||
def sync_machine_and_file(machine: Any = None) -> None:
|
||||
"""One-time-per-process reconciliation between the two stores.
|
||||
|
||||
Cheap on the common case where they already agree (single ``stat`` + dict
|
||||
compare ≈ 1ms). When the two sides diverge we merge them: each side's
|
||||
unique keys are preserved, and for keys present on both we let the newer
|
||||
side win (file mtime vs. ``Machine.modified_at``). After the merge both
|
||||
stores hold the union, so every subsequent write keeps them in lockstep
|
||||
via the full-replace mirror functions.
|
||||
|
||||
Pass ``machine`` when the caller already has a current ``Machine``
|
||||
instance (e.g. from ``Machine.current()``) to skip the 10–15ms
|
||||
``get_host_guid()`` round-trip on the cold path.
|
||||
"""
|
||||
global _INITIAL_SYNC_DONE, _MIRROR_IN_PROGRESS
|
||||
if _INITIAL_SYNC_DONE:
|
||||
return
|
||||
_INITIAL_SYNC_DONE = True
|
||||
if _MIRROR_IN_PROGRESS:
|
||||
return
|
||||
_MIRROR_IN_PROGRESS = True
|
||||
try:
|
||||
if machine is None:
|
||||
from archivebox.machine.detect import get_host_guid
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
try:
|
||||
machine = Machine.objects.filter(guid=get_host_guid()).first()
|
||||
except Exception:
|
||||
return
|
||||
if machine is None:
|
||||
return
|
||||
|
||||
file_config, file_mtime = _load_file_config_dict()
|
||||
machine_config = _coerce_to_str_dict(machine.config)
|
||||
if machine_config == file_config:
|
||||
return
|
||||
|
||||
db_mtime = machine.modified_at.timestamp() if machine.modified_at else 0.0
|
||||
file_is_newer = file_mtime is not None and file_mtime > db_mtime
|
||||
|
||||
merged: dict[str, str] = {}
|
||||
all_keys = set(machine_config) | set(file_config)
|
||||
for key in all_keys:
|
||||
in_file = key in file_config
|
||||
in_db = key in machine_config
|
||||
if in_file and in_db:
|
||||
if file_config[key] == machine_config[key]:
|
||||
merged[key] = file_config[key]
|
||||
else:
|
||||
merged[key] = file_config[key] if file_is_newer else machine_config[key]
|
||||
elif in_file:
|
||||
merged[key] = file_config[key]
|
||||
else:
|
||||
merged[key] = machine_config[key]
|
||||
|
||||
if merged != file_config:
|
||||
_write_file_if_changed(_render_config_file_content(merged))
|
||||
if merged != machine_config:
|
||||
machine.config = dict(merged)
|
||||
machine.save(update_fields=["config", "modified_at"])
|
||||
finally:
|
||||
_MIRROR_IN_PROGRESS = False
|
||||
|
||||
|
||||
def write_config_file(config: dict[str, str]) -> AttrDict:
|
||||
"""load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
|
||||
"""Merge ``config`` into ``ArchiveBox.conf``, validate, then mirror to Machine.config.
|
||||
|
||||
Backwards-compatible signature: callers (CLI ``archivebox config --set``
|
||||
and the init flow) pass a partial dict of keys to upsert.
|
||||
"""
|
||||
|
||||
from archivebox.config.common import get_all_configs
|
||||
from archivebox.hooks import discover_plugin_configs
|
||||
from archivebox.misc.system import atomic_write
|
||||
|
||||
CONFIG_HEADER = """# This is the config file for your ArchiveBox collection.
|
||||
#
|
||||
# You can add options here manually in INI format, or automatically by running:
|
||||
# archivebox config --set KEY=VALUE
|
||||
#
|
||||
# If you modify this file manually, make sure to update your archive after by running:
|
||||
# archivebox init
|
||||
#
|
||||
# A list of all possible config with documentation and examples can be found here:
|
||||
# https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
|
||||
|
||||
"""
|
||||
|
||||
config_path = CONSTANTS.CONFIG_FILE
|
||||
|
||||
if not os.access(config_path, os.F_OK):
|
||||
atomic_write(config_path, CONFIG_HEADER)
|
||||
atomic_write(config_path, CONFIG_FILE_HEADER)
|
||||
|
||||
config_file = CaseConfigParser()
|
||||
config_file.read(config_path)
|
||||
@ -43,21 +242,7 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
|
||||
|
||||
# Set up sections in empty config file
|
||||
for key, val in config.items():
|
||||
section_name = None
|
||||
for section in config_sections.values():
|
||||
if key in type(section).model_fields:
|
||||
section_name = section.toml_section_header
|
||||
break
|
||||
|
||||
if section_name is None:
|
||||
for schema in plugin_configs.values():
|
||||
if "properties" in schema and key in schema["properties"]:
|
||||
section_name = "PLUGINS"
|
||||
break
|
||||
|
||||
if section_name is None:
|
||||
raise ValueError(f"No config section found for key: {key}")
|
||||
|
||||
section_name = _resolve_section_for_key(key, config_sections, plugin_configs)
|
||||
if section_name in config_file:
|
||||
existing_config = dict(config_file[section_name])
|
||||
else:
|
||||
@ -84,4 +269,18 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
|
||||
if os.access(f"{config_path}.bak", os.F_OK):
|
||||
os.remove(f"{config_path}.bak")
|
||||
|
||||
# Mirror the post-write file state into Machine.config so the DB stays
|
||||
# 1:1 with the on-disk file. Recursion-guarded so Machine.save's own
|
||||
# mirror-back doesn't loop us.
|
||||
global _MIRROR_IN_PROGRESS
|
||||
if not _MIRROR_IN_PROGRESS:
|
||||
_MIRROR_IN_PROGRESS = True
|
||||
try:
|
||||
flat, _mtime = _load_file_config_dict()
|
||||
_mirror_file_to_machine_config(flat)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
_MIRROR_IN_PROGRESS = False
|
||||
|
||||
return AttrDict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})
|
||||
|
||||
@ -62,6 +62,47 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
|
||||
return None
|
||||
|
||||
|
||||
_SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
|
||||
SENSITIVE_CONFIG_VALUE_REDACTED = "********"
|
||||
|
||||
|
||||
def is_sensitive_config_key(key: str) -> bool:
|
||||
"""True if a config key names a credential and must be write-only in the UI.
|
||||
|
||||
Matches any key whose uppercase form contains ``TOKEN``, ``SECRET``,
|
||||
``API_KEY``, ``APIKEY``, or ``PASSWORD`` — covers ``SECRET_KEY``,
|
||||
``OPENAI_API_KEY``, ``TWOCAPTCHA_APIKEY``, ``GITHUB_TOKEN``,
|
||||
``ADMIN_PASSWORD``, etc. Centralized here so the KeyValueWidget
|
||||
(Machine/Crawl/Snapshot/Persona admin forms), the plugin config grid,
|
||||
REST API responses, and any future surface that round-trips raw config
|
||||
values all agree on which keys to redact.
|
||||
"""
|
||||
upper = (key or "").upper()
|
||||
return any(needle in upper for needle in _SENSITIVE_CONFIG_KEY_NEEDLES)
|
||||
|
||||
|
||||
def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
"""Return a copy of ``config`` with credential values replaced by ``********``.
|
||||
|
||||
Used wherever a config dict crosses an API/export/debug-dump boundary. The
|
||||
widget-side write-only treatment handles the form-render path; this helper
|
||||
handles every JSON-response path (REST schemas, ``to_json`` exports, admin
|
||||
debug views, etc.). Empty values are passed through unchanged so callers
|
||||
can still tell "unset" from "set-but-hidden."
|
||||
"""
|
||||
if config is None:
|
||||
return {}
|
||||
if not isinstance(config, Mapping):
|
||||
return {}
|
||||
redacted: dict[str, Any] = {}
|
||||
for key, value in config.items():
|
||||
if is_sensitive_config_key(str(key)) and value not in (None, ""):
|
||||
redacted[key] = SENSITIVE_CONFIG_VALUE_REDACTED
|
||||
else:
|
||||
redacted[key] = value
|
||||
return redacted
|
||||
|
||||
|
||||
def rprint(*args, file=None, **kwargs):
|
||||
console = _STDERR_CONSOLE if file is sys.stderr else _STDOUT_CONSOLE
|
||||
console.print(*args, **kwargs)
|
||||
|
||||
@ -373,7 +373,15 @@ class PluginConfigFormMixin:
|
||||
default_value = prop_schema.get("default", "")
|
||||
fallback_key = prop_schema.get("x-fallback")
|
||||
default_display = f"{{{fallback_key}}}" if fallback_key else default_value
|
||||
is_sensitive = bool(prop_schema.get("x-sensitive"))
|
||||
# A field is sensitive if either the schema explicitly marks it
|
||||
# (``x-sensitive``) or the key name matches our credential heuristic
|
||||
# (``*TOKEN*`` / ``*SECRET*`` / ``*API_KEY*`` / ``*APIKEY*``). The
|
||||
# plugin grid lives on user-facing pages, so we redact the value,
|
||||
# render a password input, and on empty submit we preserve the
|
||||
# previously-saved value in ``clean_plugin_config_overrides`` below.
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
is_sensitive = bool(prop_schema.get("x-sensitive")) or is_sensitive_config_key(config_key)
|
||||
input_value = "" if is_sensitive else _jsonish(current_value)
|
||||
field_kind = "text"
|
||||
input_type = "text"
|
||||
@ -441,7 +449,9 @@ class PluginConfigFormMixin:
|
||||
if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
|
||||
raw_value = self.data.getlist(input_name)
|
||||
|
||||
if prop_schema.get("x-sensitive") and raw_value == "":
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
if (prop_schema.get("x-sensitive") or is_sensitive_config_key(config_key)) and raw_value == "":
|
||||
continue
|
||||
|
||||
try:
|
||||
|
||||
@ -1173,12 +1173,21 @@ class AddView(UserPassesTestMixin, FormView):
|
||||
required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
|
||||
can_override_crawl_config = self._can_override_crawl_config()
|
||||
plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
|
||||
from archivebox.config.common import is_sensitive_config_key
|
||||
|
||||
sensitive_keys = {
|
||||
str(config_key)
|
||||
for schema in plugin_configs.values()
|
||||
for config_key, prop_schema in (schema.get("properties") or {}).items()
|
||||
if isinstance(prop_schema, dict) and prop_schema.get("x-sensitive")
|
||||
}
|
||||
|
||||
def _drop_sensitive(items):
|
||||
# Filter by both the schema-level ``x-sensitive`` marker (above) and
|
||||
# the key-name heuristic so a persona/effective config can never
|
||||
# ship credential values to the public ``/add`` UI.
|
||||
return {str(key): value for key, value in items if str(key) not in sensitive_keys and not is_sensitive_config_key(str(key))}
|
||||
|
||||
public_persona_config_keys = {
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS",
|
||||
"DELETE_AFTER",
|
||||
@ -1193,8 +1202,8 @@ class AddView(UserPassesTestMixin, FormView):
|
||||
for persona in persona_queryset.order_by("name"):
|
||||
effective_config = get_config(persona=persona)
|
||||
if can_override_crawl_config:
|
||||
raw_config = {str(key): value for key, value in (persona.config or {}).items() if str(key) not in sensitive_keys}
|
||||
effective_config_json = {str(key): value for key, value in effective_config.items() if str(key) not in sensitive_keys}
|
||||
raw_config = _drop_sensitive((persona.config or {}).items())
|
||||
effective_config_json = _drop_sensitive(effective_config.items())
|
||||
binary_urls = get_plugin_config_binary_urls(effective_config)
|
||||
else:
|
||||
raw_config = {}
|
||||
|
||||
@ -407,6 +407,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
Convert Crawl model instance to a JSON-serializable dict.
|
||||
"""
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return {
|
||||
"type": "Crawl",
|
||||
@ -415,7 +416,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
"urls": self.urls,
|
||||
"status": self.status,
|
||||
"max_depth": self.max_depth,
|
||||
"config": self.config or {},
|
||||
"config": redact_sensitive_config(self.config),
|
||||
"tags_str": self.tags_str,
|
||||
"label": self.label,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
|
||||
@ -9,6 +9,7 @@ from django.db.models.functions import Coalesce, Now
|
||||
from django.shortcuts import redirect
|
||||
from django.utils import timezone
|
||||
from django.utils.html import format_html
|
||||
from django.utils.safestring import mark_safe
|
||||
from django_object_actions import action
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
|
||||
@ -70,7 +71,7 @@ def _format_process_duration_seconds(started_at, ended_at) -> str:
|
||||
|
||||
class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
list_display = (
|
||||
"id",
|
||||
"id_display",
|
||||
"created_at",
|
||||
"hostname",
|
||||
"ips",
|
||||
@ -137,6 +138,17 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
{
|
||||
"fields": ("config",),
|
||||
"classes": ("card", "wide"),
|
||||
"description": mark_safe(
|
||||
'<div style="padding:8px 10px;margin-bottom:8px;background:#fff7ed;'
|
||||
"border:1px solid #fed7aa;border-left:4px solid #f59e0b;border-radius:4px;"
|
||||
'color:#7c2d12;font-size:12px;line-height:1.45;">'
|
||||
"<b>Heads up:</b> saving here also rewrites "
|
||||
"<code>data/ArchiveBox.conf</code> on disk to match — the two stores are "
|
||||
"kept in 1:1 sync, so any keys you remove here will be removed from the file "
|
||||
"too. Edits to <code>ArchiveBox.conf</code> (or <code>archivebox config --set</code>) "
|
||||
"propagate back into this field on the next request."
|
||||
"</div>",
|
||||
),
|
||||
},
|
||||
),
|
||||
(
|
||||
@ -167,6 +179,36 @@ class MachineAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
color = "green" if h >= 80 else "orange" if h >= 50 else "red"
|
||||
return format_html('<span style="color: {};">{}</span>', color, h)
|
||||
|
||||
@admin.display(description="ID", ordering="id")
|
||||
def id_display(self, machine):
|
||||
# Highlight the row representing the machine that ``Machine.current()``
|
||||
# resolves to in this process — that's the one whose ``config`` is
|
||||
# actually being applied at runtime. Important to surface here because
|
||||
# a collection can accumulate stale Machine rows from prior hosts (VM
|
||||
# snapshots, container rebuilds, hostname changes), and editing the
|
||||
# wrong one silently produces "I set BASE_URL but it didn't stick."
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
try:
|
||||
current_id = str(Machine.current().pk)
|
||||
except Exception:
|
||||
current_id = None
|
||||
|
||||
machine_id = str(machine.pk)
|
||||
short_id = machine_id[:8]
|
||||
if current_id and machine_id == current_id:
|
||||
return format_html(
|
||||
'<span style="display:inline-block;background:#16a34a;color:#fff;'
|
||||
"padding:1px 6px;border-radius:3px;font-weight:800;font-size:11px;"
|
||||
'margin-right:6px;letter-spacing:0.3px;">★ CURRENT</span>'
|
||||
'<code style="font-weight:700;">{}</code>',
|
||||
short_id,
|
||||
)
|
||||
return format_html(
|
||||
'<code style="color:#888;">{}</code>',
|
||||
short_id,
|
||||
)
|
||||
|
||||
|
||||
class NetworkInterfaceAdmin(BaseModelAdmin):
|
||||
list_display = (
|
||||
|
||||
@ -211,6 +211,16 @@ class Machine(ModelWithHealthStats):
|
||||
_CURRENT_MACHINE = None
|
||||
if _CURRENT_MACHINE:
|
||||
if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
|
||||
# One-time-per-process reconciliation between ArchiveBox.conf
|
||||
# and Machine.config. Fast-path: bool check + early-return when
|
||||
# the sync has already run, so the cached-machine return path
|
||||
# stays sub-microsecond.
|
||||
try:
|
||||
from archivebox.config.collection import sync_machine_and_file
|
||||
|
||||
sync_machine_and_file(_CURRENT_MACHINE)
|
||||
except Exception:
|
||||
pass
|
||||
return _CURRENT_MACHINE
|
||||
else:
|
||||
_CURRENT_MACHINE = None
|
||||
@ -252,7 +262,17 @@ class Machine(ModelWithHealthStats):
|
||||
"modified_at",
|
||||
],
|
||||
)
|
||||
return cls._sanitize_config(_CURRENT_MACHINE)
|
||||
machine = cls._sanitize_config(_CURRENT_MACHINE)
|
||||
# Same one-time sync as the cached-return path. Triggers here on the
|
||||
# very first ``Machine.current()`` call in a process before any
|
||||
# cached return can occur.
|
||||
try:
|
||||
from archivebox.config.collection import sync_machine_and_file
|
||||
|
||||
sync_machine_and_file(_CURRENT_MACHINE)
|
||||
except Exception:
|
||||
pass
|
||||
return machine
|
||||
|
||||
@classmethod
|
||||
def _sanitize_config(cls, machine: Machine) -> Machine:
|
||||
@ -270,6 +290,7 @@ class Machine(ModelWithHealthStats):
|
||||
Convert Machine model instance to a JSON-serializable dict.
|
||||
"""
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.common import redact_sensitive_config
|
||||
|
||||
return {
|
||||
"type": "Machine",
|
||||
@ -288,7 +309,7 @@ class Machine(ModelWithHealthStats):
|
||||
"os_kernel": self.os_kernel,
|
||||
"os_release": self.os_release,
|
||||
"stats": self.stats,
|
||||
"config": self.config or {},
|
||||
"config": redact_sensitive_config(self.config),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@ -320,11 +341,28 @@ class Machine(ModelWithHealthStats):
|
||||
# never moves forward on the cached object even when the row is
|
||||
# updated in the DB, so without this we'd keep serving stale
|
||||
# ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
|
||||
update_fields = kwargs.get("update_fields")
|
||||
super().save(*args, **kwargs)
|
||||
global _CURRENT_MACHINE
|
||||
if _CURRENT_MACHINE is not None and _CURRENT_MACHINE.pk == self.pk:
|
||||
_CURRENT_MACHINE = None
|
||||
|
||||
# Mirror Machine.config into ArchiveBox.conf so the two stores stay
|
||||
# 1:1. Skipped when ``update_fields`` is set and doesn't touch
|
||||
# ``config`` (binary autodetection + ``hostname``/``stats`` refreshes
|
||||
# save fields we don't need to disk-mirror, which keeps hot paths
|
||||
# zero-IO). Errors during mirroring are swallowed: the DB write
|
||||
# already succeeded and we don't want a config-file write hiccup to
|
||||
# turn a routine save into a 500.
|
||||
if update_fields is not None and "config" not in update_fields:
|
||||
return
|
||||
try:
|
||||
from archivebox.config.collection import mirror_machine_config_to_file
|
||||
|
||||
mirror_machine_config_to_file(self.config)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class NetworkInterfaceManager(models.Manager):
|
||||
def current(self) -> NetworkInterface:
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.33rc43",
|
||||
"version": "0.9.33rc45",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.33rc43"
|
||||
version = "0.9.33rc45"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@ -79,9 +79,9 @@ dependencies = [
|
||||
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.8", # EventBus API
|
||||
"abxpkg>=1.11.69", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.75", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.75", # shared ArchiveBox downloader package with blocking install preflight
|
||||
"abxpkg>=1.11.72", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.78", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.78", # shared ArchiveBox downloader package with blocking install preflight
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user