mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
release: v0.9.33rc52
This commit is contained in:
Nick Sweeting
parent
83d5161b3e
commit
c1e792fee6
2
.github/workflows/lint.yml
vendored
2
.github/workflows/lint.yml
vendored
@ -39,7 +39,7 @@ jobs:
|
||||
|
||||
- name: Install dependencies with uv
|
||||
run: |
|
||||
uv sync --all-extras --all-groups --no-sources --no-cache
|
||||
uv sync --extra sonic --extra debug --all-groups --no-sources --no-cache
|
||||
uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
|
||||
|
||||
- name: Run prek
|
||||
|
||||
@ -15,6 +15,8 @@ from contextlib import contextmanager
|
||||
#############################################################################################
|
||||
|
||||
DATA_DIR = Path(os.getcwd())
|
||||
if os.environ.get("DATA_DIR") and Path(os.environ["DATA_DIR"]).resolve() != DATA_DIR.resolve():
|
||||
raise SystemExit(f"[X] DATA_DIR={os.environ['DATA_DIR']} must equal cwd={DATA_DIR}; cd into the data dir before running archivebox")
|
||||
|
||||
try:
|
||||
DATA_DIR_STAT = DATA_DIR.stat()
|
||||
|
||||
@ -324,20 +324,62 @@ def result_list_tag(parser, token):
|
||||
)
|
||||
|
||||
|
||||
@register.inclusion_tag("security_mode_banner.html", takes_context=True)
|
||||
def security_mode_banner(context):
|
||||
"""Render the top-of-page warning banner for one of two conditions:
|
||||
_LOW_DISK_THRESHOLD_GB = 1.0
|
||||
_HIGH_MEMORY_THRESHOLD_PCT = 95.0
|
||||
_HIGH_LOAD_MULTIPLE = 3 # 15-min loadavg > 3 * cpu_count
|
||||
_HEALTH_CHECK_INTERVAL_SECONDS = 30
|
||||
_health_cache: dict = {"checked_at": 0.0, "stats": {}}
|
||||
|
||||
1. ``mode="unconfigured"`` — ``BASE_URL`` is empty. The server is running
|
||||
on whatever host the operator happens to be hitting; CSRF auto-derive
|
||||
and the request-host fallback in ``get_base_url`` keep things working,
|
||||
but the operator should pin ``BASE_URL`` explicitly so links stay
|
||||
stable across hosts (and the misconfig banner goes away).
|
||||
2. ``mode="unsafe"`` — ``SERVER_SECURITY_MODE`` is a non-subdomain mode.
|
||||
Archived pages share an origin with privileged routes.
|
||||
|
||||
Both conditions can hold; we show the ``unconfigured`` banner first
|
||||
because pinning ``BASE_URL`` is the more immediately actionable fix.
|
||||
def _machine_health_stats() -> dict:
|
||||
"""Cached wrapper around the Machine-admin stats util.
|
||||
|
||||
The Machine list/change pages already render disk / mem / load via
|
||||
``archivebox.machine.detect.get_host_stats()`` — we reuse the same call so
|
||||
the banner thresholds line up 1:1 with what's shown on /admin/machine/.
|
||||
Cached for 30s because the inclusion tag fires on every page render and
|
||||
``get_host_stats`` shells into several psutil probes.
|
||||
"""
|
||||
import time
|
||||
|
||||
now = time.monotonic()
|
||||
if _health_cache["stats"] and (now - _health_cache["checked_at"]) < _HEALTH_CHECK_INTERVAL_SECONDS:
|
||||
return _health_cache["stats"]
|
||||
|
||||
try:
|
||||
from archivebox.machine.detect import get_host_stats
|
||||
|
||||
stats = get_host_stats() or {}
|
||||
except Exception:
|
||||
stats = {}
|
||||
|
||||
_health_cache["checked_at"] = now
|
||||
_health_cache["stats"] = stats
|
||||
return stats
|
||||
|
||||
|
||||
@register.inclusion_tag("system_warnings_banner.html", takes_context=True)
|
||||
def system_warnings_banner(context):
|
||||
"""Render the top-of-page warning banner for one of the conditions below,
|
||||
in priority order (highest first):
|
||||
|
||||
1. ``mode="unconfigured"``— ``BASE_URL`` is empty. Security/correctness
|
||||
issue: until it's pinned, generated URLs can echo any Host the client
|
||||
sends, and admin/web/api routing has no canonical anchor.
|
||||
2. ``mode="unsafe"`` — ``SERVER_SECURITY_MODE`` is a non-subdomain
|
||||
mode. Archived pages share an origin with privileged routes.
|
||||
3. ``mode="low_disk"`` — ``DATA_DIR`` has <1 GiB free; new archive
|
||||
jobs will start failing on ENOSPC.
|
||||
4. ``mode="high_memory"`` — virtual memory utilization at/above 95%; the
|
||||
host is one OOM-kill from a crash.
|
||||
5. ``mode="high_load"`` — 15-minute load average exceeds 3 × CPU count
|
||||
(the kernel's own sustained-load EMA, so no rolling buffer of ours is
|
||||
needed).
|
||||
|
||||
Config/security warnings come first because they affect correctness +
|
||||
security and need explicit operator action; host-health warnings come
|
||||
after and reuse ``machine.detect.get_host_stats`` (the same function that
|
||||
populates the Machine admin page), cached for 30s.
|
||||
"""
|
||||
config = context.get("CONFIG")
|
||||
if config is None:
|
||||
@ -349,6 +391,30 @@ def security_mode_banner(context):
|
||||
return _unconfigured_banner_context(context.get("request"))
|
||||
if not config.USES_SUBDOMAIN_ROUTING:
|
||||
return {"mode": "unsafe"}
|
||||
|
||||
stats = _machine_health_stats()
|
||||
free_gb = stats.get("disk_data_free_gb")
|
||||
if isinstance(free_gb, (int, float)) and free_gb < _LOW_DISK_THRESHOLD_GB:
|
||||
return {"mode": "low_disk", "free_gb": f"{free_gb:.2f}"}
|
||||
|
||||
mem_pct = stats.get("mem_virt_used_pct")
|
||||
if isinstance(mem_pct, (int, float)) and mem_pct >= _HIGH_MEMORY_THRESHOLD_PCT:
|
||||
return {"mode": "high_memory", "mem_pct": f"{mem_pct:.1f}"}
|
||||
|
||||
cpu_load = stats.get("cpu_load") or ()
|
||||
cpu_count = stats.get("cpu_count") or 1
|
||||
# ``cpu_load`` is the (1min, 5min, 15min) tuple from psutil.getloadavg();
|
||||
# we take the 15-min figure because the operator's threshold was
|
||||
# "sustained for 15min" and the kernel already maintains that EMA.
|
||||
load_15 = cpu_load[2] if isinstance(cpu_load, (list, tuple)) and len(cpu_load) >= 3 else None
|
||||
if isinstance(load_15, (int, float)) and load_15 > _HIGH_LOAD_MULTIPLE * cpu_count:
|
||||
return {
|
||||
"mode": "high_load",
|
||||
"load_15": f"{load_15:.2f}",
|
||||
"cpu_count": cpu_count,
|
||||
"load_threshold": _HIGH_LOAD_MULTIPLE * cpu_count,
|
||||
}
|
||||
|
||||
return {"mode": ""}
|
||||
|
||||
|
||||
|
||||
@ -1678,7 +1678,7 @@
|
||||
|
||||
|
||||
<body class="{% if is_popup %}popup {% endif %}{% block bodyclass %}{% endblock %}" data-admin-utc-offset="{% now "Z" %}">
|
||||
{% security_mode_banner %}
|
||||
{% system_warnings_banner %}
|
||||
{% include 'progressbar.html' %}
|
||||
|
||||
<div id="container">
|
||||
|
||||
@ -19,7 +19,7 @@
|
||||
{% endblock %}
|
||||
</head>
|
||||
<body>
|
||||
{% security_mode_banner %}
|
||||
{% system_warnings_banner %}
|
||||
<div id="container">
|
||||
<div id="header">
|
||||
<div id="branding">
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
{% comment %}
|
||||
Fixed red badge that hangs off the top center of the page. Two trigger
|
||||
conditions (see ``security_mode_banner`` in core_tags.py):
|
||||
|
||||
* mode="unsafe" — server is in a non-subdomain SERVER_SECURITY_MODE.
|
||||
Archived content shares an origin with the admin UI.
|
||||
* mode="unconfigured" — BASE_URL is not set. Always shown until the
|
||||
operator pins it explicitly, even when CSRF
|
||||
auto-derive or request-host fallback are keeping
|
||||
the server functional.
|
||||
{% endcomment %}
|
||||
{% if mode == "unsafe" %}
|
||||
<div id="archivebox-security-banner" role="alert" aria-live="polite"
|
||||
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
|
||||
z-index:2147483647;background:#aa1e55;color:#fff;
|
||||
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
|
||||
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
|
||||
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
|
||||
white-space:nowrap;letter-spacing:0.3px;text-transform:uppercase;">
|
||||
<span style="display:inline-block;background:#fff;color:#aa1e55;
|
||||
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
|
||||
⚠ unsafe
|
||||
</span>
|
||||
ArchiveBox single-domain mode — archived pages share an origin with this site
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes"
|
||||
target="_blank" rel="noopener noreferrer"
|
||||
style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
|
||||
</div>
|
||||
{% elif mode == "unconfigured" %}
|
||||
<div id="archivebox-security-banner" role="alert" aria-live="polite"
|
||||
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
|
||||
z-index:2147483647;background:#dc2626;color:#fff;
|
||||
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
|
||||
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
|
||||
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
|
||||
max-width:90vw;letter-spacing:0.3px;text-transform:uppercase;">
|
||||
<span style="display:inline-block;background:#fff;color:#dc2626;
|
||||
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
|
||||
⚠ base_url not set
|
||||
</span>
|
||||
To prevent unauthorized requests, you must set your intended server URL in env or config:
|
||||
<code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">
|
||||
BASE_URL={% if suggested_base_url %}{{ suggested_base_url }}{% else %}http://*.archivebox.localhost:8000{% endif %}
|
||||
</code>
|
||||
{% if machine_admin_url %}
|
||||
<a href="{{ machine_admin_url }}#BASE_URL"
|
||||
style="display:inline-block;background:#fff;color:#dc2626;padding:1px 6px;border-radius:3px;
|
||||
font-weight:800;text-decoration:none;margin-left:6px;text-transform:uppercase;">
|
||||
pin via admin →
|
||||
</a>
|
||||
{% endif %}
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes#base_url"
|
||||
target="_blank" rel="noopener noreferrer"
|
||||
style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
115
archivebox/templates/core/system_warnings_banner.html
Normal file
115
archivebox/templates/core/system_warnings_banner.html
Normal file
@ -0,0 +1,115 @@
|
||||
{% comment %}
|
||||
Fixed red badge that hangs off the top center of the page. Five trigger
|
||||
conditions (see ``system_warnings_banner`` in core_tags.py); precedence is
|
||||
config/security first, then host-health:
|
||||
|
||||
* mode="unconfigured" — BASE_URL is not set. Always shown until the
|
||||
operator pins it explicitly, even when CSRF
|
||||
auto-derive or request-host fallback are keeping
|
||||
the server functional.
|
||||
* mode="unsafe" — server is in a non-subdomain SERVER_SECURITY_MODE.
|
||||
Archived content shares an origin with the admin UI.
|
||||
* mode="low_disk" — free space on DATA_DIR's filesystem is below 1GiB.
|
||||
Archive jobs will fail until the operator frees space.
|
||||
* mode="high_memory" — virtual memory utilization above 95%; one OOM-kill
|
||||
from a crash.
|
||||
* mode="high_load" — 15-min loadavg > 3 × cpu_count; saturated host.
|
||||
{% endcomment %}
|
||||
{% if mode == "low_disk" %}
|
||||
<div id="archivebox-system-warning-banner" role="alert" aria-live="assertive"
|
||||
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
|
||||
z-index:2147483647;background:#dc2626;color:#fff;
|
||||
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
|
||||
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
|
||||
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
|
||||
max-width:90vw;letter-spacing:0.3px;text-transform:uppercase;">
|
||||
<span style="display:inline-block;background:#fff;color:#dc2626;
|
||||
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
|
||||
⚠ low disk
|
||||
</span>
|
||||
Only <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ free_gb }} GiB</code>
|
||||
free on <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">DATA_DIR</code>
|
||||
— new archive jobs will fail until space is reclaimed.
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Disk-Usage"
|
||||
target="_blank" rel="noopener noreferrer"
|
||||
style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
|
||||
</div>
|
||||
{% elif mode == "high_memory" %}
|
||||
<div id="archivebox-system-warning-banner" role="alert" aria-live="assertive"
|
||||
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
|
||||
z-index:2147483647;background:#dc2626;color:#fff;
|
||||
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
|
||||
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
|
||||
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
|
||||
max-width:90vw;letter-spacing:0.3px;text-transform:uppercase;">
|
||||
<span style="display:inline-block;background:#fff;color:#dc2626;
|
||||
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
|
||||
⚠ high memory
|
||||
</span>
|
||||
Virtual memory at
|
||||
<code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ mem_pct }}%</code>
|
||||
— the host is close to OOM. Consider stopping crawls or scaling up.
|
||||
</div>
|
||||
{% elif mode == "high_load" %}
|
||||
<div id="archivebox-system-warning-banner" role="alert" aria-live="polite"
|
||||
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
|
||||
z-index:2147483647;background:#dc2626;color:#fff;
|
||||
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
|
||||
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
|
||||
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
|
||||
max-width:90vw;letter-spacing:0.3px;text-transform:uppercase;">
|
||||
<span style="display:inline-block;background:#fff;color:#dc2626;
|
||||
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
|
||||
⚠ high load
|
||||
</span>
|
||||
15-min loadavg
|
||||
<code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
|
||||
exceeds
|
||||
<code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_threshold }}</code>
|
||||
({{ cpu_count }} cores × 3) — the host is saturated; crawls are queuing faster than the runner can finish them.
|
||||
</div>
|
||||
{% elif mode == "unsafe" %}
|
||||
<div id="archivebox-system-warning-banner" role="alert" aria-live="polite"
|
||||
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
|
||||
z-index:2147483647;background:#aa1e55;color:#fff;
|
||||
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
|
||||
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
|
||||
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
|
||||
white-space:nowrap;letter-spacing:0.3px;text-transform:uppercase;">
|
||||
<span style="display:inline-block;background:#fff;color:#aa1e55;
|
||||
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
|
||||
⚠ unsafe
|
||||
</span>
|
||||
ArchiveBox single-domain mode — archived pages share an origin with this site
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes"
|
||||
target="_blank" rel="noopener noreferrer"
|
||||
style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
|
||||
</div>
|
||||
{% elif mode == "unconfigured" %}
|
||||
<div id="archivebox-system-warning-banner" role="alert" aria-live="polite"
|
||||
style="position:fixed;top:-10px;left:50%;transform:translateX(-50%);
|
||||
z-index:2147483647;background:#dc2626;color:#fff;
|
||||
font:600 11px/1.3 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,Arial,sans-serif;
|
||||
padding:14px 14px 4px 14px;border-radius:0 0 6px 6px;
|
||||
box-shadow:0 2px 10px rgba(0,0,0,0.35);pointer-events:auto;
|
||||
max-width:90vw;letter-spacing:0.3px;text-transform:uppercase;">
|
||||
<span style="display:inline-block;background:#fff;color:#dc2626;
|
||||
border-radius:3px;padding:1px 5px;margin-right:6px;font-weight:800;">
|
||||
⚠ base_url not set
|
||||
</span>
|
||||
To prevent unauthorized requests, you must set your intended server URL in env or config:
|
||||
<code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">
|
||||
BASE_URL={% if suggested_base_url %}{{ suggested_base_url }}{% else %}http://*.archivebox.localhost:8000{% endif %}
|
||||
</code>
|
||||
{% if machine_admin_url %}
|
||||
<a href="{{ machine_admin_url }}#BASE_URL"
|
||||
style="display:inline-block;background:#fff;color:#dc2626;padding:1px 6px;border-radius:3px;
|
||||
font-weight:800;text-decoration:none;margin-left:6px;text-transform:uppercase;">
|
||||
pin via admin →
|
||||
</a>
|
||||
{% endif %}
|
||||
<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes#base_url"
|
||||
target="_blank" rel="noopener noreferrer"
|
||||
style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
@ -328,7 +328,7 @@ wait_for_pypi() {
|
||||
}
|
||||
|
||||
run_checks() {
|
||||
uv sync --all-extras --all-groups --no-cache --upgrade
|
||||
uv sync --extra sonic --extra debug --all-groups --no-cache --upgrade
|
||||
uv build --all
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.33rc51",
|
||||
"version": "0.9.33rc52",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.33rc51"
|
||||
version = "0.9.33rc52"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@ -79,9 +79,9 @@ dependencies = [
|
||||
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.8", # EventBus API
|
||||
"abxpkg>=1.11.80", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.86", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.86", # shared ArchiveBox downloader package with blocking install preflight
|
||||
"abxpkg>=1.11.81", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
|
||||
"abx-plugins>=1.11.87", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
|
||||
"abx-dl>=1.11.87", # shared ArchiveBox downloader package with blocking install preflight
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user