The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

fix: harden admin metadata surfaces

Browse Source
This commit is contained in:
Nick Sweeting 2026-06-13 23:44:59 -07:00
parent faf2109642
commit 73ec255855
No known key found for this signature in database
12 changed files with 235 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Dockerfile
View File

@ -66,7 +66,7 @@ ENV TMP_DIR=/tmp/archivebox \
ENV HOME=/home/archivebox \
XDG_CONFIG_HOME=/home/archivebox/.config \
XDG_CACHE_HOME=/home/archivebox/.cache \
XDG_CACHE_HOME=/opt/archivebox/lib/cache \
ABXPKG_INSTALL_TIMEOUT=600 \
ABXPKG_POSTINSTALL_SCRIPTS=True \
ABXPKG_MIN_RELEASE_AGE=0 \
@ -222,9 +222,8 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..."
&& [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
&& (which sonic && sonic --version) | tee -a /VERSION.txt \
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.cache" \
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
&& chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
&& install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
&& chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
&& openssl rand -hex 16 > /etc/machine-id \
&& echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
&& echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
@ -239,7 +238,7 @@ RUN echo "[+] Initializing image collection..." \
"$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
"$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
"$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
"$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache" \
"$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
2>/dev/null || true) \
&& find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
@ -256,9 +255,6 @@ RUN "$LIB_DIR/playwright/bin/chromium" --version | tee -a /VERSION.txt \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
&& chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache" \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/uv" \
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
&& rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*

6
archivebox/config/django.py
View File

@ -41,6 +41,12 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
# TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
return
# SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
# ArchiveBox file-mode policy before any DB connection can create the file,
# otherwise a permissive parent umask can expose a just-created DB until a
# later chmod runs.
os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
# Third-party patches are only needed once Django/apps are about to load.
# Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
# cost for cheap CLI startup paths like `archivebox <cmd> --help`.

10
archivebox/core/admin_archiveresults.py
View File

@ -157,10 +157,12 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
'''
# Truncate output for display
full_output = result.output_str_for_display() or "-"
output_display = full_output[:60]
if len(full_output) > 60:
output_display += "..."
full_output_raw = result.output_str_for_display() or "-"
output_display_raw = full_output_raw[:60]
if len(full_output_raw) > 60:
output_display_raw += "..."
full_output = html.escape(full_output_raw)
output_display = html.escape(output_display_raw)
display_cmd = build_abx_dl_display_command(result)
replay_cmd = build_abx_dl_replay_command(result, config=config)

3
archivebox/core/views.py
View File

@ -52,6 +52,7 @@ from archivebox.misc.util import (
base_url,
filter_queryset_by_uuid_substring,
htmlencode,
sanitize_html_text,
ts_to_date_str,
urldecode,
validate_url,
@ -1491,7 +1492,7 @@ class AddView(UserPassesTestMixin, FormView):
config=config,
)
if notes:
crawl.safe_update({"notes": notes}, refresh=False)
crawl.safe_update({"notes": sanitize_html_text(notes)}, refresh=False)
if permissions and crawl.config.get("PERMISSIONS") != permissions:
next_config = {**crawl.config, "PERMISSIONS": permissions}
crawl.safe_update({"config": next_config}, refresh=True)

14
archivebox/core/widgets.py
View File

@ -40,6 +40,16 @@ class TagEditorWidget(forms.Widget):
normalized = f"t_{normalized}"
return normalized
def _json_for_inline_script(self, value):
"""Serialize JSON so it cannot close the surrounding inline <script> tag."""
return json.dumps(value).translate(
{
ord(">"): "\\u003E",
ord("<"): "\\u003C",
ord("&"): "\\u0026",
},
)
def _tag_style(self, value):
"""Compute a stable pastel color style for a tag value."""
tag = (value or "").strip().lower()
@ -106,6 +116,8 @@ class TagEditorWidget(forms.Widget):
</span>
'''
tags_json = self._json_for_inline_script(tags)
# Build the widget HTML
html = f'''
<div id="{widget_id}_container" class="tag-editor-container" onclick="focusTagInput_{widget_id}(event)">
@ -128,7 +140,7 @@ class TagEditorWidget(forms.Widget):
<script>
(function() {{
var currentTags_{widget_id} = {json.dumps(tags)};
var currentTags_{widget_id} = {tags_json};
var autocompleteTimeout_{widget_id} = null;
window.focusTagInput_{widget_id} = function(event) {{

46
archivebox/progressmonitor/views.py
View File

@ -4,6 +4,7 @@ from functools import lru_cache
from pathlib import Path
from typing import Literal
from django.conf import settings
from django.db.models import CharField, Count, Q, Sum
from django.db.models.functions import Cast
from django.http import HttpResponse, JsonResponse
@ -935,27 +936,26 @@ def live_progress_view(request):
except ImportError:
return JsonResponse(payload)
except Exception as e:
import traceback
error_payload = {
"error": str(e),
"orchestrator_running": False,
"total_workers": 0,
"crawls_active": 0,
"crawls_queued": 0,
"crawls_recent": 0,
"snapshots_active": 0,
"snapshots_queued": 0,
"archiveresults_active": 0,
"archiveresults_queued": 0,
"downloads_active": 0,
"downloads_queued": 0,
"indexing_active": 0,
"indexing_queued": 0,
"active_crawls": [],
"server_time": timezone.now().isoformat(),
}
if settings.DEBUG:
import traceback
return JsonResponse(
{
"error": str(e),
"traceback": traceback.format_exc(),
"orchestrator_running": False,
"total_workers": 0,
"crawls_active": 0,
"crawls_queued": 0,
"crawls_recent": 0,
"snapshots_active": 0,
"snapshots_queued": 0,
"archiveresults_active": 0,
"archiveresults_queued": 0,
"downloads_active": 0,
"downloads_queued": 0,
"indexing_active": 0,
"indexing_queued": 0,
"active_crawls": [],
"server_time": timezone.now().isoformat(),
},
status=500,
)
error_payload["traceback"] = traceback.format_exc()
return JsonResponse(error_payload, status=500)

19
archivebox/tests/test_cli_init.py
View File

@ -5,6 +5,7 @@ Verify init creates correct database schema, filesystem structure, and config.
"""
import pytest
import subprocess
from django.utils import timezone
from django.db import connections
from django.db.migrations.recorder import MigrationRecorder
@ -14,6 +15,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Machine
from archivebox.tests.conftest import run_queued_crawls, run_archivebox_cmd, cli_env
from archivebox.tests.conftest import _set_test_source_pythonpath
from archivebox.tests.test_orm_helpers import use_archivebox_db
@ -129,6 +131,23 @@ def test_init_sets_correct_file_permissions(tmp_path):
assert oct(archive_dir.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
def test_init_creates_database_with_archivebox_permissions_despite_permissive_umask(tmp_path):
"""SQLite must create index.sqlite3 with ArchiveBox's restrictive mode from first open."""
env = cli_env(disable_extractors=True)
_set_test_source_pythonpath(env)
result = subprocess.run(
["bash", "-lc", "umask 000; archivebox init"],
cwd=tmp_path,
env=env,
capture_output=True,
text=True,
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
assert oct((tmp_path / "index.sqlite3").stat().st_mode)[-3:] == get_config().OUTPUT_PERMISSIONS
def test_init_is_idempotent(tmp_path):
"""Test that running init multiple times is safe (idempotent)."""

37
archivebox/tests/test_ui_add_view.py
View File

@ -215,6 +215,43 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
assert crawl.config["ONLY_NEW"] is True
def test_add_view_sanitizes_crawl_notes_before_safe_update(client, admin_user):
client.force_login(admin_user)
malicious_notes = "</script><script id=add-notes-xss>window.__archivebox_add_notes_xss__=1</script>"
response = client.post(
reverse("add"),
data={
"url": "https://example.com/notes-xss",
"tag": "",
"depth": "0",
"max_urls": "1",
"crawl_max_size": "0",
"crawl_timeout": "0",
"timeout": "",
"snapshot_max_size": "0",
"delete_after": "0",
"crawl_max_concurrent_snapshots": "1",
"url_filters_allowlist": "",
"url_filters_denylist": "",
"notes": malicious_notes,
"schedule": "",
"persona": "Default",
"permissions": "public",
"start_paused": "",
"config": "{}",
},
HTTP_HOST=ADMIN_HOST,
)
assert response.status_code == 302, response.context["form"].errors if response.context else response.content.decode()
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
assert crawl.notes == "window.__archivebox_add_notes_xss__=1"
assert "<script" not in crawl.notes
assert "</script>" not in crawl.notes
def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user, monkeypatch):
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)

44
archivebox/tests/test_ui_admin_snapshot.py
View File

@ -8,6 +8,7 @@ from types import SimpleNamespace
import pytest
from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME
from django.core.paginator import UnorderedObjectListWarning
from django.db import connection
from django.test import RequestFactory
from django.urls import reverse
from django.utils import timezone
@ -32,6 +33,49 @@ def test_snapshot_changelist_uses_stable_ordering_without_unordered_paginator_wa
assert b"Searching matching snapshots..." in response.content
def test_snapshot_admin_tag_editor_escapes_tag_json_script_breakout(admin_client, snapshot):
from archivebox.core.models import Tag
tag = Tag.objects.create(name="legacy-safe-tag")
snapshot.tags.add(tag)
malicious_name = '</script><script id="archivebox-tag-xss">window.__archivebox_tag_xss__=1</script>'
with connection.cursor() as cursor:
cursor.execute(
f"UPDATE {Tag._meta.db_table} SET name = %s WHERE id = %s",
[malicious_name, str(tag.pk)],
)
response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST)
body = response.content
assert response.status_code == 200
assert malicious_name.encode() not in body
assert b'<script id="archivebox-tag-xss">' not in body
assert b'\\u003C/script\\u003E\\u003Cscript id=\\"archivebox-tag-xss\\"\\u003E' in body
assert b"&lt;/script&gt;&lt;script id=&quot;archivebox-tag-xss&quot;&gt;" in body
def test_snapshot_admin_archive_results_escape_extractor_output(admin_client, snapshot):
from archivebox.core.models import ArchiveResult
payload = '<img src=x onerror="window.__archivebox_archiveresult_xss__=1">'
ArchiveResult.objects.create(
snapshot=snapshot,
plugin="title",
hook_name="on_Snapshot__54_title.js",
status=ArchiveResult.StatusChoices.SUCCEEDED,
output_str=payload,
)
response = admin_client.get(reverse("admin:core_snapshot_change", args=[snapshot.pk]), HTTP_HOST=ADMIN_TEST_HOST)
body = response.content
assert response.status_code == 200
assert payload.encode() not in body
assert b'<img src=x onerror="window.__archivebox_archiveresult_xss__=1">' not in body
assert b"&lt;img src=x onerror=&quot;window.__archivebox_archiveresult_xss__=1&quot;&gt;" in body
def test_snapshot_changelist_bulk_permissions_action_updates_selected_snapshots(client, admin_user, crawl, snapshot):
client.force_login(admin_user)
url = reverse("admin:core_snapshot_changelist")

43
archivebox/tests/test_ui_live_progress.py
View File

@ -6,6 +6,8 @@ from datetime import datetime, timezone as dt_timezone
from pathlib import Path
import pytest
from django.db import connection
from django.test import override_settings
from django.urls import reverse
from django.utils import timezone
@ -15,6 +17,47 @@ pytestmark = pytest.mark.django_db
class TestLiveProgressView:
def test_live_progress_rejects_unauthenticated_unscoped_request(self, client):
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 403
assert response.json() == {"error": "Permission denied"}
assert b"orchestrator_running" not in response.content
assert b"active_crawls" not in response.content
assert b"traceback" not in response.content
def test_admin_live_progress_path_does_not_bypass_admin_auth(self, client):
response = client.get("/admin/live-progress/", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code in (302, 403, 404)
assert b"orchestrator_running" not in response.content
assert b"active_crawls" not in response.content
assert b"traceback" not in response.content
@override_settings(DEBUG=False)
def test_live_progress_error_response_hides_traceback_without_debug(self, client, admin_user, crawl):
from archivebox.crawls.models import Crawl
Crawl.objects.filter(pk=crawl.pk).update(
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
modified_at=timezone.now(),
)
with connection.cursor() as cursor:
cursor.execute(
f"UPDATE {Crawl._meta.db_table} SET created_at = %s WHERE id = %s",
["not-a-date", str(crawl.pk)],
)
client.force_login(admin_user)
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 500
payload = response.json()
assert "error" in payload
assert "traceback" not in payload
assert payload["active_crawls"] == []
def test_live_progress_excludes_old_archiveresults_from_previous_snapshot_run(self, client, admin_user, crawl, snapshot):
from datetime import timedelta
from archivebox.core.models import ArchiveResult

34
archivebox/tests/test_ui_public_snapshot.py
View File

@ -6,6 +6,7 @@ import time
import pytest
import requests
from django.db import connection
from django.test import override_settings
from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE
@ -280,6 +281,39 @@ class TestPublicIndex:
assert b"Unlisted Snapshot" not in response.content
assert b"Private Snapshot" not in response.content
@override_settings(PUBLIC_INDEX=True)
def test_public_snapshot_surfaces_escape_legacy_raw_title_and_tag_values(self, client, admin_user):
from archivebox.core.models import Snapshot, Tag
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(urls="https://public-xss.example", created_by=admin_user, config={"PERMISSIONS": "public"})
snapshot = Snapshot.objects.create(
url="https://public-xss.example",
title="Safe title before raw SQL",
crawl=crawl,
status=Snapshot.StatusChoices.SEALED,
)
tag = Tag.objects.create(name="safe-tag-before-raw-sql")
snapshot.tags.add(tag)
title_payload = "</script><script id=public-title-xss>window.__archivebox_public_title_xss__=1</script>"
tag_payload = "</script><script id=public-tag-xss>window.__archivebox_public_tag_xss__=1</script>"
with connection.cursor() as cursor:
cursor.execute(f"UPDATE {Snapshot._meta.db_table} SET title = %s WHERE id = %s", [title_payload, str(snapshot.pk)])
cursor.execute(f"UPDATE {Tag._meta.db_table} SET name = %s WHERE id = %s", [tag_payload, str(tag.pk)])
public_index = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
snapshot_detail = client.get(f"/{snapshot.archive_path}/index.html", HTTP_HOST=WEB_TEST_HOST)
assert public_index.status_code == 200
assert snapshot_detail.status_code == 200
for response in (public_index, snapshot_detail):
assert b"<script id=public-title-xss>" not in response.content
assert b"<script id=public-tag-xss>" not in response.content
assert b"&lt;/script&gt;&lt;script id=public-tag-xss&gt;" in response.content
assert b"&lt;/script&gt;&lt;script id=public-title-xss&gt;" in public_index.content
assert b"window.__archivebox_public_title_xss__=1" in snapshot_detail.content
def test_direct_snapshot_urls_allow_unlisted_but_not_private_for_guests(self, client, admin_user):
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl

7
bin/docker_entrypoint.sh
View File

@ -35,9 +35,10 @@ export ABXPKG_LIB_DIR="${ABXPKG_LIB_DIR:-$LIB_DIR}"
export ARCHIVEBOX_USER="${ARCHIVEBOX_USER:-archivebox}"
export PERSONAS_DIR="${PERSONAS_DIR:-$DATA_DIR/personas}"
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$LIB_DIR/playwright/cache}"
export ABXBUS_CACHE_DIR="${ABXBUS_CACHE_DIR:-${XDG_CACHE_HOME:-/home/archivebox/.cache}/abxbus}"
export UV_CACHE_DIR="${UV_CACHE_DIR:-${XDG_CACHE_HOME:-/home/archivebox/.cache}/uv}"
export PNPM_HOME="${PNPM_HOME:-${XDG_CACHE_HOME:-/home/archivebox/.cache}/pnpm}"
export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$LIB_DIR/cache}"
export ABXBUS_CACHE_DIR="${ABXBUS_CACHE_DIR:-$XDG_CACHE_HOME/abxbus}"
export UV_CACHE_DIR="${UV_CACHE_DIR:-$XDG_CACHE_HOME/uv}"
export PNPM_HOME="${PNPM_HOME:-$XDG_CACHE_HOME/pnpm}"
# Global default uid/gid used when /data is empty or root-owned.
export DEFAULT_ARCHIVEBOX_UID="${DEFAULT_ARCHIVEBOX_UID:-911}"