mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-13 18:46:17 +05:00
fix: rename utils.py → util.py across modules, fix add --index-only, misc cleanups
Renames (no functional change, just consistency with the rest of the codebase): - cli/cli_utils.py → cli/cli_util.py - core/host_utils.py → core/host_util.py - core/tag_utils.py → core/tag_util.py - crawls/schedule_utils.py → crawls/schedule_util.py - machine/env_utils.py → machine/env_util.py Functional fixes: - archivebox add --index-only now materializes Snapshot rows synchronously via crawl.create_snapshots_from_urls() instead of just queueing the Crawl and leaving the index empty. The previous behavior broke every test that expected --index-only to populate the index, since the runner is never started in index-only mode. - config/collection.py: add _coerce_from_str_dict as the inverse of _coerce_to_str_dict so JSON-encoded INI values are decoded back to native dict/list types when mirrored into Machine.config (a JSONField). Without this, downstream consumers like MachineEvent / abx-dl get raw JSON strings where they expect dicts. Plus matching admin / middleware / model touch-ups, the registration password_change_form template, and assorted small cleanups the user worked through while validating the deploy path. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Sweeting
parent
a8b54931cd
commit
6ce2555dfd
@ -7,7 +7,7 @@ from django.shortcuts import redirect
|
||||
from django.urls import path
|
||||
from django.views.generic.base import RedirectView
|
||||
|
||||
from archivebox.core.host_utils import build_web_url
|
||||
from archivebox.core.host_util import build_web_url
|
||||
from .v1_api import urls as v1_api_urls
|
||||
|
||||
|
||||
|
||||
@ -31,9 +31,9 @@ from archivebox.core.models import Snapshot, ArchiveResult, Tag
|
||||
from archivebox.core.permissions import public_snapshots_queryset
|
||||
from archivebox.api.auth import auth_using_token
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.host_utils import build_web_url
|
||||
from archivebox.core.host_util import build_web_url
|
||||
from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
|
||||
from archivebox.core.tag_utils import (
|
||||
from archivebox.core.tag_util import (
|
||||
add_snapshot_counts,
|
||||
build_tag_cards,
|
||||
delete_tag as delete_tag_record,
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
__package__ = "archivebox.base_models"
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from typing import NotRequired, TypedDict
|
||||
|
||||
@ -11,10 +12,41 @@ from django.contrib import admin
|
||||
from django.db import models
|
||||
from django.forms.renderers import BaseRenderer
|
||||
from django.http import HttpRequest, QueryDict
|
||||
from django.urls import path, register_converter
|
||||
from django.utils.safestring import SafeString, mark_safe
|
||||
from django_object_actions import DjangoObjectActions
|
||||
|
||||
|
||||
class HexUUIDConverter:
|
||||
"""URL path converter that canonicalizes UUIDs to their 32-char hex form.
|
||||
|
||||
Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
|
||||
(``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
|
||||
parses either), but ``to_url`` always emits the bare-hex form. This is
|
||||
what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
|
||||
``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
|
||||
hyphenated rendering — admin links throughout the app reverse through
|
||||
this converter once ``BaseModelAdmin.get_urls`` swaps in
|
||||
``<hexuuid:object_id>`` below.
|
||||
"""
|
||||
|
||||
regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
|
||||
def to_python(self, value: str) -> str:
|
||||
# Strip hyphens but stay as a string — Django admin treats the
|
||||
# captured object_id as a string and calls ``model._meta.pk.to_python``
|
||||
# itself, so we don't want to short-circuit that.
|
||||
return value.replace("-", "")
|
||||
|
||||
def to_url(self, value) -> str:
|
||||
if isinstance(value, uuid.UUID):
|
||||
return value.hex
|
||||
return str(value).replace("-", "")
|
||||
|
||||
|
||||
register_converter(HexUUIDConverter, "hexuuid")
|
||||
|
||||
|
||||
class ConfigOption(TypedDict):
|
||||
plugin: str
|
||||
type: str | list[str]
|
||||
@ -846,3 +878,30 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
|
||||
if "created_by" in form.base_fields:
|
||||
form.base_fields["created_by"].initial = request.user
|
||||
return form
|
||||
|
||||
def get_urls(self):
|
||||
"""Swap the per-object admin URLs from ``<path:object_id>`` to
|
||||
``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
|
||||
32-char hex form. The hyphenated form still resolves because the
|
||||
converter's regex accepts both — Django reverses through ``to_url``
|
||||
which always emits hex, so links in templates / changelists / inline
|
||||
formsets all canonicalize automatically.
|
||||
|
||||
Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
|
||||
won't match the converter's regex and fall back to the default
|
||||
``<path:object_id>`` patterns we still include after our swap.
|
||||
"""
|
||||
info = self.opts.app_label, self.opts.model_name
|
||||
object_routes = [
|
||||
path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
|
||||
path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
|
||||
path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
|
||||
]
|
||||
# Append after super().get_urls() so our patterns are the
|
||||
# *last-registered* ones with the canonical admin URL names — Django's
|
||||
# reverse() picks the later registration when names collide, which is
|
||||
# how we make ``reverse("admin:app_model_change", args=[obj.pk])``
|
||||
# emit the hex form. The original ``<path:object_id>`` routes stay in
|
||||
# place as a fallback for non-UUID PKs and for resolving inbound
|
||||
# hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
|
||||
return super().get_urls() + object_routes
|
||||
|
||||
@ -198,7 +198,14 @@ def add(
|
||||
# Discovered URLs become child Snapshots (depth+1)
|
||||
|
||||
if index_only:
|
||||
print("[yellow]\\[*] Index-only mode - crawl queued, runner not started[/yellow]")
|
||||
# ``--index-only`` means "add the URLs to the index without archiving
|
||||
# them now". That only holds if we actually materialize the Snapshot
|
||||
# rows here — otherwise the CLI returns success with nothing in the
|
||||
# index, which broke ``test_add_url_after_init`` & friends. Create
|
||||
# the Snapshots synchronously (the same step the runner would do)
|
||||
# but skip starting any worker so extractors don't run.
|
||||
crawl.create_snapshots_from_urls()
|
||||
print("[yellow]\\[*] Index-only mode - URLs indexed, runner not started[/yellow]")
|
||||
return crawl, crawl.snapshot_set.all()
|
||||
|
||||
# 5. Start the crawl runner to process the queue
|
||||
@ -277,7 +284,7 @@ def add(
|
||||
except Exception:
|
||||
rel_output_str = str(crawl.output_dir)
|
||||
|
||||
from archivebox.core.host_utils import build_admin_url
|
||||
from archivebox.core.host_util import build_admin_url
|
||||
|
||||
admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
|
||||
|
||||
|
||||
@ -38,7 +38,7 @@ import sys
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
|
||||
|
||||
@ -33,7 +33,7 @@ import sys
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@ -39,7 +39,7 @@ from collections.abc import Iterable
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@ -27,7 +27,7 @@ import sys
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@ -37,7 +37,7 @@ from collections.abc import Iterable
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
from archivebox.personas import importers as persona_importers
|
||||
|
||||
|
||||
|
||||
@ -30,7 +30,7 @@ import sys
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@ -28,7 +28,7 @@ def schedule(
|
||||
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.crawls.models import Crawl, CrawlSchedule
|
||||
from archivebox.crawls.schedule_utils import validate_schedule
|
||||
from archivebox.crawls.schedule_util import validate_schedule
|
||||
from archivebox.services.runner import run_pending_crawls
|
||||
|
||||
config_overrides = dict(config or {})
|
||||
|
||||
@ -162,7 +162,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
|
||||
# CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the
|
||||
# implicit BASE_URL. Surface what we picked so the user knows where their
|
||||
# links / redirects are going — and tell them how to make it explicit.
|
||||
from archivebox.core.host_utils import derive_base_url_from_csrf
|
||||
from archivebox.core.host_util import derive_base_url_from_csrf
|
||||
|
||||
csrf_derived = derive_base_url_from_csrf(config)
|
||||
if csrf_derived:
|
||||
@ -178,7 +178,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
|
||||
print()
|
||||
return
|
||||
|
||||
# BASE_URL was not set explicitly. The host_utils derivation gives one of
|
||||
# BASE_URL was not set explicitly. The host_util derivation gives one of
|
||||
# three results, with very different risk profiles — show a tailored hint
|
||||
# so new users coming from the 0.7.x single-domain world know whether the
|
||||
# default is fine for them or needs attention.
|
||||
@ -203,7 +203,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
|
||||
)
|
||||
print()
|
||||
else:
|
||||
# Loopback / wildcard bind. The host_utils default of
|
||||
# Loopback / wildcard bind. The host_util default of
|
||||
# http://archivebox.localhost:PORT works in a browser on the same
|
||||
# machine, but anything else (reverse proxy, k8s ingress, LAN client)
|
||||
# needs BASE_URL set. (Real hostnames can't reach this branch — the
|
||||
@ -298,7 +298,7 @@ def server(
|
||||
return
|
||||
|
||||
os.environ["BIND_ADDR"] = f"{host}:{port}"
|
||||
from archivebox.core.host_utils import get_base_url
|
||||
from archivebox.core.host_util import get_base_url
|
||||
|
||||
base_url = get_base_url().rstrip("/")
|
||||
admin_url = f"{base_url}/admin/"
|
||||
|
||||
@ -37,7 +37,7 @@ import rich_click as click
|
||||
from rich import print as rprint
|
||||
from django.db.models import Case, IntegerField, Q, QuerySet, When
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@ -36,7 +36,7 @@ from collections.abc import Iterable
|
||||
import rich_click as click
|
||||
from rich import print as rprint
|
||||
|
||||
from archivebox.cli.cli_utils import apply_filters
|
||||
from archivebox.cli.cli_util import apply_filters
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@ -152,6 +152,37 @@ def mirror_machine_config_to_file(config: Any) -> None:
|
||||
_MIRROR_IN_PROGRESS = False
|
||||
|
||||
|
||||
def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
|
||||
"""Inverse of ``_coerce_to_str_dict``: decode complex INI values to native.
|
||||
|
||||
``mirror_machine_config_to_file`` JSON-encodes ``dict`` / ``list`` values
|
||||
so they round-trip through INI's string-only storage. When reading the
|
||||
file back into ``Machine.config`` (a JSONField that holds native types)
|
||||
those strings have to be decoded — otherwise downstream consumers like
|
||||
``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
|
||||
where they expect a dict and raise ``TypeError``.
|
||||
Delegates to pydantic-settings' own ``field_is_complex`` /
|
||||
``prepare_field_value`` (the same machinery ``IniConfigSettingsSource``
|
||||
uses for the file-read path), so every dict/list/tuple field is
|
||||
decoded according to its declared annotation — no hardcoded type
|
||||
checks, no manual ``json.loads`` per call site.
|
||||
"""
|
||||
from archivebox.config.common import ArchiveBoxConfig
|
||||
from archivebox.config.configset import IniConfigSettingsSource
|
||||
|
||||
decoder = IniConfigSettingsSource(ArchiveBoxConfig)
|
||||
decoded: dict[str, Any] = dict(file_config)
|
||||
for field_name, field in ArchiveBoxConfig.model_fields.items():
|
||||
if field_name not in decoded:
|
||||
continue
|
||||
raw = decoded[field_name]
|
||||
if not isinstance(raw, str) or not raw:
|
||||
continue
|
||||
if decoder.field_is_complex(field):
|
||||
decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
|
||||
return decoded
|
||||
|
||||
|
||||
def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None:
|
||||
"""Copy ``ArchiveBox.conf`` contents into ``Machine.config``.
|
||||
|
||||
@ -163,7 +194,7 @@ def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None:
|
||||
machine = Machine.current()
|
||||
if _coerce_to_str_dict(machine.config) == file_config:
|
||||
return
|
||||
machine.config = dict(file_config)
|
||||
machine.config = _coerce_from_str_dict(file_config)
|
||||
machine.save(update_fields=["config", "modified_at"])
|
||||
|
||||
|
||||
@ -226,7 +257,7 @@ def sync_machine_and_file(machine: Any = None) -> None:
|
||||
if merged != file_config:
|
||||
_write_file_if_changed(_render_config_file_content(merged))
|
||||
if merged != machine_config:
|
||||
machine.config = dict(merged)
|
||||
machine.config = _coerce_from_str_dict(merged)
|
||||
machine.save(update_fields=["config", "modified_at"])
|
||||
finally:
|
||||
_MIRROR_IN_PROGRESS = False
|
||||
|
||||
@ -17,7 +17,7 @@ from pydantic import BaseModel, Field, create_model, field_validator, model_vali
|
||||
from pydantic_settings import SettingsConfigDict
|
||||
from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
|
||||
|
||||
from archivebox.config.configset import BaseConfigSet
|
||||
from archivebox.config.configset import BaseConfigSet, IniConfigSettingsSource
|
||||
from archivebox.config.configset import COMPUTED_CONFIG_KEYS
|
||||
|
||||
from .constants import CONSTANTS
|
||||
@ -654,6 +654,29 @@ def get_config(
|
||||
|
||||
config_data["ABX_RUNTIME"] = "archivebox"
|
||||
|
||||
# Decode JSON-encoded complex values (dict/list fields) that came from
|
||||
# string-only sources before validation. ``IniConfigSettingsSource`` does
|
||||
# this for the ArchiveBox.conf path, but Machine.config (mirrored from the
|
||||
# INI via ``_coerce_to_str_dict``) and plugin/env scope overrides bypass
|
||||
# pydantic-settings sources entirely — they feed JSON strings directly
|
||||
# into ``model_validate``, which rejects ``"{...}"`` for a ``dict[str, str]``
|
||||
# field. Run pydantic-settings' own complex-value decoder here so every
|
||||
# source converges on the same shape before validation.
|
||||
_complex_decoder = IniConfigSettingsSource(ArchiveBoxConfig)
|
||||
for _field_name, _field in ArchiveBoxConfig.model_fields.items():
|
||||
if _field_name not in config_data:
|
||||
continue
|
||||
_raw = config_data[_field_name]
|
||||
if not isinstance(_raw, str) or not _raw:
|
||||
continue
|
||||
if _complex_decoder.field_is_complex(_field):
|
||||
config_data[_field_name] = _complex_decoder.prepare_field_value(
|
||||
_field_name,
|
||||
_field,
|
||||
_raw,
|
||||
True,
|
||||
)
|
||||
|
||||
config = ArchiveBoxConfig.model_validate(config_data)
|
||||
os.environ["LIB_DIR"] = str(config.LIB_DIR)
|
||||
os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
|
||||
|
||||
@ -360,12 +360,27 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
|
||||
|
||||
|
||||
def get_db_binaries_by_name() -> dict[str, Binary]:
|
||||
"""Group Binary rows by a URL-safe canonical name.
|
||||
|
||||
Hooks occasionally emit ``BinaryEvent.name`` carrying an abspath rather
|
||||
than a short binary name (see ``services/binary_service.py``). That used
|
||||
to leak ``name='/Users/.../bin/foo'`` rows into the DB, which then broke
|
||||
``/admin/environment/binaries`` because the admin URL regex is
|
||||
``(?P<key>[^/]+)``. Canonicalize at the keying step so duplicates fold
|
||||
into the real binary and the admin link key stays slash-free regardless
|
||||
of legacy DB state.
|
||||
"""
|
||||
from archivebox.machine.models import _canonical_binary_name
|
||||
|
||||
grouped: dict[str, list[Binary]] = {}
|
||||
binary_name_aliases = {
|
||||
"youtube-dl": "yt-dlp",
|
||||
}
|
||||
for binary in Binary.objects.all():
|
||||
canonical_name = binary_name_aliases.get(binary.name, binary.name)
|
||||
canonical_name = _canonical_binary_name(binary.name)
|
||||
canonical_name = binary_name_aliases.get(canonical_name, canonical_name)
|
||||
if not canonical_name:
|
||||
continue
|
||||
grouped.setdefault(canonical_name, []).append(binary)
|
||||
|
||||
return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()}
|
||||
@ -628,6 +643,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
|
||||
rows = {
|
||||
"Name": [],
|
||||
"Type": [],
|
||||
"State": [],
|
||||
"PID": [],
|
||||
"Started": [],
|
||||
@ -657,12 +673,115 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
continue
|
||||
all_config[config_name] = config_data
|
||||
|
||||
# Add top row for supervisord process manager
|
||||
# Collect every PID we plan to show so we can resolve them to Process rows
|
||||
# in a single query. supervisord's per-worker description carries the pid
|
||||
# in the form ``pid 12345, uptime 0:01:23`` (or just the bare ``pid``
|
||||
# placeholder when stopped); we ignore non-numeric values.
|
||||
process_items = supervisor.getAllProcessInfo()
|
||||
if not isinstance(process_items, list):
|
||||
process_items = []
|
||||
|
||||
def _parse_worker_pid_and_uptime(description: str) -> tuple[int | None, str]:
|
||||
body = description.replace("pid ", "", 1)
|
||||
pid_part, _, uptime_part = body.partition(", uptime ")
|
||||
try:
|
||||
return int(pid_part.strip()), uptime_part.strip()
|
||||
except ValueError:
|
||||
return None, ""
|
||||
|
||||
pids: set[int] = set()
|
||||
supervisor_pid = supervisor.getPID()
|
||||
if isinstance(supervisor_pid, int):
|
||||
pids.add(supervisor_pid)
|
||||
for proc_data in process_items:
|
||||
if not isinstance(proc_data, dict):
|
||||
continue
|
||||
pid_int, _ = _parse_worker_pid_and_uptime(str(proc_data.get("description") or ""))
|
||||
if pid_int is not None:
|
||||
pids.add(pid_int)
|
||||
|
||||
pid_to_process_id: dict[int, str] = {}
|
||||
pid_to_process_type: dict[int, str] = {}
|
||||
if pids:
|
||||
try:
|
||||
from archivebox.machine.models import Machine, Process
|
||||
|
||||
for row in (
|
||||
Process.objects.filter(machine=Machine.current(), pid__in=pids)
|
||||
.order_by("pid", "-started_at", "-created_at")
|
||||
.only("id", "pid", "process_type")
|
||||
):
|
||||
if row.pid in pid_to_process_id:
|
||||
continue # keep the most recent row per PID
|
||||
pid_to_process_id[row.pid] = str(row.id)
|
||||
pid_to_process_type[row.pid] = row.process_type
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _pid_cell(pid_value: int | None, uptime_str: str = ""):
|
||||
if pid_value is None:
|
||||
return ""
|
||||
pid_text = str(pid_value)
|
||||
process_id = pid_to_process_id.get(pid_value)
|
||||
if process_id:
|
||||
link = format_html('<a href="/admin/machine/process/{}/change/">{}</a>', process_id, pid_text)
|
||||
else:
|
||||
link = format_html("{}", pid_text)
|
||||
if uptime_str:
|
||||
return format_html("{}, uptime {}", link, uptime_str)
|
||||
return link
|
||||
|
||||
# Add top row for supervisord process manager. supervisord exposes its
|
||||
# state + pid over XML-RPC but not its own start time / exit status / uptime,
|
||||
# so we read those from the OS process (or fall back to the Process row
|
||||
# recorded in _record_supervisord_process). Exit status stays blank while
|
||||
# it's RUNNING — supervisord wouldn't be answering RPC if it had exited.
|
||||
rows["Name"].append(ItemLink("supervisord", key="supervisord"))
|
||||
rows["Type"].append("supervisord")
|
||||
supervisor_state = supervisor.getState()
|
||||
rows["State"].append(str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else ""))
|
||||
rows["PID"].append(str(supervisor.getPID()))
|
||||
rows["Started"].append("-")
|
||||
state_name = str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else "")
|
||||
rows["State"].append(state_name)
|
||||
|
||||
supervisor_started = ""
|
||||
supervisor_uptime = ""
|
||||
try:
|
||||
import time as _time
|
||||
|
||||
import psutil
|
||||
|
||||
ps_proc = psutil.Process(supervisor_pid)
|
||||
create_time = ps_proc.create_time()
|
||||
supervisor_started = format_parsed_datetime(create_time)
|
||||
seconds = max(int(_time.time() - create_time), 0)
|
||||
hours, remainder = divmod(seconds, 3600)
|
||||
minutes, secs = divmod(remainder, 60)
|
||||
supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
|
||||
except Exception:
|
||||
try:
|
||||
from archivebox.machine.models import Machine, Process
|
||||
|
||||
row = (
|
||||
Process.objects.filter(
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.SUPERVISORD,
|
||||
pid=supervisor_pid,
|
||||
)
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
)
|
||||
if row and row.started_at:
|
||||
supervisor_started = row.started_at.strftime("%Y-%m-%d %H:%M:%S")
|
||||
seconds = max(int((timezone.now() - row.started_at).total_seconds()), 0)
|
||||
hours, remainder = divmod(seconds, 3600)
|
||||
minutes, secs = divmod(remainder, 60)
|
||||
supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
|
||||
rows["Started"].append(supervisor_started or "-")
|
||||
|
||||
rows["Command"].append("supervisord --configuration=tmp/supervisord.conf")
|
||||
rows["Logfile"].append(
|
||||
format_html(
|
||||
@ -671,12 +790,9 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
"logs/supervisord.log",
|
||||
),
|
||||
)
|
||||
rows["Exit Status"].append("0")
|
||||
rows["Exit Status"].append("" if state_name == "RUNNING" else "-")
|
||||
|
||||
# Add a row for each worker process managed by supervisord
|
||||
process_items = supervisor.getAllProcessInfo()
|
||||
if not isinstance(process_items, list):
|
||||
process_items = []
|
||||
for proc_data in process_items:
|
||||
if not isinstance(proc_data, dict):
|
||||
continue
|
||||
@ -685,10 +801,15 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
proc_start = proc_data.get("start")
|
||||
proc_logfile = str(proc_data.get("stdout_logfile") or "")
|
||||
proc_config = all_config.get(proc_name, {})
|
||||
pid_int, uptime_str = _parse_worker_pid_and_uptime(proc_description)
|
||||
|
||||
rows["Name"].append(ItemLink(proc_name, key=proc_name))
|
||||
# Prefer the Process row's process_type when we have one (e.g. "worker",
|
||||
# "hook"); otherwise fall back to the generic "worker" label since
|
||||
# everything in this loop is supervisord-managed.
|
||||
rows["Type"].append(pid_to_process_type.get(pid_int, "worker") if pid_int else "worker")
|
||||
rows["State"].append(str(proc_data.get("statename") or ""))
|
||||
rows["PID"].append(proc_description.replace("pid ", ""))
|
||||
rows["PID"].append(_pid_cell(pid_int, uptime_str))
|
||||
rows["Started"].append(format_parsed_datetime(proc_start))
|
||||
rows["Command"].append(str(proc_config.get("command") or ""))
|
||||
rows["Logfile"].append(
|
||||
@ -811,7 +932,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
||||
f.seek(0)
|
||||
last_lines = f.read().decode("utf-8", errors="replace").split("\n")
|
||||
non_empty_lines = [line for line in last_lines if line.strip()]
|
||||
rows["Most Recent Lines"].append(non_empty_lines[-1])
|
||||
rows["Most Recent Lines"].append(non_empty_lines[-1] if non_empty_lines else "")
|
||||
|
||||
return TableContext(
|
||||
title="Debug Log files",
|
||||
|
||||
@ -24,10 +24,10 @@ from archivebox.config.common import get_config
|
||||
from archivebox.misc.paginators import AcceleratedPaginator
|
||||
from archivebox.base_models.admin import BaseModelAdmin
|
||||
from archivebox.hooks import get_plugin_icon
|
||||
from archivebox.core.host_utils import build_snapshot_url
|
||||
from archivebox.core.host_util import build_snapshot_url
|
||||
from archivebox.core.widgets import InlineTagEditorWidget
|
||||
from archivebox.core.views import LIVE_PLUGIN_BASE_URL
|
||||
from archivebox.machine.env_utils import env_to_shell_exports
|
||||
from archivebox.machine.env_util import env_to_shell_exports
|
||||
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
|
||||
@ -28,8 +28,8 @@ from archivebox.misc.util import htmldecode, urldecode
|
||||
from archivebox.misc.paginators import AcceleratedPaginator
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key
|
||||
from archivebox.core.host_utils import build_snapshot_url, build_web_url
|
||||
from archivebox.core.tag_utils import get_or_create_tag
|
||||
from archivebox.core.host_util import build_snapshot_url, build_web_url
|
||||
from archivebox.core.tag_util import get_or_create_tag
|
||||
from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
|
||||
@ -1609,13 +1609,19 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
# on the resulting crawl so existing snapshots don't cause the crawl to
|
||||
# seal immediately with zero new snapshots (the default ONLY_NEW=True
|
||||
# would skip any URLs that have ever been archived before).
|
||||
add(urls=urls, bg=True, config={"ONLY_NEW": False})
|
||||
crawl, _ = add(urls=urls, bg=True, config={"ONLY_NEW": False})
|
||||
|
||||
messages.success(
|
||||
request,
|
||||
f"Created 1 queued crawl with {len(snapshots)} URL(s). The background runner will create snapshots and process them.",
|
||||
)
|
||||
|
||||
# Redirect to the new crawl's admin page so the user lands on the
|
||||
# work-in-progress crawl, not the old snapshot they re-archived from.
|
||||
# A snapshot-view redirect would race the runner — the new snapshot
|
||||
# may sit queued for a while before the runner creates the DB row.
|
||||
return redirect(f"/admin/crawls/crawl/{crawl.id}/change/#snapshots")
|
||||
|
||||
@admin.action(
|
||||
description="🔄 Redo",
|
||||
)
|
||||
|
||||
@ -13,7 +13,7 @@ from django.utils.safestring import mark_safe
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin
|
||||
from archivebox.core.models import SnapshotTag, Tag
|
||||
from archivebox.core.tag_utils import (
|
||||
from archivebox.core.tag_util import (
|
||||
TAG_HAS_SNAPSHOTS_CHOICES,
|
||||
TAG_SORT_CHOICES,
|
||||
build_tag_cards,
|
||||
@ -24,7 +24,7 @@ from archivebox.core.tag_utils import (
|
||||
normalize_has_snapshots_filter,
|
||||
normalize_tag_sort,
|
||||
)
|
||||
from archivebox.core.host_utils import build_snapshot_url
|
||||
from archivebox.core.host_util import build_snapshot_url
|
||||
|
||||
|
||||
class TagInline(admin.TabularInline):
|
||||
|
||||
@ -13,7 +13,7 @@ from django.utils.html import format_html
|
||||
from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
|
||||
from taggit.utils import edit_string_for_tags, parse_tags
|
||||
from archivebox.base_models.admin import KeyValueWidget
|
||||
from archivebox.crawls.schedule_utils import validate_schedule
|
||||
from archivebox.crawls.schedule_util import validate_schedule
|
||||
from archivebox.config.common import get_config, parse_delete_after
|
||||
from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
|
||||
from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
|
||||
|
||||
@ -87,27 +87,6 @@ def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwa
|
||||
return ""
|
||||
|
||||
|
||||
def request_host_is_explicitly_allowed(request_host: str, config) -> bool:
|
||||
"""True if the incoming Host is in ALLOWED_HOSTS or matches a CSRF origin.
|
||||
|
||||
Used by ``get_base_url`` to honour the request's own Host header when the
|
||||
operator hasn't pinned ``BASE_URL`` explicitly. The check is intentionally
|
||||
strict — we won't trust arbitrary Host headers, only ones the user has
|
||||
already opted into via ALLOWED_HOSTS / CSRF_TRUSTED_ORIGINS.
|
||||
"""
|
||||
if not request_host:
|
||||
return False
|
||||
host, _ = split_host_port(request_host)
|
||||
allowed = _allowed_hosts(config)
|
||||
if host in allowed:
|
||||
return True
|
||||
for origin in _csrf_trusted_origins(config):
|
||||
origin_host, _ = split_host_port(urlparse(origin).netloc)
|
||||
if origin_host == host:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
|
||||
config = config or get_config(**config_kwargs)
|
||||
return (config.BIND_ADDR or "").strip()
|
||||
@ -15,7 +15,7 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.config import VERSION
|
||||
from archivebox.config.version import get_COMMIT_HASH
|
||||
from archivebox.core.host_utils import (
|
||||
from archivebox.core.host_util import (
|
||||
build_snapshot_url,
|
||||
build_admin_url,
|
||||
build_web_url,
|
||||
|
||||
@ -648,7 +648,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
return False
|
||||
|
||||
from archivebox.core.host_utils import get_admin_host, get_api_host, get_listen_host, get_public_host, get_web_host, split_host_port
|
||||
from archivebox.core.host_util import get_admin_host, get_api_host, get_listen_host, get_public_host, get_web_host, split_host_port
|
||||
|
||||
config = get_config()
|
||||
host = parsed.hostname.lower().strip(".")
|
||||
@ -2989,7 +2989,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
|
||||
def to_dict(self, extended: bool = False) -> dict[str, Any]:
|
||||
"""Convert Snapshot to a dictionary (replacement for Link._asdict())"""
|
||||
from archivebox.core.host_utils import build_snapshot_url
|
||||
from archivebox.core.host_util import build_snapshot_url
|
||||
|
||||
archive_size = self.archive_size
|
||||
|
||||
|
||||
@ -8,13 +8,12 @@ import importlib
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf.locale.en import formats as en_formats # type: ignore
|
||||
from django.utils.crypto import get_random_string
|
||||
|
||||
import archivebox
|
||||
|
||||
from archivebox.config.constants import CONSTANTS
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.host_utils import normalize_base_url, get_admin_base_url, get_api_base_url
|
||||
from archivebox.core.host_util import normalize_base_url, get_admin_base_url, get_api_base_url
|
||||
from .settings_logging import SETTINGS_LOGGING
|
||||
|
||||
|
||||
@ -359,7 +358,34 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}
|
||||
### Security Settings
|
||||
################################################################################
|
||||
|
||||
SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_")
|
||||
# Persist SECRET_KEY on first use. Data dirs created before init wrote a
|
||||
# SECRET_KEY line — or those whose ArchiveBox.conf was hand-edited to remove
|
||||
# it — would otherwise sign session cookies with a fresh random key on every
|
||||
# boot (because the pydantic field's default_factory regenerates), logging
|
||||
# users out on every server restart. ``archivebox init`` writes this for new
|
||||
# collections; this branch is the recovery path for the rest.
|
||||
#
|
||||
# We can't check ``CONFIG.SECRET_KEY`` for "missing" — pydantic's
|
||||
# default_factory already filled it with a fresh random value. We have to
|
||||
# inspect the conf file directly to know whether the value will survive.
|
||||
SECRET_KEY = CONFIG.SECRET_KEY
|
||||
try:
|
||||
from archivebox.config.configset import BaseConfigSet as _BaseConfigSet
|
||||
|
||||
_persisted_keys = _BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
|
||||
_secret_persisted = bool((_persisted_keys.get("SECRET_KEY") or "").strip())
|
||||
except Exception:
|
||||
_secret_persisted = True # err on the side of NOT touching disk
|
||||
if not _secret_persisted:
|
||||
try:
|
||||
from archivebox.config.collection import write_config_file
|
||||
|
||||
write_config_file({"SECRET_KEY": SECRET_KEY})
|
||||
except Exception:
|
||||
# Read-only mount, missing data dir, mid-init race — fall back to the
|
||||
# in-memory random key. The user will get logged out on the next boot
|
||||
# but the server still comes up.
|
||||
pass
|
||||
|
||||
ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()]
|
||||
CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()})
|
||||
|
||||
@ -67,6 +67,40 @@ class DaphneCloseTimeoutFilter(logging.Filter):
|
||||
return True
|
||||
|
||||
|
||||
class AsyncioCancelledShieldFilter(logging.Filter):
|
||||
"""Drop asyncio's "CancelledError exception in shielded future" noise.
|
||||
|
||||
When a browser disconnects mid-request, daphne cancels the asgi task and
|
||||
asgiref's ``sync_to_async`` shields the synchronous Django view via
|
||||
``asyncio.shield(exec_coro)``. The shield wakes up to find its parent
|
||||
cancelled and re-raises ``CancelledError``; asyncio's default exception
|
||||
handler then logs the full traceback via the ``asyncio`` logger. There's
|
||||
nothing the server can do (the client is already gone), so these tracebacks
|
||||
are pure noise and cause hundreds of lines of spam per disconnect.
|
||||
|
||||
We match conservatively: only drop the specific shielded-future message
|
||||
that points back into asgiref's shield path. Other asyncio errors fall
|
||||
through unchanged.
|
||||
"""
|
||||
|
||||
def filter(self, record) -> bool:
|
||||
if record.name != "asyncio":
|
||||
return True
|
||||
logline = record.getMessage()
|
||||
if "CancelledError exception in shielded future" in logline:
|
||||
return False
|
||||
exc_info = record.exc_info
|
||||
if exc_info and exc_info[0] is not None:
|
||||
try:
|
||||
import asyncio as _asyncio
|
||||
|
||||
if issubclass(exc_info[0], _asyncio.CancelledError):
|
||||
return False
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
class CustomOutboundWebhookLogFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
result = super().format(record)
|
||||
@ -128,6 +162,9 @@ SETTINGS_LOGGING = {
|
||||
"daphneclosetimeout": {
|
||||
"()": DaphneCloseTimeoutFilter,
|
||||
},
|
||||
"asynciocancelledshield": {
|
||||
"()": AsyncioCancelledShieldFilter,
|
||||
},
|
||||
"stripansi": {
|
||||
"()": StripANSIColorCodesFilter,
|
||||
},
|
||||
@ -145,7 +182,7 @@ SETTINGS_LOGGING = {
|
||||
"level": "DEBUG",
|
||||
"markup": False,
|
||||
"rich_tracebacks": False, # Use standard Python tracebacks (no frame/box)
|
||||
"filters": ["noisyrequestsfilter", "daphneclosetimeout", "stripansi"],
|
||||
"filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
|
||||
},
|
||||
"logfile": {
|
||||
"level": "INFO",
|
||||
@ -154,7 +191,7 @@ SETTINGS_LOGGING = {
|
||||
"maxBytes": 1024 * 1024 * 25, # 25 MB
|
||||
"backupCount": 10,
|
||||
"formatter": "rich",
|
||||
"filters": ["noisyrequestsfilter", "daphneclosetimeout", "stripansi"],
|
||||
"filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
|
||||
},
|
||||
"outbound_webhooks": {
|
||||
"class": "rich.logging.RichHandler",
|
||||
|
||||
@ -10,12 +10,6 @@ from django.db.backends.sqlite3.base import DatabaseWrapper as DjangoSQLiteDatab
|
||||
from django.db.backends.sqlite3.base import SQLiteCursorWrapper as DjangoSQLiteCursorWrapper
|
||||
|
||||
|
||||
def _is_locked_error(error: BaseException) -> bool:
|
||||
from django.db import OperationalError
|
||||
|
||||
return isinstance(error, (sqlite3.OperationalError, OperationalError)) and "database is locked" in str(error).lower()
|
||||
|
||||
|
||||
def _sqlite_lock_retry_timeout() -> float:
|
||||
from django.conf import settings
|
||||
|
||||
@ -111,7 +105,9 @@ def _retry_locked_database(action, query: str, params=None, *, db_wrapper=None):
|
||||
try:
|
||||
return action()
|
||||
except (sqlite3.OperationalError, Exception) as err:
|
||||
if not _is_locked_error(err):
|
||||
from archivebox.misc.db import sqlite_lock_error
|
||||
|
||||
if not sqlite_lock_error(err):
|
||||
raise
|
||||
attempt += 1
|
||||
elapsed = time.monotonic() - started_at
|
||||
|
||||
@ -12,7 +12,7 @@ from django.http import HttpRequest
|
||||
from django.urls import reverse
|
||||
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.host_utils import build_snapshot_url, build_web_url
|
||||
from archivebox.core.host_util import build_snapshot_url, build_web_url
|
||||
from archivebox.core.models import Snapshot, SnapshotTag, Tag
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ from archivebox.hooks import (
|
||||
get_plugin_template,
|
||||
get_plugin_name,
|
||||
)
|
||||
from archivebox.core.host_utils import (
|
||||
from archivebox.core.host_util import (
|
||||
canonical_base_host_for_request,
|
||||
get_admin_base_url,
|
||||
get_public_base_url,
|
||||
|
||||
@ -66,7 +66,7 @@ from archivebox.core.permissions import (
|
||||
is_admin_user,
|
||||
public_snapshots_queryset,
|
||||
)
|
||||
from archivebox.core.host_utils import (
|
||||
from archivebox.core.host_util import (
|
||||
build_admin_url,
|
||||
build_snapshot_url,
|
||||
build_web_url,
|
||||
|
||||
@ -34,7 +34,7 @@ from archivebox.base_models.models import (
|
||||
get_or_create_system_user_pk,
|
||||
)
|
||||
from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
|
||||
from archivebox.crawls.schedule_utils import next_run_for_schedule, validate_schedule
|
||||
from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
|
||||
from archivebox.misc.util import validate_url, validate_url_length
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@ -2,6 +2,7 @@ __package__ = "archivebox.machine"
|
||||
|
||||
import json
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
|
||||
from django.contrib import admin, messages
|
||||
from django.db.models import DurationField, ExpressionWrapper, F
|
||||
@ -14,7 +15,7 @@ from django_object_actions import action
|
||||
|
||||
from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
from archivebox.machine.env_utils import env_to_dotenv_text
|
||||
from archivebox.machine.env_util import env_to_dotenv_text
|
||||
from archivebox.machine.models import Machine, NetworkInterface, Binary, Process
|
||||
|
||||
|
||||
@ -398,7 +399,7 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
"snapshot_link",
|
||||
"crawl_link",
|
||||
"cmd_str",
|
||||
"status",
|
||||
"status_badge",
|
||||
"duration_display",
|
||||
"exit_code",
|
||||
"pid",
|
||||
@ -413,7 +414,7 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
"snapshot_link",
|
||||
"crawl_link",
|
||||
"cmd_str",
|
||||
"status",
|
||||
"status_badge",
|
||||
"duration_display",
|
||||
"exit_code",
|
||||
"pid",
|
||||
@ -628,11 +629,53 @@ class ProcessAdmin(BaseModelAdmin):
|
||||
def cmd_str(self, process):
|
||||
if not process.cmd:
|
||||
return "-"
|
||||
cmd = " ".join(process.cmd[:3]) if isinstance(process.cmd, list) else str(process.cmd)
|
||||
if len(process.cmd) > 3:
|
||||
cmd += " ..."
|
||||
# Compact the list-view rendering only — the change-page ``cmd_display``
|
||||
# still shows the full original ``process.cmd`` (and the DB row is
|
||||
# untouched). If the first argv token looks like an absolute path,
|
||||
# collapse it to its basename so a row like
|
||||
# ``/Users/.../.venv/bin/python -m archivebox foo`` reads as
|
||||
# ``python -m archivebox foo`` in the column.
|
||||
if isinstance(process.cmd, list):
|
||||
parts = [str(arg) for arg in process.cmd[:3]]
|
||||
if parts and (parts[0].startswith("/") or parts[0].startswith("~")):
|
||||
parts[0] = Path(parts[0]).name
|
||||
cmd = " ".join(parts)
|
||||
if len(process.cmd) > 3:
|
||||
cmd += " ..."
|
||||
else:
|
||||
cmd = str(process.cmd)
|
||||
return format_html('<code style="font-size: 0.9em;">{}</code>', cmd[:80])
|
||||
|
||||
@admin.display(description="Status", ordering="status")
|
||||
def status_badge(self, process):
|
||||
# Pill-style badge matching the look of other admin status columns.
|
||||
# Color rules requested by the operator:
|
||||
# RUNNING → green
|
||||
# EXITED, code == 0 → grey (clean exit)
|
||||
# EXITED, code == 10 → grey (treated as a clean / "skipped" exit)
|
||||
# EXITED, code other → red (failure)
|
||||
# QUEUED / anything → amber so it stands out without screaming
|
||||
status_value = str(process.status or "").lower()
|
||||
label = (process.get_status_display() or status_value or "?").upper()
|
||||
exit_code = process.exit_code
|
||||
if status_value == Process.StatusChoices.RUNNING:
|
||||
bg, fg = "#16a34a", "#fff"
|
||||
elif status_value == Process.StatusChoices.EXITED:
|
||||
if exit_code in (0, 10):
|
||||
bg, fg = "#6b7280", "#fff"
|
||||
else:
|
||||
bg, fg = "#dc2626", "#fff"
|
||||
else:
|
||||
bg, fg = "#f59e0b", "#fff"
|
||||
return format_html(
|
||||
'<span style="display:inline-block;background:{};color:{};'
|
||||
"padding:1px 6px;border-radius:3px;font-weight:800;font-size:11px;"
|
||||
'letter-spacing:0.3px;text-transform:uppercase;">{}</span>',
|
||||
bg,
|
||||
fg,
|
||||
label,
|
||||
)
|
||||
|
||||
@admin.display(description="Duration", ordering="runtime_sort")
|
||||
def duration_display(self, process):
|
||||
return _format_process_duration_seconds(process.started_at, process.ended_at)
|
||||
|
||||
@ -144,7 +144,9 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
|
||||
|
||||
|
||||
def sqlite_lock_error(error: BaseException) -> bool:
|
||||
return isinstance(error, SQLiteOperationalError) and "database is locked" in str(error).lower()
|
||||
from django.db import OperationalError as DjangoOperationalError
|
||||
|
||||
return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and "database is locked" in str(error).lower()
|
||||
|
||||
|
||||
def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO | None = None) -> Any:
|
||||
|
||||
@ -13,7 +13,6 @@ from multiprocessing import Process
|
||||
from pathlib import Path
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional, IO, TYPE_CHECKING, cast
|
||||
from collections.abc import Iterable
|
||||
|
||||
@ -23,35 +22,12 @@ if TYPE_CHECKING:
|
||||
from rich import print
|
||||
from rich.panel import Panel
|
||||
|
||||
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
|
||||
from archivebox.config import DATA_DIR, VERSION
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.misc.system import get_dir_size
|
||||
from archivebox.misc.util import enforce_types
|
||||
from archivebox.misc.logging import ANSI
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuntimeStats:
|
||||
"""mutable stats counter for logging archiving timing info to CLI output"""
|
||||
|
||||
skipped: int = 0
|
||||
succeeded: int = 0
|
||||
failed: int = 0
|
||||
|
||||
parse_start_ts: datetime | None = None
|
||||
parse_end_ts: datetime | None = None
|
||||
|
||||
index_start_ts: datetime | None = None
|
||||
index_end_ts: datetime | None = None
|
||||
|
||||
archiving_start_ts: datetime | None = None
|
||||
archiving_end_ts: datetime | None = None
|
||||
|
||||
|
||||
# globals are bad, mmkay
|
||||
_LAST_RUN_STATS = RuntimeStats()
|
||||
|
||||
|
||||
class TimedProgress:
|
||||
"""Show a progress bar and measure elapsed time until .end() is called"""
|
||||
|
||||
@ -176,257 +152,6 @@ def log_cli_command(subcommand: str, subcommand_args: Iterable[str] = (), stdin:
|
||||
print(Panel(version_msg), file=sys.stderr)
|
||||
|
||||
|
||||
### Parsing Stage
|
||||
|
||||
|
||||
def log_importing_started(urls: str | list[str], depth: int, index_only: bool):
|
||||
_LAST_RUN_STATS.parse_start_ts = datetime.now(timezone.utc)
|
||||
print(
|
||||
"[green][+] [{}] Adding {} links to index (crawl depth={}){}...[/]".format(
|
||||
_LAST_RUN_STATS.parse_start_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
len(urls) if isinstance(urls, list) else len(urls.split("\n")),
|
||||
depth,
|
||||
" (index only)" if index_only else "",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def log_source_saved(source_file: str):
|
||||
print(" > Saved verbatim input to {}/{}".format(CONSTANTS.SOURCES_DIR_NAME, source_file.rsplit("/", 1)[-1]))
|
||||
|
||||
|
||||
def log_parsing_finished(num_parsed: int, parser_name: str):
|
||||
_LAST_RUN_STATS.parse_end_ts = datetime.now(timezone.utc)
|
||||
print(f" > Parsed {num_parsed} URLs from input ({parser_name})")
|
||||
|
||||
|
||||
def log_deduping_finished(num_new_links: int):
|
||||
print(f" > Found {num_new_links} new URLs not already in index")
|
||||
|
||||
|
||||
def log_crawl_started(new_links):
|
||||
print()
|
||||
print(f"[green][*] Starting crawl of {len(new_links)} sites 1 hop out from starting point[/]")
|
||||
|
||||
|
||||
### Indexing Stage
|
||||
|
||||
|
||||
def log_indexing_process_started(num_links: int):
|
||||
start_ts = datetime.now(timezone.utc)
|
||||
_LAST_RUN_STATS.index_start_ts = start_ts
|
||||
print()
|
||||
print(
|
||||
"[bright_black][*] [{}] Writing {} links to main index...[/]".format(
|
||||
start_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
num_links,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def log_indexing_process_finished():
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
_LAST_RUN_STATS.index_end_ts = end_ts
|
||||
|
||||
|
||||
def _display_data_path(out_path: str) -> str:
|
||||
path = Path(out_path).resolve()
|
||||
try:
|
||||
return f"./{path.relative_to(DATA_DIR)}"
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def log_indexing_started(out_path: str, config=None, **config_kwargs):
|
||||
config = config or get_config(**config_kwargs)
|
||||
if config.IS_TTY:
|
||||
sys.stdout.write(f" > {_display_data_path(out_path)}")
|
||||
|
||||
|
||||
def log_indexing_finished(out_path: str):
|
||||
print(f"\r √ {_display_data_path(out_path)}")
|
||||
|
||||
|
||||
### Archiving Stage
|
||||
|
||||
|
||||
def log_archiving_started(num_links: int, resume: float | None = None):
|
||||
|
||||
start_ts = datetime.now(timezone.utc)
|
||||
_LAST_RUN_STATS.archiving_start_ts = start_ts
|
||||
print()
|
||||
if resume:
|
||||
print(
|
||||
"[green][▶] [{}] Resuming archive updating for {} pages starting from {}...[/]".format(
|
||||
start_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
num_links,
|
||||
resume,
|
||||
),
|
||||
)
|
||||
else:
|
||||
print(
|
||||
"[green][▶] [{}] Starting archiving of {} snapshots in index...[/]".format(
|
||||
start_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
num_links,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def log_archiving_paused(num_links: int, idx: int, timestamp: str):
|
||||
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
_LAST_RUN_STATS.archiving_end_ts = end_ts
|
||||
print()
|
||||
print(
|
||||
"\n[yellow3][X] [{now}] Downloading paused on link {timestamp} ({idx}/{total})[/]".format(
|
||||
now=end_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
idx=idx + 1,
|
||||
timestamp=timestamp,
|
||||
total=num_links,
|
||||
),
|
||||
)
|
||||
print()
|
||||
print(" Continue archiving where you left off by running:")
|
||||
print(f" archivebox update --resume={timestamp}")
|
||||
|
||||
|
||||
def log_archiving_finished(num_links: int):
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
_LAST_RUN_STATS.archiving_end_ts = end_ts
|
||||
assert _LAST_RUN_STATS.archiving_start_ts is not None
|
||||
seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp()
|
||||
if seconds > 60:
|
||||
duration = f"{seconds / 60:.2f} min"
|
||||
else:
|
||||
duration = f"{seconds:.2f} sec"
|
||||
|
||||
print()
|
||||
print(
|
||||
"[green][√] [{}] Update of {} pages complete ({})[/]".format(
|
||||
end_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
num_links,
|
||||
duration,
|
||||
),
|
||||
)
|
||||
print(f" - {_LAST_RUN_STATS.skipped} links skipped")
|
||||
print(f" - {_LAST_RUN_STATS.succeeded + _LAST_RUN_STATS.failed} links updated")
|
||||
print(f" - {_LAST_RUN_STATS.failed} links had errors")
|
||||
|
||||
if Snapshot.objects.count() < 50:
|
||||
print()
|
||||
print(" [violet]Hint:[/] To manage your archive in a Web UI, run:")
|
||||
print(" archivebox server 0.0.0.0:8000")
|
||||
|
||||
|
||||
def log_snapshot_archiving_started(snapshot: "Snapshot", out_dir: str, is_new: bool):
|
||||
|
||||
# [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford"
|
||||
# http://www.benstopford.com/2015/02/14/log-structured-merge-trees/
|
||||
# > output/archive/1478739709
|
||||
|
||||
print(
|
||||
'\n[[{symbol_color}]{symbol}[/]] [[{symbol_color}]{now}[/]] "{title}"'.format(
|
||||
symbol_color="green" if is_new else "bright_black",
|
||||
symbol="+" if is_new else "√",
|
||||
now=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"),
|
||||
title=snapshot.title or snapshot.base_url,
|
||||
),
|
||||
)
|
||||
print(f" [sky_blue1]{snapshot.url}[/]")
|
||||
print(
|
||||
" {} {}".format(
|
||||
">" if is_new else "√",
|
||||
pretty_path(out_dir),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def log_snapshot_archiving_finished(snapshot: "Snapshot", out_dir: str, is_new: bool, stats: dict, start_ts: datetime):
|
||||
total = sum(stats.values())
|
||||
|
||||
if stats["failed"] > 0:
|
||||
_LAST_RUN_STATS.failed += 1
|
||||
elif stats["skipped"] == total:
|
||||
_LAST_RUN_STATS.skipped += 1
|
||||
else:
|
||||
_LAST_RUN_STATS.succeeded += 1
|
||||
|
||||
try:
|
||||
results = snapshot.archiveresult_set.only("output_files", "output_size")
|
||||
total_bytes = sum(result.output_size or result.output_size_from_files() for result in results)
|
||||
total_files = sum(result.output_file_count() for result in results)
|
||||
size = (total_bytes, 0, total_files)
|
||||
except Exception:
|
||||
try:
|
||||
size = get_dir_size(out_dir)
|
||||
except FileNotFoundError:
|
||||
size = (0, None, "0")
|
||||
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
duration = str(end_ts - start_ts).split(".")[0]
|
||||
print(f" [bright_black]{size[2]} files ({printable_filesize(size[0])}) in {duration}s [/]")
|
||||
|
||||
|
||||
def log_archive_method_started(method: str):
|
||||
print(f" > {method}")
|
||||
|
||||
|
||||
def log_archive_method_finished(result: dict):
|
||||
"""
|
||||
quote the argument with whitespace in a command so the user can
|
||||
copy-paste the outputted string directly to run the cmd
|
||||
"""
|
||||
# Prettify CMD string and make it safe to copy-paste by quoting arguments
|
||||
quoted_cmd = " ".join(f'"{arg}"' if (" " in arg) or (":" in arg) else arg for arg in result["cmd"])
|
||||
|
||||
if result["status"] == "failed":
|
||||
output = result.get("output")
|
||||
if output and output.__class__.__name__ == "TimeoutExpired":
|
||||
duration = (result["end_ts"] - result["start_ts"]).seconds
|
||||
hint_header = [
|
||||
f"[yellow3]Extractor timed out after {duration}s.[/]",
|
||||
]
|
||||
else:
|
||||
error_name = output.__class__.__name__.replace("ArchiveError", "") if output else "Error"
|
||||
hint_header = [
|
||||
"[yellow3]Extractor failed:[/]",
|
||||
f" {error_name} [red1]{output}[/]",
|
||||
]
|
||||
|
||||
# Prettify error output hints string and limit to five lines
|
||||
hints = getattr(output, "hints", None) or () if output else ()
|
||||
if hints:
|
||||
if isinstance(hints, (list, tuple, type(_ for _ in ()))):
|
||||
hints = [hint.decode() if isinstance(hint, bytes) else str(hint) for hint in hints]
|
||||
else:
|
||||
if isinstance(hints, bytes):
|
||||
hints = hints.decode()
|
||||
hints = hints.split("\n")
|
||||
|
||||
hints = (f" [yellow1]{line.strip()}[/]" for line in list(hints)[:5] if line.strip())
|
||||
|
||||
docker_hints = ()
|
||||
if os.environ.get("IN_DOCKER") in ("1", "true", "True", "TRUE", "yes"):
|
||||
docker_hints = (" docker run -it -v $PWD/data:/data archivebox/archivebox /bin/bash",)
|
||||
|
||||
# Collect and prefix output lines with indentation
|
||||
output_lines = [
|
||||
*hint_header,
|
||||
*hints,
|
||||
"[violet]Run to see full output:[/]",
|
||||
*docker_hints,
|
||||
*([" cd {};".format(result.get("pwd"))] if result.get("pwd") else []),
|
||||
f" {quoted_cmd}",
|
||||
]
|
||||
print(
|
||||
"\n".join(f" {line}" for line in output_lines if line),
|
||||
)
|
||||
print()
|
||||
|
||||
|
||||
def log_list_started(filter_patterns: list[str] | None, filter_type: str):
|
||||
print(f"[green][*] Finding links in the archive index matching these {filter_type} patterns:[/]")
|
||||
print(" {}".format(" ".join(filter_patterns or ())))
|
||||
@ -472,14 +197,6 @@ def log_removal_finished(remaining_links: int, removed_links: int):
|
||||
print(f" Index now contains {remaining_links} links.")
|
||||
|
||||
|
||||
### Search Indexing Stage
|
||||
|
||||
|
||||
def log_index_started(url: str):
|
||||
print(f"[green][*] Indexing url: {url} in the search index[/]")
|
||||
print()
|
||||
|
||||
|
||||
### Helpers
|
||||
|
||||
|
||||
|
||||
@ -2,88 +2,15 @@ __package__ = "archivebox.misc"
|
||||
|
||||
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
||||
from json import dump
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE, Popen, CalledProcessError, CompletedProcess, TimeoutExpired
|
||||
|
||||
from atomicwrites import atomic_write as lib_atomic_write
|
||||
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.misc.util import enforce_types, ExtendedEncoder
|
||||
|
||||
IS_WINDOWS = os.name == "nt"
|
||||
|
||||
|
||||
def run(cmd, *args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs):
|
||||
"""Patched of subprocess.run to kill forked child subprocesses and fix blocking io making timeout=innefective
|
||||
Mostly copied from https://github.com/python/cpython/blob/master/Lib/subprocess.py
|
||||
"""
|
||||
|
||||
cmd = [str(arg) for arg in cmd]
|
||||
|
||||
if input is not None:
|
||||
if kwargs.get("stdin") is not None:
|
||||
raise ValueError("stdin and input arguments may not both be used.")
|
||||
kwargs["stdin"] = PIPE
|
||||
|
||||
if capture_output:
|
||||
if ("stdout" in kwargs) or ("stderr" in kwargs):
|
||||
raise ValueError("stdout and stderr arguments may not be used with capture_output.")
|
||||
kwargs["stdout"] = PIPE
|
||||
kwargs["stderr"] = PIPE
|
||||
|
||||
pgid = None
|
||||
try:
|
||||
if isinstance(cmd, (list, tuple)) and cmd[0].endswith(".py"):
|
||||
PYTHON_BINARY = sys.executable
|
||||
cmd = (PYTHON_BINARY, *cmd)
|
||||
|
||||
with Popen(cmd, *args, start_new_session=start_new_session, text=text, **kwargs) as process:
|
||||
pgid = os.getpgid(process.pid)
|
||||
try:
|
||||
stdout, stderr = process.communicate(input, timeout=timeout)
|
||||
except TimeoutExpired as exc:
|
||||
process.kill()
|
||||
if IS_WINDOWS:
|
||||
# Windows accumulates the output in a single blocking
|
||||
# read() call run on child threads, with the timeout
|
||||
# being done in a join() on those threads. communicate()
|
||||
# _after_ kill() is required to collect that and add it
|
||||
# to the exception.
|
||||
timed_out_stdout, timed_out_stderr = process.communicate()
|
||||
exc.stdout = timed_out_stdout.encode() if isinstance(timed_out_stdout, str) else timed_out_stdout
|
||||
exc.stderr = timed_out_stderr.encode() if isinstance(timed_out_stderr, str) else timed_out_stderr
|
||||
else:
|
||||
# POSIX _communicate already populated the output so
|
||||
# far into the TimeoutExpired exception.
|
||||
process.wait()
|
||||
raise
|
||||
except BaseException: # Including KeyboardInterrupt, communicate handled that.
|
||||
process.kill()
|
||||
# We don't call process.wait() as .__exit__ does that for us.
|
||||
raise
|
||||
|
||||
retcode = process.poll()
|
||||
if check and retcode:
|
||||
raise CalledProcessError(
|
||||
retcode,
|
||||
process.args,
|
||||
output=stdout,
|
||||
stderr=stderr,
|
||||
)
|
||||
finally:
|
||||
# force kill any straggler subprocesses that were forked from the main proc
|
||||
try:
|
||||
if pgid is not None:
|
||||
os.killpg(pgid, signal.SIGINT)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return CompletedProcess(process.args, retcode or 0, stdout, stderr)
|
||||
|
||||
|
||||
@enforce_types
|
||||
def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool = True, config=None, **config_kwargs) -> None:
|
||||
|
||||
@ -10,13 +10,10 @@ from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from inspect import signature
|
||||
from functools import wraps
|
||||
from hashlib import sha256
|
||||
from urllib.parse import urlparse, quote, unquote
|
||||
from html import escape, unescape
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from base32_crockford import encode as base32_encode
|
||||
|
||||
from .logging import COLOR_DICT
|
||||
|
||||
|
||||
@ -41,14 +38,6 @@ def filter_queryset_by_uuid_substring(queryset, slug: str, field: str = "id"):
|
||||
return queryset.filter(Q(**{prefix: normalized}) | Q(**{suffix: normalized}))
|
||||
|
||||
|
||||
def detect_encoding(rawdata):
|
||||
try:
|
||||
import chardet # type:ignore
|
||||
except ImportError:
|
||||
return "utf-8"
|
||||
return chardet.detect(rawdata)["encoding"]
|
||||
|
||||
|
||||
### Parsing Helpers
|
||||
|
||||
# All of these are (str) -> str
|
||||
@ -66,10 +55,6 @@ fragment = lambda url: urlparse(url).fragment
|
||||
extension = lambda url: basename(url).rsplit(".", 1)[-1].lower() if "." in basename(url) else ""
|
||||
base_url = lambda url: without_scheme(url) # uniq base url used to dedupe links
|
||||
|
||||
without_www = lambda url: url.replace("://www.", "://", 1)
|
||||
without_trailing_slash = lambda url: url[:-1] if url[-1] == "/" else url.replace("/?", "?")
|
||||
hashurl = lambda url: base32_encode(int(sha256(base_url(url).encode("utf-8")).hexdigest(), 16))[:20]
|
||||
|
||||
urlencode = lambda s: s and quote(s, encoding="utf-8", errors="replace")
|
||||
urldecode = lambda s: s and unquote(s)
|
||||
htmlencode = lambda s: s and escape(s, quote=True)
|
||||
@ -475,20 +460,6 @@ def ansi_to_html(text: str) -> str:
|
||||
return COLOR_REGEX.sub(single_sub, text)
|
||||
|
||||
|
||||
@enforce_types
|
||||
def dedupe(options: list[str]) -> list[str]:
|
||||
"""
|
||||
Deduplicates the given CLI args by key=value. Options that come later override earlier.
|
||||
"""
|
||||
deduped = {}
|
||||
|
||||
for option in options:
|
||||
key = option.split("=")[0]
|
||||
deduped[key] = option
|
||||
|
||||
return list(deduped.values())
|
||||
|
||||
|
||||
class ExtendedEncoder(pyjson.JSONEncoder):
|
||||
"""
|
||||
Extended json serializer that supports serializing several model
|
||||
|
||||
@ -18,10 +18,13 @@ class BinaryService(BaseService):
|
||||
self.bus.on(BinaryEvent, self.on_BinaryEvent)
|
||||
|
||||
async def on_BinaryRequestEvent(self, event: BinaryRequestEvent) -> str | None:
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
from archivebox.machine.models import Binary, Machine, _canonical_binary_name
|
||||
|
||||
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
|
||||
existing = await Binary.objects.filter(machine=machine, name=event.name).afirst()
|
||||
binary_name = _canonical_binary_name(event.name)
|
||||
if not binary_name:
|
||||
return None
|
||||
existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst()
|
||||
cache_invalidated = False
|
||||
if existing and existing.status == Binary.StatusChoices.INSTALLED:
|
||||
changed = False
|
||||
@ -39,7 +42,7 @@ class BinaryService(BaseService):
|
||||
elif existing is None:
|
||||
await Binary.objects.acreate(
|
||||
machine=machine,
|
||||
name=event.name,
|
||||
name=binary_name,
|
||||
binproviders=event.binproviders,
|
||||
overrides=event.overrides or {},
|
||||
status=Binary.StatusChoices.QUEUED,
|
||||
@ -48,7 +51,7 @@ class BinaryService(BaseService):
|
||||
installed = None
|
||||
if not cache_invalidated:
|
||||
installed = (
|
||||
await Binary.objects.filter(machine=machine, name=event.name, status=Binary.StatusChoices.INSTALLED)
|
||||
await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED)
|
||||
.exclude(abspath="")
|
||||
.exclude(abspath__isnull=True)
|
||||
.order_by("-modified_at")
|
||||
@ -132,13 +135,16 @@ class BinaryService(BaseService):
|
||||
return None
|
||||
|
||||
async def on_BinaryEvent(self, event: BinaryEvent) -> None:
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
from archivebox.machine.models import Binary, Machine, _canonical_binary_name
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
|
||||
binary_name = _canonical_binary_name(event.name)
|
||||
if not binary_name:
|
||||
return
|
||||
binary, _ = await Binary.objects.aget_or_create(
|
||||
machine=machine,
|
||||
name=event.name,
|
||||
name=binary_name,
|
||||
defaults={
|
||||
"status": Binary.StatusChoices.QUEUED,
|
||||
},
|
||||
|
||||
@ -1242,6 +1242,24 @@
|
||||
color: #1d4ed8;
|
||||
}
|
||||
|
||||
.action-rearchive-wrapper {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.action-rearchive-select {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
bottom: 0;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
/* Object count */
|
||||
.actions .action-counter {
|
||||
color: #64748b;
|
||||
@ -1828,9 +1846,80 @@
|
||||
hidden.dispatchEvent(new Event('change', { bubbles: true }))
|
||||
}
|
||||
|
||||
const rearchiveActions = ['resnapshot_snapshot', 'update_snapshots', 'overwrite_snapshots']
|
||||
const rearchiveItems = {
|
||||
resnapshot_snapshot: ['➕', 'Create fresh snapshot(s)'],
|
||||
update_snapshots: ['▶️', 'Retry only failed subtasks'],
|
||||
overwrite_snapshots: ['♻️', 'Reset & retry from scratch'],
|
||||
}
|
||||
let rearchiveBuilt = false
|
||||
|
||||
// for each action in the dropdown, turn it into a button instead
|
||||
container.find('select[name=action] option:gt(0)').each(function () {
|
||||
const action_type = this.value
|
||||
if (rearchiveActions.includes(action_type)) {
|
||||
if (rearchiveBuilt) return
|
||||
rearchiveBuilt = true
|
||||
const wrapper = $('<span></span>')
|
||||
.addClass('action-rearchive-wrapper')
|
||||
.appendTo(buttons)
|
||||
const select = $('<select></select>')
|
||||
.addClass('action-rearchive-select')
|
||||
.attr('aria-label', 'Re-Archive')
|
||||
.appendTo(wrapper)
|
||||
$('<button>')
|
||||
.attr('type', 'button')
|
||||
.addClass('button')
|
||||
.html('Re-Archive <span aria-hidden="true">▾</span>')
|
||||
.click(function (e) {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
const el = select[0]
|
||||
if (typeof el.showPicker === 'function') {
|
||||
try { el.showPicker() } catch (err) { el.focus(); el.click() }
|
||||
} else {
|
||||
el.focus()
|
||||
el.click()
|
||||
}
|
||||
return false
|
||||
})
|
||||
.appendTo(wrapper)
|
||||
$('<option></option>').attr('value', '').attr('hidden', true).appendTo(select)
|
||||
rearchiveActions.forEach(function (action_value) {
|
||||
const opt = container.find('select[name=action] option[value="' + action_value + '"]')[0]
|
||||
if (!opt) return
|
||||
const [icon, label] = rearchiveItems[action_value]
|
||||
$('<option></option>')
|
||||
.attr('value', action_value)
|
||||
.text(icon + ' ' + label)
|
||||
.appendTo(select)
|
||||
})
|
||||
select.change(function () {
|
||||
const action_value = this.value
|
||||
if (!action_value) return
|
||||
const num_selected = (
|
||||
document.querySelector('.action-selected-count')?.textContent.split('/')[0].trim()
|
||||
|| document.querySelector('.action-counter')?.textContent.split(' ')[0]
|
||||
|| '0'
|
||||
)
|
||||
if (action_value === 'overwrite_snapshots') {
|
||||
const message = (
|
||||
'Are you sure you want to re-archive (overwrite) ' + num_selected + ' Snapshots?\n\n' +
|
||||
'This will delete all previously saved files from these Snapshots and re-archive them from scratch.\n\n'
|
||||
)
|
||||
if (!window.confirm(message)) {
|
||||
this.value = ''
|
||||
return
|
||||
}
|
||||
}
|
||||
container.find('select[name=action]')
|
||||
.val(action_value)
|
||||
.trigger('change')
|
||||
$('#changelist-form button[name="index"]').click()
|
||||
document.querySelector('#logo').outerHTML = '<div class="loader"></div>'
|
||||
})
|
||||
return
|
||||
}
|
||||
if (action_type === 'set_crawl_permissions') {
|
||||
const wrapper = $('<span></span>')
|
||||
.addClass('action-permissions-wrapper')
|
||||
|
||||
@ -37,6 +37,6 @@
|
||||
{% block content %}
|
||||
{{ block.super }}
|
||||
{% if crawl_snapshots_changelist %}
|
||||
{{ crawl_snapshots_changelist }}
|
||||
<div id="snapshots">{{ crawl_snapshots_changelist }}</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
@ -2,9 +2,10 @@
|
||||
|
||||
<div id="user-tools">
|
||||
<a href="{% url 'add' %}">Add ➕</a>
|
||||
<a href="/admin/crawls/crawl/">Crawls</a> |
|
||||
<a href="{% url 'Home' %}">Snapshots</a> |
|
||||
<a href="/admin/core/tag/">Tags</a> |
|
||||
<a href="/admin/core/archiveresult/?o=-1">Log</a>
|
||||
<a href="/admin/core/archiveresult/?o=-1">Log</a> |
|
||||
<a href="/admin/core/tag/">Tags</a>
|
||||
<a href="{% url 'Docs' %}" target="_blank" rel="noopener noreferrer">Docs</a> |
|
||||
<a href="/api/v1/docs">API</a> |
|
||||
<a href="/admin/">Admin</a>
|
||||
|
||||
@ -1150,7 +1150,7 @@
|
||||
</a>
|
||||
</div>
|
||||
<div class="header-title-line header-toggle-trigger">
|
||||
<img src="{% snapshot_url snapshot 'favicon/favicon.ico' %}" onerror="this.style.opacity=0" alt="Favicon" class="favicon"/>
|
||||
<img src="{% snapshot_url snapshot 'favicon/favicon.ico' %}" onerror="this.onerror=null;this.src="data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20' fill='none' stroke='rgba(255,255,255,0.65)' stroke-width='1.4'><circle cx='10' cy='10' r='7.25'/><ellipse cx='10' cy='10' rx='3.25' ry='7.25'/><line x1='2.75' y1='10' x2='17.25' y2='10'/></svg>"" alt="Favicon" class="favicon"/>
|
||||
<span class="header-title-text">{{title|truncatechars:120|safe}}</span>
|
||||
{% if title_tags %}
|
||||
<span class="header-tags">
|
||||
|
||||
69
archivebox/templates/registration/password_change_form.html
Normal file
69
archivebox/templates/registration/password_change_form.html
Normal file
@ -0,0 +1,69 @@
|
||||
{% extends "admin/base_site.html" %}
|
||||
{% load i18n static %}
|
||||
|
||||
{% block title %}{% if form.errors %}{% translate "Error:" %} {% endif %}{{ block.super }}{% endblock %}
|
||||
{% block extrastyle %}{{ block.super }}<link rel="stylesheet" href="{% static "admin/css/forms.css" %}">{% endblock %}
|
||||
{% block userlinks %}
|
||||
{% url 'django-admindocs-docroot' as docsroot %}{% if docsroot %}<a href="{{ docsroot }}">{% translate 'Documentation' %}</a> / {% endif %} {% translate 'Change password' %} /
|
||||
<form id="logout-form" method="post" action="{% url 'admin:logout' %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit">{% translate 'Log out' %}</button>
|
||||
</form>
|
||||
{% include "admin/color_theme_toggle.html" %}
|
||||
{% endblock %}
|
||||
{% block breadcrumbs %}
|
||||
<div class="breadcrumbs">
|
||||
<a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
|
||||
› {% translate 'Password change' %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}<div id="content-main">
|
||||
|
||||
<form method="post">{% csrf_token %}
|
||||
<div>
|
||||
{% if form.errors %}
|
||||
<p class="errornote">
|
||||
{% blocktranslate count counter=form.errors.items|length %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktranslate %}
|
||||
</p>
|
||||
{% endif %}
|
||||
|
||||
|
||||
<p>{% translate 'Please enter your old password, for security’s sake, and then enter your new password twice so we can verify you typed it in correctly.' %}</p>
|
||||
|
||||
<fieldset class="module aligned wide">
|
||||
|
||||
<div class="form-row">
|
||||
{{ form.old_password.errors }}
|
||||
<div class="flex-container">{{ form.old_password.label_tag }} {{ form.old_password }}</div>
|
||||
</div>
|
||||
|
||||
<div class="form-row">
|
||||
{{ form.new_password1.errors }}
|
||||
<div class="flex-container">{{ form.new_password1.label_tag }} {{ form.new_password1 }}</div>
|
||||
{% if form.new_password1.help_text %}
|
||||
<div class="help"{% if form.new_password1.id_for_label %} id="{{ form.new_password1.id_for_label }}_helptext"{% endif %}>{{ form.new_password1.help_text|safe }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="form-row">
|
||||
{{ form.new_password2.errors }}
|
||||
<div class="flex-container">{{ form.new_password2.label_tag }} {{ form.new_password2 }}</div>
|
||||
{% if form.new_password2.help_text %}
|
||||
<div class="help"{% if form.new_password2.id_for_label %} id="{{ form.new_password2.id_for_label }}_helptext"{% endif %}>{{ form.new_password2.help_text|safe }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
</fieldset>
|
||||
|
||||
<div class="submit-row">
|
||||
<input type="submit" value="{% translate 'Change my password' %}" class="default">
|
||||
{% if perms.auth.change_user %}
|
||||
<a href="{% url 'admin:auth_user_change' user.pk %}" class="button" style="float: right;">{% translate 'Edit other user settings' %} →</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</form></div>
|
||||
|
||||
{% endblock %}
|
||||
@ -51,7 +51,7 @@ def _build_script(body: str) -> str:
|
||||
from archivebox.core.models import Snapshot, ArchiveResult
|
||||
from archivebox.config.common import get_config
|
||||
SERVER_CONFIG = get_config()
|
||||
from archivebox.core.host_utils import (
|
||||
from archivebox.core.host_util import (
|
||||
get_admin_host,
|
||||
get_admin_base_url,
|
||||
get_base_host,
|
||||
@ -188,7 +188,7 @@ class TestUrlRouting:
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "OK" in result.stdout
|
||||
|
||||
def test_host_utils_and_public_redirect(self) -> None:
|
||||
def test_host_util_and_public_redirect(self) -> None:
|
||||
self._run(
|
||||
"""
|
||||
snapshot = get_snapshot()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user