diff --git a/archivebox/api/urls.py b/archivebox/api/urls.py index 6a21bd75..b625d139 100644 --- a/archivebox/api/urls.py +++ b/archivebox/api/urls.py @@ -7,7 +7,7 @@ from django.shortcuts import redirect from django.urls import path from django.views.generic.base import RedirectView -from archivebox.core.host_utils import build_web_url +from archivebox.core.host_util import build_web_url from .v1_api import urls as v1_api_urls diff --git a/archivebox/api/v1_core.py b/archivebox/api/v1_core.py index 8e490564..4ca3cf7b 100644 --- a/archivebox/api/v1_core.py +++ b/archivebox/api/v1_core.py @@ -31,9 +31,9 @@ from archivebox.core.models import Snapshot, ArchiveResult, Tag from archivebox.core.permissions import public_snapshots_queryset from archivebox.api.auth import auth_using_token from archivebox.config.common import get_config -from archivebox.core.host_utils import build_web_url +from archivebox.core.host_util import build_web_url from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length -from archivebox.core.tag_utils import ( +from archivebox.core.tag_util import ( add_snapshot_counts, build_tag_cards, delete_tag as delete_tag_record, diff --git a/archivebox/base_models/admin.py b/archivebox/base_models/admin.py index f1c06577..c2941248 100644 --- a/archivebox/base_models/admin.py +++ b/archivebox/base_models/admin.py @@ -3,6 +3,7 @@ __package__ = "archivebox.base_models" import json +import uuid from collections.abc import Mapping from typing import NotRequired, TypedDict @@ -11,10 +12,41 @@ from django.contrib import admin from django.db import models from django.forms.renderers import BaseRenderer from django.http import HttpRequest, QueryDict +from django.urls import path, register_converter from django.utils.safestring import SafeString, mark_safe from django_object_actions import DjangoObjectActions +class HexUUIDConverter: + """URL path converter that canonicalizes UUIDs to their 32-char hex form. + + Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex + (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField + parses either), but ``to_url`` always emits the bare-hex form. This is + what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce + ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default + hyphenated rendering — admin links throughout the app reverse through + this converter once ``BaseModelAdmin.get_urls`` swaps in + ```` below. + """ + + regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" + + def to_python(self, value: str) -> str: + # Strip hyphens but stay as a string — Django admin treats the + # captured object_id as a string and calls ``model._meta.pk.to_python`` + # itself, so we don't want to short-circuit that. + return value.replace("-", "") + + def to_url(self, value) -> str: + if isinstance(value, uuid.UUID): + return value.hex + return str(value).replace("-", "") + + +register_converter(HexUUIDConverter, "hexuuid") + + class ConfigOption(TypedDict): plugin: str type: str | list[str] @@ -846,3 +878,30 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin): if "created_by" in form.base_fields: form.base_fields["created_by"].initial = request.user return form + + def get_urls(self): + """Swap the per-object admin URLs from ```` to + ```` so canonical change/delete/history URLs use the + 32-char hex form. The hyphenated form still resolves because the + converter's regex accepts both — Django reverses through ``to_url`` + which always emits hex, so links in templates / changelists / inline + formsets all canonicalize automatically. + + Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example) + won't match the converter's regex and fall back to the default + ```` patterns we still include after our swap. + """ + info = self.opts.app_label, self.opts.model_name + object_routes = [ + path("/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info), + path("/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info), + path("/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info), + ] + # Append after super().get_urls() so our patterns are the + # *last-registered* ones with the canonical admin URL names — Django's + # reverse() picks the later registration when names collide, which is + # how we make ``reverse("admin:app_model_change", args=[obj.pk])`` + # emit the hex form. The original ```` routes stay in + # place as a fallback for non-UUID PKs and for resolving inbound + # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway). + return super().get_urls() + object_routes diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 280e0e43..196dd80e 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -198,7 +198,14 @@ def add( # Discovered URLs become child Snapshots (depth+1) if index_only: - print("[yellow]\\[*] Index-only mode - crawl queued, runner not started[/yellow]") + # ``--index-only`` means "add the URLs to the index without archiving + # them now". That only holds if we actually materialize the Snapshot + # rows here — otherwise the CLI returns success with nothing in the + # index, which broke ``test_add_url_after_init`` & friends. Create + # the Snapshots synchronously (the same step the runner would do) + # but skip starting any worker so extractors don't run. + crawl.create_snapshots_from_urls() + print("[yellow]\\[*] Index-only mode - URLs indexed, runner not started[/yellow]") return crawl, crawl.snapshot_set.all() # 5. Start the crawl runner to process the queue @@ -277,7 +284,7 @@ def add( except Exception: rel_output_str = str(crawl.output_dir) - from archivebox.core.host_utils import build_admin_url + from archivebox.core.host_util import build_admin_url admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config) diff --git a/archivebox/cli/archivebox_archiveresult.py b/archivebox/cli/archivebox_archiveresult.py index 2cabd92c..e34a413c 100644 --- a/archivebox/cli/archivebox_archiveresult.py +++ b/archivebox/cli/archivebox_archiveresult.py @@ -38,7 +38,7 @@ import sys import rich_click as click from rich import print as rprint -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict: diff --git a/archivebox/cli/archivebox_binary.py b/archivebox/cli/archivebox_binary.py index d156d8cc..de7c8a70 100644 --- a/archivebox/cli/archivebox_binary.py +++ b/archivebox/cli/archivebox_binary.py @@ -33,7 +33,7 @@ import sys import rich_click as click from rich import print as rprint -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters # ============================================================================= diff --git a/archivebox/cli/archivebox_crawl.py b/archivebox/cli/archivebox_crawl.py index 1c4919f9..740c1615 100644 --- a/archivebox/cli/archivebox_crawl.py +++ b/archivebox/cli/archivebox_crawl.py @@ -39,7 +39,7 @@ from collections.abc import Iterable import rich_click as click from rich import print as rprint -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters # ============================================================================= diff --git a/archivebox/cli/archivebox_machine.py b/archivebox/cli/archivebox_machine.py index 688216c5..405f0ccb 100644 --- a/archivebox/cli/archivebox_machine.py +++ b/archivebox/cli/archivebox_machine.py @@ -27,7 +27,7 @@ import sys import rich_click as click from rich import print as rprint -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters # ============================================================================= diff --git a/archivebox/cli/archivebox_persona.py b/archivebox/cli/archivebox_persona.py index 88691790..11b9d298 100644 --- a/archivebox/cli/archivebox_persona.py +++ b/archivebox/cli/archivebox_persona.py @@ -37,7 +37,7 @@ from collections.abc import Iterable import rich_click as click from rich import print as rprint -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters from archivebox.personas import importers as persona_importers diff --git a/archivebox/cli/archivebox_process.py b/archivebox/cli/archivebox_process.py index d307b22d..4a94a365 100644 --- a/archivebox/cli/archivebox_process.py +++ b/archivebox/cli/archivebox_process.py @@ -30,7 +30,7 @@ import sys import rich_click as click from rich import print as rprint -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters # ============================================================================= diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index f5854f86..df74748f 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -28,7 +28,7 @@ def schedule( from archivebox.base_models.models import get_or_create_system_user_pk from archivebox.crawls.models import Crawl, CrawlSchedule - from archivebox.crawls.schedule_utils import validate_schedule + from archivebox.crawls.schedule_util import validate_schedule from archivebox.services.runner import run_pending_crawls config_overrides = dict(config or {}) diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 723c9330..0c5fbb92 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -162,7 +162,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None: # CSRF_TRUSTED_ORIGINS set, get_base_url() will silently use that as the # implicit BASE_URL. Surface what we picked so the user knows where their # links / redirects are going — and tell them how to make it explicit. - from archivebox.core.host_utils import derive_base_url_from_csrf + from archivebox.core.host_util import derive_base_url_from_csrf csrf_derived = derive_base_url_from_csrf(config) if csrf_derived: @@ -178,7 +178,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None: print() return - # BASE_URL was not set explicitly. The host_utils derivation gives one of + # BASE_URL was not set explicitly. The host_util derivation gives one of # three results, with very different risk profiles — show a tailored hint # so new users coming from the 0.7.x single-domain world know whether the # default is fine for them or needs attention. @@ -203,7 +203,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None: ) print() else: - # Loopback / wildcard bind. The host_utils default of + # Loopback / wildcard bind. The host_util default of # http://archivebox.localhost:PORT works in a browser on the same # machine, but anything else (reverse proxy, k8s ingress, LAN client) # needs BASE_URL set. (Real hostnames can't reach this branch — the @@ -298,7 +298,7 @@ def server( return os.environ["BIND_ADDR"] = f"{host}:{port}" - from archivebox.core.host_utils import get_base_url + from archivebox.core.host_util import get_base_url base_url = get_base_url().rstrip("/") admin_url = f"{base_url}/admin/" diff --git a/archivebox/cli/archivebox_snapshot.py b/archivebox/cli/archivebox_snapshot.py index 4a6166a8..9f934f5f 100644 --- a/archivebox/cli/archivebox_snapshot.py +++ b/archivebox/cli/archivebox_snapshot.py @@ -37,7 +37,7 @@ import rich_click as click from rich import print as rprint from django.db.models import Case, IntegerField, Q, QuerySet, When -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters # ============================================================================= diff --git a/archivebox/cli/archivebox_tag.py b/archivebox/cli/archivebox_tag.py index 73352d5d..ff6692a7 100644 --- a/archivebox/cli/archivebox_tag.py +++ b/archivebox/cli/archivebox_tag.py @@ -36,7 +36,7 @@ from collections.abc import Iterable import rich_click as click from rich import print as rprint -from archivebox.cli.cli_utils import apply_filters +from archivebox.cli.cli_util import apply_filters # ============================================================================= diff --git a/archivebox/cli/cli_utils.py b/archivebox/cli/cli_util.py similarity index 100% rename from archivebox/cli/cli_utils.py rename to archivebox/cli/cli_util.py diff --git a/archivebox/config/collection.py b/archivebox/config/collection.py index bfe2303d..c534f534 100644 --- a/archivebox/config/collection.py +++ b/archivebox/config/collection.py @@ -152,6 +152,37 @@ def mirror_machine_config_to_file(config: Any) -> None: _MIRROR_IN_PROGRESS = False +def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]: + """Inverse of ``_coerce_to_str_dict``: decode complex INI values to native. + + ``mirror_machine_config_to_file`` JSON-encodes ``dict`` / ``list`` values + so they round-trip through INI's string-only storage. When reading the + file back into ``Machine.config`` (a JSONField that holds native types) + those strings have to be decoded — otherwise downstream consumers like + ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string + where they expect a dict and raise ``TypeError``. + Delegates to pydantic-settings' own ``field_is_complex`` / + ``prepare_field_value`` (the same machinery ``IniConfigSettingsSource`` + uses for the file-read path), so every dict/list/tuple field is + decoded according to its declared annotation — no hardcoded type + checks, no manual ``json.loads`` per call site. + """ + from archivebox.config.common import ArchiveBoxConfig + from archivebox.config.configset import IniConfigSettingsSource + + decoder = IniConfigSettingsSource(ArchiveBoxConfig) + decoded: dict[str, Any] = dict(file_config) + for field_name, field in ArchiveBoxConfig.model_fields.items(): + if field_name not in decoded: + continue + raw = decoded[field_name] + if not isinstance(raw, str) or not raw: + continue + if decoder.field_is_complex(field): + decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True) + return decoded + + def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None: """Copy ``ArchiveBox.conf`` contents into ``Machine.config``. @@ -163,7 +194,7 @@ def _mirror_file_to_machine_config(file_config: dict[str, str]) -> None: machine = Machine.current() if _coerce_to_str_dict(machine.config) == file_config: return - machine.config = dict(file_config) + machine.config = _coerce_from_str_dict(file_config) machine.save(update_fields=["config", "modified_at"]) @@ -226,7 +257,7 @@ def sync_machine_and_file(machine: Any = None) -> None: if merged != file_config: _write_file_if_changed(_render_config_file_content(merged)) if merged != machine_config: - machine.config = dict(merged) + machine.config = _coerce_from_str_dict(merged) machine.save(update_fields=["config", "modified_at"]) finally: _MIRROR_IN_PROGRESS = False diff --git a/archivebox/config/common.py b/archivebox/config/common.py index 4e474371..0a29bbc6 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -17,7 +17,7 @@ from pydantic import BaseModel, Field, create_model, field_validator, model_vali from pydantic_settings import SettingsConfigDict from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs -from archivebox.config.configset import BaseConfigSet +from archivebox.config.configset import BaseConfigSet, IniConfigSettingsSource from archivebox.config.configset import COMPUTED_CONFIG_KEYS from .constants import CONSTANTS @@ -654,6 +654,29 @@ def get_config( config_data["ABX_RUNTIME"] = "archivebox" + # Decode JSON-encoded complex values (dict/list fields) that came from + # string-only sources before validation. ``IniConfigSettingsSource`` does + # this for the ArchiveBox.conf path, but Machine.config (mirrored from the + # INI via ``_coerce_to_str_dict``) and plugin/env scope overrides bypass + # pydantic-settings sources entirely — they feed JSON strings directly + # into ``model_validate``, which rejects ``"{...}"`` for a ``dict[str, str]`` + # field. Run pydantic-settings' own complex-value decoder here so every + # source converges on the same shape before validation. + _complex_decoder = IniConfigSettingsSource(ArchiveBoxConfig) + for _field_name, _field in ArchiveBoxConfig.model_fields.items(): + if _field_name not in config_data: + continue + _raw = config_data[_field_name] + if not isinstance(_raw, str) or not _raw: + continue + if _complex_decoder.field_is_complex(_field): + config_data[_field_name] = _complex_decoder.prepare_field_value( + _field_name, + _field, + _raw, + True, + ) + config = ArchiveBoxConfig.model_validate(config_data) os.environ["LIB_DIR"] = str(config.LIB_DIR) os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR) diff --git a/archivebox/config/views.py b/archivebox/config/views.py index 9390c74e..0fb7bf27 100644 --- a/archivebox/config/views.py +++ b/archivebox/config/views.py @@ -360,12 +360,27 @@ def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]: def get_db_binaries_by_name() -> dict[str, Binary]: + """Group Binary rows by a URL-safe canonical name. + + Hooks occasionally emit ``BinaryEvent.name`` carrying an abspath rather + than a short binary name (see ``services/binary_service.py``). That used + to leak ``name='/Users/.../bin/foo'`` rows into the DB, which then broke + ``/admin/environment/binaries`` because the admin URL regex is + ``(?P[^/]+)``. Canonicalize at the keying step so duplicates fold + into the real binary and the admin link key stays slash-free regardless + of legacy DB state. + """ + from archivebox.machine.models import _canonical_binary_name + grouped: dict[str, list[Binary]] = {} binary_name_aliases = { "youtube-dl": "yt-dlp", } for binary in Binary.objects.all(): - canonical_name = binary_name_aliases.get(binary.name, binary.name) + canonical_name = _canonical_binary_name(binary.name) + canonical_name = binary_name_aliases.get(canonical_name, canonical_name) + if not canonical_name: + continue grouped.setdefault(canonical_name, []).append(binary) return {name: max(records, key=_binary_sort_key) for name, records in grouped.items()} @@ -628,6 +643,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: rows = { "Name": [], + "Type": [], "State": [], "PID": [], "Started": [], @@ -657,12 +673,115 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: continue all_config[config_name] = config_data - # Add top row for supervisord process manager + # Collect every PID we plan to show so we can resolve them to Process rows + # in a single query. supervisord's per-worker description carries the pid + # in the form ``pid 12345, uptime 0:01:23`` (or just the bare ``pid`` + # placeholder when stopped); we ignore non-numeric values. + process_items = supervisor.getAllProcessInfo() + if not isinstance(process_items, list): + process_items = [] + + def _parse_worker_pid_and_uptime(description: str) -> tuple[int | None, str]: + body = description.replace("pid ", "", 1) + pid_part, _, uptime_part = body.partition(", uptime ") + try: + return int(pid_part.strip()), uptime_part.strip() + except ValueError: + return None, "" + + pids: set[int] = set() + supervisor_pid = supervisor.getPID() + if isinstance(supervisor_pid, int): + pids.add(supervisor_pid) + for proc_data in process_items: + if not isinstance(proc_data, dict): + continue + pid_int, _ = _parse_worker_pid_and_uptime(str(proc_data.get("description") or "")) + if pid_int is not None: + pids.add(pid_int) + + pid_to_process_id: dict[int, str] = {} + pid_to_process_type: dict[int, str] = {} + if pids: + try: + from archivebox.machine.models import Machine, Process + + for row in ( + Process.objects.filter(machine=Machine.current(), pid__in=pids) + .order_by("pid", "-started_at", "-created_at") + .only("id", "pid", "process_type") + ): + if row.pid in pid_to_process_id: + continue # keep the most recent row per PID + pid_to_process_id[row.pid] = str(row.id) + pid_to_process_type[row.pid] = row.process_type + + except Exception: + pass + + def _pid_cell(pid_value: int | None, uptime_str: str = ""): + if pid_value is None: + return "" + pid_text = str(pid_value) + process_id = pid_to_process_id.get(pid_value) + if process_id: + link = format_html('{}', process_id, pid_text) + else: + link = format_html("{}", pid_text) + if uptime_str: + return format_html("{}, uptime {}", link, uptime_str) + return link + + # Add top row for supervisord process manager. supervisord exposes its + # state + pid over XML-RPC but not its own start time / exit status / uptime, + # so we read those from the OS process (or fall back to the Process row + # recorded in _record_supervisord_process). Exit status stays blank while + # it's RUNNING — supervisord wouldn't be answering RPC if it had exited. rows["Name"].append(ItemLink("supervisord", key="supervisord")) + rows["Type"].append("supervisord") supervisor_state = supervisor.getState() - rows["State"].append(str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else "")) - rows["PID"].append(str(supervisor.getPID())) - rows["Started"].append("-") + state_name = str(supervisor_state.get("statename") if isinstance(supervisor_state, dict) else "") + rows["State"].append(state_name) + + supervisor_started = "" + supervisor_uptime = "" + try: + import time as _time + + import psutil + + ps_proc = psutil.Process(supervisor_pid) + create_time = ps_proc.create_time() + supervisor_started = format_parsed_datetime(create_time) + seconds = max(int(_time.time() - create_time), 0) + hours, remainder = divmod(seconds, 3600) + minutes, secs = divmod(remainder, 60) + supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}" + except Exception: + try: + from archivebox.machine.models import Machine, Process + + row = ( + Process.objects.filter( + machine=Machine.current(), + process_type=Process.TypeChoices.SUPERVISORD, + pid=supervisor_pid, + ) + .order_by("-started_at") + .first() + ) + if row and row.started_at: + supervisor_started = row.started_at.strftime("%Y-%m-%d %H:%M:%S") + seconds = max(int((timezone.now() - row.started_at).total_seconds()), 0) + hours, remainder = divmod(seconds, 3600) + minutes, secs = divmod(remainder, 60) + supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}" + except Exception: + pass + + rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime)) + rows["Started"].append(supervisor_started or "-") + rows["Command"].append("supervisord --configuration=tmp/supervisord.conf") rows["Logfile"].append( format_html( @@ -671,12 +790,9 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: "logs/supervisord.log", ), ) - rows["Exit Status"].append("0") + rows["Exit Status"].append("" if state_name == "RUNNING" else "-") # Add a row for each worker process managed by supervisord - process_items = supervisor.getAllProcessInfo() - if not isinstance(process_items, list): - process_items = [] for proc_data in process_items: if not isinstance(proc_data, dict): continue @@ -685,10 +801,15 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: proc_start = proc_data.get("start") proc_logfile = str(proc_data.get("stdout_logfile") or "") proc_config = all_config.get(proc_name, {}) + pid_int, uptime_str = _parse_worker_pid_and_uptime(proc_description) rows["Name"].append(ItemLink(proc_name, key=proc_name)) + # Prefer the Process row's process_type when we have one (e.g. "worker", + # "hook"); otherwise fall back to the generic "worker" label since + # everything in this loop is supervisord-managed. + rows["Type"].append(pid_to_process_type.get(pid_int, "worker") if pid_int else "worker") rows["State"].append(str(proc_data.get("statename") or "")) - rows["PID"].append(proc_description.replace("pid ", "")) + rows["PID"].append(_pid_cell(pid_int, uptime_str)) rows["Started"].append(format_parsed_datetime(proc_start)) rows["Command"].append(str(proc_config.get("command") or "")) rows["Logfile"].append( @@ -811,7 +932,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext: f.seek(0) last_lines = f.read().decode("utf-8", errors="replace").split("\n") non_empty_lines = [line for line in last_lines if line.strip()] - rows["Most Recent Lines"].append(non_empty_lines[-1]) + rows["Most Recent Lines"].append(non_empty_lines[-1] if non_empty_lines else "") return TableContext( title="Debug Log files", diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py index 1970fdd4..d5fb048b 100644 --- a/archivebox/core/admin_archiveresults.py +++ b/archivebox/core/admin_archiveresults.py @@ -24,10 +24,10 @@ from archivebox.config.common import get_config from archivebox.misc.paginators import AcceleratedPaginator from archivebox.base_models.admin import BaseModelAdmin from archivebox.hooks import get_plugin_icon -from archivebox.core.host_utils import build_snapshot_url +from archivebox.core.host_util import build_snapshot_url from archivebox.core.widgets import InlineTagEditorWidget from archivebox.core.views import LIVE_PLUGIN_BASE_URL -from archivebox.machine.env_utils import env_to_shell_exports +from archivebox.machine.env_util import env_to_shell_exports from archivebox.core.models import ArchiveResult, Snapshot diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 481f0ed4..1fa12939 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -28,8 +28,8 @@ from archivebox.misc.util import htmldecode, urldecode from archivebox.misc.paginators import AcceleratedPaginator from archivebox.misc.logging_util import printable_filesize from archivebox.search.admin import SEARCH_RESULT_CACHE_TTL, SearchResultsAdminMixin, SearchResultsChangeList, get_admin_search_cache_key -from archivebox.core.host_utils import build_snapshot_url, build_web_url -from archivebox.core.tag_utils import get_or_create_tag +from archivebox.core.host_util import build_snapshot_url, build_web_url +from archivebox.core.tag_util import get_or_create_tag from archivebox.hooks import discover_hooks, get_plugin_icon, get_plugin_name, get_plugins from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin @@ -1609,13 +1609,19 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): # on the resulting crawl so existing snapshots don't cause the crawl to # seal immediately with zero new snapshots (the default ONLY_NEW=True # would skip any URLs that have ever been archived before). - add(urls=urls, bg=True, config={"ONLY_NEW": False}) + crawl, _ = add(urls=urls, bg=True, config={"ONLY_NEW": False}) messages.success( request, f"Created 1 queued crawl with {len(snapshots)} URL(s). The background runner will create snapshots and process them.", ) + # Redirect to the new crawl's admin page so the user lands on the + # work-in-progress crawl, not the old snapshot they re-archived from. + # A snapshot-view redirect would race the runner — the new snapshot + # may sit queued for a while before the runner creates the DB row. + return redirect(f"/admin/crawls/crawl/{crawl.id}/change/#snapshots") + @admin.action( description="🔄 Redo", ) diff --git a/archivebox/core/admin_tags.py b/archivebox/core/admin_tags.py index 377049d6..7c053b23 100644 --- a/archivebox/core/admin_tags.py +++ b/archivebox/core/admin_tags.py @@ -13,7 +13,7 @@ from django.utils.safestring import mark_safe from archivebox.base_models.admin import BaseModelAdmin from archivebox.core.models import SnapshotTag, Tag -from archivebox.core.tag_utils import ( +from archivebox.core.tag_util import ( TAG_HAS_SNAPSHOTS_CHOICES, TAG_SORT_CHOICES, build_tag_cards, @@ -24,7 +24,7 @@ from archivebox.core.tag_utils import ( normalize_has_snapshots_filter, normalize_tag_sort, ) -from archivebox.core.host_utils import build_snapshot_url +from archivebox.core.host_util import build_snapshot_url class TagInline(admin.TabularInline): diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index d74f6b5a..6b3d26f3 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -13,7 +13,7 @@ from django.utils.html import format_html from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes from taggit.utils import edit_string_for_tags, parse_tags from archivebox.base_models.admin import KeyValueWidget -from archivebox.crawls.schedule_utils import validate_schedule +from archivebox.crawls.schedule_util import validate_schedule from archivebox.config.common import get_config, parse_delete_after from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget diff --git a/archivebox/core/host_utils.py b/archivebox/core/host_util.py similarity index 95% rename from archivebox/core/host_utils.py rename to archivebox/core/host_util.py index 5648370a..1f395547 100644 --- a/archivebox/core/host_utils.py +++ b/archivebox/core/host_util.py @@ -87,27 +87,6 @@ def derive_base_url_from_csrf(config: dict[str, Any] | None = None, **config_kwa return "" -def request_host_is_explicitly_allowed(request_host: str, config) -> bool: - """True if the incoming Host is in ALLOWED_HOSTS or matches a CSRF origin. - - Used by ``get_base_url`` to honour the request's own Host header when the - operator hasn't pinned ``BASE_URL`` explicitly. The check is intentionally - strict — we won't trust arbitrary Host headers, only ones the user has - already opted into via ALLOWED_HOSTS / CSRF_TRUSTED_ORIGINS. - """ - if not request_host: - return False - host, _ = split_host_port(request_host) - allowed = _allowed_hosts(config) - if host in allowed: - return True - for origin in _csrf_trusted_origins(config): - origin_host, _ = split_host_port(urlparse(origin).netloc) - if origin_host == host: - return True - return False - - def get_listen_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str: config = config or get_config(**config_kwargs) return (config.BIND_ADDR or "").strip() diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py index 0c0a654e..85e6e47d 100644 --- a/archivebox/core/middleware.py +++ b/archivebox/core/middleware.py @@ -15,7 +15,7 @@ from django.http import HttpResponseForbidden, HttpResponseNotModified from archivebox.config.common import get_config from archivebox.config import VERSION from archivebox.config.version import get_COMMIT_HASH -from archivebox.core.host_utils import ( +from archivebox.core.host_util import ( build_snapshot_url, build_admin_url, build_web_url, diff --git a/archivebox/core/models.py b/archivebox/core/models.py index d2994db3..67e86c6e 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -648,7 +648,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW if parsed.scheme not in ("http", "https") or not parsed.hostname: return False - from archivebox.core.host_utils import get_admin_host, get_api_host, get_listen_host, get_public_host, get_web_host, split_host_port + from archivebox.core.host_util import get_admin_host, get_api_host, get_listen_host, get_public_host, get_web_host, split_host_port config = get_config() host = parsed.hostname.lower().strip(".") @@ -2989,7 +2989,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW def to_dict(self, extended: bool = False) -> dict[str, Any]: """Convert Snapshot to a dictionary (replacement for Link._asdict())""" - from archivebox.core.host_utils import build_snapshot_url + from archivebox.core.host_util import build_snapshot_url archive_size = self.archive_size diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 50ce16ff..f66e0122 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -8,13 +8,12 @@ import importlib from pathlib import Path from django.conf.locale.en import formats as en_formats # type: ignore -from django.utils.crypto import get_random_string import archivebox from archivebox.config.constants import CONSTANTS from archivebox.config.common import get_config -from archivebox.core.host_utils import normalize_base_url, get_admin_base_url, get_api_base_url +from archivebox.core.host_util import normalize_base_url, get_admin_base_url, get_api_base_url from .settings_logging import SETTINGS_LOGGING @@ -359,7 +358,34 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"} ### Security Settings ################################################################################ -SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, "abcdefghijklmnopqrstuvwxyz0123456789_") +# Persist SECRET_KEY on first use. Data dirs created before init wrote a +# SECRET_KEY line — or those whose ArchiveBox.conf was hand-edited to remove +# it — would otherwise sign session cookies with a fresh random key on every +# boot (because the pydantic field's default_factory regenerates), logging +# users out on every server restart. ``archivebox init`` writes this for new +# collections; this branch is the recovery path for the rest. +# +# We can't check ``CONFIG.SECRET_KEY`` for "missing" — pydantic's +# default_factory already filled it with a fresh random value. We have to +# inspect the conf file directly to know whether the value will survive. +SECRET_KEY = CONFIG.SECRET_KEY +try: + from archivebox.config.configset import BaseConfigSet as _BaseConfigSet + + _persisted_keys = _BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE) + _secret_persisted = bool((_persisted_keys.get("SECRET_KEY") or "").strip()) +except Exception: + _secret_persisted = True # err on the side of NOT touching disk +if not _secret_persisted: + try: + from archivebox.config.collection import write_config_file + + write_config_file({"SECRET_KEY": SECRET_KEY}) + except Exception: + # Read-only mount, missing data dir, mid-init race — fall back to the + # in-memory random key. The user will get logged out on the next boot + # but the server still comes up. + pass ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()] CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()}) diff --git a/archivebox/core/settings_logging.py b/archivebox/core/settings_logging.py index dc52dcae..957d2f2a 100644 --- a/archivebox/core/settings_logging.py +++ b/archivebox/core/settings_logging.py @@ -67,6 +67,40 @@ class DaphneCloseTimeoutFilter(logging.Filter): return True +class AsyncioCancelledShieldFilter(logging.Filter): + """Drop asyncio's "CancelledError exception in shielded future" noise. + + When a browser disconnects mid-request, daphne cancels the asgi task and + asgiref's ``sync_to_async`` shields the synchronous Django view via + ``asyncio.shield(exec_coro)``. The shield wakes up to find its parent + cancelled and re-raises ``CancelledError``; asyncio's default exception + handler then logs the full traceback via the ``asyncio`` logger. There's + nothing the server can do (the client is already gone), so these tracebacks + are pure noise and cause hundreds of lines of spam per disconnect. + + We match conservatively: only drop the specific shielded-future message + that points back into asgiref's shield path. Other asyncio errors fall + through unchanged. + """ + + def filter(self, record) -> bool: + if record.name != "asyncio": + return True + logline = record.getMessage() + if "CancelledError exception in shielded future" in logline: + return False + exc_info = record.exc_info + if exc_info and exc_info[0] is not None: + try: + import asyncio as _asyncio + + if issubclass(exc_info[0], _asyncio.CancelledError): + return False + except Exception: + pass + return True + + class CustomOutboundWebhookLogFormatter(logging.Formatter): def format(self, record): result = super().format(record) @@ -128,6 +162,9 @@ SETTINGS_LOGGING = { "daphneclosetimeout": { "()": DaphneCloseTimeoutFilter, }, + "asynciocancelledshield": { + "()": AsyncioCancelledShieldFilter, + }, "stripansi": { "()": StripANSIColorCodesFilter, }, @@ -145,7 +182,7 @@ SETTINGS_LOGGING = { "level": "DEBUG", "markup": False, "rich_tracebacks": False, # Use standard Python tracebacks (no frame/box) - "filters": ["noisyrequestsfilter", "daphneclosetimeout", "stripansi"], + "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"], }, "logfile": { "level": "INFO", @@ -154,7 +191,7 @@ SETTINGS_LOGGING = { "maxBytes": 1024 * 1024 * 25, # 25 MB "backupCount": 10, "formatter": "rich", - "filters": ["noisyrequestsfilter", "daphneclosetimeout", "stripansi"], + "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"], }, "outbound_webhooks": { "class": "rich.logging.RichHandler", diff --git a/archivebox/core/sqlite_backend/base.py b/archivebox/core/sqlite_backend/base.py index ff68ddf7..8659bb4c 100644 --- a/archivebox/core/sqlite_backend/base.py +++ b/archivebox/core/sqlite_backend/base.py @@ -10,12 +10,6 @@ from django.db.backends.sqlite3.base import DatabaseWrapper as DjangoSQLiteDatab from django.db.backends.sqlite3.base import SQLiteCursorWrapper as DjangoSQLiteCursorWrapper -def _is_locked_error(error: BaseException) -> bool: - from django.db import OperationalError - - return isinstance(error, (sqlite3.OperationalError, OperationalError)) and "database is locked" in str(error).lower() - - def _sqlite_lock_retry_timeout() -> float: from django.conf import settings @@ -111,7 +105,9 @@ def _retry_locked_database(action, query: str, params=None, *, db_wrapper=None): try: return action() except (sqlite3.OperationalError, Exception) as err: - if not _is_locked_error(err): + from archivebox.misc.db import sqlite_lock_error + + if not sqlite_lock_error(err): raise attempt += 1 elapsed = time.monotonic() - started_at diff --git a/archivebox/core/tag_utils.py b/archivebox/core/tag_util.py similarity index 99% rename from archivebox/core/tag_utils.py rename to archivebox/core/tag_util.py index a64cf12f..da9da86a 100644 --- a/archivebox/core/tag_utils.py +++ b/archivebox/core/tag_util.py @@ -12,7 +12,7 @@ from django.http import HttpRequest from django.urls import reverse from archivebox.config.common import get_config -from archivebox.core.host_utils import build_snapshot_url, build_web_url +from archivebox.core.host_util import build_snapshot_url, build_web_url from archivebox.core.models import Snapshot, SnapshotTag, Tag diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py index f4689440..cbd0cf94 100644 --- a/archivebox/core/templatetags/core_tags.py +++ b/archivebox/core/templatetags/core_tags.py @@ -12,7 +12,7 @@ from archivebox.hooks import ( get_plugin_template, get_plugin_name, ) -from archivebox.core.host_utils import ( +from archivebox.core.host_util import ( canonical_base_host_for_request, get_admin_base_url, get_public_base_url, diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 2842de50..1687f9d3 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -66,7 +66,7 @@ from archivebox.core.permissions import ( is_admin_user, public_snapshots_queryset, ) -from archivebox.core.host_utils import ( +from archivebox.core.host_util import ( build_admin_url, build_snapshot_url, build_web_url, diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py index 73294af0..fbd1dc41 100755 --- a/archivebox/crawls/models.py +++ b/archivebox/crawls/models.py @@ -34,7 +34,7 @@ from archivebox.base_models.models import ( get_or_create_system_user_pk, ) from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine -from archivebox.crawls.schedule_utils import next_run_for_schedule, validate_schedule +from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule from archivebox.misc.util import validate_url, validate_url_length if TYPE_CHECKING: diff --git a/archivebox/crawls/schedule_utils.py b/archivebox/crawls/schedule_util.py similarity index 100% rename from archivebox/crawls/schedule_utils.py rename to archivebox/crawls/schedule_util.py diff --git a/archivebox/machine/admin.py b/archivebox/machine/admin.py index 5613527c..13a25adb 100644 --- a/archivebox/machine/admin.py +++ b/archivebox/machine/admin.py @@ -2,6 +2,7 @@ __package__ = "archivebox.machine" import json import shlex +from pathlib import Path from django.contrib import admin, messages from django.db.models import DurationField, ExpressionWrapper, F @@ -14,7 +15,7 @@ from django_object_actions import action from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin from archivebox.misc.logging_util import printable_filesize -from archivebox.machine.env_utils import env_to_dotenv_text +from archivebox.machine.env_util import env_to_dotenv_text from archivebox.machine.models import Machine, NetworkInterface, Binary, Process @@ -398,7 +399,7 @@ class ProcessAdmin(BaseModelAdmin): "snapshot_link", "crawl_link", "cmd_str", - "status", + "status_badge", "duration_display", "exit_code", "pid", @@ -413,7 +414,7 @@ class ProcessAdmin(BaseModelAdmin): "snapshot_link", "crawl_link", "cmd_str", - "status", + "status_badge", "duration_display", "exit_code", "pid", @@ -628,11 +629,53 @@ class ProcessAdmin(BaseModelAdmin): def cmd_str(self, process): if not process.cmd: return "-" - cmd = " ".join(process.cmd[:3]) if isinstance(process.cmd, list) else str(process.cmd) - if len(process.cmd) > 3: - cmd += " ..." + # Compact the list-view rendering only — the change-page ``cmd_display`` + # still shows the full original ``process.cmd`` (and the DB row is + # untouched). If the first argv token looks like an absolute path, + # collapse it to its basename so a row like + # ``/Users/.../.venv/bin/python -m archivebox foo`` reads as + # ``python -m archivebox foo`` in the column. + if isinstance(process.cmd, list): + parts = [str(arg) for arg in process.cmd[:3]] + if parts and (parts[0].startswith("/") or parts[0].startswith("~")): + parts[0] = Path(parts[0]).name + cmd = " ".join(parts) + if len(process.cmd) > 3: + cmd += " ..." + else: + cmd = str(process.cmd) return format_html('{}', cmd[:80]) + @admin.display(description="Status", ordering="status") + def status_badge(self, process): + # Pill-style badge matching the look of other admin status columns. + # Color rules requested by the operator: + # RUNNING → green + # EXITED, code == 0 → grey (clean exit) + # EXITED, code == 10 → grey (treated as a clean / "skipped" exit) + # EXITED, code other → red (failure) + # QUEUED / anything → amber so it stands out without screaming + status_value = str(process.status or "").lower() + label = (process.get_status_display() or status_value or "?").upper() + exit_code = process.exit_code + if status_value == Process.StatusChoices.RUNNING: + bg, fg = "#16a34a", "#fff" + elif status_value == Process.StatusChoices.EXITED: + if exit_code in (0, 10): + bg, fg = "#6b7280", "#fff" + else: + bg, fg = "#dc2626", "#fff" + else: + bg, fg = "#f59e0b", "#fff" + return format_html( + '{}', + bg, + fg, + label, + ) + @admin.display(description="Duration", ordering="runtime_sort") def duration_display(self, process): return _format_process_duration_seconds(process.started_at, process.ended_at) diff --git a/archivebox/machine/env_utils.py b/archivebox/machine/env_util.py similarity index 100% rename from archivebox/machine/env_utils.py rename to archivebox/machine/env_util.py diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index 9421ae3d..159af05d 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -144,7 +144,9 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str] def sqlite_lock_error(error: BaseException) -> bool: - return isinstance(error, SQLiteOperationalError) and "database is locked" in str(error).lower() + from django.db import OperationalError as DjangoOperationalError + + return isinstance(error, (SQLiteOperationalError, DjangoOperationalError)) and "database is locked" in str(error).lower() def retry_sqlite_locks(action: Callable[[], Any], *, label: str, stderr: TextIO | None = None) -> Any: diff --git a/archivebox/misc/logging_util.py b/archivebox/misc/logging_util.py index 8d740428..15ccc1f5 100644 --- a/archivebox/misc/logging_util.py +++ b/archivebox/misc/logging_util.py @@ -13,7 +13,6 @@ from multiprocessing import Process from pathlib import Path from datetime import datetime, timezone -from dataclasses import dataclass from typing import Any, Optional, IO, TYPE_CHECKING, cast from collections.abc import Iterable @@ -23,35 +22,12 @@ if TYPE_CHECKING: from rich import print from rich.panel import Panel -from archivebox.config import CONSTANTS, DATA_DIR, VERSION +from archivebox.config import DATA_DIR, VERSION from archivebox.config.common import get_config -from archivebox.misc.system import get_dir_size from archivebox.misc.util import enforce_types from archivebox.misc.logging import ANSI -@dataclass -class RuntimeStats: - """mutable stats counter for logging archiving timing info to CLI output""" - - skipped: int = 0 - succeeded: int = 0 - failed: int = 0 - - parse_start_ts: datetime | None = None - parse_end_ts: datetime | None = None - - index_start_ts: datetime | None = None - index_end_ts: datetime | None = None - - archiving_start_ts: datetime | None = None - archiving_end_ts: datetime | None = None - - -# globals are bad, mmkay -_LAST_RUN_STATS = RuntimeStats() - - class TimedProgress: """Show a progress bar and measure elapsed time until .end() is called""" @@ -176,257 +152,6 @@ def log_cli_command(subcommand: str, subcommand_args: Iterable[str] = (), stdin: print(Panel(version_msg), file=sys.stderr) -### Parsing Stage - - -def log_importing_started(urls: str | list[str], depth: int, index_only: bool): - _LAST_RUN_STATS.parse_start_ts = datetime.now(timezone.utc) - print( - "[green][+] [{}] Adding {} links to index (crawl depth={}){}...[/]".format( - _LAST_RUN_STATS.parse_start_ts.strftime("%Y-%m-%d %H:%M:%S"), - len(urls) if isinstance(urls, list) else len(urls.split("\n")), - depth, - " (index only)" if index_only else "", - ), - ) - - -def log_source_saved(source_file: str): - print(" > Saved verbatim input to {}/{}".format(CONSTANTS.SOURCES_DIR_NAME, source_file.rsplit("/", 1)[-1])) - - -def log_parsing_finished(num_parsed: int, parser_name: str): - _LAST_RUN_STATS.parse_end_ts = datetime.now(timezone.utc) - print(f" > Parsed {num_parsed} URLs from input ({parser_name})") - - -def log_deduping_finished(num_new_links: int): - print(f" > Found {num_new_links} new URLs not already in index") - - -def log_crawl_started(new_links): - print() - print(f"[green][*] Starting crawl of {len(new_links)} sites 1 hop out from starting point[/]") - - -### Indexing Stage - - -def log_indexing_process_started(num_links: int): - start_ts = datetime.now(timezone.utc) - _LAST_RUN_STATS.index_start_ts = start_ts - print() - print( - "[bright_black][*] [{}] Writing {} links to main index...[/]".format( - start_ts.strftime("%Y-%m-%d %H:%M:%S"), - num_links, - ), - ) - - -def log_indexing_process_finished(): - end_ts = datetime.now(timezone.utc) - _LAST_RUN_STATS.index_end_ts = end_ts - - -def _display_data_path(out_path: str) -> str: - path = Path(out_path).resolve() - try: - return f"./{path.relative_to(DATA_DIR)}" - except ValueError: - return str(path) - - -def log_indexing_started(out_path: str, config=None, **config_kwargs): - config = config or get_config(**config_kwargs) - if config.IS_TTY: - sys.stdout.write(f" > {_display_data_path(out_path)}") - - -def log_indexing_finished(out_path: str): - print(f"\r √ {_display_data_path(out_path)}") - - -### Archiving Stage - - -def log_archiving_started(num_links: int, resume: float | None = None): - - start_ts = datetime.now(timezone.utc) - _LAST_RUN_STATS.archiving_start_ts = start_ts - print() - if resume: - print( - "[green][▶] [{}] Resuming archive updating for {} pages starting from {}...[/]".format( - start_ts.strftime("%Y-%m-%d %H:%M:%S"), - num_links, - resume, - ), - ) - else: - print( - "[green][▶] [{}] Starting archiving of {} snapshots in index...[/]".format( - start_ts.strftime("%Y-%m-%d %H:%M:%S"), - num_links, - ), - ) - - -def log_archiving_paused(num_links: int, idx: int, timestamp: str): - - end_ts = datetime.now(timezone.utc) - _LAST_RUN_STATS.archiving_end_ts = end_ts - print() - print( - "\n[yellow3][X] [{now}] Downloading paused on link {timestamp} ({idx}/{total})[/]".format( - now=end_ts.strftime("%Y-%m-%d %H:%M:%S"), - idx=idx + 1, - timestamp=timestamp, - total=num_links, - ), - ) - print() - print(" Continue archiving where you left off by running:") - print(f" archivebox update --resume={timestamp}") - - -def log_archiving_finished(num_links: int): - - from archivebox.core.models import Snapshot - - end_ts = datetime.now(timezone.utc) - _LAST_RUN_STATS.archiving_end_ts = end_ts - assert _LAST_RUN_STATS.archiving_start_ts is not None - seconds = end_ts.timestamp() - _LAST_RUN_STATS.archiving_start_ts.timestamp() - if seconds > 60: - duration = f"{seconds / 60:.2f} min" - else: - duration = f"{seconds:.2f} sec" - - print() - print( - "[green][√] [{}] Update of {} pages complete ({})[/]".format( - end_ts.strftime("%Y-%m-%d %H:%M:%S"), - num_links, - duration, - ), - ) - print(f" - {_LAST_RUN_STATS.skipped} links skipped") - print(f" - {_LAST_RUN_STATS.succeeded + _LAST_RUN_STATS.failed} links updated") - print(f" - {_LAST_RUN_STATS.failed} links had errors") - - if Snapshot.objects.count() < 50: - print() - print(" [violet]Hint:[/] To manage your archive in a Web UI, run:") - print(" archivebox server 0.0.0.0:8000") - - -def log_snapshot_archiving_started(snapshot: "Snapshot", out_dir: str, is_new: bool): - - # [*] [2019-03-22 13:46:45] "Log Structured Merge Trees - ben stopford" - # http://www.benstopford.com/2015/02/14/log-structured-merge-trees/ - # > output/archive/1478739709 - - print( - '\n[[{symbol_color}]{symbol}[/]] [[{symbol_color}]{now}[/]] "{title}"'.format( - symbol_color="green" if is_new else "bright_black", - symbol="+" if is_new else "√", - now=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), - title=snapshot.title or snapshot.base_url, - ), - ) - print(f" [sky_blue1]{snapshot.url}[/]") - print( - " {} {}".format( - ">" if is_new else "√", - pretty_path(out_dir), - ), - ) - - -def log_snapshot_archiving_finished(snapshot: "Snapshot", out_dir: str, is_new: bool, stats: dict, start_ts: datetime): - total = sum(stats.values()) - - if stats["failed"] > 0: - _LAST_RUN_STATS.failed += 1 - elif stats["skipped"] == total: - _LAST_RUN_STATS.skipped += 1 - else: - _LAST_RUN_STATS.succeeded += 1 - - try: - results = snapshot.archiveresult_set.only("output_files", "output_size") - total_bytes = sum(result.output_size or result.output_size_from_files() for result in results) - total_files = sum(result.output_file_count() for result in results) - size = (total_bytes, 0, total_files) - except Exception: - try: - size = get_dir_size(out_dir) - except FileNotFoundError: - size = (0, None, "0") - - end_ts = datetime.now(timezone.utc) - duration = str(end_ts - start_ts).split(".")[0] - print(f" [bright_black]{size[2]} files ({printable_filesize(size[0])}) in {duration}s [/]") - - -def log_archive_method_started(method: str): - print(f" > {method}") - - -def log_archive_method_finished(result: dict): - """ - quote the argument with whitespace in a command so the user can - copy-paste the outputted string directly to run the cmd - """ - # Prettify CMD string and make it safe to copy-paste by quoting arguments - quoted_cmd = " ".join(f'"{arg}"' if (" " in arg) or (":" in arg) else arg for arg in result["cmd"]) - - if result["status"] == "failed": - output = result.get("output") - if output and output.__class__.__name__ == "TimeoutExpired": - duration = (result["end_ts"] - result["start_ts"]).seconds - hint_header = [ - f"[yellow3]Extractor timed out after {duration}s.[/]", - ] - else: - error_name = output.__class__.__name__.replace("ArchiveError", "") if output else "Error" - hint_header = [ - "[yellow3]Extractor failed:[/]", - f" {error_name} [red1]{output}[/]", - ] - - # Prettify error output hints string and limit to five lines - hints = getattr(output, "hints", None) or () if output else () - if hints: - if isinstance(hints, (list, tuple, type(_ for _ in ()))): - hints = [hint.decode() if isinstance(hint, bytes) else str(hint) for hint in hints] - else: - if isinstance(hints, bytes): - hints = hints.decode() - hints = hints.split("\n") - - hints = (f" [yellow1]{line.strip()}[/]" for line in list(hints)[:5] if line.strip()) - - docker_hints = () - if os.environ.get("IN_DOCKER") in ("1", "true", "True", "TRUE", "yes"): - docker_hints = (" docker run -it -v $PWD/data:/data archivebox/archivebox /bin/bash",) - - # Collect and prefix output lines with indentation - output_lines = [ - *hint_header, - *hints, - "[violet]Run to see full output:[/]", - *docker_hints, - *([" cd {};".format(result.get("pwd"))] if result.get("pwd") else []), - f" {quoted_cmd}", - ] - print( - "\n".join(f" {line}" for line in output_lines if line), - ) - print() - - def log_list_started(filter_patterns: list[str] | None, filter_type: str): print(f"[green][*] Finding links in the archive index matching these {filter_type} patterns:[/]") print(" {}".format(" ".join(filter_patterns or ()))) @@ -472,14 +197,6 @@ def log_removal_finished(remaining_links: int, removed_links: int): print(f" Index now contains {remaining_links} links.") -### Search Indexing Stage - - -def log_index_started(url: str): - print(f"[green][*] Indexing url: {url} in the search index[/]") - print() - - ### Helpers diff --git a/archivebox/misc/system.py b/archivebox/misc/system.py index a35fcf2c..0f981df8 100644 --- a/archivebox/misc/system.py +++ b/archivebox/misc/system.py @@ -2,88 +2,15 @@ __package__ = "archivebox.misc" import os -import signal -import sys from json import dump from pathlib import Path -from subprocess import PIPE, Popen, CalledProcessError, CompletedProcess, TimeoutExpired from atomicwrites import atomic_write as lib_atomic_write from archivebox.config.common import get_config from archivebox.misc.util import enforce_types, ExtendedEncoder -IS_WINDOWS = os.name == "nt" - - -def run(cmd, *args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs): - """Patched of subprocess.run to kill forked child subprocesses and fix blocking io making timeout=innefective - Mostly copied from https://github.com/python/cpython/blob/master/Lib/subprocess.py - """ - - cmd = [str(arg) for arg in cmd] - - if input is not None: - if kwargs.get("stdin") is not None: - raise ValueError("stdin and input arguments may not both be used.") - kwargs["stdin"] = PIPE - - if capture_output: - if ("stdout" in kwargs) or ("stderr" in kwargs): - raise ValueError("stdout and stderr arguments may not be used with capture_output.") - kwargs["stdout"] = PIPE - kwargs["stderr"] = PIPE - - pgid = None - try: - if isinstance(cmd, (list, tuple)) and cmd[0].endswith(".py"): - PYTHON_BINARY = sys.executable - cmd = (PYTHON_BINARY, *cmd) - - with Popen(cmd, *args, start_new_session=start_new_session, text=text, **kwargs) as process: - pgid = os.getpgid(process.pid) - try: - stdout, stderr = process.communicate(input, timeout=timeout) - except TimeoutExpired as exc: - process.kill() - if IS_WINDOWS: - # Windows accumulates the output in a single blocking - # read() call run on child threads, with the timeout - # being done in a join() on those threads. communicate() - # _after_ kill() is required to collect that and add it - # to the exception. - timed_out_stdout, timed_out_stderr = process.communicate() - exc.stdout = timed_out_stdout.encode() if isinstance(timed_out_stdout, str) else timed_out_stdout - exc.stderr = timed_out_stderr.encode() if isinstance(timed_out_stderr, str) else timed_out_stderr - else: - # POSIX _communicate already populated the output so - # far into the TimeoutExpired exception. - process.wait() - raise - except BaseException: # Including KeyboardInterrupt, communicate handled that. - process.kill() - # We don't call process.wait() as .__exit__ does that for us. - raise - - retcode = process.poll() - if check and retcode: - raise CalledProcessError( - retcode, - process.args, - output=stdout, - stderr=stderr, - ) - finally: - # force kill any straggler subprocesses that were forked from the main proc - try: - if pgid is not None: - os.killpg(pgid, signal.SIGINT) - except Exception: - pass - - return CompletedProcess(process.args, retcode or 0, stdout, stderr) - @enforce_types def atomic_write(path: Path | str, contents: dict | str | bytes, overwrite: bool = True, config=None, **config_kwargs) -> None: diff --git a/archivebox/misc/util.py b/archivebox/misc/util.py index 452306b2..91c94fa8 100644 --- a/archivebox/misc/util.py +++ b/archivebox/misc/util.py @@ -10,13 +10,10 @@ from collections.abc import Callable from pathlib import Path from inspect import signature from functools import wraps -from hashlib import sha256 from urllib.parse import urlparse, quote, unquote from html import escape, unescape from datetime import datetime, timezone -from base32_crockford import encode as base32_encode - from .logging import COLOR_DICT @@ -41,14 +38,6 @@ def filter_queryset_by_uuid_substring(queryset, slug: str, field: str = "id"): return queryset.filter(Q(**{prefix: normalized}) | Q(**{suffix: normalized})) -def detect_encoding(rawdata): - try: - import chardet # type:ignore - except ImportError: - return "utf-8" - return chardet.detect(rawdata)["encoding"] - - ### Parsing Helpers # All of these are (str) -> str @@ -66,10 +55,6 @@ fragment = lambda url: urlparse(url).fragment extension = lambda url: basename(url).rsplit(".", 1)[-1].lower() if "." in basename(url) else "" base_url = lambda url: without_scheme(url) # uniq base url used to dedupe links -without_www = lambda url: url.replace("://www.", "://", 1) -without_trailing_slash = lambda url: url[:-1] if url[-1] == "/" else url.replace("/?", "?") -hashurl = lambda url: base32_encode(int(sha256(base_url(url).encode("utf-8")).hexdigest(), 16))[:20] - urlencode = lambda s: s and quote(s, encoding="utf-8", errors="replace") urldecode = lambda s: s and unquote(s) htmlencode = lambda s: s and escape(s, quote=True) @@ -475,20 +460,6 @@ def ansi_to_html(text: str) -> str: return COLOR_REGEX.sub(single_sub, text) -@enforce_types -def dedupe(options: list[str]) -> list[str]: - """ - Deduplicates the given CLI args by key=value. Options that come later override earlier. - """ - deduped = {} - - for option in options: - key = option.split("=")[0] - deduped[key] = option - - return list(deduped.values()) - - class ExtendedEncoder(pyjson.JSONEncoder): """ Extended json serializer that supports serializing several model diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py index 77209e0f..66c85afa 100644 --- a/archivebox/services/binary_service.py +++ b/archivebox/services/binary_service.py @@ -18,10 +18,13 @@ class BinaryService(BaseService): self.bus.on(BinaryEvent, self.on_BinaryEvent) async def on_BinaryRequestEvent(self, event: BinaryRequestEvent) -> str | None: - from archivebox.machine.models import Binary, Machine + from archivebox.machine.models import Binary, Machine, _canonical_binary_name machine = await sync_to_async(Machine.current, thread_sensitive=True)() - existing = await Binary.objects.filter(machine=machine, name=event.name).afirst() + binary_name = _canonical_binary_name(event.name) + if not binary_name: + return None + existing = await Binary.objects.filter(machine=machine, name=binary_name).afirst() cache_invalidated = False if existing and existing.status == Binary.StatusChoices.INSTALLED: changed = False @@ -39,7 +42,7 @@ class BinaryService(BaseService): elif existing is None: await Binary.objects.acreate( machine=machine, - name=event.name, + name=binary_name, binproviders=event.binproviders, overrides=event.overrides or {}, status=Binary.StatusChoices.QUEUED, @@ -48,7 +51,7 @@ class BinaryService(BaseService): installed = None if not cache_invalidated: installed = ( - await Binary.objects.filter(machine=machine, name=event.name, status=Binary.StatusChoices.INSTALLED) + await Binary.objects.filter(machine=machine, name=binary_name, status=Binary.StatusChoices.INSTALLED) .exclude(abspath="") .exclude(abspath__isnull=True) .order_by("-modified_at") @@ -132,13 +135,16 @@ class BinaryService(BaseService): return None async def on_BinaryEvent(self, event: BinaryEvent) -> None: - from archivebox.machine.models import Binary, Machine + from archivebox.machine.models import Binary, Machine, _canonical_binary_name from archivebox.config.common import get_config machine = await sync_to_async(Machine.current, thread_sensitive=True)() + binary_name = _canonical_binary_name(event.name) + if not binary_name: + return binary, _ = await Binary.objects.aget_or_create( machine=machine, - name=event.name, + name=binary_name, defaults={ "status": Binary.StatusChoices.QUEUED, }, diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html index 7ce687d5..aed0ef38 100644 --- a/archivebox/templates/admin/base.html +++ b/archivebox/templates/admin/base.html @@ -1242,6 +1242,24 @@ color: #1d4ed8; } + .action-rearchive-wrapper { + position: relative; + display: inline-block; + } + + .action-rearchive-select { + position: absolute; + left: 0; + bottom: 0; + width: 1px; + height: 1px; + opacity: 0; + pointer-events: none; + border: 0; + padding: 0; + margin: 0; + } + /* Object count */ .actions .action-counter { color: #64748b; @@ -1828,9 +1846,80 @@ hidden.dispatchEvent(new Event('change', { bubbles: true })) } + const rearchiveActions = ['resnapshot_snapshot', 'update_snapshots', 'overwrite_snapshots'] + const rearchiveItems = { + resnapshot_snapshot: ['➕', 'Create fresh snapshot(s)'], + update_snapshots: ['▶️', 'Retry only failed subtasks'], + overwrite_snapshots: ['♻️', 'Reset & retry from scratch'], + } + let rearchiveBuilt = false + // for each action in the dropdown, turn it into a button instead container.find('select[name=action] option:gt(0)').each(function () { const action_type = this.value + if (rearchiveActions.includes(action_type)) { + if (rearchiveBuilt) return + rearchiveBuilt = true + const wrapper = $('') + .addClass('action-rearchive-wrapper') + .appendTo(buttons) + const select = $('') + .addClass('action-rearchive-select') + .attr('aria-label', 'Re-Archive') + .appendTo(wrapper) + $(' + + {% include "admin/color_theme_toggle.html" %} +{% endblock %} +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {% translate 'Password change' %} +
+{% endblock %} + +{% block content %}
+ +
{% csrf_token %} +
+{% if form.errors %} +

+ {% blocktranslate count counter=form.errors.items|length %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktranslate %} +

+{% endif %} + + +

{% translate 'Please enter your old password, for security’s sake, and then enter your new password twice so we can verify you typed it in correctly.' %}

+ +
+ +
+ {{ form.old_password.errors }} +
{{ form.old_password.label_tag }} {{ form.old_password }}
+
+ +
+ {{ form.new_password1.errors }} +
{{ form.new_password1.label_tag }} {{ form.new_password1 }}
+ {% if form.new_password1.help_text %} +
{{ form.new_password1.help_text|safe }}
+ {% endif %} +
+ +
+ {{ form.new_password2.errors }} +
{{ form.new_password2.label_tag }} {{ form.new_password2 }}
+ {% if form.new_password2.help_text %} +
{{ form.new_password2.help_text|safe }}
+ {% endif %} +
+ +
+ +
+ + {% if perms.auth.change_user %} + {% translate 'Edit other user settings' %} → + {% endif %} +
+ +
+
+ +{% endblock %} diff --git a/archivebox/tests/test_urls.py b/archivebox/tests/test_urls.py index ac95e6e5..bf861b0c 100644 --- a/archivebox/tests/test_urls.py +++ b/archivebox/tests/test_urls.py @@ -51,7 +51,7 @@ def _build_script(body: str) -> str: from archivebox.core.models import Snapshot, ArchiveResult from archivebox.config.common import get_config SERVER_CONFIG = get_config() - from archivebox.core.host_utils import ( + from archivebox.core.host_util import ( get_admin_host, get_admin_base_url, get_base_host, @@ -188,7 +188,7 @@ class TestUrlRouting: assert result.returncode == 0, result.stderr assert "OK" in result.stdout - def test_host_utils_and_public_redirect(self) -> None: + def test_host_util_and_public_redirect(self) -> None: self._run( """ snapshot = get_snapshot()