diff --git a/archivebox/config/common.py b/archivebox/config/common.py index c6359279..b0c87ed1 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -151,7 +151,7 @@ class ArchivingConfig(BaseConfigSet): DEFAULT_PERSONA: str = Field(default="Default") - def validate(self): + def warn_if_invalid(self) -> None: if int(self.TIMEOUT) < 5: print(f"[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.TIMEOUT} seconds)[/red]", file=sys.stderr) print(" You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.", file=sys.stderr) @@ -165,10 +165,8 @@ class ArchivingConfig(BaseConfigSet): def validate_check_ssl_validity(cls, v): """SIDE EFFECT: disable "you really shouldnt disable ssl" warnings emitted by requests""" if not v: - import requests import urllib3 - requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) return v @@ -206,6 +204,7 @@ class ArchivingConfig(BaseConfigSet): ARCHIVING_CONFIG = ArchivingConfig() +ARCHIVING_CONFIG.warn_if_invalid() class SearchBackendConfig(BaseConfigSet): diff --git a/archivebox/config/django.py b/archivebox/config/django.py index 09ddcfd2..36d2655f 100644 --- a/archivebox/config/django.py +++ b/archivebox/config/django.py @@ -8,6 +8,7 @@ from datetime import datetime, timezone from rich.console import Console import django +import django.db from archivebox.misc import logging diff --git a/archivebox/config/views.py b/archivebox/config/views.py index 1e3e8f5e..c4ceb3e1 100644 --- a/archivebox/config/views.py +++ b/archivebox/config/views.py @@ -3,14 +3,15 @@ __package__ = 'archivebox.config' import os import shutil import inspect -from typing import Any, List, Dict, cast +from typing import Any, List, Dict from benedict import benedict from django.http import HttpRequest from django.utils import timezone -from django.utils.html import format_html, mark_safe +from django.utils.html import format_html +from django.utils.safestring import mark_safe -from admin_data_views.typing import TableContext, ItemContext +from admin_data_views.typing import TableContext, ItemContext, SectionData from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink from archivebox.config import CONSTANTS @@ -29,6 +30,10 @@ KNOWN_BINARIES = [ ] +def is_superuser(request: HttpRequest) -> bool: + return bool(getattr(request.user, 'is_superuser', False)) + + def obj_to_yaml(obj: Any, indent: int = 0) -> str: indent_str = " " * indent if indent == 0: @@ -132,7 +137,7 @@ def get_filesystem_plugins() -> Dict[str, Dict[str, Any]]: @render_with_table_view def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext: - assert request.user.is_superuser, 'Must be a superuser to view configuration settings.' + assert is_superuser(request), 'Must be a superuser to view configuration settings.' rows = { "Binary Name": [], @@ -177,29 +182,27 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext: @render_with_item_view def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: - - assert request.user and request.user.is_superuser, 'Must be a superuser to view configuration settings.' + assert is_superuser(request), 'Must be a superuser to view configuration settings.' # Try database first try: binary = Binary.objects.get(name=key) + section: SectionData = { + "name": binary.name, + "description": str(binary.abspath or ''), + "fields": { + 'name': binary.name, + 'binprovider': binary.binprovider, + 'abspath': str(binary.abspath), + 'version': binary.version, + 'sha256': binary.sha256, + }, + "help_texts": {}, + } return ItemContext( slug=key, title=key, - data=[ - { - "name": binary.name, - "description": str(binary.abspath or ''), - "fields": { - 'name': binary.name, - 'binprovider': binary.binprovider, - 'abspath': str(binary.abspath), - 'version': binary.version, - 'sha256': binary.sha256, - }, - "help_texts": {}, - }, - ], + data=[section], ) except Binary.DoesNotExist: pass @@ -207,47 +210,44 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: # Try to detect from PATH path = shutil.which(key) if path: + section: SectionData = { + "name": key, + "description": path, + "fields": { + 'name': key, + 'binprovider': 'PATH', + 'abspath': path, + 'version': 'unknown', + }, + "help_texts": {}, + } return ItemContext( slug=key, title=key, - data=[ - { - "name": key, - "description": path, - "fields": { - 'name': key, - 'binprovider': 'PATH', - 'abspath': path, - 'version': 'unknown', - }, - "help_texts": {}, - }, - ], + data=[section], ) + section: SectionData = { + "name": key, + "description": "Binary not found", + "fields": { + 'name': key, + 'binprovider': 'not installed', + 'abspath': 'not found', + 'version': 'N/A', + }, + "help_texts": {}, + } return ItemContext( slug=key, title=key, - data=[ - { - "name": key, - "description": "Binary not found", - "fields": { - 'name': key, - 'binprovider': 'not installed', - 'abspath': 'not found', - 'version': 'N/A', - }, - "help_texts": {}, - }, - ], + data=[section], ) @render_with_table_view def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext: - - assert request.user.is_superuser, 'Must be a superuser to view configuration settings.' + assert is_superuser(request), 'Must be a superuser to view configuration settings.' rows = { "Name": [], @@ -291,7 +291,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext: def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: import json - assert request.user.is_superuser, 'Must be a superuser to view configuration settings.' + assert is_superuser(request), 'Must be a superuser to view configuration settings.' plugins = get_filesystem_plugins() @@ -309,7 +309,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: "name": plugin['name'], "source": plugin['source'], "path": plugin['path'], - "hooks": plugin['hooks'], + "hooks": ', '.join(plugin['hooks']), } # Add config.json data if available @@ -348,7 +348,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: @render_with_table_view def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: - assert request.user.is_superuser, "Must be a superuser to view configuration settings." + assert is_superuser(request), "Must be a superuser to view configuration settings." rows = { "Name": [], @@ -369,8 +369,12 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: table=rows, ) - all_config_entries = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or []) - all_config = {config["name"]: benedict(config) for config in all_config_entries} + all_config_entries = [ + benedict(config) + for config in (supervisor.getAllConfigInfo() or []) + if isinstance(config, dict) and "name" in config + ] + all_config = {str(config["name"]): config for config in all_config_entries} # Add top row for supervisord process manager rows["Name"].append(ItemLink('supervisord', key='supervisord')) @@ -388,8 +392,10 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: rows['Exit Status'].append('0') # Add a row for each worker process managed by supervisord - for proc in cast(List[Dict[str, Any]], supervisor.getAllProcessInfo()): - proc = benedict(proc) + for proc_data in supervisor.getAllProcessInfo(): + if not isinstance(proc_data, dict): + continue + proc = benedict(proc_data) rows["Name"].append(ItemLink(proc.name, key=proc.name)) rows["State"].append(proc.statename) rows['PID'].append(proc.description.replace('pid ', '')) @@ -412,7 +418,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext: @render_with_item_view def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: - assert request.user.is_superuser, "Must be a superuser to view configuration settings." + assert is_superuser(request), "Must be a superuser to view configuration settings." from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME @@ -427,11 +433,15 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: data=[], ) - all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or []) + all_config = [ + benedict(config) + for config in (supervisor.getAllConfigInfo() or []) + if isinstance(config, dict) + ] if key == 'supervisord': relevant_config = CONFIG_FILE.read_text() - relevant_logs = cast(str, supervisor.readLog(0, 10_000_000)) + relevant_logs = str(supervisor.readLog(0, 10_000_000)) start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0] uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0] @@ -449,37 +459,37 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: ) else: proc = benedict(get_worker(supervisor, key) or {}) - relevant_config = [config for config in all_config if config['name'] == key][0] - relevant_logs = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0] + relevant_config = next((config for config in all_config if config.get('name') == key), benedict({})) + relevant_logs = str(supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0]) + + section: SectionData = { + "name": key, + "description": key, + "fields": { + "Command": str(proc.name), + "PID": str(proc.pid), + "State": str(proc.statename), + "Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "", + "Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "", + "Exit Status": str(proc.exitstatus), + "Logfile": str(proc.stdout_logfile), + "Uptime": str((proc.description or "").split("uptime ", 1)[-1]), + "Config": obj_to_yaml(dict(relevant_config)) if isinstance(relevant_config, dict) else str(relevant_config), + "Logs": relevant_logs, + }, + "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"}, + } return ItemContext( slug=key, title=key, - data=[ - { - "name": key, - "description": key, - "fields": { - "Command": proc.name, - "PID": proc.pid, - "State": proc.statename, - "Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "", - "Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "", - "Exit Status": str(proc.exitstatus), - "Logfile": proc.stdout_logfile, - "Uptime": (proc.description or "").split("uptime ", 1)[-1], - "Config": relevant_config, - "Logs": relevant_logs, - }, - "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"}, - }, - ], + data=[section], ) @render_with_table_view def log_list_view(request: HttpRequest, **kwargs) -> TableContext: - assert request.user.is_superuser, "Must be a superuser to view configuration settings." + assert is_superuser(request), "Must be a superuser to view configuration settings." log_files = CONSTANTS.LOGS_DIR.glob("*.log") @@ -516,7 +526,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext: @render_with_item_view def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: - assert request.user.is_superuser, "Must be a superuser to view configuration settings." + assert is_superuser(request), "Must be a superuser to view configuration settings." log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob('*.log') if key in logfile.name][0] diff --git a/archivebox/core/admin_archiveresults.py b/archivebox/core/admin_archiveresults.py index 70353578..8fb41405 100644 --- a/archivebox/core/admin_archiveresults.py +++ b/archivebox/core/admin_archiveresults.py @@ -4,7 +4,8 @@ import os from pathlib import Path from django.contrib import admin -from django.utils.html import format_html, mark_safe +from django.utils.html import format_html +from django.utils.safestring import mark_safe from django.core.exceptions import ValidationError from django.urls import reverse, resolve from django.utils import timezone diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 85024ed5..7e01d5f1 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -6,7 +6,8 @@ from pathlib import Path from django.contrib import admin, messages from django.urls import path -from django.utils.html import format_html, mark_safe +from django.utils.html import format_html +from django.utils.safestring import mark_safe from django.utils import timezone from django.db.models import Q, Sum, Count, Prefetch from django.db.models.functions import Coalesce @@ -110,7 +111,9 @@ class SnapshotAdminForm(forms.ModelForm): # Handle tags_editor field if commit: instance.save() - self._save_m2m() + save_m2m = getattr(self, '_save_m2m', None) + if callable(save_m2m): + save_m2m() # Parse and save tags from tags_editor tags_str = self.cleaned_data.get('tags_editor', '') @@ -200,6 +203,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): def get_actions(self, request): actions = super().get_actions(request) + if not actions: + return {} if 'delete_selected' in actions: func, name, _desc = actions['delete_selected'] actions['delete_selected'] = (func, name, 'Delete') @@ -684,22 +689,23 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin): # cl = self.get_changelist_instance(request) # Save before monkey patching to restore for changelist list view - saved_change_list_template = self.change_list_template - saved_list_per_page = self.list_per_page - saved_list_max_show_all = self.list_max_show_all + admin_cls = type(self) + saved_change_list_template = admin_cls.change_list_template + saved_list_per_page = admin_cls.list_per_page + saved_list_max_show_all = admin_cls.list_max_show_all # Monkey patch here plus core_tags.py - self.change_list_template = 'private_index_grid.html' - self.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE - self.list_max_show_all = self.list_per_page + admin_cls.change_list_template = 'private_index_grid.html' + admin_cls.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE + admin_cls.list_max_show_all = admin_cls.list_per_page # Call monkey patched view rendered_response = self.changelist_view(request, extra_context=extra_context) # Restore values - self.change_list_template = saved_change_list_template - self.list_per_page = saved_list_per_page - self.list_max_show_all = saved_list_max_show_all + admin_cls.change_list_template = saved_change_list_template + admin_cls.list_per_page = saved_list_per_page + admin_cls.list_max_show_all = saved_list_max_show_all return rendered_response diff --git a/archivebox/tests/test_cli_shell.py b/archivebox/tests/test_cli_shell.py index 818b9c5c..dbca4284 100644 --- a/archivebox/tests/test_cli_shell.py +++ b/archivebox/tests/test_cli_shell.py @@ -22,3 +22,18 @@ def test_shell_command_exists(tmp_path, process): # Should show shell help or recognize command assert result.returncode in [0, 1, 2] + + +def test_shell_c_executes_python(tmp_path, process): + """shell -c should fully initialize Django and run the provided command.""" + os.chdir(tmp_path) + + result = subprocess.run( + ['archivebox', 'shell', '-c', 'print("shell-ok")'], + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert 'shell-ok' in result.stdout diff --git a/archivebox/workers/models.py b/archivebox/workers/models.py index ec78531a..1d316e9e 100644 --- a/archivebox/workers/models.py +++ b/archivebox/workers/models.py @@ -26,8 +26,6 @@ ObjectStateList = Iterable[ObjectState] class BaseModelWithStateMachine(models.Model, MachineMixin): - id: models.UUIDField - StatusChoices: ClassVar[Type[DefaultStatusChoices]] # status: models.CharField @@ -384,6 +382,9 @@ class ModelWithStateMachine(BaseModelWithStateMachine): active_state = StatusChoices.STARTED retry_at_field_name: str = 'retry_at' + class Meta(BaseModelWithStateMachine.Meta): + abstract = True + class BaseStateMachine(StateMachine): """ Base class for all ArchiveBox state machines.