diff --git a/archivebox/config/django.py b/archivebox/config/django.py index 04e99b30..cee1f576 100644 --- a/archivebox/config/django.py +++ b/archivebox/config/django.py @@ -2,18 +2,17 @@ __package__ = "archivebox.config" import os import sys - -from datetime import datetime, timezone - -from rich.console import Console +from datetime import UTC, datetime import django import django.db +from django.core.exceptions import ImproperlyConfigured +from rich.console import Console from archivebox.misc import logging -from .constants import CONSTANTS from .common import get_config +from .constants import CONSTANTS CONFIG = get_config() @@ -30,8 +29,9 @@ logging.CONSOLE = CONSOLE DJANGO_SET_UP = False -def setup_django(check_db=False, in_memory_db=False) -> None: +def setup_django(check_db=False) -> None: from rich.panel import Panel + from archivebox.misc.checks import check_not_inside_source_dir global DJANGO_SET_UP @@ -52,8 +52,7 @@ def setup_django(check_db=False, in_memory_db=False) -> None: # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup # cost for cheap CLI startup paths like `archivebox --help`. import archivebox.misc.monkey_patches # noqa: F401 - - from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission + from archivebox.config.permissions import ARCHIVEBOX_GROUP, ARCHIVEBOX_USER, IS_ROOT, SudoPermission # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user if IS_ROOT and ARCHIVEBOX_USER != 0: @@ -79,40 +78,30 @@ def setup_django(check_db=False, in_memory_db=False) -> None: try: from django.core.management import call_command - if in_memory_db: - raise Exception("dont use this anymore") - - # some commands dont store a long-lived sqlite3 db file on disk. - # in those cases we create a temporary in-memory db and run the migrations - # immediately to get a usable in-memory-database at startup - os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:") + # Initialize the configured file-based database without running + # migrations automatically; `archivebox init` owns migrations. + try: django.setup() + # User config and import errors are reported through one CLI message. + except (ImproperlyConfigured, django.db.Error, ImportError, OSError, RuntimeError, ValueError) as e: + is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version")) + if not is_using_meta_cmd: + # show error message to user only if they're not running a meta command / just trying to get help + STDERR.print() + STDERR.print( + Panel( + f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n", + title="\n\n[red][X] Error while trying to load database![/red]", + subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]", + expand=False, + style="bold red", + ), + ) + STDERR.print() + import traceback - call_command("migrate", interactive=False, verbosity=0) - else: - # Otherwise use default sqlite3 file-based database and initialize django - # without running migrations automatically (user runs them manually by calling init) - try: - django.setup() - except Exception as e: - is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version")) - if not is_using_meta_cmd: - # show error message to user only if they're not running a meta command / just trying to get help - STDERR.print() - STDERR.print( - Panel( - f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n", - title="\n\n[red][X] Error while trying to load database![/red]", - subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]", - expand=False, - style="bold red", - ), - ) - STDERR.print() - import traceback - - traceback.print_exc() - return + traceback.print_exc() + return from archivebox.core.settings_logging import ERROR_LOG as DEFAULT_ERROR_LOG @@ -120,13 +109,13 @@ def setup_django(check_db=False, in_memory_db=False) -> None: error_log = DEFAULT_ERROR_LOG with open(error_log, "a", encoding="utf-8") as f: command = " ".join(sys.argv) - ts = datetime.now(timezone.utc).strftime("%Y-%m-%d__%H:%M:%S") + ts = datetime.now(UTC).strftime("%Y-%m-%d__%H:%M:%S") config = get_config() f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={config.IN_DOCKER} IS_TTY={config.IS_TTY}\n") if check_db: # make sure the data dir is owned by a non-root user - if CONSTANTS.DATA_DIR.stat().st_uid == 0: + if CONSTANTS.DATA_DIR.stat().st_uid == 0 and not (IS_ROOT and ARCHIVEBOX_USER == 0): STDERR.print("[red][X] Error: ArchiveBox DATA_DIR cannot be owned by root![/red]") STDERR.print(f" {CONSTANTS.DATA_DIR}") STDERR.print() @@ -159,6 +148,7 @@ def setup_django(check_db=False, in_memory_db=False) -> None: ) except KeyboardInterrupt: + DJANGO_SET_UP = False raise DJANGO_SET_UP = True diff --git a/archivebox/config/permissions.py b/archivebox/config/permissions.py index 59603d2b..8667f1d0 100644 --- a/archivebox/config/permissions.py +++ b/archivebox/config/permissions.py @@ -1,17 +1,16 @@ __package__ = "archivebox.config" import os -import pwd -import sys -import socket import platform +import pwd +import socket +import sys +from contextlib import contextmanager, suppress +from pathlib import Path from typing import cast from rich import print -from pathlib import Path -from contextlib import contextmanager - ############################################################################################# DATA_DIR = Path(os.getcwd()) @@ -30,13 +29,13 @@ RUNNING_AS_UID = os.getuid() RUNNING_AS_GID = os.getgid() EUID = os.geteuid() EGID = os.getegid() -SUDO_UID = int(os.environ.get("SUDO_UID", 0)) -SUDO_GID = int(os.environ.get("SUDO_GID", 0)) +SUDO_UID = int(os.environ.get("SUDO_UID", "0")) +SUDO_GID = int(os.environ.get("SUDO_GID", "0")) USER: str = Path("~").expanduser().resolve().name HOSTNAME: str = cast(str, max([socket.gethostname(), platform.node()], key=len)) IS_ROOT = RUNNING_AS_UID == 0 -IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes") +IN_DOCKER = os.environ.get("IN_DOCKER", "") in ("1", "true", "True", "TRUE", "yes") FALLBACK_UID = RUNNING_AS_UID or SUDO_UID FALLBACK_GID = RUNNING_AS_GID or SUDO_GID @@ -45,7 +44,7 @@ try: except KeyError: ARCHIVEBOX_ACCOUNT = None -if DATA_DIR_UID != 0: +if RUNNING_AS_UID == 0 and DATA_DIR_UID == 0 or DATA_DIR_UID != 0: ARCHIVEBOX_USER = DATA_DIR_UID ARCHIVEBOX_GROUP = DATA_DIR_GID elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None: @@ -58,8 +57,8 @@ if not USER: try: # alternative method 1 to get username USER = pwd.getpwuid(ARCHIVEBOX_USER).pw_name - except Exception: - pass + except (KeyError, OSError): + USER = "" if not USER: try: @@ -67,21 +66,21 @@ if not USER: import getpass USER = getpass.getuser() - except Exception: - pass + except OSError: + USER = "" if not USER: try: # alternative method 3 to get username USER = os.getlogin() or "archivebox" - except Exception: + except OSError: USER = "archivebox" ARCHIVEBOX_USER_EXISTS = False try: pwd.getpwuid(ARCHIVEBOX_USER) ARCHIVEBOX_USER_EXISTS = True -except Exception: +except KeyError: ARCHIVEBOX_USER_EXISTS = False @@ -93,30 +92,27 @@ def drop_privileges(): # Always run ArchiveBox as the user that owns the data dir, or as the # archivebox service account when the data dir is root-owned. - if os.getuid() == 0: - if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS: - os.seteuid(ARCHIVEBOX_USER) + if os.getuid() == 0 and os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS: + os.seteuid(ARCHIVEBOX_USER) - # update environment variables so that subprocesses dont try to write to /root - pw_record = pwd.getpwuid(ARCHIVEBOX_USER) - os.environ["HOME"] = pw_record.pw_dir - os.environ["LOGNAME"] = pw_record.pw_name - os.environ["USER"] = pw_record.pw_name - os.environ["XDG_CACHE_HOME"] = str(Path(pw_record.pw_dir) / ".cache") - os.environ["XDG_CONFIG_HOME"] = str(Path(pw_record.pw_dir) / ".config") - os.environ["XDG_DATA_HOME"] = str(Path(pw_record.pw_dir) / ".local" / "share") - os.environ.pop("XDG_RUNTIME_DIR", None) - os.environ.pop("ABXBUS_MULTIPROCESS_SEMAPHORE_DIR", None) + # update environment variables so that subprocesses dont try to write to /root + pw_record = pwd.getpwuid(ARCHIVEBOX_USER) + os.environ["HOME"] = pw_record.pw_dir + os.environ["LOGNAME"] = pw_record.pw_name + os.environ["USER"] = pw_record.pw_name + os.environ["XDG_CACHE_HOME"] = str(Path(pw_record.pw_dir) / ".cache") + os.environ["XDG_CONFIG_HOME"] = str(Path(pw_record.pw_dir) / ".config") + os.environ["XDG_DATA_HOME"] = str(Path(pw_record.pw_dir) / ".local" / "share") + os.environ.pop("XDG_RUNTIME_DIR", None) + os.environ.pop("ABXBUS_MULTIPROCESS_SEMAPHORE_DIR", None) - semaphore_dir = Path(pw_record.pw_dir) / ".cache" / "abxbus" / "semaphores" - os.environ["ABXBUS_MULTIPROCESS_SEMAPHORE_DIR"] = str(semaphore_dir) + semaphore_dir = Path(pw_record.pw_dir) / ".cache" / "abxbus" / "semaphores" + os.environ["ABXBUS_MULTIPROCESS_SEMAPHORE_DIR"] = str(semaphore_dir) - try: - from abxbus import retry as abxbus_retry - except Exception: - pass - else: - abxbus_retry.MULTIPROCESS_SEMAPHORE_DIR = semaphore_dir + with suppress(ImportError): + from abxbus import retry as abxbus_retry + + abxbus_retry.MULTIPROCESS_SEMAPHORE_DIR = semaphore_dir if ARCHIVEBOX_USER == 0 or not ARCHIVEBOX_USER_EXISTS: print( diff --git a/archivebox/core/asgi.py b/archivebox/core/asgi.py index 2d52aee9..46c19223 100644 --- a/archivebox/core/asgi.py +++ b/archivebox/core/asgi.py @@ -7,10 +7,11 @@ For more information on this file, see https://docs.djangoproject.com/en/stable/howto/deployment/asgi/ """ -from archivebox.config.django import setup_django from django.core.asgi import get_asgi_application -setup_django(in_memory_db=False, check_db=True) +from archivebox.config.django import setup_django + +setup_django(check_db=True) def _patch_thread_sensitive_context_shutdown() -> None: diff --git a/archivebox/core/wsgi.py b/archivebox/core/wsgi.py index 1b667177..e667cb7d 100644 --- a/archivebox/core/wsgi.py +++ b/archivebox/core/wsgi.py @@ -11,6 +11,6 @@ import archivebox # noqa from archivebox.config.django import setup_django from django.core.wsgi import get_wsgi_application -setup_django(in_memory_db=False, check_db=True) +setup_django(check_db=True) application = get_wsgi_application()