mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-12 19:50:57 +05:00
402 lines
20 KiB
Python
402 lines
20 KiB
Python
__package__ = "archivebox.misc"
|
|
|
|
import os
|
|
import signal
|
|
import sys
|
|
import time
|
|
import threading
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
|
|
from rich import print
|
|
from rich.panel import Panel
|
|
|
|
# DO NOT ADD ANY TOP-LEVEL IMPORTS HERE to anything other than builtin python libraries
|
|
# this file is imported by archivebox/__init__.py
|
|
# and any imports here will be imported by EVERYTHING else
|
|
# so this file should only be used for pure python checks
|
|
# that don't need to import other parts of ArchiveBox
|
|
|
|
# if a check needs to import other parts of ArchiveBox,
|
|
# the imports should be done inside the check function
|
|
# and you should make sure if you need to import any django stuff
|
|
# that the check is called after django.setup() has been called
|
|
|
|
|
|
def _migration_interrupt_message(*, before_apply: bool = False) -> str:
|
|
status = "Migration cancelled before any changes were applied." if before_apply else "Migration interrupted."
|
|
return (
|
|
f"\n[X] {status}\n"
|
|
" Database migrations are atomic; interrupted migration work is rolled back or left unapplied,\n"
|
|
" so no partially-applied migration is recorded and no data loss has occurred.\n\n"
|
|
" To continue the upgrade, run:\n"
|
|
" archivebox init\n"
|
|
)
|
|
|
|
|
|
@contextmanager
|
|
def _exit_on_migration_interrupt():
|
|
if threading.current_thread() is not threading.main_thread():
|
|
yield
|
|
return
|
|
|
|
handled_signals = (signal.SIGINT, signal.SIGTERM)
|
|
previous_handlers = {sig: signal.getsignal(sig) for sig in handled_signals}
|
|
|
|
def handle_shutdown(_signum, _frame):
|
|
try:
|
|
os.write(sys.stderr.fileno(), _migration_interrupt_message().encode())
|
|
except Exception:
|
|
pass
|
|
os._exit(130)
|
|
|
|
try:
|
|
for sig in handled_signals:
|
|
signal.signal(sig, handle_shutdown)
|
|
yield
|
|
finally:
|
|
for sig, previous_handler in previous_handlers.items():
|
|
signal.signal(sig, previous_handler)
|
|
|
|
|
|
def check_data_folder(config=None, **config_kwargs) -> None:
|
|
from archivebox import DATA_DIR
|
|
from archivebox.config import CONSTANTS
|
|
from archivebox.config.common import get_config
|
|
from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
|
|
|
|
config = config or get_config(**config_kwargs)
|
|
archive_dir = config.ARCHIVE_DIR
|
|
archive_dir_exists = os.path.isdir(archive_dir)
|
|
if not archive_dir_exists:
|
|
print("[red][X] No archivebox index found in the current directory.[/red]", file=sys.stderr)
|
|
print(f" {DATA_DIR}", file=sys.stderr)
|
|
print(file=sys.stderr)
|
|
print(" [violet]Hint[/violet]: Are you running archivebox in the right folder?", file=sys.stderr)
|
|
print(" cd path/to/your/archive/folder", file=sys.stderr)
|
|
print(" archivebox [command]", file=sys.stderr)
|
|
print(file=sys.stderr)
|
|
print(" [violet]Hint[/violet]: To create a new archive collection or import existing data in this folder, run:", file=sys.stderr)
|
|
print(" archivebox init", file=sys.stderr)
|
|
raise SystemExit(2)
|
|
|
|
# Create data dir subdirs
|
|
create_and_chown_dir(CONSTANTS.SOURCES_DIR)
|
|
create_and_chown_dir(config.USERS_DIR)
|
|
create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default")
|
|
create_and_chown_dir(CONSTANTS.LOGS_DIR)
|
|
# create_and_chown_dir(CONSTANTS.CACHE_DIR)
|
|
|
|
# Create /tmp and /lib dirs if they don't exist
|
|
get_or_create_working_tmp_dir(autofix=True, quiet=False, config=config)
|
|
get_or_create_working_lib_dir(autofix=True, quiet=False, config=config)
|
|
|
|
# Check data dir permissions, /tmp, and /lib permissions
|
|
check_data_dir_permissions(config=config)
|
|
|
|
|
|
def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_delay: int = 3) -> list[str]:
|
|
from archivebox import DATA_DIR
|
|
from archivebox.misc.db import apply_migrations, migration_state, pending_migrations
|
|
|
|
pending, missing_from_code, rollback_targets = migration_state()
|
|
is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"]) or os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1"
|
|
|
|
if missing_from_code:
|
|
print(
|
|
"[red][X] This collection was migrated by a newer version of ArchiveBox than the one currently running.[/red]",
|
|
file=sys.stderr,
|
|
)
|
|
print(f" {DATA_DIR}", file=sys.stderr)
|
|
print(file=sys.stderr)
|
|
print(" [violet]Hint:[/violet] Upgrade ArchiveBox / pull the latest Docker image, then restart:", file=sys.stderr)
|
|
print(" docker compose pull && docker compose up -d", file=sys.stderr)
|
|
print(file=sys.stderr)
|
|
print(" Applied migrations missing from this build:", file=sys.stderr)
|
|
for migration in missing_from_code[:10]:
|
|
print(f" {migration}", file=sys.stderr)
|
|
if len(missing_from_code) > 10:
|
|
print(f" ... and {len(missing_from_code) - 10} more", file=sys.stderr)
|
|
print(file=sys.stderr)
|
|
print(
|
|
" If you are intentionally trying to downgrade, switch back to the newer version temporarily",
|
|
file=sys.stderr,
|
|
)
|
|
print(
|
|
" and run this to downgrade the DB version (back up your DB first!):",
|
|
file=sys.stderr,
|
|
)
|
|
for app, target in sorted(rollback_targets.items()):
|
|
print(f" archivebox manage migrate {app} {target}", file=sys.stderr)
|
|
raise SystemExit(3)
|
|
|
|
if pending and not is_migrating:
|
|
print("[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]", file=sys.stderr)
|
|
print(f" {DATA_DIR}", file=sys.stderr)
|
|
print(file=sys.stderr)
|
|
print(
|
|
f" [violet]Hint:[/violet] To upgrade it to the latest version and apply the {len(pending)} pending migrations, run:",
|
|
file=sys.stderr,
|
|
)
|
|
print(" archivebox init", file=sys.stderr)
|
|
if auto_apply:
|
|
print(file=sys.stderr)
|
|
print(
|
|
f"[yellow][*] ArchiveBox will apply migrations automatically in {cancel_delay}s. Press CTRL+C to cancel.[/yellow]",
|
|
file=sys.stderr,
|
|
)
|
|
try:
|
|
time.sleep(cancel_delay)
|
|
except KeyboardInterrupt:
|
|
print(_migration_interrupt_message(before_apply=True), file=sys.stderr)
|
|
raise SystemExit(130) from None
|
|
|
|
# Always delegate to Django's migration executor. It records each
|
|
# migration only after it succeeds, so power loss or SIGKILL leaves
|
|
# unapplied work visible here and the next startup resumes normally.
|
|
print("[yellow][*] Applying database migrations...[/yellow]", file=sys.stderr)
|
|
try:
|
|
with _exit_on_migration_interrupt():
|
|
apply_migrations(stdout=sys.stderr, stderr=sys.stderr, verbosity=1)
|
|
except KeyboardInterrupt:
|
|
print(_migration_interrupt_message(), file=sys.stderr)
|
|
raise SystemExit(130) from None
|
|
return pending_migrations()
|
|
if blocking:
|
|
raise SystemExit(3)
|
|
return pending
|
|
|
|
|
|
def check_io_encoding():
|
|
PYTHON_ENCODING = (sys.__stdout__ or sys.stdout or sys.__stderr__ or sys.stderr).encoding.upper().replace("UTF8", "UTF-8")
|
|
|
|
if PYTHON_ENCODING != "UTF-8":
|
|
print(
|
|
f"[red][X] Your system is running python3 scripts with a bad locale setting: {PYTHON_ENCODING} (it should be UTF-8).[/red]",
|
|
file=sys.stderr,
|
|
)
|
|
print(' To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)', file=sys.stderr)
|
|
print(' Or if you\'re using ubuntu/debian, run "dpkg-reconfigure locales"', file=sys.stderr)
|
|
print("")
|
|
print(" Confirm that it's fixed by opening a new shell and running:", file=sys.stderr)
|
|
print(' python3 -c "import sys; print(sys.stdout.encoding)" # should output UTF-8', file=sys.stderr)
|
|
raise SystemExit(2)
|
|
|
|
# # hard errors: check python version
|
|
# if sys.version_info[:3] < (3, 10, 0):
|
|
# print('[red][X] Python version is not new enough: {sys.version} (>3.10 is required)[/red]', file=sys.stderr)
|
|
# print(' See https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#python for help upgrading your Python installation.', file=sys.stderr)
|
|
# raise SystemExit(2)
|
|
|
|
# # hard errors: check django version
|
|
# if int(django.VERSION[0]) < 5:
|
|
# print('[red][X] Django version is not new enough: {django.VERSION[:3]} (>=5.0 is required)[/red]', file=sys.stderr)
|
|
# print(' Upgrade django using pip or your system package manager: pip3 install --upgrade django', file=sys.stderr)
|
|
# raise SystemExit(2)
|
|
|
|
|
|
def check_not_root():
|
|
from archivebox.config.permissions import IS_ROOT
|
|
|
|
is_getting_help = "-h" in sys.argv or "--help" in sys.argv or "help" in sys.argv
|
|
is_getting_version = "--version" in sys.argv or "version" in sys.argv
|
|
|
|
if IS_ROOT and not (is_getting_help or is_getting_version):
|
|
print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
|
|
print(" Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
|
|
print(" https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
|
|
|
|
|
|
def check_not_inside_source_dir():
|
|
"""Prevent running ArchiveBox from inside its source directory (would pollute repo with data files)."""
|
|
cwd = Path(os.getcwd()).resolve()
|
|
is_source_dir = (cwd / "archivebox" / "__init__.py").exists() and (cwd / "pyproject.toml").exists()
|
|
data_dir_set_elsewhere = os.environ.get("DATA_DIR", "").strip() and Path(os.environ["DATA_DIR"]).resolve() != cwd
|
|
is_testing = "pytest" in sys.modules or "unittest" in sys.modules
|
|
|
|
if is_source_dir and not data_dir_set_elsewhere and not is_testing:
|
|
raise SystemExit("[!] Cannot run from source dir, set DATA_DIR or cd to a data folder first")
|
|
|
|
|
|
def check_data_dir_permissions(config=None, **config_kwargs):
|
|
from archivebox import DATA_DIR
|
|
from archivebox.misc.logging import STDERR
|
|
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, DEFAULT_PUID, DEFAULT_PGID, IS_ROOT, USER
|
|
from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
|
|
|
|
data_dir_stat = Path(DATA_DIR).stat()
|
|
data_dir_uid, data_dir_gid = data_dir_stat.st_uid, data_dir_stat.st_gid
|
|
data_owned_by_root = data_dir_uid == 0
|
|
|
|
# data_owned_by_default_user = data_dir_uid == DEFAULT_PUID or data_dir_gid == DEFAULT_PGID
|
|
data_owner_doesnt_match = (data_dir_uid != ARCHIVEBOX_USER and data_dir_gid != ARCHIVEBOX_GROUP) if not IS_ROOT else False
|
|
data_not_writable = not (os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.W_OK))
|
|
if data_not_writable:
|
|
STDERR.print(
|
|
f"\n[yellow]:warning: Warning: ArchiveBox [blue]DATA_DIR[/blue] is not writable by ArchiveBox user [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue] ({USER}).[/yellow]",
|
|
)
|
|
elif data_owned_by_root:
|
|
STDERR.print(
|
|
"\n[yellow]:warning: Warning: ArchiveBox [blue]DATA_DIR[/blue] appears to be owned by [red]root[/red]. If this is an NFS or mapped volume and writes work, no change is required.[/yellow]",
|
|
)
|
|
elif data_owner_doesnt_match:
|
|
STDERR.print(
|
|
f"\n[yellow]:warning: Warning: ArchiveBox [blue]DATA_DIR[/blue] is currently owned by [red]{data_dir_uid}:{data_dir_gid}[/red], but ArchiveBox user is [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue] ({USER})! (ArchiveBox may not be able to write to the data dir)[/yellow]",
|
|
)
|
|
|
|
if data_not_writable:
|
|
STDERR.print(
|
|
f"[violet]Hint:[/violet] Change the current ownership [red]{data_dir_uid}[/red]:{data_dir_gid} (PUID:PGID) to the user & group that will run ArchiveBox, e.g.:",
|
|
)
|
|
STDERR.print(f" [grey53]sudo[/grey53] chown -R [blue]{DEFAULT_PUID}:{DEFAULT_PGID}[/blue] {DATA_DIR.resolve()}")
|
|
STDERR.print(" Avoid recursive chown on very large archives unless you know the full tree needs repair.")
|
|
STDERR.print()
|
|
STDERR.print("[blue]More info:[/blue]")
|
|
STDERR.print(
|
|
" [link=https://github.com/ArchiveBox/ArchiveBox#storage-requirements]https://github.com/ArchiveBox/ArchiveBox#storage-requirements[/link]",
|
|
)
|
|
STDERR.print(
|
|
" [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
|
|
)
|
|
STDERR.print(
|
|
" [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid[/link]",
|
|
)
|
|
STDERR.print(
|
|
" [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",
|
|
)
|
|
|
|
from archivebox.config.common import get_config
|
|
|
|
config = config or get_config(**config_kwargs)
|
|
try:
|
|
tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True, config=config) or config.TMP_DIR
|
|
except Exception:
|
|
tmp_dir = config.TMP_DIR
|
|
|
|
try:
|
|
lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
|
|
except Exception:
|
|
lib_dir = config.LIB_DIR
|
|
|
|
# Check /tmp dir permissions
|
|
check_tmp_dir(tmp_dir, throw=False, must_exist=True, config=config)
|
|
|
|
# Check /lib dir permissions
|
|
check_lib_dir(lib_dir, throw=False, must_exist=True, config=config)
|
|
|
|
# Derive directory mode from file mode by OR-ing the execute bits (matches
|
|
# the old DIR_OUTPUT_PERMISSIONS=755 vs OUTPUT_PERMISSIONS=644 convention).
|
|
os.umask(0o777 - (int(config.OUTPUT_PERMISSIONS, base=8) | 0o111))
|
|
|
|
|
|
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
|
|
from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
|
|
from archivebox.misc.logging import STDERR
|
|
from archivebox.misc.logging_util import pretty_path
|
|
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
|
|
from archivebox.config.common import get_config
|
|
|
|
config = config or get_config(**config_kwargs)
|
|
tmp_dir = tmp_dir or config.TMP_DIR
|
|
socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
|
|
|
|
if not must_exist and not os.path.isdir(tmp_dir):
|
|
# just check that its viable based on its length (because dir may not exist yet, we cant check if its writable)
|
|
return len(f"file://{socket_file}") <= 96
|
|
|
|
tmp_is_valid = False
|
|
try:
|
|
tmp_is_valid = dir_is_writable(tmp_dir)
|
|
if not config.ALLOW_NO_UNIX_SOCKETS:
|
|
tmp_is_valid = tmp_is_valid and assert_dir_can_contain_unix_sockets(tmp_dir)
|
|
assert tmp_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to TMP_DIR={tmp_dir}"
|
|
assert len(f"file://{socket_file}") <= 96, (
|
|
f"ArchiveBox TMP_DIR={tmp_dir} is too long, dir containing unix socket files must be <90 chars."
|
|
)
|
|
return True
|
|
except Exception as e:
|
|
if not quiet:
|
|
STDERR.print()
|
|
ERROR_TEXT = "\n".join(
|
|
(
|
|
"",
|
|
f"[red]:cross_mark: ArchiveBox is unable to use TMP_DIR={pretty_path(tmp_dir)}[/red]",
|
|
f" [yellow]{e}[/yellow]",
|
|
"",
|
|
"[blue]Info:[/blue] [grey53]The TMP_DIR is used for the supervisord unix socket file and other temporary files.",
|
|
" - It [red]must[/red] be on a local drive (not inside a docker volume, remote network drive, or FUSE mount).",
|
|
f" - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
|
|
" - It [red]must[/red] be a *short* path (less than 90 characters) due to UNIX path length restrictions for sockets.",
|
|
" - It [yellow]should[/yellow] be able to hold at least 200MB of data (in-progress downloads can be large).[/grey53]",
|
|
"",
|
|
"[violet]Hint:[/violet] Fix it by setting TMP_DIR to a path that meets these requirements, e.g.:",
|
|
f" [green]archivebox config --set TMP_DIR={get_or_create_working_tmp_dir(autofix=False, quiet=True) or '/tmp/archivebox'}[/green]",
|
|
"",
|
|
),
|
|
)
|
|
STDERR.print(
|
|
Panel(
|
|
ERROR_TEXT,
|
|
expand=False,
|
|
border_style="red",
|
|
title="[red]:cross_mark: Error with configured TMP_DIR[/red]",
|
|
subtitle="Background workers may fail to start until fixed.",
|
|
),
|
|
)
|
|
STDERR.print()
|
|
if throw:
|
|
raise OSError(f"TMP_DIR={tmp_dir} is invalid, ArchiveBox is unable to use it and the server will fail to start!") from e
|
|
return False
|
|
|
|
|
|
def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True, config=None, **config_kwargs):
|
|
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
|
|
from archivebox.misc.logging import STDERR
|
|
from archivebox.misc.logging_util import pretty_path
|
|
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
|
|
from archivebox.config.common import get_config
|
|
|
|
config = config or get_config(**config_kwargs)
|
|
lib_dir = lib_dir or config.LIB_DIR
|
|
|
|
if not must_exist and not os.path.isdir(lib_dir):
|
|
return True
|
|
|
|
lib_is_valid = False
|
|
try:
|
|
lib_is_valid = dir_is_writable(lib_dir)
|
|
assert lib_is_valid, f"ArchiveBox user PUID={ARCHIVEBOX_USER} PGID={ARCHIVEBOX_GROUP} is unable to write to LIB_DIR={lib_dir}"
|
|
return True
|
|
except Exception as e:
|
|
if not quiet:
|
|
STDERR.print()
|
|
ERROR_TEXT = "\n".join(
|
|
(
|
|
"",
|
|
f"[red]:cross_mark: ArchiveBox is unable to use LIB_DIR={pretty_path(lib_dir)}[/red]",
|
|
f" [yellow]{e}[/yellow]",
|
|
"",
|
|
"[blue]Info:[/blue] [grey53]The LIB_DIR is used to store ArchiveBox auto-installed plugin library and binary dependencies.",
|
|
f" - It [red]must[/red] be readable and writable by the ArchiveBox user (PUID={ARCHIVEBOX_USER}, PGID={ARCHIVEBOX_GROUP}).",
|
|
" - It [yellow]should[/yellow] be on a local (ideally fast) drive like an SSD or HDD (not on a network drive or external HDD).",
|
|
" - It [yellow]should[/yellow] be able to hold at least 1GB of data (some dependencies like Chrome can be large).[/grey53]",
|
|
"",
|
|
"[violet]Hint:[/violet] Fix it by setting LIB_DIR to a path that meets these requirements, e.g.:",
|
|
f" [green]archivebox config --set LIB_DIR={get_or_create_working_lib_dir(autofix=False, quiet=True) or config.LIB_DIR}[/green]",
|
|
"",
|
|
),
|
|
)
|
|
STDERR.print(
|
|
Panel(
|
|
ERROR_TEXT,
|
|
expand=False,
|
|
border_style="red",
|
|
title="[red]:cross_mark: Error with configured LIB_DIR[/red]",
|
|
subtitle="[yellow]Dependencies may not auto-install properly until fixed.[/yellow]",
|
|
),
|
|
)
|
|
STDERR.print()
|
|
if throw:
|
|
raise OSError(f"LIB_DIR={lib_dir} is invalid, ArchiveBox is unable to use it and dependencies will fail to install.") from e
|
|
return False
|