diff --git a/archivebox/cli/archivebox_init.py b/archivebox/cli/archivebox_init.py index 8fb12824..957b78ee 100755 --- a/archivebox/cli/archivebox_init.py +++ b/archivebox/cli/archivebox_init.py @@ -29,6 +29,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non from archivebox.config.common import get_config from archivebox.config.collection import write_config_file from archivebox.misc.db import apply_migrations + from archivebox.misc.checks import check_migrations config = get_config() @@ -91,6 +92,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non from archivebox.config.django import setup_django setup_django() + check_migrations(blocking=True, auto_apply=False) for migration_line in apply_migrations(DATA_DIR): sys.stdout.write(f" {migration_line}\n") diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index 33399b45..f920136c 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -58,11 +58,35 @@ def check_data_folder(config=None, **config_kwargs) -> None: def check_migrations(*, blocking: bool = True, auto_apply: bool = False, cancel_delay: int = 3) -> list[str]: from archivebox import DATA_DIR - from archivebox.misc.db import apply_migrations, pending_migrations + from archivebox.misc.db import apply_migrations, migration_state, pending_migrations - pending = pending_migrations() + pending, missing_from_code, rollback_targets = migration_state() is_migrating = any(arg in sys.argv for arg in ["makemigrations", "migrate", "init"]) + if missing_from_code: + print( + "[red][X] This collection was migrated by a newer version of ArchiveBox than the one currently running.[/red]", + file=sys.stderr, + ) + print(f" {DATA_DIR}", file=sys.stderr) + print(file=sys.stderr) + print(" [violet]Hint:[/violet] Upgrade ArchiveBox / pull the latest Docker image, then restart:", file=sys.stderr) + print(" docker compose pull && docker compose up -d", file=sys.stderr) + print(file=sys.stderr) + print(" Applied migrations missing from this build:", file=sys.stderr) + for migration in missing_from_code[:10]: + print(f" {migration}", file=sys.stderr) + if len(missing_from_code) > 10: + print(f" ... and {len(missing_from_code) - 10} more", file=sys.stderr) + print(file=sys.stderr) + print( + " If you intentionally downgraded and need to roll the DB back, run this with a build that contains those migrations:", + file=sys.stderr, + ) + for app, target in sorted(rollback_targets.items()): + print(f" archivebox manage migrate {app} {target}", file=sys.stderr) + raise SystemExit(3) + if pending and not is_migrating: print("[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]", file=sys.stderr) print(f" {DATA_DIR}", file=sys.stderr) diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index 4da840c2..1803511b 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -122,7 +122,7 @@ def migration_lock(stdout: TextIO | None = None): @enforce_types -def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]: +def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]: """Cheaply compare migration files to django_migrations without invoking migrate.""" from django.apps import apps from django.db import connection @@ -140,6 +140,7 @@ def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]: applied = retry_sqlite_locks(applied_rows, label="checking applied migrations") disk_migrations: set[tuple[str, str]] = set() + app_labels = {app_config.label for app_config in apps.get_app_configs()} for app_config in apps.get_app_configs(): module_name, explicit = MigrationLoader.migrations_module(app_config.label) if module_name is None: @@ -156,7 +157,26 @@ def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]: for migration_file in Path(module_file).parent.glob("[0-9][0-9][0-9][0-9]_*.py"): disk_migrations.add((app_config.label, migration_file.stem)) - return [f"{app}.{name}" for app, name in sorted(disk_migrations - applied)] + applied = {(app, name) for app, name in applied if app in app_labels} + pending = [f"{app}.{name}" for app, name in sorted(disk_migrations - applied)] + missing_pairs = sorted(applied - disk_migrations) + missing_from_code = [f"{app}.{name}" for app, name in missing_pairs] + rollback_targets = { + app: ( + max(name for disk_app, name in disk_migrations if disk_app == app) + if any(disk_app == app for disk_app, _name in disk_migrations) + else "zero" + ) + for app, _name in missing_pairs + } + return pending, missing_from_code, rollback_targets + + +@enforce_types +def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]: + """Return migration files on disk that have not been applied yet.""" + pending, _missing_from_code, _rollback_targets = migration_state(out_dir=out_dir) + return pending @enforce_types diff --git a/archivebox/tests/test_cli_init.py b/archivebox/tests/test_cli_init.py index 57b65927..d6bf18bd 100644 --- a/archivebox/tests/test_cli_init.py +++ b/archivebox/tests/test_cli_init.py @@ -8,6 +8,8 @@ import os import subprocess import pytest +from django.utils import timezone +from django.db import connections from django.db.migrations.recorder import MigrationRecorder from archivebox.config.common import get_config @@ -165,6 +167,23 @@ def test_init_is_idempotent(tmp_path): assert count > 0 +def test_init_refuses_database_migrated_by_newer_code(tmp_path): + """A downgraded ArchiveBox build must fail before serving a newer DB schema.""" + os.chdir(tmp_path) + result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + assert result.returncode == 0 + + with use_archivebox_db(tmp_path): + MigrationRecorder.Migration.objects.create(app="crawls", name="9999_future_test", applied=timezone.now()) + connections["default"].commit() + + result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + assert result.returncode == 3 + assert "migrated by a newer version of ArchiveBox" in result.stderr + assert "crawls.9999_future_test" in result.stderr + assert "archivebox manage migrate crawls " in result.stderr + + def test_init_with_existing_data_preserves_snapshots(tmp_path, process, disable_extractors_dict): """Test that re-running init preserves existing snapshot data.""" os.chdir(tmp_path)