diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py index 997ae05f..9421ae3d 100644 --- a/archivebox/misc/db.py +++ b/archivebox/misc/db.py @@ -9,7 +9,6 @@ from pathlib import Path from typing import TextIO from typing import Any import fcntl -import importlib import time from collections.abc import Callable from contextlib import contextmanager @@ -212,6 +211,137 @@ def migration_lock(stdout: TextIO | None = None): fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) +# Migration names that previously existed in ArchiveBox's source tree but have +# since been deleted (squashed away, renamed, moved between apps, etc.). DBs +# upgraded incrementally through 0.8.x → 0.9.x dev rcs accumulate rows for +# these in ``django_migrations``; the newer-DB guard added in 65dc2521 would +# otherwise refuse to start with "applied migrations missing from this build" +# and brick beta-tester collections. We deliberately do NOT use Django's +# ``replaces=`` for these because Django's all-or-none replaces semantics +# splits the migration graph when only a *subset* of the replaces list is +# applied (which is exactly what happens for users at different intermediate +# dev branch states). This set is the authoritative compat list — extend it +# when squashing more migrations away. Generated from +# ``git log --diff-filter=D --name-only`` over each app's migrations/ tree. +HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset( + { + # core: 0023→0075 sequence plus every transient dev rename + ("core", "0002_auto_20190417_0739"), + ("core", "0006_auto_20200915_2006"), + ("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"), + ("core", "0023_new_schema"), + ("core", "0024_auto_20240513_1143"), + ("core", "0024_b_clear_config_fields"), + ("core", "0024_c_disable_fk_checks"), + ("core", "0024_d_fix_crawls_config"), + ("core", "0024_f_add_snapshot_config"), + ("core", "0024_snapshot_crawl"), + ("core", "0025_allow_duplicate_urls_per_crawl"), + ("core", "0025_alter_archiveresult_uuid"), + ("core", "0025_cleanup_schema"), + ("core", "0026_archiveresult_created_archiveresult_created_by_and_more"), + ("core", "0026_final_field_adjustments"), + ("core", "0026_remove_archiveresult_output_dir_and_more"), + ("core", "0027_alter_archiveresult_created_by_and_more"), + ("core", "0027_alter_archiveresult_hook_name_alter_archiveresult_id_and_more"), + ("core", "0027_update_snapshot_ids"), + ("core", "0028_alter_archiveresult_uuid"), + ("core", "0028_snapshot_fs_version"), + ("core", "0029_alter_archiveresult_id"), + ("core", "0029_archiveresult_hook_fields"), + ("core", "0030_alter_archiveresult_uuid"), + ("core", "0030_migrate_output_field"), + ("core", "0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more"), + ("core", "0031_snapshot_parent_snapshot"), + ("core", "0032_alter_archiveresult_binary_and_more"), + ("core", "0032_alter_archiveresult_id"), + ("core", "0033_rename_extractor_add_hook_name"), + ("core", "0033_rename_id_archiveresult_old_id"), + ("core", "0034_alter_archiveresult_old_id_alter_archiveresult_uuid"), + ("core", "0034_snapshot_current_step"), + ("core", "0035_remove_archiveresult_uuid_archiveresult_id"), + ("core", "0035_snapshot_crawl_non_nullable_remove_created_by"), + ("core", "0036_alter_archiveresult_id_alter_archiveresult_old_id"), + ("core", "0036_remove_archiveresult_created_by"), + ("core", "0037_remove_archiveresult_output_dir_and_more"), + ("core", "0037_rename_id_snapshot_old_id"), + ("core", "0038_fix_missing_columns"), + ("core", "0038_rename_uuid_snapshot_id"), + ("core", "0039_fix_num_uses_values"), + ("core", "0039_rename_snapshot_archiveresult_snapshot_old"), + ("core", "0040_archiveresult_snapshot"), + ("core", "0041_alter_archiveresult_snapshot_and_more"), + ("core", "0042_remove_archiveresult_snapshot_old"), + ("core", "0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more"), + ("core", "0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more"), + ("core", "0045_alter_snapshot_old_id"), + ("core", "0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more"), + ("core", "0047_alter_snapshottag_unique_together_and_more"), + ("core", "0048_alter_archiveresult_snapshot_and_more"), + ("core", "0049_rename_snapshot_snapshottag_snapshot_old_and_more"), + ("core", "0050_alter_snapshottag_snapshot_old"), + ("core", "0051_snapshottag_snapshot_alter_snapshottag_snapshot_old"), + ("core", "0052_alter_snapshottag_unique_together_and_more"), + ("core", "0053_remove_snapshottag_snapshot_old"), + ("core", "0054_alter_snapshot_timestamp"), + ("core", "0055_alter_tag_slug"), + ("core", "0056_remove_tag_uuid"), + ("core", "0057_rename_id_tag_old_id"), + ("core", "0058_alter_tag_old_id"), + ("core", "0059_tag_id"), + ("core", "0060_alter_tag_id"), + ("core", "0061_rename_tag_snapshottag_old_tag_and_more"), + ("core", "0062_alter_snapshottag_old_tag"), + ("core", "0063_snapshottag_tag_alter_snapshottag_old_tag"), + ("core", "0064_alter_snapshottag_unique_together_and_more"), + ("core", "0065_remove_snapshottag_old_tag"), + ("core", "0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id"), + ("core", "0067_alter_snapshottag_tag"), + ("core", "0068_alter_archiveresult_options"), + ("core", "0069_alter_archiveresult_created_alter_snapshot_added_and_more"), + ("core", "0070_alter_archiveresult_created_by_alter_snapshot_added_and_more"), + ("core", "0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more"), + ("core", "0072_rename_added_snapshot_bookmarked_at_and_more"), + ("core", "0073_rename_created_archiveresult_created_at_and_more"), + ("core", "0074_alter_snapshot_downloaded_at"), + ("core", "0075_crawl"), + # api: pre-squash 0001_squashed plus 0002→0009 chain + ("api", "0001_squashed"), + ("api", "0002_alter_apitoken_options"), + ("api", "0002_alter_outboundwebhook_options_and_more"), + ("api", "0003_alter_apitoken_created_by_and_more"), + ("api", "0003_rename_user_apitoken_created_by_apitoken_abid_and_more"), + ("api", "0004_alter_apitoken_id_alter_apitoken_uuid"), + ("api", "0004_rename_user_apitoken_created_by_apitoken_modified_and_more"), + ("api", "0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more"), + ("api", "0006_remove_outboundwebhook_uuid_apitoken_id_and_more"), + ("api", "0007_alter_apitoken_created_by"), + ("api", "0008_alter_apitoken_created_alter_apitoken_created_by_and_more"), + ("api", "0009_rename_created_apitoken_created_at_and_more"), + # machine: pre-squash 0001_squashed plus transient 0002→0005 renames + ("machine", "0001_squashed"), + ("machine", "0002_alter_dependency_bin_name_and_more"), + ("machine", "0002_alter_machine_stats_installedbinary"), + ("machine", "0002_process_parent_and_type"), + ("machine", "0002_rename_custom_cmds_to_overrides"), + ("machine", "0003_alter_dependency_id_alter_installedbinary_dependency_and_more"), + ("machine", "0003_alter_installedbinary_options_and_more"), + ("machine", "0004_alter_installedbinary_abspath_and_more"), + ("machine", "0004_drop_dependency_table"), + ("machine", "0004_rename_installedbinary_to_binary"), + ("machine", "0005_binary_binproviders_binary_output_dir_and_more"), + # crawls: transient dev renames around the seed-model removal + ("crawls", "0002_delete_outlink"), + ("crawls", "0002_drop_seed_model"), + ("crawls", "0002_upgrade_to_0_9_0"), + ("crawls", "0003_alter_crawl_output_dir"), + ("crawls", "0004_alter_crawl_output_dir"), + ("crawls", "0005_drop_seed_id_column"), + ("crawls", "0006_alter_crawl_config_alter_crawl_output_dir_and_more"), + }, +) + + @enforce_types def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]: """Cheaply compare migration files to django_migrations without invoking migrate.""" @@ -231,26 +361,21 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic applied = retry_sqlite_locks(applied_rows, label="checking applied migrations") disk_migrations: set[tuple[str, str]] = set() + # Names that any current migration declares it ``replaces=``. Whether or + # not we use ``replaces=`` today, supporting it costs nothing and keeps + # the checker honest if a future migration adopts it. + squashed_replaced: set[tuple[str, str]] = set() app_labels = {app_config.label for app_config in apps.get_app_configs()} - for app_config in apps.get_app_configs(): - module_name, explicit = MigrationLoader.migrations_module(app_config.label) - if module_name is None: - continue - try: - module = importlib.import_module(module_name) - except ModuleNotFoundError: - if explicit: - raise - continue - module_file = getattr(module, "__file__", None) - if not module_file: - continue - for migration_file in Path(module_file).parent.glob("[0-9][0-9][0-9][0-9]_*.py"): - disk_migrations.add((app_config.label, migration_file.stem)) + loader = MigrationLoader(connection=None, ignore_no_migrations=True, load=False) + loader.load_disk() + for (app_label, migration_name), migration in loader.disk_migrations.items(): + disk_migrations.add((app_label, migration_name)) + for replaced_app, replaced_name in getattr(migration, "replaces", ()) or (): + squashed_replaced.add((replaced_app, replaced_name)) applied = {(app, name) for app, name in applied if app in app_labels} pending = [f"{app}.{name}" for app, name in sorted(disk_migrations - applied)] - missing_pairs = sorted(applied - disk_migrations) + missing_pairs = sorted(applied - disk_migrations - squashed_replaced - HISTORICAL_GHOST_MIGRATIONS) missing_from_code = [f"{app}.{name}" for app, name in missing_pairs] rollback_targets = { app: ( diff --git a/archivebox/tests/test_cli_init.py b/archivebox/tests/test_cli_init.py index b5df81c8..1cab73a6 100644 --- a/archivebox/tests/test_cli_init.py +++ b/archivebox/tests/test_cli_init.py @@ -185,6 +185,37 @@ def test_init_refuses_database_migrated_by_newer_code(tmp_path): assert "archivebox manage migrate crawls " in result.stderr +def test_init_recovers_from_pre_squash_dev_history(tmp_path): + """Pre-squash dev DBs (rows for migrations now absorbed by ``replaces=``) + must NOT trip the newer-DB guard — every historical squash would otherwise + brick beta-tester collections that pre-date the squash commit.""" + os.chdir(tmp_path) + result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + assert result.returncode == 0 + + # Sampling — one name per affected app, all listed in the ``replaces=`` + # declarations of the current squash anchors. If any of these get treated + # as missing-from-code, dev DBs that ran the historical chain pre-squash + # would refuse to start. + historical_pre_squash_rows = [ + ("api", "0002_alter_apitoken_options"), + ("api", "0009_rename_created_apitoken_created_at_and_more"), + ("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"), + ("core", "0074_alter_snapshot_downloaded_at"), + ("core", "0075_crawl"), + ("machine", "0002_alter_machine_stats_installedbinary"), + ("machine", "0004_alter_installedbinary_abspath_and_more"), + ] + with use_archivebox_db(tmp_path): + for app, name in historical_pre_squash_rows: + MigrationRecorder.Migration.objects.create(app=app, name=name, applied=timezone.now()) + connections["default"].commit() + + result = subprocess.run(["archivebox", "init"], capture_output=True, text=True) + assert result.returncode == 0, f"init refused to recover pre-squash dev DB.\nstdout={result.stdout}\nstderr={result.stderr}" + assert "migrated by a newer version of ArchiveBox" not in result.stderr + + def test_init_with_existing_data_preserves_snapshots(tmp_path, process, disable_extractors_dict): """Test that re-running init preserves existing snapshot data.""" os.chdir(tmp_path)