The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: recover pre-squash dev DBs without bricking the newer-DB guard

Browse Source 
The newer-DB guard added in 65dc2521 refused to start on any collection
whose django_migrations table referenced migrations that have since been
squashed away from source (core/0023→0075, api/0002→0009, machine/0002→
0004, plus every transient dev-branch rename). That's the state of every
beta-tester DB that was upgraded incrementally through the 0.8.x → 0.9.x
rc chain — cabbage's demo deployment was the first concrete casualty.

Django's standard `replaces=` pattern doesn't fit here: its all-or-none
semantics split the migration graph whenever only a subset of the listed
replaces is applied, which is exactly what happens when different beta
testers stopped at different intermediate dev branches. Instead, keep an
explicit HISTORICAL_GHOST_MIGRATIONS allowlist in misc/db.py enumerating
every squashed-away name, and subtract it from missing_from_code in
migration_state(). The hard SystemExit(3) for truly unknown future names
stays — only known-historical ghosts are absorbed.

Adds test_init_recovers_from_pre_squash_dev_history covering the gap the
pre-existing migration tests missed (they only seed published 0.4/0.7/
0.8 schemas, never a post-squash dev DB).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Sweeting 2026-05-30 13:49:15 -07:00
parent 0373f73fff
commit a8b54931cd
No known key found for this signature in database
2 changed files with 173 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
archivebox/misc/db.py
View File

@ -9,7 +9,6 @@ from pathlib import Path
from typing import TextIO
from typing import Any
import fcntl
import importlib
import time
from collections.abc import Callable
from contextlib import contextmanager
@ -212,6 +211,137 @@ def migration_lock(stdout: TextIO | None = None):
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
# Migration names that previously existed in ArchiveBox's source tree but have
# since been deleted (squashed away, renamed, moved between apps, etc.). DBs
# upgraded incrementally through 0.8.x → 0.9.x dev rcs accumulate rows for
# these in ``django_migrations``; the newer-DB guard added in 65dc2521 would
# otherwise refuse to start with "applied migrations missing from this build"
# and brick beta-tester collections. We deliberately do NOT use Django's
# ``replaces=`` for these because Django's all-or-none replaces semantics
# splits the migration graph when only a *subset* of the replaces list is
# applied (which is exactly what happens for users at different intermediate
# dev branch states). This set is the authoritative compat list — extend it
# when squashing more migrations away. Generated from
# ``git log --diff-filter=D --name-only`` over each app's migrations/ tree.
HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
{
# core: 0023→0075 sequence plus every transient dev rename
("core", "0002_auto_20190417_0739"),
("core", "0006_auto_20200915_2006"),
("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
("core", "0023_new_schema"),
("core", "0024_auto_20240513_1143"),
("core", "0024_b_clear_config_fields"),
("core", "0024_c_disable_fk_checks"),
("core", "0024_d_fix_crawls_config"),
("core", "0024_f_add_snapshot_config"),
("core", "0024_snapshot_crawl"),
("core", "0025_allow_duplicate_urls_per_crawl"),
("core", "0025_alter_archiveresult_uuid"),
("core", "0025_cleanup_schema"),
("core", "0026_archiveresult_created_archiveresult_created_by_and_more"),
("core", "0026_final_field_adjustments"),
("core", "0026_remove_archiveresult_output_dir_and_more"),
("core", "0027_alter_archiveresult_created_by_and_more"),
("core", "0027_alter_archiveresult_hook_name_alter_archiveresult_id_and_more"),
("core", "0027_update_snapshot_ids"),
("core", "0028_alter_archiveresult_uuid"),
("core", "0028_snapshot_fs_version"),
("core", "0029_alter_archiveresult_id"),
("core", "0029_archiveresult_hook_fields"),
("core", "0030_alter_archiveresult_uuid"),
("core", "0030_migrate_output_field"),
("core", "0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more"),
("core", "0031_snapshot_parent_snapshot"),
("core", "0032_alter_archiveresult_binary_and_more"),
("core", "0032_alter_archiveresult_id"),
("core", "0033_rename_extractor_add_hook_name"),
("core", "0033_rename_id_archiveresult_old_id"),
("core", "0034_alter_archiveresult_old_id_alter_archiveresult_uuid"),
("core", "0034_snapshot_current_step"),
("core", "0035_remove_archiveresult_uuid_archiveresult_id"),
("core", "0035_snapshot_crawl_non_nullable_remove_created_by"),
("core", "0036_alter_archiveresult_id_alter_archiveresult_old_id"),
("core", "0036_remove_archiveresult_created_by"),
("core", "0037_remove_archiveresult_output_dir_and_more"),
("core", "0037_rename_id_snapshot_old_id"),
("core", "0038_fix_missing_columns"),
("core", "0038_rename_uuid_snapshot_id"),
("core", "0039_fix_num_uses_values"),
("core", "0039_rename_snapshot_archiveresult_snapshot_old"),
("core", "0040_archiveresult_snapshot"),
("core", "0041_alter_archiveresult_snapshot_and_more"),
("core", "0042_remove_archiveresult_snapshot_old"),
("core", "0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more"),
("core", "0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more"),
("core", "0045_alter_snapshot_old_id"),
("core", "0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more"),
("core", "0047_alter_snapshottag_unique_together_and_more"),
("core", "0048_alter_archiveresult_snapshot_and_more"),
("core", "0049_rename_snapshot_snapshottag_snapshot_old_and_more"),
("core", "0050_alter_snapshottag_snapshot_old"),
("core", "0051_snapshottag_snapshot_alter_snapshottag_snapshot_old"),
("core", "0052_alter_snapshottag_unique_together_and_more"),
("core", "0053_remove_snapshottag_snapshot_old"),
("core", "0054_alter_snapshot_timestamp"),
("core", "0055_alter_tag_slug"),
("core", "0056_remove_tag_uuid"),
("core", "0057_rename_id_tag_old_id"),
("core", "0058_alter_tag_old_id"),
("core", "0059_tag_id"),
("core", "0060_alter_tag_id"),
("core", "0061_rename_tag_snapshottag_old_tag_and_more"),
("core", "0062_alter_snapshottag_old_tag"),
("core", "0063_snapshottag_tag_alter_snapshottag_old_tag"),
("core", "0064_alter_snapshottag_unique_together_and_more"),
("core", "0065_remove_snapshottag_old_tag"),
("core", "0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id"),
("core", "0067_alter_snapshottag_tag"),
("core", "0068_alter_archiveresult_options"),
("core", "0069_alter_archiveresult_created_alter_snapshot_added_and_more"),
("core", "0070_alter_archiveresult_created_by_alter_snapshot_added_and_more"),
("core", "0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more"),
("core", "0072_rename_added_snapshot_bookmarked_at_and_more"),
("core", "0073_rename_created_archiveresult_created_at_and_more"),
("core", "0074_alter_snapshot_downloaded_at"),
("core", "0075_crawl"),
# api: pre-squash 0001_squashed plus 0002→0009 chain
("api", "0001_squashed"),
("api", "0002_alter_apitoken_options"),
("api", "0002_alter_outboundwebhook_options_and_more"),
("api", "0003_alter_apitoken_created_by_and_more"),
("api", "0003_rename_user_apitoken_created_by_apitoken_abid_and_more"),
("api", "0004_alter_apitoken_id_alter_apitoken_uuid"),
("api", "0004_rename_user_apitoken_created_by_apitoken_modified_and_more"),
("api", "0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more"),
("api", "0006_remove_outboundwebhook_uuid_apitoken_id_and_more"),
("api", "0007_alter_apitoken_created_by"),
("api", "0008_alter_apitoken_created_alter_apitoken_created_by_and_more"),
("api", "0009_rename_created_apitoken_created_at_and_more"),
# machine: pre-squash 0001_squashed plus transient 0002→0005 renames
("machine", "0001_squashed"),
("machine", "0002_alter_dependency_bin_name_and_more"),
("machine", "0002_alter_machine_stats_installedbinary"),
("machine", "0002_process_parent_and_type"),
("machine", "0002_rename_custom_cmds_to_overrides"),
("machine", "0003_alter_dependency_id_alter_installedbinary_dependency_and_more"),
("machine", "0003_alter_installedbinary_options_and_more"),
("machine", "0004_alter_installedbinary_abspath_and_more"),
("machine", "0004_drop_dependency_table"),
("machine", "0004_rename_installedbinary_to_binary"),
("machine", "0005_binary_binproviders_binary_output_dir_and_more"),
# crawls: transient dev renames around the seed-model removal
("crawls", "0002_delete_outlink"),
("crawls", "0002_drop_seed_model"),
("crawls", "0002_upgrade_to_0_9_0"),
("crawls", "0003_alter_crawl_output_dir"),
("crawls", "0004_alter_crawl_output_dir"),
("crawls", "0005_drop_seed_id_column"),
("crawls", "0006_alter_crawl_config_alter_crawl_output_dir_and_more"),
},
)
@enforce_types
def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
"""Cheaply compare migration files to django_migrations without invoking migrate."""
@ -231,26 +361,21 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
applied = retry_sqlite_locks(applied_rows, label="checking applied migrations")
disk_migrations: set[tuple[str, str]] = set()
# Names that any current migration declares it ``replaces=``. Whether or
# not we use ``replaces=`` today, supporting it costs nothing and keeps
# the checker honest if a future migration adopts it.
squashed_replaced: set[tuple[str, str]] = set()
app_labels = {app_config.label for app_config in apps.get_app_configs()}
for app_config in apps.get_app_configs():
module_name, explicit = MigrationLoader.migrations_module(app_config.label)
if module_name is None:
continue
try:
module = importlib.import_module(module_name)
except ModuleNotFoundError:
if explicit:
raise
continue
module_file = getattr(module, "__file__", None)
if not module_file:
continue
for migration_file in Path(module_file).parent.glob("[0-9][0-9][0-9][0-9]_*.py"):
disk_migrations.add((app_config.label, migration_file.stem))
loader = MigrationLoader(connection=None, ignore_no_migrations=True, load=False)
loader.load_disk()
for (app_label, migration_name), migration in loader.disk_migrations.items():
disk_migrations.add((app_label, migration_name))
for replaced_app, replaced_name in getattr(migration, "replaces", ()) or ():
squashed_replaced.add((replaced_app, replaced_name))
applied = {(app, name) for app, name in applied if app in app_labels}
pending = [f"{app}.{name}" for app, name in sorted(disk_migrations - applied)]
missing_pairs = sorted(applied - disk_migrations)
missing_pairs = sorted(applied - disk_migrations - squashed_replaced - HISTORICAL_GHOST_MIGRATIONS)
missing_from_code = [f"{app}.{name}" for app, name in missing_pairs]
rollback_targets = {
app: (

31
archivebox/tests/test_cli_init.py
View File

@ -185,6 +185,37 @@ def test_init_refuses_database_migrated_by_newer_code(tmp_path):
assert "archivebox manage migrate crawls " in result.stderr
def test_init_recovers_from_pre_squash_dev_history(tmp_path):
"""Pre-squash dev DBs (rows for migrations now absorbed by ``replaces=``)
must NOT trip the newer-DB guard — every historical squash would otherwise
brick beta-tester collections that pre-date the squash commit."""
os.chdir(tmp_path)
result = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
assert result.returncode == 0
# Sampling — one name per affected app, all listed in the ``replaces=``
# declarations of the current squash anchors. If any of these get treated
# as missing-from-code, dev DBs that ran the historical chain pre-squash
# would refuse to start.
historical_pre_squash_rows = [
("api", "0002_alter_apitoken_options"),
("api", "0009_rename_created_apitoken_created_at_and_more"),
("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
("core", "0074_alter_snapshot_downloaded_at"),
("core", "0075_crawl"),
("machine", "0002_alter_machine_stats_installedbinary"),
("machine", "0004_alter_installedbinary_abspath_and_more"),
]
with use_archivebox_db(tmp_path):
for app, name in historical_pre_squash_rows:
MigrationRecorder.Migration.objects.create(app=app, name=name, applied=timezone.now())
connections["default"].commit()
result = subprocess.run(["archivebox", "init"], capture_output=True, text=True)
assert result.returncode == 0, f"init refused to recover pre-squash dev DB.\nstdout={result.stdout}\nstderr={result.stderr}"
assert "migrated by a newer version of ArchiveBox" not in result.stderr
def test_init_with_existing_data_preserves_snapshots(tmp_path, process, disable_extractors_dict):
"""Test that re-running init preserves existing snapshot data."""
os.chdir(tmp_path)