diff --git a/archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py b/archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py index ddf16ea5..88d555e1 100644 --- a/archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py +++ b/archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py @@ -1,7 +1,7 @@ # Generated by hand on 2026-01-02 # Migrate ArchiveResult from integer PK to UUID PK (matching Snapshot) -from django.db import migrations, models, connection +from django.db import migrations, models from uuid import UUID from archivebox.uuid_compat import uuid7 @@ -32,7 +32,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor): if schema_editor.connection.vendor != "sqlite": return - cursor = connection.cursor() + cursor = schema_editor.connection.cursor() # Check if table exists and has data cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'") @@ -101,23 +101,35 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor): col_names = [col[1] for col in columns] has_uuid_column = "uuid" in col_names + used_uuids = set() + + def fresh_uuid(): + while True: + candidate = uuid7().hex + if candidate not in used_uuids: + used_uuids.add(candidate) + return candidate + + def unique_uuid(existing_uuid): + try: + candidate = UUID(str(existing_uuid)).hex if existing_uuid else None + except (TypeError, ValueError, AttributeError): + candidate = None + + if candidate and candidate not in used_uuids: + used_uuids.add(candidate) + return candidate + return fresh_uuid() + if has_uuid_column: cursor.execute("SELECT id, uuid FROM core_archiveresult") records = cursor.fetchall() - id_to_uuid = {} - for old_id, existing_uuid in records: - if existing_uuid: - # Normalize existing UUID to 32-char hex format (Django SQLite UUIDField format) - # (existing UUIDs might be stored with or without dashes in old schema) - id_to_uuid[old_id] = UUID(existing_uuid).hex - else: - # Generate new UUIDv7 (time-ordered) as 32-char hex - id_to_uuid[old_id] = uuid7().hex + id_to_uuid = {old_id: unique_uuid(existing_uuid) for old_id, existing_uuid in records} else: # 0.7.x path: no uuid column, generate new UUIDs for all records cursor.execute("SELECT id FROM core_archiveresult") records = cursor.fetchall() - id_to_uuid = {old_id: uuid7().hex for (old_id,) in records} + id_to_uuid = {old_id: fresh_uuid() for (old_id,) in records} # Step 3: Copy data with UUIDs as new primary key cursor.execute("SELECT * FROM core_archiveresult") @@ -125,7 +137,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor): # col_names already fetched in Step 2 inserted_count = 0 - for i, record in enumerate(old_records): + for record in old_records: old_id = record[col_names.index("id")] new_uuid = id_to_uuid[old_id] @@ -163,24 +175,29 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor): insert_values = [new_uuid] + [values.get(f) for f in existing_fields] - try: - cursor.execute( - f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})", - insert_values, - ) - inserted_count += 1 - if inserted_count % PROGRESS_EVERY == 0: - print(f" copied {inserted_count}/{len(old_records)} ArchiveResults...") - except Exception as e: - print(f"[0029] ERROR inserting record {old_id}: {e}") - if i == 0: - print(f"[0029] First record values: {insert_values[:5]}...") - raise + cursor.execute( + f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})", + insert_values, + ) + inserted_count += 1 + if inserted_count % PROGRESS_EVERY == 0: + print(f" copied {inserted_count}/{len(old_records)} ArchiveResults...") if old_records: print(f" copied {inserted_count}/{len(old_records)} ArchiveResults") - # Step 4: Replace old table with new table + cursor.execute("SELECT COUNT(*) FROM core_archiveresult") + source_count = cursor.fetchone()[0] + cursor.execute("SELECT COUNT(*) FROM core_archiveresult_new") + destination_count = cursor.fetchone()[0] + if source_count != row_count or destination_count != row_count or inserted_count != row_count: + raise RuntimeError( + "ArchiveResult UUID migration row-count mismatch: " + f"expected={row_count}, source={source_count}, destination={destination_count}, inserted={inserted_count}", + ) + + # Step 4: Replace old table with new table. Django runs this migration in + # one transaction, so any failure above leaves the source table untouched. cursor.execute("DROP TABLE core_archiveresult") cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult") diff --git a/archivebox/tests/test_migration_uuid_integrity.py b/archivebox/tests/test_migration_uuid_integrity.py new file mode 100644 index 00000000..fcb6f081 --- /dev/null +++ b/archivebox/tests/test_migration_uuid_integrity.py @@ -0,0 +1,56 @@ +import sqlite3 +from uuid import UUID + +from .migrations_helpers import ( + SCHEMA_0_7, + create_data_dir_structure, + run_archivebox_migration_cmd, + seed_0_7_data, +) + + +def test_0029_preserves_duplicate_and_malformed_legacy_uuids(tmp_path): + """Every 0.7.4 ArchiveResult must survive conversion to a unique UUID PK.""" + db_path = tmp_path / "index.sqlite3" + create_data_dir_structure(tmp_path) + + conn = sqlite3.connect(db_path) + conn.executescript(SCHEMA_0_7) + conn.close() + seed_0_7_data(db_path) + + duplicate_uuid = "12345678123456781234567812345678" + conn = sqlite3.connect(db_path) + conn.execute("ALTER TABLE core_archiveresult ADD COLUMN uuid CHAR(32) NOT NULL DEFAULT ''") + source_rows = conn.execute( + "SELECT id, snapshot_id, extractor, status, output FROM core_archiveresult ORDER BY id", + ).fetchall() + for row_number, (result_id, *_metadata) in enumerate(source_rows, start=1): + conn.execute( + "UPDATE core_archiveresult SET uuid = ? WHERE id = ?", + (f"{row_number:032x}", result_id), + ) + conn.execute( + "UPDATE core_archiveresult SET uuid = ? WHERE id IN (?, ?)", + (duplicate_uuid, source_rows[0][0], source_rows[1][0]), + ) + conn.execute("UPDATE core_archiveresult SET uuid = ? WHERE id = ?", ("not-a-valid-uuid", source_rows[2][0])) + conn.commit() + + expected_results = sorted(row[1:] for row in source_rows) + conn.close() + + result = run_archivebox_migration_cmd(tmp_path, ["init"], timeout=60) + assert result.returncode == 0, f"Init failed:\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + + conn = sqlite3.connect(db_path) + migrated_rows = conn.execute( + "SELECT id, snapshot_id, plugin, status, output_str FROM core_archiveresult ORDER BY snapshot_id, plugin", + ).fetchall() + conn.close() + + migrated_ids = [row[0] for row in migrated_rows] + assert len(migrated_rows) == len(source_rows) + assert len(set(migrated_ids)) == len(source_rows) + assert all(UUID(result_id).hex == result_id for result_id in migrated_ids) + assert sorted(row[1:] for row in migrated_rows) == expected_results