Preserve ArchiveResults during UUID migration

archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py

@@ -1,7 +1,7 @@
 # Generated by hand on 2026-01-02
 # Migrate ArchiveResult from integer PK to UUID PK (matching Snapshot)
 
-from django.db import migrations, models, connection
+from django.db import migrations, models
 from uuid import UUID
 from archivebox.uuid_compat import uuid7
 
@@ -32,7 +32,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     if schema_editor.connection.vendor != "sqlite":
         return
 
-    cursor = connection.cursor()
+    cursor = schema_editor.connection.cursor()
 
     # Check if table exists and has data
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'")
@@ -101,23 +101,35 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     col_names = [col[1] for col in columns]
     has_uuid_column = "uuid" in col_names
 
+    used_uuids = set()
+
+    def fresh_uuid():
+        while True:
+            candidate = uuid7().hex
+            if candidate not in used_uuids:
+                used_uuids.add(candidate)
+                return candidate
+
+    def unique_uuid(existing_uuid):
+        try:
+            candidate = UUID(str(existing_uuid)).hex if existing_uuid else None
+        except (TypeError, ValueError, AttributeError):
+            candidate = None
+
+        if candidate and candidate not in used_uuids:
+            used_uuids.add(candidate)
+            return candidate
+        return fresh_uuid()
+
     if has_uuid_column:
         cursor.execute("SELECT id, uuid FROM core_archiveresult")
         records = cursor.fetchall()
-        id_to_uuid = {}
-        for old_id, existing_uuid in records:
-            if existing_uuid:
-                # Normalize existing UUID to 32-char hex format (Django SQLite UUIDField format)
-                # (existing UUIDs might be stored with or without dashes in old schema)
-                id_to_uuid[old_id] = UUID(existing_uuid).hex
-            else:
-                # Generate new UUIDv7 (time-ordered) as 32-char hex
-                id_to_uuid[old_id] = uuid7().hex
+        id_to_uuid = {old_id: unique_uuid(existing_uuid) for old_id, existing_uuid in records}
     else:
         # 0.7.x path: no uuid column, generate new UUIDs for all records
         cursor.execute("SELECT id FROM core_archiveresult")
         records = cursor.fetchall()
-        id_to_uuid = {old_id: uuid7().hex for (old_id,) in records}
+        id_to_uuid = {old_id: fresh_uuid() for (old_id,) in records}
 
     # Step 3: Copy data with UUIDs as new primary key
     cursor.execute("SELECT * FROM core_archiveresult")
@@ -125,7 +137,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
 
     # col_names already fetched in Step 2
     inserted_count = 0
-    for i, record in enumerate(old_records):
+    for record in old_records:
         old_id = record[col_names.index("id")]
         new_uuid = id_to_uuid[old_id]
 
@@ -163,24 +175,29 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
 
         insert_values = [new_uuid] + [values.get(f) for f in existing_fields]
 
-        try:
-            cursor.execute(
-                f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})",
-                insert_values,
-            )
-            inserted_count += 1
-            if inserted_count % PROGRESS_EVERY == 0:
-                print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
-        except Exception as e:
-            print(f"[0029] ERROR inserting record {old_id}: {e}")
-            if i == 0:
-                print(f"[0029] First record values: {insert_values[:5]}...")
-                raise
+        cursor.execute(
+            f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})",
+            insert_values,
+        )
+        inserted_count += 1
+        if inserted_count % PROGRESS_EVERY == 0:
+            print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
 
     if old_records:
         print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults")
 
-    # Step 4: Replace old table with new table
+    cursor.execute("SELECT COUNT(*) FROM core_archiveresult")
+    source_count = cursor.fetchone()[0]
+    cursor.execute("SELECT COUNT(*) FROM core_archiveresult_new")
+    destination_count = cursor.fetchone()[0]
+    if source_count != row_count or destination_count != row_count or inserted_count != row_count:
+        raise RuntimeError(
+            "ArchiveResult UUID migration row-count mismatch: "
+            f"expected={row_count}, source={source_count}, destination={destination_count}, inserted={inserted_count}",
+        )
+
+    # Step 4: Replace old table with new table. Django runs this migration in
+    # one transaction, so any failure above leaves the source table untouched.
     cursor.execute("DROP TABLE core_archiveresult")
     cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult")
 

archivebox/tests/test_migration_uuid_integrity.py

@@ -0,0 +1,56 @@
+import sqlite3
+from uuid import UUID
+
+from .migrations_helpers import (
+    SCHEMA_0_7,
+    create_data_dir_structure,
+    run_archivebox_migration_cmd,
+    seed_0_7_data,
+)
+
+
+def test_0029_preserves_duplicate_and_malformed_legacy_uuids(tmp_path):
+    """Every 0.7.4 ArchiveResult must survive conversion to a unique UUID PK."""
+    db_path = tmp_path / "index.sqlite3"
+    create_data_dir_structure(tmp_path)
+
+    conn = sqlite3.connect(db_path)
+    conn.executescript(SCHEMA_0_7)
+    conn.close()
+    seed_0_7_data(db_path)
+
+    duplicate_uuid = "12345678123456781234567812345678"
+    conn = sqlite3.connect(db_path)
+    conn.execute("ALTER TABLE core_archiveresult ADD COLUMN uuid CHAR(32) NOT NULL DEFAULT ''")
+    source_rows = conn.execute(
+        "SELECT id, snapshot_id, extractor, status, output FROM core_archiveresult ORDER BY id",
+    ).fetchall()
+    for row_number, (result_id, *_metadata) in enumerate(source_rows, start=1):
+        conn.execute(
+            "UPDATE core_archiveresult SET uuid = ? WHERE id = ?",
+            (f"{row_number:032x}", result_id),
+        )
+    conn.execute(
+        "UPDATE core_archiveresult SET uuid = ? WHERE id IN (?, ?)",
+        (duplicate_uuid, source_rows[0][0], source_rows[1][0]),
+    )
+    conn.execute("UPDATE core_archiveresult SET uuid = ? WHERE id = ?", ("not-a-valid-uuid", source_rows[2][0]))
+    conn.commit()
+
+    expected_results = sorted(row[1:] for row in source_rows)
+    conn.close()
+
+    result = run_archivebox_migration_cmd(tmp_path, ["init"], timeout=60)
+    assert result.returncode == 0, f"Init failed:\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+
+    conn = sqlite3.connect(db_path)
+    migrated_rows = conn.execute(
+        "SELECT id, snapshot_id, plugin, status, output_str FROM core_archiveresult ORDER BY snapshot_id, plugin",
+    ).fetchall()
+    conn.close()
+
+    migrated_ids = [row[0] for row in migrated_rows]
+    assert len(migrated_rows) == len(source_rows)
+    assert len(set(migrated_ids)) == len(source_rows)
+    assert all(UUID(result_id).hex == result_id for result_id in migrated_ids)
+    assert sorted(row[1:] for row in migrated_rows) == expected_results