Preserve oldest filesystem migration edge cases

archivebox/core/migrations/0004_auto_20200713_1552.py

@@ -9,6 +9,12 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        # The original schema allowed NULL timestamps. Use the already-unique
+        # 32-character snapshot ID before enforcing the later NOT NULL field.
+        migrations.RunSQL(
+            sql="UPDATE core_snapshot SET timestamp = id WHERE timestamp IS NULL",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
         migrations.AlterField(
             model_name="snapshot",
             name="timestamp",

archivebox/tests/migrations_helpers.py

@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS django_migrations (
 CREATE TABLE IF NOT EXISTS core_snapshot (
     id CHAR(32) PRIMARY KEY,
     url VARCHAR(200) NOT NULL UNIQUE,
-    timestamp VARCHAR(32) NOT NULL UNIQUE,
+    timestamp VARCHAR(32) UNIQUE,
     title VARCHAR(128),
     tags VARCHAR(256),
     added DATETIME NOT NULL,
@@ -713,6 +713,29 @@ def seed_0_4_data(db_path: Path) -> dict[str, list[dict]]:
         )
         created_data["tags_str"].append(tags)
 
+    # ArchiveBox's original Django schema allowed NULL timestamps. Preserve a
+    # real row from that valid state so the oldest-schema migration cannot
+    # silently narrow the historical schema exercised by this fixture.
+    snapshot_id = generate_uuid()
+    tags = "legacy,null-timestamp"
+    cursor.execute(
+        """
+        INSERT INTO core_snapshot (id, url, timestamp, title, tags, added, updated)
+        VALUES (?, ?, NULL, ?, ?, ?, NULL)
+        """,
+        (snapshot_id, "https://example.net/no-timestamp", "Legacy Missing Timestamp", tags, "2019-05-01 03:27:00"),
+    )
+    created_data["snapshots"].append(
+        {
+            "id": snapshot_id,
+            "url": "https://example.net/no-timestamp",
+            "timestamp": None,
+            "title": "Legacy Missing Timestamp",
+            "tags": tags,
+        },
+    )
+    created_data["tags_str"].append(tags)
+
     cursor.execute("""
         INSERT INTO django_migrations (app, name, applied)
         VALUES ('core', '0001_initial', datetime('now'))

archivebox/tests/test_migrations_04_to_09.py

@@ -43,6 +43,8 @@ def test_oldest_django_collection_migrates_end_to_end_without_data_loss(tmp_path
 
     original_trees = {}
     for index, snapshot in enumerate(original["snapshots"]):
+        if snapshot["timestamp"] is None:
+            continue
         snapshot_dir = tmp_path / "archive" / snapshot["timestamp"]
         snapshot_dir.mkdir(parents=True)
         history = {}
@@ -124,3 +126,8 @@ def test_oldest_django_collection_migrates_end_to_end_without_data_loss(tmp_path
             )
         }
         assert migrated_snapshots == {snapshot["url"]: snapshot["title"] for snapshot in original["snapshots"]}
+        missing_timestamp_snapshot = next(snapshot for snapshot in original["snapshots"] if snapshot["timestamp"] is None)
+        assert connection.execute(
+            "SELECT timestamp FROM core_snapshot WHERE url = ?",
+            (missing_timestamp_snapshot["url"],),
+        ).fetchone() == (missing_timestamp_snapshot["id"],)

archivebox/tests/test_migrations_08_to_09.py

@@ -1007,3 +1007,54 @@ def test_07_filesystem_hop_preserves_complete_output_tree(tmp_path):
     crawl_snapshot_links = list((tmp_path / "archive" / "users").glob("*/crawls/*/*/*/snapshots/*/*"))
     assert crawl_snapshot_links
     assert all(path.is_symlink() for path in crawl_snapshot_links)
+
+
+@pytest.mark.parametrize("fs_version", ("0.7.0", "0.8.0", "0.8.5", "0.9.0", "0.9.1", "0.9.2", "0.9.3"))
+def test_each_declared_filesystem_hop_preserves_outputs(migration_08_data, fs_version):
+    """Every declared source version must migrate through the public update command."""
+    work_dir, db_path, original_data = migration_08_data
+    result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=90)
+    assert result.returncode == 0, result.stderr
+
+    snapshot = original_data["snapshots"][0]
+    with sqlite3.connect(db_path) as connection:
+        connection.execute("UPDATE core_snapshot SET fs_version = ? WHERE id = ?", (fs_version, snapshot["id"]))
+        username, bookmarked_at = connection.execute(
+            """
+            SELECT u.username, s.bookmarked_at
+            FROM core_snapshot s
+            JOIN crawls_crawl c ON c.id = s.crawl_id
+            JOIN auth_user u ON u.id = c.created_by_id
+            WHERE s.id = ?
+            """,
+            (snapshot["id"],),
+        ).fetchone()
+
+    if fs_version in ("0.7.0", "0.8.0", "0.8.5"):
+        source_dir = work_dir / "archive" / snapshot["timestamp"]
+    else:
+        source_dir = (
+            work_dir
+            / "archive"
+            / "users"
+            / username
+            / "snapshots"
+            / datetime.fromisoformat(bookmarked_at).strftime("%Y%m%d")
+            / urlparse(snapshot["url"]).hostname
+            / snapshot["id"]
+        )
+
+    output = source_dir / "unknown-plugin" / "payload.bin"
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_bytes(b"migration payload\x00\xff")
+    (output.parent / "payload-link").symlink_to("payload.bin")
+    (source_dir / "unknown-empty-dir" / "nested").mkdir(parents=True)
+    expected_tree = filesystem_manifest(source_dir)
+
+    result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=180)
+    assert result.returncode == 0, result.stderr
+
+    migrated_dir = source_dir.resolve()
+    assert {path: filesystem_manifest(migrated_dir).get(path) for path in expected_tree} == expected_tree
+    with sqlite3.connect(db_path) as connection:
+        assert connection.execute("SELECT fs_version FROM core_snapshot WHERE id = ?", (snapshot["id"],)).fetchone() == ("0.9.4",)