diff --git a/archivebox/core/migrations/0004_auto_20200713_1552.py b/archivebox/core/migrations/0004_auto_20200713_1552.py index 02f2738c..098ed1b0 100644 --- a/archivebox/core/migrations/0004_auto_20200713_1552.py +++ b/archivebox/core/migrations/0004_auto_20200713_1552.py @@ -9,6 +9,12 @@ class Migration(migrations.Migration): ] operations = [ + # The original schema allowed NULL timestamps. Use the already-unique + # 32-character snapshot ID before enforcing the later NOT NULL field. + migrations.RunSQL( + sql="UPDATE core_snapshot SET timestamp = id WHERE timestamp IS NULL", + reverse_sql=migrations.RunSQL.noop, + ), migrations.AlterField( model_name="snapshot", name="timestamp", diff --git a/archivebox/tests/migrations_helpers.py b/archivebox/tests/migrations_helpers.py index 1bdc3841..75f8358f 100644 --- a/archivebox/tests/migrations_helpers.py +++ b/archivebox/tests/migrations_helpers.py @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS django_migrations ( CREATE TABLE IF NOT EXISTS core_snapshot ( id CHAR(32) PRIMARY KEY, url VARCHAR(200) NOT NULL UNIQUE, - timestamp VARCHAR(32) NOT NULL UNIQUE, + timestamp VARCHAR(32) UNIQUE, title VARCHAR(128), tags VARCHAR(256), added DATETIME NOT NULL, @@ -713,6 +713,29 @@ def seed_0_4_data(db_path: Path) -> dict[str, list[dict]]: ) created_data["tags_str"].append(tags) + # ArchiveBox's original Django schema allowed NULL timestamps. Preserve a + # real row from that valid state so the oldest-schema migration cannot + # silently narrow the historical schema exercised by this fixture. + snapshot_id = generate_uuid() + tags = "legacy,null-timestamp" + cursor.execute( + """ + INSERT INTO core_snapshot (id, url, timestamp, title, tags, added, updated) + VALUES (?, ?, NULL, ?, ?, ?, NULL) + """, + (snapshot_id, "https://example.net/no-timestamp", "Legacy Missing Timestamp", tags, "2019-05-01 03:27:00"), + ) + created_data["snapshots"].append( + { + "id": snapshot_id, + "url": "https://example.net/no-timestamp", + "timestamp": None, + "title": "Legacy Missing Timestamp", + "tags": tags, + }, + ) + created_data["tags_str"].append(tags) + cursor.execute(""" INSERT INTO django_migrations (app, name, applied) VALUES ('core', '0001_initial', datetime('now')) diff --git a/archivebox/tests/test_migrations_04_to_09.py b/archivebox/tests/test_migrations_04_to_09.py index b0917e5e..b861a3f1 100644 --- a/archivebox/tests/test_migrations_04_to_09.py +++ b/archivebox/tests/test_migrations_04_to_09.py @@ -43,6 +43,8 @@ def test_oldest_django_collection_migrates_end_to_end_without_data_loss(tmp_path original_trees = {} for index, snapshot in enumerate(original["snapshots"]): + if snapshot["timestamp"] is None: + continue snapshot_dir = tmp_path / "archive" / snapshot["timestamp"] snapshot_dir.mkdir(parents=True) history = {} @@ -124,3 +126,8 @@ def test_oldest_django_collection_migrates_end_to_end_without_data_loss(tmp_path ) } assert migrated_snapshots == {snapshot["url"]: snapshot["title"] for snapshot in original["snapshots"]} + missing_timestamp_snapshot = next(snapshot for snapshot in original["snapshots"] if snapshot["timestamp"] is None) + assert connection.execute( + "SELECT timestamp FROM core_snapshot WHERE url = ?", + (missing_timestamp_snapshot["url"],), + ).fetchone() == (missing_timestamp_snapshot["id"],) diff --git a/archivebox/tests/test_migrations_08_to_09.py b/archivebox/tests/test_migrations_08_to_09.py index 8bc21d38..5235ef95 100644 --- a/archivebox/tests/test_migrations_08_to_09.py +++ b/archivebox/tests/test_migrations_08_to_09.py @@ -1007,3 +1007,54 @@ def test_07_filesystem_hop_preserves_complete_output_tree(tmp_path): crawl_snapshot_links = list((tmp_path / "archive" / "users").glob("*/crawls/*/*/*/snapshots/*/*")) assert crawl_snapshot_links assert all(path.is_symlink() for path in crawl_snapshot_links) + + +@pytest.mark.parametrize("fs_version", ("0.7.0", "0.8.0", "0.8.5", "0.9.0", "0.9.1", "0.9.2", "0.9.3")) +def test_each_declared_filesystem_hop_preserves_outputs(migration_08_data, fs_version): + """Every declared source version must migrate through the public update command.""" + work_dir, db_path, original_data = migration_08_data + result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=90) + assert result.returncode == 0, result.stderr + + snapshot = original_data["snapshots"][0] + with sqlite3.connect(db_path) as connection: + connection.execute("UPDATE core_snapshot SET fs_version = ? WHERE id = ?", (fs_version, snapshot["id"])) + username, bookmarked_at = connection.execute( + """ + SELECT u.username, s.bookmarked_at + FROM core_snapshot s + JOIN crawls_crawl c ON c.id = s.crawl_id + JOIN auth_user u ON u.id = c.created_by_id + WHERE s.id = ? + """, + (snapshot["id"],), + ).fetchone() + + if fs_version in ("0.7.0", "0.8.0", "0.8.5"): + source_dir = work_dir / "archive" / snapshot["timestamp"] + else: + source_dir = ( + work_dir + / "archive" + / "users" + / username + / "snapshots" + / datetime.fromisoformat(bookmarked_at).strftime("%Y%m%d") + / urlparse(snapshot["url"]).hostname + / snapshot["id"] + ) + + output = source_dir / "unknown-plugin" / "payload.bin" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(b"migration payload\x00\xff") + (output.parent / "payload-link").symlink_to("payload.bin") + (source_dir / "unknown-empty-dir" / "nested").mkdir(parents=True) + expected_tree = filesystem_manifest(source_dir) + + result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=180) + assert result.returncode == 0, result.stderr + + migrated_dir = source_dir.resolve() + assert {path: filesystem_manifest(migrated_dir).get(path) for path in expected_tree} == expected_tree + with sqlite3.connect(db_path) as connection: + assert connection.execute("SELECT fs_version FROM core_snapshot WHERE id = ?", (snapshot["id"],)).fetchone() == ("0.9.4",)