The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Preserve snapshot completion from 0.8 betas

Browse Source
This commit is contained in:
Nick Sweeting 2026-08-12 22:31:46 -07:00
parent b8da6ff6ec
commit be1c0b097b
No known key found for this signature in database
2 changed files with 26 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
archivebox/core/migrations/0023_upgrade_to_0_9_0.py
View File

@ -318,6 +318,22 @@ def upgrade_core_tables(apps, schema_editor):
snapshot_cols = get_table_columns("core_snapshot")
has_added = "added" in snapshot_cols
has_bookmarked_at = "bookmarked_at" in snapshot_cols
inferred_status_sql = """
CASE
WHEN EXISTS (
SELECT 1 FROM core_archiveresult
WHERE core_archiveresult.snapshot_id = core_snapshot.id
AND core_archiveresult.status IN ('queued', 'started', 'backoff')
)
THEN 'queued'
WHEN EXISTS (
SELECT 1 FROM core_archiveresult
WHERE core_archiveresult.snapshot_id = core_snapshot.id
)
THEN 'sealed'
ELSE 'queued'
END
"""
if has_added and not has_bookmarked_at:
# Migrating from v0.7.2 (has added/updated fields)
@ -343,20 +359,7 @@ def upgrade_core_tables(apps, schema_editor):
COALESCE(added, CURRENT_TIMESTAMP) as created_at,
COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
updated as downloaded_at,
CASE
WHEN EXISTS (
SELECT 1 FROM core_archiveresult
WHERE core_archiveresult.snapshot_id = core_snapshot.id
AND core_archiveresult.status IN ('queued', 'started', 'backoff')
)
THEN 'queued'
WHEN EXISTS (
SELECT 1 FROM core_archiveresult
WHERE core_archiveresult.snapshot_id = core_snapshot.id
)
THEN 'sealed'
ELSE 'queued'
END as status
{inferred_status_sql} as status
FROM core_snapshot;
""")
print(f" copied {cursor.rowcount} Snapshots")
@ -398,6 +401,9 @@ def upgrade_core_tables(apps, schema_editor):
END
""",
)
else:
insert_cols.append("status")
select_cols.append(inferred_status_sql)
if has_retry_at:
insert_cols.append("retry_at")
select_cols.append("retry_at")

8
archivebox/tests/test_migrations_08_to_09.py
View File

@ -408,7 +408,8 @@ def test_migration_preserves_foreign_keys(migration_08_data):
assert ok, msg
def test_migration_preserves_08_timestamp_meanings(migration_08_data):
@pytest.mark.parametrize("has_snapshot_status", (True, False), ids=("with-status", "without-status"))
def test_migration_preserves_08_timestamp_meanings(migration_08_data, has_snapshot_status):
"""0.8.x already has separated timestamp/bookmarked_at/created_at/downloaded_at fields."""
work_dir, db_path, original_data = migration_08_data
snapshot = original_data["snapshots"][0]
@ -420,6 +421,8 @@ def test_migration_preserves_08_timestamp_meanings(migration_08_data):
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
if not has_snapshot_status:
cursor.execute("ALTER TABLE core_snapshot DROP COLUMN status")
cursor.execute(
"""
UPDATE core_snapshot
@ -437,7 +440,7 @@ def test_migration_preserves_08_timestamp_meanings(migration_08_data):
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute(
"SELECT timestamp, bookmarked_at, created_at, modified_at, downloaded_at FROM core_snapshot WHERE id = ?",
"SELECT timestamp, bookmarked_at, created_at, modified_at, downloaded_at, status FROM core_snapshot WHERE id = ?",
(snapshot["id"],),
)
migrated = cursor.fetchone()
@ -448,6 +451,7 @@ def test_migration_preserves_08_timestamp_meanings(migration_08_data):
assert migrated[2].startswith("2024-08-28"), migrated[2]
assert migrated[3].startswith("2024-08-29"), migrated[3]
assert migrated[4].startswith("2024-08-30"), migrated[4]
assert migrated[5] == "sealed"
def test_hyphenated_crawl_ids_are_normalized_before_snapshot_saves(migration_08_data):