The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-14 11:06:13 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Migrate queued legacy snapshot files

Browse Source
This commit is contained in:
Nick Sweeting 2026-08-02 01:37:28 -07:00
parent 74a37ad836
commit 096060b591
No known key found for this signature in database
2 changed files with 35 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
archivebox/services/runner.py
View File

@ -2255,6 +2255,19 @@ def run_pending_crawls(
if _fast_forward_same_path_snapshot_fs_versions():
continue
if maintenance_only:
# Filesystem migration is independent of lifecycle status; do not
# tick queued snapshots or start their extraction work here.
filesystem_snapshot = (
Snapshot.objects.filter(retry_at__lte=timezone.now())
.exclude(fs_version=Snapshot._fs_current_version())
.order_by("retry_at", "created_at")
.first()
)
if filesystem_snapshot and Snapshot.claim_for_worker(filesystem_snapshot, lock_seconds=60):
if run_snapshot_maintenance(str(filesystem_snapshot.id)):
continue
if not maintenance_only:
active_snapshots = Snapshot.objects.filter(
retry_at__lte=timezone.now(),

46
archivebox/tests/test_cli_update_reindex_snapshots.py
View File

@ -137,46 +137,42 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized
env = cli_env(disable_extractors=True)
url = f"https://example.com/fs-{source_version}"
legacy_layout = source_version.startswith(("0.7.", "0.8."))
destination = None
if source_version == "0.8.5":
add_process = run_archivebox_cmd(["add", url], env=env, timeout=90)
assert add_process.returncode == 0, add_process.stderr
with use_archivebox_db(tmp_path):
snapshot = Snapshot.objects.get(url=url)
destination = snapshot.output_dir
timestamp = snapshot.timestamp
Snapshot.objects.filter(pk=snapshot.pk).update(fs_version=source_version)
source_dir = tmp_path / "archive" / timestamp
source_dir.mkdir(parents=True, exist_ok=True)
(destination / "existing-user-output.bin").write_bytes(b"preserve interrupted migration output")
elif legacy_layout:
timestamp = "1700000000"
source_dir = tmp_path / "archive" / timestamp
add_process = run_archivebox_cmd(["add", url], env=env, timeout=90)
assert add_process.returncode == 0, add_process.stderr
with use_archivebox_db(tmp_path):
snapshot = Snapshot.objects.get(url=url)
destination = snapshot.output_dir
Snapshot.objects.filter(pk=snapshot.pk).update(
fs_version=source_version,
status=Snapshot.StatusChoices.QUEUED,
retry_at=None,
)
snapshot.refresh_from_db()
source_dir = tmp_path / "archive" / snapshot.timestamp if legacy_layout else destination
if legacy_layout:
source_dir.mkdir(parents=True, exist_ok=True)
(source_dir / "index.jsonl").unlink(missing_ok=True)
(source_dir / "index.json").write_text(
json.dumps(
{
"url": url,
"timestamp": timestamp,
"timestamp": snapshot.timestamp,
"title": f"Filesystem {source_version}",
"fs_version": source_version,
"status": "queued",
"archive_results": [],
},
),
)
else:
add_process = run_archivebox_cmd(["add", url], env=env, timeout=90)
assert add_process.returncode == 0, add_process.stderr
with use_archivebox_db(tmp_path):
snapshot = Snapshot.objects.get(url=url)
Snapshot.objects.filter(pk=snapshot.pk).update(fs_version=source_version)
snapshot.refresh_from_db()
source_dir = snapshot.output_dir
if source_version == "0.8.5":
(destination / "existing-user-output.bin").write_bytes(b"preserve interrupted migration output")
(source_dir / "unknown" / "empty").mkdir(parents=True, exist_ok=True)
(source_dir / "unknown" / "payload.bin").write_bytes(b"filesystem migration payload\x00\xff")
(source_dir / "unknown" / "payload-link").symlink_to("payload.bin")
original_tree = filesystem_manifest(source_dir)
original_tree.pop("index.jsonl", None)
update_process = run_archivebox_cmd(["update", "--migrate-only"], env=env, timeout=90)
assert update_process.returncode == 0, f"Initial update failed: {update_process.stderr}"
@ -186,6 +182,8 @@ def test_update_migrates_every_declared_filesystem_version(tmp_path, initialized
assert snapshot.fs_version == Snapshot._fs_current_version()
migrated_dir = snapshot.output_dir.resolve()
migrated_tree = filesystem_manifest(migrated_dir)
assert snapshot.status == Snapshot.StatusChoices.QUEUED
assert snapshot.archiveresult_set.count() == 0
if source_version == "0.8.5":
assert (migrated_dir / "existing-user-output.bin").read_bytes() == b"preserve interrupted migration output"