Merge remote-tracking branch 'origin/dev' into audit-pr1827

archivebox/core/migrations/0052_unique_archiveresult_per_snapshot_plugin.py

@@ -1,89 +0,0 @@
-import hashlib
-
-from django.db import migrations, models
-
-
-STASH_KEY = "__archivebox_0052_original_row"
-TEMP_PLUGIN_PREFIX = "__abx52_"
-
-
-def _temporary_plugin_name(row_id, reserved_plugins):
-    """Return a deterministic 32-character plugin name unused by this snapshot."""
-    salt = 0
-    while True:
-        digest = hashlib.sha256(f"{row_id}:{salt}".encode()).hexdigest()[:24]
-        candidate = f"{TEMP_PLUGIN_PREFIX}{digest}"
-        if candidate not in reserved_plugins:
-            return candidate
-        salt += 1
-
-
-def consolidate_archiveresults_per_plugin(apps, schema_editor):
-    """Temporarily make plugin names unique without deleting hook history.
-
-    This migration shipped with a short-lived one-row-per-plugin model, while
-    0054 restores the durable (snapshot, plugin, hook_name) identity. Fresh
-    upgrades still traverse both published migrations, so deleting duplicate
-    rows here would lose history before 0054 gets a chance to restore the
-    correct constraint. Rename only the non-canonical rows and stash their
-    original plugin/output_json values for 0054 to restore verbatim.
-    """
-    ArchiveResult = apps.get_model("core", "ArchiveResult")
-    duplicate_groups = ArchiveResult.objects.values("snapshot_id", "plugin").annotate(count=models.Count("id")).filter(count__gt=1)
-
-    for group in duplicate_groups.iterator(chunk_size=200):
-        rows = list(
-            ArchiveResult.objects.filter(
-                snapshot_id=group["snapshot_id"],
-                plugin=group["plugin"],
-            ).order_by("created_at", "id"),
-        )
-        winner = max(
-            rows,
-            key=lambda row: (
-                bool(row.output_files),
-                int(row.output_size or 0),
-                row.modified_at,
-                str(row.id),
-            ),
-        )
-        reserved_plugins = set(
-            ArchiveResult.objects.filter(snapshot_id=group["snapshot_id"]).values_list("plugin", flat=True),
-        )
-        for row in rows:
-            if row.id == winner.id:
-                continue
-            temporary_plugin = _temporary_plugin_name(row.id, reserved_plugins)
-            reserved_plugins.add(temporary_plugin)
-            row.output_json = {
-                STASH_KEY: {
-                    "plugin": row.plugin,
-                    "output_json": row.output_json,
-                },
-            }
-            row.plugin = temporary_plugin
-            row.save(update_fields=["plugin", "output_json"])
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("core", "0051_postgres_url_pattern_ops_index"),
-    ]
-
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="archiveresult",
-            name="unique_archiveresult_per_snapshot_hook",
-        ),
-        migrations.RunPython(
-            consolidate_archiveresults_per_plugin,
-            reverse_code=migrations.RunPython.noop,
-        ),
-        migrations.AddConstraint(
-            model_name="archiveresult",
-            constraint=models.UniqueConstraint(
-                fields=("snapshot", "plugin"),
-                name="unique_archiveresult_per_snapshot_plugin",
-            ),
-        ),
-    ]

archivebox/core/migrations/0053_alter_archiveresult_options.py

@@ -5,7 +5,7 @@ from django.db import migrations
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("core", "0052_unique_archiveresult_per_snapshot_plugin"),
+        ("core", "0051_postgres_url_pattern_ops_index"),
     ]
 
     operations = [

archivebox/core/migrations/0054_restore_archiveresult_hook_identity.py

@@ -1,90 +0,0 @@
-import hashlib
-
-from django.db import migrations, models
-
-
-STASH_KEY = "__archivebox_0052_original_row"
-TEMP_PLUGIN_PREFIX = "__abx52_"
-
-
-def _temporary_plugin_name(row_id, reserved_plugins):
-    salt = 0
-    while True:
-        digest = hashlib.sha256(f"{row_id}:{salt}".encode()).hexdigest()[:24]
-        candidate = f"{TEMP_PLUGIN_PREFIX}{digest}"
-        if candidate not in reserved_plugins:
-            return candidate
-        salt += 1
-
-
-def stash_archiveresult_hook_rows(apps, schema_editor):
-    """Make plugin names temporarily unique when reversing to 0053."""
-    ArchiveResult = apps.get_model("core", "ArchiveResult")
-    duplicate_groups = ArchiveResult.objects.values("snapshot_id", "plugin").annotate(count=models.Count("id")).filter(count__gt=1)
-    for group in duplicate_groups.iterator(chunk_size=200):
-        rows = list(
-            ArchiveResult.objects.filter(snapshot_id=group["snapshot_id"], plugin=group["plugin"]).order_by("created_at", "id"),
-        )
-        winner = max(
-            rows,
-            key=lambda row: (
-                bool(row.output_files),
-                int(row.output_size or 0),
-                row.modified_at,
-                str(row.id),
-            ),
-        )
-        reserved_plugins = set(
-            ArchiveResult.objects.filter(snapshot_id=group["snapshot_id"]).values_list("plugin", flat=True),
-        )
-        for row in rows:
-            if row.id == winner.id:
-                continue
-            temporary_plugin = _temporary_plugin_name(row.id, reserved_plugins)
-            reserved_plugins.add(temporary_plugin)
-            row.output_json = {
-                STASH_KEY: {
-                    "plugin": row.plugin,
-                    "output_json": row.output_json,
-                },
-            }
-            row.plugin = temporary_plugin
-            row.save(update_fields=["plugin", "output_json"])
-
-
-def restore_archiveresult_hook_rows(apps, schema_editor):
-    """Undo 0052's temporary plugin renames after its constraint is removed."""
-    ArchiveResult = apps.get_model("core", "ArchiveResult")
-    rows = ArchiveResult.objects.filter(plugin__startswith=TEMP_PLUGIN_PREFIX)
-    for row in rows.iterator(chunk_size=200):
-        stash = row.output_json
-        original = stash.get(STASH_KEY) if isinstance(stash, dict) else None
-        if not isinstance(original, dict) or "plugin" not in original:
-            continue
-        row.plugin = original["plugin"]
-        row.output_json = original.get("output_json")
-        row.save(update_fields=["plugin", "output_json"])
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("core", "0053_alter_archiveresult_options"),
-    ]
-
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="archiveresult",
-            name="unique_archiveresult_per_snapshot_plugin",
-        ),
-        migrations.RunPython(
-            restore_archiveresult_hook_rows,
-            reverse_code=stash_archiveresult_hook_rows,
-        ),
-        migrations.AddConstraint(
-            model_name="archiveresult",
-            constraint=models.UniqueConstraint(
-                fields=("snapshot", "plugin", "hook_name"),
-                name="unique_archiveresult_per_snapshot_hook",
-            ),
-        ),
-    ]

archivebox/tests/test_migration_archiveresult_hook_identity.py

@@ -1,95 +0,0 @@
-import pytest
-from django.db import IntegrityError, connection, transaction
-from django.db.migrations.executor import MigrationExecutor
-
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-
-def test_published_plugin_constraint_preserves_existing_hook_rows():
-    try:
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0051_postgres_url_pattern_ops_index")])
-        old_apps = executor.loader.project_state([("core", "0051_postgres_url_pattern_ops_index")]).apps
-        Crawl = old_apps.get_model("crawls", "Crawl")
-        Snapshot = old_apps.get_model("core", "Snapshot")
-        ArchiveResult = old_apps.get_model("core", "ArchiveResult")
-        crawl = Crawl.objects.create(urls="https://example.com")
-        snapshot = Snapshot.objects.create(url="https://example.com/history", crawl=crawl)
-        first = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="responses",
-            hook_name="browser-upload",
-            output_json={"source": "browser"},
-        )
-        second = ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="responses",
-            hook_name="server-capture",
-            output_json={"source": "server"},
-        )
-
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0054_restore_archiveresult_hook_identity")])
-        new_apps = executor.loader.project_state([("core", "0054_restore_archiveresult_hook_identity")]).apps
-        ArchiveResult = new_apps.get_model("core", "ArchiveResult")
-        rows = list(
-            ArchiveResult.objects.filter(snapshot_id=snapshot.id, plugin="responses")
-            .order_by("hook_name")
-            .values_list("id", "hook_name", "output_json"),
-        )
-
-        assert rows == [
-            (first.id, "browser-upload", {"source": "browser"}),
-            (second.id, "server-capture", {"source": "server"}),
-        ]
-
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0053_alter_archiveresult_options")])
-        temporary_apps = executor.loader.project_state([("core", "0053_alter_archiveresult_options")]).apps
-        ArchiveResult = temporary_apps.get_model("core", "ArchiveResult")
-        temporary_rows = list(
-            ArchiveResult.objects.filter(snapshot_id=snapshot.id).values_list("plugin", "output_json"),
-        )
-        assert len(temporary_rows) == 2
-        assert len({plugin for plugin, _output_json in temporary_rows}) == 2
-
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0054_restore_archiveresult_hook_identity")])
-        restored_apps = executor.loader.project_state([("core", "0054_restore_archiveresult_hook_identity")]).apps
-        ArchiveResult = restored_apps.get_model("core", "ArchiveResult")
-        assert (
-            list(
-                ArchiveResult.objects.filter(snapshot_id=snapshot.id, plugin="responses")
-                .order_by("hook_name")
-                .values_list("id", "hook_name", "output_json"),
-            )
-            == rows
-        )
-    finally:
-        MigrationExecutor(connection).migrate([("core", "0054_restore_archiveresult_hook_identity")])
-
-
-def test_migration_restores_distinct_hook_rows_after_published_plugin_constraint():
-    try:
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0052_unique_archiveresult_per_snapshot_plugin")])
-        old_apps = executor.loader.project_state([("core", "0052_unique_archiveresult_per_snapshot_plugin")]).apps
-        Crawl = old_apps.get_model("crawls", "Crawl")
-        Snapshot = old_apps.get_model("core", "Snapshot")
-        ArchiveResult = old_apps.get_model("core", "ArchiveResult")
-        crawl = Crawl.objects.create(urls="https://example.com")
-        snapshot = Snapshot.objects.create(url="https://example.com/migration", crawl=crawl)
-        ArchiveResult.objects.create(snapshot=snapshot, plugin="responses", hook_name="browser-upload")
-
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0054_restore_archiveresult_hook_identity")])
-        new_apps = executor.loader.project_state([("core", "0054_restore_archiveresult_hook_identity")]).apps
-        ArchiveResult = new_apps.get_model("core", "ArchiveResult")
-
-        ArchiveResult.objects.create(snapshot_id=snapshot.id, plugin="responses", hook_name="server-capture")
-        assert ArchiveResult.objects.filter(snapshot_id=snapshot.id, plugin="responses").count() == 2
-        with pytest.raises(IntegrityError), transaction.atomic():
-            ArchiveResult.objects.create(snapshot_id=snapshot.id, plugin="responses", hook_name="browser-upload")
-    finally:
-        MigrationExecutor(connection).migrate([("core", "0054_restore_archiveresult_hook_identity")])

docs/codeblocks.toml

@@ -74,8 +74,8 @@ version = 2
 # docs/ArchiveBox-Architecture-Diagrams.md
 "7442dd337fd0a1dd-1" = "illustration"
 "9d6f8ac3043ff691-1" = "illustration"
-"8c8802244e05cfb7-1" = "illustration"
-"d0eb428affe5bb44-1" = "illustration"
+"9529bb7bc396b6b0-1" = "illustration"
+"66018350af060c17-1" = "illustration"
 "7d1d068e1e754a75-1" = "illustration"
 
 # docs/Changelog.md

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc381",
+  "version": "0.9.35rc383",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc381"
+version = "0.9.35rc383"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -345,7 +345,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.35rc381"
+current_version = "v0.9.35rc383"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -120,7 +120,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.35rc381"
+version = "0.9.35rc383"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl" },