diff --git a/archivebox/core/migrations/0052_unique_archiveresult_per_snapshot_plugin.py b/archivebox/core/migrations/0052_unique_archiveresult_per_snapshot_plugin.py deleted file mode 100644 index fad6b4de..00000000 --- a/archivebox/core/migrations/0052_unique_archiveresult_per_snapshot_plugin.py +++ /dev/null @@ -1,89 +0,0 @@ -import hashlib - -from django.db import migrations, models - - -STASH_KEY = "__archivebox_0052_original_row" -TEMP_PLUGIN_PREFIX = "__abx52_" - - -def _temporary_plugin_name(row_id, reserved_plugins): - """Return a deterministic 32-character plugin name unused by this snapshot.""" - salt = 0 - while True: - digest = hashlib.sha256(f"{row_id}:{salt}".encode()).hexdigest()[:24] - candidate = f"{TEMP_PLUGIN_PREFIX}{digest}" - if candidate not in reserved_plugins: - return candidate - salt += 1 - - -def consolidate_archiveresults_per_plugin(apps, schema_editor): - """Temporarily make plugin names unique without deleting hook history. - - This migration shipped with a short-lived one-row-per-plugin model, while - 0054 restores the durable (snapshot, plugin, hook_name) identity. Fresh - upgrades still traverse both published migrations, so deleting duplicate - rows here would lose history before 0054 gets a chance to restore the - correct constraint. Rename only the non-canonical rows and stash their - original plugin/output_json values for 0054 to restore verbatim. - """ - ArchiveResult = apps.get_model("core", "ArchiveResult") - duplicate_groups = ArchiveResult.objects.values("snapshot_id", "plugin").annotate(count=models.Count("id")).filter(count__gt=1) - - for group in duplicate_groups.iterator(chunk_size=200): - rows = list( - ArchiveResult.objects.filter( - snapshot_id=group["snapshot_id"], - plugin=group["plugin"], - ).order_by("created_at", "id"), - ) - winner = max( - rows, - key=lambda row: ( - bool(row.output_files), - int(row.output_size or 0), - row.modified_at, - str(row.id), - ), - ) - reserved_plugins = set( - ArchiveResult.objects.filter(snapshot_id=group["snapshot_id"]).values_list("plugin", flat=True), - ) - for row in rows: - if row.id == winner.id: - continue - temporary_plugin = _temporary_plugin_name(row.id, reserved_plugins) - reserved_plugins.add(temporary_plugin) - row.output_json = { - STASH_KEY: { - "plugin": row.plugin, - "output_json": row.output_json, - }, - } - row.plugin = temporary_plugin - row.save(update_fields=["plugin", "output_json"]) - - -class Migration(migrations.Migration): - dependencies = [ - ("core", "0051_postgres_url_pattern_ops_index"), - ] - - operations = [ - migrations.RemoveConstraint( - model_name="archiveresult", - name="unique_archiveresult_per_snapshot_hook", - ), - migrations.RunPython( - consolidate_archiveresults_per_plugin, - reverse_code=migrations.RunPython.noop, - ), - migrations.AddConstraint( - model_name="archiveresult", - constraint=models.UniqueConstraint( - fields=("snapshot", "plugin"), - name="unique_archiveresult_per_snapshot_plugin", - ), - ), - ] diff --git a/archivebox/core/migrations/0053_alter_archiveresult_options.py b/archivebox/core/migrations/0053_alter_archiveresult_options.py index a8ed522a..422a3d76 100644 --- a/archivebox/core/migrations/0053_alter_archiveresult_options.py +++ b/archivebox/core/migrations/0053_alter_archiveresult_options.py @@ -5,7 +5,7 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("core", "0052_unique_archiveresult_per_snapshot_plugin"), + ("core", "0051_postgres_url_pattern_ops_index"), ] operations = [ diff --git a/archivebox/core/migrations/0054_restore_archiveresult_hook_identity.py b/archivebox/core/migrations/0054_restore_archiveresult_hook_identity.py deleted file mode 100644 index d19299ef..00000000 --- a/archivebox/core/migrations/0054_restore_archiveresult_hook_identity.py +++ /dev/null @@ -1,90 +0,0 @@ -import hashlib - -from django.db import migrations, models - - -STASH_KEY = "__archivebox_0052_original_row" -TEMP_PLUGIN_PREFIX = "__abx52_" - - -def _temporary_plugin_name(row_id, reserved_plugins): - salt = 0 - while True: - digest = hashlib.sha256(f"{row_id}:{salt}".encode()).hexdigest()[:24] - candidate = f"{TEMP_PLUGIN_PREFIX}{digest}" - if candidate not in reserved_plugins: - return candidate - salt += 1 - - -def stash_archiveresult_hook_rows(apps, schema_editor): - """Make plugin names temporarily unique when reversing to 0053.""" - ArchiveResult = apps.get_model("core", "ArchiveResult") - duplicate_groups = ArchiveResult.objects.values("snapshot_id", "plugin").annotate(count=models.Count("id")).filter(count__gt=1) - for group in duplicate_groups.iterator(chunk_size=200): - rows = list( - ArchiveResult.objects.filter(snapshot_id=group["snapshot_id"], plugin=group["plugin"]).order_by("created_at", "id"), - ) - winner = max( - rows, - key=lambda row: ( - bool(row.output_files), - int(row.output_size or 0), - row.modified_at, - str(row.id), - ), - ) - reserved_plugins = set( - ArchiveResult.objects.filter(snapshot_id=group["snapshot_id"]).values_list("plugin", flat=True), - ) - for row in rows: - if row.id == winner.id: - continue - temporary_plugin = _temporary_plugin_name(row.id, reserved_plugins) - reserved_plugins.add(temporary_plugin) - row.output_json = { - STASH_KEY: { - "plugin": row.plugin, - "output_json": row.output_json, - }, - } - row.plugin = temporary_plugin - row.save(update_fields=["plugin", "output_json"]) - - -def restore_archiveresult_hook_rows(apps, schema_editor): - """Undo 0052's temporary plugin renames after its constraint is removed.""" - ArchiveResult = apps.get_model("core", "ArchiveResult") - rows = ArchiveResult.objects.filter(plugin__startswith=TEMP_PLUGIN_PREFIX) - for row in rows.iterator(chunk_size=200): - stash = row.output_json - original = stash.get(STASH_KEY) if isinstance(stash, dict) else None - if not isinstance(original, dict) or "plugin" not in original: - continue - row.plugin = original["plugin"] - row.output_json = original.get("output_json") - row.save(update_fields=["plugin", "output_json"]) - - -class Migration(migrations.Migration): - dependencies = [ - ("core", "0053_alter_archiveresult_options"), - ] - - operations = [ - migrations.RemoveConstraint( - model_name="archiveresult", - name="unique_archiveresult_per_snapshot_plugin", - ), - migrations.RunPython( - restore_archiveresult_hook_rows, - reverse_code=stash_archiveresult_hook_rows, - ), - migrations.AddConstraint( - model_name="archiveresult", - constraint=models.UniqueConstraint( - fields=("snapshot", "plugin", "hook_name"), - name="unique_archiveresult_per_snapshot_hook", - ), - ), - ] diff --git a/archivebox/tests/test_migration_archiveresult_hook_identity.py b/archivebox/tests/test_migration_archiveresult_hook_identity.py deleted file mode 100644 index fa290489..00000000 --- a/archivebox/tests/test_migration_archiveresult_hook_identity.py +++ /dev/null @@ -1,95 +0,0 @@ -import pytest -from django.db import IntegrityError, connection, transaction -from django.db.migrations.executor import MigrationExecutor - - -pytestmark = pytest.mark.django_db(transaction=True) - - -def test_published_plugin_constraint_preserves_existing_hook_rows(): - try: - executor = MigrationExecutor(connection) - executor.migrate([("core", "0051_postgres_url_pattern_ops_index")]) - old_apps = executor.loader.project_state([("core", "0051_postgres_url_pattern_ops_index")]).apps - Crawl = old_apps.get_model("crawls", "Crawl") - Snapshot = old_apps.get_model("core", "Snapshot") - ArchiveResult = old_apps.get_model("core", "ArchiveResult") - crawl = Crawl.objects.create(urls="https://example.com") - snapshot = Snapshot.objects.create(url="https://example.com/history", crawl=crawl) - first = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="responses", - hook_name="browser-upload", - output_json={"source": "browser"}, - ) - second = ArchiveResult.objects.create( - snapshot=snapshot, - plugin="responses", - hook_name="server-capture", - output_json={"source": "server"}, - ) - - executor = MigrationExecutor(connection) - executor.migrate([("core", "0054_restore_archiveresult_hook_identity")]) - new_apps = executor.loader.project_state([("core", "0054_restore_archiveresult_hook_identity")]).apps - ArchiveResult = new_apps.get_model("core", "ArchiveResult") - rows = list( - ArchiveResult.objects.filter(snapshot_id=snapshot.id, plugin="responses") - .order_by("hook_name") - .values_list("id", "hook_name", "output_json"), - ) - - assert rows == [ - (first.id, "browser-upload", {"source": "browser"}), - (second.id, "server-capture", {"source": "server"}), - ] - - executor = MigrationExecutor(connection) - executor.migrate([("core", "0053_alter_archiveresult_options")]) - temporary_apps = executor.loader.project_state([("core", "0053_alter_archiveresult_options")]).apps - ArchiveResult = temporary_apps.get_model("core", "ArchiveResult") - temporary_rows = list( - ArchiveResult.objects.filter(snapshot_id=snapshot.id).values_list("plugin", "output_json"), - ) - assert len(temporary_rows) == 2 - assert len({plugin for plugin, _output_json in temporary_rows}) == 2 - - executor = MigrationExecutor(connection) - executor.migrate([("core", "0054_restore_archiveresult_hook_identity")]) - restored_apps = executor.loader.project_state([("core", "0054_restore_archiveresult_hook_identity")]).apps - ArchiveResult = restored_apps.get_model("core", "ArchiveResult") - assert ( - list( - ArchiveResult.objects.filter(snapshot_id=snapshot.id, plugin="responses") - .order_by("hook_name") - .values_list("id", "hook_name", "output_json"), - ) - == rows - ) - finally: - MigrationExecutor(connection).migrate([("core", "0054_restore_archiveresult_hook_identity")]) - - -def test_migration_restores_distinct_hook_rows_after_published_plugin_constraint(): - try: - executor = MigrationExecutor(connection) - executor.migrate([("core", "0052_unique_archiveresult_per_snapshot_plugin")]) - old_apps = executor.loader.project_state([("core", "0052_unique_archiveresult_per_snapshot_plugin")]).apps - Crawl = old_apps.get_model("crawls", "Crawl") - Snapshot = old_apps.get_model("core", "Snapshot") - ArchiveResult = old_apps.get_model("core", "ArchiveResult") - crawl = Crawl.objects.create(urls="https://example.com") - snapshot = Snapshot.objects.create(url="https://example.com/migration", crawl=crawl) - ArchiveResult.objects.create(snapshot=snapshot, plugin="responses", hook_name="browser-upload") - - executor = MigrationExecutor(connection) - executor.migrate([("core", "0054_restore_archiveresult_hook_identity")]) - new_apps = executor.loader.project_state([("core", "0054_restore_archiveresult_hook_identity")]).apps - ArchiveResult = new_apps.get_model("core", "ArchiveResult") - - ArchiveResult.objects.create(snapshot_id=snapshot.id, plugin="responses", hook_name="server-capture") - assert ArchiveResult.objects.filter(snapshot_id=snapshot.id, plugin="responses").count() == 2 - with pytest.raises(IntegrityError), transaction.atomic(): - ArchiveResult.objects.create(snapshot_id=snapshot.id, plugin="responses", hook_name="browser-upload") - finally: - MigrationExecutor(connection).migrate([("core", "0054_restore_archiveresult_hook_identity")]) diff --git a/docs/codeblocks.toml b/docs/codeblocks.toml index 172ab3b1..e06d1eef 100644 --- a/docs/codeblocks.toml +++ b/docs/codeblocks.toml @@ -74,8 +74,8 @@ version = 2 # docs/ArchiveBox-Architecture-Diagrams.md "7442dd337fd0a1dd-1" = "illustration" "9d6f8ac3043ff691-1" = "illustration" -"8c8802244e05cfb7-1" = "illustration" -"d0eb428affe5bb44-1" = "illustration" +"9529bb7bc396b6b0-1" = "illustration" +"66018350af060c17-1" = "illustration" "7d1d068e1e754a75-1" = "illustration" # docs/Changelog.md diff --git a/etc/package.json b/etc/package.json index aa119069..eaa23188 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.35rc381", + "version": "0.9.35rc383", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index e92bb3a5..629a3da2 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.35rc381" +version = "0.9.35rc383" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -345,7 +345,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations" [tool.bumpver] -current_version = "v0.9.35rc381" +current_version = "v0.9.35rc383" version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]" commit_message = "bump version {old_version} -> {new_version}" tag_message = "{new_version}" diff --git a/uv.lock b/uv.lock index 6a3ac870..54d2a9ee 100644 --- a/uv.lock +++ b/uv.lock @@ -120,7 +120,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.9.35rc381" +version = "0.9.35rc383" source = { editable = "." } dependencies = [ { name = "abx-dl" },