mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 02:56:11 +05:00
Add DATABASE_ENGINE=postgres (plus DATABASE_HOST/PORT/USER/PASSWORD/NAME) config and centralize all sqlite-vs-postgres branching in archivebox.misc.db: - get_database_settings() builds DATABASES for either backend; the sqlite path is unchanged (custom lock-retry backend, same PRAGMAs). - database_exists()/ensure_database_ready() replace index.sqlite3 file checks; init auto-creates the postgres database when missing. - approximate_row_counts() serves admin index counts from sqlite_stat1 or pg_class.reltuples; missing-table detection covers both vendors. - rebuild_models_from_migration_state() lets historical sqlite-only raw SQL migrations resync postgres schema from Django migration state at every divergence point (postgres can never hold legacy data, so affected tables are empty when these run). All raw-DDL and PRAGMA migrations are now vendor-gated with sqlite behavior byte-for-byte unchanged. - A pre_save clamp truncates CharField values to max_length: sqlite never enforced varchar(n) but postgres does (e.g. long crawl labels). - Collation-sensitive URL range scans branch to escaped LIKE on postgres (with a text_pattern_ops index) since linguistic collations break bytewise range tricks; the crawl-config JSON search wave gets a jsonb-text implementation. Verified on real PostgreSQL 16: fresh init applies the entire migration graph, schema matches models exactly (column-level parity check + makemigrations --check), and add/run/list/search/status/remove all work end-to-end. New test_postgres_backend.py suite boots a real throwaway postgres cluster (initdb + pg_ctl); CI workflows install postgres server binaries. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019YHSjZM6TstSAMN2PhgfUg
100 lines
4.6 KiB
Python
100 lines
4.6 KiB
Python
# Generated by hand on 2026-01-01
|
|
# Converges machine app for 0.8.6rc0 → 0.9.x migration path
|
|
# Drops old Binary table and ensures Binary table exists
|
|
|
|
from django.db import migrations, connection
|
|
|
|
|
|
def converge_binary_table(apps, schema_editor):
|
|
"""
|
|
Drop machine_installedbinary if it exists (0.8.6rc0 path).
|
|
Create machine_binary if it doesn't exist (needed by Process model).
|
|
"""
|
|
# sqlite-only legacy convergence: on a fresh install machine_binary already
|
|
# exists with the final column set (created by 0001), so every branch below
|
|
# is a no-op schema-wise. A non-sqlite DB never carries the 0.8.x legacy
|
|
# shapes this repairs, and its Binary table already matches migration state.
|
|
if schema_editor.connection.vendor != "sqlite":
|
|
return
|
|
|
|
cursor = connection.cursor()
|
|
|
|
# Check what tables exist
|
|
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('machine_installedbinary', 'machine_binary')")
|
|
existing_tables = {row[0] for row in cursor.fetchall()}
|
|
|
|
# Drop old Binary table if it exists (0.8.6rc0 path)
|
|
if "machine_installedbinary" in existing_tables:
|
|
print(" - Removing old machine_installedbinary table...")
|
|
cursor.execute("DROP TABLE IF EXISTS machine_installedbinary")
|
|
|
|
# Create Binary table if it doesn't exist.
|
|
# This handles the case where 0.8.6rc0's 0001_initial didn't create it.
|
|
if "machine_binary" not in existing_tables:
|
|
print(" - Creating machine_binary table...")
|
|
cursor.execute("""
|
|
CREATE TABLE machine_binary (
|
|
id TEXT PRIMARY KEY NOT NULL,
|
|
created_at DATETIME NOT NULL,
|
|
modified_at DATETIME NOT NULL,
|
|
num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
|
|
num_uses_failed INTEGER NOT NULL DEFAULT 0,
|
|
machine_id TEXT NOT NULL REFERENCES machine_machine(id) ON DELETE CASCADE,
|
|
name VARCHAR(63) NOT NULL,
|
|
binproviders VARCHAR(255) NOT NULL DEFAULT 'env',
|
|
overrides TEXT NOT NULL DEFAULT '{}',
|
|
binprovider VARCHAR(63) NOT NULL DEFAULT 'env',
|
|
abspath VARCHAR(255) NOT NULL,
|
|
version VARCHAR(128) NOT NULL,
|
|
sha256 VARCHAR(64) NOT NULL DEFAULT '',
|
|
status VARCHAR(16) NOT NULL DEFAULT 'succeeded',
|
|
retry_at DATETIME NULL,
|
|
output_dir VARCHAR(255) NOT NULL DEFAULT ''
|
|
)
|
|
""")
|
|
|
|
# Create indexes
|
|
cursor.execute("CREATE INDEX machine_binary_machine_id_idx ON machine_binary(machine_id)")
|
|
cursor.execute("CREATE INDEX machine_binary_name_idx ON machine_binary(name)")
|
|
cursor.execute("CREATE INDEX machine_binary_abspath_idx ON machine_binary(abspath)")
|
|
|
|
print(" ✓ machine_binary table ready")
|
|
else:
|
|
print(" - Converging existing machine_binary table...")
|
|
cursor.execute("PRAGMA table_info(machine_binary)")
|
|
binary_cols = {row[1] for row in cursor.fetchall()}
|
|
|
|
# Old 0.8.x data dirs already have machine_binary, but with the
|
|
# pre-abxpkg shape. Converge it here before later migrations and
|
|
# runtime code expect Binary.binproviders / Binary.status to exist.
|
|
if "binproviders" not in binary_cols:
|
|
cursor.execute("ALTER TABLE machine_binary ADD COLUMN binproviders VARCHAR(255) NOT NULL DEFAULT 'env'")
|
|
if "overrides" not in binary_cols:
|
|
cursor.execute("ALTER TABLE machine_binary ADD COLUMN overrides TEXT NOT NULL DEFAULT '{}'")
|
|
if "status" not in binary_cols:
|
|
cursor.execute("ALTER TABLE machine_binary ADD COLUMN status VARCHAR(16) NOT NULL DEFAULT 'installed'")
|
|
if "retry_at" not in binary_cols:
|
|
cursor.execute("ALTER TABLE machine_binary ADD COLUMN retry_at DATETIME NULL")
|
|
if "output_dir" not in binary_cols:
|
|
cursor.execute("ALTER TABLE machine_binary ADD COLUMN output_dir VARCHAR(255) NOT NULL DEFAULT ''")
|
|
|
|
cursor.execute(
|
|
"UPDATE machine_binary SET binproviders = COALESCE(NULLIF(binproviders, ''), COALESCE(NULLIF(binprovider, ''), 'env'))",
|
|
)
|
|
cursor.execute("UPDATE machine_binary SET overrides = COALESCE(NULLIF(overrides, ''), '{}')")
|
|
cursor.execute("UPDATE machine_binary SET status = COALESCE(NULLIF(status, ''), 'installed')")
|
|
print(" ✓ machine_binary table ready")
|
|
|
|
|
|
class Migration(migrations.Migration):
|
|
dependencies = [
|
|
("machine", "0001_initial"),
|
|
]
|
|
|
|
operations = [
|
|
migrations.RunPython(
|
|
converge_binary_table,
|
|
reverse_code=migrations.RunPython.noop,
|
|
),
|
|
]
|