mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
6a8062090b
Helper reduction: - Move the Django DATABASES/SQLITE_CONNECTION_OPTIONS assembly into core/settings.py, dropping get_database_settings() and get_sqlite_connection_options() from misc/db.py. - Inline the single-use migration_table_columns() into its one migration. Review fixes: - search: match only scalar JSON *values* on postgres (jsonb_path_query over '$.**' scalar leaves), mirroring SQLite json_tree.atom so config keys no longer match. - CharField clamp now also runs in SnapshotQuerySet.bulk_create (bulk paths bypass the pre_save signal); truncate_overlong_charfields is dual-use. - Restore reverse-migration parity on postgres: crawls/machine/api initial migrations drop their rebuilt tables on reverse via drop_models_on_postgres. - docs: give DATABASE_NAME its own section so the anchor resolves correctly. - CI: only install postgres binaries on the shard that runs the postgres test. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019YHSjZM6TstSAMN2PhgfUg
290 lines
14 KiB
Python
290 lines
14 KiB
Python
# Generated by hand on 2025-12-29
|
|
# Creates Machine, Binary, NetworkInterface, and Process tables using raw SQL
|
|
|
|
from django.db import migrations, models
|
|
import django.db.models.deletion
|
|
import django.utils.timezone
|
|
from archivebox.uuid_compat import uuid7
|
|
|
|
|
|
def _run_sqlite_initial_ddl(apps, schema_editor):
|
|
# sqlite-only: raw DDL creates the real tables byte-for-byte as historically
|
|
# shipped. On other vendors the tables are built from migration state by
|
|
# _pg_sync_schema below (a fresh non-sqlite DB never holds legacy data here).
|
|
if schema_editor.connection.vendor != "sqlite":
|
|
return
|
|
migrations.RunSQL(
|
|
sql=_SQLITE_INITIAL_SQL,
|
|
reverse_sql=_SQLITE_INITIAL_REVERSE_SQL,
|
|
).database_forwards("machine", schema_editor, None, None)
|
|
|
|
|
|
def _reverse_sqlite_initial_ddl(apps, schema_editor):
|
|
if schema_editor.connection.vendor != "sqlite":
|
|
return
|
|
migrations.RunSQL(
|
|
sql=_SQLITE_INITIAL_SQL,
|
|
reverse_sql=_SQLITE_INITIAL_REVERSE_SQL,
|
|
).database_backwards("machine", schema_editor, None, None)
|
|
|
|
|
|
def _pg_sync_schema(apps, schema_editor):
|
|
from archivebox.misc.db import rebuild_models_from_migration_state
|
|
|
|
rebuild_models_from_migration_state(
|
|
apps,
|
|
schema_editor,
|
|
"machine",
|
|
["Machine", "NetworkInterface", "Binary"],
|
|
)
|
|
|
|
|
|
def _pg_drop_schema(apps, schema_editor):
|
|
from archivebox.misc.db import drop_models_on_postgres
|
|
|
|
drop_models_on_postgres(apps, schema_editor, "machine", ["Binary", "NetworkInterface", "Machine"])
|
|
|
|
|
|
_SQLITE_INITIAL_SQL = """
|
|
-- Create machine_machine table
|
|
CREATE TABLE IF NOT EXISTS machine_machine (
|
|
id TEXT PRIMARY KEY NOT NULL,
|
|
created_at DATETIME NOT NULL,
|
|
modified_at DATETIME NOT NULL,
|
|
num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
|
|
num_uses_failed INTEGER NOT NULL DEFAULT 0,
|
|
|
|
guid VARCHAR(64) NOT NULL UNIQUE,
|
|
hostname VARCHAR(63) NOT NULL,
|
|
hw_in_docker BOOLEAN NOT NULL DEFAULT 0,
|
|
hw_in_vm BOOLEAN NOT NULL DEFAULT 0,
|
|
hw_manufacturer VARCHAR(63) NOT NULL,
|
|
hw_product VARCHAR(63) NOT NULL,
|
|
hw_uuid VARCHAR(255) NOT NULL,
|
|
|
|
os_arch VARCHAR(15) NOT NULL,
|
|
os_family VARCHAR(15) NOT NULL,
|
|
os_platform VARCHAR(63) NOT NULL,
|
|
os_release VARCHAR(63) NOT NULL,
|
|
os_kernel VARCHAR(255) NOT NULL,
|
|
|
|
stats TEXT,
|
|
config TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS machine_machine_guid_idx ON machine_machine(guid);
|
|
|
|
-- Create machine_networkinterface table
|
|
CREATE TABLE IF NOT EXISTS machine_networkinterface (
|
|
id TEXT PRIMARY KEY NOT NULL,
|
|
created_at DATETIME NOT NULL,
|
|
modified_at DATETIME NOT NULL,
|
|
num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
|
|
num_uses_failed INTEGER NOT NULL DEFAULT 0,
|
|
|
|
machine_id TEXT NOT NULL,
|
|
iface VARCHAR(15) NOT NULL,
|
|
ip_public VARCHAR(39) NOT NULL,
|
|
ip_local VARCHAR(39) NOT NULL,
|
|
mac_address VARCHAR(17) NOT NULL,
|
|
dns_server VARCHAR(39) NOT NULL,
|
|
hostname VARCHAR(256) NOT NULL,
|
|
isp VARCHAR(256) NOT NULL,
|
|
city VARCHAR(100) NOT NULL,
|
|
region VARCHAR(100) NOT NULL,
|
|
country VARCHAR(100) NOT NULL,
|
|
|
|
FOREIGN KEY (machine_id) REFERENCES machine_machine(id) ON DELETE CASCADE
|
|
);
|
|
CREATE INDEX IF NOT EXISTS machine_networkinterface_machine_id_idx ON machine_networkinterface(machine_id);
|
|
|
|
-- Create machine_binary table
|
|
CREATE TABLE IF NOT EXISTS machine_binary (
|
|
id TEXT PRIMARY KEY NOT NULL,
|
|
created_at DATETIME NOT NULL,
|
|
modified_at DATETIME NOT NULL,
|
|
num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
|
|
num_uses_failed INTEGER NOT NULL DEFAULT 0,
|
|
|
|
machine_id TEXT NOT NULL,
|
|
name VARCHAR(63) NOT NULL,
|
|
binproviders VARCHAR(127) NOT NULL DEFAULT 'env',
|
|
overrides TEXT NOT NULL DEFAULT '{}',
|
|
|
|
binprovider VARCHAR(31) NOT NULL DEFAULT '',
|
|
abspath VARCHAR(255) NOT NULL DEFAULT '',
|
|
version VARCHAR(32) NOT NULL DEFAULT '',
|
|
sha256 VARCHAR(64) NOT NULL DEFAULT '',
|
|
|
|
status VARCHAR(16) NOT NULL DEFAULT 'queued',
|
|
retry_at DATETIME,
|
|
output_dir VARCHAR(255) NOT NULL DEFAULT '',
|
|
|
|
FOREIGN KEY (machine_id) REFERENCES machine_machine(id) ON DELETE CASCADE,
|
|
UNIQUE(machine_id, name, abspath, version, sha256)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS machine_binary_machine_id_idx ON machine_binary(machine_id);
|
|
CREATE INDEX IF NOT EXISTS machine_binary_name_idx ON machine_binary(name);
|
|
CREATE INDEX IF NOT EXISTS machine_binary_status_idx ON machine_binary(status);
|
|
CREATE INDEX IF NOT EXISTS machine_binary_retry_at_idx ON machine_binary(retry_at);
|
|
|
|
"""
|
|
|
|
|
|
_SQLITE_INITIAL_REVERSE_SQL = """
|
|
DROP TABLE IF EXISTS machine_binary;
|
|
DROP TABLE IF EXISTS machine_networkinterface;
|
|
DROP TABLE IF EXISTS machine_machine;
|
|
"""
|
|
|
|
|
|
class Migration(migrations.Migration):
|
|
initial = True
|
|
|
|
dependencies = []
|
|
|
|
operations = [
|
|
migrations.SeparateDatabaseAndState(
|
|
database_operations=[
|
|
migrations.RunPython(_run_sqlite_initial_ddl, _reverse_sqlite_initial_ddl),
|
|
],
|
|
state_operations=[
|
|
migrations.CreateModel(
|
|
name="Machine",
|
|
fields=[
|
|
("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
|
|
("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
|
("modified_at", models.DateTimeField(auto_now=True)),
|
|
("guid", models.CharField(default=None, editable=False, max_length=64, unique=True)),
|
|
("hostname", models.CharField(default=None, max_length=63)),
|
|
("hw_in_docker", models.BooleanField(default=False)),
|
|
("hw_in_vm", models.BooleanField(default=False)),
|
|
("hw_manufacturer", models.CharField(default=None, max_length=63)),
|
|
("hw_product", models.CharField(default=None, max_length=63)),
|
|
("hw_uuid", models.CharField(default=None, max_length=255)),
|
|
("os_arch", models.CharField(default=None, max_length=15)),
|
|
("os_family", models.CharField(default=None, max_length=15)),
|
|
("os_platform", models.CharField(default=None, max_length=63)),
|
|
("os_release", models.CharField(default=None, max_length=63)),
|
|
("os_kernel", models.CharField(default=None, max_length=255)),
|
|
("stats", models.JSONField(blank=True, default=dict, null=True)),
|
|
(
|
|
"config",
|
|
models.JSONField(
|
|
blank=True,
|
|
default=dict,
|
|
help_text="Machine-specific config overrides (e.g., resolved binary paths like WGET_BINARY)",
|
|
null=True,
|
|
),
|
|
),
|
|
("num_uses_succeeded", models.PositiveIntegerField(default=0)),
|
|
("num_uses_failed", models.PositiveIntegerField(default=0)),
|
|
],
|
|
options={
|
|
"app_label": "machine",
|
|
},
|
|
),
|
|
migrations.CreateModel(
|
|
name="NetworkInterface",
|
|
fields=[
|
|
("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
|
|
("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
|
("modified_at", models.DateTimeField(auto_now=True)),
|
|
("mac_address", models.CharField(default=None, editable=False, max_length=17)),
|
|
("ip_public", models.GenericIPAddressField(default=None, editable=False)),
|
|
("ip_local", models.GenericIPAddressField(default=None, editable=False)),
|
|
("dns_server", models.GenericIPAddressField(default=None, editable=False)),
|
|
("hostname", models.CharField(default=None, max_length=63)),
|
|
("iface", models.CharField(default=None, max_length=15)),
|
|
("isp", models.CharField(default=None, max_length=63)),
|
|
("city", models.CharField(default=None, max_length=63)),
|
|
("region", models.CharField(default=None, max_length=63)),
|
|
("country", models.CharField(default=None, max_length=63)),
|
|
("machine", models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to="machine.machine")),
|
|
("num_uses_succeeded", models.PositiveIntegerField(default=0)),
|
|
("num_uses_failed", models.PositiveIntegerField(default=0)),
|
|
],
|
|
options={
|
|
"unique_together": {("machine", "ip_public", "ip_local", "mac_address", "dns_server")},
|
|
"app_label": "machine",
|
|
},
|
|
),
|
|
migrations.CreateModel(
|
|
name="Binary",
|
|
fields=[
|
|
("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
|
|
("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
|
|
("modified_at", models.DateTimeField(auto_now=True)),
|
|
("name", models.CharField(blank=True, db_index=True, default="", max_length=63)),
|
|
(
|
|
"binproviders",
|
|
models.CharField(
|
|
blank=True,
|
|
default="env",
|
|
help_text="Comma-separated list of allowed providers: apt,brew,pip,npm,env",
|
|
max_length=127,
|
|
),
|
|
),
|
|
(
|
|
"overrides",
|
|
models.JSONField(
|
|
blank=True,
|
|
default=dict,
|
|
help_text="Provider-specific overrides: {'apt': {'install_args': ['pkg']}, ...}",
|
|
),
|
|
),
|
|
(
|
|
"binprovider",
|
|
models.CharField(
|
|
blank=True,
|
|
default="",
|
|
help_text="Provider that successfully installed this binary",
|
|
max_length=31,
|
|
),
|
|
),
|
|
("abspath", models.CharField(blank=True, default="", max_length=255)),
|
|
("version", models.CharField(blank=True, default="", max_length=32)),
|
|
("sha256", models.CharField(blank=True, default="", max_length=64)),
|
|
(
|
|
"status",
|
|
models.CharField(
|
|
choices=[("queued", "Queued"), ("started", "Started"), ("succeeded", "Succeeded"), ("failed", "Failed")],
|
|
db_index=True,
|
|
default="queued",
|
|
max_length=16,
|
|
),
|
|
),
|
|
(
|
|
"retry_at",
|
|
models.DateTimeField(
|
|
blank=True,
|
|
db_index=True,
|
|
default=django.utils.timezone.now,
|
|
help_text="When to retry this binary installation",
|
|
null=True,
|
|
),
|
|
),
|
|
(
|
|
"output_dir",
|
|
models.CharField(
|
|
blank=True,
|
|
default="",
|
|
help_text="Directory where installation hook logs are stored",
|
|
max_length=255,
|
|
),
|
|
),
|
|
("machine", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="machine.machine")),
|
|
("num_uses_succeeded", models.PositiveIntegerField(default=0)),
|
|
("num_uses_failed", models.PositiveIntegerField(default=0)),
|
|
],
|
|
options={
|
|
"verbose_name": "Binary",
|
|
"verbose_name_plural": "Binaries",
|
|
"unique_together": {("machine", "name", "abspath", "version", "sha256")},
|
|
"app_label": "machine",
|
|
},
|
|
),
|
|
],
|
|
),
|
|
migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
|
|
]
|