diff --git a/archivebox/api/migrations/0001_initial.py b/archivebox/api/migrations/0001_initial.py index 99cdeef9..b3a2aab3 100644 --- a/archivebox/api/migrations/0001_initial.py +++ b/archivebox/api/migrations/0001_initial.py @@ -72,7 +72,10 @@ def _run_sqlite_only_sql(apps, schema_editor): if schema_editor.connection.vendor != "sqlite": return migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards( - "api", schema_editor, None, None, + "api", + schema_editor, + None, + None, ) @@ -80,7 +83,10 @@ def _run_sqlite_only_sql_reverse(apps, schema_editor): if schema_editor.connection.vendor != "sqlite": return migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards( - "api", schema_editor, None, None, + "api", + schema_editor, + None, + None, ) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index b278ee2a..80619bcd 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -14,6 +14,10 @@ import archivebox from archivebox.config.constants import CONSTANTS from archivebox.config.common import get_config from archivebox.core.routes_util import get_api_base_url, get_admin_base_url, get_base_url, normalize_base_url + +# All sqlite-vs-postgres connection logic lives in archivebox.misc.db; +# DATABASE_ENGINE config selects the backend (sqlite by default). +from archivebox.misc.db import get_database_settings, get_sqlite_connection_options from .settings_logging import SETTINGS_LOGGING @@ -212,10 +216,6 @@ TEMPLATES = [ ### External Service Settings ################################################################################ -# All sqlite-vs-postgres connection logic lives in archivebox.misc.db; -# DATABASE_ENGINE config selects the backend (sqlite by default). -from archivebox.misc.db import get_database_settings, get_sqlite_connection_options - DATABASE_NAME = CONFIG.DATABASE_NAME SQLITE_JOURNAL_MODE = CONFIG.SQLITE_JOURNAL_MODE SQLITE_MMAP_SIZE = CONFIG.SQLITE_MMAP_SIZE diff --git a/archivebox/crawls/migrations/0001_initial.py b/archivebox/crawls/migrations/0001_initial.py index 6b611a3b..6a079d30 100644 --- a/archivebox/crawls/migrations/0001_initial.py +++ b/archivebox/crawls/migrations/0001_initial.py @@ -77,7 +77,10 @@ def _run_sqlite_only_sql(apps, schema_editor): if schema_editor.connection.vendor != "sqlite": return migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards( - "crawls", schema_editor, None, None, + "crawls", + schema_editor, + None, + None, ) @@ -85,7 +88,10 @@ def _run_sqlite_only_sql_reverse(apps, schema_editor): if schema_editor.connection.vendor != "sqlite": return migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards( - "crawls", schema_editor, None, None, + "crawls", + schema_editor, + None, + None, ) diff --git a/archivebox/machine/migrations/0001_initial.py b/archivebox/machine/migrations/0001_initial.py index 9425f761..cea929bb 100644 --- a/archivebox/machine/migrations/0001_initial.py +++ b/archivebox/machine/migrations/0001_initial.py @@ -32,7 +32,10 @@ def _pg_sync_schema(apps, schema_editor): from archivebox.misc.db import rebuild_models_from_migration_state rebuild_models_from_migration_state( - apps, schema_editor, "machine", ["Machine", "NetworkInterface", "Binary"] + apps, + schema_editor, + "machine", + ["Machine", "NetworkInterface", "Binary"], ) diff --git a/bin/benchmark_db_backends.py b/bin/benchmark_db_backends.py old mode 100644 new mode 100755 index 5a344f21..09418832 --- a/bin/benchmark_db_backends.py +++ b/bin/benchmark_db_backends.py @@ -129,7 +129,9 @@ def run_benchmarks(rows: int) -> dict[str, float]: benchmarks = { "exact_count": lambda: Snapshot.objects.count(), "approximate_row_counts": lambda: approximate_row_counts(connection), - "admin_list_page": lambda: list(Snapshot.objects.order_by("-bookmarked_at").values("id", "url", "title", "status", "bookmarked_at")[:40]), + "admin_list_page": lambda: list( + Snapshot.objects.order_by("-bookmarked_at").values("id", "url", "title", "status", "bookmarked_at")[:40], + ), "admin_list_page_offset_10k": lambda: list(Snapshot.objects.order_by("-bookmarked_at").values("id", "url", "title")[10_000:10_040]), "snapshot_detail_by_url": lambda: list(Snapshot.objects.filter(fragmentless_q(target_url))[:10]), "snapshot_detail_archiveresults": lambda: list( @@ -137,7 +139,9 @@ def run_benchmarks(rows: int) -> dict[str, float]: ), "url_prefix_search": lambda: list(iter_url_prefix_search_ids("https://site500.example.org/", Snapshot.objects.all())), "worker_queue_scan": lambda: list( - Snapshot.objects.filter(status="queued", retry_at__lte=now).order_by("retry_at", "created_at").values_list("id", flat=True)[:100], + Snapshot.objects.filter(status="queued", retry_at__lte=now) + .order_by("retry_at", "created_at") + .values_list("id", flat=True)[:100], ), "status_facet_counts": lambda: dict(Snapshot.objects.values_list("status").annotate(n=Count("id")).values_list("status", "n")), "tag_join_filter": lambda: list(Snapshot.objects.filter(title__icontains="page 4242").values("id")[:20]), @@ -152,7 +156,10 @@ def run_benchmarks(rows: int) -> dict[str, float]: snapshot = Snapshot.objects.filter(status="queued").order_by("retry_at").first() if snapshot is None: return 0 - return Snapshot.objects.filter(pk=snapshot.pk, retry_at=snapshot.retry_at).update(retry_at=now + timedelta(seconds=60), modified_at=now) + return Snapshot.objects.filter(pk=snapshot.pk, retry_at=snapshot.retry_at).update( + retry_at=now + timedelta(seconds=60), + modified_at=now, + ) median_ms, _ = timed(claim_one) results["worker_cas_claim"] = round(median_ms, 2)