mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-09-14 11:06:13 +05:00
Add DATABASE_ENGINE=postgres (plus DATABASE_HOST/PORT/USER/PASSWORD/NAME) config and centralize all sqlite-vs-postgres branching in archivebox.misc.db: - get_database_settings() builds DATABASES for either backend; the sqlite path is unchanged (custom lock-retry backend, same PRAGMAs). - database_exists()/ensure_database_ready() replace index.sqlite3 file checks; init auto-creates the postgres database when missing. - approximate_row_counts() serves admin index counts from sqlite_stat1 or pg_class.reltuples; missing-table detection covers both vendors. - rebuild_models_from_migration_state() lets historical sqlite-only raw SQL migrations resync postgres schema from Django migration state at every divergence point (postgres can never hold legacy data, so affected tables are empty when these run). All raw-DDL and PRAGMA migrations are now vendor-gated with sqlite behavior byte-for-byte unchanged. - A pre_save clamp truncates CharField values to max_length: sqlite never enforced varchar(n) but postgres does (e.g. long crawl labels). - Collation-sensitive URL range scans branch to escaped LIKE on postgres (with a text_pattern_ops index) since linguistic collations break bytewise range tricks; the crawl-config JSON search wave gets a jsonb-text implementation. Verified on real PostgreSQL 16: fresh init applies the entire migration graph, schema matches models exactly (column-level parity check + makemigrations --check), and add/run/list/search/status/remove all work end-to-end. New test_postgres_backend.py suite boots a real throwaway postgres cluster (initdb + pg_ctl); CI workflows install postgres server binaries. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019YHSjZM6TstSAMN2PhgfUg
201 lines
7.7 KiB
Python
201 lines
7.7 KiB
Python
__package__ = "archivebox.core"
|
|
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from django.contrib import admin
|
|
from django.contrib.auth import REDIRECT_FIELD_NAME
|
|
from django.contrib.auth.decorators import login_not_required
|
|
from django.contrib.auth.views import LoginView
|
|
from django.db import DatabaseError, connection
|
|
from django.http import HttpResponseRedirect
|
|
from django.urls import reverse
|
|
from django.utils.decorators import method_decorator
|
|
from django.utils.translation import gettext as _
|
|
from django.views.decorators.cache import never_cache
|
|
from admin_data_views.admin import (
|
|
admin_data_index_view as adv_admin_data_index_view,
|
|
get_admin_data_urls as adv_get_admin_data_urls,
|
|
get_app_list as adv_get_app_list,
|
|
)
|
|
|
|
from archivebox.config import VERSION
|
|
from archivebox.config.version import get_COMMIT_HASH
|
|
from archivebox.core.routes_util import is_allowed_archivebox_redirect_url
|
|
|
|
if TYPE_CHECKING:
|
|
from django.http import HttpRequest
|
|
from django.template.response import TemplateResponse
|
|
from django.urls import URLPattern, URLResolver
|
|
|
|
from admin_data_views.typing import AppDict
|
|
|
|
|
|
class ArchiveBoxLoginView(LoginView):
|
|
def get_redirect_url(self) -> str:
|
|
redirect_to = self.request.POST.get(
|
|
self.redirect_field_name,
|
|
self.request.GET.get(self.redirect_field_name),
|
|
)
|
|
if is_allowed_archivebox_redirect_url(redirect_to, request=self.request):
|
|
return redirect_to
|
|
return ""
|
|
|
|
|
|
class ArchiveBoxAdmin(admin.AdminSite):
|
|
site_header = "ArchiveBox"
|
|
index_title = "Admin Views"
|
|
site_title = "Admin"
|
|
namespace = "admin"
|
|
|
|
def each_context(self, request: "HttpRequest") -> dict[str, Any]:
|
|
context = super().each_context(request)
|
|
context["VERSION"] = VERSION
|
|
context["STATIC_CACHE_KEY"] = (get_COMMIT_HASH() or VERSION or "dev").strip()
|
|
return context
|
|
|
|
@staticmethod
|
|
def _format_object_count(count: int) -> tuple[int, str, str]:
|
|
if count >= 1_000_000_000:
|
|
count_label = f"{count / 1_000_000_000:.1f}B"
|
|
elif count >= 1_000_000:
|
|
count_label = f"{count / 1_000_000:.1f}M"
|
|
elif count >= 1_000:
|
|
count_label = f"{count / 1_000:.1f}K"
|
|
else:
|
|
count_label = f"{count:,}"
|
|
count_label = count_label.replace(".0", "")
|
|
return count, count_label, f"Object count: {count:,}"
|
|
|
|
def _set_model_object_count(
|
|
self,
|
|
models_by_table: dict[str, list[dict[str, Any]]],
|
|
table: str,
|
|
count: int,
|
|
title: str | None = None,
|
|
) -> None:
|
|
models = models_by_table.get(table)
|
|
if not models:
|
|
return
|
|
count, count_label, count_title = self._format_object_count(count)
|
|
if title:
|
|
count_title = title
|
|
for model in models:
|
|
model["object_count"] = count
|
|
model["object_count_label"] = count_label
|
|
model["object_count_title"] = count_title
|
|
|
|
def get_app_list(self, request: "HttpRequest", app_label: str | None = None) -> list["AppDict"]:
|
|
if app_label is None:
|
|
return adv_get_app_list(self, request)
|
|
return adv_get_app_list(self, request, app_label)
|
|
|
|
def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
|
|
return adv_admin_data_index_view(self, request, **kwargs)
|
|
|
|
@method_decorator(never_cache)
|
|
@login_not_required
|
|
def login(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
|
|
if request.method == "GET" and self.has_permission(request):
|
|
return HttpResponseRedirect(reverse("admin:index", current_app=self.name))
|
|
|
|
from django.contrib.admin.forms import AdminAuthenticationForm
|
|
|
|
context = {
|
|
**self.each_context(request),
|
|
"title": _("Log in"),
|
|
"subtitle": None,
|
|
"app_path": request.get_full_path(),
|
|
"username": request.user.get_username(),
|
|
}
|
|
if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
|
|
context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
|
|
context.update(extra_context or {})
|
|
|
|
index_path = reverse("admin:index", current_app=self.name)
|
|
request.current_app = self.name
|
|
return ArchiveBoxLoginView.as_view(
|
|
extra_context=context,
|
|
authentication_form=self.login_form or AdminAuthenticationForm,
|
|
template_name=self.login_template or "admin/login.html",
|
|
next_page=index_path,
|
|
)(request)
|
|
|
|
def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
|
|
response = super().index(request, extra_context)
|
|
|
|
models_by_table: dict[str, list[dict[str, Any]]] = {}
|
|
for app in response.context_data.get("app_list", []):
|
|
for model in app.get("models", []):
|
|
model_class = model.get("model")
|
|
if not model_class or not model.get("perms", {}).get("view"):
|
|
continue
|
|
models_by_table.setdefault(model_class._meta.db_table, []).append(model)
|
|
|
|
if not models_by_table:
|
|
return response
|
|
|
|
from archivebox.misc.db import approximate_row_counts
|
|
|
|
for table, count in approximate_row_counts(connection).items():
|
|
if table not in models_by_table:
|
|
continue
|
|
self._set_model_object_count(
|
|
models_by_table,
|
|
table,
|
|
count,
|
|
title=f"Approximate count from database stats: {count:,}",
|
|
)
|
|
models_by_table.pop(table, None)
|
|
|
|
for table in list(models_by_table):
|
|
try:
|
|
with connection.cursor() as cursor:
|
|
cursor.execute(f"SELECT COUNT(*) FROM {connection.ops.quote_name(table)}")
|
|
count = int(cursor.fetchone()[0])
|
|
except DatabaseError:
|
|
continue
|
|
self._set_model_object_count(models_by_table, table, count)
|
|
models_by_table.pop(table, None)
|
|
return response
|
|
|
|
def get_admin_data_urls(self) -> list["URLResolver | URLPattern"]:
|
|
return adv_get_admin_data_urls(self)
|
|
|
|
def get_urls(self) -> list["URLResolver | URLPattern"]:
|
|
return self.get_admin_data_urls() + super().get_urls()
|
|
|
|
|
|
archivebox_admin = ArchiveBoxAdmin()
|
|
# Note: delete_selected is enabled per-model via actions = ['delete_selected'] in each ModelAdmin
|
|
# TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
|
|
|
|
|
|
############### Admin Data View sections are defined in settings.ADMIN_DATA_VIEWS #########
|
|
|
|
|
|
def register_admin_site():
|
|
"""Replace the default admin site with our custom ArchiveBox admin site."""
|
|
from django.contrib import admin
|
|
from django.contrib.admin import sites
|
|
|
|
admin.site = archivebox_admin
|
|
sites.site = archivebox_admin
|
|
|
|
# Register admin views for each app
|
|
# (Previously handled by ABX plugin system, now called directly)
|
|
from archivebox.core.admin import register_admin as register_core_admin
|
|
from archivebox.crawls.admin import register_admin as register_crawls_admin
|
|
from archivebox.api.admin import register_admin as register_api_admin
|
|
from archivebox.machine.admin import register_admin as register_machine_admin
|
|
from archivebox.personas.admin import register_admin as register_personas_admin
|
|
from archivebox.workers.admin import register_admin as register_workers_admin
|
|
|
|
register_core_admin(archivebox_admin)
|
|
register_crawls_admin(archivebox_admin)
|
|
register_api_admin(archivebox_admin)
|
|
register_machine_admin(archivebox_admin)
|
|
register_personas_admin(archivebox_admin)
|
|
register_workers_admin(archivebox_admin)
|
|
|
|
return archivebox_admin
|