@@ -710,10 +705,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
)
def output_display(self, result):
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
# Determine output link path - use embed_path() which checks output_files
- embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+ embed_path = result.embed_path()
output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
snapshot_id = str(result.snapshot_id)
return format_html(
@@ -728,12 +723,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
if not output_text:
return "-"
- request = getattr(self, "request", None)
- live_path = result.embed_path() if hasattr(result, "embed_path") else None
+ request = self.request
+ live_path = result.embed_path()
if live_path:
return format_html(
'
{}',
- build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
+ build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
output_text,
output_text,
)
@@ -785,18 +780,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
)
def output_summary(self, result):
- snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+ snapshot_dir = Path(result.snapshot.output_dir)
output_html = format_html(
'
{}',
result.output_str_for_display(),
)
snapshot_id = str(result.snapshot_id)
- request = getattr(self, "request", None)
+ request = self.request
output_html += format_html(
'
See result files ...',
- build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+ build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
)
- embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+ embed_path = result.embed_path() or ""
path_from_embed = snapshot_dir / (embed_path or "")
output_html += format_html(
'{}/{}
',
diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py
index fbf227bf..de9dbbbb 100644
--- a/archivebox/core/admin_snapshots.py
+++ b/archivebox/core/admin_snapshots.py
@@ -200,7 +200,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
("failed", ">50% failed"),
("running", ">50% running"),
("pending", ">50% queued"),
- ("backoff", ">50% waiting to retry"),
("noresults", ">50% noresults"),
)
@@ -216,7 +215,7 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
"succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
"failed": ArchiveResult.StatusChoices.FAILED,
"running": ArchiveResult.StatusChoices.STARTED,
- "pending": ArchiveResult.StatusChoices.QUEUED,
+ "pending": (ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.BACKOFF),
"backoff": ArchiveResult.StatusChoices.BACKOFF,
"noresults": ArchiveResult.StatusChoices.NORESULTS,
}
@@ -226,21 +225,28 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
# "succeeded" is overwhelmingly common in large collections.
# Scan Snapshots in admin order and do indexed per-snapshot
# probes so page 1 can stop after list_per_page matches.
- return queryset.alias(
+ queryset = queryset.alias(
total_results=ArchiveResult.snapshot_count_expr(),
matching_results=ArchiveResult.snapshot_count_expr(status=status),
).filter(matching_results__gt=F("total_results") / 2)
+ queryset._archivebox_count_hint = "model_estimate"
+ queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+ return queryset
# Rare statuses are faster status-first: use the
# (status, snapshot_id) index to find candidate snapshots.
- return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status))
+ snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
+ queryset = queryset.filter(pk__in=snapshot_ids)
+ queryset._archivebox_count_hint = len(snapshot_ids)
+ queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+ return queryset
return queryset
class SnapshotChangeList(SearchResultsChangeList):
def __init__(self, request, *args, **kwargs):
super().__init__(request, *args, **kwargs)
- resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+ resolver_name = request.resolver_match.url_name
self.embedded_changelist = request.GET.get("_embedded") == "crawl"
self.snapshot_is_grid_view = not self.embedded_changelist and (
resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
@@ -323,9 +329,7 @@ class SnapshotAdminForm(forms.ModelForm):
# Handle tags_editor field
if commit:
instance.save()
- save_m2m = getattr(self, "_save_m2m", None)
- if callable(save_m2m):
- save_m2m()
+ self._save_m2m()
# Parse and save tags from tags_editor
tags_str = self.cleaned_data.get("tags_editor", "")
@@ -489,7 +493,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return super().get_ordering(request)
def change_view(self, request, object_id, form_url="", extra_context=None):
- request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
+ self.request = request
extra_context = extra_context or {}
extra_context["CONFIG"] = request.archivebox_config
snapshot = self.get_object(request, object_id)
@@ -504,7 +508,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
def changelist_view(self, request, extra_context=None):
self.request = request
- request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
saved_list_per_page = self.list_per_page
embedded_changelist = request.GET.get("_embedded") == "crawl"
if embedded_changelist:
@@ -540,12 +543,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return super().lookup_allowed(lookup, value, request=request)
def get_snapshot_view_url(self, obj: Snapshot) -> str:
- request = getattr(self, "request", None)
- return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
+ request = self.request
+ return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
def get_snapshot_files_url(self, obj: Snapshot) -> str:
- request = getattr(self, "request", None)
- return f"{build_snapshot_url(str(obj.id), request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+ request = self.request
+ return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
def get_snapshot_zip_url(self, obj: Snapshot) -> str:
return f"{self.get_snapshot_files_url(obj)}&download=zip"
@@ -650,7 +653,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
ordering_fields = self._get_ordering_fields(request)
needs_files_sort = "files" in ordering_fields
needs_tags_sort = "tags_inline" in ordering_fields
- is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
+ is_change_view = request.resolver_match.url_name == "core_snapshot_change"
prefetches = ["tags"]
if is_change_view:
prefetches.append(
@@ -702,7 +705,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
@admin.display(description="👁", ordering="permissions")
def permissions_badge(self, obj):
- permissions = getattr(obj, "snapshot_permissions", None)
+ permissions = obj.__dict__.get("snapshot_permissions")
if permissions is None:
if obj.permissions:
permissions = obj.permissions
@@ -872,8 +875,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
)
def status_info(self, obj):
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
return format_html(
"""
@@ -890,16 +893,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
@admin.display(description="Archive Results")
def archiveresults_list(self, obj):
- request = getattr(self, "request", None)
- return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=getattr(request, "archivebox_config", None))
+ request = self.request
+ return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)
@admin.display(
description="Title",
ordering="title",
)
def title_str(self, obj):
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
title_raw = (obj.title or "").strip()
url_raw = (obj.url or "").strip()
title_normalized = title_raw.lower()
@@ -952,8 +955,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return mark_safe(f'{tags_html}')
def _get_preview_data(self, obj):
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
results = self._get_prefetched_results(obj)
if results is not None:
has_screenshot = any(r.plugin == "screenshot" for r in results)
@@ -1005,8 +1008,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
if not preview:
return ""
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
onerror_js = (
@@ -1044,8 +1047,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
@admin.display(description=" ", empty_value="")
def snapshot_summary(self, obj):
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
preview = self._get_preview_data(obj)
stats = self._get_progress_stats(obj)
archive_size = stats["output_size"] or 0
@@ -1109,8 +1112,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
)
visible_results = sorted_results[:14]
output = []
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
for result in visible_results:
icon = mark_safe(get_plugin_icon(result.plugin))
if not icon.strip():
@@ -1141,8 +1144,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
# ordering='archiveresult_count'
)
def size(self, obj):
- request = getattr(self, "request", None)
- config = getattr(request, "archivebox_config", None)
+ request = self.request
+ config = request.archivebox_config
archive_size = self._get_progress_stats(obj)["output_size"] or 0
if archive_size:
size_txt = printable_filesize(archive_size)
@@ -1256,7 +1259,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
)
def _get_progress_stats(self, obj):
- cached_stats = getattr(obj, "_admin_progress_stats", None)
+ cached_stats = obj.__dict__.get("_admin_progress_stats")
if cached_stats is not None:
return cached_stats
@@ -1302,31 +1305,31 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
def _get_prefetched_results(self, obj):
if "_admin_archiveresults" in obj.__dict__:
return obj.__dict__["_admin_archiveresults"]
- if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
+ if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
return obj.archiveresult_set.all()
return None
def _get_expected_hook_total(self, obj) -> int:
try:
- request = getattr(self, "request", None)
- if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}:
+ request = self.request
+ if request.resolver_match.url_name in {"core_snapshot_changelist", "core_snapshot_change"}:
return 0
- crawl = getattr(obj, "crawl", None)
- snapshot_config = getattr(obj, "config", None) or {}
- crawl_config = getattr(crawl, "config", None) or {}
+ crawl = obj.crawl
+ snapshot_config = obj.config or {}
+ crawl_config = crawl.config or {}
has_scoped_config = bool(snapshot_config or crawl_config)
if request is not None and not has_scoped_config:
- cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
+ cached_total = request.__dict__.get("archivebox_expected_snapshot_hook_total")
if cached_total is None:
- config = getattr(request, "archivebox_config", None) or get_config()
+ config = request.archivebox_config
cached_total = len(discover_hooks("Snapshot", config=config))
request.archivebox_expected_snapshot_hook_total = cached_total
return cached_total
if request is not None:
- scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+ scoped_cache = request.__dict__.get("archivebox_expected_snapshot_hook_totals_by_scope")
if scoped_cache is None:
scoped_cache = {}
request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
@@ -1346,8 +1349,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
return 0
def _get_prefetched_tags(self, obj):
- if hasattr(obj, "_prefetched_objects_cache") and "tags" in obj._prefetched_objects_cache:
- return list(obj._prefetched_objects_cache["tags"])
+ prefetched_cache = obj.__dict__.get("_prefetched_objects_cache", {})
+ if "tags" in prefetched_cache:
+ return list(prefetched_cache["tags"])
return None
def _get_ordering_fields(self, request):
diff --git a/archivebox/core/admin_tags.py b/archivebox/core/admin_tags.py
index 07eb7209..db6c3dca 100644
--- a/archivebox/core/admin_tags.py
+++ b/archivebox/core/admin_tags.py
@@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):
@admin.display(description="Snapshots", ordering="num_snapshots")
def num_snapshots(self, tag: Tag):
- count = getattr(tag, "num_snapshots", None)
+ count = tag.__dict__.get("num_snapshots")
if count is None:
count = tag.snapshot_set.count()
return format_html(
diff --git a/archivebox/core/admin_users.py b/archivebox/core/admin_users.py
index bce9f6f1..d8e6ac3c 100644
--- a/archivebox/core/admin_users.py
+++ b/archivebox/core/admin_users.py
@@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):
def snapshot_count_badge(self, obj):
snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
- snapshot_count = getattr(obj, "snapshot_count", 0)
+ snapshot_count = obj.__dict__.get("snapshot_count", 0)
snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
return format_html(
(
diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py
index 57fc280d..c47823be 100644
--- a/archivebox/core/forms.py
+++ b/archivebox/core/forms.py
@@ -31,6 +31,8 @@ DEPTH_CHOICES = (
class AddLinkForm(PluginConfigFormMixin, forms.Form):
+ allow_crawl_execution_config_fields = False
+
# Basic fields
url = forms.CharField(
label="URLs",
diff --git a/archivebox/core/middleware.py b/archivebox/core/middleware.py
index cfe7b770..50f02fd5 100644
--- a/archivebox/core/middleware.py
+++ b/archivebox/core/middleware.py
@@ -119,7 +119,7 @@ def CacheControlMiddleware(get_response):
if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
if not response.get("Cache-Control"):
- config = getattr(request, "archivebox_config", None)
+ config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
@@ -136,7 +136,7 @@ def ServerSecurityModeMiddleware(get_response):
allowed_methods = {"GET", "HEAD", "OPTIONS"}
def middleware(request):
- config = getattr(request, "archivebox_config", None)
+ config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
@@ -169,7 +169,7 @@ def HostRoutingMiddleware(get_response):
return get_response(request)
request_host = (request.get_host() or "").lower()
- config = getattr(request, "archivebox_config", None)
+ config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
@@ -323,7 +323,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
header = "HTTP_REMOTE_USER"
def process_request(self, request):
- config = getattr(request, "archivebox_config", None)
+ config = request.__dict__.get("archivebox_config")
if config is None:
config = get_config(resolve_plugins=False)
request.archivebox_config = config
diff --git a/archivebox/core/migrations/0048_remove_archiveresult_config.py b/archivebox/core/migrations/0048_remove_archiveresult_config.py
new file mode 100644
index 00000000..fe95e949
--- /dev/null
+++ b/archivebox/core/migrations/0048_remove_archiveresult_config.py
@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("core", "0047_archiveresult_status_snapshot_index"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="archiveresult",
+ name="config",
+ ),
+ ]
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index 33ee9e46..8a29eeb9 100755
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -63,6 +63,13 @@ if TYPE_CHECKING:
from archivebox.config.common import ArchiveBoxBaseConfig
+class UngroupedSubquery(models.Subquery):
+ """Scalar subquery that should not be copied into the outer GROUP BY."""
+
+ def get_group_by_cols(self):
+ return []
+
+
class Tag(ModelWithUUID):
# Keep AutoField for compatibility with main branch migrations
# Don't use UUIDField here - requires complex FK transformation
@@ -203,7 +210,7 @@ class SnapshotQuerySet(models.QuerySet):
offset += chunk_size
return
- unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if getattr(field, "unique", False))}
+ unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
if not any(field_name in unique_field_names for field_name in ordered_field_names):
offset = 0
while True:
@@ -659,7 +666,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
self.sm.seal()
def get_delete_after_config_value(self):
- return get_config(snapshot=self).DELETE_AFTER
+ from archivebox.config.common import resolve_delete_after_config_value
+
+ return resolve_delete_after_config_value(self.config, self.crawl.config)
@classmethod
def missing_delete_at_candidates(cls):
@@ -818,7 +827,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
# work and crawl maintenance so SQLite commits before touching the disk.
transaction.on_commit(finish_snapshot_save)
- migration_cleanup = getattr(self, "_pending_fs_migration_cleanup", None)
+ migration_cleanup = self.__dict__.get("_pending_fs_migration_cleanup")
if migration_cleanup:
old_dir, new_dir = migration_cleanup
transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir))
@@ -890,9 +899,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
runtime_config = config or get_config()
if source_dir and current == target:
- current_dir = self.get_storage_path_for_version(target, config=runtime_config)
+ current_dir = self.get_storage_path_for_version(target)
cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir)
- crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+ crawl_dir = self.crawl.output_dir
old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -932,9 +941,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
def _fs_migrate_from_0_9_0_to_0_9_4(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None):
runtime_config = config or get_config()
- target_dir = self.get_storage_path_for_version("0.9.4", config=runtime_config)
+ target_dir = self.get_storage_path_for_version("0.9.4")
cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir or self.output_dir, target_dir=target_dir, config=runtime_config)
- crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+ crawl_dir = self.crawl.output_dir
old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -956,8 +965,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
import filecmp
import shutil
- old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0", config=config)
- new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0", config=config)
+ old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0")
+ new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0")
if old_dir == new_dir:
return None
@@ -1087,7 +1096,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
except Exception:
return "unknown"
- def get_storage_path_for_version(self, version: str, config: "ArchiveBoxBaseConfig | None" = None) -> Path:
+ def get_storage_path_for_version(self, version: str) -> Path:
"""
Calculate storage path for specific filesystem version.
Centralizes path logic so it's reusable.
@@ -1095,10 +1104,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
0.7.x/0.8.x: archive/{timestamp}
0.9.x: archive/users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/
"""
- runtime_config = config or get_config()
-
if version in ("0.7.0", "0.8.0"):
- return runtime_config.ARCHIVE_DIR / self.timestamp
+ return CONSTANTS.ARCHIVE_DIR / self.timestamp
elif version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
username = self.created_by.username
@@ -1108,10 +1115,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
domain = self.extract_domain_from_url(self.url)
- return runtime_config.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
+ return CONSTANTS.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
else:
# Unknown version - use current
- return self.get_storage_path_for_version(self._fs_current_version(), config=runtime_config)
+ return self.get_storage_path_for_version(self._fs_current_version())
# =========================================================================
# Loading and Creation from Filesystem (Used by archivebox update ONLY)
@@ -1702,7 +1709,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
f.write(json.dumps(process.to_json()) + "\n")
# Write ArchiveResult record
- f.write(json.dumps(ar.to_json()) + "\n")
+ f.write(json.dumps(ar.to_json(snapshot_output_dir=output_dir)) + "\n")
os.replace(tmp_index_path, index_path)
def read_index_jsonl(self, output_dir: Path | None = None) -> dict:
@@ -1978,7 +1985,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@admin.display(description="Tags")
def tags_str(self, nocache=True) -> str | None:
calc_tags_str = lambda: ",".join(sorted(tag.name for tag in self.tags.all()))
- prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+ prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if "tags" in prefetched_cache:
return calc_tags_str()
cache_key = f"{self.pk}-tags"
@@ -1988,12 +1995,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""Generate HTML icons showing which extractor plugins have succeeded for this snapshot"""
from django.utils.html import format_html
- compact_icons = getattr(self, "_icons_compact", False)
+ compact_icons = self.__dict__.get("_icons_compact", False)
cache_key = f"result_icons:{self.pk}:{'compact' if compact_icons else 'full'}:{(self.downloaded_at or self.modified_at or self.created_at or self.bookmarked_at).timestamp()}"
def calc_icons():
if compact_icons and self.status == self.StatusChoices.STARTED:
- progress_stats = getattr(self, "_icons_progress_stats", None) or self.get_progress_stats()
+ progress_stats = self.__dict__.get("_icons_progress_stats") or self.get_progress_stats()
total = int(progress_stats.get("total") or 0)
succeeded = int(progress_stats.get("succeeded") or 0)
failed = int(progress_stats.get("failed") or 0)
@@ -2025,8 +2032,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
running,
)
- precomputed_archive_results = getattr(self, "_icons_archive_results", None)
- prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+ precomputed_archive_results = self.__dict__.get("_icons_archive_results")
+ prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if precomputed_archive_results is not None and compact_icons:
archive_results = {plugin: True for plugin in precomputed_archive_results}
elif "archiveresult_set" in prefetched_cache:
@@ -2049,7 +2056,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
output_template = '{}'
# Get all plugins from hooks system (sorted by numeric prefix)
- all_plugins = getattr(self, "_icons_plugin_names", None)
+ all_plugins = self.__dict__.get("_icons_plugin_names")
if all_plugins is None and not compact_icons:
all_plugins = [get_plugin_name(e) for e in get_plugins()]
elif all_plugins is None:
@@ -2200,8 +2207,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""The filesystem path to the snapshot's output directory."""
import os
- runtime_config = getattr(self, "_runtime_config", None) or get_config()
- current_path = self.get_storage_path_for_version(self.fs_version, config=runtime_config)
+ current_path = self.get_storage_path_for_version(self.fs_version)
if current_path.exists():
return current_path
@@ -2212,7 +2218,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
return hyphen_path
# Check for backwards-compat symlink
- old_path = runtime_config.ARCHIVE_DIR / self.timestamp
+ old_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
if old_path.is_symlink():
link_target = Path(os.readlink(old_path))
return (old_path.parent / link_target).resolve() if not link_target.is_absolute() else link_target.resolve()
@@ -2225,7 +2231,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
"""Ensure the legacy archive/ path resolves to this snapshot."""
import os
- legacy_path = get_config().ARCHIVE_DIR / self.timestamp
+ legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
target = Path(self.get_storage_path_for_version(self._fs_current_version()))
if target == legacy_path:
@@ -2301,8 +2307,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
username = "web"
- crawl = getattr(self, "crawl", None)
- if crawl and getattr(crawl, "created_by_id", None):
+ crawl = self.crawl if self.crawl_id else None
+ if crawl and crawl.created_by_id:
username = crawl.created_by.username
if username == "system":
username = "web"
@@ -2326,7 +2332,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
output_dir = Path(self.output_dir).resolve()
try:
- rel_users_path = output_dir.relative_to(get_config().USERS_DIR)
+ rel_users_path = output_dir.relative_to(CONSTANTS.USERS_DIR)
except Exception:
rel_users_path = None
@@ -2836,7 +2842,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@cached_property
def is_archived(self) -> bool:
- cached_is_archived = getattr(self, "_is_archived_cached", None)
+ cached_is_archived = self.__dict__.get("_is_archived_cached")
if cached_is_archived is not None:
return bool(cached_is_archived)
@@ -2890,10 +2896,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@cached_property
def num_outputs(self) -> int:
- if hasattr(self, "num_outputs_cached"):
- return int(self.num_outputs_cached or 0)
+ if "num_outputs_cached" in self.__dict__:
+ return int(self.__dict__["num_outputs_cached"] or 0)
- prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+ prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if "archiveresult_set" in prefetched_cache:
return sum(1 for result in self.archiveresult_set.all() if result.status == "succeeded")
@@ -2901,10 +2907,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
@cached_property
def num_failures(self) -> int:
- if hasattr(self, "num_failures_cached"):
- return int(self.num_failures_cached or 0)
+ if "num_failures_cached" in self.__dict__:
+ return int(self.__dict__["num_failures_cached"] or 0)
- prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+ prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
if "archiveresult_set" in prefetched_cache:
return sum(1 for result in self.archiveresult_set.all() if result.status == "failed")
@@ -3458,7 +3464,7 @@ class SnapshotMachine(BaseStateMachine):
# and the runner still needs to enqueue those child snapshots.
-class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
+class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
class StatusChoices(models.TextChoices):
QUEUED = "queued", "Queued"
STARTED = "started", "Started"
@@ -3523,7 +3529,17 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
qs = cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
if status is not None:
qs = qs.filter(status=status)
- return qs.order_by().values("snapshot_id").annotate(count=models.Count("pk")).values("count")
+ return qs.order_by().values("snapshot_id").annotate(count=models.Count("*")).values("count")
+
+ @classmethod
+ def snapshot_half_count_subquery(cls, *, outer_ref: str = "snapshot_id") -> QuerySet:
+ return (
+ cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
+ .order_by()
+ .values("snapshot_id")
+ .annotate(half=models.Count("*") / models.Value(2))
+ .values("half")
+ )
@classmethod
def snapshot_count_expr(cls, *, status: str | None = None, outer_ref: str = "pk"):
@@ -3539,28 +3555,50 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
return {status: qs.filter(status=status).count() for status in (statuses or cls.StatusChoices.values)}
@classmethod
- def snapshot_ids_with_majority_status(cls, status: str) -> QuerySet:
+ def snapshot_ids_with_majority_status(cls, status: str | Iterable[str]) -> QuerySet:
"""Return Snapshot IDs where more than half of ArchiveResults have ``status``.
Start from ArchiveResult.status for every majority-status filter. The
``(status, snapshot_id)`` index keeps the plan predictable even when a
user's collection has an unusual status distribution.
"""
+ statuses = tuple(status) if not isinstance(status, str) else (status,)
+ total_half = UngroupedSubquery(cls.snapshot_half_count_subquery(outer_ref="snapshot_id"), output_field=models.IntegerField())
return (
- cls.objects.filter(status=status)
+ cls.objects.filter(status__in=statuses)
.order_by()
.values("snapshot_id")
.annotate(
- matching_results=models.Count("pk"),
- total_results=models.Subquery(
- cls.snapshot_count_subquery(outer_ref="snapshot_id"),
- output_field=models.IntegerField(),
- ),
+ matching_results=models.Count("*"),
+ total_half=total_half,
)
- .filter(matching_results__gt=models.F("total_results") / 2)
+ .filter(matching_results__gt=models.F("total_half"))
.values("snapshot_id")
)
+ @classmethod
+ def cached_snapshot_ids_with_majority_status(cls, status: str | Iterable[str], *, timeout: int = 60) -> tuple[str, ...]:
+ statuses = tuple(status) if not isinstance(status, str) else (status,)
+ cache_key = f"archivebox:archiveresult:majority_status:{':'.join(sorted(statuses))}"
+ cached_ids = cache.get(cache_key)
+ if cached_ids is not None:
+ return tuple(cached_ids)
+
+ snapshot_ids = tuple(
+ str(snapshot_id) for snapshot_id in cls.snapshot_ids_with_majority_status(statuses).values_list("snapshot_id", flat=True)
+ )
+ cache.set(cache_key, snapshot_ids, timeout=timeout)
+ return snapshot_ids
+
+ @classmethod
+ def clear_majority_status_cache(cls) -> None:
+ cache.delete_many(
+ [
+ *(f"archivebox:archiveresult:majority_status:{status}" for status in cls.StatusChoices.values),
+ f"archivebox:archiveresult:majority_status:{':'.join(sorted((cls.StatusChoices.BACKOFF, cls.StatusChoices.QUEUED)))}",
+ ],
+ )
+
# UUID primary key (migrated from integer in 0029)
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -3609,7 +3647,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
class Meta(
ModelWithDeleteAfter.Meta,
ModelWithOutputDir.Meta,
- ModelWithConfig.Meta,
ModelWithNotes.Meta,
):
app_label = "core"
@@ -3644,14 +3681,15 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
return "\n".join(self._format_output_line_for_display(line) for line in str(self.output_str or "").splitlines())
def get_delete_after_config_value(self):
- return get_config(archiveresult=self).DELETE_AFTER
+ snapshot = self.snapshot
+ from archivebox.config.common import resolve_delete_after_config_value
+
+ return resolve_delete_after_config_value(snapshot.config, snapshot.crawl.config)
@classmethod
def missing_delete_at_candidates(cls):
return cls.objects.filter(delete_at__isnull=True).filter(
- Q(config__has_key="DELETE_AFTER")
- | Q(snapshot__config__has_key="DELETE_AFTER")
- | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
+ Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
)
@property
@@ -3659,12 +3697,21 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
"""Convenience property to access the user who created this archive result via its snapshot's crawl."""
return self.snapshot.crawl.created_by
- def to_json(self) -> dict:
+ def to_json(self, *, snapshot_output_dir: Path | None = None) -> dict:
"""
Convert ArchiveResult model instance to a JSON-serializable dict.
"""
from archivebox.config import VERSION
+ process = self.process_record
+ pwd = (
+ process.pwd
+ if process and process.pwd
+ else str((snapshot_output_dir / self.plugin) if snapshot_output_dir is not None else self.output_dir)
+ )
+ cmd = process.cmd if process else []
+ cmd_version = process.cmd_version if process else ""
+
record = {
"type": "ArchiveResult",
"schema_version": VERSION,
@@ -3686,13 +3733,12 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
record["output_size"] = self.output_size
if self.output_mimetypes:
record["output_mimetypes"] = self.output_mimetypes
- if self.pwd:
- record["pwd"] = self.pwd
- if self.cmd:
- record["cmd"] = self.cmd
- if self.cmd_version:
- record["cmd_version"] = self.cmd_version
- process = self.process_record
+ if pwd:
+ record["pwd"] = pwd
+ if cmd:
+ record["cmd"] = cmd
+ if cmd_version:
+ record["cmd_version"] = cmd_version
if process:
record["process_id"] = str(process.id)
return record
@@ -3767,6 +3813,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
if refresh_snapshot_size:
snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id}
transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+ if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields:
+ transaction.on_commit(type(self).clear_majority_status_cache)
# if is_new:
# from archivebox.misc.logging_util import log_worker_event
@@ -3788,6 +3836,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
deleted = super().delete(*args, **kwargs)
if snapshot_id:
transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes({snapshot_id}))
+ transaction.on_commit(type(self).clear_majority_status_cache)
return deleted
@staticmethod
@@ -4495,7 +4544,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
def _url_passes_filters(self, url: str) -> bool:
"""Check if URL passes URL_ALLOWLIST and URL_DENYLIST config filters.
- Uses proper config hierarchy: defaults -> file -> env -> machine -> user -> crawl -> snapshot
+ Uses the centralized config resolver so frozen crawl/snapshot values
+ and live Machine/Persona execution values apply in their scoped order.
"""
return self.snapshot.crawl.url_passes_filters(url, snapshot=self.snapshot)
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index 0e97b412..4d3f559f 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -174,9 +174,9 @@ except ImportError:
STATIC_URL = "/static/"
TEMPLATES_DIR_NAME = "templates"
-CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
+CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
STATICFILES_DIRS = [
- *([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
+ *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
# *[
# str(plugin_dir / 'static')
# for plugin_dir in PLUGIN_DIRS.values()
@@ -187,7 +187,7 @@ STATICFILES_DIRS = [
]
TEMPLATE_DIRS = [
- *([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
+ *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
# *[
# str(plugin_dir / 'templates')
# for plugin_dir in PLUGIN_DIRS.values()
@@ -334,7 +334,7 @@ STORAGES = {
"BACKEND": "django.core.files.storage.FileSystemStorage",
"OPTIONS": {
"base_url": "/archive/",
- "location": CONFIG.ARCHIVE_DIR,
+ "location": CONSTANTS.ARCHIVE_DIR,
},
},
# "snapshots": {
diff --git a/archivebox/core/shutdown_util.py b/archivebox/core/shutdown_util.py
index 95bb7edf..f9273360 100644
--- a/archivebox/core/shutdown_util.py
+++ b/archivebox/core/shutdown_util.py
@@ -134,7 +134,10 @@ def foreground_shutdown_signals(
# signal for hard foreground-command shutdown. Server/update/run and
# other non-interactive commands raise immediately so their finally
# blocks can stop owned children without prompting.
- if raise_on_first_signal or already_requested:
+ if already_requested:
+ os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
+ os._exit(130)
+ if raise_on_first_signal:
raise KeyboardInterrupt
try:
@@ -143,6 +146,8 @@ def foreground_shutdown_signals(
signal.signal(sig, raise_keyboard_interrupt)
yield state
finally:
+ if state.signal_name and previous_active_state is not None and not previous_active_state.signal_name:
+ previous_active_state.signal_name = state.signal_name
_active_shutdown_state = previous_active_state
for sig, previous_handler in previous_handlers.items():
signal.signal(sig, previous_handler)
diff --git a/archivebox/core/tag_util.py b/archivebox/core/tag_util.py
index d970cb09..27289c05 100644
--- a/archivebox/core/tag_util.py
+++ b/archivebox/core/tag_util.py
@@ -246,7 +246,7 @@ def _build_snapshot_preview_map(
def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = None) -> dict[str, Any]:
- count = getattr(tag, "num_snapshots", None)
+ count = tag.__dict__.get("num_snapshots")
if count is None:
count = tag.snapshot_set.count()
return {
diff --git a/archivebox/core/templatetags/core_tags.py b/archivebox/core/templatetags/core_tags.py
index 1ff044ee..dce6857d 100644
--- a/archivebox/core/templatetags/core_tags.py
+++ b/archivebox/core/templatetags/core_tags.py
@@ -76,6 +76,12 @@ def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
return {}
+def _snapshot_id(value: Any) -> Any:
+ from archivebox.core.models import Snapshot
+
+ return value.id if isinstance(value, Snapshot) else value
+
+
def _coerce_output_file_size(value: Any) -> int | None:
try:
return max(int(value or 0), 0)
@@ -85,7 +91,7 @@ def _coerce_output_file_size(value: Any) -> int | None:
def _count_media_files(result) -> int:
try:
- output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+ output_files = _normalize_output_files(result.output_files or {})
except Exception:
output_files = {}
@@ -121,7 +127,7 @@ def _list_media_files(result) -> list[dict]:
except Exception:
return media_files
- output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+ output_files = _normalize_output_files(result.output_files or {})
candidates: list[tuple[Path, int | None]] = []
if output_files:
for path, metadata in output_files.items():
@@ -300,7 +306,7 @@ def result_list(context, cl):
"""
num_sorted_fields = 0
request = context.get("request")
- config = getattr(request, "archivebox_config", None) or context.get("CONFIG")
+ config = request.__dict__.get("archivebox_config") if request is not None else context.get("CONFIG")
results = cl.result_list
if config is not None:
for obj in results:
@@ -443,7 +449,7 @@ def _unconfigured_banner_context(request) -> dict:
# operator's clipboard already aligned with subdomain routing. The config
# parser strips the leading ``*.`` so users can paste it verbatim.
suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
- user = getattr(request, "user", None)
+ user = request.user
is_superuser = bool(user and user.is_authenticated and user.is_superuser)
machine_admin_url = ""
if is_superuser:
@@ -496,19 +502,19 @@ def public_base_url(context) -> str:
@register.simple_tag(takes_context=True)
def snapshot_base_url(context, snapshot) -> str:
- snapshot_id = getattr(snapshot, "id", snapshot)
+ snapshot_id = _snapshot_id(snapshot)
return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
@register.simple_tag(takes_context=True)
def snapshot_url(context, snapshot, path: str = "") -> str:
- snapshot_id = getattr(snapshot, "id", snapshot)
+ snapshot_id = _snapshot_id(snapshot)
return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
@register.simple_tag(takes_context=True)
def snapshot_preview_url(context, snapshot, path: str = "") -> str:
- snapshot_id = getattr(snapshot, "id", snapshot)
+ snapshot_id = _snapshot_id(snapshot)
return _build_snapshot_preview_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
@@ -538,16 +544,18 @@ def plugin_card(context, result) -> str:
- output_path: Path to output relative to snapshot dir (from embed_path())
- plugin: Plugin base name
"""
- if result is None or not hasattr(result, "plugin"):
+ from archivebox.core.models import ArchiveResult
+
+ if result is None or not isinstance(result, ArchiveResult):
return ""
plugin = get_plugin_name(result.plugin)
template_str = get_plugin_template(plugin, "card")
# Use embed_path() for the display path
- raw_output_path = result.embed_path() if hasattr(result, "embed_path") else ""
+ raw_output_path = result.embed_path() or ""
output_url = build_snapshot_url(
- str(getattr(result, "snapshot_id", "")),
+ str(result.snapshot_id),
raw_output_path or "",
request=context.get("request"),
config=context.get("CONFIG"),
@@ -558,7 +566,7 @@ def plugin_card(context, result) -> str:
media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
if media_files:
- snapshot_id = str(getattr(result, "snapshot_id", ""))
+ snapshot_id = str(result.snapshot_id)
request = context.get("request")
config = context.get("CONFIG")
for item in media_files:
@@ -592,7 +600,7 @@ def plugin_card(context, result) -> str:
pass
if force_text_preview:
- preview = _render_text_file_preview(getattr(result, "snapshot_dir", None), raw_output_path, plugin, icon_html)
+ preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
if preview:
return mark_safe(preview)
@@ -608,7 +616,7 @@ def plugin_card(context, result) -> str:
def output_card(snapshot, output_path: str, plugin: str) -> str:
plugin_name = get_plugin_name(plugin)
icon_html = get_plugin_icon(plugin_name)
- preview = _render_text_file_preview(getattr(snapshot, "output_dir", None), output_path, plugin_name, icon_html)
+ preview = _render_text_file_preview(snapshot.output_dir, output_path, plugin_name, icon_html)
if preview:
return mark_safe(preview)
@@ -624,7 +632,9 @@ def plugin_full(context, result) -> str:
Usage: {% plugin_full result %}
"""
- if result is None or not hasattr(result, "plugin"):
+ from archivebox.core.models import ArchiveResult
+
+ if result is None or not isinstance(result, ArchiveResult):
return ""
plugin = get_plugin_name(result.plugin)
@@ -634,14 +644,13 @@ def plugin_full(context, result) -> str:
return ""
raw_output_path = ""
- if hasattr(result, "embed_path_db"):
- raw_output_path = result.embed_path_db() or ""
- if not raw_output_path and hasattr(result, "embed_path"):
+ raw_output_path = result.embed_path_db() or ""
+ if not raw_output_path:
raw_output_path = result.embed_path() or ""
if _is_root_snapshot_output_path(raw_output_path):
return ""
output_url = build_snapshot_url(
- str(getattr(result, "snapshot_id", "")),
+ str(result.snapshot_id),
raw_output_path,
request=context.get("request"),
config=context.get("CONFIG"),
@@ -685,7 +694,7 @@ def api_token(context) -> str:
from archivebox.api.auth import get_or_create_api_token
request = context.get("request")
- user = getattr(request, "user", None)
+ user = request.user
if not user or not user.is_authenticated:
return ""
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index 94a5b29e..6790c87f 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -5,9 +5,7 @@ import os
import posixpath
from glob import glob, escape
from django.utils import timezone
-import inspect
from typing import cast
-from collections.abc import Callable
from pathlib import Path
from urllib.parse import quote, urlparse
@@ -30,7 +28,17 @@ from admin_data_views.utils import render_with_table_view, render_with_item_view
from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION
-from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED, get_config, get_all_configs, redact_sensitive_config
+from archivebox.config.common import (
+ SENSITIVE_CONFIG_VALUE_REDACTED,
+ find_config_default,
+ find_config_section,
+ find_config_source,
+ find_config_type,
+ get_config,
+ get_all_configs,
+ get_request_config,
+ redact_sensitive_config,
+)
from archivebox.config.configset import BaseConfigSet
from archivebox.misc.paginators import CountlessPaginator
from archivebox.misc.util import (
@@ -80,16 +88,6 @@ from archivebox.plugins.views import get_config_definition_link
from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
-def _get_request_config(request: HttpRequest, *, resolve_plugins: bool = False):
- request_config = getattr(request, "archivebox_config", None)
- request_config_resolves_plugins = bool(getattr(request, "_archivebox_config_resolves_plugins", False))
- if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
- request_config = get_config(resolve_plugins=resolve_plugins)
- request.archivebox_config = request_config
- request._archivebox_config_resolves_plugins = resolve_plugins
- return request_config
-
-
def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
target = archivefile or ""
if target == "index.html":
@@ -122,14 +120,14 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
def _admin_login_redirect_or_forbidden(request: HttpRequest):
- if _get_request_config(request).CONTROL_PLANE_ENABLED:
+ if get_request_config(request).CONTROL_PLANE_ENABLED:
return redirect(f"/admin/login/?next={request.path}")
return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
class HomepageView(View):
def get(self, request):
- request_config = _get_request_config(request)
+ request_config = get_request_config(request)
if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
return redirect("/admin/core/snapshot/")
@@ -200,7 +198,7 @@ class SnapshotView(View):
# Reuse the middleware-attached config; never re-bootstrap from env + plugin
# schemas just to render a snapshot page (that pays ~30ms for no reason).
- runtime_config = _get_request_config(request)
+ runtime_config = get_request_config(request)
snapshot._runtime_config = runtime_config
snapshot_permissions = get_snapshot_permissions(snapshot)
hidden_card_plugins = {"archivedotorg", "favicon", "title"}
@@ -734,7 +732,7 @@ def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
if not snapshot or not can_view_snapshot(request, snapshot):
return False
- request.archivebox_config = _get_request_config(request, resolve_plugins=False)
+ request.archivebox_config = get_request_config(request, resolve_plugins=False)
return True
@@ -845,11 +843,14 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind
def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
- request_config = _get_request_config(request, resolve_plugins=False)
+ rel_path = path or ""
+ request_config = get_request_config(
+ request,
+ resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
+ )
request.archivebox_config = request_config
request.archivebox_snapshot_url = snapshot.url
snapshot._runtime_config = request_config
- rel_path = path or ""
if rel_path.startswith("replay/") or rel_path == "replay":
response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
@@ -899,7 +900,8 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
- request_config = _get_request_config(request)
+ request_config = get_request_config(request, resolve_plugins=False)
+ request.archivebox_config = request_config
requested_root_index = path in ("", "index.html") or path.endswith("/")
rel_path = path or ""
if not rel_path or rel_path.endswith("/"):
@@ -909,13 +911,13 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
raise Http404
domain = domain.lower()
- match = _latest_response_match(request, domain, rel_path, data_root=request_config.USERS_DIR)
+ match = _latest_response_match(request, domain, rel_path, data_root=CONSTANTS.USERS_DIR)
if not match and "." not in Path(rel_path).name:
index_path = f"{rel_path.rstrip('/')}/index.html"
- match = _latest_response_match(request, domain, index_path, data_root=request_config.USERS_DIR)
+ match = _latest_response_match(request, domain, index_path, data_root=CONSTANTS.USERS_DIR)
if not match and "." not in Path(rel_path).name:
html_path = f"{rel_path}.html"
- match = _latest_response_match(request, domain, html_path, data_root=request_config.USERS_DIR)
+ match = _latest_response_match(request, domain, html_path, data_root=CONSTANTS.USERS_DIR)
show_indexes = bool(request.GET.get("files"))
if match:
@@ -924,7 +926,7 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
if response is not None:
return response
- responses_root = _latest_responses_root(request, domain, data_root=request_config.USERS_DIR)
+ responses_root = _latest_responses_root(request, domain, data_root=CONSTANTS.USERS_DIR)
if responses_root:
response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
if response is not None:
@@ -946,7 +948,7 @@ class SnapshotHostView(View):
"""Serve snapshot directory contents on ./."""
def get(self, request, snapshot_id: str, path: str = ""):
- request_config = _get_request_config(request)
+ request_config = get_request_config(request)
snapshot = _find_snapshot_by_ref(snapshot_id)
if not snapshot:
@@ -998,15 +1000,15 @@ class PublicIndexView(ListView):
paginator_class = CountlessPaginator
def get_paginate_by(self, queryset):
- runtime_config = getattr(self, "runtime_config", None)
+ runtime_config = self.__dict__.get("runtime_config")
if runtime_config is None:
- self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
+ self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
return runtime_config.SNAPSHOTS_PER_PAGE
def get_context_data(self, **kwargs):
- runtime_config = getattr(self, "runtime_config", None)
+ runtime_config = self.__dict__.get("runtime_config")
if runtime_config is None:
- self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
+ self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
context = {
@@ -1023,7 +1025,7 @@ class PublicIndexView(ListView):
self.request.GET.get("q")
and get_search_mode_base(search_mode, config=runtime_config) == "deep"
and search_mode_backend
- and getattr(context.get("paginator"), "count", 0) == 0,
+ and context["paginator"].count == 0,
)
snapshots = list(context.get("object_list") or ())
icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
@@ -1086,7 +1088,7 @@ class PublicIndexView(ListView):
if not query:
return qs
- runtime_config = getattr(self, "runtime_config", None)
+ runtime_config = self.__dict__.get("runtime_config")
search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
try:
return apply_snapshot_search(
@@ -1105,7 +1107,7 @@ class PublicIndexView(ListView):
def get(self, *args, **kwargs):
if self.request.user.is_authenticated:
return redirect("/admin/core/snapshot/")
- if _get_request_config(self.request).PUBLIC_INDEX:
+ if get_request_config(self.request).PUBLIC_INDEX:
response = super().get(*args, **kwargs)
return response
else:
@@ -1132,7 +1134,7 @@ class AddView(UserPassesTestMixin, FormView):
return kwargs
def test_func(self):
- return _get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
+ return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
def _can_override_crawl_config(self) -> bool:
return is_admin_user(self.request)
@@ -1146,11 +1148,11 @@ class AddView(UserPassesTestMixin, FormView):
if not self._can_override_crawl_config():
return {}
- return custom_config
+ return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
- request_config = _get_request_config(self.request, resolve_plugins=True)
+ request_config = get_request_config(self.request, resolve_plugins=True)
required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
can_override_crawl_config = self._can_override_crawl_config()
plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
@@ -1252,7 +1254,7 @@ class AddView(UserPassesTestMixin, FormView):
created_by_id = get_or_create_system_user_pk()
- created_by_name = getattr(self.request.user, "username", "web") if self.request.user.is_authenticated else "web"
+ created_by_name = self.request.user.username if self.request.user.is_authenticated else "web"
# 1. save the provided urls to sources/2024-11-05__23-59-59__web_ui_add_by_user_.txt
sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__web_ui_add_by_user_{created_by_id}.txt"
@@ -1267,8 +1269,7 @@ class AddView(UserPassesTestMixin, FormView):
config = {}
if plugins:
config["PLUGINS"] = plugins
- request_user = self.request.user if self.request.user.is_authenticated else None
- effective_config = get_config(persona=persona, user=request_user) if persona else get_config(user=request_user)
+ effective_config = get_config(persona=persona) if persona else get_config()
if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
if delete_after != str(effective_config.DELETE_AFTER):
@@ -1378,11 +1379,7 @@ class WebAddView(AddView):
return redirect(f"/{snapshot.url_path}")
request_host = (request.get_host() or "").lower()
- if (
- request.user.is_authenticated
- and not _get_request_config(request).PUBLIC_ADD_VIEW
- and host_matches(request_host, get_web_host())
- ):
+ if request.user.is_authenticated and not get_request_config(request).PUBLIC_ADD_VIEW and host_matches(request_host, get_web_host()):
return redirect(build_admin_url(request.get_full_path(), request=request))
if not self.test_func():
@@ -1461,79 +1458,11 @@ class HealthCheckView(View):
return HttpResponse("OK", content_type="text/plain", status=200)
-def find_config_section(key: str) -> str:
- CONFIGS = get_all_configs()
-
- if key in CONSTANTS_CONFIG:
- return "CONSTANT"
- matching_sections = [section_id for section_id, section in CONFIGS.items() if key in dict(section)]
- section = matching_sections[0] if matching_sections else "DYNAMIC"
- return section
-
-
-def find_config_default(key: str) -> str:
- CONFIGS = get_all_configs()
-
- if key in CONSTANTS_CONFIG:
- return str(CONSTANTS_CONFIG[key])
-
- default_val = None
-
- for config in CONFIGS.values():
- if key in dict(config):
- default_val = type(config).model_fields[key].default
- break
-
- if isinstance(default_val, Callable):
- default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
- if default_val.count(")") > default_val.count("("):
- default_val = default_val[:-1]
- else:
- default_val = str(default_val)
-
- return default_val
-
-
-def find_config_type(key: str) -> str:
- CONFIGS = get_all_configs()
-
- for config in CONFIGS.values():
- if key in type(config).model_fields:
- annotation = type(config).model_fields[key].annotation
- return getattr(annotation, "__name__", str(annotation))
- return "str"
-
-
-def find_config_source(key: str, merged_config: dict) -> str:
- """Determine where a config value comes from."""
- from archivebox.machine.models import Machine
-
- # Environment variables override all persistent config sources.
- if key in os.environ:
- return "Environment"
-
- # Machine.config overrides ArchiveBox.conf.
- try:
- machine = Machine.current()
- if machine.config and key in machine.config:
- return "Machine"
- except Exception:
- pass
-
- # Check if it's from archivebox.config.file
- file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
- if key in file_config:
- return "Config File"
-
- # Otherwise it's using the default
- return "Default"
-
-
@render_with_table_view
def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
CONFIGS = get_all_configs()
- assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
+ assert request.user.is_superuser, "Must be a superuser to view configuration settings."
merged_config = get_config(redact_sensitive=True)
@@ -1555,12 +1484,12 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
rows["Type"].append(format_html("{}", find_config_type(key)))
# Use merged config value (includes machine overrides)
- actual_value = merged_config.get(key, getattr(section, key, None))
+ actual_value = merged_config.get(key, dict(section)[key])
rows["Value"].append(mark_safe(f"{actual_value}"))
# Show where the value comes from
source = find_config_source(key, merged_config)
- source_colors = {"Machine": "purple", "Environment": "blue", "Config File": "green", "Default": "gray"}
+ source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
rows["Source"].append(format_html('{}', source_colors.get(source, "gray"), source))
rows["Default"].append(
@@ -1575,7 +1504,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
for key in CONSTANTS_CONFIG.keys():
rows["Section"].append(section) # section.replace('_', ' ').title().replace(' Config', '')
rows["Key"].append(ItemLink(key, key=key))
- rows["Type"].append(format_html("{}", getattr(type(CONSTANTS_CONFIG[key]), "__name__", str(CONSTANTS_CONFIG[key]))))
+ rows["Type"].append(format_html("{}", type(CONSTANTS_CONFIG[key]).__name__))
rows["Value"].append(format_html("{}", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
rows["Source"].append(mark_safe('Constant'))
rows["Default"].append(
@@ -1598,17 +1527,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
CONFIGS = get_all_configs()
- assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
+ assert request.user.is_superuser, "Must be a superuser to view configuration settings."
merged_config = get_config(redact_sensitive=True)
# Determine all sources for this config value
sources_info = []
- # Environment variable
- if key in os.environ:
- sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
-
# Machine config
machine = None
machine_admin_url = None
@@ -1620,11 +1545,15 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
except Exception:
pass
+ # Environment variable
+ if key in os.environ:
+ sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
+
# Config file value
if CONSTANTS.CONFIG_FILE.exists():
file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
if key in file_config:
- sources_info.append(("Config File", redact_sensitive_config(file_config).get(key), "green"))
+ sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))
# Default value
default_val = find_config_default(key)
diff --git a/archivebox/core/widgets.py b/archivebox/core/widgets.py
index 3c4d16ee..c4c60047 100644
--- a/archivebox/core/widgets.py
+++ b/archivebox/core/widgets.py
@@ -4,6 +4,8 @@ import json
import re
import hashlib
from django import forms
+from django.db.models.manager import BaseManager
+from django.db.models.query import QuerySet
from django.utils.html import escape
from django.utils.safestring import mark_safe
@@ -65,10 +67,12 @@ class TagEditorWidget(forms.Widget):
# Parse value to get list of tag names
tags = []
if value:
- if hasattr(value, "all"): # QuerySet
+ if isinstance(value, (BaseManager, QuerySet)):
tags = sorted([tag.name for tag in value.all()])
elif isinstance(value, (list, tuple)):
- if value and hasattr(value[0], "name"): # List of Tag objects
+ from archivebox.core.models import Tag
+
+ if value and isinstance(value[0], Tag): # List of Tag objects
tags = sorted([tag.name for tag in value])
else: # List of strings or IDs
# Could be tag IDs from form submission
@@ -675,12 +679,14 @@ class InlineTagEditorWidget(TagEditorWidget):
# Parse value to get list of tag dicts with id and name
tag_data = []
if value:
- if hasattr(value, "all"): # QuerySet
+ if isinstance(value, (BaseManager, QuerySet)):
for tag in value.all():
tag_data.append({"id": tag.pk, "name": tag.name})
tag_data.sort(key=lambda x: x["name"].lower())
elif isinstance(value, (list, tuple)):
- if value and hasattr(value[0], "name"):
+ from archivebox.core.models import Tag
+
+ if value and isinstance(value[0], Tag):
for tag in value:
tag_data.append({"id": tag.pk, "name": tag.name})
tag_data.sort(key=lambda x: x["name"].lower())
diff --git a/archivebox/crawls/admin.py b/archivebox/crawls/admin.py
index b0d04b32..37411e88 100644
--- a/archivebox/crawls/admin.py
+++ b/archivebox/crawls/admin.py
@@ -605,9 +605,7 @@ class CrawlAdminForm(forms.ModelForm):
if commit:
instance.save()
instance.apply_crawl_config_filters()
- save_m2m = getattr(self, "_save_m2m", None)
- if callable(save_m2m):
- save_m2m()
+ self._save_m2m()
return instance
@@ -729,7 +727,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
super().__init__(model, admin_site)
self.crawl_admin_base_config = None
self.stop_reason_cache = {}
- self.persona_limit_config_cache = {}
class Media:
css = {"all": ("admin/crawls/crawl_change.css",)}
@@ -739,9 +736,8 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
self.request = request
self.crawl_admin_base_config = request.archivebox_config
self.stop_reason_cache = {}
- self.persona_limit_config_cache = {}
response = super().changelist_view(request, extra_context)
- cl = getattr(response, "context_data", {}).get("cl") if hasattr(response, "context_data") else None
+ cl = response.context_data.get("cl")
if cl is not None and not self.should_annotate_snapshot_counts(request):
self.hydrate_visible_snapshot_counts(cl.result_list)
return response
@@ -790,7 +786,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
self.request = request
self.crawl_admin_base_config = request.archivebox_config
self.stop_reason_cache = {}
- self.persona_limit_config_cache = {}
crawl = self.get_object(request, object_id)
if crawl:
self.hydrate_visible_snapshot_counts([crawl])
@@ -982,7 +977,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
if obj.pk in self.stop_reason_cache:
return self.stop_reason_cache[obj.pk]
- output_dir = obj.output_dir_for_config(self.crawl_admin_base_config)
+ output_dir = obj.output_dir
config = self.limit_config_for_crawl(obj, output_dir)
reason = obj.stop_reason(
config=config,
@@ -994,22 +989,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
return reason
def limit_config_for_crawl(self, obj, output_dir):
- config = {
- "CRAWL_DIR": str(output_dir),
- "CRAWL_MAX_URLS": self.crawl_admin_base_config.CRAWL_MAX_URLS,
- "CRAWL_MAX_SIZE": self.crawl_admin_base_config.CRAWL_MAX_SIZE,
- "CRAWL_TIMEOUT": self.crawl_admin_base_config.CRAWL_TIMEOUT,
- "SNAPSHOT_MAX_SIZE": self.crawl_admin_base_config.SNAPSHOT_MAX_SIZE,
- }
- if obj.persona_id:
- if obj.persona_id not in self.persona_limit_config_cache:
- self.persona_limit_config_cache[obj.persona_id] = {
- key: value for key, value in obj.persona.get_derived_config().items() if key in config
- }
- config.update(self.persona_limit_config_cache[obj.persona_id])
- if obj.config:
- config.update({key: value for key, value in obj.config.items() if key in config})
- return config
+ from archivebox.config.common import get_config
+
+ return get_config(crawl=obj).for_crawl_runtime(
+ crawl=obj,
+ persona=obj.resolve_persona(),
+ crawl_output_dir=output_dir,
+ )
@admin.display(description="Status", ordering="status")
def status_with_stop_reason(self, obj):
@@ -1106,12 +1092,10 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
@admin.display(description="Snapshots")
def snapshots_changelist(self, obj):
- request = getattr(self, "request", None)
+ request = self.request
snapshot_changelist = reverse("admin:core_snapshot_changelist")
scoped_params = {"crawl_id": str(obj.pk)}
full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
- if request is None:
- return format_html('Open snapshots changelist', full_url)
snapshot_admin = self.admin_site._registry[Snapshot]
changelist_request = copy(request)
@@ -1305,7 +1289,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
return self.fieldsets
def save_model(self, request, obj, form, change):
- if not obj.created_by_id and getattr(request, "user", None) and request.user.is_authenticated:
+ if not obj.created_by_id and request.user.is_authenticated:
obj.created_by = request.user
super().save_model(request, obj, form, change)
@@ -1315,14 +1299,14 @@ class CrawlScheduleAdmin(BaseModelAdmin):
@admin.display(description="# Crawls", ordering="crawl_count")
def num_crawls(self, obj):
- count = getattr(obj, "crawl_count", None)
+ count = obj.__dict__.get("crawl_count")
if count is None:
count = obj.crawl_set.count()
return count
@admin.display(description="# Snapshots", ordering="snapshot_count")
def num_snapshots(self, obj):
- count = getattr(obj, "snapshot_count", None)
+ count = obj.__dict__.get("snapshot_count")
if count is None:
count = Snapshot.objects.filter(crawl__schedule=obj).count()
return count
@@ -1338,7 +1322,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
crawl_ids = obj.crawl_set.values_list("pk", flat=True)
return render_snapshots_list(
Snapshot.objects.filter(crawl_id__in=crawl_ids),
- request=getattr(self, "request", None),
+ request=self.request,
prefix="schedule_snapshots",
)
diff --git a/archivebox/crawls/migrations/0016_hydrate_crawl_permissions.py b/archivebox/crawls/migrations/0016_hydrate_crawl_permissions.py
index 66e5dc19..1377faac 100644
--- a/archivebox/crawls/migrations/0016_hydrate_crawl_permissions.py
+++ b/archivebox/crawls/migrations/0016_hydrate_crawl_permissions.py
@@ -1,5 +1,6 @@
import os
import json
+import uuid
from django.db import migrations
from django.db.models import Q
@@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
BATCH_SIZE = 1000
-def legacy_bool(value):
- if value is None:
- return None
- normalized = str(value).strip().lower()
- if normalized in {"1", "true", "yes", "on"}:
- return True
- if normalized in {"0", "false", "no", "off"}:
- return False
- return None
-
-
-def permissions_from_legacy_public_flags(config):
- if str(config.get("PERMISSIONS") or "").strip():
- return None
- public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
- public_index = legacy_bool(config.get("PUBLIC_INDEX"))
- if public_snapshots is False:
- return "private"
- if public_index is False:
- return "unlisted"
- if public_snapshots is True or public_index is True:
- return "public"
- return None
-
-
def normalize_permissions(value, default):
value = str(value or "").strip().lower()
return value if value in VALID_PERMISSIONS else default
@@ -59,22 +35,19 @@ def raw_base_config(apps):
def resolve_permissions(config, default):
+ from archivebox.config.common import permissions_from_legacy_public_flags
+
explicit = str(config.get("PERMISSIONS") or "").strip().lower()
if explicit in VALID_PERMISSIONS:
return explicit
return permissions_from_legacy_public_flags(config) or default
-def model_has_config(model):
- try:
- model._meta.get_field("config")
- except Exception:
- return False
- return True
-
-
def id_values(pk):
- return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
+ if isinstance(pk, uuid.UUID):
+ return str(pk), pk.hex
+ pk_str = str(pk)
+ return pk_str, pk_str.replace("-", "")
def flush_batch(cursor, table_name, batch):
@@ -125,8 +98,6 @@ def _ensure_permissions_column(cursor):
def hydrate_crawl_permissions(apps, schema_editor):
Crawl = apps.get_model("crawls", "Crawl")
- User = apps.get_model("auth", "User")
- user_has_config = model_has_config(User)
base_config = raw_base_config(apps)
default_permissions = resolve_permissions(base_config, "public")
table_name = schema_editor.quote_name(Crawl._meta.db_table)
@@ -135,17 +106,13 @@ def hydrate_crawl_permissions(apps, schema_editor):
batch = []
missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
- for crawl in Crawl.objects.filter(missing_permissions).select_related("persona", "created_by").iterator(chunk_size=BATCH_SIZE):
+ for crawl in Crawl.objects.filter(missing_permissions).select_related("persona").iterator(chunk_size=BATCH_SIZE):
config = dict(crawl.config or {})
resolved = dict(base_config)
if crawl.persona_id:
persona_config = crawl.persona.config or {}
if isinstance(persona_config, dict):
resolved.update(persona_config)
- if user_has_config:
- user_config = crawl.created_by.config or {}
- if isinstance(user_config, dict):
- resolved.update(user_config)
resolved.update(config)
config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
batch.append((crawl.id, config))
diff --git a/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py b/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py
index 05cbb79b..78f5417e 100644
--- a/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py
+++ b/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py
@@ -1,6 +1,5 @@
import json
-from django.conf import settings
from django.db import migrations
@@ -19,26 +18,31 @@ def _flush_updates(Crawl, db_alias, pending):
def freeze_existing_crawl_configs(apps, schema_editor):
from archivebox.config.common import build_crawl_config_snapshot
- from archivebox.personas.models import Persona
+ from archivebox.config.constants import CONSTANTS
+ from archivebox.personas.models import derive_persona_config
+
+ class PersonaConfigSnapshot:
+ def __init__(self, persona):
+ self.name = persona.name
+ self.config = dict(persona.config or {})
+
+ def get_derived_config(self):
+ return derive_persona_config(name=self.name, config=self.config, persona_dir=CONSTANTS.PERSONAS_DIR / self.name)
Crawl = apps.get_model("crawls", "Crawl")
- auth_app_label, auth_model_name = settings.AUTH_USER_MODEL.split(".", 1)
- User = apps.get_model(auth_app_label, auth_model_name)
+ Persona = apps.get_model("personas", "Persona")
db_alias = schema_editor.connection.alias
- rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "created_by_id", "config")
- persona_ids = {persona_id for _, persona_id, _, _ in rows if persona_id}
- user_ids = {user_id for _, _, user_id, _ in rows if user_id}
- personas = {persona.pk: persona for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
- users = {user.pk: user for user in User.objects.using(db_alias).filter(pk__in=user_ids)}
+ rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "config")
+ persona_ids = {persona_id for _, persona_id, _ in rows if persona_id}
+ personas = {persona.pk: PersonaConfigSnapshot(persona) for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
frozen_cache = {}
pending = []
- for crawl_id, persona_id, user_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
+ for crawl_id, persona_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
current_config = dict(current_config or {})
- cache_key = (persona_id, user_id, _config_cache_key(current_config))
+ cache_key = (persona_id, _config_cache_key(current_config))
if cache_key not in frozen_cache:
frozen_cache[cache_key] = build_crawl_config_snapshot(
- user=users.get(user_id),
persona=personas.get(persona_id),
overrides=current_config,
)
diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py
index ac8bb9b3..0e9017e1 100755
--- a/archivebox/crawls/models.py
+++ b/archivebox/crawls/models.py
@@ -109,11 +109,10 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
template = self.template
label = template.label or self.label
persona = template.persona if template.persona_id else None
- user = template.created_by if template.created_by_id else None
return Crawl.objects.create(
urls=template.urls,
- config=build_crawl_config_snapshot(user=user, persona=persona, overrides=self.config or {}),
+ config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
max_depth=template.max_depth,
tags_str=template.tags_str,
persona_id=template.persona_id,
@@ -197,9 +196,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
return f"[...{short_id}] {first_url[:120]}"
def get_delete_after_config_value(self):
- from archivebox.config.common import get_config
+ from archivebox.config.common import resolve_delete_after_config_value
- return get_config(crawl=self).DELETE_AFTER
+ return resolve_delete_after_config_value(self.config)
def pause(self, *, save: bool = True) -> bool:
return super().pause(save=save)
@@ -286,15 +285,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
config = dict(self.config or {})
is_new = self._state.adding or old_crawl is None
persona = self.persona if self.persona_id else None
- user = self.created_by if self.created_by_id else None
if is_new:
from archivebox.config.common import build_crawl_config_snapshot
- config = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
+ config = build_crawl_config_snapshot(persona=persona, overrides=config)
if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
from archivebox.config.common import get_config
- config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, user=user, include_machine=True).PERMISSIONS)
+ config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, include_machine=True).PERMISSIONS)
if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
if raw_concurrency in (None, ""):
@@ -469,11 +467,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
)
return crawl
- def output_dir_for_config(self, runtime_config: Mapping[str, Any] | Any) -> Path:
- """
- Construct output directory: archive/users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id}
- Domain is extracted from the first URL in the crawl.
- """
+ @property
+ def output_dir(self) -> Path:
from archivebox.config import CONSTANTS
from archivebox.core.models import Snapshot
@@ -486,14 +481,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
break
domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"
- users_dir = Path(runtime_config["USERS_DIR"]) if isinstance(runtime_config, Mapping) else runtime_config.USERS_DIR
- return users_dir / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
-
- @property
- def output_dir(self) -> Path:
- from archivebox.config.common import get_config
-
- output_dir = self.output_dir_for_config(get_config(resolve_plugins=False))
+ output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
hyphen_dir = output_dir.with_name(str(uuid.UUID(hex=self.id.hex)))
return output_dir if output_dir.exists() or not hyphen_dir.exists() else hyphen_dir
@@ -783,7 +771,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
def _config_value(config: Mapping[str, Any] | Any, key: str, default: Any = None) -> Any:
if isinstance(config, Mapping):
return config.get(key, default)
- return getattr(config, key, default)
+ return config[key] if key in config else default
@classmethod
def create_scheduler_row(cls, **kwargs) -> "Crawl":
@@ -794,17 +782,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
kwargs.setdefault("created_at", now)
kwargs.setdefault("modified_at", now)
config = normalize_config_json_values(kwargs.get("config") or {})
- user = kwargs.get("created_by")
persona = kwargs.get("persona")
- if user is None and kwargs.get("created_by_id"):
- from django.contrib.auth import get_user_model
-
- user = get_user_model().objects.filter(pk=kwargs["created_by_id"]).first()
if persona is None and kwargs.get("persona_id"):
from archivebox.personas.models import Persona
persona = Persona.objects.filter(pk=kwargs["persona_id"]).first()
- kwargs["config"] = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
+ kwargs["config"] = build_crawl_config_snapshot(persona=persona, overrides=config)
crawl = cls(**kwargs)
if crawl.delete_at is None:
crawl.set_delete_at_from_config()
@@ -820,18 +803,20 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
) -> str:
from abx_dl.limits import CrawlLimitState
+ if output_dir is None:
+ output_dir = self.output_dir
if config is None:
from archivebox.config.common import get_config
- config = get_config(crawl=self, include_machine=False)
- if output_dir is None:
- output_dir = self.output_dir
+ config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
+ crawl=self,
+ persona=self.resolve_persona(),
+ crawl_output_dir=output_dir,
+ )
limits_path = output_dir / ".abx-dl" / "limits.json"
if limits_path.exists():
- config_with_crawl_dir = {**dict(config.items())} if isinstance(config, Mapping) else config
- config_with_crawl_dir["CRAWL_DIR"] = str(output_dir)
- stop_reason = CrawlLimitState.from_config(config_with_crawl_dir).get_stop_reason()
+ stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
if stop_reason:
return stop_reason
@@ -1302,10 +1287,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
from archivebox.machine.models import Binary, Machine
def get_runtime_config():
- config = get_config(crawl=self)
- if persona_runtime_overrides:
- config.update(persona_runtime_overrides)
- return config
+ return get_config(crawl=self).for_crawl_runtime(
+ crawl=self,
+ persona=persona,
+ runtime_overrides=persona_runtime_overrides,
+ )
system_task = self.get_system_task()
if system_task == "archivebox://update":
diff --git a/archivebox/machine/admin.py b/archivebox/machine/admin.py
index 13a25adb..9fc49093 100644
--- a/archivebox/machine/admin.py
+++ b/archivebox/machine/admin.py
@@ -590,9 +590,10 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="ArchiveResult", ordering="archiveresult__plugin")
def archiveresult_link(self, process):
- if not hasattr(process, "archiveresult"):
+ try:
+ ar = process.archiveresult
+ except Process.archiveresult.RelatedObjectDoesNotExist:
return "-"
- ar = process.archiveresult
return format_html(
'{} ← {}',
ar.id,
@@ -602,9 +603,9 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="Snapshot", ordering="archiveresult__snapshot__id")
def snapshot_link(self, process):
- ar = getattr(process, "archiveresult", None)
- snapshot = getattr(ar, "snapshot", None)
- if not snapshot:
+ try:
+ snapshot = process.archiveresult.snapshot
+ except Process.archiveresult.RelatedObjectDoesNotExist:
return "-"
return format_html(
'{}',
@@ -614,10 +615,9 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="Crawl", ordering="archiveresult__snapshot__crawl__id")
def crawl_link(self, process):
- ar = getattr(process, "archiveresult", None)
- snapshot = getattr(ar, "snapshot", None)
- crawl = getattr(snapshot, "crawl", None)
- if not crawl:
+ try:
+ crawl = process.archiveresult.snapshot.crawl
+ except Process.archiveresult.RelatedObjectDoesNotExist:
return "-"
return format_html(
'{}',
@@ -682,7 +682,10 @@ class ProcessAdmin(BaseModelAdmin):
@admin.display(description="Output", ordering="archiveresult__output_size")
def output_summary(self, process):
- output_files = getattr(getattr(process, "archiveresult", None), "output_files", {}) or {}
+ try:
+ output_files = process.archiveresult.output_files or {}
+ except Process.archiveresult.RelatedObjectDoesNotExist:
+ output_files = {}
if isinstance(output_files, str):
try:
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
index a457ecfb..a3b6d164 100755
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@@ -19,6 +19,7 @@ from django.db.models import Q, QuerySet
from django.utils import timezone
from django.utils.functional import cached_property
+from archivebox.config import CONSTANTS
from archivebox.config.common import rprint
from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
from archivebox.workers.models import BaseStateMachine, ModelWithStateMachine
@@ -442,7 +443,7 @@ class NetworkInterface(ModelWithHealthStats):
if refresh or timezone.now() >= _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
updates = ["modified_at"]
for key, value in net_info.items():
- if getattr(_CURRENT_INTERFACE, key) != value:
+ if _CURRENT_INTERFACE.__dict__.get(key) != value:
setattr(_CURRENT_INTERFACE, key, value)
updates.append(key)
if len(updates) > 1:
@@ -581,10 +582,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
Get output directory for this binary's hook logs.
Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
"""
- from archivebox.config.common import get_config
-
- data_dir = get_config().DATA_DIR
- return data_dir / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
+ return CONSTANTS.DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
def to_json(self) -> dict:
"""
@@ -1144,15 +1142,13 @@ class Process(ModelWithDeleteAfter, models.Model):
return f"Process[{self.id}] {cmd_str} ({self.status})"
def get_delete_after_config_value(self):
- from archivebox.config.common import get_config
-
value = self.env.get("DELETE_AFTER")
if value not in (None, ""):
return value
value = (self.machine.config or {}).get("DELETE_AFTER")
if value not in (None, ""):
return value
- return get_config(include_machine=False).DELETE_AFTER
+ return "0"
@classmethod
def missing_delete_at_candidates(cls):
@@ -1174,17 +1170,18 @@ class Process(ModelWithDeleteAfter, models.Model):
@property
def plugin(self) -> str:
"""Get plugin name from associated ArchiveResult (if any)."""
- if hasattr(self, "archiveresult"):
- # Inline import to avoid circular dependency
+ try:
return self.archiveresult.plugin
- return ""
+ except Process.archiveresult.RelatedObjectDoesNotExist:
+ return ""
@property
def hook_name(self) -> str:
"""Get hook name from associated ArchiveResult (if any)."""
- if hasattr(self, "archiveresult"):
+ try:
return self.archiveresult.hook_name
- return ""
+ except Process.archiveresult.RelatedObjectDoesNotExist:
+ return ""
def to_json(self) -> dict:
"""
@@ -2449,16 +2446,14 @@ class Process(ModelWithDeleteAfter, models.Model):
"""
import subprocess
from importlib.resources import files
- from archivebox.config.common import get_config
chrome_utils = files("abx_plugins.plugins.chrome").joinpath("chrome_utils.js")
if not chrome_utils.exists():
return 0
- config = get_config()
crawl_roots = [
crawls_dir
- for user_dir in config.USERS_DIR.iterdir()
+ for user_dir in CONSTANTS.USERS_DIR.iterdir()
if user_dir.is_dir()
for crawls_dir in [user_dir / "crawls"]
if crawls_dir.is_dir()
diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py
index e1d97bb0..0a11bf8c 100644
--- a/archivebox/misc/checks.py
+++ b/archivebox/misc/checks.py
@@ -66,7 +66,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:
from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
config = config or get_config(**config_kwargs)
- archive_dir = config.ARCHIVE_DIR
+ archive_dir = CONSTANTS.ARCHIVE_DIR
archive_dir_exists = os.path.isdir(archive_dir)
if not archive_dir_exists:
print("[red][X] No archivebox index found in the current directory.[/red]", file=sys.stderr)
@@ -82,7 +82,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:
# Create data dir subdirs
create_and_chown_dir(CONSTANTS.SOURCES_DIR)
- create_and_chown_dir(config.USERS_DIR)
+ create_and_chown_dir(CONSTANTS.USERS_DIR)
create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default")
create_and_chown_dir(CONSTANTS.LOGS_DIR)
# create_and_chown_dir(CONSTANTS.CACHE_DIR)
@@ -211,11 +211,10 @@ def check_not_inside_source_dir():
"""Prevent running ArchiveBox from inside its source directory (would pollute repo with data files)."""
cwd = Path(os.getcwd()).resolve()
is_source_dir = (cwd / "archivebox" / "__init__.py").exists() and (cwd / "pyproject.toml").exists()
- data_dir_set_elsewhere = os.environ.get("DATA_DIR", "").strip() and Path(os.environ["DATA_DIR"]).resolve() != cwd
is_testing = "pytest" in sys.modules or "unittest" in sys.modules
- if is_source_dir and not data_dir_set_elsewhere and not is_testing:
- raise SystemExit("[!] Cannot run from source dir, set DATA_DIR or cd to a data folder first")
+ if is_source_dir and not is_testing:
+ raise SystemExit("[!] Cannot run from source dir, cd to a data folder first")
def check_data_dir_permissions(config=None, **config_kwargs):
diff --git a/archivebox/misc/db.py b/archivebox/misc/db.py
index 9288b063..a154d16b 100644
--- a/archivebox/misc/db.py
+++ b/archivebox/misc/db.py
@@ -1,7 +1,7 @@
"""
Database utility functions for ArchiveBox.
-Post-bootstrap: requires archivebox.config (DATA_DIR) and uses Django lazily
+Post-bootstrap: requires archivebox.config constants and uses Django lazily
(``from django.db import ...`` inside functions). Not safe to import pre-bootstrap.
"""
@@ -17,7 +17,7 @@ from collections.abc import Callable
from contextlib import contextmanager
from sqlite3 import OperationalError as SQLiteOperationalError
-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
from archivebox.misc.util import enforce_types
@@ -79,7 +79,7 @@ def run_db_analyze_batch(
return []
next_table, *rest = remaining
- raw_conn = getattr(connection, "connection", None)
+ raw_conn = connection.connection
progress_handler_set = False
if raw_conn is not None and max_seconds_per_table > 0:
deadline = time.monotonic() + max_seconds_per_table
@@ -117,7 +117,7 @@ def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
return " ".join([Path(parts[0]).name, *parts[1:]])[:220]
-def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]:
+def sqlite_lock_holders(db_path: Path = CONSTANTS.DATABASE_FILE) -> list[str]:
import psutil
db_path = db_path.resolve()
@@ -146,7 +146,7 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
return holders
-def log_sqlite_lock_holders(console: Any, *, db_path: Path = DATA_DIR / "index.sqlite3", limit: int = 8) -> None:
+def log_sqlite_lock_holders(console: Any, *, db_path: Path = CONSTANTS.DATABASE_FILE, limit: int = 8) -> None:
holders = sqlite_lock_holders(db_path)
if holders:
console.print("[yellow] DB holders:[/yellow]")
@@ -346,7 +346,7 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
@enforce_types
-def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
+def migration_state(out_dir: Path = CONSTANTS.DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
"""Cheaply compare migration files to django_migrations without invoking migrate."""
from django.apps import apps
from django.db import connection
@@ -373,7 +373,7 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
loader.load_disk()
for (app_label, migration_name), migration in loader.disk_migrations.items():
disk_migrations.add((app_label, migration_name))
- for replaced_app, replaced_name in getattr(migration, "replaces", ()) or ():
+ for replaced_app, replaced_name in migration.replaces or ():
squashed_replaced.add((replaced_app, replaced_name))
applied = {(app, name) for app, name in applied if app in app_labels}
@@ -392,14 +392,19 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
@enforce_types
-def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]:
+def pending_migrations(out_dir: Path = CONSTANTS.DATA_DIR) -> list[str]:
"""Return migration files on disk that have not been applied yet."""
pending, _missing_from_code, _rollback_targets = migration_state(out_dir=out_dir)
return pending
@enforce_types
-def apply_migrations(out_dir: Path = DATA_DIR, stdout: TextIO | None = None, stderr: TextIO | None = None, verbosity: int = 1) -> list[str]:
+def apply_migrations(
+ out_dir: Path = CONSTANTS.DATA_DIR,
+ stdout: TextIO | None = None,
+ stderr: TextIO | None = None,
+ verbosity: int = 1,
+) -> list[str]:
"""Apply pending Django migrations"""
from django.core.management import call_command
diff --git a/archivebox/misc/logging_util.py b/archivebox/misc/logging_util.py
index a05d8266..ac4b2718 100644
--- a/archivebox/misc/logging_util.py
+++ b/archivebox/misc/logging_util.py
@@ -1,7 +1,7 @@
__package__ = "archivebox"
# Post-bootstrap CLI logging helpers (event loggers, progress bars, formatters).
-# Requires archivebox.config to be loaded — imports DATA_DIR, get_config, and
+# Requires archivebox.config to be loaded — imports CONSTANTS/get_config and
# references Django ORM types. For pre-bootstrap logging primitives use
# misc/logging.py, which has no archivebox or Django dependencies.
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
from rich import print
from rich.panel import Panel
-from archivebox.config import DATA_DIR, VERSION
+from archivebox.config import CONSTANTS, VERSION
from archivebox.config.common import get_config
from archivebox.misc.util import enforce_types
from archivebox.misc.logging import ANSI
@@ -171,7 +171,9 @@ def log_list_finished(snapshots):
def log_removal_started(snapshots, yes: bool):
- count = snapshots.count() if hasattr(snapshots, "count") else len(snapshots)
+ from django.db.models import QuerySet
+
+ count = snapshots.count() if isinstance(snapshots, QuerySet) else len(snapshots)
print(f"[yellow3][i] Found {count} matching URLs to remove.[/]")
file_counts = [s.num_outputs for s in snapshots if os.access(s.output_dir, os.R_OK)]
print(
@@ -203,7 +205,7 @@ def log_removal_finished(remaining_links: int, removed_links: int):
@enforce_types
-def pretty_path(path: Path | str, pwd: Path | str = DATA_DIR, color: bool = True) -> str:
+def pretty_path(path: Path | str, pwd: Path | str = CONSTANTS.DATA_DIR, color: bool = True) -> str:
"""convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc"""
pwd = str(Path(pwd)) # .resolve()
path = str(path)
diff --git a/archivebox/misc/paginators.py b/archivebox/misc/paginators.py
index fd12c292..602a1082 100644
--- a/archivebox/misc/paginators.py
+++ b/archivebox/misc/paginators.py
@@ -4,6 +4,7 @@ from django.core.paginator import Paginator
from django.core.paginator import Page
from django.core.paginator import EmptyPage
from django.db import connection
+from django.db.models import QuerySet
from django.utils.functional import cached_property
@@ -19,9 +20,13 @@ class CountlessPage(Page):
class CountlessPaginator(Paginator):
has_exact_count = False
+ def __init__(self, *args, **kwargs):
+ self._count_hint = 0
+ super().__init__(*args, **kwargs)
+
@cached_property
def count(self):
- return getattr(self, "_count_hint", 0)
+ return self._count_hint
@cached_property
def num_pages(self):
@@ -56,16 +61,29 @@ class AcceleratedPaginator(Paginator):
@cached_property
def count(self):
- query = getattr(self.object_list, "query", None)
- if query is not None and (getattr(query, "distinct", False) or getattr(getattr(query, "where", None), "children", None)):
+ if not isinstance(self.object_list, QuerySet):
+ return super().count
+
+ query = self.object_list.query
+ count_hint = self.object_list.__dict__.get("_archivebox_count_hint")
+ if count_hint is None:
+ count_hint = query.__dict__.get("_archivebox_count_hint")
+ if count_hint is not None:
+ model = self.object_list.model
+ if count_hint == "model_estimate":
+ return self._model_count_estimate(model)
+ if callable(count_hint):
+ return count_hint()
+ return count_hint
+
+ if query.distinct or query.where.children:
# fallback to normal count method on filtered queryset
return super().count
- model = getattr(self.object_list, "model", None)
- if model is None:
- return super().count
-
# otherwise count total rows in a separate fast query
+ return self._model_count_estimate(self.object_list.model)
+
+ def _model_count_estimate(self, model):
if connection.vendor == "sqlite":
table_name = model._meta.db_table
with connection.cursor() as cursor:
diff --git a/archivebox/misc/serve_static.py b/archivebox/misc/serve_static.py
index 1ec3ab33..5e5bd166 100644
--- a/archivebox/misc/serve_static.py
+++ b/archivebox/misc/serve_static.py
@@ -2,6 +2,7 @@ import html
import json
import re
import os
+import sys
import stat
import asyncio
import posixpath
@@ -17,6 +18,7 @@ from pathlib import Path
from urllib.parse import urlencode
from django import template
+from django.core.handlers.asgi import ASGIRequest
from django.contrib.staticfiles import finders
from django.template import TemplateDoesNotExist, loader
from django.views import static
@@ -117,7 +119,7 @@ def _render_mhtml_preview_document(filename: str, output_path: str) -> str:
def _format_direntry_timestamp(stat_result: os.stat_result) -> str:
- timestamp = getattr(stat_result, "st_birthtime", None) or stat_result.st_mtime
+ timestamp = stat_result.st_birthtime if sys.platform == "darwin" else stat_result.st_mtime
return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M")
@@ -333,7 +335,7 @@ mimetypes.add_type("multipart/related", ".mhtml")
mimetypes.add_type("multipart/related", ".mht")
try:
- _markdown = getattr(importlib.import_module("markdown"), "markdown")
+ _markdown = importlib.import_module("markdown").markdown
except ImportError:
_markdown: Callable[..., str] | None = None
@@ -716,10 +718,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
"""
assert document_root
- config = getattr(request, "archivebox_config", None)
- if config is None:
- config = get_config(resolve_plugins=False)
- request.archivebox_config = config
+ config = request.archivebox_config
fullpath, path = _resolve_archive_path(document_root, path)
if os.access(fullpath, os.R_OK) and fullpath.is_dir():
if request.GET.get("download") == "zip" and show_indexes:
@@ -727,7 +726,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
fullpath,
path,
is_archive_replay=is_archive_replay,
- use_async_stream=hasattr(request, "scope"),
+ use_async_stream=isinstance(request, ASGIRequest),
config=config,
)
if show_indexes:
@@ -872,7 +871,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
fullpath.name,
raw_output_path,
wacz_path=fullpath,
- fallback_url=getattr(request, "archivebox_snapshot_url", "") or "",
+ fallback_url=request.archivebox_snapshot_url or "",
last_modified=http_date(statobj.st_mtime),
etag=etag or "",
cache_control=(
@@ -978,7 +977,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
# setup response object
ranged_file = RangedFileReader(open(fullpath, "rb"))
response = StreamingHttpResponse(
- _stream_ranged_file_async(ranged_file) if hasattr(request, "scope") else ranged_file,
+ _stream_ranged_file_async(ranged_file) if isinstance(request, ASGIRequest) else ranged_file,
content_type=content_type,
)
response.headers["Last-Modified"] = http_date(statobj.st_mtime)
diff --git a/archivebox/misc/toml_util.py b/archivebox/misc/toml_util.py
index 36b0d111..33daaa40 100644
--- a/archivebox/misc/toml_util.py
+++ b/archivebox/misc/toml_util.py
@@ -96,7 +96,7 @@ class JSONSchemaWithLambdas(GenerateJsonSchema):
def better_toml_dump_str(val: Any) -> str:
try:
- dump_str = cast(Callable[[Any], str], getattr(toml.encoder, "_dump_str"))
+ dump_str = cast(Callable[[Any], str], toml.encoder._dump_str)
return dump_str(val)
except Exception:
# if we hit any of toml's numerous encoding bugs,
diff --git a/archivebox/misc/util.py b/archivebox/misc/util.py
index ac5fa500..7c12b321 100644
--- a/archivebox/misc/util.py
+++ b/archivebox/misc/util.py
@@ -474,7 +474,7 @@ class ExtendedEncoder(pyjson.JSONEncoder):
def default(self, o):
cls_name = o.__class__.__name__
- if hasattr(o, "_asdict"):
+ if isinstance(o, tuple) and "_asdict" in vars(type(o)):
return o._asdict()
elif isinstance(o, bytes):
diff --git a/archivebox/personas/migrations/0004_hydrate_persona_permissions.py b/archivebox/personas/migrations/0004_hydrate_persona_permissions.py
index 7238c678..7b7bd22e 100644
--- a/archivebox/personas/migrations/0004_hydrate_persona_permissions.py
+++ b/archivebox/personas/migrations/0004_hydrate_persona_permissions.py
@@ -1,5 +1,6 @@
import os
import json
+import uuid
from django.db import migrations
from django.db.models import Q
@@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
BATCH_SIZE = 1000
-def legacy_bool(value):
- if value is None:
- return None
- normalized = str(value).strip().lower()
- if normalized in {"1", "true", "yes", "on"}:
- return True
- if normalized in {"0", "false", "no", "off"}:
- return False
- return None
-
-
-def permissions_from_legacy_public_flags(config):
- if str(config.get("PERMISSIONS") or "").strip():
- return None
- public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
- public_index = legacy_bool(config.get("PUBLIC_INDEX"))
- if public_snapshots is False:
- return "private"
- if public_index is False:
- return "unlisted"
- if public_snapshots is True or public_index is True:
- return "public"
- return None
-
-
def normalize_permissions(value, default):
value = str(value or "").strip().lower()
return value if value in VALID_PERMISSIONS else default
@@ -59,22 +35,19 @@ def raw_base_config(apps):
def resolve_permissions(config, default):
+ from archivebox.config.common import permissions_from_legacy_public_flags
+
explicit = str(config.get("PERMISSIONS") or "").strip().lower()
if explicit in VALID_PERMISSIONS:
return explicit
return permissions_from_legacy_public_flags(config) or default
-def model_has_config(model):
- try:
- model._meta.get_field("config")
- except Exception:
- return False
- return True
-
-
def id_values(pk):
- return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
+ if isinstance(pk, uuid.UUID):
+ return str(pk), pk.hex
+ pk_str = str(pk)
+ return pk_str, pk_str.replace("-", "")
def flush_batch(cursor, table_name, batch):
@@ -120,8 +93,6 @@ def _ensure_permissions_column(cursor):
def hydrate_persona_permissions(apps, schema_editor):
Persona = apps.get_model("personas", "Persona")
- User = apps.get_model("auth", "User")
- user_has_config = model_has_config(User)
base_config = raw_base_config(apps)
default_permissions = resolve_permissions(base_config, "public")
table_name = schema_editor.quote_name(Persona._meta.db_table)
@@ -130,13 +101,9 @@ def hydrate_persona_permissions(apps, schema_editor):
batch = []
missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
- for persona in Persona.objects.filter(missing_permissions).select_related("created_by").iterator(chunk_size=BATCH_SIZE):
+ for persona in Persona.objects.filter(missing_permissions).iterator(chunk_size=BATCH_SIZE):
config = dict(persona.config or {})
resolved = dict(base_config)
- if user_has_config:
- user_config = persona.created_by.config or {}
- if isinstance(user_config, dict):
- resolved.update(user_config)
resolved.update(config)
config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
batch.append((persona.id, config))
diff --git a/archivebox/personas/models.py b/archivebox/personas/models.py
index 5b471069..fffca6dc 100644
--- a/archivebox/personas/models.py
+++ b/archivebox/personas/models.py
@@ -16,6 +16,7 @@ import sys
from contextlib import contextmanager
from pathlib import Path
from typing import TYPE_CHECKING, Any
+from collections.abc import Mapping
from django.db import models
from django.db.models.fields.json import KT
@@ -59,6 +60,26 @@ VOLATILE_PROFILE_FILE_NAMES = {
}
+def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, persona_dir: Path) -> dict[str, Any]:
+ derived = dict(config or {})
+
+ if "CHROME_USER_DATA_DIR" not in derived:
+ derived["CHROME_USER_DATA_DIR"] = str(persona_dir / "chrome_profile")
+ if "CHROME_DOWNLOADS_DIR" not in derived:
+ derived["CHROME_DOWNLOADS_DIR"] = str(persona_dir / "chrome_downloads")
+
+ cookies_path = persona_dir / "cookies.txt"
+ if "COOKIES_FILE" not in derived and cookies_path.exists():
+ derived["COOKIES_FILE"] = str(cookies_path)
+
+ auth_path = persona_dir / "auth.json"
+ if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
+ derived["AUTH_STORAGE_FILE"] = str(auth_path)
+
+ derived["ACTIVE_PERSONA"] = name
+ return derived
+
+
class Persona(ModelWithConfig):
"""
Browser persona/profile for archiving sessions.
@@ -99,8 +120,7 @@ class Persona(ModelWithConfig):
if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
from archivebox.config.common import get_config
- user = self.created_by if self.created_by_id else None
- config["PERMISSIONS"] = normalize_permissions(get_config(user=user, include_machine=True).PERMISSIONS)
+ config["PERMISSIONS"] = normalize_permissions(get_config(include_machine=True).PERMISSIONS)
self.config = config
update_fields = kwargs.get("update_fields")
if update_fields is not None:
@@ -151,22 +171,7 @@ class Persona(ModelWithConfig):
- AUTH_STORAGE_FILE (derived from persona path, if file exists)
- ACTIVE_PERSONA (set to this persona's name)
"""
- derived = dict(self.config or {})
-
- # Add derived paths (don't override if explicitly set in config)
- if "CHROME_USER_DATA_DIR" not in derived:
- derived["CHROME_USER_DATA_DIR"] = self.CHROME_USER_DATA_DIR
- if "CHROME_DOWNLOADS_DIR" not in derived:
- derived["CHROME_DOWNLOADS_DIR"] = self.CHROME_DOWNLOADS_DIR
- if "COOKIES_FILE" not in derived and self.COOKIES_FILE:
- derived["COOKIES_FILE"] = self.COOKIES_FILE
- if "AUTH_STORAGE_FILE" not in derived and self.AUTH_STORAGE_FILE:
- derived["AUTH_STORAGE_FILE"] = self.AUTH_STORAGE_FILE
-
- # Always set ACTIVE_PERSONA to this persona's name
- derived["ACTIVE_PERSONA"] = self.name
-
- return derived
+ return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
def ensure_dirs(self) -> None:
"""Create persona directories if they don't exist."""
diff --git a/archivebox/plugins/discovery.py b/archivebox/plugins/discovery.py
index ecc20bfc..46d95484 100644
--- a/archivebox/plugins/discovery.py
+++ b/archivebox/plugins/discovery.py
@@ -1,7 +1,6 @@
__package__ = "archivebox.plugins"
import json
-import os
from collections.abc import Iterable
from functools import lru_cache
from pathlib import Path
@@ -26,9 +25,7 @@ class PluginSpecialConfig(TypedDict):
BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = Path(
- os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR),
-).expanduser()
+USER_PLUGINS_DIR = CONSTANTS.USER_PLUGINS_DIR
def iter_plugin_dirs() -> list[Path]:
@@ -167,7 +164,7 @@ def discover_plugins_that_provide_interface(
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
- if not all(hasattr(module, attr) for attr in required_attrs):
+ if not all(attr in vars(module) for attr in required_attrs):
continue
if plugin_prefix:
diff --git a/archivebox/plugins/forms.py b/archivebox/plugins/forms.py
index 95cae2f1..5f6b662e 100644
--- a/archivebox/plugins/forms.py
+++ b/archivebox/plugins/forms.py
@@ -9,7 +9,7 @@ from typing import Any
from django import forms
from django.utils.html import format_html
-from archivebox.config.common import get_config
+from archivebox.config.common import ArchiveBoxConfig, get_config
from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
@@ -250,6 +250,7 @@ def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> An
class PluginConfigFormMixin:
plugin_groups: list[dict[str, Any]]
+ allow_crawl_execution_config_fields = True
def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
all_plugins = get_plugins()
@@ -314,7 +315,10 @@ class PluginConfigFormMixin:
config_fields = [
self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
for config_key, prop_schema in properties.items()
- if isinstance(prop_schema, dict)
+ if (
+ isinstance(prop_schema, dict)
+ and (self.allow_crawl_execution_config_fields or ArchiveBoxConfig.scope_for_key(str(config_key)) == "crawl_frozen")
+ )
]
cards.append(
{
@@ -427,6 +431,8 @@ class PluginConfigFormMixin:
input_name = _plugin_config_input_name(plugin_name, config_key)
if input_name not in self.data:
continue
+ if not self.allow_crawl_execution_config_fields and ArchiveBoxConfig.scope_for_key(str(config_key)) != "crawl_frozen":
+ continue
raw_value: Any = self.data.get(input_name)
if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
@@ -595,5 +601,6 @@ def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str
)
if binary is None and name != value:
binary = Binary.objects.get_valid_binary(name, machine=machine)
- urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
+ binary_name = binary.name if binary is not None else name
+ urls[key] = get_installed_binary_change_url(binary_name, binary) or get_environment_binary_url(name)
return urls
diff --git a/archivebox/plugins/hooks.py b/archivebox/plugins/hooks.py
index c3118557..efb368d9 100644
--- a/archivebox/plugins/hooks.py
+++ b/archivebox/plugins/hooks.py
@@ -46,7 +46,7 @@ import json
import os
from collections.abc import Mapping
from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard
+from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, runtime_checkable
from archivebox.config.constants import CONSTANTS
from archivebox.config.version import VERSION
@@ -62,12 +62,13 @@ if TYPE_CHECKING:
from archivebox.machine.models import Process
+@runtime_checkable
class ConfigDump(Protocol):
def as_dict(self) -> dict[str, Any]: ...
def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
- return callable(getattr(config, "as_dict", None))
+ return isinstance(config, ConfigDump)
def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
@@ -259,13 +260,13 @@ def run_hook(
This is the low-level hook executor that creates a Process record and
uses Process.launch() for subprocess management.
- Config is passed to hooks via environment variables. Caller MUST use
- get_config() to merge all sources (file, env, machine, crawl, snapshot).
+ Config is passed to hooks via environment variables. Crawl/snapshot callers
+ should pass the runtime config produced by for_crawl_runtime().
Args:
script: Path to the hook script (.sh, .py, or .js)
output_dir: Working directory for the script (where output files go)
- config: Optional pre-merged config dict from get_config(crawl=..., snapshot=...).
+ config: Optional runtime config dict from for_crawl_runtime().
If omitted, pass scope/override args using kwargs prefixed with config_.
timeout: Maximum execution time in seconds
If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
@@ -277,18 +278,22 @@ def run_hook(
Example:
from archivebox.config.common import get_config
- config = get_config(crawl=my_crawl, snapshot=my_snapshot)
+ config = get_config(crawl=my_crawl, snapshot=my_snapshot).for_crawl_runtime(crawl=my_crawl, snapshot=my_snapshot)
process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
if process.status == 'exited':
records = process.get_records() # Get parsed JSONL output
"""
from archivebox.machine.models import Process, Machine, NetworkInterface
- from archivebox.config.common import get_config
+ from archivebox.config.common import get_config, normalize_runtime_config
import sys
config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
- resolved_config = get_config(overrides=_config_to_overrides(config), **config_scope)
- hook_config = resolved_config.for_crawl_execution()
+ config_overrides = _config_to_overrides(config)
+ resolved_config = get_config(overrides=config_overrides, **config_scope)
+ hook_config = normalize_runtime_config(
+ config_overrides if config is not None else resolved_config.for_crawl(),
+ json_safe=False,
+ )
# Auto-detect timeout from plugin config if not explicitly provided
if timeout is None:
@@ -363,31 +368,16 @@ def run_hook(
# Set up environment with base paths
env = os.environ.copy()
- env["DATA_DIR"] = str(resolved_config.DATA_DIR)
- env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR)
- env["ABX_RUNTIME"] = "archivebox"
+ env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
env["LIBRARY_VERSION"] = VERSION
env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
-
- resolved_output_dir = output_dir.resolve()
- snap_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
- crawl_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.CRAWLS_DIR_NAME)
+ snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
+ crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
if snap_dir:
env["SNAP_DIR"] = str(snap_dir)
if crawl_dir:
env["CRAWL_DIR"] = str(crawl_dir)
- crawl_id = kwargs.get("_crawl_id") or kwargs.get("crawl_id")
- if crawl_id:
- try:
- from archivebox.crawls.models import Crawl
-
- crawl = Crawl.objects.filter(id=crawl_id).first()
- if crawl:
- env["CRAWL_DIR"] = str(crawl.output_dir)
- except Exception:
- pass
-
# Export runtime library roots; abx-dl/abxpkg own executable lookup env.
lib_dir = resolved_config.LIB_DIR
if lib_dir:
@@ -423,7 +413,6 @@ def run_hook(
"NODE_MODULES_DIR",
"NODE_MODULE_DIR",
"DATA_DIR",
- "ARCHIVE_DIR",
"MACHINE_ID",
"SNAP_DIR",
"CRAWL_DIR",
diff --git a/archivebox/plugins/views.py b/archivebox/plugins/views.py
index 9ef503d8..fb67d21e 100644
--- a/archivebox/plugins/views.py
+++ b/archivebox/plugins/views.py
@@ -82,7 +82,7 @@ def get_machine_admin_url() -> str | None:
from archivebox.machine.models import Machine
machine = Machine.current()
- return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id}/change/"
+ return machine.admin_change_url or f"/admin/machine/machine/{machine.id}/change/"
except Exception:
return None
diff --git a/archivebox/progressmonitor/views.py b/archivebox/progressmonitor/views.py
index c6a3086d..a73ba265 100644
--- a/archivebox/progressmonitor/views.py
+++ b/archivebox/progressmonitor/views.py
@@ -345,9 +345,7 @@ def live_progress_view(request):
persona_objects_by_id = {}
persona_objects_by_name = {}
persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
- persona_names = {
- str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default") for crawl in active_crawls_list if not crawl["persona_id"]
- }
+ persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
if persona_ids or persona_names:
from archivebox.personas.models import Persona
@@ -813,7 +811,7 @@ def live_progress_view(request):
urls_preview = crawl["urls"][:60] if crawl["urls"] else None
crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
- persona_name = persona_details["name"] if persona_details else str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default")
+ persona_name = persona_details["name"] if persona_details else "Default"
persona_details = persona_details or persona_details_by_name.get(persona_name)
crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
diff --git a/archivebox/search/admin.py b/archivebox/search/admin.py
index e588130f..d7c7c698 100644
--- a/archivebox/search/admin.py
+++ b/archivebox/search/admin.py
@@ -19,8 +19,8 @@ class SearchResultsChangeList(ChangeList):
def __init__(self, request, *args, **kwargs):
"""Capture normalized search mode before Django builds results."""
- self.search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
- self.search_mode_backend = get_search_mode_backend(self.search_mode, config=getattr(request, "archivebox_config", None))
+ self.search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
+ self.search_mode_backend = get_search_mode_backend(self.search_mode, config=request.archivebox_config)
super().__init__(request, *args, **kwargs)
self.embedded_changelist = request.GET.get("_embedded") == "crawl"
@@ -31,7 +31,7 @@ class SearchResultsChangeList(ChangeList):
self.opts.model_name == "snapshot"
and self.query
and self.result_count == 0
- and get_search_mode_base(self.search_mode, config=getattr(request, "archivebox_config", None)) == "deep"
+ and get_search_mode_base(self.search_mode, config=request.archivebox_config) == "deep"
and self.search_mode_backend,
)
@@ -55,13 +55,11 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
def get_default_search_mode(self):
"""Return the default search mode for the current request config."""
- request = getattr(self, "request", None)
- return get_default_search_mode(config=getattr(request, "archivebox_config", None))
+ return get_default_search_mode(config=self.request.archivebox_config)
def get_search_mode_options(self):
"""Return selector options for the current request config."""
- request = getattr(self, "request", None)
- return get_search_mode_options(config=getattr(request, "archivebox_config", None))
+ return get_search_mode_options(config=self.request.archivebox_config)
def get_search_results(self, request, queryset, search_term: str):
"""Apply admin search semantics to a changelist queryset."""
@@ -69,14 +67,14 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
search_term = search_term.strip()
if not search_term:
return super().get_search_results(request, queryset, search_term)
- search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+ search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
cached_ids = get_cached_admin_search_ids(request)
if cached_ids is not None:
return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
return queryset.none(), False
- if get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None)) == "meta":
+ if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":
qs, use_distinct = super().get_search_results(request, queryset, search_term)
return qs, use_distinct
if request.GET.get("_embedded") == "crawl":
@@ -85,7 +83,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
pk__in=query_search_index(
search_term,
search_mode=search_mode,
- config=getattr(request, "archivebox_config", None),
+ config=request.archivebox_config,
).values("pk"),
), False
except Exception as err:
diff --git a/archivebox/search/backends.py b/archivebox/search/backends.py
index c23b4e84..135faad7 100644
--- a/archivebox/search/backends.py
+++ b/archivebox/search/backends.py
@@ -12,16 +12,16 @@ _search_backends_cache: dict | None = None
@contextmanager
def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
- """Temporarily expose resolved config through os.environ for backend code."""
+ """Temporarily expose resolved search config through os.environ for backend code."""
config = config or get_config(**config_kwargs)
updates = {}
for key, value in config.items():
+ if not str(key).startswith("SEARCH_BACKEND_"):
+ continue
if value is None:
continue
if isinstance(value, (str, int, float, bool, os.PathLike)):
updates[str(key)] = str(value)
- updates["DATA_DIR"] = str(config.DATA_DIR)
- updates["SNAP_DIR"] = str(config.USERS_DIR)
previous = {key: os.environ.get(key) for key in updates}
os.environ.update(updates)
try:
diff --git a/archivebox/search/query.py b/archivebox/search/query.py
index 15ce7091..2e01537d 100644
--- a/archivebox/search/query.py
+++ b/archivebox/search/query.py
@@ -248,10 +248,8 @@ def iter_query_search_ids(
backend = backends[backend_name]
try:
with search_backend_env(config=config):
- if hasattr(backend, "iter_search"):
+ if backend_name == "ripgrep":
ids = backend.iter_search(query, search_mode=search_mode_base)
- elif backend_name == "ripgrep":
- ids = backend.search(query, search_mode=search_mode_base)
else:
ids = backend.search(query)
for snapshot_id in ids:
diff --git a/archivebox/search/views.py b/archivebox/search/views.py
index ba868603..0c840ec8 100644
--- a/archivebox/search/views.py
+++ b/archivebox/search/views.py
@@ -101,7 +101,8 @@ def iter_admin_backend_search_ids(iterator, queryset):
def admin_snapshot_search_stream_view(model_admin, request):
"""Stream admin Snapshot search progress and cache matching IDs."""
query = (request.GET.get("q") or "").strip()
- search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+ config = request.archivebox_config
+ search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
if not query:
return StreamingHttpResponse((), content_type="text/plain")
@@ -115,12 +116,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
filter_request.path = target_url.path or request.path
filter_request.path_info = target_url.path or request.path_info
filter_request.GET = target_get
- filter_request.archivebox_config = getattr(request, "archivebox_config", None)
+ filter_request.archivebox_config = config
# Build the same filtered base queryset the changelist uses, but with the
# search params stripped. The stream intersects each wave with this queryset
# before writing IDs into the short-lived cache consumed by the changelist.
- current_request = getattr(model_admin, "request", None)
+ current_request = model_admin.__dict__.get("request")
try:
base_queryset = model_admin.get_changelist_instance(filter_request).queryset
finally:
@@ -150,12 +151,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
nonlocal last_sent
iterator = None
try:
- search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
+ search_mode_base = get_search_mode_base(search_mode, config=config)
iterator = (
iter_admin_meta_search_ids(query, base_queryset)
if search_mode_base == "meta"
else iter_admin_backend_search_ids(
- iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
+ iter_query_search_ids(query, search_mode=search_mode, config=config),
base_queryset,
)
)
diff --git a/archivebox/services/archive_result_service.py b/archivebox/services/archive_result_service.py
index 3a7f9bb5..1a2cf798 100644
--- a/archivebox/services/archive_result_service.py
+++ b/archivebox/services/archive_result_service.py
@@ -5,7 +5,7 @@ import json
from collections import defaultdict
from collections.abc import Iterable
from pathlib import Path
-from typing import Any
+from typing import Any, Protocol, runtime_checkable
from asgiref.sync import sync_to_async
from django.db import IntegrityError
@@ -18,6 +18,11 @@ from abx_dl.services.base import BaseService
from .process_service import parse_event_datetime
+@runtime_checkable
+class ModelDumpable(Protocol):
+ def model_dump(self) -> dict[str, Any]: ...
+
+
def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
output_files: dict[str, dict] = {}
@@ -97,15 +102,8 @@ def _normalize_output_files(raw_output_files: Any) -> dict[str, dict]:
if isinstance(item, str):
normalized[item] = _enrich_metadata(item, {})
continue
- if hasattr(item, "model_dump"):
+ if isinstance(item, ModelDumpable):
item = item.model_dump()
- elif hasattr(item, "path"):
- item = {
- "path": getattr(item, "path", ""),
- "extension": getattr(item, "extension", ""),
- "mimetype": getattr(item, "mimetype", ""),
- "size": getattr(item, "size", 0),
- }
if not isinstance(item, dict):
continue
path = str(item.get("path") or "").strip()
@@ -224,14 +222,18 @@ class ArchiveResultService(BaseService):
snapshot = await Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").afirst()
if snapshot is None:
return
- plugin_dir = Path(snapshot.output_dir) / event.plugin
- output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
process_started = await self.bus.find(
ProcessStartedEvent,
past=True,
future=False,
where=lambda candidate: self.bus.event_is_child_of(event, candidate),
)
+ plugin_dir = (
+ Path(process_started.output_dir)
+ if process_started is not None and process_started.output_dir
+ else Path(snapshot.output_dir) / event.plugin
+ )
+ output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
process = None
if process_started is not None:
started_at = parse_event_datetime(process_started.start_ts)
@@ -292,14 +294,15 @@ class ArchiveResultService(BaseService):
update_fields = []
for field, value in defaults.items():
- if getattr(result, field) != value:
+ if result.__dict__[field] != value:
setattr(result, field, value)
update_fields.append(field)
if update_fields:
await result.asave(update_fields=[*update_fields, "modified_at"])
if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
- next_title = _extract_snapshot_title(str(snapshot.output_dir), event.plugin, result.output_str, snapshot_url=snapshot.url)
+ title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
+ next_title = _extract_snapshot_title(str(plugin_dir.parent), event.plugin, title_output_str, snapshot_url=snapshot.url)
if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):
snapshot.title = next_title
await snapshot.asave(update_fields=["title", "modified_at"])
diff --git a/archivebox/services/machine_service.py b/archivebox/services/machine_service.py
index b91cd92e..f8a51567 100644
--- a/archivebox/services/machine_service.py
+++ b/archivebox/services/machine_service.py
@@ -40,19 +40,18 @@ class MachineService(BaseService):
self.bus.on(MachineEvent, self.on_MachineEvent__save_to_db)
async def on_MachineEvent__save_to_db(self, event: MachineEvent) -> None:
- from archivebox.machine.models import Machine, _sanitize_machine_config
- from archivebox.config.common import get_config
+ from archivebox.machine.models import Machine
if event.config_type != "derived":
return
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
- lib_dir = await sync_to_async(lambda: get_config(include_machine=False).LIB_DIR, thread_sensitive=True)()
- config = dict(machine.config or {})
+ old_config = dict(machine.config or {})
+ config = dict(old_config)
if event.config is not None:
binary_only = _strip_to_binary_keys(event.config)
- config.update(_sanitize_machine_config(binary_only, lib_dir=lib_dir))
+ config.update(binary_only)
elif event.method == "update":
key = event.key.replace("config/", "", 1).strip()
if key and _is_binary_event_key(key):
@@ -64,5 +63,7 @@ class MachineService(BaseService):
else:
return
- machine.config = _sanitize_machine_config(config, lib_dir=lib_dir)
+ if config == old_config:
+ return
+ machine.config = config
await machine.asave(update_fields=["config", "modified_at"])
diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py
index c4a8e8b2..8dcaf39e 100644
--- a/archivebox/services/runner.py
+++ b/archivebox/services/runner.py
@@ -2,15 +2,12 @@ from __future__ import annotations
import asyncio
import contextvars
-import json
import os
import signal
import shutil
-import subprocess
import sys
import threading
import time
-from collections.abc import Mapping
from contextlib import nullcontext
from datetime import timedelta
from functools import lru_cache
@@ -58,10 +55,10 @@ from abxbus import BaseEvent
from abxbus.event_bus import EventBus, get_current_event, in_handler_context
from abxbus.event_handler import EventHandlerAbortedError, EventHandlerCancelledError
-from archivebox.config.common import ArchiveBoxBaseConfig
+from archivebox.config.common import ArchiveBoxBaseConfig, normalize_runtime_config
from archivebox.core.recovery_util import recover_orchestrator_state
from archivebox.misc.db import run_db_analyze_batch
-from archivebox.core.shutdown_util import foreground_shutdown_signals
+from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
@@ -74,6 +71,9 @@ from .snapshot_service import SnapshotService, finalize_completed_snapshot
from .tag_service import TagService
+QUEUED_PLUGIN_RESULT_BATCH_SIZE = 100
+
+
def _bus_name(prefix: str, identifier: str) -> str:
normalized = "".join(ch if ch.isalnum() else "_" for ch in identifier)
return f"{prefix}_{normalized}"
@@ -110,9 +110,9 @@ def _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: st
if snapshot is not None:
label = snapshot.url
else:
- label = (getattr(crawl, "label", "") or "").strip()
+ label = (crawl.label or "").strip()
if not label:
- label = (getattr(crawl, "urls", "") or "").partition("\n")[0].strip() or str(crawl_id)
+ label = (crawl.urls or "").partition("\n")[0].strip() or str(crawl_id)
line.append(_runner_label(label, reserve=prefix_width))
Console(highlight=False).print(line)
@@ -122,14 +122,6 @@ def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str
return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
-def _normalize_runtime_config(config: ArchiveBoxBaseConfig | Mapping[str, Any] | str | None) -> dict[str, Any]:
- from archivebox.config.common import normalize_runtime_config
-
- if isinstance(config, ArchiveBoxBaseConfig):
- return config.for_crawl_execution()
- return normalize_runtime_config(config)
-
-
def _runner_task_context() -> contextvars.Context:
context = contextvars.copy_context()
context.run(EventBus.current_event_context.set, None)
@@ -149,8 +141,9 @@ async def _emit_machine_config(
derived_config: dict[str, Any],
parent_event=None,
) -> None:
- user_config = _normalize_runtime_config(config)
- derived_machine_config = _normalize_runtime_config(derived_config)
+ user_config = normalize_runtime_config(config)
+ user_config["ABX_RUNTIME"] = "archivebox"
+ derived_machine_config = normalize_runtime_config(derived_config)
user_event = MachineEvent(
config=user_config,
config_type="user",
@@ -179,7 +172,6 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
if os.environ.get("PYTEST_CURRENT_TEST") and not allow_under_pytest:
return False
- from archivebox.config import CONSTANTS
from archivebox.machine.models import Machine, Process
from archivebox.workers.supervisord_util import RUNNER_WORKER, get_existing_supervisord_process, get_worker, start_worker
@@ -205,22 +197,7 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
except Exception:
pass
- log_path = CONSTANTS.LOGS_DIR / "errors.log"
- log_path.parent.mkdir(parents=True, exist_ok=True)
- env = os.environ.copy()
- env.setdefault("DATA_DIR", str(CONSTANTS.DATA_DIR))
-
- with log_path.open("a", encoding="utf-8") as log_handle:
- subprocess.Popen(
- [sys.executable, "-m", "archivebox", "run", "--daemon"],
- cwd=str(CONSTANTS.DATA_DIR),
- env=env,
- stdin=subprocess.DEVNULL,
- stdout=log_handle,
- stderr=log_handle,
- start_new_session=True,
- )
- return True
+ return False
class CrawlRunner:
@@ -233,6 +210,7 @@ class CrawlRunner:
process_discovered_snapshots_inline: bool = True,
show_progress: bool = True,
interactive_interrupts: bool = False,
+ config_overrides: dict[str, Any] | None = None,
):
self.crawl = crawl
self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
@@ -249,6 +227,7 @@ class CrawlRunner:
self.process_discovered_snapshots_inline = process_discovered_snapshots_inline
self.show_progress = show_progress
self.interactive_interrupts = interactive_interrupts
+ self.config_overrides = dict(config_overrides or {})
async def ignore_snapshot(_snapshot_id: str) -> None:
return None
@@ -265,7 +244,7 @@ class CrawlRunner:
self.snapshot_semaphore = asyncio.Semaphore(1)
self.max_concurrent_snapshots = 1
self.persona = None
- self.base_config: dict[str, Any] = {}
+ self.base_config: ArchiveBoxBaseConfig | dict[str, Any] = {}
self.derived_config: dict[str, Any] = {}
self.primary_url = ""
self.crawl_output_dir = ""
@@ -282,6 +261,8 @@ class CrawlRunner:
self._last_lease_heartbeat_at = 0.0
def _request_abort_from_signal(self, _sig: signal.Signals) -> None:
+ if os.environ.get("ARCHIVEBOX_RUNNER_DAEMON") == "1":
+ os._exit(128 + int(_sig))
already_requested = self._signal_abort_requested
self._signal_abort_requested = True
self._skip_wait_until_idle = True
@@ -352,16 +333,19 @@ class CrawlRunner:
if self.interactive_interrupts
else "\n[🛑] Got {signal_name}, stopping gracefully...\n"
)
- # interactive_interrupts is only enabled when this runner belongs
- # to a foreground `archivebox add`. Runners owned by server/update/
- # run should use immediate graceful shutdown instead of the
- # add-specific "abort current hook or continue" flow.
+ self._run_task = asyncio.current_task()
+ # Do not raise KeyboardInterrupt directly from an OS signal while
+ # the asyncio loop is active. Python can inject it into whichever
+ # task is currently running, which produces noisy "Task exception
+ # was never retrieved" logs from unrelated abxbus housekeeping
+ # tasks. _request_abort_from_signal() cancels the runner task
+ # cooperatively instead; repeated signals still hard-exit in the
+ # shared foreground signal handler.
with foreground_shutdown_signals(
first_signal_message=first_signal_message,
on_signal=self._request_abort_from_signal,
- raise_on_first_signal=not self.interactive_interrupts,
+ raise_on_first_signal=False,
):
- self._run_task = asyncio.current_task()
snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
max_concurrent_snapshots = max(1, int(self.base_config.get("CRAWL_MAX_CONCURRENT_SNAPSHOTS", 1)))
self.max_concurrent_snapshots = max_concurrent_snapshots
@@ -372,10 +356,7 @@ class CrawlRunner:
await heartbeat.start()
await _emit_machine_config(
self.bus,
- config={
- **self.base_config,
- "ABX_RUNTIME": "archivebox",
- },
+ config=self.base_config,
derived_config=self.derived_config,
)
if snapshot_ids:
@@ -386,8 +367,9 @@ class CrawlRunner:
await heartbeat.stop()
await self.stop_snapshot_tasks()
try:
- if not self._skip_wait_until_idle:
- await self.bus.wait_until_idle(timeout=30.0)
+ await self.bus.wait_until_idle(timeout=1.0 if self._skip_wait_until_idle else 30.0)
+ except TimeoutError:
+ pass
finally:
await self.bus.destroy(clear=False)
bus_destroyed = True
@@ -548,7 +530,7 @@ class CrawlRunner:
return
await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
- config = await sync_to_async(lambda: get_config(crawl=self.crawl, include_machine=False), thread_sensitive=True)()
+ config = await sync_to_async(lambda: get_config(crawl=self.crawl), thread_sensitive=True)()
self.max_concurrent_snapshots = max(1, int(config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]))
active_snapshot_ids = [snapshot_id for snapshot_id, task in self.snapshot_tasks.items() if not task.done()]
@@ -573,20 +555,27 @@ class CrawlRunner:
from archivebox.config.common import get_config
from archivebox.core.models import Snapshot
from archivebox.plugins.hooks import discover_hooks
- from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config
+ from archivebox.machine.models import Machine, NetworkInterface, Process
self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
- current_iface = NetworkInterface.current(refresh=True)
+ current_iface = NetworkInterface.current(refresh=not self.allow_maintenance_on_inactive_crawl)
current_process = Process.current()
if current_process.iface_id != current_iface.id or current_process.machine_id != current_iface.machine_id:
current_process.iface = current_iface
current_process.machine = current_iface.machine
current_process.save(update_fields=["iface", "machine", "modified_at"])
self.persona = self.crawl.resolve_persona()
- self.base_config = get_config(crawl=self.crawl, include_machine=False)
- self.derived_config = _sanitize_machine_config(Machine.current().config, lib_dir=self.base_config["LIB_DIR"])
+ self.base_config = get_config(crawl=self.crawl)
+ self.derived_config = dict(Machine.current().config or {})
self.crawl_output_dir = str(self.crawl.output_dir)
- self.base_config["ABX_RUNTIME"] = "archivebox"
+ if self.persona:
+ self.base_config.update(
+ self.persona.prepare_runtime_for_crawl(
+ self.crawl,
+ chrome_binary=self.base_config["CHROME_BINARY"],
+ ),
+ )
+ self.base_config.update(self.config_overrides)
if self.selected_plugins is None:
raw_plugins = str(self.base_config.get("PLUGINS") or "").strip()
if raw_plugins:
@@ -597,13 +586,6 @@ class CrawlRunner:
hook.parent.name for event_name in runtime_events for hook in discover_hooks(event_name, config=self.base_config)
}
self.selected_plugins = sorted(runtime_plugins) or None
- if self.persona:
- self.base_config.update(
- self.persona.prepare_runtime_for_crawl(
- self.crawl,
- chrome_binary=self.base_config["CHROME_BINARY"],
- ),
- )
if self.initial_snapshot_ids:
# Direct snapshot maintenance paths are allowed to name paused
# snapshots explicitly. The runner still requires selected_plugins
@@ -712,31 +694,32 @@ class CrawlRunner:
return live_ui
def load_snapshot_payload(self, snapshot_id: str) -> dict[str, Any]:
- from archivebox.core.models import Snapshot
from archivebox.config.common import get_config
+ from archivebox.core.models import Snapshot
- snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+ snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").get(id=snapshot_id)
+ self.crawl = snapshot.crawl
+ self.persona = snapshot.crawl.resolve_persona()
+ self.base_config = get_config(crawl=snapshot.crawl)
+ if self.persona:
+ self.base_config.update(
+ self.persona.prepare_runtime_for_crawl(
+ snapshot.crawl,
+ chrome_binary=self.base_config["CHROME_BINARY"],
+ ),
+ )
+ self.base_config.update(self.config_overrides)
+ self.crawl_output_dir = str(snapshot.crawl.output_dir)
runtime_chrome_overrides = {
key: self.base_config[key] for key in ("CHROME_USER_DATA_DIR", "CHROME_DOWNLOADS_DIR") if self.base_config.get(key)
}
- config = get_config(
+ config = self.base_config.for_crawl_runtime(
crawl=snapshot.crawl,
snapshot=snapshot,
- base_config=self.base_config or None,
- overrides=runtime_chrome_overrides,
- include_machine=False,
+ persona=self.persona,
+ runtime_overrides=runtime_chrome_overrides,
+ extra_context={"snapshot_id": str(snapshot.id), "snapshot_depth": snapshot.depth},
)
- config["CRAWL_DIR"] = self.crawl_output_dir
- config["SNAP_DIR"] = str(snapshot.output_dir)
- extra_context: dict[str, Any] = {}
- if config.get("EXTRA_CONTEXT"):
- parsed_extra_context = json.loads(str(config["EXTRA_CONTEXT"]))
- if not isinstance(parsed_extra_context, dict):
- raise TypeError("EXTRA_CONTEXT must decode to an object")
- extra_context = parsed_extra_context
- extra_context["snapshot_id"] = str(snapshot.id)
- extra_context["snapshot_depth"] = snapshot.depth
- config["EXTRA_CONTEXT"] = json.dumps(extra_context, separators=(",", ":"), sort_keys=True)
return {
"id": str(snapshot.id),
"url": snapshot.url,
@@ -748,7 +731,7 @@ class CrawlRunner:
"depth": snapshot.depth,
"status": snapshot.status,
"output_dir": str(snapshot.output_dir),
- "config": _normalize_runtime_config(config),
+ "config": normalize_runtime_config(config),
"_snapshot": snapshot,
}
@@ -774,9 +757,10 @@ class CrawlRunner:
if parent_snapshot is None:
return
config = await sync_to_async(
- lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot, include_machine=False),
+ lambda: get_config(crawl=self.crawl, snapshot=parent_snapshot),
thread_sensitive=True,
)()
+ config = config.for_crawl_runtime(crawl=self.crawl, snapshot=parent_snapshot, persona=self.crawl.resolve_persona())
if CrawlLimitState.from_config(config).get_stop_reason() in ("crawl_max_size", "crawl_timeout"):
return
@@ -790,8 +774,8 @@ class CrawlRunner:
async def run_crawl(self, root_snapshot_id: str, snapshot_ids: list[str]) -> None:
snapshot = await sync_to_async(self.load_snapshot_payload, thread_sensitive=True)(root_snapshot_id)
- config = _normalize_runtime_config(snapshot["config"])
- derived_config = _normalize_runtime_config(self.derived_config)
+ config = normalize_runtime_config(snapshot["config"])
+ derived_config = normalize_runtime_config(self.derived_config)
output_dir = Path(self.crawl_output_dir)
plugins = self.runtime_plugins()
abx_snapshot = AbxSnapshot(
@@ -1033,8 +1017,8 @@ class CrawlRunner:
):
await sync_to_async(self.seal_snapshot_due_to_limit, thread_sensitive=True)(snapshot_id)
return
- config = _normalize_runtime_config(snapshot["config"])
- derived_config = _normalize_runtime_config(self.derived_config)
+ config = normalize_runtime_config(snapshot["config"])
+ derived_config = normalize_runtime_config(self.derived_config)
output_dir = Path(snapshot["output_dir"])
plugins = (
filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
@@ -1087,9 +1071,14 @@ class CrawlRunner:
# runner is the scheduler owner. Finalize idempotently here too
# so a completed snapshot cannot remain STARTED if the event was
# observed before its DB projector advanced the state machine.
- await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(snapshot_id)
+ crawl_limit_stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
+ await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
+ snapshot_id,
+ output_dir=output_dir,
+ crawl_limit_stop_reason=crawl_limit_stop_reason,
+ )
if snapshot["status"] == "sealed":
- await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id)
+ await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
return
await self.enqueue_discovered_snapshots_from_outputs(snapshot)
await sync_to_async(
@@ -1109,7 +1098,7 @@ class CrawlRunner:
def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
from archivebox.core.models import Snapshot
- snapshot = Snapshot.objects.filter(id=snapshot_id).first()
+ snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
if snapshot is None or snapshot.status == Snapshot.StatusChoices.SEALED:
return
if snapshot.status == Snapshot.StatusChoices.STARTED:
@@ -1129,6 +1118,7 @@ def run_crawl(
process_discovered_snapshots_inline: bool = True,
show_progress: bool = True,
interactive_interrupts: bool = False,
+ config_overrides: dict[str, Any] | None = None,
) -> None:
from archivebox.crawls.models import Crawl
from django.db import close_old_connections
@@ -1145,6 +1135,7 @@ def run_crawl(
process_discovered_snapshots_inline=process_discovered_snapshots_inline,
show_progress=show_progress,
interactive_interrupts=interactive_interrupts,
+ config_overrides=config_overrides,
).run(),
)
finally:
@@ -1171,15 +1162,15 @@ def run_crawl(
async def _run_binary(binary_id: str) -> None:
from archivebox.config.common import get_config
- from archivebox.machine.models import Binary, Machine, _sanitize_machine_config
+ from archivebox.machine.models import Binary, Machine
binary = await Binary.objects.aget(id=binary_id)
plugins = discover_plugins()
config = get_config(include_machine=False)
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
- derived_config = _normalize_runtime_config(_sanitize_machine_config(machine.config, lib_dir=config["LIB_DIR"]))
- config["ABX_RUNTIME"] = "archivebox"
- config = _normalize_runtime_config(config)
+ derived_config = normalize_runtime_config(machine.config)
+ config = config.for_crawl()
+ config = normalize_runtime_config(config)
bus = create_bus(name=_bus_name("ArchiveBox_binary", str(binary.id)), total_timeout=1800.0)
process_service = PersistedProcessService(bus)
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
@@ -1273,10 +1264,10 @@ def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
return None
-def run_snapshot_maintenance(snapshot_id: str) -> bool:
+def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None) -> bool:
from archivebox.core.models import ArchiveResult, Snapshot
- snapshot = Snapshot.objects.filter(id=snapshot_id).first()
+ snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
if snapshot is None:
return False
@@ -1290,9 +1281,23 @@ def run_snapshot_maintenance(snapshot_id: str) -> bool:
# branch can process those targeted plugin rows on the next tick
# This avoids reopening final/paused snapshots while also avoiding stranded
# queued ArchiveResults that have no independent scheduler.
- snapshot.retry_at = timezone.now() if has_queued_results else None
- snapshot.save(update_fields=["retry_at", "modified_at"])
- snapshot.write_index_jsonl()
+ current_retry_at = snapshot.retry_at
+ next_retry_at = timezone.now() if has_queued_results else None
+ snapshot.retry_at = next_retry_at
+ if snapshot.fs_migration_needed:
+ snapshot.save(update_fields=["retry_at", "modified_at"])
+ else:
+ updated = snapshot.safe_update(
+ {"retry_at": next_retry_at},
+ refresh=False,
+ extra_filter={
+ "status": snapshot.StatusChoices.SEALED,
+ "retry_at": current_retry_at,
+ },
+ )
+ if not updated:
+ return False
+ snapshot.write_index_jsonl(output_dir=output_dir)
return True
@@ -1395,8 +1400,6 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo
snapshot = Snapshot.objects.get(pk=snapshot.pk)
except Snapshot.DoesNotExist:
return False
- if runtime_config is not None:
- snapshot._runtime_config = runtime_config
parent_reconciled = snapshot.reconcile_parent_lifecycle(lock_seconds=lock_seconds)
if parent_reconciled is not None:
return parent_reconciled
@@ -1547,14 +1550,14 @@ def run_due_binary(binary, *, lock_seconds: int) -> bool:
async def _run_install(plugin_names: list[str] | None = None) -> None:
from archivebox.config.common import get_config
- from archivebox.machine.models import Machine, _sanitize_machine_config
+ from archivebox.machine.models import Machine
plugins = discover_plugins()
config = get_config(include_machine=False)
machine = await sync_to_async(Machine.current, thread_sensitive=True)()
- derived_config = _normalize_runtime_config(_sanitize_machine_config(machine.config, lib_dir=config["LIB_DIR"]))
- config["ABX_RUNTIME"] = "archivebox"
- config = _normalize_runtime_config(config)
+ derived_config = normalize_runtime_config(machine.config)
+ config = config.for_crawl()
+ config = normalize_runtime_config(config)
bus = create_bus(name="ArchiveBox_install", total_timeout=3600.0)
PersistedProcessService(bus)
BinaryCacheService(bus, backend=ArchiveBoxDBBinaryCacheBackend())
@@ -1721,24 +1724,84 @@ def _run_due_queued_plugin_result(
runtime_config,
) -> bool:
from archivebox.core.models import ArchiveResult, Snapshot
+ from django.db.models import Exists, OuterRef
if not plugin_names:
return False
- queued_results = ArchiveResult.objects.filter(
+ queued_snapshot_ids = ArchiveResult.objects.filter(
status=ArchiveResult.StatusChoices.QUEUED,
plugin__in=plugin_names,
- snapshot__status=Snapshot.StatusChoices.SEALED,
- snapshot__retry_at__lte=timezone.now(),
+ ).values("snapshot_id")
+ due_snapshots = Snapshot.objects.filter(
+ id__in=queued_snapshot_ids,
+ retry_at__lte=timezone.now(),
+ status=Snapshot.StatusChoices.SEALED,
)
if crawl_id:
- queued_results = queued_results.filter(snapshot__crawl_id=crawl_id)
- due_snapshot_id = queued_results.order_by("snapshot__retry_at", "snapshot__created_at").values_list("snapshot_id", flat=True).first()
- return _run_due_snapshot_id(
- due_snapshot_id,
- lock_seconds=lock_seconds,
- interactive_interrupts=interactive_interrupts,
- runtime_config=runtime_config,
+ due_snapshots = due_snapshots.filter(crawl_id=crawl_id)
+ due_snapshots = due_snapshots.only("id", "crawl_id", "retry_at", "status").order_by("retry_at", "created_at")
+ first_due_snapshot = due_snapshots.first()
+ if first_due_snapshot is None:
+ return False
+ root_crawl_id = str(first_due_snapshot.crawl_id)
+ batch_candidates = list(
+ due_snapshots.filter(crawl_id=root_crawl_id)[:QUEUED_PLUGIN_RESULT_BATCH_SIZE],
)
+ if not batch_candidates:
+ return False
+
+ selected_plugins: list[str] | None = None
+ claimed_snapshot_ids: list[str] = []
+ for snapshot in batch_candidates:
+ snapshot_selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
+ if not snapshot_selected_plugins:
+ continue
+ if selected_plugins is None:
+ selected_plugins = snapshot_selected_plugins
+ if snapshot_selected_plugins != selected_plugins:
+ continue
+ if not Snapshot.claim_for_worker(snapshot, lock_seconds=lock_seconds):
+ continue
+ snapshot.refresh_from_db()
+ snapshot.finalize_completed_upload_results()
+ if snapshot.fs_migration_needed:
+ run_snapshot_maintenance(str(snapshot.id))
+ snapshot.refresh_from_db()
+ if snapshot.status != Snapshot.StatusChoices.SEALED:
+ continue
+ claimed_snapshot_ids.append(str(snapshot.id))
+ _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
+
+ if not claimed_snapshot_ids or selected_plugins is None:
+ return True
+
+ run_crawl(
+ root_crawl_id,
+ snapshot_ids=claimed_snapshot_ids,
+ selected_plugins=selected_plugins,
+ process_discovered_snapshots_inline=True,
+ interactive_interrupts=interactive_interrupts,
+ config_overrides={
+ "CRAWL_MAX_CONCURRENT_SNAPSHOTS": QUEUED_PLUGIN_RESULT_BATCH_SIZE,
+ },
+ )
+ if all(plugin.startswith("search_backend_") for plugin in selected_plugins):
+ queued_results = ArchiveResult.objects.filter(
+ snapshot_id=OuterRef("pk"),
+ status=ArchiveResult.StatusChoices.QUEUED,
+ )
+ Snapshot.objects.filter(
+ id__in=claimed_snapshot_ids,
+ status=Snapshot.StatusChoices.SEALED,
+ ).annotate(
+ has_queued_results=Exists(queued_results),
+ ).filter(
+ has_queued_results=False,
+ ).update(
+ retry_at=None,
+ modified_at=timezone.now(),
+ )
+ return True
def _run_due_binary() -> bool:
@@ -1836,6 +1899,7 @@ def run_pending_crawls(
analyze_sweep_started_at = 0.0
orchestrator_started_at = time.monotonic()
while True:
+ raise_if_shutdown_requested()
now_monotonic = time.monotonic()
if now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
for model in (ArchiveResult, Snapshot, Crawl, Process):
@@ -1867,29 +1931,6 @@ def run_pending_crawls(
if _fast_forward_same_path_snapshot_fs_versions():
continue
- # Final-state snapshot maintenance comes before normal crawl work:
- # filesystem/index maintenance and upload finalization should drain
- # promptly, but pure search backend backfills are deferred below so
- # they do not starve live crawls.
- sealed_snapshots = Snapshot.objects.filter(
- retry_at__lte=timezone.now(),
- status=Snapshot.StatusChoices.SEALED,
- )
- if search_plugin_names:
- sealed_snapshots = sealed_snapshots.exclude(
- archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
- archiveresult__plugin__in=search_plugin_names,
- )
- if crawl_id:
- sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)
- if _run_due_snapshot_query(
- sealed_snapshots,
- lock_seconds=60,
- interactive_interrupts=interactive_interrupts,
- runtime_config=runtime_config,
- ):
- continue
-
if not maintenance_only:
active_snapshots = Snapshot.objects.filter(
retry_at__lte=timezone.now(),
@@ -1987,6 +2028,32 @@ def run_pending_crawls(
):
continue
+ # Broad final-state maintenance is intentionally a fallback. Specific
+ # queued plugin work above can use ArchiveResult's scheduler indexes;
+ # this branch may need to prove that no due sealed snapshot remains, so
+ # avoid paying that scan while targeted work is already available.
+ sealed_snapshots = Snapshot.objects.filter(
+ retry_at__lte=timezone.now(),
+ status=Snapshot.StatusChoices.SEALED,
+ )
+ if search_plugin_names:
+ queued_search_snapshot_ids = ArchiveResult.objects.filter(
+ status=ArchiveResult.StatusChoices.QUEUED,
+ plugin__in=search_plugin_names,
+ ).values("snapshot_id")
+ sealed_snapshots = sealed_snapshots.exclude(
+ id__in=queued_search_snapshot_ids,
+ )
+ if crawl_id:
+ sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)
+ if _run_due_snapshot_query(
+ sealed_snapshots,
+ lock_seconds=60,
+ interactive_interrupts=interactive_interrupts,
+ runtime_config=runtime_config,
+ ):
+ continue
+
if not maintenance_only:
if _run_due_crawl_status(
Crawl.StatusChoices.SEALED,
diff --git a/archivebox/services/snapshot_service.py b/archivebox/services/snapshot_service.py
index a548adb7..8ae63c82 100644
--- a/archivebox/services/snapshot_service.py
+++ b/archivebox/services/snapshot_service.py
@@ -11,7 +11,12 @@ from abx_dl.limits import CrawlLimitState
from abx_dl.services.base import BaseService
-def finalize_completed_snapshot(snapshot_id: str) -> None:
+def finalize_completed_snapshot(
+ snapshot_id: str,
+ *,
+ output_dir=None,
+ crawl_limit_stop_reason: str | None = None,
+) -> None:
from archivebox.core.models import Snapshot
snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
@@ -22,7 +27,7 @@ def finalize_completed_snapshot(snapshot_id: str) -> None:
snapshot.downloaded_at = timezone.now()
snapshot.save(update_fields=["downloaded_at", "modified_at"])
- stop_reason = _crawl_limit_stop_reason(snapshot.crawl)
+ stop_reason = crawl_limit_stop_reason if crawl_limit_stop_reason is not None else _crawl_limit_stop_reason(snapshot.crawl)
if snapshot.crawl_id and stop_reason in ("crawl_max_size", "crawl_timeout"):
Snapshot.objects.filter(
crawl_id=snapshot.crawl_id,
@@ -40,14 +45,17 @@ def finalize_completed_snapshot(snapshot_id: str) -> None:
snapshot.sm.seal()
snapshot.refresh_from_db()
- snapshot.write_index_jsonl()
+ snapshot.write_index_jsonl(output_dir=output_dir)
def _crawl_limit_stop_reason(crawl) -> str:
from archivebox.config.common import get_config
- config = get_config(crawl=crawl, include_machine=False)
- config["CRAWL_DIR"] = str(crawl.output_dir)
+ config_model = get_config(crawl=crawl)
+ config = config_model.for_crawl_runtime(
+ crawl=crawl,
+ persona=crawl.resolve_persona(),
+ )
return CrawlLimitState.from_config(config).get_stop_reason()
diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html
index 6f54dcdb..086fc5d6 100644
--- a/archivebox/templates/core/add.html
+++ b/archivebox/templates/core/add.html
@@ -304,7 +304,7 @@
{{ form.config.errors }}
{% endif %}
- Override any config option for this crawl (e.g., TIMEOUT, USER_AGENT, CHROME_BINARY, etc.). URL_ALLOWLIST, URL_DENYLIST, and PLUGINS are updated automatically from the fields above.
+ Override crawl-scoped config options (e.g., TIMEOUT, USER_AGENT, URL_ALLOWLIST, URL_DENYLIST, PLUGINS). *_BINARY paths are managed on Persona or Machine config.