diff --git a/archivebox/cli/archivebox_archiveresult.py b/archivebox/cli/archivebox_archiveresult.py index 545f7157..2cabd92c 100644 --- a/archivebox/cli/archivebox_archiveresult.py +++ b/archivebox/cli/archivebox_archiveresult.py @@ -152,7 +152,7 @@ def create_archiveresults( config = get_config(crawl=snapshot.crawl, snapshot=snapshot) hooks = discover_hooks("Snapshot", config=config) for hook_path in hooks: - hook_name = hook_path.name + hook_name = hook_path.stem plugin_name = hook_path.parent.name if not is_tty: write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status)) diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index 32820d02..8a40b438 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -685,7 +685,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): ), ) persona = forms.ModelChoiceField( - label="Persona (authentication profile)", + label="Persona (configuration profile)", required=False, queryset=Persona.objects.none(), empty_label=None, diff --git a/archivebox/core/models.py b/archivebox/core/models.py index 8702ae88..974046c3 100755 --- a/archivebox/core/models.py +++ b/archivebox/core/models.py @@ -2565,7 +2565,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW archiveresults = [] for hook_path in hooks: - hook_name = hook_path.name # e.g., 'on_Snapshot__50_wget.py' + hook_name = hook_path.stem # e.g., 'on_Snapshot__50_wget' plugin = hook_path.parent.name # e.g., 'wget' # ArchiveResult output is one filesystem directory per plugin hook, so diff --git a/archivebox/core/views.py b/archivebox/core/views.py index 4b0c54ef..e2e809bc 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -1132,6 +1132,7 @@ class AddView(UserPassesTestMixin, FormView): "effective_config": effective_config_json, "binary_urls": binary_urls, } + recent_personas = list(persona_queryset.order_by("-created_at", "name")[:5]) plugin_dependency_map = {} if can_override_crawl_config: plugin_dependency_map = { @@ -1153,6 +1154,7 @@ class AddView(UserPassesTestMixin, FormView): "required_search_plugin": required_search_plugin, "plugin_dependency_map_json": json.dumps(plugin_dependency_map, sort_keys=True), "persona_config_map_json": json.dumps(persona_config_map, sort_keys=True, default=str), + "recent_personas": recent_personas, "can_override_crawl_config": can_override_crawl_config, "stdout": "", } @@ -1621,22 +1623,20 @@ def live_progress_view(request): snapshots_paused = snapshot_status_counts.get(Snapshot.StatusChoices.PAUSED, 0) download_plugin_names, indexing_plugin_names = _live_progress_plugin_names() - archiveresult_status_counts: dict[str, int] = {} - download_status_counts: dict[str, int] = {} - indexing_status_counts: dict[str, int] = {} - for status in ( + result_statuses = ( ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.STARTED, ArchiveResult.StatusChoices.PAUSED, - ): - for row in archiveresult_scope.filter(status=status).values("plugin").annotate(count=Count("id")).order_by(): - plugin = row["plugin"] - count = row["count"] - archiveresult_status_counts[status] = archiveresult_status_counts.get(status, 0) + count - if plugin in indexing_plugin_names: - indexing_status_counts[status] = indexing_status_counts.get(status, 0) + count - elif plugin in download_plugin_names: - download_status_counts[status] = download_status_counts.get(status, 0) + count + ) + archiveresult_status_counts = count_statuses(archiveresult_scope, result_statuses) + download_scope = archiveresult_scope.filter( + plugin__in=download_plugin_names, + snapshot__status__in=(Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED), + snapshot__crawl__status__in=(Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED), + ) + indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names) + download_status_counts = count_statuses(download_scope, result_statuses) + indexing_status_counts = count_statuses(indexing_scope, result_statuses) archiveresults_pending = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0) archiveresults_started = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0) archiveresults_paused = archiveresult_status_counts.get(ArchiveResult.StatusChoices.PAUSED, 0) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index cb7c851a..d8b014dc 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -13,6 +13,7 @@ import time from collections.abc import Mapping from contextlib import nullcontext from datetime import timedelta +from functools import lru_cache from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -1185,19 +1186,43 @@ def run_binary(binary_id: str) -> None: asyncio.run(_run_binary(binary_id)) +@lru_cache(maxsize=1) +def _snapshot_hook_names_by_plugin() -> dict[str, frozenset[str]]: + return {plugin.name: frozenset(hook.name for hook in plugin.filter_hooks("Snapshot")) for plugin in discover_plugins().values()} + + def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None: from archivebox.core.models import ArchiveResult - queued_plugins = sorted( - set( - ArchiveResult.objects.filter( - snapshot_id=snapshot_id, - status=ArchiveResult.StatusChoices.QUEUED, - ) - .exclude(plugin="") - .values_list("plugin", flat=True), - ), + queued_results = list( + ArchiveResult.objects.filter( + snapshot_id=snapshot_id, + status=ArchiveResult.StatusChoices.QUEUED, + ) + .exclude(plugin="") + .only("id", "plugin", "hook_name"), ) + hooks_by_plugin = _snapshot_hook_names_by_plugin() + obsolete_result_ids = [ + result.id + for result in queued_results + if result.hook_name and result.hook_name not in hooks_by_plugin.get(result.plugin, frozenset()) + ] + if obsolete_result_ids: + # Hook names are the scheduler identity for ArchiveResults. If an old + # queued row names a hook that the current plugin model cannot run, mark + # only that row final so the Snapshot scheduler can drain normally instead + # of re-running the whole plugin forever. + ArchiveResult.objects.filter( + id__in=obsolete_result_ids, + status=ArchiveResult.StatusChoices.QUEUED, + ).update( + status=ArchiveResult.StatusChoices.SKIPPED, + output_str="Hook no longer exists in the current plugin set.", + modified_at=timezone.now(), + ) + + queued_plugins = sorted({result.plugin for result in queued_results if result.id not in obsolete_result_ids}) if queued_plugins: return queued_plugins return None diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html index 487388bd..386e8534 100644 --- a/archivebox/templates/core/add.html +++ b/archivebox/templates/core/add.html @@ -100,9 +100,9 @@
{{ form.persona.errors }}
{% endif %}
- Authentication profile (Chrome profile, cookies, etc.) to use when accessing URLs. + Authentication + configuration settings to use when saving URLs (cookies, user agent, resolution, timeouts, etc.) {% if can_override_crawl_config %} - Create new persona / import from Chrome → + Create new profile / import from Chrome -> {% endif %}
@@ -247,6 +247,9 @@
Quick Select: + {% for persona in recent_personas %} + + {% endfor %} @@ -1151,7 +1154,16 @@ 'text-only': ['wget', 'readability', 'mercury', 'htmltotext', 'title', 'favicon'] }; - document.querySelectorAll('.preset-btn').forEach(btn => { + document.querySelectorAll('.persona-preset-btn').forEach(btn => { + btn.addEventListener('click', function() { + if (!personaSelect) return; + personaSelect.value = this.dataset.persona || ''; + dispatchChange(personaSelect); + saveFormState(); + }); + }); + + document.querySelectorAll('.preset-btn[data-preset]').forEach(btn => { btn.addEventListener('click', function() { const preset = this.dataset.preset; const allCheckboxes = document.querySelectorAll('.plugin-section-toggle'); diff --git a/archivebox/templates/static/add.css b/archivebox/templates/static/add.css index 9a5a379d..4970d47a 100755 --- a/archivebox/templates/static/add.css +++ b/archivebox/templates/static/add.css @@ -766,6 +766,12 @@ select { box-shadow: none; } +.persona-preset-btn { + color: #004882; + border-color: #b7d3ea; + background-color: #f5fbff; +} + /* Advanced section (collapsible) */ .advanced-section { background-color: white; diff --git a/archivebox/tests/test_cli_run.py b/archivebox/tests/test_cli_run.py index d174f54a..441de554 100644 --- a/archivebox/tests/test_cli_run.py +++ b/archivebox/tests/test_cli_run.py @@ -797,6 +797,32 @@ class TestRecoverOrchestratorState: assert snapshot.retry_at is None assert snapshot.downloaded_at is None + def test_create_pending_archiveresults_uses_canonical_hook_names(self): + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.STARTED, + retry_at=timezone.now(), + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.QUEUED, + retry_at=timezone.now(), + ) + + snapshot.create_pending_archiveresults() + + hook_names = list(ArchiveResult.objects.filter(snapshot=snapshot).values_list("hook_name", flat=True)) + assert hook_names + assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names) + def test_run_due_snapshot_pauses_child_when_parent_is_paused(self): from django.utils import timezone @@ -1194,6 +1220,40 @@ class TestRecoverOrchestratorState: assert result.status == ArchiveResult.StatusChoices.SUCCEEDED assert snapshot.retry_at is None + def test_run_due_snapshot_skips_obsolete_queued_hook_name(self): + from django.utils import timezone + + from archivebox.base_models.models import get_or_create_system_user_pk + from archivebox.crawls.models import Crawl + from archivebox.core.models import ArchiveResult, Snapshot + from archivebox.services.runner import run_due_snapshot + + crawl = Crawl.objects.create( + urls="https://example.com", + created_by_id=get_or_create_system_user_pk(), + status=Crawl.StatusChoices.SEALED, + retry_at=None, + ) + snapshot = Snapshot.objects.create( + url="https://example.com", + crawl=crawl, + status=Snapshot.StatusChoices.SEALED, + retry_at=timezone.now(), + ) + result = ArchiveResult.objects.create( + snapshot=snapshot, + plugin="singlefile", + hook_name="on_Snapshot__50_singlefile.py", + status=ArchiveResult.StatusChoices.QUEUED, + ) + + assert run_due_snapshot(snapshot, lock_seconds=60) is True + + result.refresh_from_db() + snapshot.refresh_from_db() + assert result.status == ArchiveResult.StatusChoices.SKIPPED + assert snapshot.retry_at is None + def test_recover_orchestrator_state_ignores_sealed_downloaded_snapshot_without_results(self): from django.utils import timezone diff --git a/archivebox/tests/test_ui_admin_views.py b/archivebox/tests/test_ui_admin_views.py index 419cebf8..a0117b5a 100644 --- a/archivebox/tests/test_ui_admin_views.py +++ b/archivebox/tests/test_ui_admin_views.py @@ -1259,7 +1259,7 @@ class TestArchiveResultAdminListView: class TestLiveProgressView: def test_live_progress_hides_finished_cancelled_crawl(self, client, admin_user, crawl, snapshot): - from archivebox.core.models import Snapshot + from archivebox.core.models import ArchiveResult, Snapshot from archivebox.crawls.models import Crawl now = timezone.now() @@ -1274,12 +1274,20 @@ class TestLiveProgressView: downloaded_at=None, modified_at=now, ) + ArchiveResult.objects.create( + snapshot=snapshot, + plugin="singlefile", + hook_name="on_Snapshot__50_singlefile", + status=ArchiveResult.StatusChoices.QUEUED, + ) client.login(username="testadmin", password="testpassword") response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_HOST) assert response.status_code == 200 - assert response.json()["active_crawls"] == [] + payload = response.json() + assert payload["active_crawls"] == [] + assert payload["downloads_queued"] == 0 def test_live_progress_reports_real_orchestrator_process_running(self, client, admin_user, db): import archivebox.machine.models as machine_models diff --git a/etc/package.json b/etc/package.json index f7eee9df..0f34ee93 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.33rc38", + "version": "0.9.33rc39", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index 2106cc88..b410b761 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.33rc38" +version = "0.9.33rc39" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.8", # EventBus API - "abxpkg>=1.11.64", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.70", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.70", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.65", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.71", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.71", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]