diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py index 8a40b438..a65e0b40 100644 --- a/archivebox/core/forms.py +++ b/archivebox/core/forms.py @@ -71,6 +71,7 @@ PLUGIN_GROUP_DEFINITIONS = ( ( "staticfile", "responses", + "chrome_screencast", "ytdlp", "gallerydl", "git", @@ -697,8 +698,8 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form): initial="public", required=True, ) - index_only = forms.BooleanField( - label="Index only dry run (add crawl but don't archive yet)", + start_paused = forms.BooleanField( + label="Start paused", initial=False, required=False, ) diff --git a/archivebox/core/views.py b/archivebox/core/views.py index e2e809bc..7556a691 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -70,6 +70,7 @@ from archivebox.core.host_utils import ( ) from archivebox.core.forms import AddLinkForm, get_plugin_config_binary_urls from archivebox.crawls.models import Crawl +from archivebox.workers.models import RETRY_AT_MAX from archivebox.hooks import ( BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR, @@ -1178,7 +1179,7 @@ class AddView(UserPassesTestMixin, FormView): plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else "" schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else "" persona = form.cleaned_data.get("persona") - index_only = form.cleaned_data.get("index_only", False) if can_override_crawl_config else False + start_paused = form.cleaned_data.get("start_paused", False) if can_override_crawl_config else False notes = form.cleaned_data.get("notes", "") url_filters = form.cleaned_data.get("url_filters") or {} plugin_config = form.cleaned_data.get("plugin_config") or {} @@ -1215,8 +1216,6 @@ class AddView(UserPassesTestMixin, FormView): # Store only explicit crawl-scoped overrides. Persona/machine/plugin # defaults are resolved at hook runtime via get_config(...). config = {} - if index_only: - config["INDEX_ONLY"] = True if plugins: config["PLUGINS"] = plugins effective_config = get_config(persona=persona, user=self.request.user) if persona else get_config(user=self.request.user) @@ -1255,7 +1254,8 @@ class AddView(UserPassesTestMixin, FormView): created_by_id=created_by_id, config=config, persona_id=persona.id if persona else None, - retry_at=None if index_only else timezone.now(), + status=Crawl.StatusChoices.PAUSED if start_paused else Crawl.StatusChoices.QUEUED, + retry_at=RETRY_AT_MAX if start_paused else timezone.now(), ) # 3. create a CrawlSchedule if schedule is provided @@ -1273,7 +1273,7 @@ class AddView(UserPassesTestMixin, FormView): crawl.schedule = crawl_schedule crawl.safe_update({"schedule": crawl_schedule}, refresh=False) - if not index_only: + if not start_paused: from archivebox.services.runner import ensure_background_runner ensure_background_runner() @@ -1378,8 +1378,8 @@ class WebAddView(AddView): "config": "{}", }, ) - if defaults_form.fields["index_only"].initial: - form_data["index_only"] = "on" + if defaults_form.fields["start_paused"].initial: + form_data["start_paused"] = "on" form = self.form_class(data=form_data) if not form.is_valid(): diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html index 386e8534..c53e3419 100644 --- a/archivebox/templates/core/add.html +++ b/archivebox/templates/core/add.html @@ -247,8 +247,12 @@
Quick Select: + + 👤 {% for persona in recent_personas %} - + + + ✎ + {% endfor %} @@ -280,12 +284,12 @@
- {{ form.index_only }} - {{ form.index_only.label_tag }} - {% if form.index_only.errors %} -
{{ form.index_only.errors }}
+ {{ form.start_paused }} + {{ form.start_paused.label_tag }} + {% if form.start_paused.errors %} +
{{ form.start_paused.errors }}
{% endif %} -
Create the crawl and queue snapshots without running archive plugins yet.
+
Create the crawl in a paused state. No snapshots will be created until you resume it.
diff --git a/archivebox/templates/static/add.css b/archivebox/templates/static/add.css index 4970d47a..eb5d11c5 100755 --- a/archivebox/templates/static/add.css +++ b/archivebox/templates/static/add.css @@ -743,6 +743,8 @@ select { } .preset-btn { + display: inline-flex; + align-items: center; padding: 6px 14px; font-size: 13px; font-weight: 500; @@ -750,6 +752,8 @@ select { border: 1px solid #ced4da; border-radius: 4px; cursor: pointer; + color: inherit; + text-decoration: none; transition: all 0.2s; white-space: nowrap; } @@ -772,6 +776,33 @@ select { background-color: #f5fbff; } +.persona-preset-wrap { + display: inline-flex; + align-items: stretch; +} + +.persona-preset-wrap .persona-preset-btn { + border-top-right-radius: 0; + border-bottom-right-radius: 0; + padding-right: 10px; +} + +.persona-edit-btn { + margin-left: -1px; + padding: 6px 8px; + color: #004882; + border-color: #b7d3ea; + border-top-left-radius: 0; + border-bottom-left-radius: 0; + background-color: #eaf5ff; +} + +.persona-create-btn { + color: #116329; + border-color: #b6e3c6; + background-color: #f0fff4; +} + /* Advanced section (collapsible) */ .advanced-section { background-color: white; diff --git a/archivebox/tests/test_ui_add_view.py b/archivebox/tests/test_ui_add_view.py index b67925c3..4908623e 100644 --- a/archivebox/tests/test_ui_add_view.py +++ b/archivebox/tests/test_ui_add_view.py @@ -7,6 +7,7 @@ from archivebox.config.common import get_config from archivebox.core.models import Snapshot, Tag from archivebox.crawls.models import Crawl from archivebox.personas.models import Persona +from archivebox.workers.models import RETRY_AT_MAX pytestmark = pytest.mark.django_db @@ -48,6 +49,7 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m "snapshot_max_size", "delete_after", "crawl_max_concurrent_snapshots", + "start_paused", "notes", }.issubset(form.fields) assert b'name="url_filters_only_new"' in response.content @@ -66,6 +68,8 @@ def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypat assert response.context["can_override_crawl_config"] is True assert form.plugin_groups assert any(card["config_fields"] for group in form.plugin_groups for card in group["plugins"]) + assert b"Index only dry run" not in response.content + assert b"Start paused" in response.content def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user, monkeypatch): @@ -145,7 +149,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_ "schedule": "", "persona": "Default", "permissions": "public", - "index_only": "", + "start_paused": "", "config": "{}", }, HTTP_HOST=ADMIN_HOST, @@ -190,7 +194,7 @@ def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user, mon "schedule": "", "persona": "Default", "permissions": "public", - "index_only": "", + "start_paused": "", "config": "{}", }, HTTP_HOST=ADMIN_HOST, @@ -226,7 +230,7 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin "schedule": "", "persona": "Private", "permissions": "public", - "index_only": "", + "start_paused": "", "config": '{"DEFAULT_PERSONA": "Default"}', }, HTTP_HOST=ADMIN_HOST, @@ -264,7 +268,7 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc "schedule": "", "persona": "Default", "permissions": "public", - "index_only": "", + "start_paused": "", "main_plugins": ["wget"], "plugin_config__wget__WGET_TIMEOUT": "77", "plugin_config__wget__WGET_WARC_ENABLED": "false", @@ -305,7 +309,7 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm "schedule": "daily", "persona": "Default", "permissions": "public", - "index_only": "on", + "start_paused": "on", "main_plugins": ["wget"], "plugin_config__twocaptcha__TWOCAPTCHA_API_KEY": "posted-token", "plugin_config__wget__WGET_TIMEOUT": "77", @@ -331,6 +335,7 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm assert "NODE_BINARY" not in crawl.config assert "TWOCAPTCHA_API_KEY" not in crawl.config assert "INDEX_ONLY" not in crawl.config + assert crawl.status == Crawl.StatusChoices.QUEUED assert crawl.schedule is None @@ -354,7 +359,7 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp "schedule": "", "persona": "Default", "permissions": "public", - "index_only": "", + "start_paused": "", "config": "{}", }, HTTP_HOST=ADMIN_HOST, @@ -368,6 +373,41 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp assert crawl.snapshot_set.count() == 0 +def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, admin_user, monkeypatch): + monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") + client.force_login(admin_user) + + response = client.post( + reverse("add"), + data={ + "url": "https://example.com/paused", + "tag": "", + "depth": "1", + "max_urls": "0", + "crawl_max_size": "0", + "snapshot_max_size": "0", + "url_filters_allowlist": "", + "url_filters_denylist": "", + "url_filters_only_new": "1", + "notes": "", + "schedule": "", + "persona": "Default", + "permissions": "public", + "start_paused": "on", + "config": "{}", + }, + HTTP_HOST=ADMIN_HOST, + ) + + assert response.status_code == 302, response.context["form"].errors if response.context else response.content.decode() + crawl = Crawl.objects.order_by("-created_at").first() + assert crawl is not None + assert crawl.status == Crawl.StatusChoices.PAUSED + assert crawl.retry_at == RETRY_AT_MAX + assert crawl.snapshot_set.count() == 0 + assert "INDEX_ONLY" not in crawl.config + + def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkeypatch): monkeypatch.setenv("PUBLIC_ADD_VIEW", "true") client.force_login(admin_user) @@ -396,7 +436,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkey "schedule": "", "persona": "Default", "permissions": "public", - "index_only": "", + "start_paused": "", "config": "{}", }, HTTP_HOST=ADMIN_HOST, @@ -444,7 +484,7 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us "schedule": "", "persona": "Default", "permissions": "public", - "index_only": "", + "start_paused": "", "config": "{}", }, HTTP_HOST=ADMIN_HOST, diff --git a/archivebox/tests/test_ui_add_view_runtime.py b/archivebox/tests/test_ui_add_view_runtime.py index d8e01a03..abddad30 100644 --- a/archivebox/tests/test_ui_add_view_runtime.py +++ b/archivebox/tests/test_ui_add_view_runtime.py @@ -151,7 +151,7 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv "notes": "created from running-server web ui", "persona": "Default", "permissions": "public", - "index_only": "", + "start_paused": "", "config": "{}", "csrfmiddlewaretoken": csrf_match.group(1), }, diff --git a/archivebox/tests/test_ui_admin_views.py b/archivebox/tests/test_ui_admin_views.py index a0117b5a..214b0c1a 100644 --- a/archivebox/tests/test_ui_admin_views.py +++ b/archivebox/tests/test_ui_admin_views.py @@ -1152,13 +1152,20 @@ class TestAdminSnapshotListView: class TestCrawlScheduleAdmin: - def test_crawlschedule_add_view_renders_and_saves(self, client, admin_user, crawl): + def test_crawlschedule_change_view_renders_and_saves(self, client, admin_user, crawl): from archivebox.crawls.models import CrawlSchedule + schedule = CrawlSchedule.objects.create( + label="Nightly crawl", + notes="", + schedule="0 0 * * *", + template=crawl, + created_by=admin_user, + ) client.login(username="testadmin", password="testpassword") - add_url = reverse("admin:crawls_crawlschedule_add") - get_response = client.get(add_url, HTTP_HOST=ADMIN_HOST) + change_url = reverse("admin:crawls_crawlschedule_change", args=[schedule.pk]) + get_response = client.get(change_url, HTTP_HOST=ADMIN_HOST) assert get_response.status_code == 200 assert b"Schedule Info" in get_response.content @@ -1166,11 +1173,11 @@ class TestCrawlScheduleAdmin: assert b"No Snapshots yet..." not in get_response.content post_response = client.post( - add_url, + change_url, { - "label": "Nightly crawl", - "notes": "", - "schedule": "0 0 * * *", + "label": "Morning crawl", + "notes": "updated", + "schedule": "0 8 * * *", "template": str(crawl.pk), "created_by": str(admin_user.pk), "_save": "Save", @@ -1179,7 +1186,10 @@ class TestCrawlScheduleAdmin: ) assert post_response.status_code == 302 - schedule = CrawlSchedule.objects.get(label="Nightly crawl") + schedule.refresh_from_db() + assert schedule.label == "Morning crawl" + assert schedule.notes == "updated" + assert schedule.schedule == "0 8 * * *" assert schedule.template_id == crawl.pk assert schedule.created_by_id == admin_user.pk diff --git a/etc/package.json b/etc/package.json index 0f34ee93..6c9a117c 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.33rc39", + "version": "0.9.33rc40", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index b410b761..bb769f6e 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.33rc39" +version = "0.9.33rc40" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.8", # EventBus API - "abxpkg>=1.11.65", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.71", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.71", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.66", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.72", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.72", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]