The_Great_Archive/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-09-12 19:50:57 +05:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

release: archivebox 0.9.33rc40

Browse Source
This commit is contained in:
Nick Sweeting 2026-05-29 04:06:13 -07:00
parent 2d2b8ff047
commit dd741de767
No known key found for this signature in database
9 changed files with 123 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
archivebox/core/forms.py
View File

@ -71,6 +71,7 @@ PLUGIN_GROUP_DEFINITIONS = (
(
"staticfile",
"responses",
"chrome_screencast",
"ytdlp",
"gallerydl",
"git",
@ -697,8 +698,8 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
initial="public",
required=True,
)
index_only = forms.BooleanField(
label="Index only dry run (add crawl but don't archive yet)",
start_paused = forms.BooleanField(
label="Start paused",
initial=False,
required=False,
)

14
archivebox/core/views.py
View File

@ -70,6 +70,7 @@ from archivebox.core.host_utils import (
)
from archivebox.core.forms import AddLinkForm, get_plugin_config_binary_urls
from archivebox.crawls.models import Crawl
from archivebox.workers.models import RETRY_AT_MAX
from archivebox.hooks import (
BUILTIN_PLUGINS_DIR,
USER_PLUGINS_DIR,
@ -1178,7 +1179,7 @@ class AddView(UserPassesTestMixin, FormView):
plugins = ",".join(form.cleaned_data.get("plugins", [])) if can_override_crawl_config else ""
schedule = form.cleaned_data.get("schedule", "").strip() if can_override_crawl_config else ""
persona = form.cleaned_data.get("persona")
index_only = form.cleaned_data.get("index_only", False) if can_override_crawl_config else False
start_paused = form.cleaned_data.get("start_paused", False) if can_override_crawl_config else False
notes = form.cleaned_data.get("notes", "")
url_filters = form.cleaned_data.get("url_filters") or {}
plugin_config = form.cleaned_data.get("plugin_config") or {}
@ -1215,8 +1216,6 @@ class AddView(UserPassesTestMixin, FormView):
# Store only explicit crawl-scoped overrides. Persona/machine/plugin
# defaults are resolved at hook runtime via get_config(...).
config = {}
if index_only:
config["INDEX_ONLY"] = True
if plugins:
config["PLUGINS"] = plugins
effective_config = get_config(persona=persona, user=self.request.user) if persona else get_config(user=self.request.user)
@ -1255,7 +1254,8 @@ class AddView(UserPassesTestMixin, FormView):
created_by_id=created_by_id,
config=config,
persona_id=persona.id if persona else None,
retry_at=None if index_only else timezone.now(),
status=Crawl.StatusChoices.PAUSED if start_paused else Crawl.StatusChoices.QUEUED,
retry_at=RETRY_AT_MAX if start_paused else timezone.now(),
)
# 3. create a CrawlSchedule if schedule is provided
@ -1273,7 +1273,7 @@ class AddView(UserPassesTestMixin, FormView):
crawl.schedule = crawl_schedule
crawl.safe_update({"schedule": crawl_schedule}, refresh=False)
if not index_only:
if not start_paused:
from archivebox.services.runner import ensure_background_runner
ensure_background_runner()
@ -1378,8 +1378,8 @@ class WebAddView(AddView):
"config": "{}",
},
)
if defaults_form.fields["index_only"].initial:
form_data["index_only"] = "on"
if defaults_form.fields["start_paused"].initial:
form_data["start_paused"] = "on"
form = self.form_class(data=form_data)
if not form.is_valid():

16
archivebox/templates/core/add.html
View File

@ -247,8 +247,12 @@
<div class="plugin-presets">
<span class="preset-label">Quick Select:</span>
<a href="/admin/personas/persona/add/" class="preset-btn persona-create-btn" target="_blank" rel="noopener" title="Create new profile / import from Chrome">+ 👤</a>
{% for persona in recent_personas %}
<button type="button" class="preset-btn persona-preset-btn" data-persona="{{ persona.name }}" title="Use {{ persona.name }} persona">👤 {{ persona.name }}</button>
<span class="persona-preset-wrap">
<button type="button" class="preset-btn persona-preset-btn" data-persona="{{ persona.name }}" title="Use {{ persona.name }} persona">👤 {{ persona.name }}</button>
<a href="/admin/personas/persona/{{ persona.pk }}/change/" class="preset-btn persona-edit-btn" target="_blank" rel="noopener" title="Edit {{ persona.name }} persona">✎</a>
</span>
{% endfor %}
<button type="button" class="preset-btn" data-preset="text-only">📄 Text Only</button>
<button type="button" class="preset-btn" data-preset="select-all">✓ Select All</button>
@ -280,12 +284,12 @@
</div>
<div class="form-field checkbox-field">
{{ form.index_only }}
{{ form.index_only.label_tag }}
{% if form.index_only.errors %}
<div class="error">{{ form.index_only.errors }}</div>
{{ form.start_paused }}
{{ form.start_paused.label_tag }}
{% if form.start_paused.errors %}
<div class="error">{{ form.start_paused.errors }}</div>
{% endif %}
<div class="help-text">Create the crawl and queue snapshots without running archive plugins yet.</div>
<div class="help-text">Create the crawl in a paused state. No snapshots will be created until you resume it.</div>
</div>
<div class="form-field">

31
archivebox/templates/static/add.css
View File

@ -743,6 +743,8 @@ select {
}
.preset-btn {
display: inline-flex;
align-items: center;
padding: 6px 14px;
font-size: 13px;
font-weight: 500;
@ -750,6 +752,8 @@ select {
border: 1px solid #ced4da;
border-radius: 4px;
cursor: pointer;
color: inherit;
text-decoration: none;
transition: all 0.2s;
white-space: nowrap;
}
@ -772,6 +776,33 @@ select {
background-color: #f5fbff;
}
.persona-preset-wrap {
display: inline-flex;
align-items: stretch;
}
.persona-preset-wrap .persona-preset-btn {
border-top-right-radius: 0;
border-bottom-right-radius: 0;
padding-right: 10px;
}
.persona-edit-btn {
margin-left: -1px;
padding: 6px 8px;
color: #004882;
border-color: #b7d3ea;
border-top-left-radius: 0;
border-bottom-left-radius: 0;
background-color: #eaf5ff;
}
.persona-create-btn {
color: #116329;
border-color: #b6e3c6;
background-color: #f0fff4;
}
/* Advanced section (collapsible) */
.advanced-section {
background-color: white;

56
archivebox/tests/test_ui_add_view.py
View File

@ -7,6 +7,7 @@ from archivebox.config.common import get_config
from archivebox.core.models import Snapshot, Tag
from archivebox.crawls.models import Crawl
from archivebox.personas.models import Persona
from archivebox.workers.models import RETRY_AT_MAX
pytestmark = pytest.mark.django_db
@ -48,6 +49,7 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m
"snapshot_max_size",
"delete_after",
"crawl_max_concurrent_snapshots",
"start_paused",
"notes",
}.issubset(form.fields)
assert b'name="url_filters_only_new"' in response.content
@ -66,6 +68,8 @@ def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypat
assert response.context["can_override_crawl_config"] is True
assert form.plugin_groups
assert any(card["config_fields"] for group in form.plugin_groups for card in group["plugins"])
assert b"Index only dry run" not in response.content
assert b"Start paused" in response.content
def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user, monkeypatch):
@ -145,7 +149,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
"schedule": "",
"persona": "Default",
"permissions": "public",
"index_only": "",
"start_paused": "",
"config": "{}",
},
HTTP_HOST=ADMIN_HOST,
@ -190,7 +194,7 @@ def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user, mon
"schedule": "",
"persona": "Default",
"permissions": "public",
"index_only": "",
"start_paused": "",
"config": "{}",
},
HTTP_HOST=ADMIN_HOST,
@ -226,7 +230,7 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
"schedule": "",
"persona": "Private",
"permissions": "public",
"index_only": "",
"start_paused": "",
"config": '{"DEFAULT_PERSONA": "Default"}',
},
HTTP_HOST=ADMIN_HOST,
@ -264,7 +268,7 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc
"schedule": "",
"persona": "Default",
"permissions": "public",
"index_only": "",
"start_paused": "",
"main_plugins": ["wget"],
"plugin_config__wget__WGET_TIMEOUT": "77",
"plugin_config__wget__WGET_WARC_ENABLED": "false",
@ -305,7 +309,7 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm
"schedule": "daily",
"persona": "Default",
"permissions": "public",
"index_only": "on",
"start_paused": "on",
"main_plugins": ["wget"],
"plugin_config__twocaptcha__TWOCAPTCHA_API_KEY": "posted-token",
"plugin_config__wget__WGET_TIMEOUT": "77",
@ -331,6 +335,7 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm
assert "NODE_BINARY" not in crawl.config
assert "TWOCAPTCHA_API_KEY" not in crawl.config
assert "INDEX_ONLY" not in crawl.config
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.schedule is None
@ -354,7 +359,7 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp
"schedule": "",
"persona": "Default",
"permissions": "public",
"index_only": "",
"start_paused": "",
"config": "{}",
},
HTTP_HOST=ADMIN_HOST,
@ -368,6 +373,41 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user, monkeyp
assert crawl.snapshot_set.count() == 0
def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, admin_user, monkeypatch):
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)
response = client.post(
reverse("add"),
data={
"url": "https://example.com/paused",
"tag": "",
"depth": "1",
"max_urls": "0",
"crawl_max_size": "0",
"snapshot_max_size": "0",
"url_filters_allowlist": "",
"url_filters_denylist": "",
"url_filters_only_new": "1",
"notes": "",
"schedule": "",
"persona": "Default",
"permissions": "public",
"start_paused": "on",
"config": "{}",
},
HTTP_HOST=ADMIN_HOST,
)
assert response.status_code == 302, response.context["form"].errors if response.context else response.content.decode()
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
assert crawl.status == Crawl.StatusChoices.PAUSED
assert crawl.retry_at == RETRY_AT_MAX
assert crawl.snapshot_set.count() == 0
assert "INDEX_ONLY" not in crawl.config
def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkeypatch):
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
client.force_login(admin_user)
@ -396,7 +436,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user, monkey
"schedule": "",
"persona": "Default",
"permissions": "public",
"index_only": "",
"start_paused": "",
"config": "{}",
},
HTTP_HOST=ADMIN_HOST,
@ -444,7 +484,7 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us
"schedule": "",
"persona": "Default",
"permissions": "public",
"index_only": "",
"start_paused": "",
"config": "{}",
},
HTTP_HOST=ADMIN_HOST,

2
archivebox/tests/test_ui_add_view_runtime.py
View File

@ -151,7 +151,7 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv
"notes": "created from running-server web ui",
"persona": "Default",
"permissions": "public",
"index_only": "",
"start_paused": "",
"config": "{}",
"csrfmiddlewaretoken": csrf_match.group(1),
},

26
archivebox/tests/test_ui_admin_views.py
View File

@ -1152,13 +1152,20 @@ class TestAdminSnapshotListView:
class TestCrawlScheduleAdmin:
def test_crawlschedule_add_view_renders_and_saves(self, client, admin_user, crawl):
def test_crawlschedule_change_view_renders_and_saves(self, client, admin_user, crawl):
from archivebox.crawls.models import CrawlSchedule
schedule = CrawlSchedule.objects.create(
label="Nightly crawl",
notes="",
schedule="0 0 * * *",
template=crawl,
created_by=admin_user,
)
client.login(username="testadmin", password="testpassword")
add_url = reverse("admin:crawls_crawlschedule_add")
get_response = client.get(add_url, HTTP_HOST=ADMIN_HOST)
change_url = reverse("admin:crawls_crawlschedule_change", args=[schedule.pk])
get_response = client.get(change_url, HTTP_HOST=ADMIN_HOST)
assert get_response.status_code == 200
assert b"Schedule Info" in get_response.content
@ -1166,11 +1173,11 @@ class TestCrawlScheduleAdmin:
assert b"No Snapshots yet..." not in get_response.content
post_response = client.post(
add_url,
change_url,
{
"label": "Nightly crawl",
"notes": "",
"schedule": "0 0 * * *",
"label": "Morning crawl",
"notes": "updated",
"schedule": "0 8 * * *",
"template": str(crawl.pk),
"created_by": str(admin_user.pk),
"_save": "Save",
@ -1179,7 +1186,10 @@ class TestCrawlScheduleAdmin:
)
assert post_response.status_code == 302
schedule = CrawlSchedule.objects.get(label="Nightly crawl")
schedule.refresh_from_db()
assert schedule.label == "Morning crawl"
assert schedule.notes == "updated"
assert schedule.schedule == "0 8 * * *"
assert schedule.template_id == crawl.pk
assert schedule.created_by_id == admin_user.pk

2
etc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.33rc39",
"version": "0.9.33rc40",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {

8
pyproject.toml
View File

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.33rc39"
version = "0.9.33rc40"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus==2.5.8", # EventBus API
"abxpkg>=1.11.65", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.71", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.71", # shared ArchiveBox downloader package with blocking install preflight
"abxpkg>=1.11.66", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
"abx-plugins>=1.11.72", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
"abx-dl>=1.11.72", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]