diff --git a/archivebox/api/admin.py b/archivebox/api/admin.py
index 76586165..f54942c5 100644
--- a/archivebox/api/admin.py
+++ b/archivebox/api/admin.py
@@ -1,9 +1,12 @@
__package__ = "archivebox.api"
+from django import forms
from django.contrib import admin
from django.http import HttpRequest
-from signal_webhooks.admin import WebhookAdmin
-from signal_webhooks.utils import get_webhook_model
+from django.utils.text import capfirst
+from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
+from signal_webhooks.settings import webhook_settings
+from signal_webhooks.utils import get_webhook_model, model_from_reference
from archivebox.base_models.admin import BaseModelAdmin
@@ -50,7 +53,21 @@ class APITokenAdmin(BaseModelAdmin):
list_per_page = 100
+class OutboundWebhookAdminForm(WebhookModelForm):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.fields["ref"] = forms.ChoiceField(
+ label=self.fields["ref"].label,
+ help_text=self.fields["ref"].help_text,
+ choices=[
+ (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
+ for ref in sorted(webhook_settings.HOOKS)
+ ],
+ )
+
+
class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
+ form = OutboundWebhookAdminForm
list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
diff --git a/archivebox/api/v1_crawls.py b/archivebox/api/v1_crawls.py
index 3aff07e2..38b0597e 100644
--- a/archivebox/api/v1_crawls.py
+++ b/archivebox/api/v1_crawls.py
@@ -14,6 +14,7 @@ from ninja.errors import HttpError
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
+from archivebox.config.common import get_config
from .auth import API_AUTH_METHODS
@@ -38,11 +39,16 @@ class CrawlSchema(Schema):
max_urls: int
crawl_max_size: int
snapshot_max_size: int
+ crawl_max_concurrent_snapshots: int
tags_str: str
config: dict
# snapshots: List[SnapshotSchema]
+ @staticmethod
+ def resolve_crawl_max_concurrent_snapshots(obj):
+ return int((obj.config or {}).get("CRAWL_MAX_CONCURRENT_SNAPSHOTS") or get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS)
+
@staticmethod
def resolve_created_by_id(obj):
return str(obj.created_by_id)
@@ -74,6 +80,7 @@ class CrawlCreateSchema(Schema):
max_urls: int = 0
crawl_max_size: int = 0
snapshot_max_size: int = 0
+ crawl_max_concurrent_snapshots: int | None = None
tags: list[str] | None = None
tags_str: str = ""
label: str = ""
@@ -112,8 +119,15 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
raise HttpError(400, "crawl_max_size must be >= 0")
if data.snapshot_max_size < 0:
raise HttpError(400, "snapshot_max_size must be >= 0")
+ crawl_max_concurrent_snapshots = data.crawl_max_concurrent_snapshots
+ if crawl_max_concurrent_snapshots is None:
+ crawl_max_concurrent_snapshots = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
+ if crawl_max_concurrent_snapshots < 1:
+ raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1")
tags = normalize_tag_list(data.tags, data.tags_str)
+ config = dict(data.config or {})
+ config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
crawl = Crawl.objects.create(
urls="\n".join(urls),
max_depth=data.max_depth,
@@ -123,7 +137,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
tags_str=",".join(tags),
label=data.label,
notes=data.notes,
- config=data.config,
+ config=config,
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
created_by=request.user if isinstance(request.user, User) else None,
diff --git a/archivebox/base_models/admin.py b/archivebox/base_models/admin.py
index d6703b82..939f3171 100644
--- a/archivebox/base_models/admin.py
+++ b/archivebox/base_models/admin.py
@@ -44,10 +44,27 @@ class KeyValueWidget(forms.Widget):
def _get_config_options(self) -> dict[str, ConfigOption]:
"""Get available config options from plugins."""
try:
+ from archivebox.config.common import ArchiveBoxConfig
from archivebox.hooks import discover_plugin_configs
- plugin_configs = discover_plugin_configs()
options: dict[str, ConfigOption] = {}
+ skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
+ for key, field in ArchiveBoxConfig.model_fields.items():
+ if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
+ continue
+ default = field.default
+ try:
+ json.dumps(default)
+ except TypeError:
+ default = str(default)
+ options[key] = {
+ "plugin": "archivebox",
+ "type": str(field.annotation),
+ "default": default,
+ "description": field.description or "",
+ }
+
+ plugin_configs = discover_plugin_configs()
for plugin_name, schema in plugin_configs.items():
for key, prop in schema.get("properties", {}).items():
option: ConfigOption = {
diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py
index 0e2771c2..dd8225d7 100644
--- a/archivebox/cli/archivebox_add.py
+++ b/archivebox/cli/archivebox_add.py
@@ -53,6 +53,7 @@ def add(
max_urls: int = 0,
crawl_max_size: int | str = 0,
snapshot_max_size: int | str = 0,
+ crawl_max_concurrent_snapshots: int | None = None,
tag: str = "",
url_allowlist: str = "",
url_denylist: str = "",
@@ -83,6 +84,10 @@ def add(
max_urls = int(max_urls or 0)
crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
+ config = get_config()
+ if crawl_max_concurrent_snapshots is None:
+ crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+ crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
if depth not in (0, 1, 2, 3, 4):
raise ValueError("Depth must be 0-4")
@@ -92,6 +97,8 @@ def add(
raise ValueError("crawl_max_size must be >= 0")
if snapshot_max_size < 0:
raise ValueError("snapshot_max_size must be >= 0")
+ if crawl_max_concurrent_snapshots < 1:
+ raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
# import models once django is set up
from archivebox.crawls.models import Crawl
@@ -101,7 +108,6 @@ def add(
from archivebox.misc.system import get_dir_size
from archivebox.services.runner import run_crawl
- config = get_config()
created_by_id = created_by_id or get_or_create_system_user_pk()
started_at = timezone.now()
if update is None:
@@ -157,6 +163,7 @@ def add(
"OVERWRITE": overwrite,
"PLUGINS": plugins,
"DEFAULT_PERSONA": persona_name,
+ "CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
"PARSER": parser,
**({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
**({"URL_DENYLIST": url_denylist} if url_denylist else {}),
@@ -260,6 +267,7 @@ def add(
@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
@@ -289,6 +297,8 @@ def main(**kwargs):
kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
except ValueError as err:
raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
+ if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
+ raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
add(urls=urls, **kwargs)
diff --git a/archivebox/cli/archivebox_run.py b/archivebox/cli/archivebox_run.py
index d3ca7e40..75b8ebd2 100644
--- a/archivebox/cli/archivebox_run.py
+++ b/archivebox/cli/archivebox_run.py
@@ -259,6 +259,7 @@ def run_runner(daemon: bool = False) -> int:
Process.cleanup_stale_running()
Process.cleanup_orphaned_workers()
+ Process.cleanup_orphaned_chrome()
recover_orphaned_snapshots()
recover_orphaned_crawls()
Machine.current()
diff --git a/archivebox/config/common.py b/archivebox/config/common.py
index 7bd5ce76..0c9ed294 100644
--- a/archivebox/config/common.py
+++ b/archivebox/config/common.py
@@ -251,6 +251,10 @@ class ArchivingConfig(BaseConfigSet):
MAX_DEPTH: int = Field(default=0)
CRAWL_MAX_URLS: int = Field(default=0)
CRAWL_MAX_SIZE: int = Field(default=0)
+ CRAWL_MAX_CONCURRENT_SNAPSHOTS: int = Field(
+ default=4,
+ description="Maximum number of snapshots to archive concurrently within one crawl.",
+ )
SNAPSHOT_MAX_SIZE: int = Field(default=0)
RESOLUTION: str = Field(default="1440,2000")
diff --git a/archivebox/core/forms.py b/archivebox/core/forms.py
index 3b974fa0..74cd714f 100644
--- a/archivebox/core/forms.py
+++ b/archivebox/core/forms.py
@@ -1,7 +1,7 @@
__package__ = "archivebox.core"
import json
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
@@ -28,80 +28,123 @@ DEPTH_CHOICES = (
PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
PLUGIN_GROUP_DEFINITIONS = (
(
- "chrome_plugins",
- "Chrome-dependent plugins",
+ "main_plugins",
+ "Main",
"",
- "chrome-plugins",
- "chrome",
- {
- "accessibility",
- "chrome",
- "consolelog",
+ "",
+ "",
+ (
"dom",
- "headers",
- "parse_dom_outlinks",
- "pdf",
- "redirects",
- "responses",
"screenshot",
- "seo",
+ "pdf",
"singlefile",
- "ssl",
- "staticfile",
- "title",
- },
+ "wget",
+ "archivedotorg",
+ ),
),
(
- "archiving_plugins",
- "Archiving",
+ "page_setup_plugins",
+ "Page Setup",
"",
"",
"",
- {
- "archivedotorg",
- "defuddle",
- "favicon",
- "forumdl",
+ (
+ "chrome",
+ "infiniscroll",
+ "modalcloser",
+ "ublock",
+ "istilldontcareaboutcookies",
+ "twocaptcha",
+ "claudechrome",
+ ),
+ ),
+ (
+ "media_plugins",
+ "Media",
+ "",
+ "",
+ "",
+ (
+ "staticfile",
+ "responses",
+ "ytdlp",
"gallerydl",
"git",
- "htmltotext",
- "mercury",
- "papersdl",
- "readability",
- "trafilatura",
- "wget",
- "ytdlp",
- },
+ ),
),
(
- "parsing_plugins",
- "Parsing",
+ "text_plugins",
+ "Text",
"",
"",
"",
- {
+ (
+ "readability",
+ "htmltotext",
+ "defuddle",
+ "forumdl",
+ "mercury",
+ "trafilatura",
+ "liteparse",
+ "opendataloader",
+ "papersdl",
+ ),
+ ),
+ (
+ "metadata_plugins",
+ "Metadata",
+ "",
+ "",
+ "",
+ (
+ "title",
+ "favicon",
+ "headers",
+ "redirects",
+ "accessibility",
+ "consolelog",
+ "sslcerts",
+ "dns",
+ "seo",
+ "hashes",
+ ),
+ ),
+ (
+ "postprocessing_plugins",
+ "Postprocessing",
+ "",
+ "",
+ "",
+ (
+ "parse_dom_outlinks",
"parse_html_urls",
"parse_jsonl_urls",
"parse_netscape_urls",
"parse_rss_urls",
"parse_txt_urls",
- },
+ "claudecode",
+ "claudecodecleanup",
+ "claudecodeextract",
+ ),
),
- (
- "search_plugins",
- "Search",
- "(defaults to SEARCH_BACKEND_ENGINE)",
- "",
- "",
- {
- "search_backend_ripgrep",
- "search_backend_sonic",
- "search_backend_sqlite",
- },
- ),
- ("binary_plugins", "Binary Providers", "", "", "", {"apt", "brew", "custom", "env", "npm", "pip"}),
- ("extension_plugins", "Browser Extensions", "", "", "", {"twocaptcha", "istilldontcareaboutcookies", "ublock"}),
)
+HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
+ "apt",
+ "base",
+ "bash",
+ "brew",
+ "cargo",
+ "chromewebstore",
+ "env",
+ "media",
+ "npm",
+ "pip",
+ "puppeteer",
+ "search_backend_ripgrep",
+ "search_backend_sonic",
+ "search_backend_sqlite",
+ "ssl",
+}
def get_plugin_choices():
@@ -224,25 +267,21 @@ class PluginConfigFormMixin:
runtime_config = runtime_config or get_config()
self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
+ other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
if field_name in self.fields:
get_choice_field(self, field_name).choices = [
- (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in plugin_names
+ (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
]
if "other_plugins" in self.fields:
- get_choice_field(self, "other_plugins").choices = [
- (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p not in grouped_plugins
- ]
+ get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
- if "search_plugins" in self.fields:
- required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip()
- search_choices = [choice[0] for choice in get_choice_field(self, "search_plugins").choices]
- if required_search_plugin in search_choices:
- get_choice_field(self, "search_plugins").initial = [required_search_plugin]
-
- group_specs = (*PLUGIN_GROUP_DEFINITIONS, ("other_plugins", "Other", "", "", "", set(all_plugins) - grouped_plugins))
+ group_specs = (
+ *PLUGIN_GROUP_DEFINITIONS,
+ ("other_plugins", "Other", "", "", "", other_plugins),
+ )
self.plugin_groups = [
{
"field_name": field_name,
@@ -260,7 +299,7 @@ class PluginConfigFormMixin:
def _build_plugin_cards(
self,
field_name: str,
- plugin_names: set[str],
+ plugin_names: Iterable[str],
plugin_configs: dict[str, dict[str, Any]],
runtime_config: Mapping[str, Any],
) -> list[dict[str, Any]]:
@@ -268,7 +307,8 @@ class PluginConfigFormMixin:
choices = list(get_choice_field(self, field_name).choices)
selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
else:
- choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(get_plugins()) if p in plugin_names]
+ all_plugins = get_plugins()
+ choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
selected_values = set()
cards = []
@@ -517,6 +557,17 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
},
),
)
+ crawl_max_concurrent_snapshots = forms.IntegerField(
+ label="Max concurrent snapshots",
+ required=False,
+ min_value=1,
+ widget=forms.NumberInput(
+ attrs={
+ "min": 1,
+ "step": 1,
+ },
+ ),
+ )
notes = forms.CharField(
label="Notes",
strip=True,
@@ -534,44 +585,44 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
)
# Plugin groups
- chrome_plugins = forms.MultipleChoiceField(
- label="Chrome-dependent plugins",
+ main_plugins = forms.MultipleChoiceField(
+ label="Main",
required=False,
widget=forms.CheckboxSelectMultiple,
choices=[], # populated in __init__
)
- archiving_plugins = forms.MultipleChoiceField(
- label="Archiving",
+ page_setup_plugins = forms.MultipleChoiceField(
+ label="Page Setup",
required=False,
widget=forms.CheckboxSelectMultiple,
choices=[],
)
- parsing_plugins = forms.MultipleChoiceField(
- label="Parsing",
+ media_plugins = forms.MultipleChoiceField(
+ label="Media",
required=False,
widget=forms.CheckboxSelectMultiple,
choices=[],
)
- search_plugins = forms.MultipleChoiceField(
- label="Search",
+ text_plugins = forms.MultipleChoiceField(
+ label="Text",
required=False,
widget=forms.CheckboxSelectMultiple,
choices=[],
)
- binary_plugins = forms.MultipleChoiceField(
- label="Binary providers",
+ metadata_plugins = forms.MultipleChoiceField(
+ label="Metadata",
required=False,
widget=forms.CheckboxSelectMultiple,
choices=[],
)
- extension_plugins = forms.MultipleChoiceField(
- label="Browser extensions",
+ postprocessing_plugins = forms.MultipleChoiceField(
+ label="Postprocessing",
required=False,
widget=forms.CheckboxSelectMultiple,
choices=[],
)
other_plugins = forms.MultipleChoiceField(
- label="Other plugins",
+ label="Other",
required=False,
widget=forms.CheckboxSelectMultiple,
choices=[],
@@ -613,6 +664,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
default_persona = Persona.get_or_create_default()
self.fields["persona"].queryset = Persona.objects.order_by("name")
self.fields["persona"].initial = default_persona.name
+ self.fields["crawl_max_concurrent_snapshots"].initial = get_config(persona=default_persona).CRAWL_MAX_CONCURRENT_SNAPSHOTS
selected_persona = default_persona
if self.is_bound:
@@ -625,12 +677,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
# Combine all plugin groups into single list
all_selected_plugins = []
for field in [
- "chrome_plugins",
- "archiving_plugins",
- "parsing_plugins",
- "search_plugins",
- "binary_plugins",
- "extension_plugins",
+ "main_plugins",
+ "page_setup_plugins",
+ "media_plugins",
+ "text_plugins",
+ "metadata_plugins",
+ "postprocessing_plugins",
"other_plugins",
]:
selected = cleaned_data.get(field)
@@ -696,6 +748,15 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
return value
+ def clean_crawl_max_concurrent_snapshots(self):
+ value = self.cleaned_data.get("crawl_max_concurrent_snapshots")
+ if value in (None, ""):
+ value = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
+ value = int(value)
+ if value < 1:
+ raise forms.ValidationError("Max concurrent snapshots must be at least 1.")
+ return value
+
def clean_schedule(self):
schedule = (self.cleaned_data.get("schedule") or "").strip()
if not schedule:
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index 9e5549a8..03640266 100755
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -2614,6 +2614,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
embeddable_exts = {
"html",
"htm",
+ "mhtml",
+ "mht",
"pdf",
"txt",
"md",
@@ -3459,6 +3461,8 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
"output.html",
"content.html",
"article.html",
+ "snapshot.mhtml",
+ "snapshot.mht",
"output.pdf",
"index.pdf",
"content.txt",
@@ -3474,7 +3478,7 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
return candidate
ext_groups = (
- (".html", ".htm", ".pdf"),
+ (".html", ".htm", ".mhtml", ".mht", ".pdf"),
(".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
(".json", ".jsonl", ".txt", ".md", ".csv", ".tsv"),
(".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
index 9a80f60b..97970bb5 100644
--- a/archivebox/core/settings.py
+++ b/archivebox/core/settings.py
@@ -439,6 +439,7 @@ LOGGING = SETTINGS_LOGGING
# Add default webhook configuration to the User model
SIGNAL_WEBHOOKS_CUSTOM_MODEL = "archivebox.api.models.OutboundWebhook"
SIGNAL_WEBHOOKS: dict[str, object] = {
+ "TIMEOUT": 30,
"TASK_HANDLER": "archivebox.api.webhooks.transaction_on_commit_task_handler",
"ERROR_HANDLER": "archivebox.api.webhooks.warning_error_handler",
"HOOKS": {
diff --git a/archivebox/core/views.py b/archivebox/core/views.py
index b898d722..95e83d9e 100644
--- a/archivebox/core/views.py
+++ b/archivebox/core/views.py
@@ -231,6 +231,8 @@ class SnapshotView(View):
archiveresults.values(),
key=lambda r: all_types.index(r["name"]) if r["name"] in all_types else -r["size"],
)
+ if best_result["path"] == "about:blank" and ordered_outputs:
+ best_result = ordered_outputs[0]
non_compact_outputs = [out for out in ordered_outputs if not out.get("is_compact") and not out.get("is_metadata")]
compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")]
tag_widget = TagEditorWidget()
@@ -1058,6 +1060,7 @@ class AddView(UserPassesTestMixin, FormView):
max_urls = int(form.cleaned_data.get("max_urls") or 0)
crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0)
snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0)
+ crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"])
plugins = ",".join(form.cleaned_data.get("plugins", []))
schedule = form.cleaned_data.get("schedule", "").strip()
persona = form.cleaned_data.get("persona")
@@ -1098,6 +1101,7 @@ class AddView(UserPassesTestMixin, FormView):
"DEPTH": depth,
"PLUGINS": plugins or "",
"DEFAULT_PERSONA": persona_name,
+ "CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
}
# Merge custom config overrides
@@ -1232,6 +1236,7 @@ class WebAddView(AddView):
"max_urls": defaults_form.fields["max_urls"].initial or 0,
"crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0",
"snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0",
+ "crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial,
"persona": defaults_form.fields["persona"].initial or "Default",
"config": "{}",
},
diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py
index ef70cc1d..94bef737 100755
--- a/archivebox/crawls/models.py
+++ b/archivebox/crawls/models.py
@@ -194,6 +194,13 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
else:
config.pop("SNAPSHOT_MAX_SIZE", None)
+ if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
+ raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
+ if raw_concurrency in (None, ""):
+ config.pop("CRAWL_MAX_CONCURRENT_SNAPSHOTS", None)
+ else:
+ config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = max(1, int(raw_concurrency))
+
if config != (self.config or {}):
self.config = config
update_fields = kwargs.get("update_fields")
diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py
index 0a7df632..3a4f1a1b 100755
--- a/archivebox/machine/models.py
+++ b/archivebox/machine/models.py
@@ -2348,10 +2348,10 @@ class Process(models.Model):
Number of zombie Chrome processes killed
"""
import subprocess
- from pathlib import Path
+ from importlib.resources import files
from archivebox.config.common import get_config
- chrome_utils = Path(__file__).parent.parent / "plugins" / "chrome" / "chrome_utils.js"
+ chrome_utils = files("abx_plugins.plugins.chrome").joinpath("chrome_utils.js")
if not chrome_utils.exists():
return 0
diff --git a/archivebox/misc/serve_static.py b/archivebox/misc/serve_static.py
index 6fa7ce2a..271e8c9e 100644
--- a/archivebox/misc/serve_static.py
+++ b/archivebox/misc/serve_static.py
@@ -264,6 +264,8 @@ mimetypes.add_type("text/csv", ".csv")
mimetypes.add_type("text/tab-separated-values", ".tsv")
mimetypes.add_type("application/xml", ".xml")
mimetypes.add_type("image/svg+xml", ".svg")
+mimetypes.add_type("multipart/related", ".mhtml")
+mimetypes.add_type("multipart/related", ".mht")
try:
_markdown = getattr(importlib.import_module("markdown"), "markdown")
diff --git a/archivebox/services/binary_service.py b/archivebox/services/binary_service.py
index d34c751d..16a44cd7 100644
--- a/archivebox/services/binary_service.py
+++ b/archivebox/services/binary_service.py
@@ -48,6 +48,23 @@ class BinaryService(BaseService):
)
cached = None
if installed is not None:
+ from abxpkg import BinProvider, PROVIDER_CLASS_BY_NAME
+
+ binary_env: dict[str, str] = {}
+ provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
+ provider_class = PROVIDER_CLASS_BY_NAME.get(provider_name)
+ if provider_class is not None:
+ provider = provider_class()
+ overrides = installed.overrides if isinstance(installed.overrides, dict) else {}
+ provider_overrides = overrides.get(provider_name)
+ if isinstance(provider_overrides, dict):
+ provider = provider.get_provider_with_overrides(
+ overrides={installed.name: provider_overrides},
+ )
+ binary_env = BinProvider.build_exec_env(
+ providers=[provider],
+ base_env={},
+ )
cached = {
"abspath": installed.abspath,
"version": installed.version or "",
@@ -56,6 +73,7 @@ class BinaryService(BaseService):
"binprovider": installed.binprovider or "",
"machine_id": str(installed.machine_id),
"overrides": installed.overrides or {},
+ "env": binary_env,
}
if cached is not None:
binary_event = BinaryEvent(
@@ -68,6 +86,7 @@ class BinaryService(BaseService):
binproviders=event.binproviders or cached["binproviders"],
binprovider=cached["binprovider"],
overrides=event.overrides or cached["overrides"],
+ env=cached["env"],
binary_id=event.binary_id,
machine_id=cached["machine_id"],
)
diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py
index 5dd9fdd3..ef7ba6d9 100644
--- a/archivebox/services/runner.py
+++ b/archivebox/services/runner.py
@@ -173,8 +173,6 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
class CrawlRunner:
- MAX_CONCURRENT_SNAPSHOTS = 8
-
def __init__(
self,
crawl,
@@ -207,7 +205,7 @@ class CrawlRunner:
self.selected_plugins = selected_plugins
self.initial_snapshot_ids = snapshot_ids
self.snapshot_tasks: dict[str, asyncio.Task[None]] = {}
- self.snapshot_semaphore = asyncio.Semaphore(self.MAX_CONCURRENT_SNAPSHOTS)
+ self.snapshot_semaphore = asyncio.Semaphore(1)
self.persona = None
self.base_config: dict[str, Any] = {}
self.derived_config: dict[str, Any] = {}
@@ -244,6 +242,8 @@ class CrawlRunner:
root_snapshot_id: str | None = None
try:
snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
+ max_concurrent_snapshots = max(1, int(self.base_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS))
+ self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
live_ui = self._create_live_ui()
with live_ui if live_ui is not None else nullcontext():
await heartbeat.start()
diff --git a/archivebox/templates/core/add.html b/archivebox/templates/core/add.html
index b7193d12..2ee54339 100644
--- a/archivebox/templates/core/add.html
+++ b/archivebox/templates/core/add.html
@@ -128,6 +128,15 @@
{% endif %}
0 means unlimited per snapshot. Accepts bytes or units like 45mb and 1gb.
+
+
+ {{ form.crawl_max_concurrent_snapshots.label_tag }}
+ {{ form.crawl_max_concurrent_snapshots }}
+ {% if form.crawl_max_concurrent_snapshots.errors %}
+
{{ form.crawl_max_concurrent_snapshots.errors }}
+ {% endif %}
+
Caps how many snapshots from this crawl archive at the same time.
+
Quick Select:
-
-
@@ -292,12 +299,12 @@
function getRequiredSearchCheckbox() {
if (!requiredSearchPlugin) return null;
- return document.querySelector(`input[name="search_plugins"][value="${requiredSearchPlugin}"]`);
+ return document.querySelector(`.plugin-section-toggle[value="${requiredSearchPlugin}"]`);
}
function getPluginCheckbox(pluginName) {
if (!pluginName) return null;
- return document.querySelector(`.plugin-checkboxes input[type="checkbox"][value="${pluginName}"]`);
+ return document.querySelector(`.plugin-section-toggle[value="${pluginName}"]`);
}
function getRequiredPlugins(pluginName) {
@@ -367,7 +374,7 @@
}
function syncEnabledPluginsConfig() {
- const selectedPlugins = Array.from(document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]:checked'))
+ const selectedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
.map(cb => cb.value)
.filter(Boolean)
.sort((left, right) => left.localeCompare(right));
@@ -387,6 +394,11 @@
function applyPersonaConfig(personaName) {
const personaData = personaConfigMap[personaName];
if (!personaData) return;
+ const concurrencyInput = document.querySelector('input[name="crawl_max_concurrent_snapshots"]');
+ const concurrencyValue = personaData.effective_config?.CRAWL_MAX_CONCURRENT_SNAPSHOTS;
+ if (concurrencyInput && concurrencyValue) {
+ concurrencyInput.value = concurrencyValue;
+ }
if (typeof window.archiveboxSetPluginConfigValues === 'function') {
window.archiveboxSetPluginConfigValues(personaData.effective_config || {}, personaData.binary_urls || {});
}
@@ -429,7 +441,7 @@
}
function normalizePluginSelections() {
- const checkedPlugins = Array.from(document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]:checked'))
+ const checkedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
.map(cb => cb.value)
.filter(Boolean);
checkedPlugins.forEach(pluginName => ensureRequiredPluginsChecked(pluginName));
@@ -1041,15 +1053,13 @@
// Plugin Presets
const presetConfigs = {
- 'quick-archive': ['screenshot', 'dom', 'favicon', 'wget', 'title'],
- 'full-chrome': ['chrome', 'screenshot', 'pdf', 'dom', 'singlefile', 'consolelog', 'redirects', 'responses', 'ssl', 'headers', 'title', 'accessibility', 'seo'],
'text-only': ['wget', 'readability', 'mercury', 'htmltotext', 'title', 'favicon']
};
document.querySelectorAll('.preset-btn').forEach(btn => {
btn.addEventListener('click', function() {
const preset = this.dataset.preset;
- const allCheckboxes = document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]');
+ const allCheckboxes = document.querySelectorAll('.plugin-section-toggle');
const requiredSearchPreference = getStoredPluginPreference(requiredSearchPlugin);
if (preset === 'select-all') {
@@ -1083,7 +1093,7 @@
const group = btn.dataset.group;
const container = document.getElementById(group + '-plugins');
if (!container) return;
- const checkboxes = Array.from(container.querySelectorAll('input[type="checkbox"]'));
+ const checkboxes = Array.from(container.querySelectorAll('.plugin-section-toggle'));
const allChecked = checkboxes.length > 0 && checkboxes.every(cb => cb.checked);
btn.textContent = allChecked ? 'Deselect All Chrome' : 'Select All Chrome';
});
@@ -1095,7 +1105,7 @@
const container = document.getElementById(group + '-plugins');
if (!container) return;
- const checkboxes = Array.from(container.querySelectorAll('input[type="checkbox"]'));
+ const checkboxes = Array.from(container.querySelectorAll('.plugin-section-toggle'));
const allChecked = checkboxes.length > 0 && checkboxes.every(cb => cb.checked);
const requiredSearchPreference = getStoredPluginPreference(requiredSearchPlugin);
@@ -1112,7 +1122,7 @@
});
});
- document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]').forEach(checkbox => {
+ document.querySelectorAll('.plugin-section-toggle').forEach(checkbox => {
checkbox.addEventListener('change', function() {
if (this.checked) {
ensureRequiredPluginsChecked(this.value);
diff --git a/archivebox/tests/test_add_view.py b/archivebox/tests/test_add_view.py
index 1d2df6ae..ece5bca3 100644
--- a/archivebox/tests/test_add_view.py
+++ b/archivebox/tests/test_add_view.py
@@ -46,6 +46,7 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m
assert 'name="max_urls"' in body
assert 'name="crawl_max_size"' in body
assert 'name="snapshot_max_size"' in body
+ assert 'name="crawl_max_concurrent_snapshots"' in body
assert 'Crawl Plugins")
assert "data-url-regex=" in body
@@ -91,7 +92,7 @@ def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_
assert persona_config_map["Private"]["effective_config"]["YTDLP_COOKIES_FILE"] == "/tmp/archivebox-private-cookies.txt"
-def test_add_view_checks_configured_search_backend_by_default(client, monkeypatch):
+def test_add_view_hides_search_backend_plugins(client, monkeypatch):
monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite")
@@ -99,10 +100,10 @@ def test_add_view_checks_configured_search_backend_by_default(client, monkeypatc
body = response.content.decode()
assert response.status_code == 200
- assert re.search(
- r']* checked\b',
- body,
- )
+ assert not re.search(r']*value="search_backend_sqlite"', body)
+ assert 'data-plugin-name="search_backend_ripgrep"' not in body
+ assert 'data-plugin-name="search_backend_sonic"' not in body
+ assert 'data-plugin-name="search_backend_sqlite"' not in body
assert "const requiredSearchPlugin = 'search_backend_sqlite';" in body
@@ -119,6 +120,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
"max_urls": "3",
"crawl_max_size": "45mb",
"snapshot_max_size": "5mb",
+ "crawl_max_concurrent_snapshots": "4",
"url_filters_allowlist": "example.com\n*.example.com",
"url_filters_denylist": "cdn.example.com",
"notes": "Created from /add/",
@@ -143,6 +145,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
assert crawl.config["CRAWL_MAX_URLS"] == 3
assert crawl.config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024
assert crawl.config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024
+ assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 4
assert crawl.config["URL_ALLOWLIST"] == "example.com\n*.example.com"
assert crawl.config["URL_DENYLIST"] == "cdn.example.com"
assert "OVERWRITE" not in crawl.config
@@ -208,7 +211,7 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc
"schedule": "",
"persona": "Default",
"index_only": "",
- "archiving_plugins": ["wget"],
+ "main_plugins": ["wget"],
"plugin_config__wget__WGET_TIMEOUT": "77",
"plugin_config__wget__WGET_WARC_ENABLED": "false",
"config": "{}",
diff --git a/archivebox/tests/test_schedule_e2e.py b/archivebox/tests/test_schedule_e2e.py
index 16fc5ffb..5694ec2c 100644
--- a/archivebox/tests/test_schedule_e2e.py
+++ b/archivebox/tests/test_schedule_e2e.py
@@ -464,8 +464,8 @@ def test_web_ui_add_depth_two_crawls_and_renders_real_outputs_over_running_serve
"max_urls": "20",
"crawl_max_size": "0",
"snapshot_max_size": "0",
- "archiving_plugins": ["wget"],
- "parsing_plugins": ["parse_html_urls"],
+ "main_plugins": ["wget"],
+ "postprocessing_plugins": ["parse_html_urls"],
"tag": "web-depth-two",
"url_filters_allowlist": r"127\.0\.0\.1[:/].*",
"url_filters_denylist": "",
diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py
index 1f268390..b3fa7e79 100644
--- a/archivebox/workers/supervisord_util.py
+++ b/archivebox/workers/supervisord_util.py
@@ -41,6 +41,9 @@ RUNNER_WORKER = {
"command": _shell_join([sys.executable, "-m", "archivebox", "run", "--daemon"]),
"autostart": "false",
"autorestart": "true",
+ "stopasgroup": "true",
+ "killasgroup": "true",
+ "stopwaitsecs": "30",
"stdout_logfile": "logs/worker_runner.log",
"redirect_stderr": "true",
}
diff --git a/etc/package.json b/etc/package.json
index f6efe7b1..abfadd7f 100644
--- a/etc/package.json
+++ b/etc/package.json
@@ -1,6 +1,6 @@
{
"name": "archivebox",
- "version": "0.9.31rc41",
+ "version": "0.9.31rc42",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {
diff --git a/pyproject.toml b/pyproject.toml
index 5230e5b1..025863af 100755
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "archivebox"
-version = "0.9.31rc41"
+version = "0.9.31rc42"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
### Extractor dependencies (optional binaries detected at runtime via shutil.which)
### Binary/Package Management
"abxbus>=2.5.4", # EventBus API
- "abxpkg>=1.10.31", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
- "abx-plugins>=1.10.96", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
- "abx-dl>=1.10.96", # shared ArchiveBox downloader package with blocking install preflight
+ "abxpkg>=1.10.32", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+ "abx-plugins>=1.10.97", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+ "abx-dl>=1.10.97", # shared ArchiveBox downloader package with blocking install preflight
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
]