release: archivebox 0.9.31rc42

archivebox/api/admin.py

@@ -1,9 +1,12 @@
 __package__ = "archivebox.api"
 
+from django import forms
 from django.contrib import admin
 from django.http import HttpRequest
-from signal_webhooks.admin import WebhookAdmin
-from signal_webhooks.utils import get_webhook_model
+from django.utils.text import capfirst
+from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
+from signal_webhooks.settings import webhook_settings
+from signal_webhooks.utils import get_webhook_model, model_from_reference
 
 from archivebox.base_models.admin import BaseModelAdmin
 
@@ -50,7 +53,21 @@ class APITokenAdmin(BaseModelAdmin):
     list_per_page = 100
 
 
+class OutboundWebhookAdminForm(WebhookModelForm):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["ref"] = forms.ChoiceField(
+            label=self.fields["ref"].label,
+            help_text=self.fields["ref"].help_text,
+            choices=[
+                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
+                for ref in sorted(webhook_settings.HOOKS)
+            ],
+        )
+
+
 class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
+    form = OutboundWebhookAdminForm
     list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
     sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
     readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)

archivebox/api/v1_crawls.py

@@ -14,6 +14,7 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot
 from archivebox.crawls.models import Crawl
+from archivebox.config.common import get_config
 
 from .auth import API_AUTH_METHODS
 
@@ -38,11 +39,16 @@ class CrawlSchema(Schema):
     max_urls: int
     crawl_max_size: int
     snapshot_max_size: int
+    crawl_max_concurrent_snapshots: int
     tags_str: str
     config: dict
 
     # snapshots: List[SnapshotSchema]
 
+    @staticmethod
+    def resolve_crawl_max_concurrent_snapshots(obj):
+        return int((obj.config or {}).get("CRAWL_MAX_CONCURRENT_SNAPSHOTS") or get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS)
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -74,6 +80,7 @@ class CrawlCreateSchema(Schema):
     max_urls: int = 0
     crawl_max_size: int = 0
     snapshot_max_size: int = 0
+    crawl_max_concurrent_snapshots: int | None = None
     tags: list[str] | None = None
     tags_str: str = ""
     label: str = ""
@@ -112,8 +119,15 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
         raise HttpError(400, "crawl_max_size must be >= 0")
     if data.snapshot_max_size < 0:
         raise HttpError(400, "snapshot_max_size must be >= 0")
+    crawl_max_concurrent_snapshots = data.crawl_max_concurrent_snapshots
+    if crawl_max_concurrent_snapshots is None:
+        crawl_max_concurrent_snapshots = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
+    if crawl_max_concurrent_snapshots < 1:
+        raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1")
 
     tags = normalize_tag_list(data.tags, data.tags_str)
+    config = dict(data.config or {})
+    config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
         max_depth=data.max_depth,
@@ -123,7 +137,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
         tags_str=",".join(tags),
         label=data.label,
         notes=data.notes,
-        config=data.config,
+        config=config,
         status=Crawl.StatusChoices.QUEUED,
         retry_at=timezone.now(),
         created_by=request.user if isinstance(request.user, User) else None,

archivebox/base_models/admin.py

@@ -44,10 +44,27 @@ class KeyValueWidget(forms.Widget):
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
         try:
+            from archivebox.config.common import ArchiveBoxConfig
             from archivebox.hooks import discover_plugin_configs
 
-            plugin_configs = discover_plugin_configs()
             options: dict[str, ConfigOption] = {}
+            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
+            for key, field in ArchiveBoxConfig.model_fields.items():
+                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
+                    continue
+                default = field.default
+                try:
+                    json.dumps(default)
+                except TypeError:
+                    default = str(default)
+                options[key] = {
+                    "plugin": "archivebox",
+                    "type": str(field.annotation),
+                    "default": default,
+                    "description": field.description or "",
+                }
+
+            plugin_configs = discover_plugin_configs()
             for plugin_name, schema in plugin_configs.items():
                 for key, prop in schema.get("properties", {}).items():
                     option: ConfigOption = {

archivebox/cli/archivebox_add.py

@@ -53,6 +53,7 @@ def add(
     max_urls: int = 0,
     crawl_max_size: int | str = 0,
     snapshot_max_size: int | str = 0,
+    crawl_max_concurrent_snapshots: int | None = None,
     tag: str = "",
     url_allowlist: str = "",
     url_denylist: str = "",
@@ -83,6 +84,10 @@ def add(
     max_urls = int(max_urls or 0)
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
+    config = get_config()
+    if crawl_max_concurrent_snapshots is None:
+        crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+    crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
 
     if depth not in (0, 1, 2, 3, 4):
         raise ValueError("Depth must be 0-4")
@@ -92,6 +97,8 @@ def add(
         raise ValueError("crawl_max_size must be >= 0")
     if snapshot_max_size < 0:
         raise ValueError("snapshot_max_size must be >= 0")
+    if crawl_max_concurrent_snapshots < 1:
+        raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
 
     # import models once django is set up
     from archivebox.crawls.models import Crawl
@@ -101,7 +108,6 @@ def add(
     from archivebox.misc.system import get_dir_size
     from archivebox.services.runner import run_crawl
 
-    config = get_config()
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
     if update is None:
@@ -157,6 +163,7 @@ def add(
             "OVERWRITE": overwrite,
             "PLUGINS": plugins,
             "DEFAULT_PERSONA": persona_name,
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
             "PARSER": parser,
             **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
             **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
@@ -260,6 +267,7 @@ def add(
 @click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
 @click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
 @click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
 @click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
 @click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
@@ -289,6 +297,8 @@ def main(**kwargs):
         kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
     except ValueError as err:
         raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
+    if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
+        raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
 
     add(urls=urls, **kwargs)
 

archivebox/cli/archivebox_run.py

@@ -259,6 +259,7 @@ def run_runner(daemon: bool = False) -> int:
 
     Process.cleanup_stale_running()
     Process.cleanup_orphaned_workers()
+    Process.cleanup_orphaned_chrome()
     recover_orphaned_snapshots()
     recover_orphaned_crawls()
     Machine.current()

archivebox/config/common.py

@@ -251,6 +251,10 @@ class ArchivingConfig(BaseConfigSet):
     MAX_DEPTH: int = Field(default=0)
     CRAWL_MAX_URLS: int = Field(default=0)
     CRAWL_MAX_SIZE: int = Field(default=0)
+    CRAWL_MAX_CONCURRENT_SNAPSHOTS: int = Field(
+        default=4,
+        description="Maximum number of snapshots to archive concurrently within one crawl.",
+    )
     SNAPSHOT_MAX_SIZE: int = Field(default=0)
 
     RESOLUTION: str = Field(default="1440,2000")

archivebox/core/forms.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 from pathlib import Path
 from typing import Any
 
@@ -28,80 +28,123 @@ DEPTH_CHOICES = (
 PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
 PLUGIN_GROUP_DEFINITIONS = (
     (
-        "chrome_plugins",
-        "Chrome-dependent plugins",
+        "main_plugins",
+        "Main",
         "",
-        "chrome-plugins",
-        "chrome",
-        {
-            "accessibility",
-            "chrome",
-            "consolelog",
+        "",
+        "",
+        (
             "dom",
-            "headers",
-            "parse_dom_outlinks",
-            "pdf",
-            "redirects",
-            "responses",
             "screenshot",
-            "seo",
+            "pdf",
             "singlefile",
-            "ssl",
-            "staticfile",
-            "title",
-        },
+            "wget",
+            "archivedotorg",
+        ),
     ),
     (
-        "archiving_plugins",
-        "Archiving",
+        "page_setup_plugins",
+        "Page Setup",
         "",
         "",
         "",
-        {
-            "archivedotorg",
-            "defuddle",
-            "favicon",
-            "forumdl",
+        (
+            "chrome",
+            "infiniscroll",
+            "modalcloser",
+            "ublock",
+            "istilldontcareaboutcookies",
+            "twocaptcha",
+            "claudechrome",
+        ),
+    ),
+    (
+        "media_plugins",
+        "Media",
+        "",
+        "",
+        "",
+        (
+            "staticfile",
+            "responses",
+            "ytdlp",
             "gallerydl",
             "git",
-            "htmltotext",
-            "mercury",
-            "papersdl",
-            "readability",
-            "trafilatura",
-            "wget",
-            "ytdlp",
-        },
+        ),
     ),
     (
-        "parsing_plugins",
-        "Parsing",
+        "text_plugins",
+        "Text",
         "",
         "",
         "",
-        {
+        (
+            "readability",
+            "htmltotext",
+            "defuddle",
+            "forumdl",
+            "mercury",
+            "trafilatura",
+            "liteparse",
+            "opendataloader",
+            "papersdl",
+        ),
+    ),
+    (
+        "metadata_plugins",
+        "Metadata",
+        "",
+        "",
+        "",
+        (
+            "title",
+            "favicon",
+            "headers",
+            "redirects",
+            "accessibility",
+            "consolelog",
+            "sslcerts",
+            "dns",
+            "seo",
+            "hashes",
+        ),
+    ),
+    (
+        "postprocessing_plugins",
+        "Postprocessing",
+        "",
+        "",
+        "",
+        (
+            "parse_dom_outlinks",
             "parse_html_urls",
             "parse_jsonl_urls",
             "parse_netscape_urls",
             "parse_rss_urls",
             "parse_txt_urls",
-        },
+            "claudecode",
+            "claudecodecleanup",
+            "claudecodeextract",
+        ),
     ),
-    (
-        "search_plugins",
-        "Search",
-        "(defaults to SEARCH_BACKEND_ENGINE)",
-        "",
-        "",
-        {
-            "search_backend_ripgrep",
-            "search_backend_sonic",
-            "search_backend_sqlite",
-        },
-    ),
-    ("binary_plugins", "Binary Providers", "", "", "", {"apt", "brew", "custom", "env", "npm", "pip"}),
-    ("extension_plugins", "Browser Extensions", "", "", "", {"twocaptcha", "istilldontcareaboutcookies", "ublock"}),
 )
+HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
+    "apt",
+    "base",
+    "bash",
+    "brew",
+    "cargo",
+    "chromewebstore",
+    "env",
+    "media",
+    "npm",
+    "pip",
+    "puppeteer",
+    "search_backend_ripgrep",
+    "search_backend_sonic",
+    "search_backend_sqlite",
+    "ssl",
+}
 
 
 def get_plugin_choices():
@@ -224,25 +267,21 @@ class PluginConfigFormMixin:
         runtime_config = runtime_config or get_config()
         self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
         grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
+        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
 
         for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
             if field_name in self.fields:
                 get_choice_field(self, field_name).choices = [
-                    (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p in plugin_names
+                    (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
                 ]
 
         if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [
-                (p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(all_plugins) if p not in grouped_plugins
-            ]
+            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
 
-        if "search_plugins" in self.fields:
-            required_search_plugin = f"search_backend_{get_config().SEARCH_BACKEND_ENGINE}".strip()
-            search_choices = [choice[0] for choice in get_choice_field(self, "search_plugins").choices]
-            if required_search_plugin in search_choices:
-                get_choice_field(self, "search_plugins").initial = [required_search_plugin]
-
-        group_specs = (*PLUGIN_GROUP_DEFINITIONS, ("other_plugins", "Other", "", "", "", set(all_plugins) - grouped_plugins))
+        group_specs = (
+            *PLUGIN_GROUP_DEFINITIONS,
+            ("other_plugins", "Other", "", "", "", other_plugins),
+        )
         self.plugin_groups = [
             {
                 "field_name": field_name,
@@ -260,7 +299,7 @@ class PluginConfigFormMixin:
     def _build_plugin_cards(
         self,
         field_name: str,
-        plugin_names: set[str],
+        plugin_names: Iterable[str],
         plugin_configs: dict[str, dict[str, Any]],
         runtime_config: Mapping[str, Any],
     ) -> list[dict[str, Any]]:
@@ -268,7 +307,8 @@ class PluginConfigFormMixin:
             choices = list(get_choice_field(self, field_name).choices)
             selected_values = set(self.data.getlist(field_name)) if self.is_bound else set(get_choice_field(self, field_name).initial or [])
         else:
-            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in sorted(get_plugins()) if p in plugin_names]
+            all_plugins = get_plugins()
+            choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins]
             selected_values = set()
 
         cards = []
@@ -517,6 +557,17 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             },
         ),
     )
+    crawl_max_concurrent_snapshots = forms.IntegerField(
+        label="Max concurrent snapshots",
+        required=False,
+        min_value=1,
+        widget=forms.NumberInput(
+            attrs={
+                "min": 1,
+                "step": 1,
+            },
+        ),
+    )
     notes = forms.CharField(
         label="Notes",
         strip=True,
@@ -534,44 +585,44 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
     )
 
     # Plugin groups
-    chrome_plugins = forms.MultipleChoiceField(
-        label="Chrome-dependent plugins",
+    main_plugins = forms.MultipleChoiceField(
+        label="Main",
         required=False,
         widget=forms.CheckboxSelectMultiple,
         choices=[],  # populated in __init__
     )
-    archiving_plugins = forms.MultipleChoiceField(
-        label="Archiving",
+    page_setup_plugins = forms.MultipleChoiceField(
+        label="Page Setup",
         required=False,
         widget=forms.CheckboxSelectMultiple,
         choices=[],
     )
-    parsing_plugins = forms.MultipleChoiceField(
-        label="Parsing",
+    media_plugins = forms.MultipleChoiceField(
+        label="Media",
         required=False,
         widget=forms.CheckboxSelectMultiple,
         choices=[],
     )
-    search_plugins = forms.MultipleChoiceField(
-        label="Search",
+    text_plugins = forms.MultipleChoiceField(
+        label="Text",
         required=False,
         widget=forms.CheckboxSelectMultiple,
         choices=[],
     )
-    binary_plugins = forms.MultipleChoiceField(
-        label="Binary providers",
+    metadata_plugins = forms.MultipleChoiceField(
+        label="Metadata",
         required=False,
         widget=forms.CheckboxSelectMultiple,
         choices=[],
     )
-    extension_plugins = forms.MultipleChoiceField(
-        label="Browser extensions",
+    postprocessing_plugins = forms.MultipleChoiceField(
+        label="Postprocessing",
         required=False,
         widget=forms.CheckboxSelectMultiple,
         choices=[],
     )
     other_plugins = forms.MultipleChoiceField(
-        label="Other plugins",
+        label="Other",
         required=False,
         widget=forms.CheckboxSelectMultiple,
         choices=[],
@@ -613,6 +664,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         default_persona = Persona.get_or_create_default()
         self.fields["persona"].queryset = Persona.objects.order_by("name")
         self.fields["persona"].initial = default_persona.name
+        self.fields["crawl_max_concurrent_snapshots"].initial = get_config(persona=default_persona).CRAWL_MAX_CONCURRENT_SNAPSHOTS
 
         selected_persona = default_persona
         if self.is_bound:
@@ -625,12 +677,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         # Combine all plugin groups into single list
         all_selected_plugins = []
         for field in [
-            "chrome_plugins",
-            "archiving_plugins",
-            "parsing_plugins",
-            "search_plugins",
-            "binary_plugins",
-            "extension_plugins",
+            "main_plugins",
+            "page_setup_plugins",
+            "media_plugins",
+            "text_plugins",
+            "metadata_plugins",
+            "postprocessing_plugins",
             "other_plugins",
         ]:
             selected = cleaned_data.get(field)
@@ -696,6 +748,15 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             raise forms.ValidationError("Max snapshot size must be 0 or a positive number of bytes.")
         return value
 
+    def clean_crawl_max_concurrent_snapshots(self):
+        value = self.cleaned_data.get("crawl_max_concurrent_snapshots")
+        if value in (None, ""):
+            value = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        value = int(value)
+        if value < 1:
+            raise forms.ValidationError("Max concurrent snapshots must be at least 1.")
+        return value
+
     def clean_schedule(self):
         schedule = (self.cleaned_data.get("schedule") or "").strip()
         if not schedule:

archivebox/core/models.py

@@ -2614,6 +2614,8 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHea
         embeddable_exts = {
             "html",
             "htm",
+            "mhtml",
+            "mht",
             "pdf",
             "txt",
             "md",
@@ -3459,6 +3461,8 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
             "output.html",
             "content.html",
             "article.html",
+            "snapshot.mhtml",
+            "snapshot.mht",
             "output.pdf",
             "index.pdf",
             "content.txt",
@@ -3474,7 +3478,7 @@ class ArchiveResult(ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
                     return candidate
 
         ext_groups = (
-            (".html", ".htm", ".pdf"),
+            (".html", ".htm", ".mhtml", ".mht", ".pdf"),
             (".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
             (".json", ".jsonl", ".txt", ".md", ".csv", ".tsv"),
             (".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),

archivebox/core/settings.py

@@ -439,6 +439,7 @@ LOGGING = SETTINGS_LOGGING
 # Add default webhook configuration to the User model
 SIGNAL_WEBHOOKS_CUSTOM_MODEL = "archivebox.api.models.OutboundWebhook"
 SIGNAL_WEBHOOKS: dict[str, object] = {
+    "TIMEOUT": 30,
     "TASK_HANDLER": "archivebox.api.webhooks.transaction_on_commit_task_handler",
     "ERROR_HANDLER": "archivebox.api.webhooks.warning_error_handler",
     "HOOKS": {

archivebox/core/views.py

@@ -231,6 +231,8 @@ class SnapshotView(View):
             archiveresults.values(),
             key=lambda r: all_types.index(r["name"]) if r["name"] in all_types else -r["size"],
         )
+        if best_result["path"] == "about:blank" and ordered_outputs:
+            best_result = ordered_outputs[0]
         non_compact_outputs = [out for out in ordered_outputs if not out.get("is_compact") and not out.get("is_metadata")]
         compact_outputs = [out for out in ordered_outputs if out.get("is_compact") or out.get("is_metadata")]
         tag_widget = TagEditorWidget()
@@ -1058,6 +1060,7 @@ class AddView(UserPassesTestMixin, FormView):
         max_urls = int(form.cleaned_data.get("max_urls") or 0)
         crawl_max_size = int(form.cleaned_data.get("crawl_max_size") or 0)
         snapshot_max_size = int(form.cleaned_data.get("snapshot_max_size") or 0)
+        crawl_max_concurrent_snapshots = int(form.cleaned_data["crawl_max_concurrent_snapshots"])
         plugins = ",".join(form.cleaned_data.get("plugins", []))
         schedule = form.cleaned_data.get("schedule", "").strip()
         persona = form.cleaned_data.get("persona")
@@ -1098,6 +1101,7 @@ class AddView(UserPassesTestMixin, FormView):
             "DEPTH": depth,
             "PLUGINS": plugins or "",
             "DEFAULT_PERSONA": persona_name,
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
         }
 
         # Merge custom config overrides
@@ -1232,6 +1236,7 @@ class WebAddView(AddView):
                 "max_urls": defaults_form.fields["max_urls"].initial or 0,
                 "crawl_max_size": defaults_form.fields["crawl_max_size"].initial or "0",
                 "snapshot_max_size": defaults_form.fields["snapshot_max_size"].initial or "0",
+                "crawl_max_concurrent_snapshots": defaults_form.fields["crawl_max_concurrent_snapshots"].initial,
                 "persona": defaults_form.fields["persona"].initial or "Default",
                 "config": "{}",
             },

archivebox/crawls/models.py

@@ -194,6 +194,13 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
         else:
             config.pop("SNAPSHOT_MAX_SIZE", None)
 
+        if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
+            raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
+            if raw_concurrency in (None, ""):
+                config.pop("CRAWL_MAX_CONCURRENT_SNAPSHOTS", None)
+            else:
+                config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = max(1, int(raw_concurrency))
+
         if config != (self.config or {}):
             self.config = config
             update_fields = kwargs.get("update_fields")

archivebox/machine/models.py

@@ -2348,10 +2348,10 @@ class Process(models.Model):
             Number of zombie Chrome processes killed
         """
         import subprocess
-        from pathlib import Path
+        from importlib.resources import files
         from archivebox.config.common import get_config
 
-        chrome_utils = Path(__file__).parent.parent / "plugins" / "chrome" / "chrome_utils.js"
+        chrome_utils = files("abx_plugins.plugins.chrome").joinpath("chrome_utils.js")
         if not chrome_utils.exists():
             return 0
 

archivebox/misc/serve_static.py

@@ -264,6 +264,8 @@ mimetypes.add_type("text/csv", ".csv")
 mimetypes.add_type("text/tab-separated-values", ".tsv")
 mimetypes.add_type("application/xml", ".xml")
 mimetypes.add_type("image/svg+xml", ".svg")
+mimetypes.add_type("multipart/related", ".mhtml")
+mimetypes.add_type("multipart/related", ".mht")
 
 try:
     _markdown = getattr(importlib.import_module("markdown"), "markdown")

archivebox/services/binary_service.py

@@ -48,6 +48,23 @@ class BinaryService(BaseService):
         )
         cached = None
         if installed is not None:
+            from abxpkg import BinProvider, PROVIDER_CLASS_BY_NAME
+
+            binary_env: dict[str, str] = {}
+            provider_name = (installed.binprovider or installed.binproviders.split(",", 1)[0]).strip()
+            provider_class = PROVIDER_CLASS_BY_NAME.get(provider_name)
+            if provider_class is not None:
+                provider = provider_class()
+                overrides = installed.overrides if isinstance(installed.overrides, dict) else {}
+                provider_overrides = overrides.get(provider_name)
+                if isinstance(provider_overrides, dict):
+                    provider = provider.get_provider_with_overrides(
+                        overrides={installed.name: provider_overrides},
+                    )
+                binary_env = BinProvider.build_exec_env(
+                    providers=[provider],
+                    base_env={},
+                )
             cached = {
                 "abspath": installed.abspath,
                 "version": installed.version or "",
@@ -56,6 +73,7 @@ class BinaryService(BaseService):
                 "binprovider": installed.binprovider or "",
                 "machine_id": str(installed.machine_id),
                 "overrides": installed.overrides or {},
+                "env": binary_env,
             }
         if cached is not None:
             binary_event = BinaryEvent(
@@ -68,6 +86,7 @@ class BinaryService(BaseService):
                 binproviders=event.binproviders or cached["binproviders"],
                 binprovider=cached["binprovider"],
                 overrides=event.overrides or cached["overrides"],
+                env=cached["env"],
                 binary_id=event.binary_id,
                 machine_id=cached["machine_id"],
             )

archivebox/services/runner.py

@@ -173,8 +173,6 @@ def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
 
 
 class CrawlRunner:
-    MAX_CONCURRENT_SNAPSHOTS = 8
-
     def __init__(
         self,
         crawl,
@@ -207,7 +205,7 @@ class CrawlRunner:
         self.selected_plugins = selected_plugins
         self.initial_snapshot_ids = snapshot_ids
         self.snapshot_tasks: dict[str, asyncio.Task[None]] = {}
-        self.snapshot_semaphore = asyncio.Semaphore(self.MAX_CONCURRENT_SNAPSHOTS)
+        self.snapshot_semaphore = asyncio.Semaphore(1)
         self.persona = None
         self.base_config: dict[str, Any] = {}
         self.derived_config: dict[str, Any] = {}
@@ -244,6 +242,8 @@ class CrawlRunner:
         root_snapshot_id: str | None = None
         try:
             snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
+            max_concurrent_snapshots = max(1, int(self.base_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS))
+            self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
             live_ui = self._create_live_ui()
             with live_ui if live_ui is not None else nullcontext():
                 await heartbeat.start()

archivebox/templates/core/add.html

@@ -128,6 +128,15 @@
                                     {% endif %}
                                     <div class="help-text">0 means unlimited per snapshot. Accepts bytes or units like <code>45mb</code> and <code>1gb</code>.</div>
                                 </div>
+
+                                <div class="crawl-limit-field">
+                                    {{ form.crawl_max_concurrent_snapshots.label_tag }}
+                                    {{ form.crawl_max_concurrent_snapshots }}
+                                    {% if form.crawl_max_concurrent_snapshots.errors %}
+                                        <div class="error">{{ form.crawl_max_concurrent_snapshots.errors }}</div>
+                                    {% endif %}
+                                    <div class="help-text">Caps how many snapshots from this crawl archive at the same time.</div>
+                                </div>
                             </div>
                         </div>
 
@@ -171,8 +180,6 @@
 
                     <div class="plugin-presets">
                         <span class="preset-label">Quick Select:</span>
-                        <button type="button" class="preset-btn" data-preset="quick-archive">📦 Quick Archive</button>
-                        <button type="button" class="preset-btn" data-preset="full-chrome">🌐 Full Chrome</button>
                         <button type="button" class="preset-btn" data-preset="text-only">📄 Text Only</button>
                         <button type="button" class="preset-btn" data-preset="select-all">✓ Select All</button>
                         <button type="button" class="preset-btn" data-preset="clear-all">✗ Clear All</button>
@@ -292,12 +299,12 @@
 
                 function getRequiredSearchCheckbox() {
                     if (!requiredSearchPlugin) return null;
-                    return document.querySelector(`input[name="search_plugins"][value="${requiredSearchPlugin}"]`);
+                    return document.querySelector(`.plugin-section-toggle[value="${requiredSearchPlugin}"]`);
                 }
 
                 function getPluginCheckbox(pluginName) {
                     if (!pluginName) return null;
-                    return document.querySelector(`.plugin-checkboxes input[type="checkbox"][value="${pluginName}"]`);
+                    return document.querySelector(`.plugin-section-toggle[value="${pluginName}"]`);
                 }
 
                 function getRequiredPlugins(pluginName) {
@@ -367,7 +374,7 @@
                 }
 
                 function syncEnabledPluginsConfig() {
-                    const selectedPlugins = Array.from(document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]:checked'))
+                    const selectedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
                         .map(cb => cb.value)
                         .filter(Boolean)
                         .sort((left, right) => left.localeCompare(right));
@@ -387,6 +394,11 @@
                 function applyPersonaConfig(personaName) {
                     const personaData = personaConfigMap[personaName];
                     if (!personaData) return;
+                    const concurrencyInput = document.querySelector('input[name="crawl_max_concurrent_snapshots"]');
+                    const concurrencyValue = personaData.effective_config?.CRAWL_MAX_CONCURRENT_SNAPSHOTS;
+                    if (concurrencyInput && concurrencyValue) {
+                        concurrencyInput.value = concurrencyValue;
+                    }
                     if (typeof window.archiveboxSetPluginConfigValues === 'function') {
                         window.archiveboxSetPluginConfigValues(personaData.effective_config || {}, personaData.binary_urls || {});
                     }
@@ -429,7 +441,7 @@
                 }
 
                 function normalizePluginSelections() {
-                    const checkedPlugins = Array.from(document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]:checked'))
+                    const checkedPlugins = Array.from(document.querySelectorAll('.plugin-section-toggle:checked'))
                         .map(cb => cb.value)
                         .filter(Boolean);
                     checkedPlugins.forEach(pluginName => ensureRequiredPluginsChecked(pluginName));
@@ -1041,15 +1053,13 @@
 
                 // Plugin Presets
                 const presetConfigs = {
-                    'quick-archive': ['screenshot', 'dom', 'favicon', 'wget', 'title'],
-                    'full-chrome': ['chrome', 'screenshot', 'pdf', 'dom', 'singlefile', 'consolelog', 'redirects', 'responses', 'ssl', 'headers', 'title', 'accessibility', 'seo'],
                     'text-only': ['wget', 'readability', 'mercury', 'htmltotext', 'title', 'favicon']
                 };
 
                 document.querySelectorAll('.preset-btn').forEach(btn => {
                     btn.addEventListener('click', function() {
                         const preset = this.dataset.preset;
-                        const allCheckboxes = document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]');
+                        const allCheckboxes = document.querySelectorAll('.plugin-section-toggle');
                         const requiredSearchPreference = getStoredPluginPreference(requiredSearchPlugin);
 
                         if (preset === 'select-all') {
@@ -1083,7 +1093,7 @@
                         const group = btn.dataset.group;
                         const container = document.getElementById(group + '-plugins');
                         if (!container) return;
-                        const checkboxes = Array.from(container.querySelectorAll('input[type="checkbox"]'));
+                        const checkboxes = Array.from(container.querySelectorAll('.plugin-section-toggle'));
                         const allChecked = checkboxes.length > 0 && checkboxes.every(cb => cb.checked);
                         btn.textContent = allChecked ? 'Deselect All Chrome' : 'Select All Chrome';
                     });
@@ -1095,7 +1105,7 @@
                         const container = document.getElementById(group + '-plugins');
                         if (!container) return;
 
-                        const checkboxes = Array.from(container.querySelectorAll('input[type="checkbox"]'));
+                        const checkboxes = Array.from(container.querySelectorAll('.plugin-section-toggle'));
                         const allChecked = checkboxes.length > 0 && checkboxes.every(cb => cb.checked);
                         const requiredSearchPreference = getStoredPluginPreference(requiredSearchPlugin);
 
@@ -1112,7 +1122,7 @@
                     });
                 });
 
-                document.querySelectorAll('.plugin-checkboxes input[type="checkbox"]').forEach(checkbox => {
+                document.querySelectorAll('.plugin-section-toggle').forEach(checkbox => {
                     checkbox.addEventListener('change', function() {
                         if (this.checked) {
                             ensureRequiredPluginsChecked(this.value);

archivebox/tests/test_add_view.py

@@ -46,6 +46,7 @@ def test_add_view_renders_tag_editor_and_url_filter_fields(client, admin_user, m
     assert 'name="max_urls"' in body
     assert 'name="crawl_max_size"' in body
     assert 'name="snapshot_max_size"' in body
+    assert 'name="crawl_max_concurrent_snapshots"' in body
     assert '<input type="text" name="notes"' in body
     assert body.index('name="persona"') < body.index("<h3>Crawl Plugins</h3>")
     assert "data-url-regex=" in body
@@ -91,7 +92,7 @@ def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_
     assert persona_config_map["Private"]["effective_config"]["YTDLP_COOKIES_FILE"] == "/tmp/archivebox-private-cookies.txt"
 
 
-def test_add_view_checks_configured_search_backend_by_default(client, monkeypatch):
+def test_add_view_hides_search_backend_plugins(client, monkeypatch):
     monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
     monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite")
 
@@ -99,10 +100,10 @@ def test_add_view_checks_configured_search_backend_by_default(client, monkeypatc
     body = response.content.decode()
 
     assert response.status_code == 200
-    assert re.search(
-        r'<input type="checkbox" name="search_plugins" value="search_backend_sqlite"[^>]* checked\b',
-        body,
-    )
+    assert not re.search(r'<input type="checkbox"[^>]*value="search_backend_sqlite"', body)
+    assert 'data-plugin-name="search_backend_ripgrep"' not in body
+    assert 'data-plugin-name="search_backend_sonic"' not in body
+    assert 'data-plugin-name="search_backend_sqlite"' not in body
     assert "const requiredSearchPlugin = 'search_backend_sqlite';" in body
 
 
@@ -119,6 +120,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
             "max_urls": "3",
             "crawl_max_size": "45mb",
             "snapshot_max_size": "5mb",
+            "crawl_max_concurrent_snapshots": "4",
             "url_filters_allowlist": "example.com\n*.example.com",
             "url_filters_denylist": "cdn.example.com",
             "notes": "Created from /add/",
@@ -143,6 +145,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
     assert crawl.config["CRAWL_MAX_URLS"] == 3
     assert crawl.config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024
     assert crawl.config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024
+    assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 4
     assert crawl.config["URL_ALLOWLIST"] == "example.com\n*.example.com"
     assert crawl.config["URL_DENYLIST"] == "cdn.example.com"
     assert "OVERWRITE" not in crawl.config
@@ -208,7 +211,7 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc
             "schedule": "",
             "persona": "Default",
             "index_only": "",
-            "archiving_plugins": ["wget"],
+            "main_plugins": ["wget"],
             "plugin_config__wget__WGET_TIMEOUT": "77",
             "plugin_config__wget__WGET_WARC_ENABLED": "false",
             "config": "{}",

archivebox/tests/test_schedule_e2e.py

@@ -464,8 +464,8 @@ def test_web_ui_add_depth_two_crawls_and_renders_real_outputs_over_running_serve
                 "max_urls": "20",
                 "crawl_max_size": "0",
                 "snapshot_max_size": "0",
-                "archiving_plugins": ["wget"],
-                "parsing_plugins": ["parse_html_urls"],
+                "main_plugins": ["wget"],
+                "postprocessing_plugins": ["parse_html_urls"],
                 "tag": "web-depth-two",
                 "url_filters_allowlist": r"127\.0\.0\.1[:/].*",
                 "url_filters_denylist": "",

archivebox/workers/supervisord_util.py

@@ -41,6 +41,9 @@ RUNNER_WORKER = {
     "command": _shell_join([sys.executable, "-m", "archivebox", "run", "--daemon"]),
     "autostart": "false",
     "autorestart": "true",
+    "stopasgroup": "true",
+    "killasgroup": "true",
+    "stopwaitsecs": "30",
     "stdout_logfile": "logs/worker_runner.log",
     "redirect_stderr": "true",
 }

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.31rc41",
+  "version": "0.9.31rc42",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.31rc41"
+version = "0.9.31rc42"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (optional binaries detected at runtime via shutil.which)
     ### Binary/Package Management
     "abxbus>=2.5.4",          # EventBus API
-    "abxpkg>=1.10.31",         # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
-    "abx-plugins>=1.10.96",   # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
-    "abx-dl>=1.10.96",        # shared ArchiveBox downloader package with blocking install preflight
+    "abxpkg>=1.10.32",         # for: detecting, versioning, and installing binaries via apt/brew/pip/npm
+    "abx-plugins>=1.10.97",   # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring
+    "abx-dl>=1.10.97",        # shared ArchiveBox downloader package with blocking install preflight
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]