Fix crawl event flow and UI archiving

Dockerfile

@@ -299,7 +299,7 @@ ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH" \
     CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
     CHROME_HEADLESS=true \
     CHROME_SANDBOX=false \
-    CHROME_ISOLATION=snapshot \
+    CHROME_ISOLATION=crawl \
     CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
 USER $ARCHIVEBOX_USER
 WORKDIR "/home/$ARCHIVEBOX_USER/.npm"

archivebox/machine/models.py

@@ -52,6 +52,7 @@ PID_REUSE_WINDOW = timedelta(hours=24)  # Max age for considering a PID match va
 PROCESS_TIMEOUT_GRACE = timedelta(seconds=30)  # Extra margin before force-cleaning timed-out RUNNING rows
 START_TIME_TOLERANCE = 5.0  # Seconds tolerance for start time matching
 LEGACY_MACHINE_CONFIG_KEYS = frozenset({"CHROMIUM_VERSION"})
+MACHINE_CONFIG_ALWAYS_ALLOWED_KEYS = frozenset({"ABX_INSTALL_CACHE"})
 
 
 def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None:
@@ -124,7 +125,7 @@ def _sanitize_machine_config(config: dict[str, Any] | None) -> dict[str, Any]:
     if not isinstance(config, dict):
         return {}
 
-    sanitized = dict(config)
+    sanitized = {key: value for key, value in config.items() if key in MACHINE_CONFIG_ALWAYS_ALLOWED_KEYS or str(key).endswith("_BINARY")}
     for key in LEGACY_MACHINE_CONFIG_KEYS:
         sanitized.pop(key, None)
     return sanitized

archivebox/misc/util.py

@@ -193,11 +193,18 @@ def fix_url_from_markdown(url_str: str) -> str:
     This assumption is true 99.9999% of the time, and for the rare edge case the user can use url_list parser.
     """
     trimmed_url = url_str
+    if len(trimmed_url) > 2048:
+        return trimmed_url
 
     # cut off one trailing character at a time
     # until parens are balanced e.g. /a(b)c).x(y)z -> /a(b)c
-    while trimmed_url and not parens_are_matched(trimmed_url):
+    trim_attempts = 0
+    while trimmed_url and not parens_are_matched(trimmed_url) and trim_attempts < 256:
         trimmed_url = trimmed_url[:-1]
+        trim_attempts += 1
+
+    if not trimmed_url or not parens_are_matched(trimmed_url):
+        return url_str
 
     # make sure trimmed url is still valid
     if any(match == trimmed_url for match in re.findall(URL_REGEX, trimmed_url)):
@@ -689,7 +696,7 @@ def chrome_cleanup():
     """
     Cleans up any state or runtime files that Chrome leaves behind when killed by
     a timeout or other error. Handles:
-    - All persona chrome_user_data directories (via Persona.cleanup_chrome_all())
+    - All persona chrome_profile directories (via Persona.cleanup_chrome_all())
     - Explicit CHROME_USER_DATA_DIR from config
     - Legacy Docker chromium path
     """

archivebox/personas/admin.py

@@ -73,7 +73,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Chrome Profile")
     def chrome_profile_state(self, obj: Persona) -> str:
-        return "yes" if (obj.path / "chrome_user_data").exists() else "no"
+        return "yes" if (obj.path / "chrome_profile").exists() else "no"
 
     @admin.display(description="cookies.txt")
     def cookies_state(self, obj: Persona) -> str:
@@ -88,7 +88,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             "<div class='abx-persona-path-list'>"
             "<div><strong>Persona root</strong><code>{}</code></div>"
-            "<div><strong>chrome_user_data</strong><code>{}</code></div>"
+            "<div><strong>chrome_profile</strong><code>{}</code></div>"
             "<div><strong>chrome_extensions</strong><code>{}</code></div>"
             "<div><strong>chrome_downloads</strong><code>{}</code></div>"
             "<div><strong>cookies.txt</strong><code>{}</code></div>"
@@ -105,7 +105,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin):
     @admin.display(description="Import Artifacts")
     def import_artifact_status(self, obj: Persona) -> str:
         entries = [
-            ("Browser profile", (obj.path / "chrome_user_data").exists(), obj.CHROME_USER_DATA_DIR),
+            ("Browser profile", (obj.path / "chrome_profile").exists(), obj.CHROME_USER_DATA_DIR),
             ("cookies.txt", bool(obj.COOKIES_FILE), obj.COOKIES_FILE or (obj.path / "cookies.txt")),
             ("auth.json", bool(obj.AUTH_STORAGE_FILE), obj.AUTH_STORAGE_FILE or (obj.path / "auth.json")),
         ]

archivebox/personas/forms.py

@@ -74,7 +74,7 @@ class PersonaAdminForm(forms.ModelForm):
         required=False,
         initial=True,
         label="Copy browser profile into this persona",
-        help_text="Copies the chosen Chromium user-data tree into `chrome_user_data` for future archiving runs.",
+        help_text="Copies the chosen Chromium user-data tree into `chrome_profile` for future archiving runs.",
     )
     import_extract_cookies = forms.BooleanField(
         required=False,

archivebox/personas/importers.py

@@ -536,7 +536,7 @@ def import_persona_from_source(
             resolved_persona_root = persona_chrome_dir.resolve()
             if resolved_source_root == resolved_persona_root:
                 result.warnings.append(
-                    "Skipped profile copy because the selected source is already this persona's chrome_user_data directory.",
+                    "Skipped profile copy because the selected source is already this persona's chrome_profile directory.",
                 )
             else:
                 copy_browser_user_data_dir(resolved_source_root, resolved_persona_root)

archivebox/personas/models.py

@@ -76,7 +76,7 @@ class Persona(ModelWithConfig):
 
         # Or access directly from persona
         persona = Persona.objects.get(name='Default')
-        persona.CHROME_USER_DATA_DIR  # -> Path to chrome_user_data
+        persona.CHROME_USER_DATA_DIR  # -> Path to chrome_profile
     """
 
     id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
@@ -100,7 +100,7 @@ class Persona(ModelWithConfig):
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
         """Derived path to Chrome user data directory for this persona."""
-        return str(self.path / "chrome_user_data")
+        return str(self.path / "chrome_profile")
 
     @property
     def CHROME_EXTENSIONS_DIR(self) -> str:
@@ -159,7 +159,7 @@ class Persona(ModelWithConfig):
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
         self.path.mkdir(parents=True, exist_ok=True)
-        (self.path / "chrome_user_data").mkdir(parents=True, exist_ok=True)
+        (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_extensions").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
 
@@ -196,7 +196,7 @@ class Persona(ModelWithConfig):
 
     def cleanup_chrome(self) -> bool:
         """Clean up volatile Chrome state for this persona's base profile."""
-        return self.cleanup_chrome_profile(self.path / "chrome_user_data")
+        return self.cleanup_chrome_profile(self.path / "chrome_profile")
 
     @contextmanager
     def lock_runtime_for_crawl(self):
@@ -216,7 +216,7 @@ class Persona(ModelWithConfig):
         return Path(crawl.output_dir) / ".persona" / self.name
 
     def runtime_profile_dir_for_crawl(self, crawl) -> Path:
-        return self.runtime_root_for_crawl(crawl) / "chrome_user_data"
+        return self.runtime_root_for_crawl(crawl) / "chrome_profile"
 
     def runtime_downloads_dir_for_crawl(self, crawl) -> Path:
         return self.runtime_root_for_crawl(crawl) / "chrome_downloads"

archivebox/services/crawl_service.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from asgiref.sync import sync_to_async
 from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, CrawlSetupEvent, CrawlStartEvent
 from abx_dl.services.base import BaseService
 
@@ -38,7 +39,10 @@ class CrawlService(BaseService):
         from archivebox.crawls.models import Crawl
 
         crawl = await Crawl.objects.aget(id=self.crawl_id)
-        if crawl.status != Crawl.StatusChoices.SEALED:
+        is_finished = await sync_to_async(crawl.is_finished, thread_sensitive=True)()
+        if is_finished:
+            crawl.status = Crawl.StatusChoices.SEALED
+        elif crawl.status != Crawl.StatusChoices.SEALED:
             crawl.status = Crawl.StatusChoices.STARTED
         crawl.retry_at = None
         await crawl.asave(update_fields=["status", "retry_at", "modified_at"])
@@ -47,6 +51,14 @@ class CrawlService(BaseService):
         from archivebox.crawls.models import Crawl
 
         crawl = await Crawl.objects.aget(id=self.crawl_id)
+        is_finished = await sync_to_async(crawl.is_finished, thread_sensitive=True)()
+        if not is_finished:
+            if crawl.status != Crawl.StatusChoices.SEALED:
+                crawl.status = Crawl.StatusChoices.STARTED
+            crawl.retry_at = None
+            await crawl.asave(update_fields=["status", "retry_at", "modified_at"])
+            return
+
         crawl.status = Crawl.StatusChoices.SEALED
         crawl.retry_at = None
         await crawl.asave(update_fields=["status", "retry_at", "modified_at"])

archivebox/services/machine_service.py

@@ -17,6 +17,9 @@ class MachineService(BaseService):
     async def on_MachineEvent__save_to_db(self, event: MachineEvent) -> None:
         from archivebox.machine.models import Machine, _sanitize_machine_config
 
+        if event.config_type != "derived":
+            return
+
         machine = await sync_to_async(Machine.current, thread_sensitive=True)()
         config = dict(machine.config or {})
 

archivebox/services/runner.py

@@ -101,6 +101,13 @@ async def _emit_machine_config(
         ).now()
 
 
+async def _run_event_now(event, timeout: float | None = None):
+    await event.now(timeout=timeout)
+    await event.wait(timeout=timeout)
+    await event.event_results_list()
+    return event
+
+
 def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool:
     if os.environ.get("PYTEST_CURRENT_TEST") and not allow_under_pytest:
         return False
@@ -245,7 +252,7 @@ class CrawlRunner:
     def load_run_state(self) -> list[str]:
         from archivebox.config.configset import get_config
         from archivebox.hooks import discover_hooks
-        from archivebox.machine.models import Machine, NetworkInterface, Process
+        from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config
 
         self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
         current_iface = NetworkInterface.current(refresh=True)
@@ -256,10 +263,9 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl)
-        self.derived_config = dict(Machine.current().config)
+        self.derived_config = _sanitize_machine_config(Machine.current().config)
         self.crawl_output_dir = str(self.crawl.output_dir)
         self.base_config["ABX_RUNTIME"] = "archivebox"
-        self.base_config["CHROME_ISOLATION"] = "snapshot"
         if self.selected_plugins is None:
             raw_plugins = str(self.base_config.get("PLUGINS") or "").strip()
             if raw_plugins:
@@ -390,6 +396,7 @@ class CrawlRunner:
         from archivebox.core.models import Snapshot
         from archivebox.hooks import collect_urls_from_plugins
 
+        await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)()
         if int(snapshot_payload["depth"]) >= self.crawl.max_depth:
             return
         if CrawlLimitState.from_config(snapshot_payload["config"]).get_stop_reason() == "max_size":
@@ -477,8 +484,7 @@ class CrawlRunner:
                 event_handler_slow_timeout=slow_warning_timeout(install_phase_timeout),
             ),
         )
-        await install_event.now()
-        await install_event.wait()
+        await _run_event_now(install_event, install_phase_timeout)
         crawl_event = CrawlEvent(
             url=snapshot["url"],
             snapshot_id=snapshot["id"],
@@ -487,7 +493,7 @@ class CrawlRunner:
             event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout),
         )
         self.root_crawl_event_id = crawl_event.event_id
-        await self.bus.emit(crawl_event).now()
+        await _run_event_now(self.bus.emit(crawl_event), crawl_setup_phase_timeout)
         for plugin, hook in setup_hooks:
             if hook.is_background:
                 continue
@@ -512,6 +518,8 @@ class CrawlRunner:
             )
             if completed_process is None:
                 raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} did not complete")
+            await completed_process.wait(timeout=crawl_setup_phase_timeout)
+            await completed_process.event_results_list()
             if completed_process.status == "failed":
                 raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} failed")
 
@@ -524,16 +532,19 @@ class CrawlRunner:
         plugins = self.runtime_plugins()
         setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")]
         crawl_cleanup_phase_timeout = compute_phase_timeout(setup_hooks, config)
-        await self.bus.emit(
-            CrawlCleanupEvent(
-                url=snapshot["url"],
-                snapshot_id=snapshot["id"],
-                output_dir=str(output_dir),
-                event_parent_id=self.root_crawl_event_id,
-                event_timeout=crawl_cleanup_phase_timeout,
-                event_handler_slow_timeout=slow_warning_timeout(crawl_cleanup_phase_timeout),
+        await _run_event_now(
+            self.bus.emit(
+                CrawlCleanupEvent(
+                    url=snapshot["url"],
+                    snapshot_id=snapshot["id"],
+                    output_dir=str(output_dir),
+                    event_parent_id=self.root_crawl_event_id,
+                    event_timeout=crawl_cleanup_phase_timeout,
+                    event_handler_slow_timeout=slow_warning_timeout(crawl_cleanup_phase_timeout),
+                ),
             ),
-        ).now()
+            crawl_cleanup_phase_timeout,
+        )
 
     async def run_snapshot(self, snapshot_id: str) -> None:
         async with self.snapshot_semaphore:
@@ -574,7 +585,7 @@ class CrawlRunner:
                     event_timeout=snapshot_phase_timeout,
                     event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
                 )
-                await self.bus.emit(crawl_start_event).now()
+                await _run_event_now(self.bus.emit(crawl_start_event), snapshot_phase_timeout)
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
                     snapshot_id=snapshot["id"],
@@ -585,7 +596,7 @@ class CrawlRunner:
                     event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout),
                 )
                 emitted_snapshot_event = self.bus.emit(snapshot_event)
-                await emitted_snapshot_event.now()
+                await _run_event_now(emitted_snapshot_event, snapshot_phase_timeout)
                 completed_snapshot = await self.bus.find(
                     SnapshotCompletedEvent,
                     child_of=emitted_snapshot_event,
@@ -594,6 +605,9 @@ class CrawlRunner:
                 )
                 if completed_snapshot is None:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
+                await completed_snapshot.now(timeout=snapshot_phase_timeout)
+                await completed_snapshot.wait(timeout=snapshot_phase_timeout)
+                await completed_snapshot.event_results_list()
                 await self.enqueue_discovered_snapshots_from_outputs(snapshot)
             finally:
                 current_task = asyncio.current_task()

archivebox/services/snapshot_service.py

@@ -80,6 +80,7 @@ class SnapshotService(BaseService):
             await self.schedule_snapshot(snapshot_id)
 
     async def on_SnapshotCompletedEvent(self, event: SnapshotCompletedEvent) -> None:
+        from archivebox.crawls.models import Crawl
         from archivebox.core.models import Snapshot
 
         snapshot = await Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=event.snapshot_id).afirst()
@@ -107,24 +108,31 @@ class SnapshotService(BaseService):
         if snapshot_id:
             snapshot = await Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").afirst()
             if snapshot is not None:
-                await sync_to_async(snapshot.write_index_jsonl, thread_sensitive=True)()
-                await sync_to_async(snapshot.write_json_details, thread_sensitive=True)()
-                await sync_to_async(snapshot.write_html_details, thread_sensitive=True)()
-                stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(snapshot.crawl)
-                if snapshot.depth < snapshot.crawl.max_depth and stop_reason != "max_size":
-                    from archivebox.hooks import collect_urls_from_plugins
+                try:
+                    await sync_to_async(snapshot.write_index_jsonl, thread_sensitive=True)()
+                    await sync_to_async(snapshot.write_json_details, thread_sensitive=True)()
+                    await sync_to_async(snapshot.write_html_details, thread_sensitive=True)()
+                    stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(snapshot.crawl)
+                    if snapshot.depth < snapshot.crawl.max_depth and stop_reason != "max_size":
+                        from archivebox.hooks import collect_urls_from_plugins
 
-                    discovered_urls = await sync_to_async(collect_urls_from_plugins, thread_sensitive=True)(Path(snapshot.output_dir))
-                    for record in discovered_urls:
-                        discovered_snapshot_id = await self._upsert_discovered_snapshot(
-                            snapshot,
-                            url=str(record.get("url") or "").strip(),
-                            depth=snapshot.depth + 1,
-                            title=str(record.get("title") or "").strip(),
-                            tags=str(record.get("tags") or "").strip(),
-                        )
-                        if discovered_snapshot_id:
-                            await self.schedule_snapshot(discovered_snapshot_id)
+                        discovered_urls = await sync_to_async(collect_urls_from_plugins, thread_sensitive=True)(Path(snapshot.output_dir))
+                        for record in discovered_urls:
+                            discovered_snapshot_id = await self._upsert_discovered_snapshot(
+                                snapshot,
+                                url=str(record.get("url") or "").strip(),
+                                depth=snapshot.depth + 1,
+                                title=str(record.get("title") or "").strip(),
+                                tags=str(record.get("tags") or "").strip(),
+                            )
+                            if discovered_snapshot_id:
+                                await self.schedule_snapshot(discovered_snapshot_id)
+                finally:
+                    is_finished = await sync_to_async(snapshot.crawl.is_finished, thread_sensitive=True)()
+                    if is_finished and snapshot.crawl.status != Crawl.StatusChoices.SEALED:
+                        snapshot.crawl.status = Crawl.StatusChoices.SEALED
+                        snapshot.crawl.retry_at = None
+                        await snapshot.crawl.asave(update_fields=["status", "retry_at", "modified_at"])
 
     def _crawl_limit_stop_reason(self, crawl) -> str:
         config = dict(crawl.config or {})

archivebox/templates/admin/base.html

@@ -1639,8 +1639,12 @@
         {% endcomment %}
         
         <script>
-            $ = django.jQuery;
-            $.fn.reverse = [].reverse;
+            const archiveboxAdminJQuery = (window.django && window.django.jQuery) || window.jQuery;
+            const $ = archiveboxAdminJQuery;
+            if ($) {
+                window.$ = $;
+                $.fn.reverse = [].reverse;
+            }
 
             // hide images that fail to load
             document.querySelector('body').addEventListener('error', function (e) {
@@ -1724,7 +1728,9 @@
                         .appendTo(buttons)
                 })
                 console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
-                jQuery('select[multiple]').select2();
+                if (window.jQuery && window.jQuery.fn.select2) {
+                    window.jQuery('select[multiple]').select2();
+                }
             }
             function updateTagWidgetVisibility() {
                 const tagContainer = document.querySelector('.actions-tags');
@@ -1851,18 +1857,30 @@
                 })
                 return false
             }
-            $(document).ready(function() {
-                fix_actions()
-                updateTagWidgetVisibility()
-                const form = document.querySelector('#changelist-form')
-                if (form) {
-                    form.addEventListener('change', updateTagWidgetVisibility)
-                }
-                fixInlineAddRow()
-                setupSnapshotGridListToggle()
-                setTimeOffset()
-                selectSnapshotIfHotlinked()
-            })
+            if ($) {
+                $(document).ready(function() {
+                    fix_actions()
+                    updateTagWidgetVisibility()
+                    const form = document.querySelector('#changelist-form')
+                    if (form) {
+                        form.addEventListener('change', updateTagWidgetVisibility)
+                    }
+                    fixInlineAddRow()
+                    setupSnapshotGridListToggle()
+                    setTimeOffset()
+                    selectSnapshotIfHotlinked()
+                })
+            } else {
+                document.addEventListener('DOMContentLoaded', function() {
+                    updateTagWidgetVisibility()
+                    const form = document.querySelector('#changelist-form')
+                    if (form) {
+                        form.addEventListener('change', updateTagWidgetVisibility)
+                    }
+                    setTimeOffset()
+                    selectSnapshotIfHotlinked()
+                })
+            }
         </script>
         <script src="{% static 'admin-inline-tags.js' %}"></script>
     </body>

archivebox/templates/admin/personas/persona/change_form.html

@@ -239,7 +239,7 @@ document.addEventListener('DOMContentLoaded', function () {
         </div>
         <div class="persona-import-hero__stat">
             <span>Persona artifacts</span>
-            <code>chrome_user_data</code>
+            <code>chrome_profile</code>
             <code>cookies.txt</code>
             <code>auth.json</code>
         </div>

archivebox/templates/core/progressbar.html

@@ -17,27 +17,32 @@
 </style>
 <script>
     // Page Loading Bar
+    const archiveboxJQuery = (window.django && window.django.jQuery) || window.jQuery;
     window.loadStart = function(distance) {
+        if (!archiveboxJQuery) return;
         var distance = distance || 0;
         // only add progrstess bar if not already present
-        if (django.jQuery("#loading-bar").length == 0) {
-            django.jQuery("body").add("<div id=\"loading-bar\"></div>");
+        if (archiveboxJQuery("#loading-bar").length == 0) {
+            archiveboxJQuery("body").add("<div id=\"loading-bar\"></div>");
         }
-        if (django.jQuery("#progress").length === 0) {
-            django.jQuery("body").append(django.jQuery("<div></div>").attr("id", "progress"));
+        if (archiveboxJQuery("#progress").length === 0) {
+            archiveboxJQuery("body").append(archiveboxJQuery("<div></div>").attr("id", "progress"));
             let last_distance = (distance || (30 + (Math.random() * 30)))
-            django.jQuery("#progress").width(last_distance + "%");
+            archiveboxJQuery("#progress").width(last_distance + "%");
             setInterval(function() {
                 last_distance += Math.random()
-                django.jQuery("#progress").width(last_distance + "%");
+                archiveboxJQuery("#progress").width(last_distance + "%");
             }, 1000)
         }
     };
 
     window.loadFinish = function() {
-        django.jQuery("#progress").width("101%").delay(200).fadeOut(400, function() {
-            django.jQuery(this).remove();
-        });
+        if (!archiveboxJQuery) return;
+        const progress = archiveboxJQuery("#progress");
+        progress.width("101%");
+        window.setTimeout(function() {
+            progress.remove();
+        }, 600);
     };
     window.loadStart();
     window.addEventListener('beforeunload', function() {window.loadStart(27)});

archivebox/tests/test_cli_add.py

@@ -237,7 +237,7 @@ def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extrac
 
     assert persona_id
     assert default_persona == "Default"
-    assert (tmp_path / "personas" / "Default" / "chrome_user_data").is_dir()
+    assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir()
 
 
 def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_extractors_dict):

archivebox/tests/test_machine_models.py

@@ -129,13 +129,17 @@ class TestMachineModel(TestCase):
         result = Machine.from_json({"invalid": "record"})
         self.assertIsNone(result)
 
-    def test_machine_current_strips_legacy_chromium_version(self):
-        """Machine.current() should clean legacy browser version keys from persisted config."""
+    def test_machine_current_keeps_only_derived_runtime_cache(self):
+        """Machine.current() should keep derived cache entries, not runtime config."""
         import archivebox.machine.models as models
 
         machine = Machine.current()
         machine.config = {
             "CHROME_BINARY": "/tmp/chromium",
+            "NODE_BINARY": "/tmp/node",
+            "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+            "CHROME_ISOLATION": "snapshot",
+            "CHROME_USER_DATA_DIR": "/tmp/profile",
             "CHROMIUM_VERSION": "123.4.5",
         }
         machine.save(update_fields=["config"])
@@ -144,6 +148,10 @@ class TestMachineModel(TestCase):
         refreshed = Machine.current()
 
         self.assertEqual(refreshed.config.get("CHROME_BINARY"), "/tmp/chromium")
+        self.assertEqual(refreshed.config.get("NODE_BINARY"), "/tmp/node")
+        self.assertEqual(refreshed.config.get("ABX_INSTALL_CACHE"), {"wget": "2026-03-24T00:00:00+00:00"})
+        self.assertNotIn("CHROME_ISOLATION", refreshed.config)
+        self.assertNotIn("CHROME_USER_DATA_DIR", refreshed.config)
         self.assertNotIn("CHROMIUM_VERSION", refreshed.config)
 
     def test_machine_manager_current(self):

archivebox/tests/test_persona_runtime.py

@@ -68,7 +68,7 @@ def test_persona_prepare_runtime_for_crawl_clones_and_cleans_profile(initialized
     assert payload["cache_removed"] is True
     assert payload["log_removed"] is True
     assert payload["persona_name_recorded"] == "Default"
-    assert payload["template_dir_recorded"].endswith("/personas/Default/chrome_user_data")
+    assert payload["template_dir_recorded"].endswith("/personas/Default/chrome_profile")
     assert payload["chrome_binary_recorded"] == "/Applications/Chromium.app/Contents/MacOS/Chromium"
 
 
@@ -113,6 +113,45 @@ def test_persona_cleanup_runtime_for_crawl_removes_only_runtime_copy(initialized
     assert payload["template_still_exists"] is True
 
 
+def test_crawl_runner_respects_chrome_isolation_config(initialized_archive):
+    script = textwrap.dedent(
+        """
+        import json
+        import os
+
+        os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'archivebox.core.settings')
+        import django
+        django.setup()
+
+        from archivebox.crawls.models import Crawl
+        from archivebox.services.runner import CrawlRunner
+
+        crawl_default = Crawl.objects.create(urls='https://example.com')
+        runner_default = CrawlRunner(crawl_default)
+        runner_default.load_run_state()
+
+        crawl_snapshot = Crawl.objects.create(
+            urls='https://example.com/explicit',
+            config={'CHROME_ISOLATION': 'snapshot'},
+        )
+        runner_snapshot = CrawlRunner(crawl_snapshot)
+        runner_snapshot.load_run_state()
+
+        print(json.dumps({
+            'default_isolation': runner_default.base_config.get('CHROME_ISOLATION'),
+            'explicit_isolation': runner_snapshot.base_config.get('CHROME_ISOLATION'),
+        }))
+        """,
+    )
+
+    stdout, stderr, code = run_python_cwd(script, cwd=initialized_archive, timeout=60)
+    assert code == 0, stderr
+
+    payload = json.loads(stdout.strip().splitlines()[-1])
+    assert payload["default_isolation"] == "crawl"
+    assert payload["explicit_isolation"] == "snapshot"
+
+
 def test_crawl_resolve_persona_raises_for_missing_persona_id(initialized_archive):
     script = textwrap.dedent(
         """

archivebox/tests/test_runner.py

@@ -38,14 +38,14 @@ class _DummyBus:
                 from abx_dl.events import SnapshotCompletedEvent, SnapshotEvent
 
                 if isinstance(event, SnapshotEvent):
-                    bus.emitted.append(
-                        SnapshotCompletedEvent(
-                            url=event.url,
-                            snapshot_id=event.snapshot_id,
-                            output_dir=event.output_dir,
-                            event_parent_id=event.event_id,
-                        ),
+                    completed = SnapshotCompletedEvent(
+                        url=event.url,
+                        snapshot_id=event.snapshot_id,
+                        output_dir=event.output_dir,
+                        event_parent_id=event.event_id,
                     )
+                    completed._mark_completed()
+                    bus.emitted.append(completed)
                 return event
 
             async def wait(self, *args, **kwargs):
@@ -154,6 +154,7 @@ def test_run_snapshot_reuses_crawl_bus_for_all_snapshots(monkeypatch):
         },
     }
     monkeypatch.setattr(crawl_runner, "load_snapshot_payload", lambda snapshot_id: snapshot_data[snapshot_id])
+    monkeypatch.setattr(crawl_runner, "enqueue_discovered_snapshots_from_outputs", lambda snapshot: asyncio.sleep(0))
 
     async def run_both():
         await asyncio.gather(
@@ -209,6 +210,50 @@ def test_run_snapshot_does_not_wait_for_crawl_background_daemons(monkeypatch):
     asyncio.run(crawl_runner.run_snapshot(str(snapshot.id)))
 
 
+@pytest.mark.django_db(transaction=True)
+def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.services.runner import CrawlRunner
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        max_depth=0,
+        max_urls=5,
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    snapshot = Snapshot.objects.create(
+        url="https://example.com",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        depth=0,
+    )
+    parser_dir = Path(snapshot.output_dir) / "parse_html_urls"
+    parser_dir.mkdir(parents=True, exist_ok=True)
+    (parser_dir / "urls.jsonl").write_text(
+        "\n".join(
+            [
+                json.dumps({"type": "Snapshot", "url": "https://example.com/child-a", "depth": 1}),
+                json.dumps({"type": "Snapshot", "url": "https://example.com/child-b", "depth": 1}),
+                "",
+            ],
+        ),
+    )
+
+    runner = CrawlRunner(crawl)
+    Crawl.objects.filter(id=crawl.id).update(max_depth=1)
+    payload = runner.load_snapshot_payload(str(snapshot.id))
+
+    asyncio.run(runner.enqueue_discovered_snapshots_from_outputs(payload))
+
+    child_snapshots = list(crawl.snapshot_set.filter(depth=1).order_by("url").values_list("url", "status"))
+    assert child_snapshots == [
+        ("https://example.com/child-a", Snapshot.StatusChoices.QUEUED),
+        ("https://example.com/child-b", Snapshot.StatusChoices.QUEUED),
+    ]
+
+
 def test_ensure_background_runner_starts_when_none_running(monkeypatch):
     import archivebox.machine.models as machine_models
     from archivebox.services import runner as runner_module
@@ -283,14 +328,17 @@ def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
     bus.on(CrawlEvent, on_crawl)
 
     async def run_test():
-        await bus.emit(
-            CrawlEvent(
-                url="https://example.com",
-                snapshot_id="snapshot-1",
-                output_dir=str(tmp_path),
-            ),
-        ).now()
-        await bus.wait_until_idle()
+        try:
+            await bus.emit(
+                CrawlEvent(
+                    url="https://example.com",
+                    snapshot_id="snapshot-1",
+                    output_dir=str(tmp_path),
+                ),
+            ).now()
+            await bus.wait_until_idle()
+        finally:
+            await bus.destroy()
 
     asyncio.run(run_test())
 
@@ -300,6 +348,58 @@ def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
     ]
 
 
+@pytest.mark.django_db(transaction=True)
+def test_machine_service_persists_only_derived_config_events():
+    from abx_dl.events import MachineEvent
+    from abx_dl.orchestrator import create_bus
+    from archivebox.machine.models import Machine
+    from archivebox.services.machine_service import MachineService
+
+    machine = Machine.current()
+    machine.config = {}
+    machine.save(update_fields=["config"])
+
+    async def run_test():
+        bus = create_bus(name="test_machine_service_persists_only_derived_config_events")
+        try:
+            MachineService(bus)
+            user_event = bus.emit(
+                MachineEvent(
+                    config={
+                        "CHROME_ISOLATION": "snapshot",
+                        "CHROME_USER_DATA_DIR": "/tmp/stale-profile",
+                        "ABX_RUNTIME": "archivebox",
+                    },
+                    config_type="user",
+                ),
+            )
+            await user_event.now()
+            await user_event.event_results_list()
+            derived_event = bus.emit(
+                MachineEvent(
+                    config={
+                        "WGET_BINARY": "/tmp/wget",
+                        "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+                        "CHROME_USER_DATA_DIR": "/tmp/stale-derived-profile",
+                    },
+                    config_type="derived",
+                ),
+            )
+            await derived_event.now()
+            await derived_event.event_results_list()
+            await bus.wait_until_idle()
+        finally:
+            await bus.destroy()
+
+    asyncio.run(run_test())
+
+    machine.refresh_from_db()
+    assert machine.config == {
+        "WGET_BINARY": "/tmp/wget",
+        "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+    }
+
+
 def test_runner_prepare_refreshes_network_interface_and_attaches_current_process(monkeypatch):
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
@@ -376,7 +476,12 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
         os_release="14.0",
         os_kernel="Darwin",
         stats={},
-        config={"WGET_BINARY": "/tmp/wget", "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"}},
+        config={
+            "WGET_BINARY": "/tmp/wget",
+            "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+            "CHROME_ISOLATION": "snapshot",
+            "CHROME_USER_DATA_DIR": "/tmp/stale-profile",
+        },
     )
     crawl = Crawl.objects.create(
         urls="https://example.com",
@@ -396,7 +501,10 @@ def test_load_run_state_uses_machine_config_as_derived_config(monkeypatch):
     crawl_runner = runner_module.CrawlRunner(crawl)
     crawl_runner.load_run_state()
 
-    assert crawl_runner.derived_config == machine.config
+    assert crawl_runner.derived_config == {
+        "WGET_BINARY": "/tmp/wget",
+        "ABX_INSTALL_CACHE": {"wget": "2026-03-24T00:00:00+00:00"},
+    }
 
 
 def test_load_run_state_does_not_force_chrome_keepalive(monkeypatch):
@@ -598,7 +706,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_max_size():
         encoding="utf-8",
     )
 
-    bus = create_bus(name="test_snapshot_limit_cancel")
+    bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}")
     service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: None)
     try:
 
@@ -614,6 +722,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_max_size():
         asyncio.run(emit_event())
     finally:
         asyncio.run(bus.wait_until_idle())
+        asyncio.run(bus.destroy())
 
     root.refresh_from_db()
     child.refresh_from_db()
@@ -882,21 +991,24 @@ def test_abx_process_service_background_process_finishes_after_process_exit(monk
     pid_file.write_text("12345")
 
     async def run_test():
-        event = ProcessEvent(
-            plugin_name="chrome",
-            hook_name="on_CrawlSetup__90_chrome_launch.daemon.bg",
-            hook_path=sys.executable,
-            hook_args=["-c", "pass"],
-            env={},
-            output_dir=str(plugin_output_dir),
-            timeout=60,
-            is_background=True,
-            url="https://example.org/",
-            process_type="hook",
-            worker_type="hook",
-        )
-        await asyncio.wait_for(bus.emit(event).now(), timeout=0.5)
-        await bus.wait_until_idle()
+        try:
+            event = ProcessEvent(
+                plugin_name="chrome",
+                hook_name="on_CrawlSetup__90_chrome_launch.daemon.bg",
+                hook_path=sys.executable,
+                hook_args=["-c", "pass"],
+                env={},
+                output_dir=str(plugin_output_dir),
+                timeout=60,
+                is_background=True,
+                url="https://example.org/",
+                process_type="hook",
+                worker_type="hook",
+            )
+            await asyncio.wait_for(bus.emit(event).now(), timeout=0.5)
+            await bus.wait_until_idle()
+        finally:
+            await bus.destroy()
 
     asyncio.run(run_test())
 
@@ -1029,3 +1141,193 @@ def test_run_pending_crawls_prioritizes_queued_crawl_before_unrelated_binary_bac
 
     assert run_calls == [(str(queued_crawl.id), None, True)]
     assert binary_calls == []
+
+
+@pytest.mark.django_db(transaction=True)
+def test_crawl_completed_event_does_not_seal_active_snapshots():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.services.crawl_service import CrawlService
+    from abx_dl.events import CrawlCompletedEvent
+    from abx_dl.orchestrator import create_bus
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.STARTED,
+        retry_at=None,
+    )
+    Snapshot.objects.create(
+        url="https://example.com",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.STARTED,
+        retry_at=None,
+    )
+
+    bus = create_bus(name=f"test_crawl_completed_active_snapshots_{str(crawl.id).replace('-', '_')}")
+    CrawlService(bus, crawl_id=str(crawl.id))
+    try:
+
+        async def emit_completed() -> None:
+            event = CrawlCompletedEvent(
+                url="https://example.com",
+                snapshot_id="",
+                output_dir=str(crawl.output_dir),
+            )
+            emitted = bus.emit(event)
+            await emitted.now()
+            await emitted.event_results_list()
+
+        asyncio.run(emit_completed())
+    finally:
+        asyncio.run(bus.wait_until_idle())
+        asyncio.run(bus.destroy())
+
+    crawl.refresh_from_db()
+    assert crawl.status == Crawl.StatusChoices.STARTED
+    assert crawl.retry_at is None
+
+
+@pytest.mark.django_db(transaction=True)
+def test_crawl_cleanup_event_seals_finished_crawl():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.services.crawl_service import CrawlService
+    from abx_dl.events import CrawlCleanupEvent
+    from abx_dl.orchestrator import create_bus
+    from django.utils import timezone
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.STARTED,
+        retry_at=timezone.now(),
+    )
+    snapshot = Snapshot.objects.create(
+        url="https://example.com",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+    )
+
+    bus = create_bus(name=f"test_crawl_cleanup_finished_crawl_{str(crawl.id).replace('-', '_')}")
+    CrawlService(bus, crawl_id=str(crawl.id))
+    try:
+
+        async def emit_cleanup() -> None:
+            event = CrawlCleanupEvent(
+                url="https://example.com",
+                snapshot_id=str(snapshot.id),
+                output_dir=str(crawl.output_dir),
+            )
+            emitted = bus.emit(event)
+            await emitted.now()
+            await emitted.event_results_list()
+
+        asyncio.run(emit_cleanup())
+    finally:
+        asyncio.run(bus.wait_until_idle())
+        asyncio.run(bus.destroy())
+
+    crawl.refresh_from_db()
+    assert crawl.status == Crawl.StatusChoices.SEALED
+    assert crawl.retry_at is None
+
+
+@pytest.mark.django_db(transaction=True)
+def test_snapshot_completed_event_seals_finished_crawl():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.services.snapshot_service import SnapshotService
+    from abx_dl.events import SnapshotCompletedEvent
+    from abx_dl.orchestrator import create_bus
+    from django.utils import timezone
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.STARTED,
+        retry_at=timezone.now(),
+    )
+    snapshot = Snapshot.objects.create(
+        url="https://example.com",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.STARTED,
+        retry_at=None,
+    )
+
+    bus = create_bus(name=f"test_snapshot_completed_finished_crawl_{str(crawl.id).replace('-', '_')}")
+    service = SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0))
+    try:
+
+        async def emit_completed() -> None:
+            await service.on_SnapshotCompletedEvent(
+                SnapshotCompletedEvent(
+                    url="https://example.com",
+                    snapshot_id=str(snapshot.id),
+                    output_dir=str(snapshot.output_dir),
+                ),
+            )
+
+        asyncio.run(emit_completed())
+    finally:
+        asyncio.run(bus.destroy())
+
+    snapshot.refresh_from_db()
+    crawl.refresh_from_db()
+    assert snapshot.status == Snapshot.StatusChoices.SEALED
+    assert crawl.status == Crawl.StatusChoices.SEALED
+    assert crawl.retry_at is None
+
+
+@pytest.mark.django_db(transaction=True)
+def test_snapshot_completed_event_bus_seals_finished_crawl():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.services.snapshot_service import SnapshotService
+    from abx_dl.events import SnapshotCompletedEvent
+    from abx_dl.orchestrator import create_bus
+    from django.utils import timezone
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by_id=get_or_create_system_user_pk(),
+        status=Crawl.StatusChoices.STARTED,
+        retry_at=timezone.now(),
+    )
+    snapshot = Snapshot.objects.create(
+        url="https://example.com",
+        crawl=crawl,
+        status=Snapshot.StatusChoices.STARTED,
+        retry_at=None,
+    )
+
+    bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}")
+    SnapshotService(bus, crawl_id=str(crawl.id), schedule_snapshot=lambda snapshot_id: asyncio.sleep(0))
+    try:
+
+        async def emit_completed() -> None:
+            emitted = bus.emit(
+                SnapshotCompletedEvent(
+                    url="https://example.com",
+                    snapshot_id=str(snapshot.id),
+                    output_dir=str(snapshot.output_dir),
+                ),
+            )
+            await emitted.now()
+            await emitted.event_results_list()
+
+        asyncio.run(emit_completed())
+    finally:
+        asyncio.run(bus.wait_until_idle())
+        asyncio.run(bus.destroy())
+
+    snapshot.refresh_from_db()
+    crawl.refresh_from_db()
+    assert snapshot.status == Snapshot.StatusChoices.SEALED
+    assert crawl.status == Crawl.StatusChoices.SEALED
+    assert crawl.retry_at is None

docker-compose.yml

@@ -39,7 +39,7 @@ services:
             - CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile
             - CHROME_HEADLESS=true
             - CHROME_SANDBOX=false
-            - CHROME_ISOLATION=snapshot
+            - CHROME_ISOLATION=crawl
             - CHROME_ARGS_EXTRA=["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]
             # - PUID=911                        # set to your host user's UID & GID if you encounter permissions issues
             # - PGID=911                        # UID/GIDs lower than 500 may clash with system uids and are not recommended