diff --git a/Dockerfile b/Dockerfile index 720f65a9..db20c670 100644 --- a/Dockerfile +++ b/Dockerfile @@ -299,7 +299,7 @@ ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH" \ CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \ CHROME_HEADLESS=true \ CHROME_SANDBOX=false \ - CHROME_ISOLATION=snapshot \ + CHROME_ISOLATION=crawl \ CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]' USER $ARCHIVEBOX_USER WORKDIR "/home/$ARCHIVEBOX_USER/.npm" diff --git a/archivebox/machine/models.py b/archivebox/machine/models.py index 10444b36..52ad1d0f 100755 --- a/archivebox/machine/models.py +++ b/archivebox/machine/models.py @@ -52,6 +52,7 @@ PID_REUSE_WINDOW = timedelta(hours=24) # Max age for considering a PID match va PROCESS_TIMEOUT_GRACE = timedelta(seconds=30) # Extra margin before force-cleaning timed-out RUNNING rows START_TIME_TOLERANCE = 5.0 # Seconds tolerance for start time matching LEGACY_MACHINE_CONFIG_KEYS = frozenset({"CHROMIUM_VERSION"}) +MACHINE_CONFIG_ALWAYS_ALLOWED_KEYS = frozenset({"ABX_INSTALL_CACHE"}) def _find_existing_binary_for_reference(machine: Machine, reference: str) -> Binary | None: @@ -124,7 +125,7 @@ def _sanitize_machine_config(config: dict[str, Any] | None) -> dict[str, Any]: if not isinstance(config, dict): return {} - sanitized = dict(config) + sanitized = {key: value for key, value in config.items() if key in MACHINE_CONFIG_ALWAYS_ALLOWED_KEYS or str(key).endswith("_BINARY")} for key in LEGACY_MACHINE_CONFIG_KEYS: sanitized.pop(key, None) return sanitized diff --git a/archivebox/misc/util.py b/archivebox/misc/util.py index 648cdb88..aebb172b 100644 --- a/archivebox/misc/util.py +++ b/archivebox/misc/util.py @@ -193,11 +193,18 @@ def fix_url_from_markdown(url_str: str) -> str: This assumption is true 99.9999% of the time, and for the rare edge case the user can use url_list parser. """ trimmed_url = url_str + if len(trimmed_url) > 2048: + return trimmed_url # cut off one trailing character at a time # until parens are balanced e.g. /a(b)c).x(y)z -> /a(b)c - while trimmed_url and not parens_are_matched(trimmed_url): + trim_attempts = 0 + while trimmed_url and not parens_are_matched(trimmed_url) and trim_attempts < 256: trimmed_url = trimmed_url[:-1] + trim_attempts += 1 + + if not trimmed_url or not parens_are_matched(trimmed_url): + return url_str # make sure trimmed url is still valid if any(match == trimmed_url for match in re.findall(URL_REGEX, trimmed_url)): @@ -689,7 +696,7 @@ def chrome_cleanup(): """ Cleans up any state or runtime files that Chrome leaves behind when killed by a timeout or other error. Handles: - - All persona chrome_user_data directories (via Persona.cleanup_chrome_all()) + - All persona chrome_profile directories (via Persona.cleanup_chrome_all()) - Explicit CHROME_USER_DATA_DIR from config - Legacy Docker chromium path """ diff --git a/archivebox/personas/admin.py b/archivebox/personas/admin.py index cdf7df7f..60740ded 100644 --- a/archivebox/personas/admin.py +++ b/archivebox/personas/admin.py @@ -73,7 +73,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin): @admin.display(description="Chrome Profile") def chrome_profile_state(self, obj: Persona) -> str: - return "yes" if (obj.path / "chrome_user_data").exists() else "no" + return "yes" if (obj.path / "chrome_profile").exists() else "no" @admin.display(description="cookies.txt") def cookies_state(self, obj: Persona) -> str: @@ -88,7 +88,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin): return format_html( "
" "
Persona root{}
" - "
chrome_user_data{}
" + "
chrome_profile{}
" "
chrome_extensions{}
" "
chrome_downloads{}
" "
cookies.txt{}
" @@ -105,7 +105,7 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin): @admin.display(description="Import Artifacts") def import_artifact_status(self, obj: Persona) -> str: entries = [ - ("Browser profile", (obj.path / "chrome_user_data").exists(), obj.CHROME_USER_DATA_DIR), + ("Browser profile", (obj.path / "chrome_profile").exists(), obj.CHROME_USER_DATA_DIR), ("cookies.txt", bool(obj.COOKIES_FILE), obj.COOKIES_FILE or (obj.path / "cookies.txt")), ("auth.json", bool(obj.AUTH_STORAGE_FILE), obj.AUTH_STORAGE_FILE or (obj.path / "auth.json")), ] diff --git a/archivebox/personas/forms.py b/archivebox/personas/forms.py index 3781a0ec..894f7af9 100644 --- a/archivebox/personas/forms.py +++ b/archivebox/personas/forms.py @@ -74,7 +74,7 @@ class PersonaAdminForm(forms.ModelForm): required=False, initial=True, label="Copy browser profile into this persona", - help_text="Copies the chosen Chromium user-data tree into `chrome_user_data` for future archiving runs.", + help_text="Copies the chosen Chromium user-data tree into `chrome_profile` for future archiving runs.", ) import_extract_cookies = forms.BooleanField( required=False, diff --git a/archivebox/personas/importers.py b/archivebox/personas/importers.py index ea63790f..d82d20ec 100644 --- a/archivebox/personas/importers.py +++ b/archivebox/personas/importers.py @@ -536,7 +536,7 @@ def import_persona_from_source( resolved_persona_root = persona_chrome_dir.resolve() if resolved_source_root == resolved_persona_root: result.warnings.append( - "Skipped profile copy because the selected source is already this persona's chrome_user_data directory.", + "Skipped profile copy because the selected source is already this persona's chrome_profile directory.", ) else: copy_browser_user_data_dir(resolved_source_root, resolved_persona_root) diff --git a/archivebox/personas/models.py b/archivebox/personas/models.py index 2b875b9b..da4d6556 100644 --- a/archivebox/personas/models.py +++ b/archivebox/personas/models.py @@ -76,7 +76,7 @@ class Persona(ModelWithConfig): # Or access directly from persona persona = Persona.objects.get(name='Default') - persona.CHROME_USER_DATA_DIR # -> Path to chrome_user_data + persona.CHROME_USER_DATA_DIR # -> Path to chrome_profile """ id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True) @@ -100,7 +100,7 @@ class Persona(ModelWithConfig): @property def CHROME_USER_DATA_DIR(self) -> str: """Derived path to Chrome user data directory for this persona.""" - return str(self.path / "chrome_user_data") + return str(self.path / "chrome_profile") @property def CHROME_EXTENSIONS_DIR(self) -> str: @@ -159,7 +159,7 @@ class Persona(ModelWithConfig): def ensure_dirs(self) -> None: """Create persona directories if they don't exist.""" self.path.mkdir(parents=True, exist_ok=True) - (self.path / "chrome_user_data").mkdir(parents=True, exist_ok=True) + (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True) (self.path / "chrome_extensions").mkdir(parents=True, exist_ok=True) (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True) @@ -196,7 +196,7 @@ class Persona(ModelWithConfig): def cleanup_chrome(self) -> bool: """Clean up volatile Chrome state for this persona's base profile.""" - return self.cleanup_chrome_profile(self.path / "chrome_user_data") + return self.cleanup_chrome_profile(self.path / "chrome_profile") @contextmanager def lock_runtime_for_crawl(self): @@ -216,7 +216,7 @@ class Persona(ModelWithConfig): return Path(crawl.output_dir) / ".persona" / self.name def runtime_profile_dir_for_crawl(self, crawl) -> Path: - return self.runtime_root_for_crawl(crawl) / "chrome_user_data" + return self.runtime_root_for_crawl(crawl) / "chrome_profile" def runtime_downloads_dir_for_crawl(self, crawl) -> Path: return self.runtime_root_for_crawl(crawl) / "chrome_downloads" diff --git a/archivebox/services/crawl_service.py b/archivebox/services/crawl_service.py index fd81f7e6..5e1b1c3b 100644 --- a/archivebox/services/crawl_service.py +++ b/archivebox/services/crawl_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +from asgiref.sync import sync_to_async from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, CrawlSetupEvent, CrawlStartEvent from abx_dl.services.base import BaseService @@ -38,7 +39,10 @@ class CrawlService(BaseService): from archivebox.crawls.models import Crawl crawl = await Crawl.objects.aget(id=self.crawl_id) - if crawl.status != Crawl.StatusChoices.SEALED: + is_finished = await sync_to_async(crawl.is_finished, thread_sensitive=True)() + if is_finished: + crawl.status = Crawl.StatusChoices.SEALED + elif crawl.status != Crawl.StatusChoices.SEALED: crawl.status = Crawl.StatusChoices.STARTED crawl.retry_at = None await crawl.asave(update_fields=["status", "retry_at", "modified_at"]) @@ -47,6 +51,14 @@ class CrawlService(BaseService): from archivebox.crawls.models import Crawl crawl = await Crawl.objects.aget(id=self.crawl_id) + is_finished = await sync_to_async(crawl.is_finished, thread_sensitive=True)() + if not is_finished: + if crawl.status != Crawl.StatusChoices.SEALED: + crawl.status = Crawl.StatusChoices.STARTED + crawl.retry_at = None + await crawl.asave(update_fields=["status", "retry_at", "modified_at"]) + return + crawl.status = Crawl.StatusChoices.SEALED crawl.retry_at = None await crawl.asave(update_fields=["status", "retry_at", "modified_at"]) diff --git a/archivebox/services/machine_service.py b/archivebox/services/machine_service.py index f451ab36..7367f58c 100644 --- a/archivebox/services/machine_service.py +++ b/archivebox/services/machine_service.py @@ -17,6 +17,9 @@ class MachineService(BaseService): async def on_MachineEvent__save_to_db(self, event: MachineEvent) -> None: from archivebox.machine.models import Machine, _sanitize_machine_config + if event.config_type != "derived": + return + machine = await sync_to_async(Machine.current, thread_sensitive=True)() config = dict(machine.config or {}) diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index b6d50615..8ff0efad 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -101,6 +101,13 @@ async def _emit_machine_config( ).now() +async def _run_event_now(event, timeout: float | None = None): + await event.now(timeout=timeout) + await event.wait(timeout=timeout) + await event.event_results_list() + return event + + def ensure_background_runner(*, allow_under_pytest: bool = False) -> bool: if os.environ.get("PYTEST_CURRENT_TEST") and not allow_under_pytest: return False @@ -245,7 +252,7 @@ class CrawlRunner: def load_run_state(self) -> list[str]: from archivebox.config.configset import get_config from archivebox.hooks import discover_hooks - from archivebox.machine.models import Machine, NetworkInterface, Process + from archivebox.machine.models import Machine, NetworkInterface, Process, _sanitize_machine_config self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else "" current_iface = NetworkInterface.current(refresh=True) @@ -256,10 +263,9 @@ class CrawlRunner: current_process.save(update_fields=["iface", "machine", "modified_at"]) self.persona = self.crawl.resolve_persona() self.base_config = get_config(crawl=self.crawl) - self.derived_config = dict(Machine.current().config) + self.derived_config = _sanitize_machine_config(Machine.current().config) self.crawl_output_dir = str(self.crawl.output_dir) self.base_config["ABX_RUNTIME"] = "archivebox" - self.base_config["CHROME_ISOLATION"] = "snapshot" if self.selected_plugins is None: raw_plugins = str(self.base_config.get("PLUGINS") or "").strip() if raw_plugins: @@ -390,6 +396,7 @@ class CrawlRunner: from archivebox.core.models import Snapshot from archivebox.hooks import collect_urls_from_plugins + await sync_to_async(self.crawl.refresh_from_db, thread_sensitive=True)() if int(snapshot_payload["depth"]) >= self.crawl.max_depth: return if CrawlLimitState.from_config(snapshot_payload["config"]).get_stop_reason() == "max_size": @@ -477,8 +484,7 @@ class CrawlRunner: event_handler_slow_timeout=slow_warning_timeout(install_phase_timeout), ), ) - await install_event.now() - await install_event.wait() + await _run_event_now(install_event, install_phase_timeout) crawl_event = CrawlEvent( url=snapshot["url"], snapshot_id=snapshot["id"], @@ -487,7 +493,7 @@ class CrawlRunner: event_handler_slow_timeout=slow_warning_timeout(crawl_setup_phase_timeout), ) self.root_crawl_event_id = crawl_event.event_id - await self.bus.emit(crawl_event).now() + await _run_event_now(self.bus.emit(crawl_event), crawl_setup_phase_timeout) for plugin, hook in setup_hooks: if hook.is_background: continue @@ -512,6 +518,8 @@ class CrawlRunner: ) if completed_process is None: raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} did not complete") + await completed_process.wait(timeout=crawl_setup_phase_timeout) + await completed_process.event_results_list() if completed_process.status == "failed": raise RuntimeError(f"Crawl setup hook {plugin.name}:{hook.name} failed") @@ -524,16 +532,19 @@ class CrawlRunner: plugins = self.runtime_plugins() setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")] crawl_cleanup_phase_timeout = compute_phase_timeout(setup_hooks, config) - await self.bus.emit( - CrawlCleanupEvent( - url=snapshot["url"], - snapshot_id=snapshot["id"], - output_dir=str(output_dir), - event_parent_id=self.root_crawl_event_id, - event_timeout=crawl_cleanup_phase_timeout, - event_handler_slow_timeout=slow_warning_timeout(crawl_cleanup_phase_timeout), + await _run_event_now( + self.bus.emit( + CrawlCleanupEvent( + url=snapshot["url"], + snapshot_id=snapshot["id"], + output_dir=str(output_dir), + event_parent_id=self.root_crawl_event_id, + event_timeout=crawl_cleanup_phase_timeout, + event_handler_slow_timeout=slow_warning_timeout(crawl_cleanup_phase_timeout), + ), ), - ).now() + crawl_cleanup_phase_timeout, + ) async def run_snapshot(self, snapshot_id: str) -> None: async with self.snapshot_semaphore: @@ -574,7 +585,7 @@ class CrawlRunner: event_timeout=snapshot_phase_timeout, event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), ) - await self.bus.emit(crawl_start_event).now() + await _run_event_now(self.bus.emit(crawl_start_event), snapshot_phase_timeout) snapshot_event = SnapshotEvent( url=snapshot["url"], snapshot_id=snapshot["id"], @@ -585,7 +596,7 @@ class CrawlRunner: event_handler_slow_timeout=slow_warning_timeout(snapshot_phase_timeout), ) emitted_snapshot_event = self.bus.emit(snapshot_event) - await emitted_snapshot_event.now() + await _run_event_now(emitted_snapshot_event, snapshot_phase_timeout) completed_snapshot = await self.bus.find( SnapshotCompletedEvent, child_of=emitted_snapshot_event, @@ -594,6 +605,9 @@ class CrawlRunner: ) if completed_snapshot is None: raise RuntimeError(f"Snapshot {snapshot_id} did not complete") + await completed_snapshot.now(timeout=snapshot_phase_timeout) + await completed_snapshot.wait(timeout=snapshot_phase_timeout) + await completed_snapshot.event_results_list() await self.enqueue_discovered_snapshots_from_outputs(snapshot) finally: current_task = asyncio.current_task() diff --git a/archivebox/services/snapshot_service.py b/archivebox/services/snapshot_service.py index f12994ab..d0df650d 100644 --- a/archivebox/services/snapshot_service.py +++ b/archivebox/services/snapshot_service.py @@ -80,6 +80,7 @@ class SnapshotService(BaseService): await self.schedule_snapshot(snapshot_id) async def on_SnapshotCompletedEvent(self, event: SnapshotCompletedEvent) -> None: + from archivebox.crawls.models import Crawl from archivebox.core.models import Snapshot snapshot = await Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=event.snapshot_id).afirst() @@ -107,24 +108,31 @@ class SnapshotService(BaseService): if snapshot_id: snapshot = await Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").afirst() if snapshot is not None: - await sync_to_async(snapshot.write_index_jsonl, thread_sensitive=True)() - await sync_to_async(snapshot.write_json_details, thread_sensitive=True)() - await sync_to_async(snapshot.write_html_details, thread_sensitive=True)() - stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(snapshot.crawl) - if snapshot.depth < snapshot.crawl.max_depth and stop_reason != "max_size": - from archivebox.hooks import collect_urls_from_plugins + try: + await sync_to_async(snapshot.write_index_jsonl, thread_sensitive=True)() + await sync_to_async(snapshot.write_json_details, thread_sensitive=True)() + await sync_to_async(snapshot.write_html_details, thread_sensitive=True)() + stop_reason = await sync_to_async(self._crawl_limit_stop_reason, thread_sensitive=True)(snapshot.crawl) + if snapshot.depth < snapshot.crawl.max_depth and stop_reason != "max_size": + from archivebox.hooks import collect_urls_from_plugins - discovered_urls = await sync_to_async(collect_urls_from_plugins, thread_sensitive=True)(Path(snapshot.output_dir)) - for record in discovered_urls: - discovered_snapshot_id = await self._upsert_discovered_snapshot( - snapshot, - url=str(record.get("url") or "").strip(), - depth=snapshot.depth + 1, - title=str(record.get("title") or "").strip(), - tags=str(record.get("tags") or "").strip(), - ) - if discovered_snapshot_id: - await self.schedule_snapshot(discovered_snapshot_id) + discovered_urls = await sync_to_async(collect_urls_from_plugins, thread_sensitive=True)(Path(snapshot.output_dir)) + for record in discovered_urls: + discovered_snapshot_id = await self._upsert_discovered_snapshot( + snapshot, + url=str(record.get("url") or "").strip(), + depth=snapshot.depth + 1, + title=str(record.get("title") or "").strip(), + tags=str(record.get("tags") or "").strip(), + ) + if discovered_snapshot_id: + await self.schedule_snapshot(discovered_snapshot_id) + finally: + is_finished = await sync_to_async(snapshot.crawl.is_finished, thread_sensitive=True)() + if is_finished and snapshot.crawl.status != Crawl.StatusChoices.SEALED: + snapshot.crawl.status = Crawl.StatusChoices.SEALED + snapshot.crawl.retry_at = None + await snapshot.crawl.asave(update_fields=["status", "retry_at", "modified_at"]) def _crawl_limit_stop_reason(self, crawl) -> str: config = dict(crawl.config or {}) diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html index 1603e926..8654c2cd 100644 --- a/archivebox/templates/admin/base.html +++ b/archivebox/templates/admin/base.html @@ -1639,8 +1639,12 @@ {% endcomment %} diff --git a/archivebox/templates/admin/personas/persona/change_form.html b/archivebox/templates/admin/personas/persona/change_form.html index 262c66c9..1bdde87a 100644 --- a/archivebox/templates/admin/personas/persona/change_form.html +++ b/archivebox/templates/admin/personas/persona/change_form.html @@ -239,7 +239,7 @@ document.addEventListener('DOMContentLoaded', function () {
Persona artifacts - chrome_user_data + chrome_profile cookies.txt auth.json
diff --git a/archivebox/templates/core/progressbar.html b/archivebox/templates/core/progressbar.html index 34d6ce98..01884e4c 100644 --- a/archivebox/templates/core/progressbar.html +++ b/archivebox/templates/core/progressbar.html @@ -17,27 +17,32 @@