diff --git a/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py b/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py index b341e18a..05cbb79b 100644 --- a/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py +++ b/archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py @@ -1,29 +1,59 @@ +import json + +from django.conf import settings from django.db import migrations +BATCH_SIZE = 1000 + + +def _config_cache_key(config): + return json.dumps(config or {}, sort_keys=True, separators=(",", ":"), default=str) + + +def _flush_updates(Crawl, db_alias, pending): + if pending: + Crawl.objects.using(db_alias).bulk_update(pending, ["config"], batch_size=BATCH_SIZE) + pending.clear() + + def freeze_existing_crawl_configs(apps, schema_editor): from archivebox.config.common import build_crawl_config_snapshot from archivebox.personas.models import Persona - from django.contrib.auth import get_user_model Crawl = apps.get_model("crawls", "Crawl") - User = get_user_model() + auth_app_label, auth_model_name = settings.AUTH_USER_MODEL.split(".", 1) + User = apps.get_model(auth_app_label, auth_model_name) db_alias = schema_editor.connection.alias + rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "created_by_id", "config") + persona_ids = {persona_id for _, persona_id, _, _ in rows if persona_id} + user_ids = {user_id for _, _, user_id, _ in rows if user_id} + personas = {persona.pk: persona for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)} + users = {user.pk: user for user in User.objects.using(db_alias).filter(pk__in=user_ids)} - for crawl in Crawl.objects.using(db_alias).select_related("persona", "created_by").iterator(chunk_size=200): - current_config = dict(crawl.config or {}) - persona = Persona.objects.using(db_alias).filter(pk=crawl.persona_id).first() - user = User.objects.using(db_alias).filter(pk=crawl.created_by_id).first() - frozen_config = build_crawl_config_snapshot( - user=user, - persona=persona, - overrides=current_config, - ) + frozen_cache = {} + pending = [] + for crawl_id, persona_id, user_id, current_config in rows.iterator(chunk_size=BATCH_SIZE): + current_config = dict(current_config or {}) + cache_key = (persona_id, user_id, _config_cache_key(current_config)) + if cache_key not in frozen_cache: + frozen_cache[cache_key] = build_crawl_config_snapshot( + user=users.get(user_id), + persona=personas.get(persona_id), + overrides=current_config, + ) + frozen_config = frozen_cache[cache_key] if frozen_config != current_config: - Crawl.objects.using(db_alias).filter(pk=crawl.pk).update(config=frozen_config) + pending.append(Crawl(id=crawl_id, config=frozen_config)) + if len(pending) >= BATCH_SIZE: + _flush_updates(Crawl, db_alias, pending) + + _flush_updates(Crawl, db_alias, pending) class Migration(migrations.Migration): + atomic = False + dependencies = [ ("crawls", "0017_drop_stale_crawl_limit_columns"), ] diff --git a/archivebox/personas/admin.py b/archivebox/personas/admin.py index 2685b22e..143cf457 100644 --- a/archivebox/personas/admin.py +++ b/archivebox/personas/admin.py @@ -92,14 +92,12 @@ class PersonaAdmin(ConfigEditorMixin, BaseModelAdmin): "
" "
Persona root{}
" "
chrome_profile{}
" - "
chrome_extensions{}
" "
chrome_downloads{}
" "
cookies.txt{}
" "
auth.json{}
" "
", obj.path, obj.CHROME_USER_DATA_DIR, - obj.CHROME_EXTENSIONS_DIR, obj.CHROME_DOWNLOADS_DIR, obj.COOKIES_FILE or (obj.path / "cookies.txt"), obj.AUTH_STORAGE_FILE or (obj.path / "auth.json"), diff --git a/archivebox/personas/models.py b/archivebox/personas/models.py index 1fe18761..5b471069 100644 --- a/archivebox/personas/models.py +++ b/archivebox/personas/models.py @@ -4,7 +4,6 @@ Persona management for ArchiveBox. A Persona represents a browser profile/identity used for archiving. Each persona has its own: - Chrome user data directory (for cookies, localStorage, extensions, etc.) -- Chrome extensions directory - Cookies file - Config overrides """ @@ -66,7 +65,6 @@ class Persona(ModelWithConfig): Each persona provides: - CHROME_USER_DATA_DIR: Chrome profile directory - - CHROME_EXTENSIONS_DIR: Installed extensions directory - CHROME_DOWNLOADS_DIR: Chrome downloads directory - COOKIES_FILE: Cookies file for wget/curl - config: JSON field with persona-specific config overrides @@ -124,11 +122,6 @@ class Persona(ModelWithConfig): """Derived path to Chrome user data directory for this persona.""" return str(self.path / "chrome_profile") - @property - def CHROME_EXTENSIONS_DIR(self) -> str: - """Derived path to Chrome extensions directory for this persona.""" - return str(self.path / "chrome_extensions") - @property def CHROME_DOWNLOADS_DIR(self) -> str: """Derived path to Chrome downloads directory for this persona.""" @@ -153,7 +146,6 @@ class Persona(ModelWithConfig): Returns dict with: - All values from self.config JSONField - CHROME_USER_DATA_DIR (derived from persona path) - - CHROME_EXTENSIONS_DIR (derived from persona path) - CHROME_DOWNLOADS_DIR (derived from persona path) - COOKIES_FILE (derived from persona path, if file exists) - AUTH_STORAGE_FILE (derived from persona path, if file exists) @@ -164,8 +156,6 @@ class Persona(ModelWithConfig): # Add derived paths (don't override if explicitly set in config) if "CHROME_USER_DATA_DIR" not in derived: derived["CHROME_USER_DATA_DIR"] = self.CHROME_USER_DATA_DIR - if "CHROME_EXTENSIONS_DIR" not in derived: - derived["CHROME_EXTENSIONS_DIR"] = self.CHROME_EXTENSIONS_DIR if "CHROME_DOWNLOADS_DIR" not in derived: derived["CHROME_DOWNLOADS_DIR"] = self.CHROME_DOWNLOADS_DIR if "COOKIES_FILE" not in derived and self.COOKIES_FILE: @@ -182,7 +172,6 @@ class Persona(ModelWithConfig): """Create persona directories if they don't exist.""" self.path.mkdir(parents=True, exist_ok=True) (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True) - (self.path / "chrome_extensions").mkdir(parents=True, exist_ok=True) (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True) def cleanup_chrome_profile(self, profile_dir: Path) -> bool: diff --git a/archivebox/services/runner.py b/archivebox/services/runner.py index 123e118b..3ce6bb7f 100644 --- a/archivebox/services/runner.py +++ b/archivebox/services/runner.py @@ -1273,8 +1273,6 @@ def run_snapshot_maintenance(snapshot_id: str) -> bool: snapshot.retry_at = timezone.now() if has_queued_results else None snapshot.save(update_fields=["retry_at", "modified_at"]) snapshot.write_index_jsonl() - snapshot.write_json_details() - snapshot.write_html_details() return True @@ -1433,7 +1431,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo snapshot.finalize_completed_upload_results() maintenance_ran = False if snapshot.fs_migration_needed: - # Final snapshots can still need filesystem/json maintenance after + # Final snapshots can still need filesystem/index maintenance after # a data-dir migration, but queued ArchiveResult rows are the actual # runnable work. Do the metadata rewrite first, then continue into # the targeted plugin path in the same tick so large migrations do @@ -1442,6 +1440,7 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo snapshot.refresh_from_db() selected_plugins = queued_plugins_for_snapshot(str(snapshot.id)) if selected_plugins: + search_only_plugins = all(plugin.startswith("search_backend_") for plugin in selected_plugins) _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot) run_crawl( str(snapshot.crawl_id), @@ -1450,6 +1449,21 @@ def run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: boo process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts, ) + if search_only_plugins: + from archivebox.core.models import ArchiveResult + + has_queued_results = ArchiveResult.objects.filter( + snapshot_id=snapshot.id, + status=ArchiveResult.StatusChoices.QUEUED, + ).exists() + if not has_queued_results: + type(snapshot).objects.filter( + pk=snapshot.pk, + status=snapshot.StatusChoices.SEALED, + ).update( + retry_at=None, + modified_at=timezone.now(), + ) return True if maintenance_ran: return True @@ -1678,8 +1692,8 @@ def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrup return True -def _run_due_queued_download_result( - download_plugin_names: frozenset[str], +def _run_due_queued_plugin_result( + plugin_names: frozenset[str], *, crawl_id: str | None, lock_seconds: int, @@ -1688,11 +1702,11 @@ def _run_due_queued_download_result( ) -> bool: from archivebox.core.models import ArchiveResult, Snapshot - if not download_plugin_names: + if not plugin_names: return False queued_results = ArchiveResult.objects.filter( status=ArchiveResult.StatusChoices.QUEUED, - plugin__in=download_plugin_names, + plugin__in=plugin_names, snapshot__status=Snapshot.StatusChoices.SEALED, snapshot__retry_at__lte=timezone.now(), ) @@ -1726,6 +1740,53 @@ def _run_due_binary() -> bool: return True +def _fast_forward_same_path_snapshot_fs_versions(batch_size: int = 10000) -> bool: + from django.db import connection + + from archivebox.core.models import Snapshot, ArchiveResult + + now = timezone.now() + current_version = Snapshot._fs_current_version() + same_path_versions = ("0.9.0", "0.9.1", "0.9.2", "0.9.3") + with connection.cursor() as cursor: + cursor.execute( + """ + UPDATE core_snapshot + SET fs_version = %s, + retry_at = CASE + WHEN EXISTS ( + SELECT 1 + FROM core_archiveresult + WHERE core_archiveresult.snapshot_id = core_snapshot.id + AND core_archiveresult.status = %s + ) + THEN retry_at + ELSE NULL + END, + modified_at = %s + WHERE id IN ( + SELECT id + FROM core_snapshot + WHERE status = %s + AND retry_at <= %s + AND fs_version IN (%s, %s, %s, %s) + ORDER BY retry_at, created_at + LIMIT %s + ) + """, + [ + current_version, + ArchiveResult.StatusChoices.QUEUED, + now, + Snapshot.StatusChoices.SEALED, + now, + *same_path_versions, + batch_size, + ], + ) + return bool(cursor.rowcount) + + def run_pending_crawls( *, daemon: bool = False, @@ -1747,6 +1808,7 @@ def run_pending_crawls( for plugin_name, plugin_config in plugin_configs.items() if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_") ) + search_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_")) last_recovery_at = 0.0 last_retention_at = 0.0 last_analyze_at = 0.0 @@ -1773,7 +1835,7 @@ def run_pending_crawls( # Final-state download rows are always first: they have no parent crawl # scheduler of their own, and leaving them behind makes the global # counters report stale queued work while new crawls continue. - if _run_due_queued_download_result( + if _run_due_queued_plugin_result( download_plugin_names, crawl_id=crawl_id, lock_seconds=60, @@ -1782,13 +1844,22 @@ def run_pending_crawls( ): continue - # Other final-state snapshot work comes next: search backfills, - # filesystem/json maintenance, and upload finalization should drain - # before starting or resuming regular crawl work. + if _fast_forward_same_path_snapshot_fs_versions(): + continue + + # Final-state snapshot maintenance comes before normal crawl work: + # filesystem/index maintenance and upload finalization should drain + # promptly, but pure search backend backfills are deferred below so + # they do not starve live crawls. sealed_snapshots = Snapshot.objects.filter( retry_at__lte=timezone.now(), status=Snapshot.StatusChoices.SEALED, ) + if search_plugin_names: + sealed_snapshots = sealed_snapshots.exclude( + archiveresult__status=ArchiveResult.StatusChoices.QUEUED, + archiveresult__plugin__in=search_plugin_names, + ) if crawl_id: sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id) if _run_due_snapshot_query( @@ -1868,11 +1939,13 @@ def run_pending_crawls( ): continue - # Final fallback uses only the retry_at scheduler index and selects an - # id first. The active/paused/sealed parent-specific branches above get - # first priority, so this stays broad without hydrating wide rows or - # forcing SQLite into a slow status/join plan. - due_snapshots = Snapshot.objects.filter(retry_at__lte=timezone.now()) + # Final active-state fallback uses only the retry_at scheduler index and + # selects an id first. Keep final SEALED rows out of this broad path so + # large filesystem/index backfills cannot starve newly queued crawls. + due_snapshots = Snapshot.objects.filter( + retry_at__lte=timezone.now(), + status__in=Snapshot.OPEN_STATES, + ) if maintenance_only: due_snapshots = due_snapshots.filter(status=Snapshot.StatusChoices.PAUSED) if crawl_id: @@ -1885,6 +1958,15 @@ def run_pending_crawls( ): continue + if _run_due_queued_plugin_result( + search_plugin_names, + crawl_id=crawl_id, + lock_seconds=60, + interactive_interrupts=interactive_interrupts, + runtime_config=runtime_config, + ): + continue + if not maintenance_only: if _run_due_crawl_status( Crawl.StatusChoices.SEALED, diff --git a/archivebox/services/snapshot_service.py b/archivebox/services/snapshot_service.py index a983c1b3..a548adb7 100644 --- a/archivebox/services/snapshot_service.py +++ b/archivebox/services/snapshot_service.py @@ -41,8 +41,6 @@ def finalize_completed_snapshot(snapshot_id: str) -> None: snapshot.refresh_from_db() snapshot.write_index_jsonl() - snapshot.write_json_details() - snapshot.write_html_details() def _crawl_limit_stop_reason(crawl) -> str: diff --git a/archivebox/tests/test_cli_add.py b/archivebox/tests/test_cli_add.py index 2fa72f88..d25f2471 100644 --- a/archivebox/tests/test_cli_add.py +++ b/archivebox/tests/test_cli_add.py @@ -296,7 +296,7 @@ def test_add_records_url_filter_overrides_on_crawl(tmp_path, process, disable_ex assert crawl.config["URL_ALLOWLIST"] == "example.com,*.example.com" assert crawl.config["URL_DENYLIST"] == "static.example.com" - assert (tmp_path / "personas" / "Default" / "chrome_extensions").is_dir() + assert not (tmp_path / "personas" / "Default" / "chrome_extensions").exists() def test_add_duplicate_url_creates_separate_crawls(tmp_path, process, disable_extractors_dict): diff --git a/archivebox/tests/test_cli_piping.py b/archivebox/tests/test_cli_piping.py index 0d581fab..a58f24ef 100644 --- a/archivebox/tests/test_cli_piping.py +++ b/archivebox/tests/test_cli_piping.py @@ -272,6 +272,7 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive): snapshot_stdout, snapshot_stderr, snapshot_code = run_archivebox_cmd( ["snapshot", "create", url], data_dir=initialized_archive, + env=PIPE_TEST_ENV, ) assert snapshot_code == 0, snapshot_stderr @@ -279,6 +280,7 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive): ["archiveresult", "create", "--plugin=favicon"], stdin=snapshot_stdout, data_dir=initialized_archive, + env=PIPE_TEST_ENV, ) assert ar_create_code == 0, ar_create_stderr @@ -293,6 +295,7 @@ def test_archiveresult_list_stdout_pipes_into_run(initialized_archive): list_stdout, list_stderr, list_code = run_archivebox_cmd( ["archiveresult", "list", "--plugin=favicon"], data_dir=initialized_archive, + env=PIPE_TEST_ENV, ) assert list_code == 0, list_stderr _assert_stdout_is_jsonl_only(list_stdout) diff --git a/etc/package.json b/etc/package.json index eb363c7c..b2d2a746 100644 --- a/etc/package.json +++ b/etc/package.json @@ -1,6 +1,6 @@ { "name": "archivebox", - "version": "0.9.33rc67", + "version": "0.9.33rc68", "repository": "github:ArchiveBox/ArchiveBox", "license": "MIT", "dependencies": { diff --git a/pyproject.toml b/pyproject.toml index fdf74c93..78d6dff0 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.9.33rc67" +version = "0.9.33rc68" requires-python = ">=3.13" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -79,9 +79,9 @@ dependencies = [ ### Extractor dependencies (optional binaries detected at runtime via shutil.which) ### Binary/Package Management "abxbus==2.5.8", # EventBus API - "abxpkg>=1.11.94", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm - "abx-plugins>=1.11.101", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring - "abx-dl>=1.11.101", # shared ArchiveBox downloader package with blocking install preflight + "abxpkg>=1.11.113", # for: detecting, versioning, and installing binaries via apt/brew/pip/npm + "abx-plugins>=1.11.116", # shared ArchiveBox plugin package with Chrome/Puppeteer dependency wiring + "abx-dl>=1.11.116", # shared ArchiveBox downloader package with blocking install preflight ### UUID7 backport for Python <3.14 "uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13 ]