diff --git a/archivebox/config/common.py b/archivebox/config/common.py index b4b3d7ea..2e0f3fe5 100644 --- a/archivebox/config/common.py +++ b/archivebox/config/common.py @@ -546,7 +546,7 @@ class SearchBackendConfig(BaseConfigSet): toml_section_header: str = "SEARCH_BACKEND_CONFIG" _scope: str = PrivateAttr(default=_SCOPE_SERVER) - SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION}) + SEARCH_BACKEND_ENGINE: str = Field(default="sonic", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION}) def _plugin_user_config_value(value: Any) -> str: diff --git a/archivebox/tests/test_frozen_crawl_config.py b/archivebox/tests/test_frozen_crawl_config.py index fd92ebb4..eef202dd 100644 --- a/archivebox/tests/test_frozen_crawl_config.py +++ b/archivebox/tests/test_frozen_crawl_config.py @@ -144,7 +144,10 @@ def test_config_scopes_are_derived_from_section_and_field_metadata(): def test_search_backend_engine_derives_default_backend_enabled_without_entering_hook_env(): from archivebox.config.common import ArchiveBoxConfig - default_runtime_config = ArchiveBoxConfig().for_crawl_runtime(extra_context={"snapshot_id": "default-runtime-config"}) + default_config = ArchiveBoxConfig() + assert default_config.SEARCH_BACKEND_ENGINE == "sonic" + + default_runtime_config = default_config.for_crawl_runtime(extra_context={"snapshot_id": "default-runtime-config"}) assert default_runtime_config["SEARCH_BACKEND_RIPGREP_ENABLED"] is True assert default_runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is False assert default_runtime_config["SEARCH_BACKEND_SONIC_ENABLED"] is True diff --git a/docs/Configuration.md b/docs/Configuration.md index 618b3cf3..2ff5154e 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -770,7 +770,7 @@ Lower values retry more aggressively (useful if you expect locks to clear quickl *Options for full-text search backend configuration.* -ArchiveBox can index Snapshot text/HTML output into a searchable index that powers the search bar in the Web UI and the `archivebox search ` CLI command. Multiple backend engines are supported — pick the one that best matches your collection size, available system resources, and tolerance for extra moving parts. +ArchiveBox can index Snapshot text/HTML output into searchable indexes that power the search bar in the Web UI and the `archivebox search ` CLI command. Multiple backend engines can be enabled at once; `SEARCH_BACKEND_ENGINE` selects the default used by the UI and CLI. > [!NOTE] > Each backend has its own tuning knobs (e.g. [Sonic](https://archivebox.github.io/abx-plugins/#search_backend_sonic) host/port, [ripgrep](https://archivebox.github.io/abx-plugins/#search_backend_ripgrep) flags, [SQLite FTS](https://archivebox.github.io/abx-plugins/#search_backend_sqlite) database path). Those backend-specific options now live with the plugin that implements them — see the [abx-plugins docs](https://archivebox.github.io/abx-plugins/) for the full per-backend schema. @@ -781,9 +781,9 @@ ArchiveBox can index Snapshot text/HTML output into a searchable index that powe Which search backend engine to use when running `archivebox search` and rendering the Web UI search bar. -- **`ripgrep`** *(default)* — Pure filesystem grep across each Snapshot's archived output (HTML, text, metadata) via the [`search_backend_ripgrep`](https://archivebox.github.io/abx-plugins/#search_backend_ripgrep) plugin. No extra daemon, no extra database to maintain — just install `rg` and it works. Slow on very large collections (each query re-scans the disk) but always 100% correct: results reflect what's actually on disk *right now*, no stale index. Best choice for small-to-medium collections (≲50k snapshots) and for users who don't want to run extra services. +- **`ripgrep`** — Pure filesystem grep across each Snapshot's archived output (HTML, text, metadata) via the [`search_backend_ripgrep`](https://archivebox.github.io/abx-plugins/#search_backend_ripgrep) plugin. No extra daemon, no extra database to maintain — just install `rg` and it works. Slow on very large collections (each query re-scans the disk) but always 100% correct: results reflect what's actually on disk *right now*, no stale index. ArchiveBox keeps it enabled as the fallback when Sonic is unavailable. -- **`sonic`** — Fast, suggest-style fuzzy search via a running [Sonic](https://github.com/valeriansaliou/sonic) daemon (configured via the [`search_backend_sonic`](https://archivebox.github.io/abx-plugins/#search_backend_sonic) plugin). ArchiveBox pushes text into Sonic at index time and queries it at search time. Sub-millisecond queries even at very large scale; ArchiveBox starts the managed service automatically when this backend is selected. Best choice for large collections (≳100k snapshots) when query latency matters. +- **`sonic`** *(default)* — Fast, suggest-style fuzzy search via a running [Sonic](https://github.com/valeriansaliou/sonic) daemon (configured via the [`search_backend_sonic`](https://archivebox.github.io/abx-plugins/#search_backend_sonic) plugin). ArchiveBox pushes text into Sonic at index time and queries it at search time. Sub-millisecond queries even at very large scale; ArchiveBox starts the managed service automatically when this backend is selected. - **`sqlite`** — FTS5 full-text index stored alongside ArchiveBox's main `index.sqlite3`, configured via the [`search_backend_sqlite`](https://archivebox.github.io/abx-plugins/#search_backend_sqlite) plugin. No extra processes, no extra binary — uses the SQLite already shipped with Python. Faster than `ripgrep` on large collections, slightly slower than `sonic`, but no daemon to babysit. Good middle ground for users who want a real index without operational overhead. diff --git a/docs/Docker.md b/docs/Docker.md index 05ecee2d..71ed4a98 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -68,10 +68,9 @@ docker compose run archivebox init docker compose run archivebox manage createsuperuser ``` -ArchiveBox installs and enables both ripgrep and [Sonic](https://github.com/valeriansaliou/sonic). To make Sonic the default engine selected in the UI while keeping both backends available: +ArchiveBox installs and enables both ripgrep and [Sonic](https://github.com/valeriansaliou/sonic). Sonic is selected by default in the UI, while ripgrep remains available as the fallback. To select ripgrep explicitly: ```bash -docker compose run archivebox config --set SEARCH_BACKEND_ENGINE=sonic -docker compose run archivebox update --index-only +docker compose run archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep ```
diff --git a/docs/Setting-up-Search.md b/docs/Setting-up-Search.md index 29eb647b..89fbbf81 100644 --- a/docs/Setting-up-Search.md +++ b/docs/Setting-up-Search.md @@ -31,31 +31,27 @@ ArchiveBox search works by doing substring matches in `Snapshot` metadata fields ArchiveBox provides a number of "Search Backend Engines" to tune its performance & behavior for different use-cases. ```bash # this setting controls which search backend ArchiveBox uses -archivebox config --set SEARCH_BACKEND_ENGINE=ripgrep +archivebox config --set SEARCH_BACKEND_ENGINE=sonic # to see information about the backend you are currently using, run: archivebox version archivebox config --get SEARCH_BACKEND_ENGINE ``` -By default out-of-the-box, the selected engine is a simple but efficient tool similar to `grep -r` called [`ripgrep`](https://github.com/BurntSushi/ripgrep). - -Ripgrep is [currently the fastest](https://blog.burntsushi.net/ripgrep/) available *filesystem search* tool that scans over the raw archived files on every search. We chose it as the default so that beginners and 95% of users with small collections can have an experience that "just works", without needing to install and maintain complex additional dependencies or background workers. - -However, there are some fundamental limitations of scanning through every file on disk each time a search is done, so ArchiveBox provides a number of additional search backend options for when users outgrow `ripgrep`. +ArchiveBox installs and enables both [Sonic](https://github.com/valeriansaliou/sonic) and [`ripgrep`](https://github.com/BurntSushi/ripgrep). Sonic is selected by default for fast indexed search, while ripgrep remains available as the filesystem fallback when Sonic is unavailable or explicitly selected. > [!TIP] -> **You should consider switching ArchiveBox to use `sonic` or another backend IF:** +> **You should consider selecting `ripgrep` instead of Sonic if:** > -> - you have more than 1,000 Snapshots saved in your archive -> - your archive data is stored on a slower filesystem like a spinning hard drive or remote network mount -> - you want more advanced search features like stemming, boolean operators, and ability to search PDFs, eBooks, ZIP/tar files, etc. +> - Sonic is unavailable on your platform +> - you prefer filesystem scanning without a background daemon +> - you need ripgrep-compatible regular expressions
-### `ripgrep` *(the default)* +### `ripgrep` *(fallback)* ArchiveBox resolves `ripgrep` through `abxpkg`: a compatible host installation is used first, otherwise a managed copy is installed. diff --git a/docs/Usage.md b/docs/Usage.md index c33a7eab..fd5bcf15 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -175,7 +175,7 @@ Click the Favicon under the "Files" column to go to the details page for each li A logged-in admin user may select ☑️ one or more snapshots from the list and perform Snapshot actions: -- Search Search text in the Snapshot title, URL, tags, or archived content (supports regex with the default ripgrep search backend, or enable the [Sonic](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docker-compose.yml#L35) full-text search backend in `docker-compose.yml` and set `SEARCH_BACKEND_ENGINE=sonic`, `SEARCH_BACKEND_HOST`, `SEARCH_BACKEND_PASSWORD` for full-text fuzzy searching) https://github.com/ArchiveBox/ArchiveBox/issues/956 +- Search Search text in the Snapshot title, URL, tags, or archived content. [Sonic](https://github.com/valeriansaliou/sonic) is the default indexed search backend, with ripgrep enabled as the filesystem fallback and regex-search option. https://github.com/ArchiveBox/ArchiveBox/issues/956 - Tags Start typing in the field to select some tags, then click `+` to add them or `-` remove them from the checked snapshots (`Tags` can be created/edited from the `/admin/core/tag/` page) - Title Pull the latest title and favicon without doing a full snapshot. (helpful to quickly ping any URLs that are stuck showing up as `Pending...` or are missing a title) - Pull Finish downloading the Snapshot, pulls any missing/failed outputs/extractors methods (pdf, wget... etc). Resumes running the same archiving steps as when you add new URL. Useful to finish pulling when previous import was paused or interrupted by a reboot or something. https://github.com/ArchiveBox/ArchiveBox#output-formats