diff --git a/archivebox/cli/archivebox_install.py b/archivebox/cli/archivebox_install.py index 509396db..b9c666fe 100755 --- a/archivebox/cli/archivebox_install.py +++ b/archivebox/cli/archivebox_install.py @@ -167,6 +167,10 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo install_plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip()) if install_plugin_names or not binaries: + if not binaries: + from archivebox.workers.supervisord_util import require_dependency_install_memory + + require_dependency_install_memory() print("[+] Running plugin installer via abx-dl bus...") print() diff --git a/archivebox/tests/test_cli_install.py b/archivebox/tests/test_cli_install.py index 9ce7a4b9..7ae0c110 100644 --- a/archivebox/tests/test_cli_install.py +++ b/archivebox/tests/test_cli_install.py @@ -79,6 +79,22 @@ def test_install_target_resolver_uses_declared_binary_aliases(): assert _resolve_install_targets(("not-a-real-tool",)) == ([], ["not-a-real-tool"]) +def test_default_dependency_install_memory_preflight_fails_before_installers(capsys): + from archivebox.workers.supervisord_util import ( + MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES, + require_dependency_install_memory, + ) + + require_dependency_install_memory(MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES) + with pytest.raises(SystemExit) as err: + require_dependency_install_memory(MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES - 1) + + assert err.value.code == 1 + output = capsys.readouterr().err + assert "Not enough available memory to install ArchiveBox dependencies" in output + assert "No plugin dependency installers were started" in output + + def test_install_shows_binary_status(initialized_archive): """Test that install shows status of binaries.""" diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py index 26631706..e4c99b83 100644 --- a/archivebox/workers/supervisord_util.py +++ b/archivebox/workers/supervisord_util.py @@ -46,6 +46,7 @@ _RUNTIME_COMPONENT_ORDER = ("orchestrator", "server", "sonic") _SUPERVISORD_ERRORS = (XmlRpcError, OSError, RuntimeError, TimeoutError) _PROCESS_STATE_ERRORS = (DatabaseError, OSError, RuntimeError, ValueError, psutil.Error) MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES = 256 * 1024 * 1024 +MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES = MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES def _shell_join(args: list[str]) -> str: @@ -81,22 +82,41 @@ def effective_available_memory_bytes() -> int: return available -def require_server_worker_memory(available_bytes: int | None = None) -> None: - available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes - if available_bytes >= MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES: +def _require_available_memory(operation: str, idle_message: str, available_bytes: int, required_bytes: int) -> None: + if available_bytes >= required_bytes: return available_mib = available_bytes // (1024 * 1024) - required_mib = MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES // (1024 * 1024) - STDERR.print("[red][X] Not enough available memory to start ArchiveBox safely.[/red]") + required_mib = required_bytes // (1024 * 1024) + STDERR.print(f"[red][X] Not enough available memory to {operation} safely.[/red]") STDERR.print( - f" Available RAM + swap: {available_mib} MiB; at least {required_mib} MiB must be free before starting the server workers.", + f" Available RAM + swap: {available_mib} MiB; at least {required_mib} MiB must be free before this operation.", ) - STDERR.print(" No server, runner, or Sonic workers were started.") + STDERR.print(idle_message) STDERR.print(" Use a host/container with at least 1 GB RAM or configure swap, then run the same command again.") raise SystemExit(1) +def require_server_worker_memory(available_bytes: int | None = None) -> None: + available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes + _require_available_memory( + "start ArchiveBox", + " No server, runner, or Sonic workers were started.", + available_bytes, + MIN_SERVER_WORKER_AVAILABLE_MEMORY_BYTES, + ) + + +def require_dependency_install_memory(available_bytes: int | None = None) -> None: + available_bytes = effective_available_memory_bytes() if available_bytes is None else available_bytes + _require_available_memory( + "install ArchiveBox dependencies", + " No plugin dependency installers were started.", + available_bytes, + MIN_DEPENDENCY_INSTALL_AVAILABLE_MEMORY_BYTES, + ) + + def archivebox_cmd(*args: str) -> list[str]: return [str(resolve_env_binary("archivebox")), *args]