fix(docker): terminate hung hardware probes on timeout

Opt-in local process-group SIGKILL and remote SSH stream/client cleanup
so getServerHardware cannot wait indefinitely on stalled docker/df/nvidia-smi.

apps/dokploy/__test__/server/exec-async-timeout.test.ts

@@ -0,0 +1,324 @@
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { findServerById } from "@dokploy/server/services/server";
+import {
+	ExecError,
+	execAsync,
+	execAsyncRemote,
+} from "@dokploy/server/utils/process/execAsync";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { ssh, MockClient } = vi.hoisted(() => {
+	class MockStream {
+		close = vi.fn(() => {
+			this.closed = true;
+		});
+		destroy = vi.fn(() => {
+			this.destroyed = true;
+		});
+		closed = false;
+		destroyed = false;
+		handlers: Record<string, Array<(...args: unknown[]) => void>> = {};
+
+		on(event: string, cb: (...args: unknown[]) => void) {
+			if (!this.handlers[event]) this.handlers[event] = [];
+			this.handlers[event].push(cb);
+			return this;
+		}
+
+		stderr = {
+			on: (_event: string, _cb: (...args: unknown[]) => void) => this,
+		};
+
+		emitClose(code: number) {
+			for (const cb of this.handlers.close ?? []) cb(code, null);
+		}
+
+		emitData(data: string) {
+			for (const cb of this.handlers.data ?? []) cb(data);
+		}
+	}
+
+	class MockClient {
+		ended = false;
+		destroyed = false;
+		execCommand = "";
+		execShouldFail: Error | null = null;
+		stream = new MockStream();
+		end = vi.fn(() => {
+			this.ended = true;
+		});
+		destroy = vi.fn(() => {
+			this.destroyed = true;
+		});
+		handlers: Record<string, Array<(...args: unknown[]) => void>> = {};
+
+		constructor() {
+			ssh.instances.push(this);
+		}
+
+		once(event: string, cb: (...args: unknown[]) => void) {
+			if (!this.handlers[event]) this.handlers[event] = [];
+			this.handlers[event].push(cb);
+			return this;
+		}
+
+		on(event: string, cb: (...args: unknown[]) => void) {
+			return this.once(event, cb);
+		}
+
+		connect() {
+			return this;
+		}
+
+		exec(command: string, cb: (err: Error | null, stream: MockStream) => void) {
+			this.execCommand = command;
+			if (this.execShouldFail) {
+				cb(this.execShouldFail, this.stream);
+				return;
+			}
+			cb(null, this.stream);
+		}
+
+		emitReady() {
+			for (const cb of this.handlers.ready ?? []) cb();
+		}
+
+		emitError(err: Error & { level?: string }) {
+			for (const cb of this.handlers.error ?? []) cb(err);
+		}
+	}
+
+	const ssh = { instances: [] as MockClient[] };
+	return { ssh, MockClient };
+});
+
+vi.mock("ssh2", () => ({
+	Client: MockClient,
+}));
+
+vi.mock("@dokploy/server/services/server", () => ({
+	findServerById: vi.fn(),
+}));
+
+const pidAlive = (pid: number) => {
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch {
+		return false;
+	}
+};
+
+const readPid = (file: string) => Number(readFileSync(file, "utf8").trim());
+
+describe("execAsync timeout", () => {
+	const dirs: string[] = [];
+
+	afterEach(() => {
+		for (const dir of dirs) {
+			rmSync(dir, { recursive: true, force: true });
+		}
+		dirs.length = 0;
+	});
+
+	it("does not apply a timeout unless one is requested", async () => {
+		const result = await execAsync("printf ok");
+		expect(result.stdout).toContain("ok");
+	});
+
+	it("rejects a hanging local command and kills the process group", async () => {
+		const dir = mkdtempSync(path.join(tmpdir(), "dokploy-exec-timeout-"));
+		dirs.push(dir);
+		const shellPidFile = path.join(dir, "shell.pid");
+		const sleepPidFile = path.join(dir, "sleep.pid");
+		const started = Date.now();
+		const command = `echo $$ > "${shellPidFile}"; sleep 30 & echo $! > "${sleepPidFile}"; wait`;
+
+		await expect(execAsync(command, { timeout: 300 })).rejects.toSatisfy(
+			(error: unknown) =>
+				error instanceof ExecError &&
+				/timed out after 300ms/.test(error.message),
+		);
+		expect(Date.now() - started).toBeLessThan(2000);
+
+		await vi.waitFor(
+			() => {
+				expect(pidAlive(readPid(shellPidFile))).toBe(false);
+				expect(pidAlive(readPid(sleepPidFile))).toBe(false);
+			},
+			{ timeout: 2000, interval: 50 },
+		);
+	});
+
+	it("rejects nonzero exit before the timeout without treating it as a hang", async () => {
+		await expect(execAsync("exit 7", { timeout: 2000 })).rejects.toSatisfy(
+			(error: unknown) =>
+				error instanceof ExecError &&
+				!/timed out/.test(error.message) &&
+				error.exitCode === 7,
+		);
+	});
+});
+
+describe("execAsyncRemote timeout", () => {
+	const server = {
+		sshKeyId: "key-1",
+		ipAddress: "203.0.113.10",
+		port: 22,
+		username: "root",
+		sshKey: { privateKey: "fake-key" },
+	};
+
+	beforeEach(() => {
+		ssh.instances.length = 0;
+		vi.mocked(findServerById).mockReset();
+		vi.mocked(findServerById).mockResolvedValue(server as never);
+		vi.useFakeTimers();
+	});
+
+	afterEach(() => {
+		vi.clearAllTimers();
+		vi.useRealTimers();
+	});
+
+	const flush = () => Promise.resolve();
+
+	const startRemote = (onData?: (data: string) => void, timeout = 1000) =>
+		execAsyncRemote("server-1", "nvidia-smi", onData, { timeout });
+
+	it("times out before ready and closes the SSH client", async () => {
+		const pending = startRemote();
+		const rejected = expect(pending).rejects.toSatisfy(
+			(error: unknown) =>
+				error instanceof ExecError &&
+				/timed out after 1000ms/.test(error.message),
+		);
+		await flush();
+		const client = ssh.instances[0];
+		expect(client).toBeDefined();
+		await vi.advanceTimersByTimeAsync(1000);
+		await rejected;
+		expect(client?.stream.close).not.toHaveBeenCalled();
+		expect(client?.end).toHaveBeenCalled();
+	});
+
+	it("times out after the stream exists and closes stream plus client", async () => {
+		const pending = startRemote();
+		const rejected = expect(pending).rejects.toMatchObject({
+			name: "ExecError",
+		});
+		await flush();
+		const client = ssh.instances[0];
+		client?.emitReady();
+		expect(client?.execCommand).toBe("nvidia-smi");
+		await vi.advanceTimersByTimeAsync(1000);
+		await rejected;
+		expect(client?.stream.close).toHaveBeenCalledTimes(1);
+		expect(client?.end).toHaveBeenCalledTimes(1);
+		client?.stream.emitClose(0);
+		expect(client?.end).toHaveBeenCalledTimes(1);
+	});
+
+	it("clears the timeout timer after a successful remote command", async () => {
+		const pending = startRemote(undefined, 5000);
+		await flush();
+		expect(vi.getTimerCount()).toBeGreaterThanOrEqual(1);
+		const client = ssh.instances[0];
+		client?.emitReady();
+		const timersBeforeClose = vi.getTimerCount();
+		client?.stream.emitClose(0);
+		await expect(pending).resolves.toEqual({ stdout: "", stderr: "" });
+		expect(vi.getTimerCount()).toBeLessThan(timersBeforeClose);
+	});
+
+	it("resolves on success and does not reject later when the timer would have fired", async () => {
+		const rejections: unknown[] = [];
+		const onUnhandled = (reason: unknown) => {
+			rejections.push(reason);
+		};
+		process.on("unhandledRejection", onUnhandled);
+
+		const pending = startRemote(undefined, 5000);
+		await flush();
+		const client = ssh.instances[0];
+		client?.emitReady();
+		client?.stream.emitClose(0);
+		await expect(pending).resolves.toEqual({ stdout: "", stderr: "" });
+		await vi.advanceTimersByTimeAsync(10_000);
+		process.off("unhandledRejection", onUnhandled);
+		expect(rejections).toEqual([]);
+		expect(client?.ended).toBe(true);
+	});
+
+	it("settles only once when close and timeout race", async () => {
+		const pending = startRemote();
+		await flush();
+		const client = ssh.instances[0];
+		client?.emitReady();
+		client?.stream.emitClose(0);
+		await vi.advanceTimersByTimeAsync(1000);
+		await expect(pending).resolves.toEqual({ stdout: "", stderr: "" });
+	});
+
+	it("rejects SSH errors before timeout and ignores the later timer", async () => {
+		const rejections: unknown[] = [];
+		const onUnhandled = (reason: unknown) => {
+			rejections.push(reason);
+		};
+		process.on("unhandledRejection", onUnhandled);
+
+		const pending = startRemote();
+		const rejected = expect(pending).rejects.toSatisfy(
+			(error: unknown) =>
+				error instanceof ExecError &&
+				error.message.includes("SSH connection error"),
+		);
+		await flush();
+		const client = ssh.instances[0];
+		client?.emitError(
+			Object.assign(new Error("offline"), { level: "client-socket" }),
+		);
+		await rejected;
+		await vi.advanceTimersByTimeAsync(1000);
+		process.off("unhandledRejection", onUnhandled);
+		expect(rejections).toEqual([]);
+		expect(client?.end).toHaveBeenCalled();
+	});
+
+	it("rejects exec callback errors before timeout", async () => {
+		const pending = startRemote();
+		const rejected = expect(pending).rejects.toSatisfy(
+			(error: unknown) =>
+				error instanceof ExecError &&
+				error.message.includes("Remote command execution failed"),
+		);
+		await flush();
+		const client = ssh.instances[0];
+		if (client) client.execShouldFail = new Error("cannot exec");
+		client?.emitReady();
+		await rejected;
+		await vi.advanceTimersByTimeAsync(1000);
+		expect(client?.end).toHaveBeenCalled();
+	});
+
+	it("still delivers onData chunks if the command hangs afterward", async () => {
+		const chunks: string[] = [];
+		const pending = startRemote((data) => {
+			chunks.push(data);
+		});
+		const rejected = expect(pending).rejects.toMatchObject({
+			name: "ExecError",
+		});
+		await flush();
+		const client = ssh.instances[0];
+		client?.emitReady();
+		client?.stream.emitData("partial");
+		await vi.advanceTimersByTimeAsync(1000);
+		await rejected;
+		expect(chunks).toEqual(["partial"]);
+		expect(client?.stream.close).toHaveBeenCalled();
+		expect(client?.end).toHaveBeenCalled();
+	});
+});

apps/dokploy/__test__/server/server-hardware.test.ts

@@ -6,6 +6,7 @@ import {
 	buildLocalHardwareScript,
 	buildRemoteHardwareScript,
 	getServerHardware,
+	HARDWARE_PROBE_TIMEOUT_MS,
 	parseNvidiaGpuLine,
 	parseServerHardware,
 } from "@dokploy/server/services/server-hardware";
@@ -799,6 +800,7 @@ describe("getServerHardware", () => {
 		const result = await getServerHardware();
 		expect(execAsync).toHaveBeenCalledWith(
 			expect.stringContaining("docker info --format"),
+			expect.objectContaining({ timeout: HARDWARE_PROBE_TIMEOUT_MS }),
 		);
 		expect(execAsyncRemote).not.toHaveBeenCalled();
 		expect(result.cpu.count).toBe(8);
@@ -820,6 +822,8 @@ describe("getServerHardware", () => {
 		expect(execAsyncRemote).toHaveBeenCalledWith(
 			"remote-server",
 			expect.stringContaining("/proc/meminfo"),
+			undefined,
+			expect.objectContaining({ timeout: HARDWARE_PROBE_TIMEOUT_MS }),
 		);
 		expect(execAsync).not.toHaveBeenCalled();
 		expect(result.gpu.devices[0]?.uuid).toBe("GPU-aaa");
@@ -844,4 +848,16 @@ describe("getServerHardware", () => {
 		expect(result.gpu.detection).toBe("unavailable");
 		expect(execAsync).not.toHaveBeenCalled();
 	});
+
+	it("treats a timed-out probe as empty hardware, not zero GPUs", async () => {
+		vi.mocked(execAsync).mockRejectedValue(
+			new Error("Command execution timed out after 30000ms"),
+		);
+		const result = await getServerHardware();
+		expect(result.error).toContain("timed out");
+		expect(result.cpu.count).toBeNull();
+		expect(result.memory.totalBytes).toBeNull();
+		expect(result.disk.totalBytes).toBeNull();
+		expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
+	});
 });

packages/server/src/services/server-hardware.ts

@@ -5,6 +5,10 @@ import {
 import { TRPCError } from "@trpc/server";
 import { IS_CLOUD } from "../constants";
 
+// Matches remoteStream SSH readyTimeout. Hardware commands are fast; this
+// bounds stalled docker/nvidia-smi/df/SSH so the API cannot wait forever.
+export const HARDWARE_PROBE_TIMEOUT_MS = 30_000;
+
 const NVIDIA_INVENTORY_QUERY =
 	"nvidia-smi --query-gpu=index,uuid,name,memory.total,memory.free,driver_version --format=csv,noheader,nounits";
 const NVIDIA_COMPUTE_CAP_QUERY =
@@ -365,8 +369,10 @@ export const getServerHardware = async (
 		: buildLocalHardwareScript();
 	try {
 		const result = serverId
-			? await execAsyncRemote(serverId, script)
-			: await execAsync(script);
+			? await execAsyncRemote(serverId, script, undefined, {
+					timeout: HARDWARE_PROBE_TIMEOUT_MS,
+				})
+			: await execAsync(script, { timeout: HARDWARE_PROBE_TIMEOUT_MS });
 		return parseServerHardware(result.stdout);
 	} catch (error) {
 		return emptyHardware(error);

packages/server/src/utils/process/execAsync.ts

@@ -1,4 +1,4 @@
-import { exec, execFile } from "node:child_process";
+import { type ChildProcess, exec, execFile, spawn } from "node:child_process";
 import util from "node:util";
 import { findServerById } from "@dokploy/server/services/server";
 import { Client } from "ssh2";
@@ -23,12 +23,119 @@ export { ExecError } from "./ExecError";
 
 const execAsyncBase = util.promisify(exec);
 
+export interface ExecAsyncOptions {
+	cwd?: string;
+	env?: NodeJS.ProcessEnv;
+	shell?: string;
+	timeout?: number;
+}
+
+export interface ExecAsyncRemoteOptions {
+	timeout?: number;
+}
+
+const killProcessGroup = (child: ChildProcess) => {
+	if (child.pid == null) return;
+	if (process.platform === "win32") {
+		child.kill("SIGKILL");
+		return;
+	}
+	try {
+		process.kill(-child.pid, "SIGKILL");
+	} catch {
+		try {
+			child.kill("SIGKILL");
+		} catch {
+			// already exited
+		}
+	}
+};
+
+const execAsyncTimed = (
+	command: string,
+	options: ExecAsyncOptions & { timeout: number },
+): Promise<{ stdout: string; stderr: string }> => {
+	const { timeout, cwd, env, shell } = options;
+	const isWin = process.platform === "win32";
+	const shellPath =
+		shell ?? (isWin ? process.env.ComSpec || "cmd.exe" : "/bin/sh");
+	const args = isWin ? ["/d", "/s", "/c", command] : ["-c", command];
+
+	return new Promise((resolve, reject) => {
+		let stdout = "";
+		let stderr = "";
+		let settled = false;
+		const child = spawn(shellPath, args, {
+			cwd,
+			env,
+			detached: !isWin,
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+
+		const timer = setTimeout(() => {
+			killProcessGroup(child);
+			finish(
+				new ExecError(`Command execution timed out after ${timeout}ms`, {
+					command,
+					stdout,
+					stderr,
+				}),
+			);
+		}, timeout);
+
+		const finish = (error?: ExecError) => {
+			if (settled) return;
+			settled = true;
+			clearTimeout(timer);
+			if (error) reject(error);
+			else resolve({ stdout, stderr });
+		};
+
+		child.stdout?.on("data", (data) => {
+			stdout += data.toString();
+		});
+		child.stderr?.on("data", (data) => {
+			stderr += data.toString();
+		});
+		child.on("error", (error) => {
+			finish(
+				new ExecError(`Command execution failed: ${error.message}`, {
+					command,
+					stdout,
+					stderr,
+					originalError: error,
+				}),
+			);
+		});
+		child.on("close", (code) => {
+			if (code === 0) {
+				finish();
+				return;
+			}
+			finish(
+				new ExecError(`Command execution failed: Command failed: ${command}`, {
+					command,
+					stdout,
+					stderr,
+					exitCode: code ?? undefined,
+				}),
+			);
+		});
+	});
+};
+
 export const execAsync = async (
 	command: string,
-	options?: { cwd?: string; env?: NodeJS.ProcessEnv; shell?: string },
+	options?: ExecAsyncOptions,
 ): Promise<{ stdout: string; stderr: string }> => {
+	const timeout = options?.timeout;
+	if (timeout != null && timeout > 0) {
+		return execAsyncTimed(command, { ...options, timeout });
+	}
+
+	const { timeout: _timeout, ...execOptions } = options ?? {};
 	try {
-		const result = await execAsyncBase(command, options);
+		const result = await execAsyncBase(command, execOptions);
 		return {
 			stdout: result.stdout.toString(),
 			stderr: result.stderr.toString(),
@@ -156,6 +263,7 @@ export const execAsyncRemote = async (
 	serverId: string | null,
 	command: string,
 	onData?: (data: string) => void,
+	options?: ExecAsyncRemoteOptions,
 ): Promise<{ stdout: string; stderr: string }> => {
 	if (!serverId) return { stdout: "", stderr: "" };
 	const server = await findServerById(serverId);
@@ -163,16 +271,84 @@ export const execAsyncRemote = async (
 
 	let stdout = "";
 	let stderr = "";
+	const timeoutMs = options?.timeout;
 	return new Promise((resolve, reject) => {
 		const conn = new Client();
+		let stream: { close: () => void; destroy: () => void } | undefined;
+		let settled = false;
+		let timer: ReturnType<typeof setTimeout> | undefined;
+
+		const settle = (
+			error?: ExecError,
+			value?: { stdout: string; stderr: string },
+		) => {
+			if (settled) return;
+			settled = true;
+			if (timer !== undefined) {
+				clearTimeout(timer);
+				timer = undefined;
+			}
+			if (error) {
+				try {
+					stream?.close();
+				} catch {
+					// already closed
+				}
+				try {
+					stream?.destroy();
+				} catch {
+					// already destroyed
+				}
+				try {
+					conn.end();
+				} catch {
+					// already ended
+				}
+				try {
+					conn.destroy();
+				} catch {
+					// already destroyed
+				}
+				reject(error);
+				return;
+			}
+			try {
+				conn.end();
+			} catch {
+				// already ended
+			}
+			resolve(value ?? { stdout, stderr });
+		};
+
+		if (timeoutMs != null && timeoutMs > 0) {
+			timer = setTimeout(() => {
+				settle(
+					new ExecError(`Command execution timed out after ${timeoutMs}ms`, {
+						command,
+						stdout,
+						stderr,
+						serverId,
+					}),
+				);
+			}, timeoutMs);
+		}
 
 		sleep(1000);
 		conn
 			.once("ready", () => {
-				conn.exec(command, (err, stream) => {
+				if (settled) return;
+				conn.exec(command, (err, commandStream) => {
+					if (settled) {
+						try {
+							commandStream?.close();
+						} catch {
+							// already closed
+						}
+						return;
+					}
 					if (err) {
 						onData?.(err.message);
-						reject(
+						settle(
 							new ExecError(`Remote command execution failed: ${err.message}`, {
 								command,
 								serverId,
@@ -181,13 +357,13 @@ export const execAsyncRemote = async (
 						);
 						return;
 					}
-					stream
+					stream = commandStream;
+					commandStream
 						.on("close", (code: number, _signal: string) => {
-							conn.end();
 							if (code === 0) {
-								resolve({ stdout, stderr });
+								settle(undefined, { stdout, stderr });
 							} else {
-								reject(
+								settle(
 									new ExecError(
 										`Remote command failed with exit code ${code}`,
 										{
@@ -212,7 +388,7 @@ export const execAsyncRemote = async (
 				});
 			})
 			.on("error", (err) => {
-				conn.end();
+				if (settled) return;
 				if (err.level === "client-authentication") {
 					const technicalDetail = `Error: ${err.message} ${err.level}`;
 					const friendlyMessage = [
@@ -228,9 +404,8 @@ export const execAsyncRemote = async (
 						"  • Try generating a new SSH key in Dokploy and add only the public key to the server, then try again.",
 						"  • Make sure to follow the instructions on the Setup Server Button on the SSH Keys tab and then click on deployments tab and check the logs for more details.",
 					].join("\n");
-					const errorMsg = `Authentication failed: Invalid SSH private key. ❌ Error: ${err.message} ${err.level}`;
 					onData?.(friendlyMessage);
-					reject(
+					settle(
 						new ExecError(
 							`Authentication failed: Invalid SSH private key. ${friendlyMessage}`,
 							{
@@ -243,7 +418,7 @@ export const execAsyncRemote = async (
 				} else {
 					const errorMsg = `SSH connection error: ${err.message}`;
 					onData?.(errorMsg);
-					reject(
+					settle(
 						new ExecError(errorMsg, {
 							command,
 							serverId,
@@ -257,7 +432,10 @@ export const execAsyncRemote = async (
 				port: server.port,
 				username: server.username,
 				privateKey: server.sshKey?.privateKey,
-				timeout: 99999,
+				timeout: timeoutMs != null && timeoutMs > 0 ? timeoutMs : 99999,
+				...(timeoutMs != null && timeoutMs > 0
+					? { readyTimeout: timeoutMs }
+					: {}),
 			});
 	});
 };