diff --git a/apps/dokploy/__test__/server/exec-async-timeout.test.ts b/apps/dokploy/__test__/server/exec-async-timeout.test.ts new file mode 100644 index 000000000..3f30a40db --- /dev/null +++ b/apps/dokploy/__test__/server/exec-async-timeout.test.ts @@ -0,0 +1,324 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { findServerById } from "@dokploy/server/services/server"; +import { + ExecError, + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { ssh, MockClient } = vi.hoisted(() => { + class MockStream { + close = vi.fn(() => { + this.closed = true; + }); + destroy = vi.fn(() => { + this.destroyed = true; + }); + closed = false; + destroyed = false; + handlers: Record void>> = {}; + + on(event: string, cb: (...args: unknown[]) => void) { + if (!this.handlers[event]) this.handlers[event] = []; + this.handlers[event].push(cb); + return this; + } + + stderr = { + on: (_event: string, _cb: (...args: unknown[]) => void) => this, + }; + + emitClose(code: number) { + for (const cb of this.handlers.close ?? []) cb(code, null); + } + + emitData(data: string) { + for (const cb of this.handlers.data ?? []) cb(data); + } + } + + class MockClient { + ended = false; + destroyed = false; + execCommand = ""; + execShouldFail: Error | null = null; + stream = new MockStream(); + end = vi.fn(() => { + this.ended = true; + }); + destroy = vi.fn(() => { + this.destroyed = true; + }); + handlers: Record void>> = {}; + + constructor() { + ssh.instances.push(this); + } + + once(event: string, cb: (...args: unknown[]) => void) { + if (!this.handlers[event]) this.handlers[event] = []; + this.handlers[event].push(cb); + return this; + } + + on(event: string, cb: (...args: unknown[]) => void) { + return this.once(event, cb); + } + + connect() { + return this; + } + + exec(command: string, cb: (err: Error | null, stream: MockStream) => void) { + this.execCommand = command; + if (this.execShouldFail) { + cb(this.execShouldFail, this.stream); + return; + } + cb(null, this.stream); + } + + emitReady() { + for (const cb of this.handlers.ready ?? []) cb(); + } + + emitError(err: Error & { level?: string }) { + for (const cb of this.handlers.error ?? []) cb(err); + } + } + + const ssh = { instances: [] as MockClient[] }; + return { ssh, MockClient }; +}); + +vi.mock("ssh2", () => ({ + Client: MockClient, +})); + +vi.mock("@dokploy/server/services/server", () => ({ + findServerById: vi.fn(), +})); + +const pidAlive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +const readPid = (file: string) => Number(readFileSync(file, "utf8").trim()); + +describe("execAsync timeout", () => { + const dirs: string[] = []; + + afterEach(() => { + for (const dir of dirs) { + rmSync(dir, { recursive: true, force: true }); + } + dirs.length = 0; + }); + + it("does not apply a timeout unless one is requested", async () => { + const result = await execAsync("printf ok"); + expect(result.stdout).toContain("ok"); + }); + + it("rejects a hanging local command and kills the process group", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "dokploy-exec-timeout-")); + dirs.push(dir); + const shellPidFile = path.join(dir, "shell.pid"); + const sleepPidFile = path.join(dir, "sleep.pid"); + const started = Date.now(); + const command = `echo $$ > "${shellPidFile}"; sleep 30 & echo $! > "${sleepPidFile}"; wait`; + + await expect(execAsync(command, { timeout: 300 })).rejects.toSatisfy( + (error: unknown) => + error instanceof ExecError && + /timed out after 300ms/.test(error.message), + ); + expect(Date.now() - started).toBeLessThan(2000); + + await vi.waitFor( + () => { + expect(pidAlive(readPid(shellPidFile))).toBe(false); + expect(pidAlive(readPid(sleepPidFile))).toBe(false); + }, + { timeout: 2000, interval: 50 }, + ); + }); + + it("rejects nonzero exit before the timeout without treating it as a hang", async () => { + await expect(execAsync("exit 7", { timeout: 2000 })).rejects.toSatisfy( + (error: unknown) => + error instanceof ExecError && + !/timed out/.test(error.message) && + error.exitCode === 7, + ); + }); +}); + +describe("execAsyncRemote timeout", () => { + const server = { + sshKeyId: "key-1", + ipAddress: "203.0.113.10", + port: 22, + username: "root", + sshKey: { privateKey: "fake-key" }, + }; + + beforeEach(() => { + ssh.instances.length = 0; + vi.mocked(findServerById).mockReset(); + vi.mocked(findServerById).mockResolvedValue(server as never); + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + }); + + const flush = () => Promise.resolve(); + + const startRemote = (onData?: (data: string) => void, timeout = 1000) => + execAsyncRemote("server-1", "nvidia-smi", onData, { timeout }); + + it("times out before ready and closes the SSH client", async () => { + const pending = startRemote(); + const rejected = expect(pending).rejects.toSatisfy( + (error: unknown) => + error instanceof ExecError && + /timed out after 1000ms/.test(error.message), + ); + await flush(); + const client = ssh.instances[0]; + expect(client).toBeDefined(); + await vi.advanceTimersByTimeAsync(1000); + await rejected; + expect(client?.stream.close).not.toHaveBeenCalled(); + expect(client?.end).toHaveBeenCalled(); + }); + + it("times out after the stream exists and closes stream plus client", async () => { + const pending = startRemote(); + const rejected = expect(pending).rejects.toMatchObject({ + name: "ExecError", + }); + await flush(); + const client = ssh.instances[0]; + client?.emitReady(); + expect(client?.execCommand).toBe("nvidia-smi"); + await vi.advanceTimersByTimeAsync(1000); + await rejected; + expect(client?.stream.close).toHaveBeenCalledTimes(1); + expect(client?.end).toHaveBeenCalledTimes(1); + client?.stream.emitClose(0); + expect(client?.end).toHaveBeenCalledTimes(1); + }); + + it("clears the timeout timer after a successful remote command", async () => { + const pending = startRemote(undefined, 5000); + await flush(); + expect(vi.getTimerCount()).toBeGreaterThanOrEqual(1); + const client = ssh.instances[0]; + client?.emitReady(); + const timersBeforeClose = vi.getTimerCount(); + client?.stream.emitClose(0); + await expect(pending).resolves.toEqual({ stdout: "", stderr: "" }); + expect(vi.getTimerCount()).toBeLessThan(timersBeforeClose); + }); + + it("resolves on success and does not reject later when the timer would have fired", async () => { + const rejections: unknown[] = []; + const onUnhandled = (reason: unknown) => { + rejections.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + + const pending = startRemote(undefined, 5000); + await flush(); + const client = ssh.instances[0]; + client?.emitReady(); + client?.stream.emitClose(0); + await expect(pending).resolves.toEqual({ stdout: "", stderr: "" }); + await vi.advanceTimersByTimeAsync(10_000); + process.off("unhandledRejection", onUnhandled); + expect(rejections).toEqual([]); + expect(client?.ended).toBe(true); + }); + + it("settles only once when close and timeout race", async () => { + const pending = startRemote(); + await flush(); + const client = ssh.instances[0]; + client?.emitReady(); + client?.stream.emitClose(0); + await vi.advanceTimersByTimeAsync(1000); + await expect(pending).resolves.toEqual({ stdout: "", stderr: "" }); + }); + + it("rejects SSH errors before timeout and ignores the later timer", async () => { + const rejections: unknown[] = []; + const onUnhandled = (reason: unknown) => { + rejections.push(reason); + }; + process.on("unhandledRejection", onUnhandled); + + const pending = startRemote(); + const rejected = expect(pending).rejects.toSatisfy( + (error: unknown) => + error instanceof ExecError && + error.message.includes("SSH connection error"), + ); + await flush(); + const client = ssh.instances[0]; + client?.emitError( + Object.assign(new Error("offline"), { level: "client-socket" }), + ); + await rejected; + await vi.advanceTimersByTimeAsync(1000); + process.off("unhandledRejection", onUnhandled); + expect(rejections).toEqual([]); + expect(client?.end).toHaveBeenCalled(); + }); + + it("rejects exec callback errors before timeout", async () => { + const pending = startRemote(); + const rejected = expect(pending).rejects.toSatisfy( + (error: unknown) => + error instanceof ExecError && + error.message.includes("Remote command execution failed"), + ); + await flush(); + const client = ssh.instances[0]; + if (client) client.execShouldFail = new Error("cannot exec"); + client?.emitReady(); + await rejected; + await vi.advanceTimersByTimeAsync(1000); + expect(client?.end).toHaveBeenCalled(); + }); + + it("still delivers onData chunks if the command hangs afterward", async () => { + const chunks: string[] = []; + const pending = startRemote((data) => { + chunks.push(data); + }); + const rejected = expect(pending).rejects.toMatchObject({ + name: "ExecError", + }); + await flush(); + const client = ssh.instances[0]; + client?.emitReady(); + client?.stream.emitData("partial"); + await vi.advanceTimersByTimeAsync(1000); + await rejected; + expect(chunks).toEqual(["partial"]); + expect(client?.stream.close).toHaveBeenCalled(); + expect(client?.end).toHaveBeenCalled(); + }); +}); diff --git a/apps/dokploy/__test__/server/server-hardware.test.ts b/apps/dokploy/__test__/server/server-hardware.test.ts index 3f14c858d..094147fd7 100644 --- a/apps/dokploy/__test__/server/server-hardware.test.ts +++ b/apps/dokploy/__test__/server/server-hardware.test.ts @@ -6,6 +6,7 @@ import { buildLocalHardwareScript, buildRemoteHardwareScript, getServerHardware, + HARDWARE_PROBE_TIMEOUT_MS, parseNvidiaGpuLine, parseServerHardware, } from "@dokploy/server/services/server-hardware"; @@ -799,6 +800,7 @@ describe("getServerHardware", () => { const result = await getServerHardware(); expect(execAsync).toHaveBeenCalledWith( expect.stringContaining("docker info --format"), + expect.objectContaining({ timeout: HARDWARE_PROBE_TIMEOUT_MS }), ); expect(execAsyncRemote).not.toHaveBeenCalled(); expect(result.cpu.count).toBe(8); @@ -820,6 +822,8 @@ describe("getServerHardware", () => { expect(execAsyncRemote).toHaveBeenCalledWith( "remote-server", expect.stringContaining("/proc/meminfo"), + undefined, + expect.objectContaining({ timeout: HARDWARE_PROBE_TIMEOUT_MS }), ); expect(execAsync).not.toHaveBeenCalled(); expect(result.gpu.devices[0]?.uuid).toBe("GPU-aaa"); @@ -844,4 +848,16 @@ describe("getServerHardware", () => { expect(result.gpu.detection).toBe("unavailable"); expect(execAsync).not.toHaveBeenCalled(); }); + + it("treats a timed-out probe as empty hardware, not zero GPUs", async () => { + vi.mocked(execAsync).mockRejectedValue( + new Error("Command execution timed out after 30000ms"), + ); + const result = await getServerHardware(); + expect(result.error).toContain("timed out"); + expect(result.cpu.count).toBeNull(); + expect(result.memory.totalBytes).toBeNull(); + expect(result.disk.totalBytes).toBeNull(); + expect(result.gpu).toEqual({ detection: "unavailable", devices: [] }); + }); }); diff --git a/packages/server/src/services/server-hardware.ts b/packages/server/src/services/server-hardware.ts index e7c05ce4a..a77a1e568 100644 --- a/packages/server/src/services/server-hardware.ts +++ b/packages/server/src/services/server-hardware.ts @@ -5,6 +5,10 @@ import { import { TRPCError } from "@trpc/server"; import { IS_CLOUD } from "../constants"; +// Matches remoteStream SSH readyTimeout. Hardware commands are fast; this +// bounds stalled docker/nvidia-smi/df/SSH so the API cannot wait forever. +export const HARDWARE_PROBE_TIMEOUT_MS = 30_000; + const NVIDIA_INVENTORY_QUERY = "nvidia-smi --query-gpu=index,uuid,name,memory.total,memory.free,driver_version --format=csv,noheader,nounits"; const NVIDIA_COMPUTE_CAP_QUERY = @@ -365,8 +369,10 @@ export const getServerHardware = async ( : buildLocalHardwareScript(); try { const result = serverId - ? await execAsyncRemote(serverId, script) - : await execAsync(script); + ? await execAsyncRemote(serverId, script, undefined, { + timeout: HARDWARE_PROBE_TIMEOUT_MS, + }) + : await execAsync(script, { timeout: HARDWARE_PROBE_TIMEOUT_MS }); return parseServerHardware(result.stdout); } catch (error) { return emptyHardware(error); diff --git a/packages/server/src/utils/process/execAsync.ts b/packages/server/src/utils/process/execAsync.ts index 37b0b7f91..9696d2aa6 100644 --- a/packages/server/src/utils/process/execAsync.ts +++ b/packages/server/src/utils/process/execAsync.ts @@ -1,4 +1,4 @@ -import { exec, execFile } from "node:child_process"; +import { type ChildProcess, exec, execFile, spawn } from "node:child_process"; import util from "node:util"; import { findServerById } from "@dokploy/server/services/server"; import { Client } from "ssh2"; @@ -23,12 +23,119 @@ export { ExecError } from "./ExecError"; const execAsyncBase = util.promisify(exec); +export interface ExecAsyncOptions { + cwd?: string; + env?: NodeJS.ProcessEnv; + shell?: string; + timeout?: number; +} + +export interface ExecAsyncRemoteOptions { + timeout?: number; +} + +const killProcessGroup = (child: ChildProcess) => { + if (child.pid == null) return; + if (process.platform === "win32") { + child.kill("SIGKILL"); + return; + } + try { + process.kill(-child.pid, "SIGKILL"); + } catch { + try { + child.kill("SIGKILL"); + } catch { + // already exited + } + } +}; + +const execAsyncTimed = ( + command: string, + options: ExecAsyncOptions & { timeout: number }, +): Promise<{ stdout: string; stderr: string }> => { + const { timeout, cwd, env, shell } = options; + const isWin = process.platform === "win32"; + const shellPath = + shell ?? (isWin ? process.env.ComSpec || "cmd.exe" : "/bin/sh"); + const args = isWin ? ["/d", "/s", "/c", command] : ["-c", command]; + + return new Promise((resolve, reject) => { + let stdout = ""; + let stderr = ""; + let settled = false; + const child = spawn(shellPath, args, { + cwd, + env, + detached: !isWin, + stdio: ["ignore", "pipe", "pipe"], + }); + + const timer = setTimeout(() => { + killProcessGroup(child); + finish( + new ExecError(`Command execution timed out after ${timeout}ms`, { + command, + stdout, + stderr, + }), + ); + }, timeout); + + const finish = (error?: ExecError) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); + else resolve({ stdout, stderr }); + }; + + child.stdout?.on("data", (data) => { + stdout += data.toString(); + }); + child.stderr?.on("data", (data) => { + stderr += data.toString(); + }); + child.on("error", (error) => { + finish( + new ExecError(`Command execution failed: ${error.message}`, { + command, + stdout, + stderr, + originalError: error, + }), + ); + }); + child.on("close", (code) => { + if (code === 0) { + finish(); + return; + } + finish( + new ExecError(`Command execution failed: Command failed: ${command}`, { + command, + stdout, + stderr, + exitCode: code ?? undefined, + }), + ); + }); + }); +}; + export const execAsync = async ( command: string, - options?: { cwd?: string; env?: NodeJS.ProcessEnv; shell?: string }, + options?: ExecAsyncOptions, ): Promise<{ stdout: string; stderr: string }> => { + const timeout = options?.timeout; + if (timeout != null && timeout > 0) { + return execAsyncTimed(command, { ...options, timeout }); + } + + const { timeout: _timeout, ...execOptions } = options ?? {}; try { - const result = await execAsyncBase(command, options); + const result = await execAsyncBase(command, execOptions); return { stdout: result.stdout.toString(), stderr: result.stderr.toString(), @@ -156,6 +263,7 @@ export const execAsyncRemote = async ( serverId: string | null, command: string, onData?: (data: string) => void, + options?: ExecAsyncRemoteOptions, ): Promise<{ stdout: string; stderr: string }> => { if (!serverId) return { stdout: "", stderr: "" }; const server = await findServerById(serverId); @@ -163,16 +271,84 @@ export const execAsyncRemote = async ( let stdout = ""; let stderr = ""; + const timeoutMs = options?.timeout; return new Promise((resolve, reject) => { const conn = new Client(); + let stream: { close: () => void; destroy: () => void } | undefined; + let settled = false; + let timer: ReturnType | undefined; + + const settle = ( + error?: ExecError, + value?: { stdout: string; stderr: string }, + ) => { + if (settled) return; + settled = true; + if (timer !== undefined) { + clearTimeout(timer); + timer = undefined; + } + if (error) { + try { + stream?.close(); + } catch { + // already closed + } + try { + stream?.destroy(); + } catch { + // already destroyed + } + try { + conn.end(); + } catch { + // already ended + } + try { + conn.destroy(); + } catch { + // already destroyed + } + reject(error); + return; + } + try { + conn.end(); + } catch { + // already ended + } + resolve(value ?? { stdout, stderr }); + }; + + if (timeoutMs != null && timeoutMs > 0) { + timer = setTimeout(() => { + settle( + new ExecError(`Command execution timed out after ${timeoutMs}ms`, { + command, + stdout, + stderr, + serverId, + }), + ); + }, timeoutMs); + } sleep(1000); conn .once("ready", () => { - conn.exec(command, (err, stream) => { + if (settled) return; + conn.exec(command, (err, commandStream) => { + if (settled) { + try { + commandStream?.close(); + } catch { + // already closed + } + return; + } if (err) { onData?.(err.message); - reject( + settle( new ExecError(`Remote command execution failed: ${err.message}`, { command, serverId, @@ -181,13 +357,13 @@ export const execAsyncRemote = async ( ); return; } - stream + stream = commandStream; + commandStream .on("close", (code: number, _signal: string) => { - conn.end(); if (code === 0) { - resolve({ stdout, stderr }); + settle(undefined, { stdout, stderr }); } else { - reject( + settle( new ExecError( `Remote command failed with exit code ${code}`, { @@ -212,7 +388,7 @@ export const execAsyncRemote = async ( }); }) .on("error", (err) => { - conn.end(); + if (settled) return; if (err.level === "client-authentication") { const technicalDetail = `Error: ${err.message} ${err.level}`; const friendlyMessage = [ @@ -228,9 +404,8 @@ export const execAsyncRemote = async ( " • Try generating a new SSH key in Dokploy and add only the public key to the server, then try again.", " • Make sure to follow the instructions on the Setup Server Button on the SSH Keys tab and then click on deployments tab and check the logs for more details.", ].join("\n"); - const errorMsg = `Authentication failed: Invalid SSH private key. ❌ Error: ${err.message} ${err.level}`; onData?.(friendlyMessage); - reject( + settle( new ExecError( `Authentication failed: Invalid SSH private key. ${friendlyMessage}`, { @@ -243,7 +418,7 @@ export const execAsyncRemote = async ( } else { const errorMsg = `SSH connection error: ${err.message}`; onData?.(errorMsg); - reject( + settle( new ExecError(errorMsg, { command, serverId, @@ -257,7 +432,10 @@ export const execAsyncRemote = async ( port: server.port, username: server.username, privateKey: server.sshKey?.privateKey, - timeout: 99999, + timeout: timeoutMs != null && timeoutMs > 0 ? timeoutMs : 99999, + ...(timeoutMs != null && timeoutMs > 0 + ? { readyTimeout: timeoutMs } + : {}), }); }); };