diff --git a/apps/dokploy/__test__/server/server-hardware.test.ts b/apps/dokploy/__test__/server/server-hardware.test.ts index 094147fd7..0052ac76a 100644 --- a/apps/dokploy/__test__/server/server-hardware.test.ts +++ b/apps/dokploy/__test__/server/server-hardware.test.ts @@ -3,19 +3,21 @@ import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { - buildLocalHardwareScript, - buildRemoteHardwareScript, getServerHardware, HARDWARE_PROBE_TIMEOUT_MS, parseNvidiaGpuLine, parseServerHardware, } from "@dokploy/server/services/server-hardware"; +import { buildHardwareScripts } from "@dokploy/server/services/server-hardware-scripts"; import { execAsync, execAsyncRemote, } from "@dokploy/server/utils/process/execAsync"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +const buildLocalHardwareScript = () => buildHardwareScripts(false).join("\n"); +const buildRemoteHardwareScript = () => buildHardwareScripts(true).join("\n"); + const cloud = { enabled: false }; vi.mock("@dokploy/server/constants", async (importOriginal) => { @@ -123,11 +125,21 @@ afterEach(() => { sandboxes.length = 0; }); -const runScript = (script: string, sandboxPath: string) => - execFileSync(resolveBin("sh"), ["-c", script], { +const runScript = (script: string, sandboxPath: string) => { + const stdout = execFileSync(resolveBin("sh"), ["-c", script], { encoding: "utf8", env: { ...process.env, PATH: sandboxPath }, }); + return JSON.stringify( + Object.assign( + { kind: script.includes("docker info") ? "local" : "remote" }, + ...stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + ), + ); +}; const inventoryQueries = (script: string) => [...script.matchAll(/nvidia-smi --query-gpu=(\S+)/g)].map( @@ -861,3 +873,124 @@ describe("getServerHardware", () => { expect(result.gpu).toEqual({ detection: "unavailable", devices: [] }); }); }); + +describe("hardware timeout isolation regressions", () => { + it.each(["inventory", "compute", "engine"])( + "preserves independent local facts when %s hangs", + async (hung) => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/process/execAsync") + >("@dokploy/server/utils/process/execAsync"); + const sandbox = makeSandbox({ + docker: `#!/bin/sh +${hung === "engine" ? "/bin/sleep 60" : ""} +printf '%s' '{"ncpu":8,"memTotal":16000000000,"arch":"x86_64"}' +`, + "nvidia-smi": `#!/bin/sh +case "$*" in + *compute_cap*) ${hung === "compute" ? "/bin/sleep 60" : ""} +printf '%s' '0, 7.5';; + *) ${hung === "inventory" ? "/bin/sleep 60" : ""} +printf '%s' '${t4}';; +esac +`, + }); + vi.mocked(execAsync).mockImplementation((command, options) => { + expect(options?.timeout).toBeGreaterThan(0); + expect(options?.timeout).toBeLessThanOrEqual(30_000); + return actual.execAsync(command, { + ...options, + timeout: 3_000, + env: { NODE_ENV: "test", PATH: sandbox }, + }); + }); + const result = await getServerHardware(); + expect(result.error).toContain("timed out"); + expect(result.cpu.count).toBe(hung === "engine" ? null : 8); + expect(result.memory.totalBytes).toBe( + hung === "engine" ? null : 16_000_000_000, + ); + expect(result.gpu.devices).toHaveLength(hung === "inventory" ? 0 : 1); + if (hung === "compute") + expect(result.gpu.devices[0]?.computeCapability).toBeNull(); + }, + ); + it.each(["disk", "inventory", "compute", "cpu", "memory", "arch"])( + "preserves remote sibling facts when %s hangs", + async (hung) => { + const actual = await vi.importActual< + typeof import("@dokploy/server/utils/process/execAsync") + >("@dokploy/server/utils/process/execAsync"); + const sandbox = makeSandbox({ + awk: `#!/bin/sh +case "$*" in +*MemTotal*) ${hung === "memory" ? "/bin/sleep 60" : ""}; printf '4096';; +*MemAvailable*) printf '1024';; +*) exec ${resolveBin("awk")} "$@";; +esac +`.replace("; printf", "\nprintf"), + nproc: `#!/bin/sh +${hung === "cpu" ? "/bin/sleep 60" : ""} +printf '4' +`, + uname: `#!/bin/sh +${hung === "arch" ? "/bin/sleep 60" : ""} +printf 'aarch64' +`, + df: `#!/bin/sh +${hung === "disk" ? "/bin/sleep 60" : ""} +printf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on' '/dev/test 1000 750 250 75% /' +`, + "nvidia-smi": `#!/bin/sh +case "$*" in +*compute_cap*) ${hung === "compute" ? "/bin/sleep 60" : ""} +printf '0, 7.5';; +*) ${hung === "inventory" ? "/bin/sleep 60" : ""} +printf '%s' '${t4}';; +esac +`, + }); + vi.mocked(execAsyncRemote).mockImplementation( + (_id, command, _onData, options) => { + expect(options?.timeout).toBeGreaterThan(0); + expect(options?.timeout).toBeLessThanOrEqual(30_000); + return actual.execAsync(command, { + timeout: 3_000, + env: { NODE_ENV: "test", PATH: sandbox }, + }); + }, + ); + const result = await getServerHardware("fixture-server"); + expect(result.error).toContain("timed out"); + expect(result.cpu).toEqual({ + count: hung === "cpu" ? null : 4, + arch: hung === "arch" ? null : "aarch64", + }); + expect(result.memory.totalBytes).toBe( + hung === "memory" ? null : 4096 * 1024, + ); + expect(result.disk.totalBytes).toBe(hung === "disk" ? null : 1000 * 1024); + expect(result.gpu.devices).toHaveLength(hung === "inventory" ? 0 : 1); + if (hung === "compute") + expect(result.gpu.devices[0]?.computeCapability).toBeNull(); + }, + ); + it("does not report malformed nonempty inventory as successfully empty", () => { + expect( + parseServerHardware(localEnvelope({ gpuExit: 0, gpuCsv: "malformed" })) + .gpu.detection, + ).toBe("unavailable"); + }); + it("leaves unsupported compute capability unknown", () => { + expect( + parseServerHardware( + localEnvelope({ + gpuExit: 0, + gpuCsv: t4, + capExit: 0, + capCsv: "0, [N/A]", + }), + ).gpu.devices[0]?.computeCapability, + ).toBeNull(); + }); +}); diff --git a/packages/server/src/services/server-hardware-scripts.ts b/packages/server/src/services/server-hardware-scripts.ts new file mode 100644 index 000000000..e9c511374 --- /dev/null +++ b/packages/server/src/services/server-hardware-scripts.ts @@ -0,0 +1,56 @@ +const NVIDIA_INVENTORY_QUERY = + "nvidia-smi --query-gpu=index,uuid,name,memory.total,memory.free,driver_version --format=csv,noheader,nounits"; +const NVIDIA_COMPUTE_CAP_QUERY = + "nvidia-smi --query-gpu=index,compute_cap --format=csv,noheader,nounits"; + +// Each invocation emits one independent result and receives its own executor deadline. +const engine = ` +engineOutput=$(docker info --format '{"ncpu":{{json .NCPU}},"memTotal":{{json .MemTotal}},"arch":{{json .Architecture}}}' 2>/dev/null) +engineExit=$? +engineB64=$(printf '%s' "$engineOutput" | base64 2>/dev/null | tr -d '\\n') +printf '{"engineExit":%s,"engineBase64":"%s"}\\n' "$engineExit" "$engineB64" +`; + +const gpu = ` +gpuCsv=$(${NVIDIA_INVENTORY_QUERY} 2>/dev/null) +gpuExit=$? +gpuB64=$(printf '%s' "$gpuCsv" | base64 2>/dev/null | tr -d '\\n') +printf '{"gpuExit":%s,"gpuBase64":"%s"}\\n' "$gpuExit" "$gpuB64" +`; + +const cap = ` +capCsv=$(${NVIDIA_COMPUTE_CAP_QUERY} 2>/dev/null) +capExit=$? +capB64=$(printf '%s' "$capCsv" | base64 2>/dev/null | tr -d '\\n') +printf '{"capExit":%s,"capBase64":"%s"}\\n' "$capExit" "$capB64" +`; + +const memory = ` +memTotalKb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null) +memAvailKb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null) +printf '{"memTotalKb":"%s","memAvailKb":"%s"}\\n' "$memTotalKb" "$memAvailKb" +`; + +const cpu = ` +cpuCount=$(nproc 2>/dev/null) +if [ -z "$cpuCount" ]; then + cpuCount=$(grep -c '^processor' /proc/cpuinfo 2>/dev/null) +fi +printf '{"cpuCount":"%s"}\\n' "$cpuCount" +`; + +const arch = ` +arch=$(uname -m 2>/dev/null) +printf '{"arch":"%s"}\\n' "$arch" +`; + +const disk = ` +diskOutput=$(df -Pk / 2>/dev/null) +diskExit=$? +diskTotalK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $2}') +diskAvailK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $4}') +printf '{"diskExit":%s,"diskTotalK":"%s","diskAvailK":"%s"}\\n' "$diskExit" "$diskTotalK" "$diskAvailK" +`; + +export const buildHardwareScripts = (remote: boolean): readonly string[] => + remote ? [memory, cpu, arch, disk, gpu, cap] : [engine, gpu, cap]; diff --git a/packages/server/src/services/server-hardware.ts b/packages/server/src/services/server-hardware.ts index a77a1e568..6a6e6b0f7 100644 --- a/packages/server/src/services/server-hardware.ts +++ b/packages/server/src/services/server-hardware.ts @@ -4,16 +4,12 @@ import { } from "@dokploy/server/utils/process/execAsync"; import { TRPCError } from "@trpc/server"; import { IS_CLOUD } from "../constants"; +import { buildHardwareScripts } from "./server-hardware-scripts"; // Matches remoteStream SSH readyTimeout. Hardware commands are fast; this // bounds stalled docker/nvidia-smi/df/SSH so the API cannot wait forever. export const HARDWARE_PROBE_TIMEOUT_MS = 30_000; -const NVIDIA_INVENTORY_QUERY = - "nvidia-smi --query-gpu=index,uuid,name,memory.total,memory.free,driver_version --format=csv,noheader,nounits"; -const NVIDIA_COMPUTE_CAP_QUERY = - "nvidia-smi --query-gpu=index,compute_cap --format=csv,noheader,nounits"; - export interface ServerHardwareGpu { index: number; uuid: string | null; @@ -92,8 +88,8 @@ const emptyHardware = (error?: unknown): ServerHardware => ({ : {}), }); -const nullIfEmpty = (value: string | null | undefined): string | null => { - const trimmed = value?.trim(); +const nullIfEmpty = (value: unknown): string | null => { + const trimmed = typeof value === "string" ? value.trim() : ""; return trimmed ? trimmed : null; }; @@ -213,7 +209,12 @@ export const parseComputeCapLine = ( if (parts.length < 2) return null; const index = parseNonNegativeInt(parts[0]); const computeCapability = nullIfEmpty(parts.slice(1).join(",")); - if (index == null || !computeCapability) return null; + if ( + index == null || + !computeCapability || + !/^\d+\.\d+$/.test(computeCapability) + ) + return null; return { index, computeCapability }; }; @@ -243,40 +244,6 @@ export const mergeComputeCapabilities = ( })); }; -export const buildLocalHardwareScript = () => ` -engineOutput=$(docker info --format '{"ncpu":{{json .NCPU}},"memTotal":{{json .MemTotal}},"arch":{{json .Architecture}}}' 2>/dev/null) -engineExit=$? -engineB64=$(printf '%s' "$engineOutput" | base64 2>/dev/null | tr -d '\\n') -gpuCsv=$(${NVIDIA_INVENTORY_QUERY} 2>/dev/null) -gpuExit=$? -gpuB64=$(printf '%s' "$gpuCsv" | base64 2>/dev/null | tr -d '\\n') -capCsv=$(${NVIDIA_COMPUTE_CAP_QUERY} 2>/dev/null) -capExit=$? -capB64=$(printf '%s' "$capCsv" | base64 2>/dev/null | tr -d '\\n') -printf '{"kind":"local","engineExit":%s,"engineBase64":"%s","gpuExit":%s,"gpuBase64":"%s","capExit":%s,"capBase64":"%s"}' "$engineExit" "$engineB64" "$gpuExit" "$gpuB64" "$capExit" "$capB64" -`; - -export const buildRemoteHardwareScript = () => ` -memTotalKb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null) -memAvailKb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null) -cpuCount=$(nproc 2>/dev/null) -if [ -z "$cpuCount" ]; then - cpuCount=$(grep -c '^processor' /proc/cpuinfo 2>/dev/null) -fi -arch=$(uname -m 2>/dev/null) -diskOutput=$(df -Pk / 2>/dev/null) -diskExit=$? -diskTotalK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $2}') -diskAvailK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $4}') -gpuCsv=$(${NVIDIA_INVENTORY_QUERY} 2>/dev/null) -gpuExit=$? -gpuB64=$(printf '%s' "$gpuCsv" | base64 2>/dev/null | tr -d '\\n') -capCsv=$(${NVIDIA_COMPUTE_CAP_QUERY} 2>/dev/null) -capExit=$? -capB64=$(printf '%s' "$capCsv" | base64 2>/dev/null | tr -d '\\n') -printf '{"kind":"remote","memTotalKb":"%s","memAvailKb":"%s","cpuCount":"%s","arch":"%s","diskTotalK":"%s","diskAvailK":"%s","diskExit":%s,"gpuExit":%s,"gpuBase64":"%s","capExit":%s,"capBase64":"%s"}' "$memTotalKb" "$memAvailKb" "$cpuCount" "$arch" "$diskTotalK" "$diskAvailK" "$diskExit" "$gpuExit" "$gpuB64" "$capExit" "$capB64" -`; - const gpuFromProbe = ( gpuExit: number, gpuCsv: string, @@ -288,7 +255,8 @@ const gpuFromProbe = ( } const devices = parseNvidiaGpuCsv(gpuCsv); return { - detection: "available", + detection: + gpuCsv.trim() && devices.length === 0 ? "unavailable" : "available", devices: capExit === 0 ? mergeComputeCapabilities(devices, capCsv) : devices, }; @@ -364,17 +332,35 @@ export const getServerHardware = async ( }); } - const script = serverId - ? buildRemoteHardwareScript() - : buildLocalHardwareScript(); - try { - const result = serverId - ? await execAsyncRemote(serverId, script, undefined, { - timeout: HARDWARE_PROBE_TIMEOUT_MS, - }) - : await execAsync(script, { timeout: HARDWARE_PROBE_TIMEOUT_MS }); - return parseServerHardware(result.stdout); - } catch (error) { - return emptyHardware(error); + const results = await Promise.allSettled( + buildHardwareScripts(Boolean(serverId)).map(async (script) => { + const result = serverId + ? await execAsyncRemote(serverId, script, undefined, { + timeout: HARDWARE_PROBE_TIMEOUT_MS, + }) + : await execAsync(script, { timeout: HARDWARE_PROBE_TIMEOUT_MS }); + const parsed: unknown = JSON.parse(result.stdout); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("Could not parse server hardware output"); + } + return parsed; + }), + ); + const fields = {}; + const errors: string[] = []; + for (const result of results) { + if (result.status === "fulfilled") Object.assign(fields, result.value); + else + errors.push( + result.reason instanceof Error + ? result.reason.message + : "Could not read server hardware", + ); } + return { + ...parseServerHardware( + JSON.stringify({ ...fields, kind: serverId ? "remote" : "local" }), + ), + ...(errors.length ? { error: [...new Set(errors)].join("; ") } : {}), + }; };