mirror of
https://github.com/Dokploy/dokploy.git
synced 2026-09-12 19:51:00 +05:00
fix(docker): preserve independent hardware probe results
This commit is contained in:
EgerDev
parent
c126ce99be
commit
a98a68f19e
@ -3,19 +3,21 @@ import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
buildLocalHardwareScript,
|
||||
buildRemoteHardwareScript,
|
||||
getServerHardware,
|
||||
HARDWARE_PROBE_TIMEOUT_MS,
|
||||
parseNvidiaGpuLine,
|
||||
parseServerHardware,
|
||||
} from "@dokploy/server/services/server-hardware";
|
||||
import { buildHardwareScripts } from "@dokploy/server/services/server-hardware-scripts";
|
||||
import {
|
||||
execAsync,
|
||||
execAsyncRemote,
|
||||
} from "@dokploy/server/utils/process/execAsync";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const buildLocalHardwareScript = () => buildHardwareScripts(false).join("\n");
|
||||
const buildRemoteHardwareScript = () => buildHardwareScripts(true).join("\n");
|
||||
|
||||
const cloud = { enabled: false };
|
||||
|
||||
vi.mock("@dokploy/server/constants", async (importOriginal) => {
|
||||
@ -123,11 +125,21 @@ afterEach(() => {
|
||||
sandboxes.length = 0;
|
||||
});
|
||||
|
||||
const runScript = (script: string, sandboxPath: string) =>
|
||||
execFileSync(resolveBin("sh"), ["-c", script], {
|
||||
const runScript = (script: string, sandboxPath: string) => {
|
||||
const stdout = execFileSync(resolveBin("sh"), ["-c", script], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, PATH: sandboxPath },
|
||||
});
|
||||
return JSON.stringify(
|
||||
Object.assign(
|
||||
{ kind: script.includes("docker info") ? "local" : "remote" },
|
||||
...stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line)),
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
const inventoryQueries = (script: string) =>
|
||||
[...script.matchAll(/nvidia-smi --query-gpu=(\S+)/g)].map(
|
||||
@ -861,3 +873,124 @@ describe("getServerHardware", () => {
|
||||
expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("hardware timeout isolation regressions", () => {
|
||||
it.each(["inventory", "compute", "engine"])(
|
||||
"preserves independent local facts when %s hangs",
|
||||
async (hung) => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("@dokploy/server/utils/process/execAsync")
|
||||
>("@dokploy/server/utils/process/execAsync");
|
||||
const sandbox = makeSandbox({
|
||||
docker: `#!/bin/sh
|
||||
${hung === "engine" ? "/bin/sleep 60" : ""}
|
||||
printf '%s' '{"ncpu":8,"memTotal":16000000000,"arch":"x86_64"}'
|
||||
`,
|
||||
"nvidia-smi": `#!/bin/sh
|
||||
case "$*" in
|
||||
*compute_cap*) ${hung === "compute" ? "/bin/sleep 60" : ""}
|
||||
printf '%s' '0, 7.5';;
|
||||
*) ${hung === "inventory" ? "/bin/sleep 60" : ""}
|
||||
printf '%s' '${t4}';;
|
||||
esac
|
||||
`,
|
||||
});
|
||||
vi.mocked(execAsync).mockImplementation((command, options) => {
|
||||
expect(options?.timeout).toBeGreaterThan(0);
|
||||
expect(options?.timeout).toBeLessThanOrEqual(30_000);
|
||||
return actual.execAsync(command, {
|
||||
...options,
|
||||
timeout: 3_000,
|
||||
env: { NODE_ENV: "test", PATH: sandbox },
|
||||
});
|
||||
});
|
||||
const result = await getServerHardware();
|
||||
expect(result.error).toContain("timed out");
|
||||
expect(result.cpu.count).toBe(hung === "engine" ? null : 8);
|
||||
expect(result.memory.totalBytes).toBe(
|
||||
hung === "engine" ? null : 16_000_000_000,
|
||||
);
|
||||
expect(result.gpu.devices).toHaveLength(hung === "inventory" ? 0 : 1);
|
||||
if (hung === "compute")
|
||||
expect(result.gpu.devices[0]?.computeCapability).toBeNull();
|
||||
},
|
||||
);
|
||||
it.each(["disk", "inventory", "compute", "cpu", "memory", "arch"])(
|
||||
"preserves remote sibling facts when %s hangs",
|
||||
async (hung) => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("@dokploy/server/utils/process/execAsync")
|
||||
>("@dokploy/server/utils/process/execAsync");
|
||||
const sandbox = makeSandbox({
|
||||
awk: `#!/bin/sh
|
||||
case "$*" in
|
||||
*MemTotal*) ${hung === "memory" ? "/bin/sleep 60" : ""}; printf '4096';;
|
||||
*MemAvailable*) printf '1024';;
|
||||
*) exec ${resolveBin("awk")} "$@";;
|
||||
esac
|
||||
`.replace("; printf", "\nprintf"),
|
||||
nproc: `#!/bin/sh
|
||||
${hung === "cpu" ? "/bin/sleep 60" : ""}
|
||||
printf '4'
|
||||
`,
|
||||
uname: `#!/bin/sh
|
||||
${hung === "arch" ? "/bin/sleep 60" : ""}
|
||||
printf 'aarch64'
|
||||
`,
|
||||
df: `#!/bin/sh
|
||||
${hung === "disk" ? "/bin/sleep 60" : ""}
|
||||
printf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on' '/dev/test 1000 750 250 75% /'
|
||||
`,
|
||||
"nvidia-smi": `#!/bin/sh
|
||||
case "$*" in
|
||||
*compute_cap*) ${hung === "compute" ? "/bin/sleep 60" : ""}
|
||||
printf '0, 7.5';;
|
||||
*) ${hung === "inventory" ? "/bin/sleep 60" : ""}
|
||||
printf '%s' '${t4}';;
|
||||
esac
|
||||
`,
|
||||
});
|
||||
vi.mocked(execAsyncRemote).mockImplementation(
|
||||
(_id, command, _onData, options) => {
|
||||
expect(options?.timeout).toBeGreaterThan(0);
|
||||
expect(options?.timeout).toBeLessThanOrEqual(30_000);
|
||||
return actual.execAsync(command, {
|
||||
timeout: 3_000,
|
||||
env: { NODE_ENV: "test", PATH: sandbox },
|
||||
});
|
||||
},
|
||||
);
|
||||
const result = await getServerHardware("fixture-server");
|
||||
expect(result.error).toContain("timed out");
|
||||
expect(result.cpu).toEqual({
|
||||
count: hung === "cpu" ? null : 4,
|
||||
arch: hung === "arch" ? null : "aarch64",
|
||||
});
|
||||
expect(result.memory.totalBytes).toBe(
|
||||
hung === "memory" ? null : 4096 * 1024,
|
||||
);
|
||||
expect(result.disk.totalBytes).toBe(hung === "disk" ? null : 1000 * 1024);
|
||||
expect(result.gpu.devices).toHaveLength(hung === "inventory" ? 0 : 1);
|
||||
if (hung === "compute")
|
||||
expect(result.gpu.devices[0]?.computeCapability).toBeNull();
|
||||
},
|
||||
);
|
||||
it("does not report malformed nonempty inventory as successfully empty", () => {
|
||||
expect(
|
||||
parseServerHardware(localEnvelope({ gpuExit: 0, gpuCsv: "malformed" }))
|
||||
.gpu.detection,
|
||||
).toBe("unavailable");
|
||||
});
|
||||
it("leaves unsupported compute capability unknown", () => {
|
||||
expect(
|
||||
parseServerHardware(
|
||||
localEnvelope({
|
||||
gpuExit: 0,
|
||||
gpuCsv: t4,
|
||||
capExit: 0,
|
||||
capCsv: "0, [N/A]",
|
||||
}),
|
||||
).gpu.devices[0]?.computeCapability,
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
56
packages/server/src/services/server-hardware-scripts.ts
Normal file
56
packages/server/src/services/server-hardware-scripts.ts
Normal file
@ -0,0 +1,56 @@
|
||||
const NVIDIA_INVENTORY_QUERY =
|
||||
"nvidia-smi --query-gpu=index,uuid,name,memory.total,memory.free,driver_version --format=csv,noheader,nounits";
|
||||
const NVIDIA_COMPUTE_CAP_QUERY =
|
||||
"nvidia-smi --query-gpu=index,compute_cap --format=csv,noheader,nounits";
|
||||
|
||||
// Each invocation emits one independent result and receives its own executor deadline.
|
||||
const engine = `
|
||||
engineOutput=$(docker info --format '{"ncpu":{{json .NCPU}},"memTotal":{{json .MemTotal}},"arch":{{json .Architecture}}}' 2>/dev/null)
|
||||
engineExit=$?
|
||||
engineB64=$(printf '%s' "$engineOutput" | base64 2>/dev/null | tr -d '\\n')
|
||||
printf '{"engineExit":%s,"engineBase64":"%s"}\\n' "$engineExit" "$engineB64"
|
||||
`;
|
||||
|
||||
const gpu = `
|
||||
gpuCsv=$(${NVIDIA_INVENTORY_QUERY} 2>/dev/null)
|
||||
gpuExit=$?
|
||||
gpuB64=$(printf '%s' "$gpuCsv" | base64 2>/dev/null | tr -d '\\n')
|
||||
printf '{"gpuExit":%s,"gpuBase64":"%s"}\\n' "$gpuExit" "$gpuB64"
|
||||
`;
|
||||
|
||||
const cap = `
|
||||
capCsv=$(${NVIDIA_COMPUTE_CAP_QUERY} 2>/dev/null)
|
||||
capExit=$?
|
||||
capB64=$(printf '%s' "$capCsv" | base64 2>/dev/null | tr -d '\\n')
|
||||
printf '{"capExit":%s,"capBase64":"%s"}\\n' "$capExit" "$capB64"
|
||||
`;
|
||||
|
||||
const memory = `
|
||||
memTotalKb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null)
|
||||
memAvailKb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null)
|
||||
printf '{"memTotalKb":"%s","memAvailKb":"%s"}\\n' "$memTotalKb" "$memAvailKb"
|
||||
`;
|
||||
|
||||
const cpu = `
|
||||
cpuCount=$(nproc 2>/dev/null)
|
||||
if [ -z "$cpuCount" ]; then
|
||||
cpuCount=$(grep -c '^processor' /proc/cpuinfo 2>/dev/null)
|
||||
fi
|
||||
printf '{"cpuCount":"%s"}\\n' "$cpuCount"
|
||||
`;
|
||||
|
||||
const arch = `
|
||||
arch=$(uname -m 2>/dev/null)
|
||||
printf '{"arch":"%s"}\\n' "$arch"
|
||||
`;
|
||||
|
||||
const disk = `
|
||||
diskOutput=$(df -Pk / 2>/dev/null)
|
||||
diskExit=$?
|
||||
diskTotalK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $2}')
|
||||
diskAvailK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $4}')
|
||||
printf '{"diskExit":%s,"diskTotalK":"%s","diskAvailK":"%s"}\\n' "$diskExit" "$diskTotalK" "$diskAvailK"
|
||||
`;
|
||||
|
||||
export const buildHardwareScripts = (remote: boolean): readonly string[] =>
|
||||
remote ? [memory, cpu, arch, disk, gpu, cap] : [engine, gpu, cap];
|
||||
@ -4,16 +4,12 @@ import {
|
||||
} from "@dokploy/server/utils/process/execAsync";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { IS_CLOUD } from "../constants";
|
||||
import { buildHardwareScripts } from "./server-hardware-scripts";
|
||||
|
||||
// Matches remoteStream SSH readyTimeout. Hardware commands are fast; this
|
||||
// bounds stalled docker/nvidia-smi/df/SSH so the API cannot wait forever.
|
||||
export const HARDWARE_PROBE_TIMEOUT_MS = 30_000;
|
||||
|
||||
const NVIDIA_INVENTORY_QUERY =
|
||||
"nvidia-smi --query-gpu=index,uuid,name,memory.total,memory.free,driver_version --format=csv,noheader,nounits";
|
||||
const NVIDIA_COMPUTE_CAP_QUERY =
|
||||
"nvidia-smi --query-gpu=index,compute_cap --format=csv,noheader,nounits";
|
||||
|
||||
export interface ServerHardwareGpu {
|
||||
index: number;
|
||||
uuid: string | null;
|
||||
@ -92,8 +88,8 @@ const emptyHardware = (error?: unknown): ServerHardware => ({
|
||||
: {}),
|
||||
});
|
||||
|
||||
const nullIfEmpty = (value: string | null | undefined): string | null => {
|
||||
const trimmed = value?.trim();
|
||||
const nullIfEmpty = (value: unknown): string | null => {
|
||||
const trimmed = typeof value === "string" ? value.trim() : "";
|
||||
return trimmed ? trimmed : null;
|
||||
};
|
||||
|
||||
@ -213,7 +209,12 @@ export const parseComputeCapLine = (
|
||||
if (parts.length < 2) return null;
|
||||
const index = parseNonNegativeInt(parts[0]);
|
||||
const computeCapability = nullIfEmpty(parts.slice(1).join(","));
|
||||
if (index == null || !computeCapability) return null;
|
||||
if (
|
||||
index == null ||
|
||||
!computeCapability ||
|
||||
!/^\d+\.\d+$/.test(computeCapability)
|
||||
)
|
||||
return null;
|
||||
return { index, computeCapability };
|
||||
};
|
||||
|
||||
@ -243,40 +244,6 @@ export const mergeComputeCapabilities = (
|
||||
}));
|
||||
};
|
||||
|
||||
export const buildLocalHardwareScript = () => `
|
||||
engineOutput=$(docker info --format '{"ncpu":{{json .NCPU}},"memTotal":{{json .MemTotal}},"arch":{{json .Architecture}}}' 2>/dev/null)
|
||||
engineExit=$?
|
||||
engineB64=$(printf '%s' "$engineOutput" | base64 2>/dev/null | tr -d '\\n')
|
||||
gpuCsv=$(${NVIDIA_INVENTORY_QUERY} 2>/dev/null)
|
||||
gpuExit=$?
|
||||
gpuB64=$(printf '%s' "$gpuCsv" | base64 2>/dev/null | tr -d '\\n')
|
||||
capCsv=$(${NVIDIA_COMPUTE_CAP_QUERY} 2>/dev/null)
|
||||
capExit=$?
|
||||
capB64=$(printf '%s' "$capCsv" | base64 2>/dev/null | tr -d '\\n')
|
||||
printf '{"kind":"local","engineExit":%s,"engineBase64":"%s","gpuExit":%s,"gpuBase64":"%s","capExit":%s,"capBase64":"%s"}' "$engineExit" "$engineB64" "$gpuExit" "$gpuB64" "$capExit" "$capB64"
|
||||
`;
|
||||
|
||||
export const buildRemoteHardwareScript = () => `
|
||||
memTotalKb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null)
|
||||
memAvailKb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null)
|
||||
cpuCount=$(nproc 2>/dev/null)
|
||||
if [ -z "$cpuCount" ]; then
|
||||
cpuCount=$(grep -c '^processor' /proc/cpuinfo 2>/dev/null)
|
||||
fi
|
||||
arch=$(uname -m 2>/dev/null)
|
||||
diskOutput=$(df -Pk / 2>/dev/null)
|
||||
diskExit=$?
|
||||
diskTotalK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $2}')
|
||||
diskAvailK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $4}')
|
||||
gpuCsv=$(${NVIDIA_INVENTORY_QUERY} 2>/dev/null)
|
||||
gpuExit=$?
|
||||
gpuB64=$(printf '%s' "$gpuCsv" | base64 2>/dev/null | tr -d '\\n')
|
||||
capCsv=$(${NVIDIA_COMPUTE_CAP_QUERY} 2>/dev/null)
|
||||
capExit=$?
|
||||
capB64=$(printf '%s' "$capCsv" | base64 2>/dev/null | tr -d '\\n')
|
||||
printf '{"kind":"remote","memTotalKb":"%s","memAvailKb":"%s","cpuCount":"%s","arch":"%s","diskTotalK":"%s","diskAvailK":"%s","diskExit":%s,"gpuExit":%s,"gpuBase64":"%s","capExit":%s,"capBase64":"%s"}' "$memTotalKb" "$memAvailKb" "$cpuCount" "$arch" "$diskTotalK" "$diskAvailK" "$diskExit" "$gpuExit" "$gpuB64" "$capExit" "$capB64"
|
||||
`;
|
||||
|
||||
const gpuFromProbe = (
|
||||
gpuExit: number,
|
||||
gpuCsv: string,
|
||||
@ -288,7 +255,8 @@ const gpuFromProbe = (
|
||||
}
|
||||
const devices = parseNvidiaGpuCsv(gpuCsv);
|
||||
return {
|
||||
detection: "available",
|
||||
detection:
|
||||
gpuCsv.trim() && devices.length === 0 ? "unavailable" : "available",
|
||||
devices:
|
||||
capExit === 0 ? mergeComputeCapabilities(devices, capCsv) : devices,
|
||||
};
|
||||
@ -364,17 +332,35 @@ export const getServerHardware = async (
|
||||
});
|
||||
}
|
||||
|
||||
const script = serverId
|
||||
? buildRemoteHardwareScript()
|
||||
: buildLocalHardwareScript();
|
||||
try {
|
||||
const result = serverId
|
||||
? await execAsyncRemote(serverId, script, undefined, {
|
||||
timeout: HARDWARE_PROBE_TIMEOUT_MS,
|
||||
})
|
||||
: await execAsync(script, { timeout: HARDWARE_PROBE_TIMEOUT_MS });
|
||||
return parseServerHardware(result.stdout);
|
||||
} catch (error) {
|
||||
return emptyHardware(error);
|
||||
const results = await Promise.allSettled(
|
||||
buildHardwareScripts(Boolean(serverId)).map(async (script) => {
|
||||
const result = serverId
|
||||
? await execAsyncRemote(serverId, script, undefined, {
|
||||
timeout: HARDWARE_PROBE_TIMEOUT_MS,
|
||||
})
|
||||
: await execAsync(script, { timeout: HARDWARE_PROBE_TIMEOUT_MS });
|
||||
const parsed: unknown = JSON.parse(result.stdout);
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
throw new Error("Could not parse server hardware output");
|
||||
}
|
||||
return parsed;
|
||||
}),
|
||||
);
|
||||
const fields = {};
|
||||
const errors: string[] = [];
|
||||
for (const result of results) {
|
||||
if (result.status === "fulfilled") Object.assign(fields, result.value);
|
||||
else
|
||||
errors.push(
|
||||
result.reason instanceof Error
|
||||
? result.reason.message
|
||||
: "Could not read server hardware",
|
||||
);
|
||||
}
|
||||
return {
|
||||
...parseServerHardware(
|
||||
JSON.stringify({ ...fields, kind: serverId ? "remote" : "local" }),
|
||||
),
|
||||
...(errors.length ? { error: [...new Set(errors)].join("; ") } : {}),
|
||||
};
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user