The_Great_Archive/dokploy
1
0
Fork 0
mirror of https://github.com/Dokploy/dokploy.git synced 2026-09-14 11:06:15 +05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge 50d0a40722 into 853ca33659

Browse Source
This commit is contained in:
EgerDev 2026-09-12 09:08:30 +00:00 committed by GitHub
commit 8bab867e4b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 3411 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

211
apps/dokploy/__test__/server/exec-async-remote-lifecycle.test.ts Normal file
View File

@ -0,0 +1,211 @@
import { type ChildProcess, execFileSync, spawn } from "node:child_process";
import { findServerById } from "@dokploy/server/services/server";
import {
ExecError,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { type Connection, Server, utils } from "ssh2";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@dokploy/server/services/server", () => ({ findServerById: vi.fn() }));
const alive = (pid: number) => {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
};
describe.skipIf(process.platform === "win32")(
"real loopback SSH process lifecycle",
() => {
const key = utils.generateKeyPairSync("ed25519");
let server: Server;
const connections: Connection[] = [];
const processes: ChildProcess[] = [];
const observedPids = new Set<number>();
const observeGroup = (group: number) => {
for (const line of execFileSync("ps", ["-axo", "pid=,pgid="], {
encoding: "utf8",
}).split("\n")) {
const [pid, pgid] = line.trim().split(/\s+/).map(Number);
if (pid && pgid === group) observedPids.add(pid);
}
};
beforeEach(async () => {
observedPids.clear();
server = new Server({ hostKeys: [key.private] }, (connection) => {
connections.push(connection);
connection.on("error", () => {});
connection.on("authentication", (context) => context.accept());
connection.on("ready", () =>
connection.on("session", (accept) => {
const session = accept();
// Ignore SSH signals deliberately: the remote deadline must survive transport loss.
session.on("signal", () => {});
session.on("exec", (acceptExec, _reject, info) => {
const channel = acceptExec();
const child = spawn("/bin/sh", ["-c", info.command], {
detached: true,
stdio: ["ignore", "pipe", "pipe"],
});
processes.push(child);
if (child.pid) observedPids.add(child.pid);
child.stdout?.on("data", (chunk: Buffer) => {
if (child.pid) observeGroup(child.pid);
if (!channel.destroyed) channel.write(chunk);
});
child.stderr?.on("data", (chunk: Buffer) => {
if (!channel.destroyed) channel.stderr.write(chunk);
});
child.on("close", (code, signal) => {
if (channel.destroyed) return;
if (signal) channel.exit(signal);
else channel.exit(code ?? 1);
channel.end();
});
});
}),
);
});
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", resolve);
});
const address = server.address();
if (!address || typeof address === "string")
throw new Error("Missing fixture port");
vi.mocked(findServerById).mockResolvedValue({
sshKeyId: "fixture",
ipAddress: "127.0.0.1",
port: address.port,
username: "fixture",
sshKey: { privateKey: key.private },
} as never);
});
afterEach(async () => {
for (const child of processes.splice(0)) {
if (!child.pid) continue;
try {
process.kill(-child.pid, "SIGKILL");
} catch {
/* already gone */
}
}
for (const connection of connections.splice(0)) connection.end();
await new Promise<void>((resolve) => server.close(() => resolve()));
});
it("kills the remote shell, child, grandchild and watchdog after timeout even when signals are ignored", async () => {
const command =
"echo shell=$$; sh -c 'echo child=$$; sleep 30 & echo grandchild=$!; echo partial >&2; wait' & wait";
const started = Date.now();
await expect(
execAsyncRemote("fixture", command, undefined, { timeout: 2000 }),
).rejects.toSatisfy(
(error: unknown) =>
error instanceof ExecError &&
/timed out/.test(error.message) &&
(error.stdout?.includes("grandchild=") ?? false) &&
(error.stderr?.includes("partial") ?? false),
);
expect(Date.now() - started).toBeLessThan(3500);
expect(observedPids.size).toBeGreaterThanOrEqual(3);
await vi.waitFor(
() => {
expect([...observedPids].filter(alive)).toEqual([]);
},
{ timeout: 2000, interval: 30 },
);
});
it("retains remote cleanup after the SSH transport disconnects", async () => {
let disconnected = false;
const pending = execAsyncRemote(
"fixture",
"echo shell=$$; sleep 30 & echo grandchild=$!; wait",
() => {
if (disconnected) return;
disconnected = true;
connections[0]?.end();
},
{ timeout: 2000 },
);
await expect(pending).rejects.toBeInstanceOf(ExecError);
expect(observedPids.size).toBeGreaterThanOrEqual(3);
await vi.waitFor(
() => expect([...observedPids].filter(alive)).toEqual([]),
{ timeout: 2000, interval: 30 },
);
});
it.each(["1", "2"])(
"retains the deadline while an orphan grandchild holds output fd %s",
async (fd) => {
await expect(
execAsyncRemote(
"fixture",
`sleep 30 >&${fd} & echo grandchild=$!`,
undefined,
{ timeout: 2000 },
),
).rejects.toSatisfy(
(error: unknown) =>
error instanceof ExecError && /timed out/.test(error.message),
);
await vi.waitFor(
() => expect([...observedPids].filter(alive)).toEqual([]),
{ timeout: 2000, interval: 30 },
);
},
);
it.each(["exit 0", "exec sh -c 'exit 0'"])(
"does not leak the private exit status for %s",
async (command) => {
await expect(
execAsyncRemote("fixture", command, undefined, { timeout: 2000 }),
).resolves.toEqual({ stdout: "", stderr: "" });
},
);
it("preserves large interleaved stdout and stderr independently", async () => {
const command =
"i=0; while [ $i -lt 2048 ]; do printf 'stdout\\n'; printf 'stderr\\n' >&2; i=$((i+1)); done";
await expect(
execAsyncRemote("fixture", command, undefined, { timeout: 5000 }),
).resolves.toEqual({
stdout: "stdout\n".repeat(2048),
stderr: "stderr\n".repeat(2048),
});
});
it.each([0, 7])(
"clears every watchdog process on normal exit %i",
async (code) => {
const pending = execAsyncRemote(
"fixture",
`echo shell=$$; sleep 0.1; exit ${code}`,
undefined,
{ timeout: 30_000 },
);
if (code === 0)
await expect(pending).resolves.toMatchObject({ stderr: "" });
else
await expect(pending).rejects.toMatchObject({
exitCode: code,
stderr: "",
});
await vi.waitFor(
() => expect([...observedPids].filter(alive)).toEqual([]),
{ timeout: 1000 },
);
},
);
},
);

448
apps/dokploy/__test__/server/exec-async-timeout.test.ts Normal file
View File

@ -0,0 +1,448 @@
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { findServerById } from "@dokploy/server/services/server";
import {
ExecError,
execAsync,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { ssh, MockClient } = vi.hoisted(() => {
class MockStream {
signal = vi.fn();
close = vi.fn(() => {
this.closed = true;
});
destroy = vi.fn(() => {
this.destroyed = true;
});
closed = false;
destroyed = false;
handlers: Record<string, Array<(...args: unknown[]) => void>> = {};
on(event: string, cb: (...args: unknown[]) => void) {
if (!this.handlers[event]) this.handlers[event] = [];
this.handlers[event].push(cb);
return this;
}
stderrHandlers: Array<(...args: unknown[]) => void> = [];
stderr = {
on: (_event: string, cb: (...args: unknown[]) => void) => {
this.stderrHandlers.push(cb);
return this;
},
};
emitStderr(data: string) {
for (const cb of this.stderrHandlers) cb(data);
}
emitClose(code: number) {
for (const cb of this.handlers.close ?? []) cb(code, null);
}
emitError(error: Error) {
for (const cb of this.handlers.error ?? []) cb(error);
}
emitData(data: string) {
for (const cb of this.handlers.data ?? []) cb(data);
}
}
class MockClient {
ended = false;
destroyed = false;
execCommand = "";
execShouldFail: Error | null = null;
execShouldThrow: Error | null = null;
deferExec = false;
execCallback?: (err: Error | null, stream: MockStream) => void;
stream = new MockStream();
end = vi.fn(() => {
this.ended = true;
});
destroy = vi.fn(() => {
this.destroyed = true;
});
handlers: Record<string, Array<(...args: unknown[]) => void>> = {};
constructor() {
ssh.instances.push(this);
}
once(event: string, cb: (...args: unknown[]) => void) {
if (!this.handlers[event]) this.handlers[event] = [];
this.handlers[event].push(cb);
return this;
}
on(event: string, cb: (...args: unknown[]) => void) {
return this.once(event, cb);
}
connect() {
if (ssh.connectError) throw ssh.connectError;
return this;
}
exec(command: string, cb: (err: Error | null, stream: MockStream) => void) {
this.execCommand = command;
if (this.execShouldThrow) throw this.execShouldThrow;
if (this.deferExec) {
this.execCallback = cb;
return;
}
if (this.execShouldFail) {
cb(this.execShouldFail, this.stream);
return;
}
cb(null, this.stream);
}
emitReady() {
for (const cb of this.handlers.ready ?? []) cb();
}
emitError(err: Error & { level?: string }) {
for (const cb of this.handlers.error ?? []) cb(err);
}
}
const ssh = {
instances: [] as MockClient[],
connectError: null as Error | null,
};
return { ssh, MockClient };
});
vi.mock("ssh2", () => ({
Client: MockClient,
}));
vi.mock("@dokploy/server/services/server", () => ({
findServerById: vi.fn(),
}));
const pidAlive = (pid: number) => {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
};
const readPid = (file: string) => Number(readFileSync(file, "utf8").trim());
describe("execAsync timeout", () => {
const dirs: string[] = [];
afterEach(() => {
for (const dir of dirs) {
rmSync(dir, { recursive: true, force: true });
}
dirs.length = 0;
});
it("does not apply a timeout unless one is requested", async () => {
const result = await execAsync("printf ok");
expect(result.stdout).toContain("ok");
});
it("rejects a hanging local command and kills the process group", async () => {
const dir = mkdtempSync(path.join(tmpdir(), "dokploy-exec-timeout-"));
dirs.push(dir);
const shellPidFile = path.join(dir, "shell.pid");
const sleepPidFile = path.join(dir, "sleep.pid");
const started = Date.now();
const command = `echo $$ > "${shellPidFile}"; sleep 30 & echo $! > "${sleepPidFile}"; wait`;
await expect(execAsync(command, { timeout: 300 })).rejects.toSatisfy(
(error: unknown) =>
error instanceof ExecError &&
/timed out after 300ms/.test(error.message),
);
expect(Date.now() - started).toBeLessThan(2000);
await vi.waitFor(
() => {
expect(pidAlive(readPid(shellPidFile))).toBe(false);
expect(pidAlive(readPid(sleepPidFile))).toBe(false);
},
{ timeout: 2000, interval: 50 },
);
});
it("rejects nonzero exit before the timeout without treating it as a hang", async () => {
await expect(execAsync("exit 7", { timeout: 2000 })).rejects.toSatisfy(
(error: unknown) =>
error instanceof ExecError &&
!/timed out/.test(error.message) &&
error.exitCode === 7,
);
});
});
describe("execAsyncRemote timeout", () => {
const server = {
sshKeyId: "key-1",
ipAddress: "203.0.113.10",
port: 22,
username: "root",
sshKey: { privateKey: "fake-key" },
};
beforeEach(() => {
ssh.instances.length = 0;
ssh.connectError = null;
vi.mocked(findServerById).mockReset();
vi.mocked(findServerById).mockResolvedValue(server as never);
vi.useFakeTimers();
});
afterEach(() => {
vi.clearAllTimers();
vi.useRealTimers();
});
const flush = () => Promise.resolve();
const startRemote = (onData?: (data: string) => void, timeout = 1000) =>
execAsyncRemote("server-1", "nvidia-smi", onData, { timeout });
it("times out before ready and closes the SSH client", async () => {
const pending = startRemote();
const rejected = expect(pending).rejects.toSatisfy(
(error: unknown) =>
error instanceof ExecError &&
/timed out after 1000ms/.test(error.message),
);
await flush();
const client = ssh.instances[0];
expect(client).toBeDefined();
await vi.advanceTimersByTimeAsync(1000);
await rejected;
expect(client?.stream.close).not.toHaveBeenCalled();
expect(client?.end).toHaveBeenCalled();
});
it("times out after the stream exists and closes stream plus client", async () => {
const pending = startRemote();
const rejected = expect(pending).rejects.toMatchObject({
name: "ExecError",
});
await flush();
const client = ssh.instances[0];
client?.emitReady();
expect(client?.execCommand).toContain("nvidia-smi");
await vi.advanceTimersByTimeAsync(1000);
await rejected;
expect(client?.stream.signal).toHaveBeenCalledWith("KILL");
expect(client?.stream.close).toHaveBeenCalledTimes(1);
expect(client?.end).toHaveBeenCalledTimes(1);
client?.stream.emitClose(0);
expect(client?.end).toHaveBeenCalledTimes(1);
});
it("clears the timeout timer after a successful remote command", async () => {
const pending = startRemote(undefined, 5000);
await flush();
expect(vi.getTimerCount()).toBeGreaterThanOrEqual(1);
const client = ssh.instances[0];
client?.emitReady();
const timersBeforeClose = vi.getTimerCount();
client?.stream.emitClose(0);
await expect(pending).resolves.toEqual({ stdout: "", stderr: "" });
expect(vi.getTimerCount()).toBeLessThan(timersBeforeClose);
expect(vi.getTimerCount()).toBe(0);
});
it("resolves on success and does not reject later when the timer would have fired", async () => {
const rejections: unknown[] = [];
const onUnhandled = (reason: unknown) => {
rejections.push(reason);
};
process.on("unhandledRejection", onUnhandled);
const pending = startRemote(undefined, 5000);
await flush();
const client = ssh.instances[0];
client?.emitReady();
client?.stream.emitClose(0);
await expect(pending).resolves.toEqual({ stdout: "", stderr: "" });
await vi.advanceTimersByTimeAsync(10_000);
process.off("unhandledRejection", onUnhandled);
expect(rejections).toEqual([]);
expect(client?.ended).toBe(true);
});
it("settles only once when close and timeout race", async () => {
const pending = startRemote();
await flush();
const client = ssh.instances[0];
client?.emitReady();
client?.stream.emitClose(0);
await vi.advanceTimersByTimeAsync(1000);
await expect(pending).resolves.toEqual({ stdout: "", stderr: "" });
});
it("rejects SSH errors before timeout and ignores the later timer", async () => {
const rejections: unknown[] = [];
const onUnhandled = (reason: unknown) => {
rejections.push(reason);
};
process.on("unhandledRejection", onUnhandled);
const pending = startRemote();
const rejected = expect(pending).rejects.toSatisfy(
(error: unknown) =>
error instanceof ExecError &&
error.message.includes("SSH connection error"),
);
await flush();
const client = ssh.instances[0];
client?.emitError(
Object.assign(new Error("offline"), { level: "client-socket" }),
);
await rejected;
await vi.advanceTimersByTimeAsync(1000);
process.off("unhandledRejection", onUnhandled);
expect(rejections).toEqual([]);
expect(client?.end).toHaveBeenCalled();
});
it("rejects exec callback errors before timeout", async () => {
const pending = startRemote();
const rejected = expect(pending).rejects.toSatisfy(
(error: unknown) =>
error instanceof ExecError &&
error.message.includes("Remote command execution failed"),
);
await flush();
const client = ssh.instances[0];
if (client) client.execShouldFail = new Error("cannot exec");
client?.emitReady();
await rejected;
await vi.advanceTimersByTimeAsync(1000);
expect(client?.end).toHaveBeenCalled();
});
it("cleans timers and clients when connect throws synchronously", async () => {
ssh.connectError = new Error("invalid key");
await expect(startRemote()).rejects.toMatchObject({
message: "SSH connection failed",
});
expect(vi.getTimerCount()).toBe(0);
expect(ssh.instances[0]?.destroy).toHaveBeenCalledOnce();
});
it("cleans timers and clients when exec throws synchronously", async () => {
const pending = startRemote();
const rejected = expect(pending).rejects.toMatchObject({
message: "Remote command execution failed",
});
await flush();
const client = ssh.instances[0];
if (client) client.execShouldThrow = new Error("channel unavailable");
client?.emitReady();
await rejected;
expect(vi.getTimerCount()).toBe(0);
expect(client?.destroy).toHaveBeenCalledOnce();
});
it("closes a late channel even when its signal method throws", async () => {
const pending = startRemote();
const rejected = expect(pending).rejects.toBeInstanceOf(ExecError);
await flush();
const client = ssh.instances[0];
if (client) client.deferExec = true;
client?.emitReady();
await vi.advanceTimersByTimeAsync(1000);
await rejected;
client?.stream.signal.mockImplementation(() => {
throw new Error("closed transport");
});
if (client) client.execCallback?.(null, client.stream);
expect(client?.stream.close).toHaveBeenCalledOnce();
expect(client?.stream.destroy).toHaveBeenCalledOnce();
expect(vi.getTimerCount()).toBe(0);
});
it("preserves both partial streams and ignores late output and errors", async () => {
const onData = vi.fn();
const pending = startRemote(onData);
const rejected = expect(pending).rejects.toMatchObject({
stdout: "out",
stderr: "err",
});
await flush();
const client = ssh.instances[0];
client?.emitReady();
client?.stream.emitData("out");
client?.stream.emitStderr("err");
await vi.advanceTimersByTimeAsync(1000);
await rejected;
client?.stream.emitData("late out");
client?.stream.emitStderr("late err");
client?.emitError(new Error("late error"));
client?.stream.emitError(new Error("late stream error"));
client?.stream.emitClose(0);
expect(onData.mock.calls).toEqual([["out"], ["err"]]);
expect(client?.end).toHaveBeenCalledTimes(1);
expect(client?.destroy).toHaveBeenCalledTimes(1);
expect(vi.getTimerCount()).toBe(0);
});
it.each([undefined, 0, -1, Number.NaN, Number.POSITIVE_INFINITY])(
"does not wrap commands without a positive finite timeout: %s",
async (timeout) => {
const pending = execAsyncRemote(
"server-1",
"original command",
undefined,
{ timeout },
);
await flush();
const client = ssh.instances[0];
client?.emitReady();
expect(client?.execCommand).toBe("original command");
expect(vi.getTimerCount()).toBe(0);
client?.stream.emitClose(0);
await pending;
expect(client?.stream.signal).not.toHaveBeenCalled();
},
);
it("keeps a normal nonzero exit distinct from timeout", async () => {
const pending = startRemote();
const rejected = expect(pending).rejects.toMatchObject({ exitCode: 7 });
await flush();
const client = ssh.instances[0];
client?.emitReady();
client?.stream.emitClose(7);
await rejected;
expect(vi.getTimerCount()).toBe(0);
});
it("still delivers onData chunks if the command hangs afterward", async () => {
const chunks: string[] = [];
const pending = startRemote((data) => {
chunks.push(data);
});
const rejected = expect(pending).rejects.toMatchObject({
name: "ExecError",
});
await flush();
const client = ssh.instances[0];
client?.emitReady();
client?.stream.emitData("partial");
await vi.advanceTimersByTimeAsync(1000);
await rejected;
expect(chunks).toEqual(["partial"]);
expect(client?.stream.close).toHaveBeenCalled();
expect(client?.end).toHaveBeenCalled();
});
});

695
apps/dokploy/__test__/server/model-runner.test.ts Normal file
View File

@ -0,0 +1,695 @@
import { execFileSync, execSync } from "node:child_process";
import {
chmodSync,
existsSync,
mkdtempSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import {
buildModelRunnerScript,
composeModelsSupported,
getModelRunnerCapability,
parseModelRunnerCapability,
} from "@dokploy/server/services/model-runner";
import {
execAsync,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const cloud = { enabled: false };
vi.mock("@dokploy/server/constants", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@dokploy/server/constants")>();
return {
...actual,
get IS_CLOUD() {
return cloud.enabled;
},
};
});
vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => ({
...(await importOriginal<
typeof import("@dokploy/server/utils/process/execAsync")
>()),
execAsync: vi.fn(),
execAsyncRemote: vi.fn(),
}));
const resolveBin = (name: string) =>
execSync(`command -v ${name}`, { encoding: "utf8" }).trim();
const b64 = (value: unknown) =>
Buffer.from(
typeof value === "string" ? value : JSON.stringify(value),
).toString("base64");
const envelope = (opts: {
dockerPresent?: boolean;
engineExit?: number;
engine?: unknown;
pluginsExit?: number;
plugins?: unknown;
metadata?: unknown;
containerStatus?: string;
}) =>
JSON.stringify({
dockerPresent: opts.dockerPresent ?? true,
engineExit: opts.engineExit ?? 0,
engineBase64: opts.engine === undefined ? "" : b64(opts.engine),
pluginsExit: opts.pluginsExit ?? 0,
pluginsBase64:
opts.metadata !== undefined
? b64(opts.metadata)
: opts.plugins === undefined
? ""
: b64({ plugins: opts.plugins, errors: null }),
containerStatus: opts.containerStatus ?? "",
});
const plugin = (
name: string,
version?: string,
err?: unknown,
extra: Record<string, unknown> = {},
) => ({
Name: name,
...(version !== undefined ? { Version: version } : {}),
...(err !== undefined ? { Err: err } : {}),
...extra,
});
const engine = (
server: { version?: string; os?: string; arch?: string } = {},
) => ({
serverVersion: server.version ?? "28.5.2",
os: server.os ?? "linux",
arch: server.arch ?? "amd64",
});
const sandboxes: string[] = [];
const makeSandbox = (dockerShim?: string) => {
const dir = mkdtempSync(path.join(tmpdir(), "dokploy-model-runner-"));
sandboxes.push(dir);
for (const tool of ["tr", "base64"]) {
const shim = path.join(dir, tool);
writeFileSync(shim, `#!/bin/sh\nexec ${resolveBin(tool)} "$@"\n`);
chmodSync(shim, 0o755);
}
if (dockerShim) {
const shim = path.join(dir, "docker");
writeFileSync(shim, dockerShim);
chmodSync(shim, 0o755);
}
return dir;
};
afterEach(() => {
for (const dir of sandboxes) {
rmSync(dir, { recursive: true, force: true });
expect(existsSync(dir)).toBe(false);
}
sandboxes.length = 0;
});
const fakeDocker = (opts: {
engine?: unknown;
engineExit?: number;
plugins?: unknown;
pluginsExit?: number;
inspect?: string;
}) => {
const engineJson =
opts.engine === undefined ? "" : JSON.stringify(opts.engine);
const pluginsJson =
opts.plugins === undefined
? ""
: JSON.stringify({ plugins: opts.plugins, errors: null });
const inspect =
opts.inspect === undefined
? "exit 1"
: `printf '%s\\n' ${JSON.stringify(opts.inspect)}`;
return `#!/bin/sh
if [ "$1" = "info" ]; then
case "$*" in
*ClientInfo.Plugins*)
${pluginsJson ? `printf '%s\\n' ${JSON.stringify(pluginsJson)}` : ":"}
exit ${opts.pluginsExit ?? 0}
;;
esac
${engineJson ? `printf '%s\\n' ${JSON.stringify(engineJson)}` : ":"}
exit ${opts.engineExit ?? 0}
fi
if [ "$1" = "inspect" ]; then
${inspect}
exit 0
fi
exit 1
`;
};
const runScript = (sandboxPath: string) =>
execFileSync(resolveBin("bash"), ["-c", buildModelRunnerScript()], {
encoding: "utf8",
env: { ...process.env, PATH: sandboxPath },
});
describe("buildModelRunnerScript", () => {
it("separates engine and plugin docker info probes and keeps their exit statuses", () => {
const script = buildModelRunnerScript();
expect(script).toContain("command -v docker");
expect(script).toContain("{{json .ServerVersion}}");
expect(script).toContain("{{json .ClientInfo.Plugins}}");
expect(script).toContain("{{json .ClientErrors}}");
expect(script).toContain("engineExit=$?");
expect(script).toContain("pluginsExit=$?");
expect(script).toContain(
"docker inspect -f '{{.State.Status}}' docker-model-runner",
);
expect(script).not.toMatch(/docker info --format.*\| base64/);
expect(script).not.toContain("docker model version");
expect(script).not.toContain("docker compose version");
expect(script).not.toMatch(/docker model status/);
expect(script).not.toMatch(
/install-runner|start-runner|restart-runner|uninstall-runner/,
);
expect(script).not.toContain("12434");
});
});
describe("composeModelsSupported", () => {
it.each([
[null, null],
["garbage", null],
["2.37.3", false],
["v2.38.0-rc.1", false],
["2.38.0-beta.1", false],
["2.38.0", true],
["v2.39.1", true],
["v2.40.3-desktop.1", true],
["v5.0.0", true],
] as const)("%s => %s", (version, supported) => {
expect(composeModelsSupported(version)).toBe(supported);
});
});
describe("parseModelRunnerCapability", () => {
it.each([
null,
[],
"malformed",
{ plugins: [] },
{ plugins: [], errors: "bad" },
])("rejects malformed metadata envelope %j", (metadata) => {
const result = parseModelRunnerCapability(
envelope({ engine: engine(), metadata }),
);
expect(result.docker.available).toBe(true);
expect(result.compose.available).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
expect(result.error).toBe("Could not parse Docker CLI plugin metadata");
});
it("retains both Engine and discovery errors without exposing details", () => {
const result = parseModelRunnerCapability(
envelope({ engineExit: 1, pluginsExit: 1 }),
);
expect(result.docker.available).toBe(false);
expect(result.compose.available).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
expect(result.error).toBe(
"Could not read Docker Engine info; Could not discover Docker CLI plugins",
);
});
it.each([
"null",
"[]",
"{}",
'{"dockerPresent":"yes"}',
'{"dockerPresent":true,"containerStatus":1}',
])("returns unknown plugins for malformed outer envelope %s", (stdout) => {
const result = parseModelRunnerCapability(stdout);
expect(result.error).toBe("Could not parse model runner probe output");
expect(result.compose.available).toBeNull();
expect(result.compose.modelsSupported).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
});
it.each([
["HTTP 403", engine({ version: "", os: "", arch: "" })],
["HTTP 500", engine({ version: "", os: "", arch: "" })],
["CLI 27 unreachable", engine({ version: "", os: "", arch: "" })],
["empty HTTP 200", {}],
["malformed info", "not-json"],
["wrong version type", { serverVersion: 42 }],
])("requires positive Engine evidence for exit-zero %s", (_name, info) => {
const result = parseModelRunnerCapability(
envelope({
engine: info,
plugins: [plugin("compose", "2.38.0"), plugin("model", "1.2.6")],
}),
);
expect(result.docker.available).toBe(false);
expect(result.error).toBe("Could not read Docker Engine info");
expect(result.compose.available).toBe(true);
expect(result.modelRunner.cliAvailable).toBe(true);
});
it("accepts vendor Engine versions without semver coercion", () => {
const result = parseModelRunnerCapability(
envelope({
engine: engine({ version: "vendor-engine-release" }),
plugins: [],
}),
);
expect(result.docker.available).toBe(true);
expect(result.docker.version).toBe("vendor-engine-release");
});
it.each([null, []])(
"recognizes successful empty enumeration %s",
(plugins) => {
const result = parseModelRunnerCapability(
envelope({ engine: engine(), plugins }),
);
expect(result.error).toBeUndefined();
expect(result.compose.available).toBe(false);
expect(result.compose.modelsSupported).toBe(false);
expect(result.modelRunner.cliAvailable).toBe(false);
},
);
it.each([
{},
[null],
[42],
[{ Name: 42 }],
[{ Name: "compose", Version: 42 }],
])("keeps malformed plugin metadata unknown: %j", (plugins) => {
const result = parseModelRunnerCapability(
envelope({ engine: engine(), plugins }),
);
expect(result.docker.available).toBe(true);
expect(result.compose.available).toBeNull();
expect(result.compose.modelsSupported).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
expect(result.error).toBe("Could not parse Docker CLI plugin metadata");
});
it("does not treat exit-zero ClientErrors as successful empty enumeration", () => {
const result = parseModelRunnerCapability(
JSON.stringify({
dockerPresent: true,
engineExit: 0,
engineBase64: b64(engine()),
pluginsExit: 0,
pluginsBase64: b64({
plugins: null,
errors: ["private/path: permission denied"],
}),
containerStatus: "running",
}),
);
expect(result.docker.available).toBe(true);
expect(result.compose.available).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
expect(result.modelRunner.standaloneRunnerContainerStatus).toBe("running");
expect(result.error).toBe("Could not discover Docker CLI plugins");
expect(JSON.stringify(result)).not.toContain("private/path");
});
it.each([undefined, "", "nonsense"])(
"keeps valid versionless compose support unknown: %s",
(version) => {
const result = parseModelRunnerCapability(
envelope({
engine: engine(),
plugins: [plugin("compose", version), plugin("model")],
}),
);
expect(result.compose.available).toBe(true);
expect(result.compose.modelsSupported).toBeNull();
expect(result.modelRunner.cliAvailable).toBe(true);
expect(result.modelRunner.cliVersion).toBeNull();
expect(result.error).toBeUndefined();
},
);
it.each(["compose", "model"])(
"distinguishes invalid %s from absent without erasing the other plugin",
(name) => {
const result = parseModelRunnerCapability(
envelope({
engine: engine(),
plugins: [
plugin(
"compose",
"2.38.0",
name === "compose" ? "private failure" : undefined,
),
plugin(
"model",
"1.2.6",
name === "model" ? { message: "private failure" } : undefined,
),
],
}),
);
expect(result.compose.available).toBe(name !== "compose");
expect(result.modelRunner.cliAvailable).toBe(name !== "model");
expect(
name === "compose" ? result.compose.error : result.modelRunner.error,
).toBe(`Docker ${name} CLI plugin is invalid`);
expect(JSON.stringify(result)).not.toContain("private failure");
},
);
it("treats a missing Docker CLI as a normal negative, not an error", () => {
const result = parseModelRunnerCapability(
envelope({ dockerPresent: false }),
);
expect(result.error).toBeUndefined();
expect(result.docker.available).toBe(false);
expect(result.compose.available).toBe(false);
expect(result.modelRunner.cliAvailable).toBe(false);
});
it("treats a failed engine probe as unreachable even if plugins succeed", () => {
const result = parseModelRunnerCapability(
envelope({
engineExit: 1,
plugins: [plugin("compose", "v2.38.0")],
containerStatus: "exited",
}),
);
expect(result.error).toBe("Could not read Docker Engine info");
expect(result.docker.available).toBe(false);
expect(result.compose.available).toBe(true);
expect(result.modelRunner.standaloneRunnerContainerStatus).toBe("exited");
});
it("keeps the engine available when plugin metadata fails", () => {
const result = parseModelRunnerCapability(
envelope({
engine: engine(),
pluginsExit: 1,
}),
);
expect(result.error).toBe("Could not discover Docker CLI plugins");
expect(result.docker).toEqual({
available: true,
version: "28.5.2",
os: "linux",
arch: "amd64",
});
expect(result.compose.available).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
});
it("returns error for a malformed probe envelope", () => {
const result = parseModelRunnerCapability("not-json");
expect(result.error).toBe("Could not parse model runner probe output");
expect(result.docker.available).toBe(false);
});
it("treats missing plugins as compose and model unavailable", () => {
const result = parseModelRunnerCapability(
envelope({ engine: engine(), plugins: [] }),
);
expect(result.error).toBeUndefined();
expect(result.docker.available).toBe(true);
expect(result.compose.available).toBe(false);
expect(result.modelRunner.cliAvailable).toBe(false);
});
it.each([
["absent", []],
["invalid Err", [plugin("compose", "v2.38.0", "cannot exec")]],
["malformed version", [plugin("compose", "not-a-version")]],
] as const)("compose plugin %s", (_name, plugins) => {
const result = parseModelRunnerCapability(
envelope({ engine: engine(), plugins: [...plugins] }),
);
if (_name === "malformed version") {
expect(result.compose.available).toBe(true);
expect(result.compose.version).toBe("not-a-version");
expect(result.compose.modelsSupported).toBeNull();
return;
}
expect(result.compose.available).toBe(false);
expect(result.compose.modelsSupported).toBe(false);
expect(result.error).toBeUndefined();
});
it.each([
["2.37.3", false],
["v2.38.0-rc.1", false],
["2.38.0-beta.1", false],
["2.38.0", true],
["v2.39.1", true],
["v2.40.3-desktop.1", true],
["v5.0.0", true],
] as const)("compose %s => modelsSupported=%s", (version, supported) => {
const result = parseModelRunnerCapability(
envelope({
engine: engine(),
plugins: [plugin("compose", version)],
}),
);
expect(result.compose).toEqual({
available: true,
version,
modelsSupported: supported,
});
});
it.each([
["absent", []],
["invalid Err", [plugin("model", "v1.0.2", { message: "broken" })]],
] as const)("model plugin %s", (_name, plugins) => {
const result = parseModelRunnerCapability(
envelope({ engine: engine(), plugins: [...plugins] }),
);
expect(result.modelRunner.cliAvailable).toBe(false);
expect(result.modelRunner.cliVersion).toBeNull();
expect(result.error).toBeUndefined();
});
it.each([
["", null],
["running", "running"],
["exited", "exited"],
] as const)("standalone runner %s", (status, expected) => {
const result = parseModelRunnerCapability(
envelope({
engine: engine(),
plugins: [plugin("model", "v1.0.2")],
containerStatus: status,
}),
);
expect(result.modelRunner.cliAvailable).toBe(true);
expect(result.modelRunner.standaloneRunnerContainerStatus).toBe(expected);
});
it("does not leak plugin Path or unrelated docker-info fields", () => {
const result = parseModelRunnerCapability(
envelope({
engine: engine(),
plugins: [
plugin("compose", "v2.38.0", undefined, {
Path: "/usr/libexec/docker/cli-plugins/docker-compose",
Vendor: "Docker Inc.",
}),
plugin("model", "v1.0.2", undefined, {
Path: "/usr/libexec/docker/cli-plugins/docker-model",
}),
],
containerStatus: "running",
}),
);
const serialized = JSON.stringify(result);
expect(serialized).not.toContain("Path");
expect(serialized).not.toContain("/usr/libexec");
expect(serialized).not.toContain("Vendor");
expect(result.modelRunner).toEqual({
cliAvailable: true,
cliVersion: "v1.0.2",
standaloneRunnerContainerStatus: "running",
});
});
});
describe("model runner probe script", () => {
it("reports Docker missing when the docker binary is absent", () => {
const result = parseModelRunnerCapability(runScript(makeSandbox()));
expect(result.error).toBeUndefined();
expect(result.docker.available).toBe(false);
expect(result.modelRunner.cliAvailable).toBe(false);
});
it("records a non-zero engine probe as unreachable", () => {
const result = parseModelRunnerCapability(
runScript(
makeSandbox(
fakeDocker({
engineExit: 1,
plugins: [plugin("compose", "v2.38.0")],
}),
),
),
);
expect(result.error).toBe("Could not read Docker Engine info");
expect(result.docker.available).toBe(false);
expect(result.compose.available).toBe(true);
});
it("does not report engine unreachable when only ClientInfo.Plugins fails", () => {
const result = parseModelRunnerCapability(
runScript(
makeSandbox(
fakeDocker({
engine: engine(),
pluginsExit: 1,
}),
),
),
);
expect(result.error).toBe("Could not discover Docker CLI plugins");
expect(result.docker.available).toBe(true);
expect(result.docker.version).toBe("28.5.2");
expect(result.compose.available).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
});
it("reads inspect status even when the model plugin is absent", () => {
const result = parseModelRunnerCapability(
runScript(
makeSandbox(
fakeDocker({
engine: engine(),
plugins: [plugin("compose", "v2.38.0")],
inspect: "running",
}),
),
),
);
expect(result.modelRunner.cliAvailable).toBe(false);
expect(result.modelRunner.standaloneRunnerContainerStatus).toBe("running");
expect(result.error).toBeUndefined();
});
});
describe("getModelRunnerCapability", () => {
const probeJson = envelope({
engine: engine(),
plugins: [plugin("compose", "v2.38.0"), plugin("model", "v1.0.2")],
containerStatus: "running",
});
beforeEach(() => {
cloud.enabled = false;
vi.mocked(execAsync).mockReset();
vi.mocked(execAsyncRemote).mockReset();
});
it.each([undefined, "remote-server"])(
"settles a hanging probe through the executor deadline (%s)",
async (serverId) => {
vi.useFakeTimers();
const hanging = (timeout?: number) =>
new Promise<{ stdout: string; stderr: string }>((_resolve, reject) => {
if (timeout)
setTimeout(
() =>
reject(
new Error(
"Command execution timed out with private host detail",
),
),
timeout,
);
});
vi.mocked(execAsync).mockImplementation((_script, options) =>
hanging(options?.timeout),
);
vi.mocked(execAsyncRemote).mockImplementation(
(_serverId, _script, _onData, options) => hanging(options?.timeout),
);
let result:
| Awaited<ReturnType<typeof getModelRunnerCapability>>
| undefined;
try {
void getModelRunnerCapability(serverId).then((value) => {
result = value;
});
await vi.advanceTimersByTimeAsync(30_000);
expect(result).toMatchObject({
docker: { available: false },
compose: { available: null, modelsSupported: null },
modelRunner: {
cliAvailable: null,
standaloneRunnerContainerStatus: null,
},
error: "Could not read model runner capability",
});
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
}
},
);
it("uses execAsync for the self-hosted local path and never execAsyncRemote", async () => {
vi.mocked(execAsync).mockResolvedValue({ stdout: probeJson, stderr: "" });
const result = await getModelRunnerCapability();
expect(execAsync).toHaveBeenCalledWith(
expect.stringContaining("docker info --format"),
{ timeout: 30_000 },
);
expect(execAsyncRemote).not.toHaveBeenCalled();
expect(result.docker.available).toBe(true);
expect(result.error).toBeUndefined();
});
it("uses execAsyncRemote for a remote serverId and never execAsync", async () => {
vi.mocked(execAsyncRemote).mockResolvedValue({
stdout: probeJson,
stderr: "",
});
const result = await getModelRunnerCapability("remote-server");
expect(execAsyncRemote).toHaveBeenCalledWith(
"remote-server",
expect.stringContaining("{{json .ClientInfo.Plugins}}"),
undefined,
{ timeout: 30_000 },
);
expect(execAsync).not.toHaveBeenCalled();
expect(result.modelRunner.cliAvailable).toBe(true);
});
it("rejects a local probe on cloud without executing docker", async () => {
cloud.enabled = true;
await expect(getModelRunnerCapability()).rejects.toMatchObject({
name: "TRPCError",
code: "BAD_REQUEST",
message: "Server is required",
});
expect(execAsync).not.toHaveBeenCalled();
expect(execAsyncRemote).not.toHaveBeenCalled();
});
it("returns empty capability plus error when the remote probe throws", async () => {
vi.mocked(execAsyncRemote).mockRejectedValue(new Error("SSH failed"));
const result = await getModelRunnerCapability("remote-server");
expect(result.error).toBe("Could not read model runner capability");
expect(result.compose.available).toBeNull();
expect(result.modelRunner.cliAvailable).toBeNull();
expect(result.docker.available).toBe(false);
expect(execAsync).not.toHaveBeenCalled();
});
});

996
apps/dokploy/__test__/server/server-hardware.test.ts Normal file
View File

@ -0,0 +1,996 @@
import { execFileSync, execSync } from "node:child_process";
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import {
getServerHardware,
HARDWARE_PROBE_TIMEOUT_MS,
parseNvidiaGpuLine,
parseServerHardware,
} from "@dokploy/server/services/server-hardware";
import { buildHardwareScripts } from "@dokploy/server/services/server-hardware-scripts";
import {
execAsync,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const buildLocalHardwareScript = () => buildHardwareScripts(false).join("\n");
const buildRemoteHardwareScript = () => buildHardwareScripts(true).join("\n");
const cloud = { enabled: false };
vi.mock("@dokploy/server/constants", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@dokploy/server/constants")>();
return {
...actual,
get IS_CLOUD() {
return cloud.enabled;
},
};
});
vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => ({
...(await importOriginal<
typeof import("@dokploy/server/utils/process/execAsync")
>()),
execAsync: vi.fn(),
execAsyncRemote: vi.fn(),
}));
const resolveBin = (name: string) =>
execSync(`command -v ${name}`, { encoding: "utf8" }).trim();
const b64 = (value: string) => Buffer.from(value).toString("base64");
const localEnvelope = (opts: {
engineExit?: number;
engine?: { ncpu?: number; memTotal?: number; arch?: string };
gpuExit?: number;
gpuCsv?: string;
capExit?: number;
capCsv?: string;
}) =>
JSON.stringify({
kind: "local",
engineExit: opts.engineExit ?? 0,
engineBase64: opts.engine ? b64(JSON.stringify(opts.engine)) : "",
gpuExit: opts.gpuExit ?? 1,
gpuBase64: opts.gpuCsv ? b64(opts.gpuCsv) : "",
capExit: opts.capExit ?? 1,
capBase64: opts.capCsv ? b64(opts.capCsv) : "",
});
const remoteEnvelope = (opts: {
memTotalKb?: string;
memAvailKb?: string;
cpuCount?: string;
arch?: string;
diskTotalK?: string;
diskAvailK?: string;
diskExit?: number;
gpuExit?: number;
gpuCsv?: string;
capExit?: number;
capCsv?: string;
}) =>
JSON.stringify({
kind: "remote",
memTotalKb: opts.memTotalKb ?? "",
memAvailKb: opts.memAvailKb ?? "",
cpuCount: opts.cpuCount ?? "",
arch: opts.arch ?? "",
diskTotalK: opts.diskTotalK ?? "",
diskAvailK: opts.diskAvailK ?? "",
diskExit: opts.diskExit ?? 0,
gpuExit: opts.gpuExit ?? 1,
gpuBase64: opts.gpuCsv ? b64(opts.gpuCsv) : "",
capExit: opts.capExit ?? 1,
capBase64: opts.capCsv ? b64(opts.capCsv) : "",
});
const t4 = "0, GPU-aaa, Tesla T4, 15360, 14000, 575.57.08";
const a100 = "1, GPU-bbb, NVIDIA A100-SXM4-40GB, 40960, 20000, 575.57.08";
const commaName =
"2, GPU-ccc, NVIDIA Graphics Device, 16GB, 16384, 15000, 550.54.14";
const t4Cap = "0, 7.5";
const a100Cap = "1, 8.0";
const commaCap = "2, 8.9";
const sandboxes: string[] = [];
const makeSandbox = (bins: Record<string, string>) => {
const dir = mkdtempSync(path.join(tmpdir(), "dokploy-hardware-"));
sandboxes.push(dir);
for (const tool of ["tr", "base64", "printf"]) {
if (bins[tool]) continue;
const shim = path.join(dir, tool);
writeFileSync(shim, `#!/bin/sh\nexec ${resolveBin(tool)} "$@"\n`);
chmodSync(shim, 0o755);
}
for (const [name, body] of Object.entries(bins)) {
const shim = path.join(dir, name);
writeFileSync(shim, body);
chmodSync(shim, 0o755);
}
return dir;
};
afterEach(() => {
cloud.enabled = false;
for (const dir of sandboxes) {
rmSync(dir, { recursive: true, force: true });
}
sandboxes.length = 0;
});
const runScript = (script: string, sandboxPath: string) => {
const stdout = execFileSync(resolveBin("sh"), ["-c", script], {
encoding: "utf8",
env: { ...process.env, PATH: sandboxPath },
});
return JSON.stringify(
Object.assign(
{ kind: script.includes("docker info") ? "local" : "remote" },
...stdout
.trim()
.split("\n")
.map((line) => JSON.parse(line)),
),
);
};
const inventoryQueries = (script: string) =>
[...script.matchAll(/nvidia-smi --query-gpu=(\S+)/g)].map(
(match) => match[1],
);
describe("hardware scripts", () => {
it("does not use container free/nproc/df or /proc/meminfo locally", () => {
const script = buildLocalHardwareScript();
expect(script).toContain("docker info --format");
expect(inventoryQueries(script)).toEqual([
"index,uuid,name,memory.total,memory.free,driver_version",
"index,compute_cap",
]);
expect(script).not.toContain("$(free");
expect(script).not.toContain("free -");
expect(script).not.toContain("$(nproc");
expect(script).not.toContain("nproc ");
expect(script).not.toContain("/proc/meminfo");
expect(script).not.toContain("df -P /");
expect(script).not.toContain("df -Pk /");
expect(script).not.toContain("DockerRootDir");
});
it("reads remote host /proc and root df -Pk, not DockerRootDir", () => {
const script = buildRemoteHardwareScript();
expect(script).toContain("/proc/meminfo");
expect(script).toContain("df -Pk /");
expect(script).toContain("diskOutput=$(df -Pk /");
expect(script).toContain("diskExit=$?");
expect(script).not.toMatch(/df -P \//);
expect(script).toContain("uname -m");
expect(script).not.toContain("DockerRootDir");
expect(inventoryQueries(script)).toEqual([
"index,uuid,name,memory.total,memory.free,driver_version",
"index,compute_cap",
]);
});
});
describe("parseNvidiaGpuLine", () => {
it("parses one GPU including uuid, leaving compute capability unset", () => {
expect(parseNvidiaGpuLine(t4)).toEqual({
index: 0,
uuid: "GPU-aaa",
name: "Tesla T4",
vendor: "nvidia",
computeCapability: null,
memoryTotalMiB: 15360,
memoryFreeMiB: 14000,
driverVersion: "575.57.08",
});
});
it("keeps a GPU name that contains a comma", () => {
expect(parseNvidiaGpuLine(commaName)).toEqual({
index: 2,
uuid: "GPU-ccc",
name: "NVIDIA Graphics Device, 16GB",
vendor: "nvidia",
computeCapability: null,
memoryTotalMiB: 16384,
memoryFreeMiB: 15000,
driverVersion: "550.54.14",
});
});
it("skips malformed lines", () => {
expect(parseNvidiaGpuLine("not-a-gpu")).toBeNull();
expect(parseNvidiaGpuLine("x,y,z")).toBeNull();
expect(
parseNvidiaGpuLine("0abc, GPU-aaa, Tesla T4, 15360, 14000, 575.57.08"),
).toBeNull();
expect(
parseNvidiaGpuLine("-1, GPU-aaa, Tesla T4, 15360, 14000, 575.57.08"),
).toBeNull();
});
it("keeps the row when VRAM is malformed instead of coercing it to 0", () => {
expect(
parseNvidiaGpuLine("0, GPU-aaa, Tesla T4, abc, 14000, 575.57.08"),
).toEqual({
index: 0,
uuid: "GPU-aaa",
name: "Tesla T4",
vendor: "nvidia",
computeCapability: null,
memoryTotalMiB: null,
memoryFreeMiB: 14000,
driverVersion: "575.57.08",
});
});
});
describe("parseServerHardware", () => {
it("reads local Docker Engine facts and leaves available memory and disk null", () => {
const result = parseServerHardware(
localEnvelope({
engine: { ncpu: 8, memTotal: 16_000_000_000, arch: "x86_64" },
}),
);
expect(result.error).toBeUndefined();
expect(result.cpu).toEqual({ count: 8, arch: "x86_64" });
expect(result.memory).toEqual({
totalBytes: 16_000_000_000,
availableBytes: null,
});
expect(result.disk).toEqual({ totalBytes: null, availableBytes: null });
expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
});
it("treats local nvidia-smi success as available, including zero rows", () => {
const empty = parseServerHardware(
localEnvelope({
engine: { ncpu: 4, memTotal: 1, arch: "aarch64" },
gpuExit: 0,
gpuCsv: "",
}),
);
expect(empty.gpu).toEqual({ detection: "available", devices: [] });
const one = parseServerHardware(
localEnvelope({
engine: { ncpu: 4, memTotal: 1, arch: "aarch64" },
gpuExit: 0,
gpuCsv: t4,
}),
);
expect(one.gpu.detection).toBe("available");
expect(one.gpu.devices).toHaveLength(1);
expect(one.gpu.devices[0]?.uuid).toBe("GPU-aaa");
});
it("does not treat nvidia-smi failure as zero GPUs", () => {
const result = parseServerHardware(
localEnvelope({
engine: { ncpu: 2, memTotal: 100, arch: "x86_64" },
gpuExit: 127,
gpuCsv: "",
}),
);
expect(result.cpu.count).toBe(2);
expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
});
it("ignores GPU stdout when the primary nvidia-smi exit is nonzero", () => {
const result = parseServerHardware(
localEnvelope({
engine: { ncpu: 8, memTotal: 100, arch: "x86_64" },
gpuExit: 1,
gpuCsv: t4,
}),
);
expect(result.cpu.count).toBe(8);
expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
});
it("ignores Engine JSON when docker info exit is nonzero", () => {
const result = parseServerHardware(
localEnvelope({
engineExit: 1,
engine: { ncpu: 8, memTotal: 16_000_000_000, arch: "x86_64" },
}),
);
expect(result.cpu).toEqual({ count: null, arch: null });
expect(result.memory.totalBytes).toBeNull();
});
it.each([0, -1, 1.5, "0", "-1", "foo", Number.NaN, Number.POSITIVE_INFINITY])(
"does not report cpu.count=%s",
(cpuCount) => {
const local = parseServerHardware(
localEnvelope({
engine: { ncpu: cpuCount as number, memTotal: 100, arch: "x86_64" },
}),
);
expect(local.cpu.count).toBeNull();
const remote = parseServerHardware(
remoteEnvelope({
cpuCount: String(cpuCount),
arch: "x86_64",
memTotalKb: "4096",
}),
);
expect(remote.cpu.count).toBeNull();
expect(remote.memory.totalBytes).toBe(4096 * 1024);
},
);
it("does not report zero or invalid memory and disk totals", () => {
const localZero = parseServerHardware(
localEnvelope({
engine: { ncpu: 8, memTotal: 0, arch: "x86_64" },
}),
);
expect(localZero.cpu.count).toBe(8);
expect(localZero.memory.totalBytes).toBeNull();
const localNeg = parseServerHardware(
localEnvelope({
engine: { ncpu: 8, memTotal: -1, arch: "x86_64" },
}),
);
expect(localNeg.memory.totalBytes).toBeNull();
const remote = parseServerHardware(
remoteEnvelope({
memTotalKb: "0",
memAvailKb: "0",
cpuCount: "2",
arch: "x86_64",
diskTotalK: "0",
diskAvailK: "0",
}),
);
expect(remote.cpu.count).toBe(2);
expect(remote.memory.totalBytes).toBeNull();
expect(remote.memory.availableBytes).toBe(0);
expect(remote.disk.totalBytes).toBeNull();
expect(remote.disk.availableBytes).toBe(0);
});
it("does not leak raw probe envelope fields", () => {
const result = parseServerHardware(
localEnvelope({
engine: { ncpu: 2, memTotal: 100, arch: "x86_64" },
gpuExit: 0,
gpuCsv: t4,
}),
);
expect(result).not.toHaveProperty("engineBase64");
expect(result).not.toHaveProperty("gpuBase64");
expect(result).not.toHaveProperty("capBase64");
expect(JSON.stringify(result)).not.toContain("DockerRootDir");
});
it("keeps MemTotal when MemAvailable is malformed", () => {
const result = parseServerHardware(
remoteEnvelope({
memTotalKb: "4096",
memAvailKb: "nope",
cpuCount: "2",
arch: "x86_64",
}),
);
expect(result.memory.totalBytes).toBe(4096 * 1024);
expect(result.memory.availableBytes).toBeNull();
expect(result.cpu.count).toBe(2);
});
it("ignores disk stdout when df exits nonzero", () => {
const result = parseServerHardware(
remoteEnvelope({
memTotalKb: "4096",
cpuCount: "4",
arch: "x86_64",
diskTotalK: "1000000",
diskAvailK: "250000",
diskExit: 1,
}),
);
expect(result.cpu.count).toBe(4);
expect(result.memory.totalBytes).toBe(4096 * 1024);
expect(result.disk).toEqual({ totalBytes: null, availableBytes: null });
});
it("does not turn oversized memory values into Infinity", () => {
const result = parseServerHardware(
remoteEnvelope({
memTotalKb: "1e308",
cpuCount: "2",
arch: "x86_64",
}),
);
expect(result.memory.totalBytes).toBeNull();
expect(result.cpu.count).toBe(2);
});
it("keeps GPU inventory when compute_cap stdout is malformed", () => {
const result = parseServerHardware(
localEnvelope({
engine: { ncpu: 4, memTotal: 1, arch: "x86_64" },
gpuExit: 0,
gpuCsv: t4,
capExit: 0,
capCsv: "not-a-cap-row",
}),
);
expect(result.gpu.detection).toBe("available");
expect(result.gpu.devices).toHaveLength(1);
expect(result.gpu.devices[0]?.computeCapability).toBeNull();
});
it("converts remote meminfo kB and df 1K-blocks to bytes", () => {
const result = parseServerHardware(
remoteEnvelope({
memTotalKb: "8000000",
memAvailKb: "2000000",
cpuCount: "16",
arch: "x86_64",
diskTotalK: "1000000",
diskAvailK: "250000",
}),
);
expect(result.memory).toEqual({
totalBytes: 8000000 * 1024,
availableBytes: 2000000 * 1024,
});
expect(result.disk).toEqual({
totalBytes: 1000000 * 1024,
availableBytes: 250000 * 1024,
});
expect(result.cpu).toEqual({ count: 16, arch: "x86_64" });
expect(result.gpu.detection).toBe("unavailable");
});
it("parses heterogeneous GPUs and skips a malformed row", () => {
const result = parseServerHardware(
remoteEnvelope({
memTotalKb: "1",
cpuCount: "8",
arch: "x86_64",
gpuExit: 0,
gpuCsv: [t4, "bad-line", a100, commaName].join("\n"),
capExit: 0,
capCsv: [t4Cap, a100Cap, commaCap].join("\n"),
}),
);
expect(result.gpu.detection).toBe("available");
expect(result.gpu.devices.map((d) => d.name)).toEqual([
"Tesla T4",
"NVIDIA A100-SXM4-40GB",
"NVIDIA Graphics Device, 16GB",
]);
expect(result.gpu.devices.map((d) => d.computeCapability)).toEqual([
"7.5",
"8.0",
"8.9",
]);
expect(result.gpu.devices[1]?.memoryTotalMiB).toBe(40960);
expect(result.gpu.devices[1]?.memoryFreeMiB).toBe(20000);
expect(result.cpu.count).toBe(8);
});
it("merges compute capability by GPU index, not row order", () => {
const gpu3 = "3, GPU-ddd, NVIDIA RTX 4090, 24576, 24000, 575.57.08";
const result = parseServerHardware(
remoteEnvelope({
cpuCount: "8",
arch: "x86_64",
gpuExit: 0,
gpuCsv: [t4, gpu3].join("\n"),
capExit: 0,
capCsv: ["3, 8.9", "0, 7.5"].join("\n"),
}),
);
expect(result.gpu.detection).toBe("available");
expect(result.gpu.devices).toEqual([
expect.objectContaining({
index: 0,
uuid: "GPU-aaa",
computeCapability: "7.5",
}),
expect.objectContaining({
index: 3,
uuid: "GPU-ddd",
name: "NVIDIA RTX 4090",
computeCapability: "8.9",
}),
]);
});
it("keeps inventory when compute_cap is unsupported", () => {
const result = parseServerHardware(
localEnvelope({
engine: { ncpu: 4, memTotal: 1, arch: "x86_64" },
gpuExit: 0,
gpuCsv: [t4, a100].join("\n"),
capExit: 1,
capCsv: "",
}),
);
expect(result.gpu.detection).toBe("available");
expect(result.gpu.devices).toHaveLength(2);
expect(result.gpu.devices[0]?.name).toBe("Tesla T4");
expect(result.gpu.devices.every((d) => d.computeCapability === null)).toBe(
true,
);
});
it("keeps CPU and memory when GPU detection fails", () => {
const result = parseServerHardware(
remoteEnvelope({
memTotalKb: "4096",
memAvailKb: "1024",
cpuCount: "4",
arch: "aarch64",
diskTotalK: "10",
diskAvailK: "5",
gpuExit: 1,
}),
);
expect(result.cpu.count).toBe(4);
expect(result.memory.totalBytes).toBe(4096 * 1024);
expect(result.disk.totalBytes).toBe(10 * 1024);
expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
expect(result.error).toBeUndefined();
});
it("keeps CPU and memory when disk fields are missing", () => {
const result = parseServerHardware(
remoteEnvelope({
memTotalKb: "2048",
cpuCount: "2",
arch: "x86_64",
}),
);
expect(result.cpu.count).toBe(2);
expect(result.memory.totalBytes).toBe(2048 * 1024);
expect(result.memory.availableBytes).toBeNull();
expect(result.disk).toEqual({ totalBytes: null, availableBytes: null });
});
it("returns error for a malformed envelope", () => {
const result = parseServerHardware("not-json");
expect(result.error).toBe("Could not parse server hardware output");
expect(result.cpu.count).toBeNull();
});
});
describe("hardware probe scripts", () => {
it("reads local Engine JSON and treats missing nvidia-smi as unavailable", () => {
const sandbox = makeSandbox({
docker: `#!/bin/sh
printf '%s\\n' '{"ncpu":8,"memTotal":16000000000,"arch":"x86_64"}'
exit 0
`,
});
const result = parseServerHardware(
runScript(buildLocalHardwareScript(), sandbox),
);
expect(result.cpu).toEqual({ count: 8, arch: "x86_64" });
expect(result.memory.availableBytes).toBeNull();
expect(result.disk.totalBytes).toBeNull();
expect(result.gpu.detection).toBe("unavailable");
expect(result.gpu.devices).toEqual([]);
});
it("parses local nvidia-smi when the binary is present", () => {
const sandbox = makeSandbox({
docker: `#!/bin/sh
printf '%s\\n' '{"ncpu":4,"memTotal":1,"arch":"aarch64"}'
exit 0
`,
"nvidia-smi": `#!/bin/sh
case "$*" in
*compute_cap*)
printf '%s\\n' '${t4Cap}'
exit 0
;;
esac
printf '%s\\n' '${t4}'
exit 0
`,
});
const result = parseServerHardware(
runScript(buildLocalHardwareScript(), sandbox),
);
expect(result.gpu.detection).toBe("available");
expect(result.gpu.devices[0]?.name).toBe("Tesla T4");
expect(result.gpu.devices[0]?.computeCapability).toBe("7.5");
});
it("keeps local inventory when compute_cap query fails", () => {
const sandbox = makeSandbox({
docker: `#!/bin/sh
printf '%s\\n' '{"ncpu":4,"memTotal":1,"arch":"aarch64"}'
exit 0
`,
"nvidia-smi": `#!/bin/sh
case "$*" in
*compute_cap*)
exit 1
;;
esac
printf '%s\\n' '${t4}'
exit 0
`,
});
const result = parseServerHardware(
runScript(buildLocalHardwareScript(), sandbox),
);
expect(result.gpu.detection).toBe("available");
expect(result.gpu.devices).toHaveLength(1);
expect(result.gpu.devices[0]?.name).toBe("Tesla T4");
expect(result.gpu.devices[0]?.computeCapability).toBeNull();
});
it("ignores docker info stdout when the command exits nonzero", () => {
const sandbox = makeSandbox({
docker: `#!/bin/sh
printf '%s\\n' '{"ncpu":8,"memTotal":16000000000,"arch":"x86_64"}'
exit 1
`,
});
const result = parseServerHardware(
runScript(buildLocalHardwareScript(), sandbox),
);
expect(result.cpu).toEqual({ count: null, arch: null });
expect(result.memory.totalBytes).toBeNull();
});
it("ignores nvidia-smi stdout when the inventory command exits nonzero", () => {
const sandbox = makeSandbox({
docker: `#!/bin/sh
printf '%s\\n' '{"ncpu":4,"memTotal":1,"arch":"x86_64"}'
exit 0
`,
"nvidia-smi": `#!/bin/sh
printf '%s\\n' '${t4}'
exit 1
`,
});
const result = parseServerHardware(
runScript(buildLocalHardwareScript(), sandbox),
);
expect(result.cpu.count).toBe(4);
expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
});
it("reads remote host facts through the generated script", () => {
const sandbox = makeSandbox({
awk: `#!/bin/sh
case "$*" in
*MemTotal*)
printf '%s\\n' '8000000'
exit 0
;;
*MemAvailable*)
printf '%s\\n' '2000000'
exit 0
;;
esac
exec ${resolveBin("awk")} "$@"
`,
nproc: `#!/bin/sh
printf '%s\\n' '16'
exit 0
`,
uname: `#!/bin/sh
printf '%s\\n' 'x86_64'
exit 0
`,
df: `#!/bin/sh
case " $* " in
*" -Pk "*) ;;
*)
printf '%s\\n' 'df: expected -Pk' >&2
exit 2
;;
esac
printf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
printf '%s\\n' '/dev/sda1 1000000 750000 250000 75% /'
exit 0
`,
});
const result = parseServerHardware(
runScript(buildRemoteHardwareScript(), sandbox),
);
expect(result.cpu).toEqual({ count: 16, arch: "x86_64" });
expect(result.memory).toEqual({
totalBytes: 8000000 * 1024,
availableBytes: 2000000 * 1024,
});
expect(result.disk).toEqual({
totalBytes: 1000000 * 1024,
availableBytes: 250000 * 1024,
});
expect(result.gpu.detection).toBe("unavailable");
});
it("keeps remote CPU and memory when df prints a table then exits 1", () => {
const sandbox = makeSandbox({
awk: `#!/bin/sh
case "$*" in
*MemTotal*)
printf '%s\\n' '4096'
exit 0
;;
*MemAvailable*)
printf '%s\\n' '1024'
exit 0
;;
esac
exec ${resolveBin("awk")} "$@"
`,
nproc: `#!/bin/sh
printf '%s\\n' '4'
exit 0
`,
uname: `#!/bin/sh
printf '%s\\n' 'aarch64'
exit 0
`,
df: `#!/bin/sh
printf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
printf '%s\\n' '/dev/sda1 1000000 750000 250000 75% /'
exit 1
`,
});
const result = parseServerHardware(
runScript(buildRemoteHardwareScript(), sandbox),
);
expect(result.cpu).toEqual({ count: 4, arch: "aarch64" });
expect(result.memory.totalBytes).toBe(4096 * 1024);
expect(result.disk).toEqual({ totalBytes: null, availableBytes: null });
});
it("falls back to cpuinfo when nproc is missing", () => {
const sandbox = makeSandbox({
awk: `#!/bin/sh
case "$*" in
*MemTotal*)
printf '%s\\n' '2048'
exit 0
;;
esac
exec ${resolveBin("awk")} "$@"
`,
grep: `#!/bin/sh
case "$*" in
*processor*)
printf '%s\\n' '8'
exit 0
;;
esac
exit 1
`,
uname: `#!/bin/sh
printf '%s\\n' 'x86_64'
exit 0
`,
df: `#!/bin/sh
printf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
printf '%s\\n' '/dev/sda1 10 5 5 50% /'
exit 0
`,
});
const result = parseServerHardware(
runScript(buildRemoteHardwareScript(), sandbox),
);
expect(result.cpu.count).toBe(8);
expect(result.memory.totalBytes).toBe(2048 * 1024);
});
});
describe("getServerHardware", () => {
beforeEach(() => {
cloud.enabled = false;
vi.mocked(execAsync).mockReset();
vi.mocked(execAsyncRemote).mockReset();
});
it("uses execAsync for the local path and never execAsyncRemote", async () => {
vi.mocked(execAsync).mockResolvedValue({
stdout: localEnvelope({
engine: { ncpu: 8, memTotal: 100, arch: "x86_64" },
}),
stderr: "",
});
const result = await getServerHardware();
expect(execAsync).toHaveBeenCalledWith(
expect.stringContaining("docker info --format"),
expect.objectContaining({ timeout: HARDWARE_PROBE_TIMEOUT_MS }),
);
expect(execAsyncRemote).not.toHaveBeenCalled();
expect(result.cpu.count).toBe(8);
expect(result.memory.availableBytes).toBeNull();
});
it("uses execAsyncRemote for a remote serverId and never execAsync", async () => {
vi.mocked(execAsyncRemote).mockResolvedValue({
stdout: remoteEnvelope({
memTotalKb: "1000",
cpuCount: "2",
arch: "x86_64",
gpuExit: 0,
gpuCsv: t4,
}),
stderr: "",
});
const result = await getServerHardware("remote-server");
expect(execAsyncRemote).toHaveBeenCalledWith(
"remote-server",
expect.stringContaining("/proc/meminfo"),
undefined,
expect.objectContaining({ timeout: HARDWARE_PROBE_TIMEOUT_MS }),
);
expect(execAsync).not.toHaveBeenCalled();
expect(result.gpu.devices[0]?.uuid).toBe("GPU-aaa");
});
it("rejects a local probe on cloud without executing commands", async () => {
cloud.enabled = true;
await expect(getServerHardware()).rejects.toMatchObject({
name: "TRPCError",
code: "BAD_REQUEST",
message: "Server is required",
});
expect(execAsync).not.toHaveBeenCalled();
expect(execAsyncRemote).not.toHaveBeenCalled();
});
it("returns empty hardware plus error when SSH fails", async () => {
vi.mocked(execAsyncRemote).mockRejectedValue(new Error("SSH failed"));
const result = await getServerHardware("remote-server");
expect(result.error).toBe("SSH failed");
expect(result.cpu.count).toBeNull();
expect(result.gpu.detection).toBe("unavailable");
expect(execAsync).not.toHaveBeenCalled();
});
it("treats a timed-out probe as empty hardware, not zero GPUs", async () => {
vi.mocked(execAsync).mockRejectedValue(
new Error("Command execution timed out after 30000ms"),
);
const result = await getServerHardware();
expect(result.error).toContain("timed out");
expect(result.cpu.count).toBeNull();
expect(result.memory.totalBytes).toBeNull();
expect(result.disk.totalBytes).toBeNull();
expect(result.gpu).toEqual({ detection: "unavailable", devices: [] });
});
});
describe("hardware timeout isolation regressions", () => {
it.each(["inventory", "compute", "engine"])(
"preserves independent local facts when %s hangs",
async (hung) => {
const actual = await vi.importActual<
typeof import("@dokploy/server/utils/process/execAsync")
>("@dokploy/server/utils/process/execAsync");
const sandbox = makeSandbox({
docker: `#!/bin/sh
${hung === "engine" ? "/bin/sleep 60" : ""}
printf '%s' '{"ncpu":8,"memTotal":16000000000,"arch":"x86_64"}'
`,
"nvidia-smi": `#!/bin/sh
case "$*" in
*compute_cap*) ${hung === "compute" ? "/bin/sleep 60" : ""}
printf '%s' '0, 7.5';;
*) ${hung === "inventory" ? "/bin/sleep 60" : ""}
printf '%s' '${t4}';;
esac
`,
});
vi.mocked(execAsync).mockImplementation((command, options) => {
expect(options?.timeout).toBeGreaterThan(0);
expect(options?.timeout).toBeLessThanOrEqual(30_000);
return actual.execAsync(command, {
...options,
timeout: 3_000,
env: { NODE_ENV: "test", PATH: sandbox },
});
});
const result = await getServerHardware();
expect(result.error).toContain("timed out");
expect(result.cpu.count).toBe(hung === "engine" ? null : 8);
expect(result.memory.totalBytes).toBe(
hung === "engine" ? null : 16_000_000_000,
);
expect(result.gpu.devices).toHaveLength(hung === "inventory" ? 0 : 1);
if (hung === "compute")
expect(result.gpu.devices[0]?.computeCapability).toBeNull();
},
);
it.each(["disk", "inventory", "compute", "cpu", "memory", "arch"])(
"preserves remote sibling facts when %s hangs",
async (hung) => {
const actual = await vi.importActual<
typeof import("@dokploy/server/utils/process/execAsync")
>("@dokploy/server/utils/process/execAsync");
const sandbox = makeSandbox({
awk: `#!/bin/sh
case "$*" in
*MemTotal*) ${hung === "memory" ? "/bin/sleep 60" : ""}; printf '4096';;
*MemAvailable*) printf '1024';;
*) exec ${resolveBin("awk")} "$@";;
esac
`.replace("; printf", "\nprintf"),
nproc: `#!/bin/sh
${hung === "cpu" ? "/bin/sleep 60" : ""}
printf '4'
`,
uname: `#!/bin/sh
${hung === "arch" ? "/bin/sleep 60" : ""}
printf 'aarch64'
`,
df: `#!/bin/sh
${hung === "disk" ? "/bin/sleep 60" : ""}
printf '%s\\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on' '/dev/test 1000 750 250 75% /'
`,
"nvidia-smi": `#!/bin/sh
case "$*" in
*compute_cap*) ${hung === "compute" ? "/bin/sleep 60" : ""}
printf '0, 7.5';;
*) ${hung === "inventory" ? "/bin/sleep 60" : ""}
printf '%s' '${t4}';;
esac
`,
});
vi.mocked(execAsyncRemote).mockImplementation(
(_id, command, _onData, options) => {
expect(options?.timeout).toBeGreaterThan(0);
expect(options?.timeout).toBeLessThanOrEqual(30_000);
return actual.execAsync(command, {
timeout: 3_000,
env: { NODE_ENV: "test", PATH: sandbox },
});
},
);
const result = await getServerHardware("fixture-server");
expect(result.error).toContain("timed out");
expect(result.cpu).toEqual({
count: hung === "cpu" ? null : 4,
arch: hung === "arch" ? null : "aarch64",
});
expect(result.memory.totalBytes).toBe(
hung === "memory" ? null : 4096 * 1024,
);
expect(result.disk.totalBytes).toBe(hung === "disk" ? null : 1000 * 1024);
expect(result.gpu.devices).toHaveLength(hung === "inventory" ? 0 : 1);
if (hung === "compute")
expect(result.gpu.devices[0]?.computeCapability).toBeNull();
},
);
it("does not report malformed nonempty inventory as successfully empty", () => {
expect(
parseServerHardware(localEnvelope({ gpuExit: 0, gpuCsv: "malformed" }))
.gpu.detection,
).toBe("unavailable");
});
it("leaves unsupported compute capability unknown", () => {
expect(
parseServerHardware(
localEnvelope({
gpuExit: 0,
gpuCsv: t4,
capExit: 0,
capCsv: "0, [N/A]",
}),
).gpu.devices[0]?.computeCapability,
).toBeNull();
});
});

36
apps/dokploy/server/api/routers/docker.ts
View File

@ -11,6 +11,8 @@ import {
getContainersByAppLabel,
getContainersByAppNameMatch,
getDockerEvents,
getModelRunnerCapability as getModelRunnerCapabilityData,
getServerHardware as getServerHardwareData,
getServerHealth as getServerHealthData,
getServiceContainersByAppName,
getStackContainersByAppName,
@ -77,6 +79,40 @@ export const dockerRouter = createTRPCRouter({
);
}),
getServerHardware: protectedProcedure
.input(
z.object({
serverId: z.string().optional(),
}),
)
.query(async ({ input, ctx }) => {
await checkPermission(ctx, { docker: ["read"], server: ["read"] });
if (input.serverId) {
const server = await findServerById(input.serverId);
if (server.organizationId !== ctx.session?.activeOrganizationId) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
}
return await getServerHardwareData(input.serverId);
}),
getModelRunnerCapability: protectedProcedure
.input(
z.object({
serverId: z.string().optional(),
}),
)
.query(async ({ input, ctx }) => {
await checkPermission(ctx, { docker: ["read"], server: ["read"] });
if (input.serverId) {
const server = await findServerById(input.serverId);
if (server.organizationId !== ctx.session?.activeOrganizationId) {
throw new TRPCError({ code: "UNAUTHORIZED" });
}
}
return await getModelRunnerCapabilityData(input.serverId);
}),
restartContainer: withPermission("docker", "read")
.input(
z.object({

8
packages/server/src/index.ts
View File

@ -28,6 +28,10 @@ export * from "./services/github";
export * from "./services/gitlab";
export * from "./services/libsql";
export * from "./services/mariadb";
export {
getModelRunnerCapability,
type ModelRunnerCapability,
} from "./services/model-runner";
export * from "./services/mongo";
export * from "./services/mount";
export * from "./services/mysql";
@ -51,6 +55,10 @@ export * from "./services/rollbacks";
export * from "./services/schedule";
export * from "./services/security";
export * from "./services/server";
export {
getServerHardware,
type ServerHardware,
} from "./services/server-hardware";
export * from "./services/server-health";
export * from "./services/settings";
export * from "./services/ssh-key";

221
packages/server/src/services/model-runner.ts Normal file
View File

@ -0,0 +1,221 @@
import {
execAsync,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { TRPCError } from "@trpc/server";
import semver from "semver";
import { z } from "zod";
import { IS_CLOUD } from "../constants";
const COMPOSE_MODELS_MIN_VERSION = "2.38.0";
export const MODEL_RUNNER_PROBE_TIMEOUT_MS = 30_000;
export interface ModelRunnerCapability {
checkedAt: string;
docker: {
/** True only after a successful, usable Engine info response. */
available: boolean;
version: string | null;
os: string | null;
arch: string | null;
};
compose: {
/** Null means discovery could not establish availability. */
available: boolean | null;
version: string | null;
modelsSupported: boolean | null;
error?: string;
};
modelRunner: {
/** CLI availability is independent of the Engine's runner container. */
cliAvailable: boolean | null;
cliVersion: string | null;
/** Null means no status was observed, not necessarily an absent runner. */
standaloneRunnerContainerStatus: string | null;
error?: string;
};
error?: string;
}
const envelopeSchema = z.object({
dockerPresent: z.boolean(),
engineExit: z.number().int(),
engineBase64: z.string(),
pluginsExit: z.number().int(),
pluginsBase64: z.string(),
containerStatus: z.string(),
});
const engineSchema = z.object({
serverVersion: z.string().trim().min(1),
os: z.string().nullish(),
arch: z.string().nullish(),
});
const pluginSchema = z.object({
Name: z.string().min(1),
Version: z.string().nullish(),
Err: z.unknown().optional(),
});
const metadataSchema = z.object({
plugins: z.array(pluginSchema).nullable(),
errors: z.array(z.string()).nullable(),
});
const nullIfEmpty = (value: string | null | undefined): string | null =>
value?.trim() || null;
export const composeModelsSupported = (
version: string | null,
): boolean | null => {
if (!version) return null;
const parsed = semver.clean(version);
return parsed ? semver.gte(parsed, COMPOSE_MODELS_MIN_VERSION) : null;
};
const parseJson = (text: string): unknown => {
try {
return JSON.parse(text);
} catch {
return undefined;
}
};
const readPlugin = (
plugins: z.infer<typeof pluginSchema>[] | null,
name: string,
) => {
if (plugins === null) return { available: null, version: null };
const plugin = plugins.find((entry) => entry.Name === name);
if (!plugin) return { available: false, version: null };
if (plugin.Err != null && plugin.Err !== "") {
return {
available: false,
version: null,
error: `Docker ${name} CLI plugin is invalid`,
};
}
return { available: true, version: nullIfEmpty(plugin.Version) };
};
const emptyCapability = (error?: string): ModelRunnerCapability => ({
checkedAt: new Date().toISOString(),
docker: { available: false, version: null, os: null, arch: null },
compose: { available: null, version: null, modelsSupported: null },
modelRunner: {
cliAvailable: null,
cliVersion: null,
standaloneRunnerContainerStatus: null,
},
...(error ? { error } : {}),
});
/**
* Read-only capability probe. Engine reachability and CLI plugin metadata are
* separate `docker info` templates so an older CLI missing ClientInfo.Plugins
* cannot be reported as an unreachable engine. Plugin discovery uses Docker's
* metadata subcommand. No install/configuration/model-lifecycle/server-mutation
* commands. Each docker info stdout is captured before base64 so exit status
* is preserved.
*/
export const buildModelRunnerScript = () => `
dockerPresent=false
engineExit=0
engineB64=""
pluginsExit=0
pluginsB64=""
containerStatus=""
if command -v docker >/dev/null 2>&1; then
dockerPresent=true
engineOutput=$(docker info --format '{"serverVersion":{{json .ServerVersion}},"os":{{json .OSType}},"arch":{{json .Architecture}}}' 2>/dev/null)
engineExit=$?
engineB64=$(printf '%s' "$engineOutput" | base64 2>/dev/null | tr -d '\\n')
pluginsOutput=$(docker info --format '{"plugins":{{json .ClientInfo.Plugins}},"errors":{{json .ClientErrors}}}' 2>/dev/null)
pluginsExit=$?
pluginsB64=$(printf '%s' "$pluginsOutput" | base64 2>/dev/null | tr -d '\\n')
containerStatus=$(docker inspect -f '{{.State.Status}}' docker-model-runner 2>/dev/null | tr -d '\\n')
fi
printf '{"dockerPresent":%s,"engineExit":%s,"engineBase64":"%s","pluginsExit":%s,"pluginsBase64":"%s","containerStatus":"%s"}' "$dockerPresent" "$engineExit" "$engineB64" "$pluginsExit" "$pluginsB64" "$containerStatus"
`;
export const parseModelRunnerCapability = (
stdout: string,
): ModelRunnerCapability => {
const parsed = envelopeSchema.safeParse(parseJson(stdout));
if (!parsed.success)
return emptyCapability("Could not parse model runner probe output");
const envelope = parsed.data;
const result = emptyCapability();
if (!envelope.dockerPresent) {
result.compose = {
available: false,
version: null,
modelsSupported: false,
};
result.modelRunner.cliAvailable = false;
return result;
}
const errors: string[] = [];
const engine = engineSchema.safeParse(
parseJson(Buffer.from(envelope.engineBase64, "base64").toString("utf8")),
);
if (envelope.engineExit === 0 && engine.success) {
result.docker = {
available: true,
version: engine.data.serverVersion,
os: nullIfEmpty(engine.data.os),
arch: nullIfEmpty(engine.data.arch),
};
} else {
errors.push("Could not read Docker Engine info");
}
const metadata = metadataSchema.safeParse(
parseJson(Buffer.from(envelope.pluginsBase64, "base64").toString("utf8")),
);
let plugins: z.infer<typeof pluginSchema>[] | null = null;
if (
envelope.pluginsExit !== 0 ||
(metadata.success && metadata.data.errors?.length)
) {
errors.push("Could not discover Docker CLI plugins");
} else if (!metadata.success) {
errors.push("Could not parse Docker CLI plugin metadata");
} else {
plugins = metadata.data.plugins ?? [];
}
const compose = readPlugin(plugins, "compose");
const model = readPlugin(plugins, "model");
result.compose = {
...compose,
modelsSupported:
compose.available === false
? false
: composeModelsSupported(compose.version),
};
result.modelRunner = {
cliAvailable: model.available,
cliVersion: model.version,
standaloneRunnerContainerStatus: nullIfEmpty(envelope.containerStatus),
...(model.error ? { error: model.error } : {}),
};
if (errors.length) result.error = errors.join("; ");
return result;
};
export const getModelRunnerCapability = async (
serverId?: string,
): Promise<ModelRunnerCapability> => {
if (IS_CLOUD && !serverId) {
throw new TRPCError({ code: "BAD_REQUEST", message: "Server is required" });
}
const script = buildModelRunnerScript();
try {
const options = { timeout: MODEL_RUNNER_PROBE_TIMEOUT_MS };
const result = serverId
? await execAsyncRemote(serverId, script, undefined, options)
: await execAsync(script, options);
return parseModelRunnerCapability(result.stdout);
} catch {
return emptyCapability("Could not read model runner capability");
}
};

56
packages/server/src/services/server-hardware-scripts.ts Normal file
View File

@ -0,0 +1,56 @@
const NVIDIA_INVENTORY_QUERY =
"nvidia-smi --query-gpu=index,uuid,name,memory.total,memory.free,driver_version --format=csv,noheader,nounits";
const NVIDIA_COMPUTE_CAP_QUERY =
"nvidia-smi --query-gpu=index,compute_cap --format=csv,noheader,nounits";
// Each invocation emits one independent result and receives its own executor deadline.
const engine = `
engineOutput=$(docker info --format '{"ncpu":{{json .NCPU}},"memTotal":{{json .MemTotal}},"arch":{{json .Architecture}}}' 2>/dev/null)
engineExit=$?
engineB64=$(printf '%s' "$engineOutput" | base64 2>/dev/null | tr -d '\\n')
printf '{"engineExit":%s,"engineBase64":"%s"}\\n' "$engineExit" "$engineB64"
`;
const gpu = `
gpuCsv=$(${NVIDIA_INVENTORY_QUERY} 2>/dev/null)
gpuExit=$?
gpuB64=$(printf '%s' "$gpuCsv" | base64 2>/dev/null | tr -d '\\n')
printf '{"gpuExit":%s,"gpuBase64":"%s"}\\n' "$gpuExit" "$gpuB64"
`;
const cap = `
capCsv=$(${NVIDIA_COMPUTE_CAP_QUERY} 2>/dev/null)
capExit=$?
capB64=$(printf '%s' "$capCsv" | base64 2>/dev/null | tr -d '\\n')
printf '{"capExit":%s,"capBase64":"%s"}\\n' "$capExit" "$capB64"
`;
const memory = `
memTotalKb=$(awk '/^MemTotal:/{print $2}' /proc/meminfo 2>/dev/null)
memAvailKb=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo 2>/dev/null)
printf '{"memTotalKb":"%s","memAvailKb":"%s"}\\n' "$memTotalKb" "$memAvailKb"
`;
const cpu = `
cpuCount=$(nproc 2>/dev/null)
if [ -z "$cpuCount" ]; then
cpuCount=$(grep -c '^processor' /proc/cpuinfo 2>/dev/null)
fi
printf '{"cpuCount":"%s"}\\n' "$cpuCount"
`;
const arch = `
arch=$(uname -m 2>/dev/null)
printf '{"arch":"%s"}\\n' "$arch"
`;
const disk = `
diskOutput=$(df -Pk / 2>/dev/null)
diskExit=$?
diskTotalK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $2}')
diskAvailK=$(printf '%s\\n' "$diskOutput" | awk 'NR==2{print $4}')
printf '{"diskExit":%s,"diskTotalK":"%s","diskAvailK":"%s"}\\n' "$diskExit" "$diskTotalK" "$diskAvailK"
`;
export const buildHardwareScripts = (remote: boolean): readonly string[] =>
remote ? [memory, cpu, arch, disk, gpu, cap] : [engine, gpu, cap];

366
packages/server/src/services/server-hardware.ts Normal file
View File

@ -0,0 +1,366 @@
import {
execAsync,
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { TRPCError } from "@trpc/server";
import { IS_CLOUD } from "../constants";
import { buildHardwareScripts } from "./server-hardware-scripts";
// Matches remoteStream SSH readyTimeout. Hardware commands are fast; this
// bounds stalled docker/nvidia-smi/df/SSH so the API cannot wait forever.
export const HARDWARE_PROBE_TIMEOUT_MS = 30_000;
export interface ServerHardwareGpu {
index: number;
uuid: string | null;
name: string;
vendor: "nvidia";
computeCapability: string | null;
memoryTotalMiB: number | null;
memoryFreeMiB: number | null;
driverVersion: string | null;
}
export interface ServerHardware {
checkedAt: string;
cpu: {
count: number | null;
arch: string | null;
};
memory: {
totalBytes: number | null;
availableBytes: number | null;
};
/**
* Remote values come from `df -Pk /` on the SSH host (root filesystem,
* 1024-byte blocks). That is not Docker/model storage capacity. Local
* values are always null.
*/
disk: {
totalBytes: number | null;
availableBytes: number | null;
};
gpu: {
detection: "available" | "unavailable";
devices: ServerHardwareGpu[];
};
error?: string;
}
interface HardwareEnvelope {
kind?: string;
engineExit?: number;
engineBase64?: string;
gpuExit?: number;
gpuBase64?: string;
capExit?: number;
capBase64?: string;
memTotalKb?: string;
memAvailKb?: string;
cpuCount?: string;
arch?: string;
diskTotalK?: string;
diskAvailK?: string;
diskExit?: number;
}
interface EngineInfo {
ncpu?: number | null;
memTotal?: number | null;
arch?: string | null;
}
const emptyHardware = (error?: unknown): ServerHardware => ({
checkedAt: new Date().toISOString(),
cpu: { count: null, arch: null },
memory: { totalBytes: null, availableBytes: null },
disk: { totalBytes: null, availableBytes: null },
gpu: { detection: "unavailable", devices: [] },
...(error !== undefined
? {
error:
error instanceof Error
? error.message
: typeof error === "string"
? error
: "Could not read server hardware",
}
: {}),
});
const nullIfEmpty = (value: unknown): string | null => {
const trimmed = typeof value === "string" ? value.trim() : "";
return trimmed ? trimmed : null;
};
const parseFinite = (value: string | null | undefined): number | null => {
if (value == null || value.trim() === "") return null;
const n = Number(value.trim());
return Number.isFinite(n) ? n : null;
};
const parseNonNegativeFinite = (
value: string | null | undefined,
): number | null => {
const n = parseFinite(value);
return n != null && n >= 0 && n <= Number.MAX_SAFE_INTEGER ? n : null;
};
const parseNonNegativeInt = (
value: string | null | undefined,
): number | null => {
const trimmed = value?.trim() ?? "";
if (!/^\d+$/.test(trimmed)) return null;
const n = Number(trimmed);
return Number.isInteger(n) && n >= 0 && n <= Number.MAX_SAFE_INTEGER
? n
: null;
};
const parsePositiveInt = (value: unknown): number | null => {
if (typeof value === "number") {
return Number.isInteger(value) &&
value > 0 &&
value <= Number.MAX_SAFE_INTEGER
? value
: null;
}
const n = parseNonNegativeInt(String(value ?? ""));
return n != null && n > 0 ? n : null;
};
const parseExitCode = (value: unknown, whenMissing: number): number => {
if (value === 0 || value === "0") return 0;
if (typeof value === "number") {
return Number.isInteger(value) && value >= 0 ? value : whenMissing;
}
if (typeof value === "string" && /^\d+$/.test(value.trim())) {
return Number(value.trim());
}
return whenMissing;
};
const kbToBytes = (kb: string | null | undefined): number | null => {
const n = parseNonNegativeFinite(kb);
if (n == null) return null;
const bytes = Math.round(n * 1024);
return Number.isSafeInteger(bytes) ? bytes : null;
};
const positiveBytes = (value: number | null): number | null =>
value != null && value > 0 && Number.isSafeInteger(value) ? value : null;
const engineMemTotalBytes = (value: unknown): number | null => {
if (typeof value !== "number" || !Number.isFinite(value)) return null;
return positiveBytes(Math.round(value));
};
const b64Decode = (value?: string): string => {
if (!value) return "";
try {
return Buffer.from(value, "base64").toString("utf-8");
} catch {
return "";
}
};
const parseJson = (text: string): unknown => {
try {
return JSON.parse(text);
} catch {
return null;
}
};
/**
* Primary inventory csv,noheader,nounits:
* index, uuid, name (may contain commas), memory.total, memory.free, driver_version
*/
export const parseNvidiaGpuLine = (line: string): ServerHardwareGpu | null => {
const parts = line.split(",").map((part) => part.trim());
if (parts.length < 6) return null;
const index = parseNonNegativeInt(parts[0]);
if (index == null) return null;
const uuid = nullIfEmpty(parts[1]);
const driverVersion = nullIfEmpty(parts[parts.length - 1]);
const memoryFreeMiB = parseNonNegativeFinite(parts[parts.length - 2]);
const memoryTotal = parseNonNegativeFinite(parts[parts.length - 3]);
const name = parts
.slice(2, parts.length - 3)
.join(", ")
.trim();
if (!name) return null;
return {
index,
uuid,
name,
vendor: "nvidia",
computeCapability: null,
memoryTotalMiB: memoryTotal != null && memoryTotal > 0 ? memoryTotal : null,
memoryFreeMiB,
driverVersion,
};
};
export const parseComputeCapLine = (
line: string,
): { index: number; computeCapability: string } | null => {
const parts = line.split(",").map((part) => part.trim());
if (parts.length < 2) return null;
const index = parseNonNegativeInt(parts[0]);
const computeCapability = nullIfEmpty(parts.slice(1).join(","));
if (
index == null ||
!computeCapability ||
!/^\d+\.\d+$/.test(computeCapability)
)
return null;
return { index, computeCapability };
};
const csvLines = (csv: string) =>
csv
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
export const parseNvidiaGpuCsv = (csv: string): ServerHardwareGpu[] =>
csvLines(csv)
.map(parseNvidiaGpuLine)
.filter((row): row is ServerHardwareGpu => row !== null);
export const mergeComputeCapabilities = (
devices: ServerHardwareGpu[],
capCsv: string,
): ServerHardwareGpu[] => {
const caps = new Map<number, string>();
for (const line of csvLines(capCsv)) {
const parsed = parseComputeCapLine(line);
if (parsed) caps.set(parsed.index, parsed.computeCapability);
}
return devices.map((device) => ({
...device,
computeCapability: caps.get(device.index) ?? null,
}));
};
const gpuFromProbe = (
gpuExit: number,
gpuCsv: string,
capExit: number,
capCsv: string,
): ServerHardware["gpu"] => {
if (gpuExit !== 0) {
return { detection: "unavailable", devices: [] };
}
const devices = parseNvidiaGpuCsv(gpuCsv);
return {
detection:
gpuCsv.trim() && devices.length === 0 ? "unavailable" : "available",
devices:
capExit === 0 ? mergeComputeCapabilities(devices, capCsv) : devices,
};
};
export const parseServerHardware = (stdout: string): ServerHardware => {
let envelope: HardwareEnvelope;
try {
envelope = JSON.parse(stdout.trim());
} catch {
return emptyHardware(new Error("Could not parse server hardware output"));
}
const gpu = gpuFromProbe(
parseExitCode(envelope.gpuExit, 1),
b64Decode(envelope.gpuBase64),
parseExitCode(envelope.capExit, 1),
b64Decode(envelope.capBase64),
);
if (envelope.kind === "local") {
const engineExit = parseExitCode(envelope.engineExit, 1);
const engine =
engineExit === 0
? (parseJson(
b64Decode(envelope.engineBase64).trim(),
) as EngineInfo | null)
: null;
return {
checkedAt: new Date().toISOString(),
cpu: {
count: parsePositiveInt(engine?.ncpu),
arch: nullIfEmpty(engine?.arch ?? undefined),
},
memory: {
totalBytes: engineMemTotalBytes(engine?.memTotal),
availableBytes: null,
},
disk: { totalBytes: null, availableBytes: null },
gpu,
};
}
const diskExit = parseExitCode(envelope.diskExit, 0);
return {
checkedAt: new Date().toISOString(),
cpu: {
count: parsePositiveInt(envelope.cpuCount),
arch: nullIfEmpty(envelope.arch),
},
memory: {
totalBytes: positiveBytes(kbToBytes(envelope.memTotalKb)),
availableBytes: kbToBytes(envelope.memAvailKb),
},
disk:
diskExit === 0
? {
totalBytes: positiveBytes(kbToBytes(envelope.diskTotalK)),
availableBytes: kbToBytes(envelope.diskAvailK),
}
: { totalBytes: null, availableBytes: null },
gpu,
};
};
export const getServerHardware = async (
serverId?: string,
): Promise<ServerHardware> => {
if (IS_CLOUD && !serverId) {
throw new TRPCError({
code: "BAD_REQUEST",
message: "Server is required",
});
}
const results = await Promise.allSettled(
buildHardwareScripts(Boolean(serverId)).map(async (script) => {
const result = serverId
? await execAsyncRemote(serverId, script, undefined, {
timeout: HARDWARE_PROBE_TIMEOUT_MS,
})
: await execAsync(script, { timeout: HARDWARE_PROBE_TIMEOUT_MS });
const parsed: unknown = JSON.parse(result.stdout);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
throw new Error("Could not parse server hardware output");
}
return parsed;
}),
);
const fields = {};
const errors: string[] = [];
for (const result of results) {
if (result.status === "fulfilled") Object.assign(fields, result.value);
else
errors.push(
result.reason instanceof Error
? result.reason.message
: "Could not read server hardware",
);
}
return {
...parseServerHardware(
JSON.stringify({ ...fields, kind: serverId ? "remote" : "local" }),
),
...(errors.length ? { error: [...new Set(errors)].join("; ") } : {}),
};
};

425
packages/server/src/utils/process/execAsync.ts
View File

@ -1,8 +1,9 @@
import { exec, execFile } from "node:child_process";
import { type ChildProcess, exec, execFile, spawn } from "node:child_process";
import util from "node:util";
import { findServerById } from "@dokploy/server/services/server";
import { Client } from "ssh2";
import { Client, type ClientChannel } from "ssh2";
import { ExecError } from "./ExecError";
import { withRemoteDeadline } from "./remote-deadline";
export class WriteFileRemoteError extends Error {
constructor(
@ -23,12 +24,119 @@ export { ExecError } from "./ExecError";
const execAsyncBase = util.promisify(exec);
export interface ExecAsyncOptions {
cwd?: string;
env?: NodeJS.ProcessEnv;
shell?: string;
timeout?: number;
}
export interface ExecAsyncRemoteOptions {
timeout?: number;
}
const killProcessGroup = (child: ChildProcess) => {
if (child.pid == null) return;
if (process.platform === "win32") {
child.kill("SIGKILL");
return;
}
try {
process.kill(-child.pid, "SIGKILL");
} catch {
try {
child.kill("SIGKILL");
} catch {
// already exited
}
}
};
const execAsyncTimed = (
command: string,
options: ExecAsyncOptions & { timeout: number },
): Promise<{ stdout: string; stderr: string }> => {
const { timeout, cwd, env, shell } = options;
const isWin = process.platform === "win32";
const shellPath =
shell ?? (isWin ? process.env.ComSpec || "cmd.exe" : "/bin/sh");
const args = isWin ? ["/d", "/s", "/c", command] : ["-c", command];
return new Promise((resolve, reject) => {
let stdout = "";
let stderr = "";
let settled = false;
const child = spawn(shellPath, args, {
cwd,
env,
detached: !isWin,
stdio: ["ignore", "pipe", "pipe"],
});
const timer = setTimeout(() => {
killProcessGroup(child);
finish(
new ExecError(`Command execution timed out after ${timeout}ms`, {
command,
stdout,
stderr,
}),
);
}, timeout);
const finish = (error?: ExecError) => {
if (settled) return;
settled = true;
clearTimeout(timer);
if (error) reject(error);
else resolve({ stdout, stderr });
};
child.stdout?.on("data", (data) => {
stdout += data.toString();
});
child.stderr?.on("data", (data) => {
stderr += data.toString();
});
child.on("error", (error) => {
finish(
new ExecError(`Command execution failed: ${error.message}`, {
command,
stdout,
stderr,
originalError: error,
}),
);
});
child.on("close", (code) => {
if (code === 0) {
finish();
return;
}
finish(
new ExecError(`Command execution failed: Command failed: ${command}`, {
command,
stdout,
stderr,
exitCode: code ?? undefined,
}),
);
});
});
};
export const execAsync = async (
command: string,
options?: { cwd?: string; env?: NodeJS.ProcessEnv; shell?: string },
options?: ExecAsyncOptions,
): Promise<{ stdout: string; stderr: string }> => {
const timeout = options?.timeout;
if (timeout != null && Number.isFinite(timeout) && timeout > 0) {
return execAsyncTimed(command, { ...options, timeout });
}
const { timeout: _timeout, ...execOptions } = options ?? {};
try {
const result = await execAsyncBase(command, options);
const result = await execAsyncBase(command, execOptions);
return {
stdout: result.stdout.toString(),
stderr: result.stderr.toString(),
@ -156,6 +264,7 @@ export const execAsyncRemote = async (
serverId: string | null,
command: string,
onData?: (data: string) => void,
options?: ExecAsyncRemoteOptions,
): Promise<{ stdout: string; stderr: string }> => {
if (!serverId) return { stdout: "", stderr: "" };
const server = await findServerById(serverId);
@ -163,102 +272,238 @@ export const execAsyncRemote = async (
let stdout = "";
let stderr = "";
const requestedTimeout = options?.timeout;
const timeoutMs =
requestedTimeout != null &&
Number.isFinite(requestedTimeout) &&
requestedTimeout > 0
? requestedTimeout
: undefined;
const started = Date.now();
return new Promise((resolve, reject) => {
const conn = new Client();
let stream: ClientChannel | undefined;
let settled = false;
let timer: ReturnType<typeof setTimeout> | undefined;
sleep(1000);
conn
.once("ready", () => {
conn.exec(command, (err, stream) => {
if (err) {
onData?.(err.message);
reject(
new ExecError(`Remote command execution failed: ${err.message}`, {
const closeStream = (channel?: ClientChannel) => {
if (timeoutMs !== undefined) {
try {
channel?.signal("KILL");
} catch {
/* remote watchdog owns cleanup */
}
}
try {
channel?.close();
} catch {
/* already closed */
}
try {
channel?.destroy();
} catch {
/* already destroyed */
}
};
const settle = (
error?: ExecError,
value?: { stdout: string; stderr: string },
) => {
if (settled) return;
settled = true;
if (timer !== undefined) {
clearTimeout(timer);
timer = undefined;
}
if (error) {
closeStream(stream);
try {
conn.end();
} catch {
// already ended
}
try {
conn.destroy();
} catch {
// already destroyed
}
reject(error);
return;
}
try {
conn.end();
} catch {
// already ended
}
resolve(value ?? { stdout, stderr });
};
if (timeoutMs != null && timeoutMs > 0) {
timer = setTimeout(() => {
settle(
new ExecError(`Command execution timed out after ${timeoutMs}ms`, {
command,
stdout,
stderr,
serverId,
}),
);
}, timeoutMs);
}
try {
conn
.once("ready", () => {
if (settled) return;
const remoteCommand =
timeoutMs === undefined
? command
: withRemoteDeadline(
command,
Math.max(1, timeoutMs - (Date.now() - started)),
);
try {
conn.exec(remoteCommand, (err, commandStream) => {
if (settled) {
closeStream(commandStream);
return;
}
if (err) {
onData?.(err.message);
settle(
new ExecError(
`Remote command execution failed: ${err.message}`,
{
command,
serverId,
originalError: err,
},
),
);
return;
}
stream = commandStream;
commandStream
.on("error", (error: Error) => {
if (settled) return;
settle(
new ExecError(
`Remote command stream failed: ${error.message}`,
{
command,
stdout,
stderr,
serverId,
originalError: error,
},
),
);
})
.on("close", (code: number, _signal: string) => {
if (code === 0) {
settle(undefined, { stdout, stderr });
} else {
settle(
new ExecError(
`Remote command failed with exit code ${code}`,
{
command,
stdout,
stderr,
exitCode: code,
serverId,
},
),
);
}
})
.on("data", (data: string) => {
if (settled) return;
stdout += data.toString();
onData?.(data.toString());
})
.stderr.on("data", (data) => {
if (settled) return;
stderr += data.toString();
onData?.(data.toString());
});
});
} catch (error) {
settle(
new ExecError("Remote command execution failed", {
command,
stdout,
stderr,
serverId,
originalError: error instanceof Error ? error : undefined,
}),
);
}
})
.on("error", (err) => {
if (settled) return;
if (err.level === "client-authentication") {
const technicalDetail = `Error: ${err.message} ${err.level}`;
const friendlyMessage = [
"",
"❌ Couldn't connect to your server — the SSH key was not accepted.",
"",
"This usually means the key doesn't match what's on the server, or the key format is invalid.",
"",
`Technical details: ${technicalDetail}`,
"",
"💡 Hints:",
" • Check that the SSH key you added in Dokploy is the same one installed on the server (e.g. in ~/.ssh/authorized_keys).",
" • Try generating a new SSH key in Dokploy and add only the public key to the server, then try again.",
" • Make sure to follow the instructions on the Setup Server Button on the SSH Keys tab and then click on deployments tab and check the logs for more details.",
].join("\n");
onData?.(friendlyMessage);
settle(
new ExecError(
`Authentication failed: Invalid SSH private key. ${friendlyMessage}`,
{
command,
serverId,
originalError: err,
},
),
);
} else {
const errorMsg = `SSH connection error: ${err.message}`;
onData?.(errorMsg);
settle(
new ExecError(errorMsg, {
stdout,
stderr,
command,
serverId,
originalError: err,
}),
);
return;
}
stream
.on("close", (code: number, _signal: string) => {
conn.end();
if (code === 0) {
resolve({ stdout, stderr });
} else {
reject(
new ExecError(
`Remote command failed with exit code ${code}`,
{
command,
stdout,
stderr,
exitCode: code,
serverId,
},
),
);
}
})
.on("data", (data: string) => {
stdout += data.toString();
onData?.(data.toString());
})
.stderr.on("data", (data) => {
stderr += data.toString();
onData?.(data.toString());
});
})
.connect({
host: server.ipAddress,
port: server.port,
username: server.username,
privateKey: server.sshKey?.privateKey,
timeout: timeoutMs != null && timeoutMs > 0 ? timeoutMs : 99999,
...(timeoutMs != null && timeoutMs > 0
? { readyTimeout: timeoutMs }
: {}),
});
})
.on("error", (err) => {
conn.end();
if (err.level === "client-authentication") {
const technicalDetail = `Error: ${err.message} ${err.level}`;
const friendlyMessage = [
"",
"❌ Couldn't connect to your server — the SSH key was not accepted.",
"",
"This usually means the key doesn't match what's on the server, or the key format is invalid.",
"",
`Technical details: ${technicalDetail}`,
"",
"💡 Hints:",
" • Check that the SSH key you added in Dokploy is the same one installed on the server (e.g. in ~/.ssh/authorized_keys).",
" • Try generating a new SSH key in Dokploy and add only the public key to the server, then try again.",
" • Make sure to follow the instructions on the Setup Server Button on the SSH Keys tab and then click on deployments tab and check the logs for more details.",
].join("\n");
const errorMsg = `Authentication failed: Invalid SSH private key. ❌ Error: ${err.message} ${err.level}`;
onData?.(friendlyMessage);
reject(
new ExecError(
`Authentication failed: Invalid SSH private key. ${friendlyMessage}`,
{
command,
serverId,
originalError: err,
},
),
);
} else {
const errorMsg = `SSH connection error: ${err.message}`;
onData?.(errorMsg);
reject(
new ExecError(errorMsg, {
command,
serverId,
originalError: err,
}),
);
}
})
.connect({
host: server.ipAddress,
port: server.port,
username: server.username,
privateKey: server.sshKey?.privateKey,
timeout: 99999,
});
} catch (error) {
settle(
new ExecError("SSH connection failed", {
command,
stdout,
stderr,
serverId,
originalError: error instanceof Error ? error : undefined,
}),
);
}
});
};

39
packages/server/src/utils/process/remote-deadline.ts Normal file
View File

@ -0,0 +1,39 @@
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
/** OpenSSH gives each non-PTY command its own process group, including its children. */
export const withRemoteDeadline = (
command: string,
timeout: number,
): string => {
const seconds = timeout / 1000;
// The status pipe is private; output relays keep the deadline alive until inherited pipes drain.
return `exec sh -c ${quote(`
kill -0 -$$ 2>/dev/null || { echo 'Timed SSH command requires an isolated process group' >&2; exit 125; }
command -v cat >/dev/null 2>&1 || { echo 'Timed SSH command requires cat' >&2; exit 125; }
if sleep 0.001 2>/dev/null; then
duration=${seconds}
elif sleep 0 2>/dev/null; then
duration=${Math.ceil(seconds)}
else
echo 'Timed SSH command requires sleep' >&2
exit 125
fi
sleep "$duration" &
deadline=$!
(
result=$(
exec 3>&1
{
{ sh -c ${quote(command)} 3>&- 4>&- 5>&-; printf '%s' "$?" >&3; } | cat >&4
} 2>&1 | cat >&5
)
kill "$deadline" 2>/dev/null
exit "$result"
) 4>&1 5>&2 &
command_pid=$!
if wait "$deadline" 2>/dev/null; then
kill -s KILL 0
fi
wait "$command_pid"
`)}`;
};