From 25582298788430911b591a97f94b2521763a6b66 Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Tue, 25 Aug 2026 11:14:12 +0000 Subject: [PATCH 1/7] fix(backups): support databases on swarm workers --- .../backups/redact-credentials.test.ts | 12 + .../backups/swarm-backup-executor.test.ts | 559 ++++++++++++++++++ packages/server/src/services/deployment.ts | 3 +- packages/server/src/utils/backups/executor.ts | 258 ++++++++ packages/server/src/utils/backups/libsql.ts | 30 +- packages/server/src/utils/backups/mariadb.ts | 30 +- packages/server/src/utils/backups/mongo.ts | 31 +- packages/server/src/utils/backups/mysql.ts | 31 +- packages/server/src/utils/backups/postgres.ts | 30 +- packages/server/src/utils/backups/redact.ts | 10 +- packages/server/src/utils/backups/utils.ts | 42 +- packages/server/src/utils/backups/worker.ts | 240 ++++++++ 12 files changed, 1154 insertions(+), 122 deletions(-) create mode 100644 apps/dokploy/__test__/backups/swarm-backup-executor.test.ts create mode 100644 packages/server/src/utils/backups/executor.ts create mode 100644 packages/server/src/utils/backups/worker.ts diff --git a/apps/dokploy/__test__/backups/redact-credentials.test.ts b/apps/dokploy/__test__/backups/redact-credentials.test.ts index 5fff508cc..b7ab34870 100644 --- a/apps/dokploy/__test__/backups/redact-credentials.test.ts +++ b/apps/dokploy/__test__/backups/redact-credentials.test.ts @@ -47,4 +47,16 @@ describe("redactRcloneCredentials (#4621)", () => { expect(redacted).not.toContain("MYSECRET"); expect(redacted).toContain("[REDACTED]"); }); + + it("should redact shell-quoted and unquoted credential values", () => { + const cmd = + "rclone rcat --s3-access-key-id=plain-key --s3-secret-access-key='secret with spaces' --s3-access-key-id='key'\\''with-quote' :s3:bucket/file.gz"; + const redacted = redactRcloneCredentials(cmd); + + expect(redacted).not.toContain("plain-key"); + expect(redacted).not.toContain("secret with spaces"); + expect(redacted).not.toContain("with-quote"); + expect(redacted).toContain('--s3-access-key-id="[REDACTED]"'); + expect(redacted).toContain('--s3-secret-access-key="[REDACTED]"'); + }); }); diff --git a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts new file mode 100644 index 000000000..49a04f19c --- /dev/null +++ b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts @@ -0,0 +1,559 @@ +import type { BackupSchedule } from "@dokploy/server/services/backup"; +import { executeBackup } from "@dokploy/server/utils/backups/executor"; +import { getBackupCommand } from "@dokploy/server/utils/backups/utils"; +import { + getBackupResourceNames, + getBackupTargetServiceName, + getBackupWorkerServiceSpec, + waitForBackupWorkerTask, +} from "@dokploy/server/utils/backups/worker"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + execAsync: vi.fn(), + execAsyncRemote: vi.fn(), + getRemoteDocker: vi.fn(), + loggerError: vi.fn(), + loggerInfo: vi.fn(), + sleep: vi.fn(), +})); + +vi.mock("@dokploy/server/lib/logger", () => ({ + logger: { + error: mocks.loggerError, + info: mocks.loggerInfo, + }, +})); + +vi.mock("@dokploy/server/utils/process/execAsync", async (importOriginal) => { + const original = + await importOriginal< + typeof import("@dokploy/server/utils/process/execAsync") + >(); + return { + ...original, + execAsync: mocks.execAsync, + execAsyncRemote: mocks.execAsyncRemote, + sleep: mocks.sleep, + }; +}); + +vi.mock("@dokploy/server/utils/servers/remote-docker", () => ({ + getRemoteDocker: mocks.getRemoteDocker, +})); + +const containerId = "a".repeat(64); + +const postgresBackup = (overrides: Record = {}) => + ({ + backupId: "backup-1", + backupType: "database", + database: "app", + databaseType: "postgres", + postgres: { + appName: "postgres-service", + databaseUser: "postgres", + serverId: null, + }, + ...overrides, + }) as BackupSchedule; + +const databaseBackup = ( + databaseType: "postgres" | "mysql" | "mariadb" | "mongo" | "libsql", +) => { + const databaseConfig = { + postgres: { appName: "postgres-service", databaseUser: "postgres" }, + mysql: { appName: "mysql-service", databaseRootPassword: "mysql-pass" }, + mariadb: { + appName: "mariadb-service", + databasePassword: "mariadb-pass", + databaseUser: "mariadb", + }, + mongo: { + appName: "mongo-service", + databasePassword: "mongo-pass", + databaseUser: "mongo", + }, + libsql: { appName: "libsql-service" }, + }[databaseType]; + + return { + backupId: `backup-${databaseType}`, + backupType: "database", + database: "app", + databaseType, + [databaseType]: databaseConfig, + } as unknown as BackupSchedule; +}; + +const input = (overrides: Record = {}) => ({ + backup: postgresBackup(), + executionId: "deployment-1", + logPath: "/etc/dokploy/logs/postgres backup.log", + rcloneDestination: ":s3:backups/postgres.sql.gz", + rcloneFlags: [ + "--s3-access-key-id=ACCESS_KEY", + "--s3-secret-access-key=SECRET_KEY", + ], + ...overrides, +}); + +const runningDatabaseTask = { + ID: "database-task", + NodeID: "worker-node-id", + Status: { + State: "running", + ContainerStatus: { ContainerID: containerId }, + }, + Version: { Index: 1 }, +}; + +const createDockerMock = () => { + const secretRemove = vi.fn().mockResolvedValue(undefined); + const serviceRemove = vi.fn().mockResolvedValue(undefined); + const docker = { + createSecret: vi.fn().mockResolvedValue({ ID: "secret-id" }), + createService: vi.fn().mockResolvedValue({ ID: "service-id" }), + getSecret: vi.fn(() => ({ remove: secretRemove })), + getService: vi.fn(() => ({ remove: serviceRemove })), + listTasks: vi + .fn() + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "complete", ContainerStatus: { ExitCode: 0 } }, + Version: { Index: 2 }, + }, + ]), + }; + + return { docker, secretRemove, serviceRemove }; +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.sleep.mockResolvedValue(undefined); + mocks.execAsync.mockImplementation(async (command: string) => { + if (command.includes("nohup docker service logs")) { + return { stdout: "4321\n", stderr: "" }; + } + if (command.startsWith("if ps -p")) { + return { stdout: "running\n", stderr: "" }; + } + return { stdout: "", stderr: "" }; + }); + mocks.execAsyncRemote.mockResolvedValue({ stdout: "", stderr: "" }); +}); + +describe("backup worker target and service specification", () => { + it.each([ + ["postgres", "postgres-service"], + ["mysql", "mysql-service"], + ["mariadb", "mariadb-service"], + ["mongo", "mongo-service"], + ["libsql", "libsql-service"], + ] as const)("resolves the %s Swarm service", (databaseType, appName) => { + const backup = { + backupType: "database", + databaseType, + [databaseType]: { appName }, + } as unknown as BackupSchedule; + + expect(getBackupTargetServiceName(backup)).toBe(appName); + }); + + it("creates deterministic, collision-resistant names per execution", () => { + const first = getBackupResourceNames("deployment-1"); + const retry = getBackupResourceNames("deployment-1"); + const concurrent = getBackupResourceNames("deployment-2"); + + expect(first).toEqual(retry); + expect(first.serviceName).toMatch(/^dokploy-backup-[a-f0-9]{16}$/); + expect(first.secretName).toBe(`${first.serviceName}-script`); + expect(concurrent.serviceName).not.toBe(first.serviceName); + }); + + it("pins a one-shot worker to the database node without exposing credentials", () => { + const spec = getBackupWorkerServiceSpec({ + backupId: "backup-1", + executionId: "deployment-1", + nodeId: "worker-node-id", + secretId: "secret-id", + secretName: "worker-script", + serviceName: "worker-service", + }); + const taskTemplate = spec.TaskTemplate; + const containerSpec = + taskTemplate && "ContainerSpec" in taskTemplate + ? taskTemplate.ContainerSpec + : undefined; + + expect(spec.Name).toBe("worker-service"); + expect(spec.Mode).toEqual({ Replicated: { Replicas: 1 } }); + expect(spec.TaskTemplate?.Placement?.Constraints).toEqual([ + "node.id==worker-node-id", + ]); + expect(spec.TaskTemplate?.RestartPolicy?.Condition).toBe("none"); + expect(containerSpec?.Mounts).toContainEqual({ + Type: "bind", + Source: "/var/run/docker.sock", + Target: "/var/run/docker.sock", + }); + expect(containerSpec?.Secrets).toContainEqual( + expect.objectContaining({ SecretID: "secret-id" }), + ); + expect(JSON.stringify(spec)).not.toContain("ACCESS_KEY"); + expect(JSON.stringify(spec)).not.toContain("SECRET_KEY"); + }); +}); + +describe("backup worker task lifecycle", () => { + it("waits through scheduling and running until completion", async () => { + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([ + { Status: { State: "pending" }, Version: { Index: 1 } }, + ]) + .mockResolvedValueOnce([ + { Status: { State: "running" }, Version: { Index: 2 } }, + ]) + .mockResolvedValueOnce([ + { Status: { State: "complete" }, Version: { Index: 3 } }, + ]), + }; + const sleepFn = vi.fn().mockResolvedValue(undefined); + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + pollIntervalMs: 0, + sleepFn, + }), + ).resolves.toBeUndefined(); + expect(sleepFn).toHaveBeenCalledTimes(2); + }); + + it("reports the terminal task error and exit code", async () => { + const docker = { + listTasks: vi.fn().mockResolvedValue([ + { + Status: { + State: "failed", + Err: "image pull failed", + ContainerStatus: { ExitCode: 1 }, + }, + Version: { Index: 1 }, + }, + ]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id"), + ).rejects.toThrow( + "Backup worker task failed: image pull failed, exit code 1", + ); + }); + + it("times out when the worker never starts", async () => { + const docker = { + listTasks: vi + .fn() + .mockResolvedValue([ + { Status: { State: "pending" }, Version: { Index: 1 } }, + ]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + startTimeoutMs: 0, + }), + ).rejects.toThrow("Backup worker did not start within 0 seconds"); + }); + + it("fails when a task disappears after it started", async () => { + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([ + { Status: { State: "running" }, Version: { Index: 1 } }, + ]) + .mockResolvedValue([]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + missingTaskGracePolls: 2, + pollIntervalMs: 0, + sleepFn: vi.fn().mockResolvedValue(undefined), + }), + ).rejects.toThrow("Backup worker task disappeared after starting"); + }); +}); + +describe("executeBackup", () => { + it("keeps the existing direct path when the database container is local", async () => { + const { docker } = createDockerMock(); + mocks.getRemoteDocker.mockResolvedValue(docker); + mocks.execAsync.mockResolvedValueOnce({ + stdout: `${containerId}\n`, + stderr: "", + }); + + await expect(executeBackup(input())).resolves.toEqual({ mode: "direct" }); + + expect(mocks.execAsync).toHaveBeenCalledTimes(2); + expect(mocks.execAsync.mock.calls[1]?.[0]).toContain( + "Starting backup process", + ); + expect(mocks.execAsync.mock.calls[1]?.[0]).toContain( + `CONTAINER_ID=${containerId};`, + ); + expect(mocks.execAsync.mock.calls[1]?.[0]).toContain( + "docker inspect --format '{{.State.Running}}'", + ); + expect(mocks.execAsync.mock.calls[1]?.[0]).toContain( + "label=com.docker.swarm.service.name", + ); + expect(docker.createService).not.toHaveBeenCalled(); + }); + + it("uses the exact worker container and removes all temporary resources", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).resolves.toEqual({ + mode: "swarm-worker", + }); + + const targetFilters = JSON.parse( + docker.listTasks.mock.calls[0]?.[0].filters, + ); + expect(targetFilters).toEqual({ + service: ["postgres-service"], + "desired-state": ["running"], + }); + + const secretSpec = docker.createSecret.mock.calls[0]?.[0]; + const workerScript = Buffer.from(secretSpec.Data, "base64").toString(); + expect(workerScript).toContain(`CONTAINER_ID=${containerId};`); + expect(workerScript).toContain("/proc/1/fd/1"); + expect(workerScript).toContain( + "label=com.docker.swarm.service.name=postgres-service", + ); + + const serviceSpec = docker.createService.mock.calls[0]?.[0]; + expect(serviceSpec.TaskTemplate.Placement.Constraints).toEqual([ + "node.id==worker-node-id", + ]); + expect(JSON.stringify(serviceSpec)).not.toContain("ACCESS_KEY"); + expect(JSON.stringify(serviceSpec)).not.toContain("SECRET_KEY"); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + const serviceLogCommands = mocks.execAsync.mock.calls.filter(([command]) => + command.includes("docker service logs --raw"), + ); + expect(serviceLogCommands).toHaveLength(1); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes("docker service logs --raw --follow"), + ), + ).toBe(false); + }); + + it("removes the worker and secret after a failed backup task", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + Status: { + State: "failed", + Err: "database command failed", + ContainerStatus: { ExitCode: 1 }, + }, + Version: { Index: 2 }, + }, + ]); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "Backup worker task failed: database command failed, exit code 1", + ); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes("❌ Error: %s"), + ), + ).toBe(true); + }); + + it("does not hide the backup failure when cleanup also fails", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + Status: { State: "rejected", Err: "worker rejected" }, + Version: { Index: 2 }, + }, + ]); + serviceRemove.mockRejectedValue(new Error("cleanup unavailable")); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "Backup worker task rejected: worker rejected", + ); + expect(secretRemove).toHaveBeenCalledOnce(); + expect(mocks.loggerError).toHaveBeenCalledWith( + expect.objectContaining({ + errors: expect.arrayContaining([ + "Failed to remove backup worker service: cleanup unavailable", + ]), + }), + "Backup worker cleanup also failed", + ); + }); + + it("fails a successful backup if its temporary resources cannot be cleaned", async () => { + const { docker, serviceRemove } = createDockerMock(); + serviceRemove.mockRejectedValue(new Error("cleanup unavailable")); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "Failed to remove backup worker service: cleanup unavailable", + ); + }); + + it("fails before creating resources when no running database task exists", async () => { + const { docker } = createDockerMock(); + docker.listTasks.mockReset(); + docker.listTasks.mockResolvedValue([]); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "No running Swarm task found for database service postgres-service", + ); + expect(docker.createSecret).not.toHaveBeenCalled(); + expect(docker.createService).not.toHaveBeenCalled(); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes("No running Swarm task found"), + ), + ).toBe(true); + }); + + it("attempts name-based cleanup when service creation loses its response", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + docker.createService.mockRejectedValue( + new Error("connection dropped after create"), + ); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "connection dropped after create", + ); + expect(docker.getService).toHaveBeenCalledWith( + getBackupResourceNames("deployment-1").serviceName, + ); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + }); + + it("attempts name-based cleanup when secret creation loses its response", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + docker.createSecret.mockRejectedValue( + new Error("connection dropped after secret create"), + ); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "connection dropped after secret create", + ); + expect(docker.getSecret).toHaveBeenCalledWith( + getBackupResourceNames("deployment-1").secretName, + ); + expect(secretRemove).toHaveBeenCalledOnce(); + expect(serviceRemove).not.toHaveBeenCalled(); + }); + + it("uses the remote manager for discovery, logs, and direct execution", async () => { + const { docker } = createDockerMock(); + mocks.getRemoteDocker.mockResolvedValue(docker); + mocks.execAsyncRemote.mockResolvedValueOnce({ + stdout: `${containerId}\n`, + stderr: "", + }); + + await expect( + executeBackup(input({ serverId: "remote-server" })), + ).resolves.toEqual({ mode: "direct" }); + + expect(mocks.execAsyncRemote).toHaveBeenCalledTimes(2); + expect(mocks.execAsync).not.toHaveBeenCalled(); + }); + + it("uses the remote manager throughout the worker lifecycle", async () => { + const { docker } = createDockerMock(); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect( + executeBackup(input({ serverId: "remote-server" })), + ).resolves.toEqual({ mode: "swarm-worker" }); + + expect(mocks.getRemoteDocker).toHaveBeenCalledWith("remote-server"); + expect(mocks.execAsyncRemote).toHaveBeenCalledTimes(3); + expect(mocks.execAsync).not.toHaveBeenCalled(); + expect( + mocks.execAsyncRemote.mock.calls.some(([, command]) => + command.includes("docker service logs --raw"), + ), + ).toBe(true); + }); +}); + +describe("worker backup command safety", () => { + it.each([ + ["postgres", "pg_dump"], + ["mysql", "mysqldump"], + ["mariadb", "mariadb-dump"], + ["mongo", "mongodump"], + ["libsql", "tar cf"], + ] as const)( + "reuses the existing %s backup pipeline", + (databaseType, expectedCommand) => { + const command = getBackupCommand( + databaseBackup(databaseType), + ["--s3-provider=AWS"], + ":s3:bucket/database.gz", + "/proc/1/fd/1", + { containerId }, + ); + + expect(command).toContain(expectedCommand); + expect(command).toContain(`CONTAINER_ID=${containerId};`); + expect(command).toContain("rclone rcat"); + }, + ); + + it("quotes log paths and validates explicit container IDs", () => { + const command = getBackupCommand( + postgresBackup(), + ["--s3-provider=AWS"], + ":s3:backups/postgres.sql.gz", + "/tmp/log path;touch injected", + { containerId }, + ); + + expect(command).toContain("'/tmp/log path;touch injected'"); + expect(command).toContain(`CONTAINER_ID=${containerId};`); + expect(() => + getBackupCommand(postgresBackup(), [], ":s3:bucket/file", "/tmp/log", { + containerId: "$(touch /tmp/injected)", + }), + ).toThrow("Invalid backup container ID"); + }); +}); diff --git a/packages/server/src/services/deployment.ts b/packages/server/src/services/deployment.ts index fd61fc106..ab798e2ff 100644 --- a/packages/server/src/services/deployment.ts +++ b/packages/server/src/services/deployment.ts @@ -374,7 +374,8 @@ export const createDeploymentBackup = async ( backup.postgres?.serverId || backup.mariadb?.serverId || backup.mysql?.serverId || - backup.mongo?.serverId; + backup.mongo?.serverId || + backup.libsql?.serverId; } else if (backup.backupType === "compose") { serverId = backup.compose?.serverId; } diff --git a/packages/server/src/utils/backups/executor.ts b/packages/server/src/utils/backups/executor.ts new file mode 100644 index 000000000..e45e3f969 --- /dev/null +++ b/packages/server/src/utils/backups/executor.ts @@ -0,0 +1,258 @@ +import { logger } from "@dokploy/server/lib/logger"; +import type { BackupSchedule } from "@dokploy/server/services/backup"; +import { quote } from "shell-quote"; +import { ExecError, execAsync, execAsyncRemote } from "../process/execAsync"; +import { getRemoteDocker } from "../servers/remote-docker"; +import { redactRcloneCredentials } from "./redact"; +import { getBackupCommand, getContainerSearchCommand } from "./utils"; +import { + findRunningServiceTask, + getBackupResourceNames, + getBackupTargetServiceName, + getBackupWorkerServiceSpec, + waitForBackupWorkerTask, +} from "./worker"; + +type ExecuteBackupInput = { + backup: BackupSchedule; + executionId: string; + logPath: string; + rcloneDestination: string; + rcloneFlags: string[]; + serverId?: string | null; +}; + +const runHostCommand = ( + serverId: string | null | undefined, + command: string, +) => { + if (serverId) { + return execAsyncRemote(serverId, command); + } + + return execAsync(command, { shell: "/bin/bash" }); +}; + +const isNotFoundError = (error: unknown) => + typeof error === "object" && + error !== null && + "statusCode" in error && + error.statusCode === 404; + +const getSafeErrorMessage = (error: unknown) => { + if (error instanceof ExecError) { + const output = error.stderr?.trim() || error.stdout?.trim(); + return redactRcloneCredentials( + output || `command exited with code ${error.exitCode ?? "unknown"}`, + ); + } + return redactRcloneCredentials( + error instanceof Error ? error.message : String(error), + ); +}; + +const appendBackupError = async ( + serverId: string | null | undefined, + logPath: string, + error: unknown, +) => { + const message = getSafeErrorMessage(error); + const command = `printf '[%s] ❌ Error: %s\\n' "$(date)" ${quote([message])} >> ${quote([logPath])}`; + try { + await runHostCommand(serverId, command); + } catch (logError) { + logger.error( + { error: getSafeErrorMessage(logError) }, + "Failed to append backup worker error to deployment log", + ); + } +}; + +const appendBackupMessage = async ( + serverId: string | null | undefined, + logPath: string, + message: string, +) => { + const command = `printf '[%s] %s\n' "$(date)" ${quote([message])} >> ${quote([logPath])}`; + try { + await runHostCommand(serverId, command); + } catch (logError) { + logger.error( + { error: getSafeErrorMessage(logError) }, + "Failed to append backup worker progress to deployment log", + ); + } +}; + +const collectServiceLogs = ( + serverId: string | null | undefined, + serviceName: string, + logPath: string, +) => { + const quotedServiceName = quote([serviceName]); + return runHostCommand( + serverId, + `if docker service inspect ${quotedServiceName} >/dev/null 2>&1; then docker service logs --raw ${quotedServiceName} >> ${quote([logPath])} 2>&1; fi`, + ); +}; + +const removeResource = async ( + remove: () => Promise, + resource: "secret" | "service", +) => { + try { + await remove(); + return null; + } catch (error) { + if (isNotFoundError(error)) { + return null; + } + return new Error( + `Failed to remove backup worker ${resource}: ${getSafeErrorMessage(error)}`, + ); + } +}; + +const runBackupOnWorker = async ({ + backup, + executionId, + logPath, + rcloneDestination, + rcloneFlags, + serverId, +}: ExecuteBackupInput) => { + const serviceTarget = getBackupTargetServiceName(backup); + if (!serviceTarget) { + throw new Error("Could not determine the Swarm service for this backup"); + } + + const docker = await getRemoteDocker(serverId); + const { containerId, nodeId } = await findRunningServiceTask( + docker, + serviceTarget, + ); + const { secretName, serviceName } = getBackupResourceNames(executionId); + const labels = { + "dokploy.backup.id": backup.backupId, + "dokploy.deployment.id": executionId, + "dokploy.managed": "true", + "dokploy.resource": "database-backup-worker", + }; + // Keep database and S3 credentials out of the inspectable service arguments. + const workerCommand = getBackupCommand( + backup, + rcloneFlags, + rcloneDestination, + "/proc/1/fd/1", + { containerId }, + ); + const service = docker.getService(serviceName); + const secret = docker.getSecret(secretName); + + let serviceMayExist = false; + let secretMayExist = false; + let primaryError: unknown; + + try { + await appendBackupMessage( + serverId, + logPath, + `Preparing backup worker on node ${nodeId}...`, + ); + + secretMayExist = true; + const createdSecret = (await docker.createSecret({ + Name: secretName, + Labels: labels, + Data: Buffer.from(workerCommand).toString("base64"), + })) as { ID?: string }; + const secretId = + createdSecret.ID || ((await secret.inspect()) as { ID?: string }).ID; + if (!secretId) { + throw new Error("Docker did not return a backup worker secret ID"); + } + + serviceMayExist = true; + const createdService = (await docker.createService( + getBackupWorkerServiceSpec({ + backupId: backup.backupId, + executionId, + nodeId, + secretId, + secretName, + serviceName, + }), + )) as { ID?: string }; + await waitForBackupWorkerTask(docker, createdService.ID || serviceName); + } catch (error) { + primaryError = error; + } finally { + const cleanupErrors: Error[] = []; + + if (serviceMayExist) { + // Collect once after the terminal task state to avoid follower races or duplicates. + try { + await collectServiceLogs(serverId, serviceName, logPath); + } catch (error) { + cleanupErrors.push( + new Error( + `Failed to collect backup worker logs: ${getSafeErrorMessage(error)}`, + ), + ); + } + + const error = await removeResource(() => service.remove(), "service"); + if (error) cleanupErrors.push(error); + } + if (secretMayExist) { + const error = await removeResource(() => secret.remove(), "secret"); + if (error) cleanupErrors.push(error); + } + + if (primaryError) { + if (cleanupErrors.length > 0) { + logger.error( + { errors: cleanupErrors.map((error) => error.message) }, + "Backup worker cleanup also failed", + ); + } + } else if (cleanupErrors.length > 0) { + primaryError = new Error( + cleanupErrors.map((error) => error.message).join("; "), + ); + } + } + + if (primaryError) { + throw primaryError; + } +}; + +export const executeBackup = async (input: ExecuteBackupInput) => { + try { + const containerSearch = getContainerSearchCommand(input.backup); + if (!containerSearch) { + throw new Error("Could not build the database container search command"); + } + + const { stdout } = await runHostCommand(input.serverId, containerSearch); + const directContainerId = stdout.trim().split(/\s+/)[0]; + if (directContainerId) { + const command = getBackupCommand( + input.backup, + input.rcloneFlags, + input.rcloneDestination, + input.logPath, + { containerId: directContainerId }, + ); + await runHostCommand(input.serverId, command); + return { mode: "direct" as const }; + } + + await runBackupOnWorker(input); + return { mode: "swarm-worker" as const }; + } catch (error) { + await appendBackupError(input.serverId, input.logPath, error); + throw error; + } +}; diff --git a/packages/server/src/utils/backups/libsql.ts b/packages/server/src/utils/backups/libsql.ts index c4ece6ed6..106e0f8da 100644 --- a/packages/server/src/utils/backups/libsql.ts +++ b/packages/server/src/utils/backups/libsql.ts @@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Libsql } from "@dokploy/server/services/libsql"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; -import { - getBackupCommand, - getBackupTimestamp, - getS3Credentials, - normalizeS3Path, -} from "./utils"; +import { executeBackup } from "./executor"; +import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils"; export const runLibsqlBackup = async ( libsql: Libsql, @@ -36,19 +31,14 @@ export const runLibsqlBackup = async ( try { const rcloneFlags = getS3Credentials(destination); const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; - const backupCommand = getBackupCommand( + await executeBackup({ backup, - rcloneFlags, + executionId: deployment.deploymentId, + logPath: deployment.logPath, rcloneDestination, - deployment.logPath, - ); - if (libsql.serverId) { - await execAsyncRemote(libsql.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + rcloneFlags, + serverId: libsql.serverId, + }); await sendDatabaseBackupNotifications({ applicationName: name, @@ -66,8 +56,8 @@ export const runLibsqlBackup = async ( projectName: project.name, databaseType: "libsql", type: "error", - // @ts-ignore - errorMessage: error?.message || "Error message not provided", + errorMessage: + error instanceof Error ? error.message : "Error message not provided", organizationId: project.organizationId, databaseName: backup.database, }); diff --git a/packages/server/src/utils/backups/mariadb.ts b/packages/server/src/utils/backups/mariadb.ts index 4a3f7dc26..877d0e03d 100644 --- a/packages/server/src/utils/backups/mariadb.ts +++ b/packages/server/src/utils/backups/mariadb.ts @@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Mariadb } from "@dokploy/server/services/mariadb"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; -import { - getBackupCommand, - getBackupTimestamp, - getS3Credentials, - normalizeS3Path, -} from "./utils"; +import { executeBackup } from "./executor"; +import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils"; export const runMariadbBackup = async ( mariadb: Mariadb, @@ -35,19 +30,14 @@ export const runMariadbBackup = async ( try { const rcloneFlags = getS3Credentials(destination); const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; - const backupCommand = getBackupCommand( + await executeBackup({ backup, - rcloneFlags, + executionId: deployment.deploymentId, + logPath: deployment.logPath, rcloneDestination, - deployment.logPath, - ); - if (mariadb.serverId) { - await execAsyncRemote(mariadb.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + rcloneFlags, + serverId: mariadb.serverId, + }); await sendDatabaseBackupNotifications({ applicationName: name, @@ -65,8 +55,8 @@ export const runMariadbBackup = async ( projectName: project.name, databaseType: "mariadb", type: "error", - // @ts-ignore - errorMessage: error?.message || "Error message not provided", + errorMessage: + error instanceof Error ? error.message : "Error message not provided", organizationId: project.organizationId, databaseName: backup.database, }); diff --git a/packages/server/src/utils/backups/mongo.ts b/packages/server/src/utils/backups/mongo.ts index 6b80ccc86..8dc5832ae 100644 --- a/packages/server/src/utils/backups/mongo.ts +++ b/packages/server/src/utils/backups/mongo.ts @@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Mongo } from "@dokploy/server/services/mongo"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; -import { - getBackupCommand, - getBackupTimestamp, - getS3Credentials, - normalizeS3Path, -} from "./utils"; +import { executeBackup } from "./executor"; +import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils"; export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { const { environmentId, name, appName } = mongo; @@ -32,20 +27,14 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { try { const rcloneFlags = getS3Credentials(destination); const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; - const backupCommand = getBackupCommand( + await executeBackup({ backup, - rcloneFlags, + executionId: deployment.deploymentId, + logPath: deployment.logPath, rcloneDestination, - deployment.logPath, - ); - - if (mongo.serverId) { - await execAsyncRemote(mongo.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + rcloneFlags, + serverId: mongo.serverId, + }); await sendDatabaseBackupNotifications({ applicationName: name, @@ -63,8 +52,8 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { projectName: project.name, databaseType: "mongodb", type: "error", - // @ts-ignore - errorMessage: error?.message || "Error message not provided", + errorMessage: + error instanceof Error ? error.message : "Error message not provided", organizationId: project.organizationId, databaseName: backup.database, }); diff --git a/packages/server/src/utils/backups/mysql.ts b/packages/server/src/utils/backups/mysql.ts index 2ac9228be..80bc12fa9 100644 --- a/packages/server/src/utils/backups/mysql.ts +++ b/packages/server/src/utils/backups/mysql.ts @@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { MySql } from "@dokploy/server/services/mysql"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; -import { - getBackupCommand, - getBackupTimestamp, - getS3Credentials, - normalizeS3Path, -} from "./utils"; +import { executeBackup } from "./executor"; +import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils"; export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { const { environmentId, name, appName } = mysql; @@ -33,20 +28,14 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { try { const rcloneFlags = getS3Credentials(destination); const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; - const backupCommand = getBackupCommand( + await executeBackup({ backup, - rcloneFlags, + executionId: deployment.deploymentId, + logPath: deployment.logPath, rcloneDestination, - deployment.logPath, - ); - - if (mysql.serverId) { - await execAsyncRemote(mysql.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + rcloneFlags, + serverId: mysql.serverId, + }); await sendDatabaseBackupNotifications({ applicationName: name, projectName: project.name, @@ -63,8 +52,8 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { projectName: project.name, databaseType: "mysql", type: "error", - // @ts-ignore - errorMessage: error?.message || "Error message not provided", + errorMessage: + error instanceof Error ? error.message : "Error message not provided", organizationId: project.organizationId, databaseName: backup.database, }); diff --git a/packages/server/src/utils/backups/postgres.ts b/packages/server/src/utils/backups/postgres.ts index de7ef5500..efb6e6546 100644 --- a/packages/server/src/utils/backups/postgres.ts +++ b/packages/server/src/utils/backups/postgres.ts @@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Postgres } from "@dokploy/server/services/postgres"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; -import { - getBackupCommand, - getBackupTimestamp, - getS3Credentials, - normalizeS3Path, -} from "./utils"; +import { executeBackup } from "./executor"; +import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils"; export const runPostgresBackup = async ( postgres: Postgres, @@ -36,19 +31,14 @@ export const runPostgresBackup = async ( try { const rcloneFlags = getS3Credentials(destination); const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`; - const backupCommand = getBackupCommand( + await executeBackup({ backup, - rcloneFlags, + executionId: deployment.deploymentId, + logPath: deployment.logPath, rcloneDestination, - deployment.logPath, - ); - if (postgres.serverId) { - await execAsyncRemote(postgres.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + rcloneFlags, + serverId: postgres.serverId, + }); await sendDatabaseBackupNotifications({ applicationName: name, @@ -66,8 +56,8 @@ export const runPostgresBackup = async ( projectName: project.name, databaseType: "postgres", type: "error", - // @ts-ignore - errorMessage: error?.message || "Error message not provided", + errorMessage: + error instanceof Error ? error.message : "Error message not provided", organizationId: project.organizationId, databaseName: backup.database, }); diff --git a/packages/server/src/utils/backups/redact.ts b/packages/server/src/utils/backups/redact.ts index 065e76d8c..719883c31 100644 --- a/packages/server/src/utils/backups/redact.ts +++ b/packages/server/src/utils/backups/redact.ts @@ -2,11 +2,11 @@ * Redacts S3 credentials from rclone command strings. * * Used to prevent credential leakage in structured logs and error output. - * Matches the flag format produced by `getS3Credentials()`: - * --s3-access-key-id="VALUE" and --s3-secret-access-key="VALUE" + * Matches quoted and unquoted flag values produced by `getS3Credentials()`. */ export const redactRcloneCredentials = (command: string): string => { - return command - .replace(/(--s3-access-key-id=)"[^"]*"/g, '$1"[REDACTED]"') - .replace(/(--s3-secret-access-key=)"[^"]*"/g, '$1"[REDACTED]"'); + return command.replace( + /(--s3-(?:access-key-id|secret-access-key)=)(?:(?:"[^"]*"|'[^']*'|\\.|[^\s'"])+)/g, + '$1"[REDACTED]"', + ); }; diff --git a/packages/server/src/utils/backups/utils.ts b/packages/server/src/utils/backups/utils.ts index 4253a3876..ee3795f71 100644 --- a/packages/server/src/utils/backups/utils.ts +++ b/packages/server/src/utils/backups/utils.ts @@ -145,7 +145,7 @@ export const getComposeContainerCommand = ( return `docker ps -q --filter "status=running" --filter "label=com.docker.compose.project=${appName}" --filter "label=com.docker.compose.service=${serviceName}" | head -n 1`; }; -const getContainerSearchCommand = (backup: BackupSchedule) => { +export const getContainerSearchCommand = (backup: BackupSchedule) => { const { backupType, postgres, @@ -262,16 +262,30 @@ export const getBackupCommand = ( rcloneFlags: string[], rcloneDestination: string, logPath: string, + options: { containerId?: string } = {}, ) => { const containerSearch = getContainerSearchCommand(backup); const backupCommand = generateBackupCommand(backup); - const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`; - const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} "${rcloneDestination}"`; + const quotedRcloneDestination = quote([rcloneDestination]); + const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} ${quotedRcloneDestination}`; + const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} ${quotedRcloneDestination}`; + const quotedLogPath = quote([logPath]); + const containerId = options.containerId; + if (containerId && !/^[a-f0-9]{12,64}$/i.test(containerId)) { + throw new Error("Invalid backup container ID"); + } + // A Swarm task can restart while the helper image starts, so re-resolve the + // original service locally if the captured task container is no longer running. + const containerAssignment = containerId + ? `CONTAINER_ID=${quote([containerId])}; + if [ "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID" 2>/dev/null)" != "true" ]; then + CONTAINER_ID=$(${containerSearch}); + fi;` + : `CONTAINER_ID=$(${containerSearch});`; logger.info( { containerSearch, - backupCommand, rcloneCommand: redactRcloneCredentials(rcloneCommand), logPath, }, @@ -280,26 +294,26 @@ export const getBackupCommand = ( return ` set -eo pipefail; - echo "[$(date)] Starting backup process..." >> ${logPath}; - echo "[$(date)] Executing backup command..." >> ${logPath}; - CONTAINER_ID=$(${containerSearch}); + echo "[$(date)] Starting backup process..." >> ${quotedLogPath}; + echo "[$(date)] Executing backup command..." >> ${quotedLogPath}; + ${containerAssignment} if [ -z "$CONTAINER_ID" ]; then - echo "[$(date)] ❌ Error: Container not found" >> ${logPath}; + echo "[$(date)] ❌ Error: Container not found" >> ${quotedLogPath}; exit 1; fi; - echo "[$(date)] Container Up: $CONTAINER_ID" >> ${logPath}; - echo "[$(date)] Starting backup and upload to S3..." >> ${logPath}; + echo "[$(date)] Container Up: $CONTAINER_ID" >> ${quotedLogPath}; + echo "[$(date)] Starting backup and upload to S3..." >> ${quotedLogPath}; UPLOAD_OUTPUT=$({ ${backupCommand} | ${rcloneCommand}; } 2>&1 >/dev/null) || { - echo "[$(date)] ❌ Error: Backup failed" >> ${logPath}; - echo "Error: $UPLOAD_OUTPUT" >> ${logPath}; + echo "[$(date)] ❌ Error: Backup failed" >> ${quotedLogPath}; + echo "Error: $UPLOAD_OUTPUT" >> ${quotedLogPath}; ${rcloneDeleteCommand} >/dev/null 2>&1 || true; exit 1; }; - echo "[$(date)] ✅ Backup uploaded to S3 successfully" >> ${logPath}; - echo "Backup done ✅" >> ${logPath}; + echo "[$(date)] ✅ Backup uploaded to S3 successfully" >> ${quotedLogPath}; + echo "Backup done ✅" >> ${quotedLogPath}; `; }; diff --git a/packages/server/src/utils/backups/worker.ts b/packages/server/src/utils/backups/worker.ts new file mode 100644 index 000000000..57a961f7d --- /dev/null +++ b/packages/server/src/utils/backups/worker.ts @@ -0,0 +1,240 @@ +import { createHash } from "node:crypto"; +import type { BackupSchedule } from "@dokploy/server/services/backup"; +import type { CreateServiceOptions } from "dockerode"; +import { sleep } from "../process/execAsync"; +import type { getRemoteDocker } from "../servers/remote-docker"; + +const BACKUP_WORKER_IMAGE = "docker:28.5.2-cli"; +const BACKUP_SCRIPT_PATH = "/run/secrets/dokploy-backup-script"; +const DEFAULT_POLL_INTERVAL_MS = 1_000; +const DEFAULT_START_TIMEOUT_MS = 5 * 60 * 1_000; +const DEFAULT_MISSING_TASK_GRACE_POLLS = 3; + +const terminalFailureStates = new Set([ + "failed", + "rejected", + "shutdown", + "orphaned", + "remove", +]); + +export type DockerClient = Awaited>; + +type SwarmTask = { + ID?: string; + NodeID?: string; + Version?: { Index?: number }; + Status?: { + State?: string; + Err?: string; + Message?: string; + ContainerStatus?: { + ContainerID?: string; + ExitCode?: number; + }; + }; +}; + +type WaitForTaskOptions = { + missingTaskGracePolls?: number; + pollIntervalMs?: number; + startTimeoutMs?: number; + sleepFn?: (milliseconds: number) => Promise; +}; + +export const getBackupTargetServiceName = ( + backup: BackupSchedule, +): string | null => { + if (backup.backupType === "database") { + return ( + backup.postgres?.appName || + backup.mysql?.appName || + backup.mariadb?.appName || + backup.mongo?.appName || + backup.libsql?.appName || + null + ); + } + + return null; +}; + +export const getBackupResourceNames = (executionId: string) => { + const suffix = createHash("sha256") + .update(executionId) + .digest("hex") + .slice(0, 16); + const serviceName = `dokploy-backup-${suffix}`; + + return { + secretName: `${serviceName}-script`, + serviceName, + }; +}; + +export const findRunningServiceTask = async ( + docker: DockerClient, + serviceName: string, +) => { + const tasks = (await docker.listTasks({ + filters: JSON.stringify({ + service: [serviceName], + "desired-state": ["running"], + }), + })) as SwarmTask[]; + + const task = tasks.find( + (candidate) => + candidate.Status?.State === "running" && + candidate.NodeID && + candidate.Status.ContainerStatus?.ContainerID, + ); + + const nodeId = task?.NodeID; + const containerId = task?.Status?.ContainerStatus?.ContainerID; + if (!nodeId || !containerId) { + throw new Error( + `No running Swarm task found for database service ${serviceName}`, + ); + } + + return { containerId, nodeId }; +}; + +const getLatestTask = (tasks: SwarmTask[]) => + [...tasks].sort( + (left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0), + )[0]; + +const getTaskFailureMessage = (task: SwarmTask) => { + const state = task.Status?.State ?? "unknown"; + const detail = task.Status?.Err || task.Status?.Message; + const exitCode = task.Status?.ContainerStatus?.ExitCode; + const suffix = [ + detail, + exitCode !== undefined ? `exit code ${exitCode}` : undefined, + ] + .filter(Boolean) + .join(", "); + + return `Backup worker task ${state}${suffix ? `: ${suffix}` : ""}`; +}; + +export const waitForBackupWorkerTask = async ( + docker: DockerClient, + serviceId: string, + options: WaitForTaskOptions = {}, +) => { + const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const startTimeoutMs = options.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS; + const missingTaskGracePolls = Math.max( + 1, + options.missingTaskGracePolls ?? DEFAULT_MISSING_TASK_GRACE_POLLS, + ); + const sleepFn = options.sleepFn ?? sleep; + const startedAt = Date.now(); + let hasStarted = false; + let missingTaskPolls = 0; + + while (true) { + const tasks = (await docker.listTasks({ + filters: JSON.stringify({ service: [serviceId] }), + })) as SwarmTask[]; + const task = getLatestTask(tasks); + const state = task?.Status?.State; + + if (state === "complete") { + return; + } + if (task && state && terminalFailureStates.has(state)) { + throw new Error(getTaskFailureMessage(task)); + } + if (state === "running") { + hasStarted = true; + } + if (hasStarted && !task) { + missingTaskPolls += 1; + if (missingTaskPolls >= missingTaskGracePolls) { + throw new Error("Backup worker task disappeared after starting"); + } + } else { + missingTaskPolls = 0; + } + + if (!hasStarted && Date.now() - startedAt >= startTimeoutMs) { + throw new Error( + `Backup worker did not start within ${Math.round(startTimeoutMs / 1_000)} seconds`, + ); + } + + await sleepFn(pollIntervalMs); + } +}; + +export const getBackupWorkerServiceSpec = ({ + backupId, + executionId, + nodeId, + secretId, + secretName, + serviceName, +}: { + backupId: string; + executionId: string; + nodeId: string; + secretId: string; + secretName: string; + serviceName: string; +}): CreateServiceOptions => { + const labels = { + "dokploy.backup.id": backupId, + "dokploy.deployment.id": executionId, + "dokploy.managed": "true", + "dokploy.resource": "database-backup-worker", + }; + + return { + Name: serviceName, + Labels: labels, + TaskTemplate: { + ContainerSpec: { + Image: BACKUP_WORKER_IMAGE, + Command: ["/bin/sh", "-c"], + Args: [ + `apk add --no-cache bash rclone >/dev/null && exec /bin/bash ${BACKUP_SCRIPT_PATH}`, + ], + Labels: labels, + Mounts: [ + { + Type: "bind", + Source: "/var/run/docker.sock", + Target: "/var/run/docker.sock", + }, + ], + Secrets: [ + { + SecretID: secretId, + SecretName: secretName, + File: { + Name: BACKUP_SCRIPT_PATH.replace("/run/secrets/", ""), + UID: "0", + GID: "0", + Mode: 0o400, + }, + }, + ], + }, + Placement: { + Constraints: [`node.id==${nodeId}`], + }, + RestartPolicy: { + Condition: "none", + }, + }, + Mode: { + Replicated: { + Replicas: 1, + }, + }, + }; +}; From 4f2c61dc64e5c1df6dbf05227401e9ca1ad064e6 Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:36:48 +0000 Subject: [PATCH 2/7] fix(backups): harden worker lifecycle and logs --- .../backups/swarm-backup-executor.test.ts | 297 +++++++++++++++++- packages/server/src/utils/backups/executor.ts | 13 +- packages/server/src/utils/backups/worker.ts | 93 +++++- 3 files changed, 376 insertions(+), 27 deletions(-) diff --git a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts index 49a04f19c..bb3473464 100644 --- a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts +++ b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts @@ -134,15 +134,7 @@ const createDockerMock = () => { beforeEach(() => { vi.clearAllMocks(); mocks.sleep.mockResolvedValue(undefined); - mocks.execAsync.mockImplementation(async (command: string) => { - if (command.includes("nohup docker service logs")) { - return { stdout: "4321\n", stderr: "" }; - } - if (command.startsWith("if ps -p")) { - return { stdout: "running\n", stderr: "" }; - } - return { stdout: "", stderr: "" }; - }); + mocks.execAsync.mockResolvedValue({ stdout: "", stderr: "" }); mocks.execAsyncRemote.mockResolvedValue({ stdout: "", stderr: "" }); }); @@ -195,6 +187,13 @@ describe("backup worker target and service specification", () => { "node.id==worker-node-id", ]); expect(spec.TaskTemplate?.RestartPolicy?.Condition).toBe("none"); + expect(spec.TaskTemplate?.LogDriver).toEqual({ + Name: "json-file", + Options: { + "max-size": "10m", + "max-file": "1", + }, + }); expect(containerSpec?.Mounts).toContainEqual({ Type: "bind", Source: "/var/run/docker.sock", @@ -214,13 +213,25 @@ describe("backup worker task lifecycle", () => { listTasks: vi .fn() .mockResolvedValueOnce([ - { Status: { State: "pending" }, Version: { Index: 1 } }, + { + ID: "worker-task", + Status: { State: "pending" }, + Version: { Index: 1 }, + }, ]) .mockResolvedValueOnce([ - { Status: { State: "running" }, Version: { Index: 2 } }, + { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 2 }, + }, ]) .mockResolvedValueOnce([ - { Status: { State: "complete" }, Version: { Index: 3 } }, + { + ID: "worker-task", + Status: { State: "complete" }, + Version: { Index: 3 }, + }, ]), }; const sleepFn = vi.fn().mockResolvedValue(undefined); @@ -276,7 +287,11 @@ describe("backup worker task lifecycle", () => { listTasks: vi .fn() .mockResolvedValueOnce([ - { Status: { State: "running" }, Version: { Index: 1 } }, + { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }, ]) .mockResolvedValue([]), }; @@ -289,6 +304,234 @@ describe("backup worker task lifecycle", () => { }), ).rejects.toThrow("Backup worker task disappeared after starting"); }); + + it("fails instead of waiting forever on a replacement pending task", async () => { + const replacementTask = { + ID: "replacement-task", + Status: { State: "pending" }, + Version: { Index: 2 }, + }; + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }, + ]) + .mockResolvedValueOnce([replacementTask]) + .mockResolvedValueOnce([replacementTask]) + .mockResolvedValue([ + { + ...replacementTask, + Status: { State: "failed", Err: "replacement failed" }, + Version: { Index: 3 }, + }, + ]), + }; + const sleepFn = vi.fn().mockResolvedValue(undefined); + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + missingTaskGracePolls: 2, + pollIntervalMs: 0, + sleepFn, + }), + ).rejects.toThrow( + "Backup worker task worker-task was replaced after starting by task replacement-task (pending)", + ); + expect(docker.listTasks).toHaveBeenCalledTimes(3); + expect(sleepFn).toHaveBeenCalledTimes(2); + }); + + it("fails when a replacement appears beside the task already running", async () => { + const runningTask = { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }; + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([runningTask]) + .mockResolvedValueOnce([ + runningTask, + { + ID: "replacement-task", + Status: { State: "pending" }, + Version: { Index: 2 }, + }, + ]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + pollIntervalMs: 0, + sleepFn: vi.fn().mockResolvedValue(undefined), + }), + ).rejects.toThrow( + "Backup worker task worker-task was replaced after starting by task replacement-task (pending)", + ); + }); + + it("detects a replacement that completes between lifecycle polls", async () => { + const runningTask = { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }; + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([runningTask]) + .mockResolvedValueOnce([ + runningTask, + { + ID: "replacement-task", + Status: { State: "complete" }, + Version: { Index: 2 }, + }, + ]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + pollIntervalMs: 0, + sleepFn: vi.fn().mockResolvedValue(undefined), + }), + ).rejects.toThrow( + "Backup worker task worker-task was replaced after starting by task replacement-task (complete)", + ); + }); + + it("rejects duplicate completed task attempts", async () => { + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }, + ]) + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "complete" }, + Version: { Index: 2 }, + }, + { + ID: "replacement-task", + Status: { State: "complete" }, + Version: { Index: 3 }, + }, + ]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + pollIntervalMs: 0, + sleepFn: vi.fn().mockResolvedValue(undefined), + }), + ).rejects.toThrow( + "Backup worker task worker-task was replaced after starting by task replacement-task (complete)", + ); + }); + + it("fails closed when multiple task attempts exist on the first poll", async () => { + const docker = { + listTasks: vi.fn().mockResolvedValue([ + { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }, + { + ID: "replacement-task", + Status: { State: "pending" }, + Version: { Index: 2 }, + }, + ]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id"), + ).rejects.toThrow( + "Backup worker service reported multiple task attempts before execution could be tracked: replacement-task (pending), worker-task (running)", + ); + }); + + it("allows one running task to outlive the start timeout", async () => { + const runningTask = { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }; + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([runningTask]) + .mockResolvedValueOnce([runningTask]) + .mockResolvedValueOnce([runningTask]) + .mockResolvedValueOnce([ + { + ...runningTask, + Status: { State: "complete" }, + Version: { Index: 2 }, + }, + ]), + }; + const sleepFn = vi.fn().mockResolvedValue(undefined); + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + pollIntervalMs: 0, + sleepFn, + startTimeoutMs: 0, + }), + ).resolves.toBeUndefined(); + expect(sleepFn).toHaveBeenCalledTimes(3); + }); + + it("reports the started task failure instead of following its replacement", async () => { + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "running" }, + Version: { Index: 1 }, + }, + ]) + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { + State: "failed", + Err: "node lost", + ContainerStatus: { ExitCode: 1 }, + }, + Version: { Index: 2 }, + }, + { + ID: "replacement-task", + Status: { State: "pending" }, + Version: { Index: 3 }, + }, + ]), + }; + + await expect( + waitForBackupWorkerTask(docker as never, "service-id", { + pollIntervalMs: 0, + sleepFn: vi.fn().mockResolvedValue(undefined), + }), + ).rejects.toThrow("Backup worker task failed: node lost, exit code 1"); + }); }); describe("executeBackup", () => { @@ -428,6 +671,34 @@ describe("executeBackup", () => { ); }); + it("keeps a completed backup successful when service logs cannot be read", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + mocks.getRemoteDocker.mockResolvedValue(docker); + mocks.execAsync.mockImplementation(async (command: string) => { + if (command.includes("docker service logs --raw")) { + throw new Error("service logging driver cannot be read"); + } + return { stdout: "", stderr: "" }; + }); + + await expect(executeBackup(input())).resolves.toEqual({ + mode: "swarm-worker", + }); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + expect(mocks.loggerError).toHaveBeenCalledWith( + { error: "service logging driver cannot be read" }, + "Failed to collect backup worker logs", + ); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes( + "⚠️ Warning: Could not collect backup worker logs: service logging driver cannot be read", + ), + ), + ).toBe(true); + }); + it("fails before creating resources when no running database task exists", async () => { const { docker } = createDockerMock(); docker.listTasks.mockReset(); diff --git a/packages/server/src/utils/backups/executor.ts b/packages/server/src/utils/backups/executor.ts index e45e3f969..f764d69d9 100644 --- a/packages/server/src/utils/backups/executor.ts +++ b/packages/server/src/utils/backups/executor.ts @@ -194,10 +194,15 @@ const runBackupOnWorker = async ({ try { await collectServiceLogs(serverId, serviceName, logPath); } catch (error) { - cleanupErrors.push( - new Error( - `Failed to collect backup worker logs: ${getSafeErrorMessage(error)}`, - ), + const message = getSafeErrorMessage(error); + logger.error( + { error: message }, + "Failed to collect backup worker logs", + ); + await appendBackupMessage( + serverId, + logPath, + `⚠️ Warning: Could not collect backup worker logs: ${message}`, ); } diff --git a/packages/server/src/utils/backups/worker.ts b/packages/server/src/utils/backups/worker.ts index 57a961f7d..9381b301d 100644 --- a/packages/server/src/utils/backups/worker.ts +++ b/packages/server/src/utils/backups/worker.ts @@ -106,6 +106,31 @@ const getLatestTask = (tasks: SwarmTask[]) => (left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0), )[0]; +const getReplacementTask = (tasks: SwarmTask[], startedTaskId: string) => + getLatestTask(tasks.filter((task) => task.ID !== startedTaskId)); + +const getUniqueTaskAttempts = (tasks: SwarmTask[]) => { + const seen = new Set(); + return [...tasks] + .sort( + (left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0), + ) + .filter((task) => { + if (!task.ID || seen.has(task.ID)) return false; + seen.add(task.ID); + return true; + }); +}; + +const getTaskSummary = (task: SwarmTask) => + `${task.ID ?? "unknown"} (${task.Status?.State ?? "unknown"})`; + +const getTaskReplacementMessage = ( + startedTaskId: string, + replacement: SwarmTask, +) => + `Backup worker task ${startedTaskId} was replaced after starting by task ${getTaskSummary(replacement)}`; + const getTaskFailureMessage = (task: SwarmTask) => { const state = task.Status?.State ?? "unknown"; const detail = task.Status?.Err || task.Status?.Message; @@ -133,13 +158,59 @@ export const waitForBackupWorkerTask = async ( ); const sleepFn = options.sleepFn ?? sleep; const startedAt = Date.now(); - let hasStarted = false; + let startedTaskId: string | null = null; let missingTaskPolls = 0; while (true) { const tasks = (await docker.listTasks({ filters: JSON.stringify({ service: [serviceId] }), })) as SwarmTask[]; + + if (startedTaskId) { + const task = tasks.find((candidate) => candidate.ID === startedTaskId); + const state = task?.Status?.State; + + if (task && state && terminalFailureStates.has(state)) { + throw new Error(getTaskFailureMessage(task)); + } + + const replacement = getReplacementTask(tasks, startedTaskId); + if (task && replacement) { + throw new Error(getTaskReplacementMessage(startedTaskId, replacement)); + } + + if (state === "complete") { + return; + } + if (state === "running") { + missingTaskPolls = 0; + } else if (task) { + throw new Error( + `Backup worker task ${startedTaskId} entered ${state ?? "an unknown state"} after starting`, + ); + } else { + missingTaskPolls += 1; + if (missingTaskPolls >= missingTaskGracePolls) { + if (replacement) { + throw new Error( + getTaskReplacementMessage(startedTaskId, replacement), + ); + } + throw new Error("Backup worker task disappeared after starting"); + } + } + + await sleepFn(pollIntervalMs); + continue; + } + + const taskAttempts = getUniqueTaskAttempts(tasks); + if (taskAttempts.length > 1) { + throw new Error( + `Backup worker service reported multiple task attempts before execution could be tracked: ${taskAttempts.map(getTaskSummary).join(", ")}`, + ); + } + const task = getLatestTask(tasks); const state = task?.Status?.State; @@ -150,18 +221,13 @@ export const waitForBackupWorkerTask = async ( throw new Error(getTaskFailureMessage(task)); } if (state === "running") { - hasStarted = true; - } - if (hasStarted && !task) { - missingTaskPolls += 1; - if (missingTaskPolls >= missingTaskGracePolls) { - throw new Error("Backup worker task disappeared after starting"); + if (!task?.ID) { + throw new Error("Backup worker running task did not provide an ID"); } - } else { - missingTaskPolls = 0; + startedTaskId = task.ID; } - if (!hasStarted && Date.now() - startedAt >= startTimeoutMs) { + if (!startedTaskId && Date.now() - startedAt >= startTimeoutMs) { throw new Error( `Backup worker did not start within ${Math.round(startTimeoutMs / 1_000)} seconds`, ); @@ -224,6 +290,13 @@ export const getBackupWorkerServiceSpec = ({ }, ], }, + LogDriver: { + Name: "json-file", + Options: { + "max-size": "10m", + "max-file": "1", + }, + }, Placement: { Constraints: [`node.id==${nodeId}`], }, From 21a4dbaf51da7e6af92697c65f55dfeb102a970a Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:42:41 +0000 Subject: [PATCH 3/7] fix(backups): use dockerode resource ids --- .../__test__/backups/swarm-backup-executor.test.ts | 11 +++++++++-- packages/server/src/utils/backups/executor.ts | 14 ++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts index bb3473464..813bce60d 100644 --- a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts +++ b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts @@ -112,8 +112,8 @@ const createDockerMock = () => { const secretRemove = vi.fn().mockResolvedValue(undefined); const serviceRemove = vi.fn().mockResolvedValue(undefined); const docker = { - createSecret: vi.fn().mockResolvedValue({ ID: "secret-id" }), - createService: vi.fn().mockResolvedValue({ ID: "service-id" }), + createSecret: vi.fn().mockResolvedValue({ id: "secret-id" }), + createService: vi.fn().mockResolvedValue({ id: "service-id" }), getSecret: vi.fn(() => ({ remove: secretRemove })), getService: vi.fn(() => ({ remove: serviceRemove })), listTasks: vi @@ -589,6 +589,13 @@ describe("executeBackup", () => { expect(serviceSpec.TaskTemplate.Placement.Constraints).toEqual([ "node.id==worker-node-id", ]); + expect(serviceSpec.TaskTemplate.ContainerSpec.Secrets).toEqual([ + expect.objectContaining({ SecretID: "secret-id" }), + ]); + const workerTaskFilters = JSON.parse( + docker.listTasks.mock.calls[1]?.[0].filters, + ); + expect(workerTaskFilters).toEqual({ service: ["service-id"] }); expect(JSON.stringify(serviceSpec)).not.toContain("ACCESS_KEY"); expect(JSON.stringify(serviceSpec)).not.toContain("SECRET_KEY"); expect(serviceRemove).toHaveBeenCalledOnce(); diff --git a/packages/server/src/utils/backups/executor.ts b/packages/server/src/utils/backups/executor.ts index f764d69d9..07b91fd58 100644 --- a/packages/server/src/utils/backups/executor.ts +++ b/packages/server/src/utils/backups/executor.ts @@ -165,15 +165,14 @@ const runBackupOnWorker = async ({ Name: secretName, Labels: labels, Data: Buffer.from(workerCommand).toString("base64"), - })) as { ID?: string }; - const secretId = - createdSecret.ID || ((await secret.inspect()) as { ID?: string }).ID; + })) as { id?: string }; + const secretId = createdSecret.id; if (!secretId) { throw new Error("Docker did not return a backup worker secret ID"); } serviceMayExist = true; - const createdService = (await docker.createService( + const createdService = await docker.createService( getBackupWorkerServiceSpec({ backupId: backup.backupId, executionId, @@ -182,8 +181,11 @@ const runBackupOnWorker = async ({ secretName, serviceName, }), - )) as { ID?: string }; - await waitForBackupWorkerTask(docker, createdService.ID || serviceName); + ); + if (!createdService.id) { + throw new Error("Docker did not return a backup worker service ID"); + } + await waitForBackupWorkerTask(docker, createdService.id); } catch (error) { primaryError = error; } finally { From 2b8d2cb9d66caabd96b72573094f628a1e13358a Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:45:07 +0000 Subject: [PATCH 4/7] [autofix.ci] apply automated fixes --- scripts/find-free-port.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/find-free-port.mjs b/scripts/find-free-port.mjs index 2e647f504..9f9a2cc66 100644 --- a/scripts/find-free-port.mjs +++ b/scripts/find-free-port.mjs @@ -18,6 +18,9 @@ async function findFreePort(start) { return port; } -const start = Number.parseInt(process.argv[2] || process.env.PORT || "3000", 10); +const start = Number.parseInt( + process.argv[2] || process.env.PORT || "3000", + 10, +); const port = await findFreePort(start); process.stdout.write(String(port)); From 3ff286e03ef1e7d5bb733444ecb81ee89eed7c4d Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:01:03 +0000 Subject: [PATCH 5/7] fix(backups): retry worker backup after task relocation --- .../backups/swarm-backup-executor.test.ts | 431 +++++++++++++++++- packages/server/src/utils/backups/executor.ts | 172 +++++-- packages/server/src/utils/backups/utils.ts | 29 +- packages/server/src/utils/backups/worker.ts | 103 ++++- 4 files changed, 672 insertions(+), 63 deletions(-) diff --git a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts index 813bce60d..82433be77 100644 --- a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts +++ b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts @@ -1,12 +1,19 @@ import type { BackupSchedule } from "@dokploy/server/services/backup"; import { executeBackup } from "@dokploy/server/utils/backups/executor"; -import { getBackupCommand } from "@dokploy/server/utils/backups/utils"; import { + BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + getBackupCommand, +} from "@dokploy/server/utils/backups/utils"; +import { + BackupWorkerTaskError, + findRunningServiceTask, getBackupResourceNames, getBackupTargetServiceName, getBackupWorkerServiceSpec, waitForBackupWorkerTask, + waitForReplacementServiceTask, } from "@dokploy/server/utils/backups/worker"; +import { ExecError } from "@dokploy/server/utils/process/execAsync"; import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ @@ -157,15 +164,102 @@ describe("backup worker target and service specification", () => { it("creates deterministic, collision-resistant names per execution", () => { const first = getBackupResourceNames("deployment-1"); - const retry = getBackupResourceNames("deployment-1"); + const repeated = getBackupResourceNames("deployment-1"); + const retry = getBackupResourceNames("deployment-1", 1); const concurrent = getBackupResourceNames("deployment-2"); - expect(first).toEqual(retry); + expect(first).toEqual(repeated); expect(first.serviceName).toMatch(/^dokploy-backup-[a-f0-9]{16}$/); expect(first.secretName).toBe(`${first.serviceName}-script`); + expect(retry.serviceName).not.toBe(first.serviceName); expect(concurrent.serviceName).not.toBe(first.serviceName); }); + it("chooses the newest running database task", async () => { + const docker = { + listTasks: vi.fn().mockResolvedValue([ + { + ...runningDatabaseTask, + ID: "old-database-task", + NodeID: "old-node", + Version: { Index: 1 }, + }, + { + ...runningDatabaseTask, + ID: "new-database-task", + NodeID: "new-node", + Status: { + State: "running", + ContainerStatus: { ContainerID: "b".repeat(64) }, + }, + Version: { Index: 2 }, + }, + ]), + }; + + await expect( + findRunningServiceTask(docker as never, "postgres-service"), + ).resolves.toEqual({ + containerId: "b".repeat(64), + nodeId: "new-node", + }); + }); + + it("waits past a stale full task ID when discovery returned a short ID", async () => { + const replacementContainerId = "b".repeat(64); + const docker = { + listTasks: vi + .fn() + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ...runningDatabaseTask, + ID: "replacement-database-task", + NodeID: "new-node", + Status: { + State: "running", + ContainerStatus: { ContainerID: replacementContainerId }, + }, + Version: { Index: 2 }, + }, + ]), + }; + const sleepFn = vi.fn().mockResolvedValue(undefined); + + await expect( + waitForReplacementServiceTask( + docker as never, + "postgres-service", + containerId.slice(0, 12), + { maxPolls: 2, pollIntervalMs: 0, sleepFn }, + ), + ).resolves.toEqual({ + containerId: replacementContainerId, + nodeId: "new-node", + }); + expect(sleepFn).toHaveBeenCalledOnce(); + }); + + it("bounds the wait for a replacement database task", async () => { + const docker = { + listTasks: vi.fn().mockResolvedValue([runningDatabaseTask]), + }; + const sleepFn = vi.fn().mockResolvedValue(undefined); + + await expect( + waitForReplacementServiceTask( + docker as never, + "postgres-service", + containerId, + { maxPolls: 2, pollIntervalMs: 0, sleepFn }, + ), + ).rejects.toThrow( + "No replacement Swarm task became ready for database service postgres-service after relocation", + ); + expect(docker.listTasks).toHaveBeenCalledTimes(2); + expect(sleepFn).toHaveBeenCalledOnce(); + }); + it("pins a one-shot worker to the database node without exposing credentials", () => { const spec = getBackupWorkerServiceSpec({ backupId: "backup-1", @@ -202,6 +296,11 @@ describe("backup worker target and service specification", () => { expect(containerSpec?.Secrets).toContainEqual( expect.objectContaining({ SecretID: "secret-id" }), ); + expect(containerSpec?.Args).toEqual([ + expect.stringContaining( + "apk add --no-cache bash rclone >/dev/null || exit 1; exec /bin/bash", + ), + ]); expect(JSON.stringify(spec)).not.toContain("ACCESS_KEY"); expect(JSON.stringify(spec)).not.toContain("SECRET_KEY"); }); @@ -259,9 +358,17 @@ describe("backup worker task lifecycle", () => { ]), }; - await expect( - waitForBackupWorkerTask(docker as never, "service-id"), - ).rejects.toThrow( + const error = await waitForBackupWorkerTask( + docker as never, + "service-id", + ).catch((caught) => caught); + + expect(error).toBeInstanceOf(BackupWorkerTaskError); + expect(error).toMatchObject({ + exitCode: 1, + state: "failed", + }); + expect((error as Error).message).toBe( "Backup worker task failed: image pull failed, exit code 1", ); }); @@ -558,9 +665,96 @@ describe("executeBackup", () => { expect(mocks.execAsync.mock.calls[1]?.[0]).toContain( "label=com.docker.swarm.service.name", ); + expect(mocks.execAsync.mock.calls[1]?.[0]).toContain("exit 75;"); expect(docker.createService).not.toHaveBeenCalled(); }); + it("does not switch to a worker after a direct backup failure", async () => { + const { docker } = createDockerMock(); + mocks.getRemoteDocker.mockResolvedValue(docker); + mocks.execAsync + .mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "" }) + .mockRejectedValueOnce( + new ExecError("direct backup failed", { + command: "backup", + exitCode: 1, + stderr: "dump failed", + }), + ); + + await expect(executeBackup(input())).rejects.toThrow( + "direct backup failed", + ); + + expect(mocks.getRemoteDocker).not.toHaveBeenCalled(); + expect(docker.createService).not.toHaveBeenCalled(); + }); + + it("switches to a worker when the direct database task moves away", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + const relocatedContainerId = "b".repeat(64); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ...runningDatabaseTask, + ID: "relocated-database-task", + NodeID: "worker-node-b", + Status: { + State: "running", + ContainerStatus: { ContainerID: relocatedContainerId }, + }, + Version: { Index: 2 }, + }, + ]) + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "complete", ContainerStatus: { ExitCode: 0 } }, + Version: { Index: 3 }, + }, + ]); + mocks.getRemoteDocker.mockResolvedValue(docker); + mocks.execAsync + .mockResolvedValueOnce({ + stdout: `${containerId.slice(0, 12)}\n`, + stderr: "", + }) + .mockRejectedValueOnce( + new ExecError("direct database task moved", { + command: "backup", + exitCode: BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + }), + ); + + await expect(executeBackup(input())).resolves.toEqual({ + mode: "swarm-worker", + }); + + expect(docker.createService).toHaveBeenCalledOnce(); + expect( + docker.createService.mock.calls[0]?.[0].TaskTemplate.Placement + .Constraints, + ).toEqual(["node.id==worker-node-b"]); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + expect(mocks.execAsync.mock.calls[1]?.[0]).toContain("exit 75;"); + expect(mocks.execAsync.mock.calls[0]?.[0]).toContain( + "docker ps -q --no-trunc", + ); + expect(mocks.execAsync.mock.calls[1]?.[0]).toContain( + "Database container moved before the backup started", + ); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes( + "Database task moved off this node before the backup started; switching to a backup worker", + ), + ), + ).toBe(true); + }); + it("uses the exact worker container and removes all temporary resources", async () => { const { docker, secretRemove, serviceRemove } = createDockerMock(); mocks.getRemoteDocker.mockResolvedValue(docker); @@ -611,6 +805,192 @@ describe("executeBackup", () => { ).toBe(false); }); + it("rediscovers the database node and retries once after relocation", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + const relocatedContainerId = "b".repeat(64); + docker.createSecret + .mockResolvedValueOnce({ id: "secret-id-a" }) + .mockResolvedValueOnce({ id: "secret-id-b" }); + docker.createService + .mockResolvedValueOnce({ id: "service-id-a" }) + .mockResolvedValueOnce({ id: "service-id-b" }); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ID: "worker-task-a", + Status: { + State: "failed", + ContainerStatus: { + ExitCode: BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + }, + }, + Version: { Index: 2 }, + }, + ]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + ...runningDatabaseTask, + NodeID: "worker-node-b", + Status: { + State: "running", + ContainerStatus: { ContainerID: relocatedContainerId }, + }, + Version: { Index: 3 }, + }, + ]) + .mockResolvedValueOnce([ + { + ID: "worker-task-b", + Status: { State: "complete", ContainerStatus: { ExitCode: 0 } }, + Version: { Index: 4 }, + }, + ]); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).resolves.toEqual({ + mode: "swarm-worker", + }); + + expect(docker.createSecret).toHaveBeenCalledTimes(2); + expect(docker.createService).toHaveBeenCalledTimes(2); + const firstNames = getBackupResourceNames("deployment-1"); + const secondNames = getBackupResourceNames("deployment-1", 1); + expect(docker.createService.mock.calls[0]?.[0].Name).toBe( + firstNames.serviceName, + ); + expect(docker.createService.mock.calls[1]?.[0].Name).toBe( + secondNames.serviceName, + ); + expect(docker.getService).toHaveBeenNthCalledWith( + 1, + firstNames.serviceName, + ); + expect(docker.getService).toHaveBeenNthCalledWith( + 2, + secondNames.serviceName, + ); + expect(docker.getSecret).toHaveBeenNthCalledWith(1, firstNames.secretName); + expect(docker.getSecret).toHaveBeenNthCalledWith(2, secondNames.secretName); + expect( + docker.createService.mock.calls[0]?.[0].TaskTemplate.Placement + .Constraints, + ).toEqual(["node.id==worker-node-id"]); + expect( + docker.createService.mock.calls[1]?.[0].TaskTemplate.Placement + .Constraints, + ).toEqual(["node.id==worker-node-b"]); + const firstScript = Buffer.from( + docker.createSecret.mock.calls[0]?.[0].Data, + "base64", + ).toString(); + const secondScript = Buffer.from( + docker.createSecret.mock.calls[1]?.[0].Data, + "base64", + ).toString(); + expect(firstScript).toContain(`CONTAINER_ID=${containerId};`); + expect(secondScript).toContain(`CONTAINER_ID=${relocatedContainerId};`); + expect(serviceRemove).toHaveBeenCalledTimes(2); + expect(secretRemove).toHaveBeenCalledTimes(2); + expect(serviceRemove.mock.invocationCallOrder[0]).toBeLessThan( + docker.listTasks.mock.invocationCallOrder[2] ?? Number.POSITIVE_INFINITY, + ); + expect(secretRemove.mock.invocationCallOrder[0]).toBeLessThan( + docker.listTasks.mock.invocationCallOrder[2] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.sleep).toHaveBeenCalledOnce(); + expect( + mocks.execAsync.mock.calls.filter(([command]) => + command.includes("docker service logs --raw"), + ), + ).toHaveLength(1); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes( + "Database task moved before the backup started; rediscovering its node and retrying", + ), + ), + ).toBe(true); + }); + + it("stops after one relocation retry and cleans both attempts", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + const relocationFailure = { + Status: { + State: "failed", + ContainerStatus: { + ExitCode: BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + }, + }, + Version: { Index: 2 }, + }; + docker.createSecret + .mockResolvedValueOnce({ id: "secret-id-a" }) + .mockResolvedValueOnce({ id: "secret-id-b" }); + docker.createService + .mockResolvedValueOnce({ id: "service-id-a" }) + .mockResolvedValueOnce({ id: "service-id-b" }); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([relocationFailure]) + .mockResolvedValueOnce([ + { + ...runningDatabaseTask, + NodeID: "worker-node-b", + Status: { + State: "running", + ContainerStatus: { ContainerID: "b".repeat(64) }, + }, + Version: { Index: 3 }, + }, + ]) + .mockResolvedValueOnce([{ ...relocationFailure, Version: { Index: 4 } }]); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "Database task moved while the backup worker was starting; retry limit reached", + ); + expect(docker.createService).toHaveBeenCalledTimes(2); + expect(serviceRemove).toHaveBeenCalledTimes(2); + expect(secretRemove).toHaveBeenCalledTimes(2); + expect( + mocks.execAsync.mock.calls.filter(([command]) => + command.includes("docker service logs --raw"), + ), + ).toHaveLength(0); + }); + + it("does not retry relocation when cleanup is incomplete", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + Status: { + State: "failed", + ContainerStatus: { + ExitCode: BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + }, + }, + Version: { Index: 2 }, + }, + ]); + serviceRemove.mockRejectedValue(new Error("cleanup unavailable")); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "Failed to remove backup worker service: cleanup unavailable", + ); + expect(docker.createService).toHaveBeenCalledOnce(); + expect(docker.listTasks).toHaveBeenCalledTimes(2); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + }); + it("removes the worker and secret after a failed backup task", async () => { const { docker, secretRemove, serviceRemove } = createDockerMock(); docker.listTasks.mockReset(); @@ -633,6 +1013,8 @@ describe("executeBackup", () => { ); expect(serviceRemove).toHaveBeenCalledOnce(); expect(secretRemove).toHaveBeenCalledOnce(); + expect(docker.createService).toHaveBeenCalledOnce(); + expect(docker.listTasks).toHaveBeenCalledTimes(2); expect( mocks.execAsync.mock.calls.some(([command]) => command.includes("❌ Error: %s"), @@ -834,4 +1216,41 @@ describe("worker backup command safety", () => { }), ).toThrow("Invalid backup container ID"); }); + + it("uses the relocation exit code only before backup work begins", () => { + const command = getBackupCommand( + postgresBackup(), + ["--s3-provider=AWS"], + ":s3:backups/postgres.sql.gz", + "/proc/1/fd/1", + { + containerId, + containerNotFoundExitCode: BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + }, + ); + + expect(command.match(/exit 75;/g)).toHaveLength(1); + expect(command.match(/exit 1;/g)).toHaveLength(1); + expect(command).toContain( + "Database container moved before the backup started", + ); + expect(command).not.toContain("❌ Error: Container not found"); + expect(command.indexOf("exit 75;")).toBeLessThan( + command.indexOf("UPLOAD_OUTPUT="), + ); + expect(command.indexOf("exit 1;")).toBeGreaterThan( + command.indexOf("UPLOAD_OUTPUT="), + ); + }); + + it("rejects invalid container-not-found exit codes", () => { + for (const containerNotFoundExitCode of [0, 256, 1.5]) { + expect(() => + getBackupCommand(postgresBackup(), [], ":s3:bucket/file", "/tmp/log", { + containerId, + containerNotFoundExitCode, + }), + ).toThrow("Invalid container-not-found exit code"); + } + }); }); diff --git a/packages/server/src/utils/backups/executor.ts b/packages/server/src/utils/backups/executor.ts index 07b91fd58..04657fdca 100644 --- a/packages/server/src/utils/backups/executor.ts +++ b/packages/server/src/utils/backups/executor.ts @@ -4,15 +4,24 @@ import { quote } from "shell-quote"; import { ExecError, execAsync, execAsyncRemote } from "../process/execAsync"; import { getRemoteDocker } from "../servers/remote-docker"; import { redactRcloneCredentials } from "./redact"; -import { getBackupCommand, getContainerSearchCommand } from "./utils"; import { + BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + getBackupCommand, + getContainerSearchCommand, +} from "./utils"; +import { + BackupWorkerTaskError, + type DockerClient, findRunningServiceTask, getBackupResourceNames, getBackupTargetServiceName, getBackupWorkerServiceSpec, waitForBackupWorkerTask, + waitForReplacementServiceTask, } from "./worker"; +const MAX_BACKUP_WORKER_ATTEMPTS = 2; + type ExecuteBackupInput = { backup: BackupSchedule; executionId: string; @@ -113,25 +122,36 @@ const removeResource = async ( } }; -const runBackupOnWorker = async ({ - backup, - executionId, - logPath, - rcloneDestination, - rcloneFlags, - serverId, -}: ExecuteBackupInput) => { - const serviceTarget = getBackupTargetServiceName(backup); - if (!serviceTarget) { - throw new Error("Could not determine the Swarm service for this backup"); - } +const isDatabaseTaskRelocationError = (error: unknown) => + error instanceof BackupWorkerTaskError && + error.exitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE; - const docker = await getRemoteDocker(serverId); - const { containerId, nodeId } = await findRunningServiceTask( - docker, - serviceTarget, +const isDirectDatabaseTaskRelocationError = (error: unknown) => + error instanceof ExecError && + error.exitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE; + +const runBackupOnWorkerAttempt = async ({ + input: { + backup, + executionId, + logPath, + rcloneDestination, + rcloneFlags, + serverId, + }, + attempt, + databaseTask: { containerId, nodeId }, + docker, +}: { + input: ExecuteBackupInput; + attempt: number; + databaseTask: { containerId: string; nodeId: string }; + docker: DockerClient; +}) => { + const { secretName, serviceName } = getBackupResourceNames( + executionId, + attempt, ); - const { secretName, serviceName } = getBackupResourceNames(executionId); const labels = { "dokploy.backup.id": backup.backupId, "dokploy.deployment.id": executionId, @@ -144,7 +164,10 @@ const runBackupOnWorker = async ({ rcloneFlags, rcloneDestination, "/proc/1/fd/1", - { containerId }, + { + containerId, + containerNotFoundExitCode: BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + }, ); const service = docker.getService(serviceName); const secret = docker.getSecret(secretName); @@ -157,7 +180,7 @@ const runBackupOnWorker = async ({ await appendBackupMessage( serverId, logPath, - `Preparing backup worker on node ${nodeId}...`, + `Preparing backup worker attempt ${attempt + 1} of ${MAX_BACKUP_WORKER_ATTEMPTS} on node ${nodeId}...`, ); secretMayExist = true; @@ -193,19 +216,21 @@ const runBackupOnWorker = async ({ if (serviceMayExist) { // Collect once after the terminal task state to avoid follower races or duplicates. - try { - await collectServiceLogs(serverId, serviceName, logPath); - } catch (error) { - const message = getSafeErrorMessage(error); - logger.error( - { error: message }, - "Failed to collect backup worker logs", - ); - await appendBackupMessage( - serverId, - logPath, - `⚠️ Warning: Could not collect backup worker logs: ${message}`, - ); + if (!isDatabaseTaskRelocationError(primaryError)) { + try { + await collectServiceLogs(serverId, serviceName, logPath); + } catch (error) { + const message = getSafeErrorMessage(error); + logger.error( + { error: message }, + "Failed to collect backup worker logs", + ); + await appendBackupMessage( + serverId, + logPath, + `⚠️ Warning: Could not collect backup worker logs: ${message}`, + ); + } } const error = await removeResource(() => service.remove(), "service"); @@ -222,6 +247,11 @@ const runBackupOnWorker = async ({ { errors: cleanupErrors.map((error) => error.message) }, "Backup worker cleanup also failed", ); + if (isDatabaseTaskRelocationError(primaryError)) { + primaryError = new Error( + `${getSafeErrorMessage(primaryError)}; ${cleanupErrors.map((error) => error.message).join("; ")}`, + ); + } } } else if (cleanupErrors.length > 0) { primaryError = new Error( @@ -235,8 +265,60 @@ const runBackupOnWorker = async ({ } }; +const runBackupOnWorker = async ( + input: ExecuteBackupInput, + previousContainerId?: string, +) => { + const serviceTarget = getBackupTargetServiceName(input.backup); + if (!serviceTarget) { + throw new Error("Could not determine the Swarm service for this backup"); + } + + const docker = await getRemoteDocker(input.serverId); + let databaseTask = previousContainerId + ? await waitForReplacementServiceTask( + docker, + serviceTarget, + previousContainerId, + ) + : await findRunningServiceTask(docker, serviceTarget); + for (let attempt = 0; attempt < MAX_BACKUP_WORKER_ATTEMPTS; attempt += 1) { + try { + await runBackupOnWorkerAttempt({ + attempt, + databaseTask, + docker, + input, + }); + return; + } catch (error) { + if (!isDatabaseTaskRelocationError(error)) { + throw error; + } + if (attempt === MAX_BACKUP_WORKER_ATTEMPTS - 1) { + throw new Error( + `Database task moved while the backup worker was starting; retry limit reached (${getSafeErrorMessage(error)})`, + { cause: error }, + ); + } + + await appendBackupMessage( + input.serverId, + input.logPath, + "Database task moved before the backup started; rediscovering its node and retrying...", + ); + databaseTask = await waitForReplacementServiceTask( + docker, + serviceTarget, + databaseTask.containerId, + ); + } + } +}; + export const executeBackup = async (input: ExecuteBackupInput) => { try { + let previousContainerId: string | undefined; const containerSearch = getContainerSearchCommand(input.backup); if (!containerSearch) { throw new Error("Could not build the database container search command"); @@ -250,13 +332,29 @@ export const executeBackup = async (input: ExecuteBackupInput) => { input.rcloneFlags, input.rcloneDestination, input.logPath, - { containerId: directContainerId }, + { + containerId: directContainerId, + containerNotFoundExitCode: + BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE, + }, ); - await runHostCommand(input.serverId, command); - return { mode: "direct" as const }; + try { + await runHostCommand(input.serverId, command); + return { mode: "direct" as const }; + } catch (error) { + if (!isDirectDatabaseTaskRelocationError(error)) { + throw error; + } + await appendBackupMessage( + input.serverId, + input.logPath, + "Database task moved off this node before the backup started; switching to a backup worker...", + ); + previousContainerId = directContainerId; + } } - await runBackupOnWorker(input); + await runBackupOnWorker(input, previousContainerId); return { mode: "swarm-worker" as const }; } catch (error) { await appendBackupError(input.serverId, input.logPath, error); diff --git a/packages/server/src/utils/backups/utils.ts b/packages/server/src/utils/backups/utils.ts index ee3795f71..fabd9d492 100644 --- a/packages/server/src/utils/backups/utils.ts +++ b/packages/server/src/utils/backups/utils.ts @@ -131,7 +131,7 @@ export const getLibsqlBackupCommand = (database: string) => { }; export const getServiceContainerCommand = (appName: string) => { - return `docker ps -q --filter "status=running" --filter "label=com.docker.swarm.service.name=${appName}" | head -n 1`; + return `docker ps -q --no-trunc --filter "status=running" --filter "label=com.docker.swarm.service.name=${appName}" | head -n 1`; }; export const getComposeContainerCommand = ( @@ -140,9 +140,9 @@ export const getComposeContainerCommand = ( composeType: "stack" | "docker-compose" | undefined, ) => { if (composeType === "stack") { - return `docker ps -q --filter "status=running" --filter "label=com.docker.stack.namespace=${appName}" --filter "label=com.docker.swarm.service.name=${appName}_${serviceName}" | head -n 1`; + return `docker ps -q --no-trunc --filter "status=running" --filter "label=com.docker.stack.namespace=${appName}" --filter "label=com.docker.swarm.service.name=${appName}_${serviceName}" | head -n 1`; } - return `docker ps -q --filter "status=running" --filter "label=com.docker.compose.project=${appName}" --filter "label=com.docker.compose.service=${serviceName}" | head -n 1`; + return `docker ps -q --no-trunc --filter "status=running" --filter "label=com.docker.compose.project=${appName}" --filter "label=com.docker.compose.service=${serviceName}" | head -n 1`; }; export const getContainerSearchCommand = (backup: BackupSchedule) => { @@ -257,12 +257,17 @@ export const generateBackupCommand = (backup: BackupSchedule) => { return null; }; +export const BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE = 75; + export const getBackupCommand = ( backup: BackupSchedule, rcloneFlags: string[], rcloneDestination: string, logPath: string, - options: { containerId?: string } = {}, + options: { + containerId?: string; + containerNotFoundExitCode?: number; + } = {}, ) => { const containerSearch = getContainerSearchCommand(backup); const backupCommand = generateBackupCommand(backup); @@ -271,9 +276,21 @@ export const getBackupCommand = ( const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} ${quotedRcloneDestination}`; const quotedLogPath = quote([logPath]); const containerId = options.containerId; + const containerNotFoundExitCode = options.containerNotFoundExitCode ?? 1; + const containerNotFoundMessage = + containerNotFoundExitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE + ? "Database container moved before the backup started" + : "❌ Error: Container not found"; if (containerId && !/^[a-f0-9]{12,64}$/i.test(containerId)) { throw new Error("Invalid backup container ID"); } + if ( + !Number.isInteger(containerNotFoundExitCode) || + containerNotFoundExitCode < 1 || + containerNotFoundExitCode > 255 + ) { + throw new Error("Invalid container-not-found exit code"); + } // A Swarm task can restart while the helper image starts, so re-resolve the // original service locally if the captured task container is no longer running. const containerAssignment = containerId @@ -299,8 +316,8 @@ export const getBackupCommand = ( ${containerAssignment} if [ -z "$CONTAINER_ID" ]; then - echo "[$(date)] ❌ Error: Container not found" >> ${quotedLogPath}; - exit 1; + echo "[$(date)] ${containerNotFoundMessage}" >> ${quotedLogPath}; + exit ${containerNotFoundExitCode}; fi; echo "[$(date)] Container Up: $CONTAINER_ID" >> ${quotedLogPath}; diff --git a/packages/server/src/utils/backups/worker.ts b/packages/server/src/utils/backups/worker.ts index 9381b301d..d58299050 100644 --- a/packages/server/src/utils/backups/worker.ts +++ b/packages/server/src/utils/backups/worker.ts @@ -9,6 +9,7 @@ const BACKUP_SCRIPT_PATH = "/run/secrets/dokploy-backup-script"; const DEFAULT_POLL_INTERVAL_MS = 1_000; const DEFAULT_START_TIMEOUT_MS = 5 * 60 * 1_000; const DEFAULT_MISSING_TASK_GRACE_POLLS = 3; +const DEFAULT_REPLACEMENT_TASK_POLLS = 5 * 60; const terminalFailureStates = new Set([ "failed", @@ -42,6 +43,12 @@ type WaitForTaskOptions = { sleepFn?: (milliseconds: number) => Promise; }; +type WaitForReplacementTaskOptions = { + maxPolls?: number; + pollIntervalMs?: number; + sleepFn?: (milliseconds: number) => Promise; +}; + export const getBackupTargetServiceName = ( backup: BackupSchedule, ): string | null => { @@ -59,9 +66,9 @@ export const getBackupTargetServiceName = ( return null; }; -export const getBackupResourceNames = (executionId: string) => { +export const getBackupResourceNames = (executionId: string, attempt = 0) => { const suffix = createHash("sha256") - .update(executionId) + .update(attempt === 0 ? executionId : `${executionId}:retry:${attempt}`) .digest("hex") .slice(0, 16); const serviceName = `dokploy-backup-${suffix}`; @@ -72,6 +79,13 @@ export const getBackupResourceNames = (executionId: string) => { }; }; +export class RunningServiceTaskNotFoundError extends Error { + constructor(serviceName: string) { + super(`No running Swarm task found for database service ${serviceName}`); + this.name = "RunningServiceTaskNotFoundError"; + } +} + export const findRunningServiceTask = async ( docker: DockerClient, serviceName: string, @@ -83,24 +97,67 @@ export const findRunningServiceTask = async ( }), })) as SwarmTask[]; - const task = tasks.find( - (candidate) => - candidate.Status?.State === "running" && - candidate.NodeID && - candidate.Status.ContainerStatus?.ContainerID, - ); + const task = [...tasks] + .sort( + (left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0), + ) + .find( + (candidate) => + candidate.Status?.State === "running" && + candidate.NodeID && + candidate.Status.ContainerStatus?.ContainerID, + ); const nodeId = task?.NodeID; const containerId = task?.Status?.ContainerStatus?.ContainerID; if (!nodeId || !containerId) { - throw new Error( - `No running Swarm task found for database service ${serviceName}`, - ); + throw new RunningServiceTaskNotFoundError(serviceName); } return { containerId, nodeId }; }; +const isSameContainerId = (left: string, right: string) => + left === right || + (left.length >= 12 && + right.length >= 12 && + (left.startsWith(right) || right.startsWith(left))); + +export const waitForReplacementServiceTask = async ( + docker: DockerClient, + serviceName: string, + previousContainerId: string, + options: WaitForReplacementTaskOptions = {}, +) => { + const maxPolls = Math.max( + 1, + Math.floor(options.maxPolls ?? DEFAULT_REPLACEMENT_TASK_POLLS), + ); + const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const sleepFn = options.sleepFn ?? sleep; + + for (let poll = 0; poll < maxPolls; poll += 1) { + try { + const task = await findRunningServiceTask(docker, serviceName); + if (!isSameContainerId(task.containerId, previousContainerId)) { + return task; + } + } catch (error) { + if (!(error instanceof RunningServiceTaskNotFoundError)) { + throw error; + } + } + + if (poll < maxPolls - 1) { + await sleepFn(pollIntervalMs); + } + } + + throw new Error( + `No replacement Swarm task became ready for database service ${serviceName} after relocation`, + ); +}; + const getLatestTask = (tasks: SwarmTask[]) => [...tasks].sort( (left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0), @@ -145,6 +202,24 @@ const getTaskFailureMessage = (task: SwarmTask) => { return `Backup worker task ${state}${suffix ? `: ${suffix}` : ""}`; }; +export class BackupWorkerTaskError extends Error { + constructor( + message: string, + readonly state: string, + readonly exitCode?: number, + ) { + super(message); + this.name = "BackupWorkerTaskError"; + } +} + +const getTaskFailureError = (task: SwarmTask) => + new BackupWorkerTaskError( + getTaskFailureMessage(task), + task.Status?.State ?? "unknown", + task.Status?.ContainerStatus?.ExitCode, + ); + export const waitForBackupWorkerTask = async ( docker: DockerClient, serviceId: string, @@ -171,7 +246,7 @@ export const waitForBackupWorkerTask = async ( const state = task?.Status?.State; if (task && state && terminalFailureStates.has(state)) { - throw new Error(getTaskFailureMessage(task)); + throw getTaskFailureError(task); } const replacement = getReplacementTask(tasks, startedTaskId); @@ -218,7 +293,7 @@ export const waitForBackupWorkerTask = async ( return; } if (task && state && terminalFailureStates.has(state)) { - throw new Error(getTaskFailureMessage(task)); + throw getTaskFailureError(task); } if (state === "running") { if (!task?.ID) { @@ -267,7 +342,7 @@ export const getBackupWorkerServiceSpec = ({ Image: BACKUP_WORKER_IMAGE, Command: ["/bin/sh", "-c"], Args: [ - `apk add --no-cache bash rclone >/dev/null && exec /bin/bash ${BACKUP_SCRIPT_PATH}`, + `apk add --no-cache bash rclone >/dev/null || exit 1; exec /bin/bash ${BACKUP_SCRIPT_PATH}`, ], Labels: labels, Mounts: [ From d75d8f5e57e44b4fa5f91660bde98836c0bb6240 Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:35:58 +0000 Subject: [PATCH 6/7] fix(backups): harden worker handoff failures --- .../backups/swarm-backup-executor.test.ts | 151 ++++++++++++++++-- packages/server/src/utils/backups/executor.ts | 70 ++++++-- packages/server/src/utils/backups/worker.ts | 50 ++++-- 3 files changed, 232 insertions(+), 39 deletions(-) diff --git a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts index 82433be77..c3edd7de0 100644 --- a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts +++ b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts @@ -5,6 +5,7 @@ import { getBackupCommand, } from "@dokploy/server/utils/backups/utils"; import { + BackupWorkerPreStartError, BackupWorkerTaskError, findRunningServiceTask, getBackupResourceNames, @@ -373,7 +374,7 @@ describe("backup worker task lifecycle", () => { ); }); - it("times out when the worker never starts", async () => { + it("keeps a pending-task timeout non-retryable", async () => { const docker = { listTasks: vi .fn() @@ -382,11 +383,37 @@ describe("backup worker task lifecycle", () => { ]), }; - await expect( - waitForBackupWorkerTask(docker as never, "service-id", { - startTimeoutMs: 0, - }), - ).rejects.toThrow("Backup worker did not start within 0 seconds"); + const error = await waitForBackupWorkerTask(docker as never, "service-id", { + startTimeoutMs: 0, + }).catch((caught) => caught); + + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(BackupWorkerPreStartError); + expect((error as Error).message).toBe( + "Backup worker did not start within 0 seconds", + ); + }); + + it("does not mark a starting-task timeout as safe to retry", async () => { + const docker = { + listTasks: vi.fn().mockResolvedValue([ + { + ID: "worker-task", + Status: { State: "starting" }, + Version: { Index: 1 }, + }, + ]), + }; + + const error = await waitForBackupWorkerTask(docker as never, "service-id", { + startTimeoutMs: 0, + }).catch((caught) => caught); + + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(BackupWorkerPreStartError); + expect((error as Error).message).toBe( + "Backup worker did not start within 0 seconds", + ); }); it("fails when a task disappears after it started", async () => { @@ -669,23 +696,47 @@ describe("executeBackup", () => { expect(docker.createService).not.toHaveBeenCalled(); }); - it("does not switch to a worker after a direct backup failure", async () => { + it("sanitizes direct backup failures without switching to a worker", async () => { const { docker } = createDockerMock(); + const exposedSecret = "notification-secret"; + const credentialBearingCommand = `rclone --s3-secret-access-key=${exposedSecret}`; + const originalError = new Error(`original error: ${exposedSecret}`); mocks.getRemoteDocker.mockResolvedValue(docker); mocks.execAsync .mockResolvedValueOnce({ stdout: `${containerId}\n`, stderr: "" }) .mockRejectedValueOnce( - new ExecError("direct backup failed", { - command: "backup", + new ExecError(`direct backup failed: ${credentialBearingCommand}`, { + command: credentialBearingCommand, exitCode: 1, - stderr: "dump failed", + stdout: `stdout echoed ${exposedSecret}`, + stderr: `stderr echoed ${exposedSecret}`, + originalError, }), ); - await expect(executeBackup(input())).rejects.toThrow( - "direct backup failed", - ); + const error = await executeBackup(input()).catch((caught) => caught); + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(ExecError); + expect(error).toMatchObject({ + message: "Backup command failed with exit code 1", + }); + expect(error).not.toHaveProperty("command"); + expect(error).not.toHaveProperty("stdout"); + expect(error).not.toHaveProperty("stderr"); + expect(error).not.toHaveProperty("originalError"); + const publicRepresentation = [ + String(error), + (error as Error).stack ?? "", + JSON.stringify(error), + ].join("\n"); + expect(publicRepresentation).not.toContain(exposedSecret); + const appendedLogCommand = + mocks.execAsync.mock.calls[mocks.execAsync.mock.calls.length - 1]?.[0]; + expect(appendedLogCommand).toContain( + "Backup command failed with exit code 1", + ); + expect(appendedLogCommand).not.toContain(exposedSecret); expect(mocks.getRemoteDocker).not.toHaveBeenCalled(); expect(docker.createService).not.toHaveBeenCalled(); }); @@ -915,6 +966,78 @@ describe("executeBackup", () => { ).toBe(true); }); + it("retries on the replacement node when the first worker cannot start", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + const relocatedContainerId = "b".repeat(64); + docker.createSecret + .mockResolvedValueOnce({ id: "secret-id-a" }) + .mockResolvedValueOnce({ id: "secret-id-b" }); + docker.createService + .mockResolvedValueOnce({ id: "service-id-a" }) + .mockResolvedValueOnce({ id: "service-id-b" }); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ID: "worker-task-a", + Status: { State: "rejected", Err: "node unavailable" }, + Version: { Index: 2 }, + }, + ]) + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ...runningDatabaseTask, + ID: "relocated-database-task", + NodeID: "worker-node-b", + Status: { + State: "running", + ContainerStatus: { ContainerID: relocatedContainerId }, + }, + Version: { Index: 3 }, + }, + ]) + .mockResolvedValueOnce([ + { + ID: "worker-task-b", + Status: { State: "complete", ContainerStatus: { ExitCode: 0 } }, + Version: { Index: 4 }, + }, + ]); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).resolves.toEqual({ + mode: "swarm-worker", + }); + + expect(docker.createService).toHaveBeenCalledTimes(2); + expect( + docker.createService.mock.calls[0]?.[0].TaskTemplate.Placement + .Constraints, + ).toEqual(["node.id==worker-node-id"]); + expect( + docker.createService.mock.calls[1]?.[0].TaskTemplate.Placement + .Constraints, + ).toEqual(["node.id==worker-node-b"]); + expect(serviceRemove).toHaveBeenCalledTimes(2); + expect(secretRemove).toHaveBeenCalledTimes(2); + expect(serviceRemove.mock.invocationCallOrder[0]).toBeLessThan( + docker.listTasks.mock.invocationCallOrder[2] ?? Number.POSITIVE_INFINITY, + ); + expect(secretRemove.mock.invocationCallOrder[0]).toBeLessThan( + docker.listTasks.mock.invocationCallOrder[2] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.sleep).toHaveBeenCalledOnce(); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes( + "Database task moved while the backup worker was waiting to start; retrying on its new node", + ), + ), + ).toBe(true); + }); + it("stops after one relocation retry and cleans both attempts", async () => { const { docker, secretRemove, serviceRemove } = createDockerMock(); const relocationFailure = { @@ -1040,6 +1163,8 @@ describe("executeBackup", () => { "Backup worker task rejected: worker rejected", ); expect(secretRemove).toHaveBeenCalledOnce(); + expect(docker.createService).toHaveBeenCalledOnce(); + expect(docker.listTasks).toHaveBeenCalledTimes(2); expect(mocks.loggerError).toHaveBeenCalledWith( expect.objectContaining({ errors: expect.arrayContaining([ diff --git a/packages/server/src/utils/backups/executor.ts b/packages/server/src/utils/backups/executor.ts index 04657fdca..5b19f0e21 100644 --- a/packages/server/src/utils/backups/executor.ts +++ b/packages/server/src/utils/backups/executor.ts @@ -10,12 +10,14 @@ import { getContainerSearchCommand, } from "./utils"; import { + BackupWorkerPreStartError, BackupWorkerTaskError, type DockerClient, findRunningServiceTask, getBackupResourceNames, getBackupTargetServiceName, getBackupWorkerServiceSpec, + ReplacementServiceTaskNotFoundError, waitForBackupWorkerTask, waitForReplacementServiceTask, } from "./worker"; @@ -60,6 +62,9 @@ const getSafeErrorMessage = (error: unknown) => { ); }; +const getPublicExecErrorMessage = (error: ExecError) => + `Backup command failed with exit code ${error.exitCode ?? "unknown"}`; + const appendBackupError = async ( serverId: string | null | undefined, logPath: string, @@ -126,6 +131,10 @@ const isDatabaseTaskRelocationError = (error: unknown) => error instanceof BackupWorkerTaskError && error.exitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE; +const isRetryableWorkerHandoffError = (error: unknown) => + isDatabaseTaskRelocationError(error) || + error instanceof BackupWorkerPreStartError; + const isDirectDatabaseTaskRelocationError = (error: unknown) => error instanceof ExecError && error.exitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE; @@ -216,7 +225,7 @@ const runBackupOnWorkerAttempt = async ({ if (serviceMayExist) { // Collect once after the terminal task state to avoid follower races or duplicates. - if (!isDatabaseTaskRelocationError(primaryError)) { + if (!isRetryableWorkerHandoffError(primaryError)) { try { await collectServiceLogs(serverId, serviceName, logPath); } catch (error) { @@ -247,7 +256,7 @@ const runBackupOnWorkerAttempt = async ({ { errors: cleanupErrors.map((error) => error.message) }, "Backup worker cleanup also failed", ); - if (isDatabaseTaskRelocationError(primaryError)) { + if (isRetryableWorkerHandoffError(primaryError)) { primaryError = new Error( `${getSafeErrorMessage(primaryError)}; ${cleanupErrors.map((error) => error.message).join("; ")}`, ); @@ -292,26 +301,51 @@ const runBackupOnWorker = async ( }); return; } catch (error) { - if (!isDatabaseTaskRelocationError(error)) { + const isRelocationError = isDatabaseTaskRelocationError(error); + const isPreStartError = error instanceof BackupWorkerPreStartError; + if (!isRelocationError && !isPreStartError) { throw error; } if (attempt === MAX_BACKUP_WORKER_ATTEMPTS - 1) { + if (isPreStartError) { + throw error; + } throw new Error( `Database task moved while the backup worker was starting; retry limit reached (${getSafeErrorMessage(error)})`, { cause: error }, ); } - await appendBackupMessage( - input.serverId, - input.logPath, - "Database task moved before the backup started; rediscovering its node and retrying...", - ); - databaseTask = await waitForReplacementServiceTask( - docker, - serviceTarget, - databaseTask.containerId, - ); + if (isPreStartError) { + try { + databaseTask = await waitForReplacementServiceTask( + docker, + serviceTarget, + databaseTask.containerId, + ); + } catch (replacementError) { + if (replacementError instanceof ReplacementServiceTaskNotFoundError) { + throw error; + } + throw replacementError; + } + await appendBackupMessage( + input.serverId, + input.logPath, + "Database task moved while the backup worker was waiting to start; retrying on its new node...", + ); + } else { + await appendBackupMessage( + input.serverId, + input.logPath, + "Database task moved before the backup started; rediscovering its node and retrying...", + ); + databaseTask = await waitForReplacementServiceTask( + docker, + serviceTarget, + databaseTask.containerId, + ); + } } } }; @@ -357,7 +391,13 @@ export const executeBackup = async (input: ExecuteBackupInput) => { await runBackupOnWorker(input, previousContainerId); return { mode: "swarm-worker" as const }; } catch (error) { - await appendBackupError(input.serverId, input.logPath, error); - throw error; + // ExecError retains the full shell command, which contains backup credentials. + // Its output may echo that command too, so only expose non-secret metadata. + const publicError = + error instanceof ExecError + ? new Error(getPublicExecErrorMessage(error)) + : error; + await appendBackupError(input.serverId, input.logPath, publicError); + throw publicError; } }; diff --git a/packages/server/src/utils/backups/worker.ts b/packages/server/src/utils/backups/worker.ts index d58299050..347c4e74a 100644 --- a/packages/server/src/utils/backups/worker.ts +++ b/packages/server/src/utils/backups/worker.ts @@ -86,6 +86,15 @@ export class RunningServiceTaskNotFoundError extends Error { } } +export class ReplacementServiceTaskNotFoundError extends Error { + constructor(serviceName: string) { + super( + `No replacement Swarm task became ready for database service ${serviceName} after relocation`, + ); + this.name = "ReplacementServiceTaskNotFoundError"; + } +} + export const findRunningServiceTask = async ( docker: DockerClient, serviceName: string, @@ -153,9 +162,7 @@ export const waitForReplacementServiceTask = async ( } } - throw new Error( - `No replacement Swarm task became ready for database service ${serviceName} after relocation`, - ); + throw new ReplacementServiceTaskNotFoundError(serviceName); }; const getLatestTask = (tasks: SwarmTask[]) => @@ -213,12 +220,34 @@ export class BackupWorkerTaskError extends Error { } } -const getTaskFailureError = (task: SwarmTask) => - new BackupWorkerTaskError( +export class BackupWorkerPreStartError extends Error { + constructor( + message: string, + readonly state: string, + ) { + super(message); + this.name = "BackupWorkerPreStartError"; + } +} + +const getTaskFailureError = (task: SwarmTask, observedRunning = false) => { + const state = task.Status?.State ?? "unknown"; + const containerStatus = task.Status?.ContainerStatus; + if ( + !observedRunning && + state === "rejected" && + !containerStatus?.ContainerID && + containerStatus?.ExitCode === undefined + ) { + return new BackupWorkerPreStartError(getTaskFailureMessage(task), state); + } + + return new BackupWorkerTaskError( getTaskFailureMessage(task), - task.Status?.State ?? "unknown", - task.Status?.ContainerStatus?.ExitCode, + state, + containerStatus?.ExitCode, ); +}; export const waitForBackupWorkerTask = async ( docker: DockerClient, @@ -246,7 +275,7 @@ export const waitForBackupWorkerTask = async ( const state = task?.Status?.State; if (task && state && terminalFailureStates.has(state)) { - throw getTaskFailureError(task); + throw getTaskFailureError(task, true); } const replacement = getReplacementTask(tasks, startedTaskId); @@ -303,9 +332,8 @@ export const waitForBackupWorkerTask = async ( } if (!startedTaskId && Date.now() - startedAt >= startTimeoutMs) { - throw new Error( - `Backup worker did not start within ${Math.round(startTimeoutMs / 1_000)} seconds`, - ); + const message = `Backup worker did not start within ${Math.round(startTimeoutMs / 1_000)} seconds`; + throw new Error(message); } await sleepFn(pollIntervalMs); From 6c078d79184bd1856d8e4d452d285f0ee047e623 Mon Sep 17 00:00:00 2001 From: Aditya Nandlal <73009776+bestmaa@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:45:38 +0000 Subject: [PATCH 7/7] fix(backups): fail fast on unrelated worker rejection --- .../backups/swarm-backup-executor.test.ts | 32 +++++++++++++++++++ packages/server/src/utils/backups/executor.ts | 27 ++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts index c3edd7de0..2cca3dc4b 100644 --- a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts +++ b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts @@ -985,6 +985,7 @@ describe("executeBackup", () => { Version: { Index: 2 }, }, ]) + .mockResolvedValueOnce([]) .mockResolvedValueOnce([runningDatabaseTask]) .mockResolvedValueOnce([ { @@ -1038,6 +1039,37 @@ describe("executeBackup", () => { ).toBe(true); }); + it("fails a rejected worker immediately when the database has not moved", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "rejected", Err: "image pull failed" }, + Version: { Index: 2 }, + }, + ]) + .mockResolvedValueOnce([runningDatabaseTask]); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "Backup worker task rejected: image pull failed", + ); + + expect(docker.createService).toHaveBeenCalledOnce(); + expect(docker.listTasks).toHaveBeenCalledTimes(3); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + expect(mocks.sleep).not.toHaveBeenCalled(); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes("retrying on its new node"), + ), + ).toBe(false); + }); + it("stops after one relocation retry and cleans both attempts", async () => { const { docker, secretRemove, serviceRemove } = createDockerMock(); const relocationFailure = { diff --git a/packages/server/src/utils/backups/executor.ts b/packages/server/src/utils/backups/executor.ts index 5b19f0e21..4aaa4bcc5 100644 --- a/packages/server/src/utils/backups/executor.ts +++ b/packages/server/src/utils/backups/executor.ts @@ -18,6 +18,7 @@ import { getBackupTargetServiceName, getBackupWorkerServiceSpec, ReplacementServiceTaskNotFoundError, + RunningServiceTaskNotFoundError, waitForBackupWorkerTask, waitForReplacementServiceTask, } from "./worker"; @@ -317,12 +318,34 @@ const runBackupOnWorker = async ( } if (isPreStartError) { + let currentDatabaseTask: Awaited< + ReturnType + > | null = null; try { - databaseTask = await waitForReplacementServiceTask( + currentDatabaseTask = await findRunningServiceTask( docker, serviceTarget, - databaseTask.containerId, ); + } catch (discoveryError) { + if (!(discoveryError instanceof RunningServiceTaskNotFoundError)) { + throw discoveryError; + } + } + + // A rejected helper is not evidence of relocation by itself. Fail fast + // when the database is still on the node selected for this attempt. + if (currentDatabaseTask?.containerId === databaseTask.containerId) { + throw error; + } + + try { + databaseTask = + currentDatabaseTask ?? + (await waitForReplacementServiceTask( + docker, + serviceTarget, + databaseTask.containerId, + )); } catch (replacementError) { if (replacementError instanceof ReplacementServiceTaskNotFoundError) { throw error;