diff --git a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts index c3edd7de0..2cca3dc4b 100644 --- a/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts +++ b/apps/dokploy/__test__/backups/swarm-backup-executor.test.ts @@ -985,6 +985,7 @@ describe("executeBackup", () => { Version: { Index: 2 }, }, ]) + .mockResolvedValueOnce([]) .mockResolvedValueOnce([runningDatabaseTask]) .mockResolvedValueOnce([ { @@ -1038,6 +1039,37 @@ describe("executeBackup", () => { ).toBe(true); }); + it("fails a rejected worker immediately when the database has not moved", async () => { + const { docker, secretRemove, serviceRemove } = createDockerMock(); + docker.listTasks.mockReset(); + docker.listTasks + .mockResolvedValueOnce([runningDatabaseTask]) + .mockResolvedValueOnce([ + { + ID: "worker-task", + Status: { State: "rejected", Err: "image pull failed" }, + Version: { Index: 2 }, + }, + ]) + .mockResolvedValueOnce([runningDatabaseTask]); + mocks.getRemoteDocker.mockResolvedValue(docker); + + await expect(executeBackup(input())).rejects.toThrow( + "Backup worker task rejected: image pull failed", + ); + + expect(docker.createService).toHaveBeenCalledOnce(); + expect(docker.listTasks).toHaveBeenCalledTimes(3); + expect(serviceRemove).toHaveBeenCalledOnce(); + expect(secretRemove).toHaveBeenCalledOnce(); + expect(mocks.sleep).not.toHaveBeenCalled(); + expect( + mocks.execAsync.mock.calls.some(([command]) => + command.includes("retrying on its new node"), + ), + ).toBe(false); + }); + it("stops after one relocation retry and cleans both attempts", async () => { const { docker, secretRemove, serviceRemove } = createDockerMock(); const relocationFailure = { diff --git a/packages/server/src/utils/backups/executor.ts b/packages/server/src/utils/backups/executor.ts index 5b19f0e21..4aaa4bcc5 100644 --- a/packages/server/src/utils/backups/executor.ts +++ b/packages/server/src/utils/backups/executor.ts @@ -18,6 +18,7 @@ import { getBackupTargetServiceName, getBackupWorkerServiceSpec, ReplacementServiceTaskNotFoundError, + RunningServiceTaskNotFoundError, waitForBackupWorkerTask, waitForReplacementServiceTask, } from "./worker"; @@ -317,12 +318,34 @@ const runBackupOnWorker = async ( } if (isPreStartError) { + let currentDatabaseTask: Awaited< + ReturnType + > | null = null; try { - databaseTask = await waitForReplacementServiceTask( + currentDatabaseTask = await findRunningServiceTask( docker, serviceTarget, - databaseTask.containerId, ); + } catch (discoveryError) { + if (!(discoveryError instanceof RunningServiceTaskNotFoundError)) { + throw discoveryError; + } + } + + // A rejected helper is not evidence of relocation by itself. Fail fast + // when the database is still on the node selected for this attempt. + if (currentDatabaseTask?.containerId === databaseTask.containerId) { + throw error; + } + + try { + databaseTask = + currentDatabaseTask ?? + (await waitForReplacementServiceTask( + docker, + serviceTarget, + databaseTask.containerId, + )); } catch (replacementError) { if (replacementError instanceof ReplacementServiceTaskNotFoundError) { throw error;