The_Great_Archive/dokploy
1
0
Fork 0
mirror of https://github.com/Dokploy/dokploy.git synced 2026-09-12 19:51:00 +05:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

fix(backups): fail fast on unrelated worker rejection

Browse Source
This commit is contained in:
Aditya Nandlal 2026-08-25 20:45:38 +00:00
parent d75d8f5e57
commit 6c078d7918
2 changed files with 57 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
apps/dokploy/__test__/backups/swarm-backup-executor.test.ts
View File

@ -985,6 +985,7 @@ describe("executeBackup", () => {
Version: { Index: 2 },
},
])
.mockResolvedValueOnce([])
.mockResolvedValueOnce([runningDatabaseTask])
.mockResolvedValueOnce([
{
@ -1038,6 +1039,37 @@ describe("executeBackup", () => {
).toBe(true);
});
it("fails a rejected worker immediately when the database has not moved", async () => {
const { docker, secretRemove, serviceRemove } = createDockerMock();
docker.listTasks.mockReset();
docker.listTasks
.mockResolvedValueOnce([runningDatabaseTask])
.mockResolvedValueOnce([
{
ID: "worker-task",
Status: { State: "rejected", Err: "image pull failed" },
Version: { Index: 2 },
},
])
.mockResolvedValueOnce([runningDatabaseTask]);
mocks.getRemoteDocker.mockResolvedValue(docker);
await expect(executeBackup(input())).rejects.toThrow(
"Backup worker task rejected: image pull failed",
);
expect(docker.createService).toHaveBeenCalledOnce();
expect(docker.listTasks).toHaveBeenCalledTimes(3);
expect(serviceRemove).toHaveBeenCalledOnce();
expect(secretRemove).toHaveBeenCalledOnce();
expect(mocks.sleep).not.toHaveBeenCalled();
expect(
mocks.execAsync.mock.calls.some(([command]) =>
command.includes("retrying on its new node"),
),
).toBe(false);
});
it("stops after one relocation retry and cleans both attempts", async () => {
const { docker, secretRemove, serviceRemove } = createDockerMock();
const relocationFailure = {

27
packages/server/src/utils/backups/executor.ts
View File

@ -18,6 +18,7 @@ import {
getBackupTargetServiceName,
getBackupWorkerServiceSpec,
ReplacementServiceTaskNotFoundError,
RunningServiceTaskNotFoundError,
waitForBackupWorkerTask,
waitForReplacementServiceTask,
} from "./worker";
@ -317,12 +318,34 @@ const runBackupOnWorker = async (
}
if (isPreStartError) {
let currentDatabaseTask: Awaited<
ReturnType<typeof findRunningServiceTask>
> | null = null;
try {
databaseTask = await waitForReplacementServiceTask(
currentDatabaseTask = await findRunningServiceTask(
docker,
serviceTarget,
databaseTask.containerId,
);
} catch (discoveryError) {
if (!(discoveryError instanceof RunningServiceTaskNotFoundError)) {
throw discoveryError;
}
}
// A rejected helper is not evidence of relocation by itself. Fail fast
// when the database is still on the node selected for this attempt.
if (currentDatabaseTask?.containerId === databaseTask.containerId) {
throw error;
}
try {
databaseTask =
currentDatabaseTask ??
(await waitForReplacementServiceTask(
docker,
serviceTarget,
databaseTask.containerId,
));
} catch (replacementError) {
if (replacementError instanceof ReplacementServiceTaskNotFoundError) {
throw error;