From 5fa963fe87796b6e12b402bec72d26672d39c3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Narciso=20E=2E=20N=C3=BA=C3=B1ez=20Arias?= Date: Tue, 18 Aug 2026 18:40:08 -0400 Subject: [PATCH] Merge pull request #5082 from bestmaa/fix/volume-backup-restart fix(volume-backups): restart services after backup failure (cherry picked from commit 3054cf53be2a5f6cf48943e3406b476133ccc53f) [skip ci] --- .../backups/volume-backup-restart.test.ts | 82 +++++++++++++++++++ .../server/src/utils/volume-backups/backup.ts | 75 +++++++++++++---- 2 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 apps/dokploy/__test__/backups/volume-backup-restart.test.ts diff --git a/apps/dokploy/__test__/backups/volume-backup-restart.test.ts b/apps/dokploy/__test__/backups/volume-backup-restart.test.ts new file mode 100644 index 000000000..ba2fc081e --- /dev/null +++ b/apps/dokploy/__test__/backups/volume-backup-restart.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from "node:child_process"; +import { createRestartSafeBackupCommand } from "@dokploy/server/utils/volume-backups/backup"; +import { describe, expect, it } from "vitest"; + +const runCommand = (command: string) => + spawnSync("bash", ["-c", command], { + encoding: "utf8", + }); + +const outputLines = (stdout: string) => + stdout + .trim() + .split("\n") + .filter((line) => line.length > 0); + +describe("createRestartSafeBackupCommand", () => { + it("restarts the service and preserves the backup error status", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"; exit 23', + startCommand: 'echo "start"', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(23); + expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]); + }); + + it("preserves the backup error status when the restart also fails", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"; exit 23', + startCommand: 'echo "start"; exit 17', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(23); + expect(outputLines(result.stdout)).toEqual([ + "stop", + "backup", + "start", + "Service restart also failed with exit code 17", + ]); + }); + + it("returns the restart error status when the backup succeeds", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"', + startCommand: 'echo "start"; exit 17', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(17); + expect(outputLines(result.stdout)).toEqual(["stop", "backup", "start"]); + }); + + it("uploads only after a successful backup and service restart", () => { + const result = runCommand( + createRestartSafeBackupCommand({ + stopCommand: 'echo "stop"', + backupCommand: 'echo "backup"', + startCommand: 'echo "start"', + uploadCommand: 'echo "upload"', + }), + ); + + expect(result.status).toBe(0); + expect(outputLines(result.stdout)).toEqual([ + "stop", + "backup", + "start", + "upload", + ]); + }); +}); diff --git a/packages/server/src/utils/volume-backups/backup.ts b/packages/server/src/utils/volume-backups/backup.ts index a9240f2dd..3172c9c21 100644 --- a/packages/server/src/utils/volume-backups/backup.ts +++ b/packages/server/src/utils/volume-backups/backup.ts @@ -9,6 +9,43 @@ import { normalizeS3Path, } from "../backups/utils"; +interface RestartSafeBackupCommandOptions { + stopCommand: string; + backupCommand: string; + startCommand: string; + uploadCommand: string; +} + +export const createRestartSafeBackupCommand = ({ + stopCommand, + backupCommand, + startCommand, + uploadCommand, +}: RestartSafeBackupCommandOptions) => ` + ${stopCommand} + set +e + ( + ${backupCommand} + ) + DOKPLOY_VOLUME_BACKUP_STATUS=$? + ( + set -e + ${startCommand} + ) + DOKPLOY_VOLUME_RESTART_STATUS=$? + set -e + if [ "$DOKPLOY_VOLUME_BACKUP_STATUS" -ne 0 ]; then + if [ "$DOKPLOY_VOLUME_RESTART_STATUS" -ne 0 ]; then + echo "Service restart also failed with exit code $DOKPLOY_VOLUME_RESTART_STATUS" + fi + exit "$DOKPLOY_VOLUME_BACKUP_STATUS" + fi + if [ "$DOKPLOY_VOLUME_RESTART_STATUS" -ne 0 ]; then + exit "$DOKPLOY_VOLUME_RESTART_STATUS" + fi + ${uploadCommand} +`; + export const getVolumeServiceAppName = ( volumeBackup: Awaited>, ): string => { @@ -117,16 +154,20 @@ export const backupVolume = async ( ); if (serviceType === "application") { - return lockWrapper(` - echo "Stopping application to 0 replicas" - ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}") - echo "Actual replicas: $ACTUAL_REPLICAS" - docker service update --replicas=0 ${volumeBackup.application?.appName} - ${backupCommand} - echo "Starting application to $ACTUAL_REPLICAS replicas" - docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName} - ${uploadCommand} - `); + return lockWrapper( + createRestartSafeBackupCommand({ + stopCommand: ` + echo "Stopping application to 0 replicas" + ACTUAL_REPLICAS=$(docker service inspect ${volumeBackup.application?.appName} --format "{{.Spec.Mode.Replicated.Replicas}}") + echo "Actual replicas: $ACTUAL_REPLICAS" + docker service update --replicas=0 ${volumeBackup.application?.appName}`, + backupCommand, + startCommand: ` + echo "Starting application to $ACTUAL_REPLICAS replicas" + docker service update --replicas=$ACTUAL_REPLICAS --with-registry-auth ${volumeBackup.application?.appName}`, + uploadCommand, + }), + ); } if (serviceType === "compose") { const compose = await findComposeById( @@ -158,11 +199,13 @@ export const backupVolume = async ( echo "Compose container started" `; } - return lockWrapper(` - ${stopCommand} - ${backupCommand} - ${startCommand} - ${uploadCommand} - `); + return lockWrapper( + createRestartSafeBackupCommand({ + stopCommand, + backupCommand, + startCommand, + uploadCommand, + }), + ); } };