Merge 6c078d7918 into 853ca33659

apps/dokploy/__test__/backups/redact-credentials.test.ts

@@ -47,4 +47,16 @@ describe("redactRcloneCredentials (#4621)", () => {
 		expect(redacted).not.toContain("MYSECRET");
 		expect(redacted).toContain("[REDACTED]");
 	});
+
+	it("should redact shell-quoted and unquoted credential values", () => {
+		const cmd =
+			"rclone rcat --s3-access-key-id=plain-key --s3-secret-access-key='secret with spaces' --s3-access-key-id='key'\\''with-quote' :s3:bucket/file.gz";
+		const redacted = redactRcloneCredentials(cmd);
+
+		expect(redacted).not.toContain("plain-key");
+		expect(redacted).not.toContain("secret with spaces");
+		expect(redacted).not.toContain("with-quote");
+		expect(redacted).toContain('--s3-access-key-id="[REDACTED]"');
+		expect(redacted).toContain('--s3-secret-access-key="[REDACTED]"');
+	});
 });

packages/server/src/services/deployment.ts

@@ -374,7 +374,8 @@ export const createDeploymentBackup = async (
 			backup.postgres?.serverId ||
 			backup.mariadb?.serverId ||
 			backup.mysql?.serverId ||
-			backup.mongo?.serverId;
+			backup.mongo?.serverId ||
+			backup.libsql?.serverId;
 	} else if (backup.backupType === "compose") {
 		serverId = backup.compose?.serverId;
 	}

packages/server/src/utils/backups/executor.ts

@@ -0,0 +1,426 @@
+import { logger } from "@dokploy/server/lib/logger";
+import type { BackupSchedule } from "@dokploy/server/services/backup";
+import { quote } from "shell-quote";
+import { ExecError, execAsync, execAsyncRemote } from "../process/execAsync";
+import { getRemoteDocker } from "../servers/remote-docker";
+import { redactRcloneCredentials } from "./redact";
+import {
+	BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE,
+	getBackupCommand,
+	getContainerSearchCommand,
+} from "./utils";
+import {
+	BackupWorkerPreStartError,
+	BackupWorkerTaskError,
+	type DockerClient,
+	findRunningServiceTask,
+	getBackupResourceNames,
+	getBackupTargetServiceName,
+	getBackupWorkerServiceSpec,
+	ReplacementServiceTaskNotFoundError,
+	RunningServiceTaskNotFoundError,
+	waitForBackupWorkerTask,
+	waitForReplacementServiceTask,
+} from "./worker";
+
+const MAX_BACKUP_WORKER_ATTEMPTS = 2;
+
+type ExecuteBackupInput = {
+	backup: BackupSchedule;
+	executionId: string;
+	logPath: string;
+	rcloneDestination: string;
+	rcloneFlags: string[];
+	serverId?: string | null;
+};
+
+const runHostCommand = (
+	serverId: string | null | undefined,
+	command: string,
+) => {
+	if (serverId) {
+		return execAsyncRemote(serverId, command);
+	}
+
+	return execAsync(command, { shell: "/bin/bash" });
+};
+
+const isNotFoundError = (error: unknown) =>
+	typeof error === "object" &&
+	error !== null &&
+	"statusCode" in error &&
+	error.statusCode === 404;
+
+const getSafeErrorMessage = (error: unknown) => {
+	if (error instanceof ExecError) {
+		const output = error.stderr?.trim() || error.stdout?.trim();
+		return redactRcloneCredentials(
+			output || `command exited with code ${error.exitCode ?? "unknown"}`,
+		);
+	}
+	return redactRcloneCredentials(
+		error instanceof Error ? error.message : String(error),
+	);
+};
+
+const getPublicExecErrorMessage = (error: ExecError) =>
+	`Backup command failed with exit code ${error.exitCode ?? "unknown"}`;
+
+const appendBackupError = async (
+	serverId: string | null | undefined,
+	logPath: string,
+	error: unknown,
+) => {
+	const message = getSafeErrorMessage(error);
+	const command = `printf '[%s] ❌ Error: %s\\n' "$(date)" ${quote([message])} >> ${quote([logPath])}`;
+	try {
+		await runHostCommand(serverId, command);
+	} catch (logError) {
+		logger.error(
+			{ error: getSafeErrorMessage(logError) },
+			"Failed to append backup worker error to deployment log",
+		);
+	}
+};
+
+const appendBackupMessage = async (
+	serverId: string | null | undefined,
+	logPath: string,
+	message: string,
+) => {
+	const command = `printf '[%s] %s\n' "$(date)" ${quote([message])} >> ${quote([logPath])}`;
+	try {
+		await runHostCommand(serverId, command);
+	} catch (logError) {
+		logger.error(
+			{ error: getSafeErrorMessage(logError) },
+			"Failed to append backup worker progress to deployment log",
+		);
+	}
+};
+
+const collectServiceLogs = (
+	serverId: string | null | undefined,
+	serviceName: string,
+	logPath: string,
+) => {
+	const quotedServiceName = quote([serviceName]);
+	return runHostCommand(
+		serverId,
+		`if docker service inspect ${quotedServiceName} >/dev/null 2>&1; then docker service logs --raw ${quotedServiceName} >> ${quote([logPath])} 2>&1; fi`,
+	);
+};
+
+const removeResource = async (
+	remove: () => Promise<unknown>,
+	resource: "secret" | "service",
+) => {
+	try {
+		await remove();
+		return null;
+	} catch (error) {
+		if (isNotFoundError(error)) {
+			return null;
+		}
+		return new Error(
+			`Failed to remove backup worker ${resource}: ${getSafeErrorMessage(error)}`,
+		);
+	}
+};
+
+const isDatabaseTaskRelocationError = (error: unknown) =>
+	error instanceof BackupWorkerTaskError &&
+	error.exitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE;
+
+const isRetryableWorkerHandoffError = (error: unknown) =>
+	isDatabaseTaskRelocationError(error) ||
+	error instanceof BackupWorkerPreStartError;
+
+const isDirectDatabaseTaskRelocationError = (error: unknown) =>
+	error instanceof ExecError &&
+	error.exitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE;
+
+const runBackupOnWorkerAttempt = async ({
+	input: {
+		backup,
+		executionId,
+		logPath,
+		rcloneDestination,
+		rcloneFlags,
+		serverId,
+	},
+	attempt,
+	databaseTask: { containerId, nodeId },
+	docker,
+}: {
+	input: ExecuteBackupInput;
+	attempt: number;
+	databaseTask: { containerId: string; nodeId: string };
+	docker: DockerClient;
+}) => {
+	const { secretName, serviceName } = getBackupResourceNames(
+		executionId,
+		attempt,
+	);
+	const labels = {
+		"dokploy.backup.id": backup.backupId,
+		"dokploy.deployment.id": executionId,
+		"dokploy.managed": "true",
+		"dokploy.resource": "database-backup-worker",
+	};
+	// Keep database and S3 credentials out of the inspectable service arguments.
+	const workerCommand = getBackupCommand(
+		backup,
+		rcloneFlags,
+		rcloneDestination,
+		"/proc/1/fd/1",
+		{
+			containerId,
+			containerNotFoundExitCode: BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE,
+		},
+	);
+	const service = docker.getService(serviceName);
+	const secret = docker.getSecret(secretName);
+
+	let serviceMayExist = false;
+	let secretMayExist = false;
+	let primaryError: unknown;
+
+	try {
+		await appendBackupMessage(
+			serverId,
+			logPath,
+			`Preparing backup worker attempt ${attempt + 1} of ${MAX_BACKUP_WORKER_ATTEMPTS} on node ${nodeId}...`,
+		);
+
+		secretMayExist = true;
+		const createdSecret = (await docker.createSecret({
+			Name: secretName,
+			Labels: labels,
+			Data: Buffer.from(workerCommand).toString("base64"),
+		})) as { id?: string };
+		const secretId = createdSecret.id;
+		if (!secretId) {
+			throw new Error("Docker did not return a backup worker secret ID");
+		}
+
+		serviceMayExist = true;
+		const createdService = await docker.createService(
+			getBackupWorkerServiceSpec({
+				backupId: backup.backupId,
+				executionId,
+				nodeId,
+				secretId,
+				secretName,
+				serviceName,
+			}),
+		);
+		if (!createdService.id) {
+			throw new Error("Docker did not return a backup worker service ID");
+		}
+		await waitForBackupWorkerTask(docker, createdService.id);
+	} catch (error) {
+		primaryError = error;
+	} finally {
+		const cleanupErrors: Error[] = [];
+
+		if (serviceMayExist) {
+			// Collect once after the terminal task state to avoid follower races or duplicates.
+			if (!isRetryableWorkerHandoffError(primaryError)) {
+				try {
+					await collectServiceLogs(serverId, serviceName, logPath);
+				} catch (error) {
+					const message = getSafeErrorMessage(error);
+					logger.error(
+						{ error: message },
+						"Failed to collect backup worker logs",
+					);
+					await appendBackupMessage(
+						serverId,
+						logPath,
+						`⚠️ Warning: Could not collect backup worker logs: ${message}`,
+					);
+				}
+			}
+
+			const error = await removeResource(() => service.remove(), "service");
+			if (error) cleanupErrors.push(error);
+		}
+		if (secretMayExist) {
+			const error = await removeResource(() => secret.remove(), "secret");
+			if (error) cleanupErrors.push(error);
+		}
+
+		if (primaryError) {
+			if (cleanupErrors.length > 0) {
+				logger.error(
+					{ errors: cleanupErrors.map((error) => error.message) },
+					"Backup worker cleanup also failed",
+				);
+				if (isRetryableWorkerHandoffError(primaryError)) {
+					primaryError = new Error(
+						`${getSafeErrorMessage(primaryError)}; ${cleanupErrors.map((error) => error.message).join("; ")}`,
+					);
+				}
+			}
+		} else if (cleanupErrors.length > 0) {
+			primaryError = new Error(
+				cleanupErrors.map((error) => error.message).join("; "),
+			);
+		}
+	}
+
+	if (primaryError) {
+		throw primaryError;
+	}
+};
+
+const runBackupOnWorker = async (
+	input: ExecuteBackupInput,
+	previousContainerId?: string,
+) => {
+	const serviceTarget = getBackupTargetServiceName(input.backup);
+	if (!serviceTarget) {
+		throw new Error("Could not determine the Swarm service for this backup");
+	}
+
+	const docker = await getRemoteDocker(input.serverId);
+	let databaseTask = previousContainerId
+		? await waitForReplacementServiceTask(
+				docker,
+				serviceTarget,
+				previousContainerId,
+			)
+		: await findRunningServiceTask(docker, serviceTarget);
+	for (let attempt = 0; attempt < MAX_BACKUP_WORKER_ATTEMPTS; attempt += 1) {
+		try {
+			await runBackupOnWorkerAttempt({
+				attempt,
+				databaseTask,
+				docker,
+				input,
+			});
+			return;
+		} catch (error) {
+			const isRelocationError = isDatabaseTaskRelocationError(error);
+			const isPreStartError = error instanceof BackupWorkerPreStartError;
+			if (!isRelocationError && !isPreStartError) {
+				throw error;
+			}
+			if (attempt === MAX_BACKUP_WORKER_ATTEMPTS - 1) {
+				if (isPreStartError) {
+					throw error;
+				}
+				throw new Error(
+					`Database task moved while the backup worker was starting; retry limit reached (${getSafeErrorMessage(error)})`,
+					{ cause: error },
+				);
+			}
+
+			if (isPreStartError) {
+				let currentDatabaseTask: Awaited<
+					ReturnType<typeof findRunningServiceTask>
+				> | null = null;
+				try {
+					currentDatabaseTask = await findRunningServiceTask(
+						docker,
+						serviceTarget,
+					);
+				} catch (discoveryError) {
+					if (!(discoveryError instanceof RunningServiceTaskNotFoundError)) {
+						throw discoveryError;
+					}
+				}
+
+				// A rejected helper is not evidence of relocation by itself. Fail fast
+				// when the database is still on the node selected for this attempt.
+				if (currentDatabaseTask?.containerId === databaseTask.containerId) {
+					throw error;
+				}
+
+				try {
+					databaseTask =
+						currentDatabaseTask ??
+						(await waitForReplacementServiceTask(
+							docker,
+							serviceTarget,
+							databaseTask.containerId,
+						));
+				} catch (replacementError) {
+					if (replacementError instanceof ReplacementServiceTaskNotFoundError) {
+						throw error;
+					}
+					throw replacementError;
+				}
+				await appendBackupMessage(
+					input.serverId,
+					input.logPath,
+					"Database task moved while the backup worker was waiting to start; retrying on its new node...",
+				);
+			} else {
+				await appendBackupMessage(
+					input.serverId,
+					input.logPath,
+					"Database task moved before the backup started; rediscovering its node and retrying...",
+				);
+				databaseTask = await waitForReplacementServiceTask(
+					docker,
+					serviceTarget,
+					databaseTask.containerId,
+				);
+			}
+		}
+	}
+};
+
+export const executeBackup = async (input: ExecuteBackupInput) => {
+	try {
+		let previousContainerId: string | undefined;
+		const containerSearch = getContainerSearchCommand(input.backup);
+		if (!containerSearch) {
+			throw new Error("Could not build the database container search command");
+		}
+
+		const { stdout } = await runHostCommand(input.serverId, containerSearch);
+		const directContainerId = stdout.trim().split(/\s+/)[0];
+		if (directContainerId) {
+			const command = getBackupCommand(
+				input.backup,
+				input.rcloneFlags,
+				input.rcloneDestination,
+				input.logPath,
+				{
+					containerId: directContainerId,
+					containerNotFoundExitCode:
+						BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE,
+				},
+			);
+			try {
+				await runHostCommand(input.serverId, command);
+				return { mode: "direct" as const };
+			} catch (error) {
+				if (!isDirectDatabaseTaskRelocationError(error)) {
+					throw error;
+				}
+				await appendBackupMessage(
+					input.serverId,
+					input.logPath,
+					"Database task moved off this node before the backup started; switching to a backup worker...",
+				);
+				previousContainerId = directContainerId;
+			}
+		}
+
+		await runBackupOnWorker(input, previousContainerId);
+		return { mode: "swarm-worker" as const };
+	} catch (error) {
+		// ExecError retains the full shell command, which contains backup credentials.
+		// Its output may echo that command too, so only expose non-secret metadata.
+		const publicError =
+			error instanceof ExecError
+				? new Error(getPublicExecErrorMessage(error))
+				: error;
+		await appendBackupError(input.serverId, input.logPath, publicError);
+		throw publicError;
+	}
+};

packages/server/src/utils/backups/libsql.ts

@@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
 import type { Libsql } from "@dokploy/server/services/libsql";
 import { findProjectById } from "@dokploy/server/services/project";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
-import { execAsync, execAsyncRemote } from "../process/execAsync";
-import {
-	getBackupCommand,
-	getBackupTimestamp,
-	getS3Credentials,
-	normalizeS3Path,
-} from "./utils";
+import { executeBackup } from "./executor";
+import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runLibsqlBackup = async (
 	libsql: Libsql,
@@ -36,19 +31,14 @@ export const runLibsqlBackup = async (
 	try {
 		const rcloneFlags = getS3Credentials(destination);
 		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
-		const backupCommand = getBackupCommand(
+		await executeBackup({
 			backup,
-			rcloneFlags,
+			executionId: deployment.deploymentId,
+			logPath: deployment.logPath,
 			rcloneDestination,
-			deployment.logPath,
-		);
-		if (libsql.serverId) {
-			await execAsyncRemote(libsql.serverId, backupCommand);
-		} else {
-			await execAsync(backupCommand, {
-				shell: "/bin/bash",
-			});
-		}
+			rcloneFlags,
+			serverId: libsql.serverId,
+		});
 
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
@@ -66,8 +56,8 @@ export const runLibsqlBackup = async (
 			projectName: project.name,
 			databaseType: "libsql",
 			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
+			errorMessage:
+				error instanceof Error ? error.message : "Error message not provided",
 			organizationId: project.organizationId,
 			databaseName: backup.database,
 		});

packages/server/src/utils/backups/mariadb.ts

@@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
 import type { Mariadb } from "@dokploy/server/services/mariadb";
 import { findProjectById } from "@dokploy/server/services/project";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
-import { execAsync, execAsyncRemote } from "../process/execAsync";
-import {
-	getBackupCommand,
-	getBackupTimestamp,
-	getS3Credentials,
-	normalizeS3Path,
-} from "./utils";
+import { executeBackup } from "./executor";
+import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runMariadbBackup = async (
 	mariadb: Mariadb,
@@ -35,19 +30,14 @@ export const runMariadbBackup = async (
 	try {
 		const rcloneFlags = getS3Credentials(destination);
 		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
-		const backupCommand = getBackupCommand(
+		await executeBackup({
 			backup,
-			rcloneFlags,
+			executionId: deployment.deploymentId,
+			logPath: deployment.logPath,
 			rcloneDestination,
-			deployment.logPath,
-		);
-		if (mariadb.serverId) {
-			await execAsyncRemote(mariadb.serverId, backupCommand);
-		} else {
-			await execAsync(backupCommand, {
-				shell: "/bin/bash",
-			});
-		}
+			rcloneFlags,
+			serverId: mariadb.serverId,
+		});
 
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
@@ -65,8 +55,8 @@ export const runMariadbBackup = async (
 			projectName: project.name,
 			databaseType: "mariadb",
 			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
+			errorMessage:
+				error instanceof Error ? error.message : "Error message not provided",
 			organizationId: project.organizationId,
 			databaseName: backup.database,
 		});

packages/server/src/utils/backups/mongo.ts

@@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
 import type { Mongo } from "@dokploy/server/services/mongo";
 import { findProjectById } from "@dokploy/server/services/project";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
-import { execAsync, execAsyncRemote } from "../process/execAsync";
-import {
-	getBackupCommand,
-	getBackupTimestamp,
-	getS3Credentials,
-	normalizeS3Path,
-} from "./utils";
+import { executeBackup } from "./executor";
+import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
 	const { environmentId, name, appName } = mongo;
@@ -32,20 +27,14 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
 	try {
 		const rcloneFlags = getS3Credentials(destination);
 		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
-		const backupCommand = getBackupCommand(
+		await executeBackup({
 			backup,
-			rcloneFlags,
+			executionId: deployment.deploymentId,
+			logPath: deployment.logPath,
 			rcloneDestination,
-			deployment.logPath,
-		);
-
-		if (mongo.serverId) {
-			await execAsyncRemote(mongo.serverId, backupCommand);
-		} else {
-			await execAsync(backupCommand, {
-				shell: "/bin/bash",
-			});
-		}
+			rcloneFlags,
+			serverId: mongo.serverId,
+		});
 
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
@@ -63,8 +52,8 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
 			projectName: project.name,
 			databaseType: "mongodb",
 			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
+			errorMessage:
+				error instanceof Error ? error.message : "Error message not provided",
 			organizationId: project.organizationId,
 			databaseName: backup.database,
 		});

packages/server/src/utils/backups/mysql.ts

@@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
 import type { MySql } from "@dokploy/server/services/mysql";
 import { findProjectById } from "@dokploy/server/services/project";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
-import { execAsync, execAsyncRemote } from "../process/execAsync";
-import {
-	getBackupCommand,
-	getBackupTimestamp,
-	getS3Credentials,
-	normalizeS3Path,
-} from "./utils";
+import { executeBackup } from "./executor";
+import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
 	const { environmentId, name, appName } = mysql;
@@ -33,20 +28,14 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
 	try {
 		const rcloneFlags = getS3Credentials(destination);
 		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
-		const backupCommand = getBackupCommand(
+		await executeBackup({
 			backup,
-			rcloneFlags,
+			executionId: deployment.deploymentId,
+			logPath: deployment.logPath,
 			rcloneDestination,
-			deployment.logPath,
-		);
-
-		if (mysql.serverId) {
-			await execAsyncRemote(mysql.serverId, backupCommand);
-		} else {
-			await execAsync(backupCommand, {
-				shell: "/bin/bash",
-			});
-		}
+			rcloneFlags,
+			serverId: mysql.serverId,
+		});
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
 			projectName: project.name,
@@ -63,8 +52,8 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
 			projectName: project.name,
 			databaseType: "mysql",
 			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
+			errorMessage:
+				error instanceof Error ? error.message : "Error message not provided",
 			organizationId: project.organizationId,
 			databaseName: backup.database,
 		});

packages/server/src/utils/backups/postgres.ts

@@ -8,13 +8,8 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
 import type { Postgres } from "@dokploy/server/services/postgres";
 import { findProjectById } from "@dokploy/server/services/project";
 import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
-import { execAsync, execAsyncRemote } from "../process/execAsync";
-import {
-	getBackupCommand,
-	getBackupTimestamp,
-	getS3Credentials,
-	normalizeS3Path,
-} from "./utils";
+import { executeBackup } from "./executor";
+import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
 
 export const runPostgresBackup = async (
 	postgres: Postgres,
@@ -36,19 +31,14 @@ export const runPostgresBackup = async (
 	try {
 		const rcloneFlags = getS3Credentials(destination);
 		const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
-		const backupCommand = getBackupCommand(
+		await executeBackup({
 			backup,
-			rcloneFlags,
+			executionId: deployment.deploymentId,
+			logPath: deployment.logPath,
 			rcloneDestination,
-			deployment.logPath,
-		);
-		if (postgres.serverId) {
-			await execAsyncRemote(postgres.serverId, backupCommand);
-		} else {
-			await execAsync(backupCommand, {
-				shell: "/bin/bash",
-			});
-		}
+			rcloneFlags,
+			serverId: postgres.serverId,
+		});
 
 		await sendDatabaseBackupNotifications({
 			applicationName: name,
@@ -66,8 +56,8 @@ export const runPostgresBackup = async (
 			projectName: project.name,
 			databaseType: "postgres",
 			type: "error",
-			// @ts-ignore
-			errorMessage: error?.message || "Error message not provided",
+			errorMessage:
+				error instanceof Error ? error.message : "Error message not provided",
 			organizationId: project.organizationId,
 			databaseName: backup.database,
 		});

packages/server/src/utils/backups/redact.ts

@@ -2,11 +2,11 @@
  * Redacts S3 credentials from rclone command strings.
  *
  * Used to prevent credential leakage in structured logs and error output.
- * Matches the flag format produced by `getS3Credentials()`:
- *   --s3-access-key-id="VALUE"  and  --s3-secret-access-key="VALUE"
+ * Matches quoted and unquoted flag values produced by `getS3Credentials()`.
  */
 export const redactRcloneCredentials = (command: string): string => {
-	return command
-		.replace(/(--s3-access-key-id=)"[^"]*"/g, '$1"[REDACTED]"')
-		.replace(/(--s3-secret-access-key=)"[^"]*"/g, '$1"[REDACTED]"');
+	return command.replace(
+		/(--s3-(?:access-key-id|secret-access-key)=)(?:(?:"[^"]*"|'[^']*'|\\.|[^\s'"])+)/g,
+		'$1"[REDACTED]"',
+	);
 };

packages/server/src/utils/backups/utils.ts

@@ -131,7 +131,7 @@ export const getLibsqlBackupCommand = (database: string) => {
 };
 
 export const getServiceContainerCommand = (appName: string) => {
-	return `docker ps -q --filter "status=running" --filter "label=com.docker.swarm.service.name=${appName}" | head -n 1`;
+	return `docker ps -q --no-trunc --filter "status=running" --filter "label=com.docker.swarm.service.name=${appName}" | head -n 1`;
 };
 
 export const getComposeContainerCommand = (
@@ -140,12 +140,12 @@ export const getComposeContainerCommand = (
 	composeType: "stack" | "docker-compose" | undefined,
 ) => {
 	if (composeType === "stack") {
-		return `docker ps -q --filter "status=running" --filter "label=com.docker.stack.namespace=${appName}" --filter "label=com.docker.swarm.service.name=${appName}_${serviceName}" | head -n 1`;
+		return `docker ps -q --no-trunc --filter "status=running" --filter "label=com.docker.stack.namespace=${appName}" --filter "label=com.docker.swarm.service.name=${appName}_${serviceName}" | head -n 1`;
 	}
-	return `docker ps -q --filter "status=running" --filter "label=com.docker.compose.project=${appName}" --filter "label=com.docker.compose.service=${serviceName}" | head -n 1`;
+	return `docker ps -q --no-trunc --filter "status=running" --filter "label=com.docker.compose.project=${appName}" --filter "label=com.docker.compose.service=${serviceName}" | head -n 1`;
 };
 
-const getContainerSearchCommand = (backup: BackupSchedule) => {
+export const getContainerSearchCommand = (backup: BackupSchedule) => {
 	const {
 		backupType,
 		postgres,
@@ -257,21 +257,52 @@ export const generateBackupCommand = (backup: BackupSchedule) => {
 	return null;
 };
 
+export const BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE = 75;
+
 export const getBackupCommand = (
 	backup: BackupSchedule,
 	rcloneFlags: string[],
 	rcloneDestination: string,
 	logPath: string,
+	options: {
+		containerId?: string;
+		containerNotFoundExitCode?: number;
+	} = {},
 ) => {
 	const containerSearch = getContainerSearchCommand(backup);
 	const backupCommand = generateBackupCommand(backup);
-	const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
-	const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
+	const quotedRcloneDestination = quote([rcloneDestination]);
+	const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} ${quotedRcloneDestination}`;
+	const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} ${quotedRcloneDestination}`;
+	const quotedLogPath = quote([logPath]);
+	const containerId = options.containerId;
+	const containerNotFoundExitCode = options.containerNotFoundExitCode ?? 1;
+	const containerNotFoundMessage =
+		containerNotFoundExitCode === BACKUP_WORKER_CONTAINER_NOT_FOUND_EXIT_CODE
+			? "Database container moved before the backup started"
+			: "❌ Error: Container not found";
+	if (containerId && !/^[a-f0-9]{12,64}$/i.test(containerId)) {
+		throw new Error("Invalid backup container ID");
+	}
+	if (
+		!Number.isInteger(containerNotFoundExitCode) ||
+		containerNotFoundExitCode < 1 ||
+		containerNotFoundExitCode > 255
+	) {
+		throw new Error("Invalid container-not-found exit code");
+	}
+	// A Swarm task can restart while the helper image starts, so re-resolve the
+	// original service locally if the captured task container is no longer running.
+	const containerAssignment = containerId
+		? `CONTAINER_ID=${quote([containerId])};
+	if [ "$(docker inspect --format '{{.State.Running}}' "$CONTAINER_ID" 2>/dev/null)" != "true" ]; then
+		CONTAINER_ID=$(${containerSearch});
+	fi;`
+		: `CONTAINER_ID=$(${containerSearch});`;
 
 	logger.info(
 		{
 			containerSearch,
-			backupCommand,
 			rcloneCommand: redactRcloneCredentials(rcloneCommand),
 			logPath,
 		},
@@ -280,26 +311,26 @@ export const getBackupCommand = (
 
 	return `
 	set -eo pipefail;
-	echo "[$(date)] Starting backup process..." >> ${logPath};
-	echo "[$(date)] Executing backup command..." >> ${logPath};
-	CONTAINER_ID=$(${containerSearch});
+	echo "[$(date)] Starting backup process..." >> ${quotedLogPath};
+	echo "[$(date)] Executing backup command..." >> ${quotedLogPath};
+	${containerAssignment}
 
 	if [ -z "$CONTAINER_ID" ]; then
-		echo "[$(date)] ❌ Error: Container not found" >> ${logPath};
-		exit 1;
+		echo "[$(date)] ${containerNotFoundMessage}" >> ${quotedLogPath};
+		exit ${containerNotFoundExitCode};
 	fi;
 
-	echo "[$(date)] Container Up: $CONTAINER_ID" >> ${logPath};
-	echo "[$(date)] Starting backup and upload to S3..." >> ${logPath};
+	echo "[$(date)] Container Up: $CONTAINER_ID" >> ${quotedLogPath};
+	echo "[$(date)] Starting backup and upload to S3..." >> ${quotedLogPath};
 
 	UPLOAD_OUTPUT=$({ ${backupCommand} | ${rcloneCommand}; } 2>&1 >/dev/null) || {
-		echo "[$(date)] ❌ Error: Backup failed" >> ${logPath};
-		echo "Error: $UPLOAD_OUTPUT" >> ${logPath};
+		echo "[$(date)] ❌ Error: Backup failed" >> ${quotedLogPath};
+		echo "Error: $UPLOAD_OUTPUT" >> ${quotedLogPath};
 		${rcloneDeleteCommand} >/dev/null 2>&1 || true;
 		exit 1;
 	};
 
-	echo "[$(date)] ✅ Backup uploaded to S3 successfully" >> ${logPath};
-	echo "Backup done ✅" >> ${logPath};
+	echo "[$(date)] ✅ Backup uploaded to S3 successfully" >> ${quotedLogPath};
+	echo "Backup done ✅" >> ${quotedLogPath};
 	`;
 };

packages/server/src/utils/backups/worker.ts

@@ -0,0 +1,416 @@
+import { createHash } from "node:crypto";
+import type { BackupSchedule } from "@dokploy/server/services/backup";
+import type { CreateServiceOptions } from "dockerode";
+import { sleep } from "../process/execAsync";
+import type { getRemoteDocker } from "../servers/remote-docker";
+
+const BACKUP_WORKER_IMAGE = "docker:28.5.2-cli";
+const BACKUP_SCRIPT_PATH = "/run/secrets/dokploy-backup-script";
+const DEFAULT_POLL_INTERVAL_MS = 1_000;
+const DEFAULT_START_TIMEOUT_MS = 5 * 60 * 1_000;
+const DEFAULT_MISSING_TASK_GRACE_POLLS = 3;
+const DEFAULT_REPLACEMENT_TASK_POLLS = 5 * 60;
+
+const terminalFailureStates = new Set([
+	"failed",
+	"rejected",
+	"shutdown",
+	"orphaned",
+	"remove",
+]);
+
+export type DockerClient = Awaited<ReturnType<typeof getRemoteDocker>>;
+
+type SwarmTask = {
+	ID?: string;
+	NodeID?: string;
+	Version?: { Index?: number };
+	Status?: {
+		State?: string;
+		Err?: string;
+		Message?: string;
+		ContainerStatus?: {
+			ContainerID?: string;
+			ExitCode?: number;
+		};
+	};
+};
+
+type WaitForTaskOptions = {
+	missingTaskGracePolls?: number;
+	pollIntervalMs?: number;
+	startTimeoutMs?: number;
+	sleepFn?: (milliseconds: number) => Promise<unknown>;
+};
+
+type WaitForReplacementTaskOptions = {
+	maxPolls?: number;
+	pollIntervalMs?: number;
+	sleepFn?: (milliseconds: number) => Promise<unknown>;
+};
+
+export const getBackupTargetServiceName = (
+	backup: BackupSchedule,
+): string | null => {
+	if (backup.backupType === "database") {
+		return (
+			backup.postgres?.appName ||
+			backup.mysql?.appName ||
+			backup.mariadb?.appName ||
+			backup.mongo?.appName ||
+			backup.libsql?.appName ||
+			null
+		);
+	}
+
+	return null;
+};
+
+export const getBackupResourceNames = (executionId: string, attempt = 0) => {
+	const suffix = createHash("sha256")
+		.update(attempt === 0 ? executionId : `${executionId}:retry:${attempt}`)
+		.digest("hex")
+		.slice(0, 16);
+	const serviceName = `dokploy-backup-${suffix}`;
+
+	return {
+		secretName: `${serviceName}-script`,
+		serviceName,
+	};
+};
+
+export class RunningServiceTaskNotFoundError extends Error {
+	constructor(serviceName: string) {
+		super(`No running Swarm task found for database service ${serviceName}`);
+		this.name = "RunningServiceTaskNotFoundError";
+	}
+}
+
+export class ReplacementServiceTaskNotFoundError extends Error {
+	constructor(serviceName: string) {
+		super(
+			`No replacement Swarm task became ready for database service ${serviceName} after relocation`,
+		);
+		this.name = "ReplacementServiceTaskNotFoundError";
+	}
+}
+
+export const findRunningServiceTask = async (
+	docker: DockerClient,
+	serviceName: string,
+) => {
+	const tasks = (await docker.listTasks({
+		filters: JSON.stringify({
+			service: [serviceName],
+			"desired-state": ["running"],
+		}),
+	})) as SwarmTask[];
+
+	const task = [...tasks]
+		.sort(
+			(left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0),
+		)
+		.find(
+			(candidate) =>
+				candidate.Status?.State === "running" &&
+				candidate.NodeID &&
+				candidate.Status.ContainerStatus?.ContainerID,
+		);
+
+	const nodeId = task?.NodeID;
+	const containerId = task?.Status?.ContainerStatus?.ContainerID;
+	if (!nodeId || !containerId) {
+		throw new RunningServiceTaskNotFoundError(serviceName);
+	}
+
+	return { containerId, nodeId };
+};
+
+const isSameContainerId = (left: string, right: string) =>
+	left === right ||
+	(left.length >= 12 &&
+		right.length >= 12 &&
+		(left.startsWith(right) || right.startsWith(left)));
+
+export const waitForReplacementServiceTask = async (
+	docker: DockerClient,
+	serviceName: string,
+	previousContainerId: string,
+	options: WaitForReplacementTaskOptions = {},
+) => {
+	const maxPolls = Math.max(
+		1,
+		Math.floor(options.maxPolls ?? DEFAULT_REPLACEMENT_TASK_POLLS),
+	);
+	const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+	const sleepFn = options.sleepFn ?? sleep;
+
+	for (let poll = 0; poll < maxPolls; poll += 1) {
+		try {
+			const task = await findRunningServiceTask(docker, serviceName);
+			if (!isSameContainerId(task.containerId, previousContainerId)) {
+				return task;
+			}
+		} catch (error) {
+			if (!(error instanceof RunningServiceTaskNotFoundError)) {
+				throw error;
+			}
+		}
+
+		if (poll < maxPolls - 1) {
+			await sleepFn(pollIntervalMs);
+		}
+	}
+
+	throw new ReplacementServiceTaskNotFoundError(serviceName);
+};
+
+const getLatestTask = (tasks: SwarmTask[]) =>
+	[...tasks].sort(
+		(left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0),
+	)[0];
+
+const getReplacementTask = (tasks: SwarmTask[], startedTaskId: string) =>
+	getLatestTask(tasks.filter((task) => task.ID !== startedTaskId));
+
+const getUniqueTaskAttempts = (tasks: SwarmTask[]) => {
+	const seen = new Set<string>();
+	return [...tasks]
+		.sort(
+			(left, right) => (right.Version?.Index ?? 0) - (left.Version?.Index ?? 0),
+		)
+		.filter((task) => {
+			if (!task.ID || seen.has(task.ID)) return false;
+			seen.add(task.ID);
+			return true;
+		});
+};
+
+const getTaskSummary = (task: SwarmTask) =>
+	`${task.ID ?? "unknown"} (${task.Status?.State ?? "unknown"})`;
+
+const getTaskReplacementMessage = (
+	startedTaskId: string,
+	replacement: SwarmTask,
+) =>
+	`Backup worker task ${startedTaskId} was replaced after starting by task ${getTaskSummary(replacement)}`;
+
+const getTaskFailureMessage = (task: SwarmTask) => {
+	const state = task.Status?.State ?? "unknown";
+	const detail = task.Status?.Err || task.Status?.Message;
+	const exitCode = task.Status?.ContainerStatus?.ExitCode;
+	const suffix = [
+		detail,
+		exitCode !== undefined ? `exit code ${exitCode}` : undefined,
+	]
+		.filter(Boolean)
+		.join(", ");
+
+	return `Backup worker task ${state}${suffix ? `: ${suffix}` : ""}`;
+};
+
+export class BackupWorkerTaskError extends Error {
+	constructor(
+		message: string,
+		readonly state: string,
+		readonly exitCode?: number,
+	) {
+		super(message);
+		this.name = "BackupWorkerTaskError";
+	}
+}
+
+export class BackupWorkerPreStartError extends Error {
+	constructor(
+		message: string,
+		readonly state: string,
+	) {
+		super(message);
+		this.name = "BackupWorkerPreStartError";
+	}
+}
+
+const getTaskFailureError = (task: SwarmTask, observedRunning = false) => {
+	const state = task.Status?.State ?? "unknown";
+	const containerStatus = task.Status?.ContainerStatus;
+	if (
+		!observedRunning &&
+		state === "rejected" &&
+		!containerStatus?.ContainerID &&
+		containerStatus?.ExitCode === undefined
+	) {
+		return new BackupWorkerPreStartError(getTaskFailureMessage(task), state);
+	}
+
+	return new BackupWorkerTaskError(
+		getTaskFailureMessage(task),
+		state,
+		containerStatus?.ExitCode,
+	);
+};
+
+export const waitForBackupWorkerTask = async (
+	docker: DockerClient,
+	serviceId: string,
+	options: WaitForTaskOptions = {},
+) => {
+	const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+	const startTimeoutMs = options.startTimeoutMs ?? DEFAULT_START_TIMEOUT_MS;
+	const missingTaskGracePolls = Math.max(
+		1,
+		options.missingTaskGracePolls ?? DEFAULT_MISSING_TASK_GRACE_POLLS,
+	);
+	const sleepFn = options.sleepFn ?? sleep;
+	const startedAt = Date.now();
+	let startedTaskId: string | null = null;
+	let missingTaskPolls = 0;
+
+	while (true) {
+		const tasks = (await docker.listTasks({
+			filters: JSON.stringify({ service: [serviceId] }),
+		})) as SwarmTask[];
+
+		if (startedTaskId) {
+			const task = tasks.find((candidate) => candidate.ID === startedTaskId);
+			const state = task?.Status?.State;
+
+			if (task && state && terminalFailureStates.has(state)) {
+				throw getTaskFailureError(task, true);
+			}
+
+			const replacement = getReplacementTask(tasks, startedTaskId);
+			if (task && replacement) {
+				throw new Error(getTaskReplacementMessage(startedTaskId, replacement));
+			}
+
+			if (state === "complete") {
+				return;
+			}
+			if (state === "running") {
+				missingTaskPolls = 0;
+			} else if (task) {
+				throw new Error(
+					`Backup worker task ${startedTaskId} entered ${state ?? "an unknown state"} after starting`,
+				);
+			} else {
+				missingTaskPolls += 1;
+				if (missingTaskPolls >= missingTaskGracePolls) {
+					if (replacement) {
+						throw new Error(
+							getTaskReplacementMessage(startedTaskId, replacement),
+						);
+					}
+					throw new Error("Backup worker task disappeared after starting");
+				}
+			}
+
+			await sleepFn(pollIntervalMs);
+			continue;
+		}
+
+		const taskAttempts = getUniqueTaskAttempts(tasks);
+		if (taskAttempts.length > 1) {
+			throw new Error(
+				`Backup worker service reported multiple task attempts before execution could be tracked: ${taskAttempts.map(getTaskSummary).join(", ")}`,
+			);
+		}
+
+		const task = getLatestTask(tasks);
+		const state = task?.Status?.State;
+
+		if (state === "complete") {
+			return;
+		}
+		if (task && state && terminalFailureStates.has(state)) {
+			throw getTaskFailureError(task);
+		}
+		if (state === "running") {
+			if (!task?.ID) {
+				throw new Error("Backup worker running task did not provide an ID");
+			}
+			startedTaskId = task.ID;
+		}
+
+		if (!startedTaskId && Date.now() - startedAt >= startTimeoutMs) {
+			const message = `Backup worker did not start within ${Math.round(startTimeoutMs / 1_000)} seconds`;
+			throw new Error(message);
+		}
+
+		await sleepFn(pollIntervalMs);
+	}
+};
+
+export const getBackupWorkerServiceSpec = ({
+	backupId,
+	executionId,
+	nodeId,
+	secretId,
+	secretName,
+	serviceName,
+}: {
+	backupId: string;
+	executionId: string;
+	nodeId: string;
+	secretId: string;
+	secretName: string;
+	serviceName: string;
+}): CreateServiceOptions => {
+	const labels = {
+		"dokploy.backup.id": backupId,
+		"dokploy.deployment.id": executionId,
+		"dokploy.managed": "true",
+		"dokploy.resource": "database-backup-worker",
+	};
+
+	return {
+		Name: serviceName,
+		Labels: labels,
+		TaskTemplate: {
+			ContainerSpec: {
+				Image: BACKUP_WORKER_IMAGE,
+				Command: ["/bin/sh", "-c"],
+				Args: [
+					`apk add --no-cache bash rclone >/dev/null || exit 1; exec /bin/bash ${BACKUP_SCRIPT_PATH}`,
+				],
+				Labels: labels,
+				Mounts: [
+					{
+						Type: "bind",
+						Source: "/var/run/docker.sock",
+						Target: "/var/run/docker.sock",
+					},
+				],
+				Secrets: [
+					{
+						SecretID: secretId,
+						SecretName: secretName,
+						File: {
+							Name: BACKUP_SCRIPT_PATH.replace("/run/secrets/", ""),
+							UID: "0",
+							GID: "0",
+							Mode: 0o400,
+						},
+					},
+				],
+			},
+			LogDriver: {
+				Name: "json-file",
+				Options: {
+					"max-size": "10m",
+					"max-file": "1",
+				},
+			},
+			Placement: {
+				Constraints: [`node.id==${nodeId}`],
+			},
+			RestartPolicy: {
+				Condition: "none",
+			},
+		},
+		Mode: {
+			Replicated: {
+				Replicas: 1,
+			},
+		},
+	};
+};