This commit is contained in:
Furox 2026-09-11 13:28:31 +00:00 committed by GitHub
commit 746044367d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 1492 additions and 409 deletions

View File

@ -1,4 +1,8 @@
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
import {
getSafeRcloneErrorMessage,
redactRcloneCredentials,
} from "@dokploy/server/utils/backups/redact";
import { quote } from "shell-quote";
import { describe, expect, it } from "vitest";
describe("redactRcloneCredentials (#4621)", () => {
@ -27,6 +31,23 @@ describe("redactRcloneCredentials (#4621)", () => {
expect(redacted).toContain('--s3-region="us-east-1"');
});
it("should redact FTP and SFTP obscured password flags", () => {
for (const backend of ["ftp", "sftp"] as const) {
const cmd = `rclone lsf --${backend}-host=storage.example.com --${backend}-pass='obscured-secret' :${backend}:backups`;
const redacted = redactRcloneCredentials(cmd);
expect(redacted).not.toContain("obscured-secret");
expect(redacted).toContain(`--${backend}-pass="[REDACTED]"`);
}
});
it("should redact unquoted FTP and SFTP password flags", () => {
const cmd =
"rclone lsf --ftp-pass=ftp-secret --sftp-pass=sftp-secret :ftp:backups";
const redacted = redactRcloneCredentials(cmd);
expect(redacted).not.toContain("ftp-secret");
expect(redacted).not.toContain("sftp-secret");
});
it("should not modify non-credential flags", () => {
const cmd =
'rclone rcat --s3-region="eu-west-1" --s3-endpoint="https://s3.example.com" --s3-no-check-bucket :s3:bucket/file.gz';
@ -47,4 +68,32 @@ describe("redactRcloneCredentials (#4621)", () => {
expect(redacted).not.toContain("MYSECRET");
expect(redacted).toContain("[REDACTED]");
});
it("should sanitize rclone credentials from propagated backup errors", () => {
const error = new Error(
"Command failed: rclone rcat --s3-access-key-id=ACCESS_VALUE_9X --s3-secret-access-key=S3_VALUE_LEAK_9X --ftp-pass=FTP_VALUE_LEAK_9X --sftp-pass=SFTP_VALUE_LEAK_9X --sftp-key-file-pass=KEY_VALUE_LEAK_9X :ftp:backups/file.gz",
);
const safe = getSafeRcloneErrorMessage(error);
for (const secret of [
"ACCESS_VALUE_9X",
"S3_VALUE_LEAK_9X",
"FTP_VALUE_LEAK_9X",
"SFTP_VALUE_LEAK_9X",
"KEY_VALUE_LEAK_9X",
]) {
expect(safe).not.toContain(secret);
}
expect(safe.match(/\[REDACTED\]/g)?.length).toBe(5);
});
it("should fully redact shell-quote output with embedded quotes and whitespace", () => {
const secret = "PART_A' PART_B\" $PART_C;\\PART_D";
const cmd = `rclone lsf --s3-secret-access-key=${quote([secret])} --s3-region=us-east-1 :s3:bucket`;
const redacted = redactRcloneCredentials(cmd);
for (const fragment of ["PART_A", "PART_B", "PART_C", "PART_D"]) {
expect(redacted).not.toContain(fragment);
}
expect(redacted).toContain('--s3-secret-access-key="[REDACTED]"');
expect(redacted).toContain("--s3-region=us-east-1");
});
});

View File

@ -1,4 +1,10 @@
import { normalizeS3Path } from "@dokploy/server/utils/backups/utils";
import { apiCreateDestination } from "@dokploy/server/db/schema/destination";
import { RCLONE_DESTINATION_PROVIDERS } from "@dokploy/server/db/validations/destination";
import {
getRclonePathAndFlags,
normalizeS3Path,
} from "@dokploy/server/utils/backups/utils";
import { normalizeVolumeBackupFilePath } from "@dokploy/server/utils/volume-backups/restore";
import { describe, expect, test } from "vitest";
describe("normalizeS3Path", () => {
@ -59,3 +65,363 @@ describe("normalizeS3Path", () => {
expect(normalizeS3Path("instance-backups")).toBe("instance-backups/");
});
});
describe("normalizeVolumeBackupFilePath", () => {
test("accepts a destination-relative backup path", () => {
expect(normalizeVolumeBackupFilePath("app/prefix/volume-2026.tar")).toBe(
"app/prefix/volume-2026.tar",
);
});
test.each([
"/absolute/backup.tar",
"../backup.tar",
"app/../backup.tar",
"backup.tar; touch /tmp/pwned",
"backup.tar$(touch /tmp/pwned)",
"backup.tar`touch /tmp/pwned`",
"backup.tar\nmalicious-command",
])("rejects unsafe restore path %s", (value) => {
expect(() => normalizeVolumeBackupFilePath(value)).toThrow(
"Invalid volume backup file path",
);
});
});
const destination = (overrides: Record<string, unknown> = {}) =>
({
destinationId: "destination-id",
name: "Test",
provider: "AWS",
accessKey: "access",
secretAccessKey: "secret",
bucket: "bucket",
region: "us-east-1",
endpoint: "https://s3.example.com",
additionalFlags: [],
organizationId: "organization-id",
createdAt: new Date(0),
...overrides,
}) as any;
describe("FTP destination validation", () => {
const input = {
name: "FTP backups",
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
accessKey: "backup-user",
secretAccessKey: "secret",
bucket: "backups",
region: "",
endpoint: "storage.example.com",
};
test("rejects plaintext FTP", () => {
expect(
apiCreateDestination.safeParse({ ...input, additionalFlags: [] }).success,
).toBe(false);
});
test.each(["--ftp-explicit-tls", "--ftp-tls"])(
"accepts encrypted FTP with %s",
(flag) => {
const result = apiCreateDestination.safeParse({
...input,
additionalFlags: [flag],
});
expect(
result.success,
result.success ? undefined : JSON.stringify(result.error.issues),
).toBe(true);
},
);
test("rejects conflicting implicit and explicit FTPS", () => {
expect(
apiCreateDestination.safeParse({
...input,
additionalFlags: ["--ftp-explicit-tls", "--ftp-tls"],
}).success,
).toBe(false);
});
});
describe("SFTP destination validation", () => {
const input = {
name: "SFTP backups",
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
accessKey: "backup-user",
secretAccessKey: "",
bucket: "backups",
region: "",
endpoint: "storage.example.com",
};
test("rejects SFTP without host-key verification", () => {
expect(
apiCreateDestination.safeParse({ ...input, additionalFlags: [] }).success,
).toBe(false);
});
test("rejects explicitly disabled SFTP host-key verification", () => {
expect(
apiCreateDestination.safeParse({
...input,
additionalFlags: ["--sftp-known-hosts-file=none"],
}).success,
).toBe(false);
});
test("accepts SFTP with a known-hosts file", () => {
const result = apiCreateDestination.safeParse({
...input,
additionalFlags: ["--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts"],
});
expect(
result.success,
result.success ? undefined : JSON.stringify(result.error.issues),
).toBe(true);
});
});
describe("getRclonePathAndFlags", () => {
test("preserves the existing S3 destination behavior", async () => {
const result = await getRclonePathAndFlags(
destination(),
"service/prefix/backup.sql.gz",
);
expect(result.path).toBe(":s3:bucket/service/prefix/backup.sql.gz");
expect(result.flags).toContain("--s3-provider=AWS");
expect(result.flags).toContain("--s3-access-key-id=access");
expect(result.flags).toContain("--s3-secret-access-key=secret");
});
test.each([
RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
RCLONE_DESTINATION_PROVIDERS.ONEDRIVE,
RCLONE_DESTINATION_PROVIDERS.REMOTE,
])("builds a safe named rclone remote for %s", async (provider) => {
const result = await getRclonePathAndFlags(
destination({
provider,
endpoint: "team-drive",
bucket: "/dokploy/",
accessKey: "",
secretAccessKey: "",
region: "",
additionalFlags: ["--transfers=2"],
}),
"/service/backup.sql.gz",
);
expect(result.path).toBe("team-drive:dokploy/service/backup.sql.gz");
expect(result.flags).toEqual(["--transfers=2"]);
});
test("rejects unsafe named remote names", async () => {
await expect(
getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
endpoint: "drive;touch /tmp/pwned",
bucket: "",
}),
),
).rejects.toThrow("Invalid rclone remote name");
});
test("rejects unsafe stored additional flags at runtime", async () => {
await expect(
getRclonePathAndFlags(
destination({ additionalFlags: ["--transfers=2;touch /tmp/pwned"] }),
),
).rejects.toThrow("Invalid flag format");
});
test.each([
["--ftp-explicit-tls", "21"],
["--ftp-tls", "990"],
] as const)(
"uses secure FTP mode %s with default port %s",
async (tlsFlag, defaultPort) => {
const result = await getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
endpoint: "storage.example.com",
accessKey: "backup-user",
secretAccessKey: "",
region: "",
bucket: "/backups/",
additionalFlags: [tlsFlag],
}),
"service/backup.tar",
);
expect(result.path).toBe(":ftp:backups/service/backup.tar");
expect(result.flags).toContain("--ftp-host=storage.example.com");
expect(result.flags).toContain("--ftp-user=backup-user");
expect(result.flags).toContain(`--ftp-port=${defaultPort}`);
expect(result.flags).toContain(tlsFlag);
},
);
test("rejects plaintext FTP at runtime", async () => {
await expect(
getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
endpoint: "storage.example.com",
accessKey: "backup-user",
secretAccessKey: "",
region: "",
bucket: "/backups/",
additionalFlags: [],
}),
),
).rejects.toThrow("FTP destinations must use TLS");
});
test("builds SFTP flags with host-key verification and port 22", async () => {
const result = await getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
endpoint: "storage.example.com",
accessKey: "backup-user",
secretAccessKey: "",
region: "",
bucket: "/backups/",
additionalFlags: ["--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts"],
}),
"service/backup.tar",
);
expect(result.path).toBe(":sftp:backups/service/backup.tar");
expect(result.flags).toContain("--sftp-host=storage.example.com");
expect(result.flags).toContain("--sftp-user=backup-user");
expect(result.flags).toContain("--sftp-port=22");
expect(result.flags).toContain(
"--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts",
);
});
test("rejects SFTP without host-key verification at runtime", async () => {
await expect(
getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
endpoint: "storage.example.com",
accessKey: "backup-user",
secretAccessKey: "",
region: "",
bucket: "/backups/",
additionalFlags: [],
}),
),
).rejects.toThrow("SFTP destinations must verify the server host key");
});
});
describe("FTP TLS certificate verification", () => {
const input = {
name: "FTP backups",
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
accessKey: "backup-user",
secretAccessKey: "secret",
bucket: "backups",
region: "",
endpoint: "storage.example.com",
};
test.each([
"--ftp-no-check-certificate",
"--ftp-no-check-certificate=true",
"--ftp-no-check-certificate=TRUE",
"--ftp-no-check-certificate=1",
"--ftp-no-check-certificate=t",
"--no-check-certificate",
"--no-check-certificate=true",
"--no-check-certificate=TRUE",
"--no-check-certificate=1",
"--no-check-certificate=t",
])(
"rejects certificate-verification bypass %s at schema validation",
(flag) => {
expect(
apiCreateDestination.safeParse({
...input,
additionalFlags: ["--ftp-explicit-tls", flag],
}).success,
).toBe(false);
},
);
test.each([
"--ftp-no-check-certificate",
"--ftp-no-check-certificate=TRUE",
"--ftp-no-check-certificate=1",
"--no-check-certificate",
"--no-check-certificate=TRUE",
"--no-check-certificate=1",
])("rejects certificate-verification bypass %s at runtime", async (flag) => {
await expect(
getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
endpoint: "storage.example.com",
accessKey: "backup-user",
secretAccessKey: "",
region: "",
bucket: "backups",
additionalFlags: ["--ftp-explicit-tls", flag],
}),
),
).rejects.toThrow("FTP TLS certificate verification cannot be disabled");
});
test.each([
"--ftp-no-check-certificate=false",
"--ftp-no-check-certificate=FALSE",
"--ftp-no-check-certificate=0",
"--ftp-no-check-certificate=f",
"--no-check-certificate=false",
"--no-check-certificate=FALSE",
"--no-check-certificate=0",
"--no-check-certificate=f",
])("allows explicitly safe certificate flag %s", (flag) => {
expect(
apiCreateDestination.safeParse({
...input,
additionalFlags: ["--ftp-explicit-tls", flag],
}).success,
).toBe(true);
});
test.each([
"--ftp-explicit-tls=TRUE",
"--ftp-explicit-tls=1",
"--ftp-explicit-tls=t",
])("accepts pflag-compatible TLS true value %s", (flag) => {
expect(
apiCreateDestination.safeParse({
...input,
additionalFlags: [flag],
}).success,
).toBe(true);
});
test("forces certificate verification on after user flags", async () => {
const result = await getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
endpoint: "storage.example.com",
accessKey: "backup-user",
secretAccessKey: "",
region: "",
bucket: "backups",
additionalFlags: ["--ftp-explicit-tls"],
}),
);
expect(result.flags.slice(-2)).toEqual([
"--ftp-no-check-certificate=false",
"--no-check-certificate=false",
]);
});
});

View File

@ -0,0 +1,137 @@
import { apiCreateDestination } from "@dokploy/server/db/schema/destination";
import { RCLONE_DESTINATION_PROVIDERS } from "@dokploy/server/db/validations/destination";
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
import {
assertSafeRclonePath,
getRclonePathAndFlags,
} from "@dokploy/server/utils/backups/utils";
import {
normalizeDockerVolumeName,
normalizeVolumeBackupFilePath,
} from "@dokploy/server/utils/volume-backups/restore";
import { describe, expect, test } from "vitest";
const destination = (overrides: Record<string, unknown> = {}) =>
({
destinationId: "destination-id",
name: "Test",
provider: RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
accessKey: "",
secretAccessKey: "",
bucket: "dokploy",
region: "",
endpoint: "team-drive",
additionalFlags: [],
organizationId: "organization-id",
createdAt: new Date(0),
...overrides,
}) as any;
describe("issue #416 rclone path safety", () => {
test.each([
"../backup.sql.gz",
"app/../backup.sql.gz",
"app/./backup.sql.gz",
"..\\backup.sql.gz",
"app\\..\\backup.sql.gz",
"backup.sql.gz\nother",
"backup.sql.gz\rboom",
"backup.sql.gz\0boom",
])("rejects unsafe destination-relative path %s", (value) => {
expect(() => assertSafeRclonePath(value)).toThrow("Invalid rclone path");
});
test.each([
"service/backup.sql.gz",
"/service/backup.sql.gz",
"service/backup..sql.gz",
"service/file name.sql.gz",
])("keeps valid backup path %s", (value) => {
expect(() => assertSafeRclonePath(value)).not.toThrow();
});
test.each([
RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
RCLONE_DESTINATION_PROVIDERS.ONEDRIVE,
RCLONE_DESTINATION_PROVIDERS.REMOTE,
RCLONE_DESTINATION_PROVIDERS.FTP,
RCLONE_DESTINATION_PROVIDERS.SFTP,
"AWS",
])("blocks traversal before building a %s target", async (provider) => {
await expect(
getRclonePathAndFlags(destination({ provider }), "app/../outside.sql.gz"),
).rejects.toThrow("Invalid rclone path");
});
});
describe("issue #416 SFTP host-key safety", () => {
const conflictingKnownHostsFlags = [
"--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts",
"--sftp-known-hosts-file=none",
];
test("rejects conflicting host-key flags during destination validation", () => {
const result = apiCreateDestination.safeParse({
name: "SFTP backups",
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
accessKey: "backup-user",
secretAccessKey: "",
bucket: "backups",
region: "",
endpoint: "storage.example.com",
additionalFlags: conflictingKnownHostsFlags,
});
expect(result.success).toBe(false);
});
test("rejects conflicting host-key flags at runtime", async () => {
await expect(
getRclonePathAndFlags(
destination({
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
endpoint: "storage.example.com",
accessKey: "backup-user",
secretAccessKey: "",
region: "",
bucket: "backups",
additionalFlags: conflictingKnownHostsFlags,
}),
),
).rejects.toThrow("SFTP destinations must verify the server host key");
});
});
describe("issue #416 credential redaction", () => {
test("redacts an SFTP private-key passphrase flag", () => {
const command =
"rclone lsf --sftp-key-file=/root/.ssh/id_rsa --sftp-key-file-pass=obscured-secret :sftp:backups";
const redacted = redactRcloneCredentials(command);
expect(redacted).not.toContain("obscured-secret");
expect(redacted).toContain('--sftp-key-file-pass="[REDACTED]"');
});
});
describe("issue #416 volume name and backup path shell safety", () => {
test("accepts valid docker volume names", () => {
expect(normalizeDockerVolumeName("app_data")).toBe("app_data");
expect(normalizeDockerVolumeName("App.Data-1")).toBe("App.Data-1");
});
test.each(["", "../evil", "vol;rm -rf /", "vol$(id)", "-sneaky"])(
"rejects unsafe docker volume name %s",
(value) => {
expect(() => normalizeDockerVolumeName(value)).toThrow(
"Invalid docker volume name",
);
},
);
test("normalizes generated backup file names", () => {
expect(
normalizeVolumeBackupFilePath("app_data-2026-01-01T00-00-00.tar"),
).toBe("app_data-2026-01-01T00-00-00.tar");
});
});

View File

@ -1,3 +1,5 @@
import { RCLONE_DESTINATION_PROVIDERS } from "@dokploy/server/db/validations/destination";
export const S3_PROVIDERS: Array<{
key: string;
name: string;
@ -131,3 +133,30 @@ export const S3_PROVIDERS: Array<{
name: "Any other S3 compatible provider",
},
];
export const DESTINATION_PROVIDERS: Array<{
key: string;
name: string;
}> = [
{
key: RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
name: "Google Drive (configured rclone remote)",
},
{
key: RCLONE_DESTINATION_PROVIDERS.ONEDRIVE,
name: "Microsoft OneDrive (configured rclone remote)",
},
{
key: RCLONE_DESTINATION_PROVIDERS.SFTP,
name: "SFTP",
},
{
key: RCLONE_DESTINATION_PROVIDERS.FTP,
name: "FTP",
},
{
key: RCLONE_DESTINATION_PROVIDERS.REMOTE,
name: "Other configured rclone remote",
},
...S3_PROVIDERS,
];

View File

@ -1,6 +1,11 @@
import {
ADDITIONAL_FLAG_ERROR,
ADDITIONAL_FLAG_REGEX,
getDestinationValidationIssues,
getFtpTlsState,
isNamedRcloneDestinationProvider,
isRcloneDestinationProvider,
RCLONE_DESTINATION_PROVIDERS,
} from "@dokploy/server/db/validations/destination";
import { standardSchemaResolver as zodResolver } from "@hookform/resolvers/standard-schema";
import { PenBoxIcon, PlusIcon, Trash2 } from "lucide-react";
@ -39,28 +44,62 @@ import {
} from "@/components/ui/select";
import { cn } from "@/lib/utils";
import { api } from "@/utils/api";
import { S3_PROVIDERS } from "./constants";
import { DESTINATION_PROVIDERS } from "./constants";
const addDestination = z.object({
name: z.string().min(1, "Name is required"),
provider: z.string().min(1, "Provider is required"),
accessKeyId: z.string().min(1, "Access Key Id is required"),
secretAccessKey: z.string().min(1, "Secret Access Key is required"),
bucket: z.string().min(1, "Bucket is required"),
region: z.string(),
endpoint: z.string().min(1, "Endpoint is required"),
serverId: z.string().optional(),
additionalFlags: z
.array(
z.object({
value: z
.string()
.min(1, "Flag cannot be empty")
.regex(ADDITIONAL_FLAG_REGEX, ADDITIONAL_FLAG_ERROR),
}),
)
.optional(),
});
const addDestination = z
.object({
name: z.string().min(1, "Name is required"),
provider: z.string().min(1, "Provider is required"),
accessKeyId: z.string(),
secretAccessKey: z.string(),
bucket: z.string(),
region: z.string(),
endpoint: z.string(),
serverId: z.string().optional(),
additionalFlags: z
.array(
z.object({
value: z
.string()
.min(1, "Flag cannot be empty")
.regex(ADDITIONAL_FLAG_REGEX, ADDITIONAL_FLAG_ERROR),
}),
)
.optional(),
})
.superRefine((data, ctx) => {
const additionalFlags = data.additionalFlags?.map((flag) => flag.value) ?? [];
for (const issue of getDestinationValidationIssues({
provider: data.provider,
accessKey: data.accessKeyId,
region: data.region,
endpoint: data.endpoint,
additionalFlags,
})) {
ctx.addIssue({
code: "custom",
path: [issue.field === "accessKey" ? "accessKeyId" : issue.field],
message: issue.message,
});
}
if (isRcloneDestinationProvider(data.provider)) return;
for (const [field, label] of [
["accessKeyId", "Access Key Id"],
["secretAccessKey", "Secret Access Key"],
["bucket", "Bucket"],
["endpoint", "Endpoint"],
] as const) {
if (!data[field].trim()) {
ctx.addIssue({
code: "custom",
path: [field],
message: `${label} is required`,
});
}
}
});
type AddDestination = z.infer<typeof addDestination>;
@ -108,6 +147,19 @@ export const HandleDestinations = ({ destinationId }: Props) => {
resolver: zodResolver(addDestination),
});
const currentProvider = form.watch("provider");
const currentAdditionalFlags =
form.watch("additionalFlags")?.map((flag) => flag.value) ?? [];
const { implicitTlsEnabled: implicitFtpTlsEnabled } = getFtpTlsState(
currentAdditionalFlags,
);
const isNamedRemote = isNamedRcloneDestinationProvider(currentProvider);
const isFileTransfer =
currentProvider === RCLONE_DESTINATION_PROVIDERS.FTP ||
currentProvider === RCLONE_DESTINATION_PROVIDERS.SFTP;
const hasRemoteServers = (servers?.length ?? 0) > 0;
const showServerSelector = Boolean(isCloud) || hasRemoteServers;
const { fields, append, remove } = useFieldArray({
control: form.control,
name: "additionalFlags",
@ -131,17 +183,25 @@ export const HandleDestinations = ({ destinationId }: Props) => {
}
}, [form, form.reset, form.formState.isSubmitSuccessful, destination]);
const getPayload = (data: AddDestination) => ({
provider: data.provider,
accessKey: isNamedRcloneDestinationProvider(data.provider)
? ""
: data.accessKeyId,
bucket: data.bucket,
endpoint: data.endpoint.trim(),
name: data.name,
region: isNamedRcloneDestinationProvider(data.provider) ? "" : data.region,
secretAccessKey: isNamedRcloneDestinationProvider(data.provider)
? ""
: data.secretAccessKey,
additionalFlags: data.additionalFlags?.map((f) => f.value) ?? [],
});
const onSubmit = async (data: AddDestination) => {
await mutateAsync({
provider: data.provider || "",
accessKey: data.accessKeyId,
bucket: data.bucket,
endpoint: data.endpoint,
name: data.name,
region: data.region,
secretAccessKey: data.secretAccessKey,
...getPayload(data),
destinationId: destinationId || "",
additionalFlags: data.additionalFlags?.map((f) => f.value) ?? [],
})
.then(async () => {
toast.success(`Destination ${destinationId ? "Updated" : "Created"}`);
@ -162,14 +222,7 @@ export const HandleDestinations = ({ destinationId }: Props) => {
};
const handleTestConnection = async (serverId?: string) => {
const result = await form.trigger([
"provider",
"accessKeyId",
"secretAccessKey",
"bucket",
"endpoint",
"additionalFlags",
]);
const result = await form.trigger();
if (!result) {
const errors = form.formState.errors;
@ -184,38 +237,23 @@ export const HandleDestinations = ({ destinationId }: Props) => {
return;
}
if (isCloud && !serverId) {
const selectedServerId = serverId === "none" ? undefined : serverId;
if (isCloud && !selectedServerId) {
toast.error("Please select a server");
return;
}
const provider = form.getValues("provider");
const accessKey = form.getValues("accessKeyId");
const secretKey = form.getValues("secretAccessKey");
const bucket = form.getValues("bucket");
const endpoint = form.getValues("endpoint");
const region = form.getValues("region");
const connectionString = `:s3,provider=${provider},access_key_id=${accessKey},secret_access_key=${secretKey},endpoint=${endpoint}${region ? `,region=${region}` : ""}:${bucket}`;
await testConnection({
provider,
accessKey,
bucket,
endpoint,
...getPayload(form.getValues()),
name: "Test",
region,
secretAccessKey: secretKey,
serverId,
additionalFlags:
form.getValues("additionalFlags")?.map((f) => f.value) ?? [],
serverId: selectedServerId,
})
.then(() => {
toast.success("Connection Success");
})
.catch((e) => {
toast.error("Error connecting to provider", {
description: `${e.message}\n\nTry manually: rclone ls ${connectionString}`,
description: e.message,
});
});
};
@ -244,9 +282,9 @@ export const HandleDestinations = ({ destinationId }: Props) => {
{destinationId ? "Update" : "Add"} Destination
</DialogTitle>
<DialogDescription>
In this section, you can configure and add new destinations for your
backups. Please ensure that you provide the correct information to
guarantee secure and efficient storage.
Configure a backup destination. Google Drive, OneDrive, and generic
rclone destinations use a named rclone remote configured on the
machine that executes the backup.
</DialogDescription>
</DialogHeader>
{(isError || isErrorConnection) && (
@ -264,123 +302,113 @@ export const HandleDestinations = ({ destinationId }: Props) => {
<FormField
control={form.control}
name="name"
render={({ field }) => {
return (
<FormItem>
<FormLabel>Name</FormLabel>
<FormControl>
<Input placeholder={"S3 Bucket"} {...field} />
</FormControl>
<FormMessage />
</FormItem>
);
}}
/>
<FormField
control={form.control}
name="provider"
render={({ field }) => {
return (
<FormItem>
<FormLabel>Provider</FormLabel>
<FormControl>
<Select
onValueChange={field.onChange}
defaultValue={field.value}
value={field.value}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a S3 Provider" />
</SelectTrigger>
</FormControl>
<SelectContent>
{S3_PROVIDERS.map((s3Provider) => (
<SelectItem
key={s3Provider.key}
value={s3Provider.key}
>
{s3Provider.name}
</SelectItem>
))}
</SelectContent>
</Select>
</FormControl>
<FormMessage />
</FormItem>
);
}}
/>
<FormField
control={form.control}
name="accessKeyId"
render={({ field }) => {
return (
<FormItem>
<FormLabel>Access Key Id</FormLabel>
<FormControl>
<Input placeholder={"xcas41dasde"} {...field} />
</FormControl>
<FormMessage />
</FormItem>
);
}}
/>
<FormField
control={form.control}
name="secretAccessKey"
render={({ field }) => (
<FormItem>
<div className="space-y-0.5">
<FormLabel>Secret Access Key</FormLabel>
</div>
<FormLabel>Name</FormLabel>
<FormControl>
<Input placeholder={"asd123asdasw"} {...field} />
<Input placeholder="Backups" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="provider"
render={({ field }) => (
<FormItem>
<FormLabel>Provider</FormLabel>
<FormControl>
<Select
onValueChange={field.onChange}
defaultValue={field.value}
value={field.value}
>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a destination provider" />
</SelectTrigger>
</FormControl>
<SelectContent>
{DESTINATION_PROVIDERS.map((provider) => (
<SelectItem key={provider.key} value={provider.key}>
{provider.name}
</SelectItem>
))}
</SelectContent>
</Select>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
{!isNamedRemote && (
<FormField
control={form.control}
name="accessKeyId"
render={({ field }) => (
<FormItem>
<FormLabel>
{isFileTransfer ? "Username" : "Access Key Id"}
</FormLabel>
<FormControl>
<Input
placeholder={
isFileTransfer ? "username" : "Access Key ID"
}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}
{!isNamedRemote && (
<FormField
control={form.control}
name="secretAccessKey"
render={({ field }) => (
<FormItem>
<FormLabel>
{isFileTransfer
? "Password (Optional)"
: "Secret Access Key"}
</FormLabel>
<FormControl>
<Input
type={isFileTransfer ? "password" : "text"}
placeholder={
isFileTransfer ? "password" : "Secret Access Key"
}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}
<FormField
control={form.control}
name="bucket"
render={({ field }) => (
<FormItem>
<div className="space-y-0.5">
<FormLabel>Bucket</FormLabel>
</div>
<FormControl>
<Input placeholder={"dokploy-bucket"} {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="region"
render={({ field }) => (
<FormItem>
<div className="space-y-0.5">
<FormLabel>Region</FormLabel>
</div>
<FormControl>
<Input placeholder={"us-east-1"} {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="endpoint"
render={({ field }) => (
<FormItem>
<FormLabel>Endpoint</FormLabel>
<FormLabel>
{isNamedRemote
? "Remote Base Path (Optional)"
: isFileTransfer
? "Base Path / Directory (Optional)"
: "Bucket"}
</FormLabel>
<FormControl>
<Input
placeholder={"https://us.bucket.aws/s3"}
placeholder={
isNamedRemote || isFileTransfer
? "dokploy-backups"
: "dokploy-bucket"
}
{...field}
/>
</FormControl>
@ -388,9 +416,78 @@ export const HandleDestinations = ({ destinationId }: Props) => {
</FormItem>
)}
/>
{!isNamedRemote && (
<FormField
control={form.control}
name="region"
render={({ field }) => (
<FormItem>
<FormLabel>{isFileTransfer ? "Port" : "Region"}</FormLabel>
<FormControl>
<Input
placeholder={
isFileTransfer
? currentProvider ===
RCLONE_DESTINATION_PROVIDERS.SFTP
? "22"
: implicitFtpTlsEnabled
? "990"
: "21"
: "us-east-1"
}
{...field}
/>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}
<FormField
control={form.control}
name="endpoint"
render={({ field }) => (
<FormItem>
<FormLabel>
{isNamedRemote
? "Rclone Remote Name"
: isFileTransfer
? "Host"
: "Endpoint"}
</FormLabel>
<FormControl>
<Input
placeholder={
isNamedRemote
? currentProvider ===
RCLONE_DESTINATION_PROVIDERS.ONEDRIVE
? "onedrive"
: "gdrive"
: isFileTransfer
? "storage.example.com"
: "https://us.bucket.aws/s3"
}
{...field}
/>
</FormControl>
{isNamedRemote && (
<p className="text-xs text-muted-foreground">
Configure this remote with rclone on the machine that runs
the backup, then enter only its remote name here (without
a colon).
</p>
)}
<FormMessage />
</FormItem>
)}
/>
<div className="flex flex-col gap-2">
<div className="flex items-center justify-between">
<FormLabel>Additional Flags (Optional)</FormLabel>
<FormLabel>
{isFileTransfer
? "Security / Additional Flags"
: "Additional Flags (Optional)"}
</FormLabel>
<Button
type="button"
variant="ghost"
@ -401,6 +498,18 @@ export const HandleDestinations = ({ destinationId }: Props) => {
Add Flag
</Button>
</div>
{currentProvider === RCLONE_DESTINATION_PROVIDERS.FTP && (
<p className="text-xs text-muted-foreground">
Required: --ftp-explicit-tls for explicit FTPS (port 21) or
--ftp-tls for implicit FTPS (port 990).
</p>
)}
{currentProvider === RCLONE_DESTINATION_PROVIDERS.SFTP && (
<p className="text-xs text-muted-foreground">
Required: --sftp-known-hosts-file=/path/to/known_hosts to
verify the server host key.
</p>
)}
{fields.map((field, index) => (
<FormField
key={field.id}
@ -410,10 +519,7 @@ export const HandleDestinations = ({ destinationId }: Props) => {
<FormItem>
<div className="flex items-center gap-2">
<FormControl>
<Input
placeholder="--s3-sign-accept-encoding=false"
{...field}
/>
<Input placeholder="--flag=value" {...field} />
</FormControl>
<Button
type="button"
@ -434,15 +540,15 @@ export const HandleDestinations = ({ destinationId }: Props) => {
<DialogFooter
className={cn(
isCloud ? "flex-col!" : "flex-row",
showServerSelector ? "flex-col!" : "flex-row",
"flex w-full justify-between! gap-4",
)}
>
{isCloud ? (
{showServerSelector ? (
<div className="flex flex-col gap-4 border p-2 rounded-lg">
<span className="text-sm text-muted-foreground">
Select a server to test the destination. If you don't have a
server choose the default one.
Select the server that will execute the backup so the
destination can be tested from the same environment.
</span>
<FormField
control={form.control}
@ -469,19 +575,22 @@ export const HandleDestinations = ({ destinationId }: Props) => {
{server.name}
</SelectItem>
))}
<SelectItem value={"none"}>None</SelectItem>
{!isCloud && (
<SelectItem value="none">
Dokploy Server (Local)
</SelectItem>
)}
</SelectGroup>
</SelectContent>
</Select>
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<Button
type="button"
variant={"secondary"}
variant="secondary"
isLoading={isPendingConnection}
onClick={async () => {
await handleTestConnection(form.getValues("serverId"));
@ -515,4 +624,4 @@ export const HandleDestinations = ({ destinationId }: Props) => {
</DialogContent>
</Dialog>
);
};
};

View File

@ -31,8 +31,9 @@ import {
import { findDestinationById } from "@dokploy/server/services/destination";
import { checkServicePermissionAndAccess } from "@dokploy/server/services/permission";
import { runComposeBackup } from "@dokploy/server/utils/backups/compose";
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
import {
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "@dokploy/server/utils/backups/utils";
import {
@ -79,6 +80,20 @@ interface RcloneFile {
};
}
const assertDestinationAccess = async (
destinationId: string,
organizationId: string,
) => {
const destination = await findDestinationById(destinationId);
if (destination.organizationId !== organizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You don't have access to this destination.",
});
}
return destination;
};
export const backupRouter = createTRPCRouter({
create: protectedProcedure
.input(apiCreateBackup)
@ -96,6 +111,10 @@ export const backupRouter = createTRPCRouter({
backup: ["create"],
});
}
await assertDestinationAccess(
input.destinationId,
ctx.session.activeOrganizationId,
);
if (IS_CLOUD) {
const dbType = (
@ -161,7 +180,7 @@ export const backupRouter = createTRPCRouter({
} catch (error) {
console.error(error);
throw new TRPCError({
code: "BAD_REQUEST",
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
message:
error instanceof Error
? error.message
@ -207,6 +226,10 @@ export const backupRouter = createTRPCRouter({
backup: ["update"],
});
}
await assertDestinationAccess(
input.destinationId,
ctx.session.activeOrganizationId,
);
await updateBackupById(input.backupId, input);
const backup = await findBackupById(input.backupId);
@ -242,7 +265,7 @@ export const backupRouter = createTRPCRouter({
const message =
error instanceof Error ? error.message : "Error updating this Backup";
throw new TRPCError({
code: "BAD_REQUEST",
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
message,
});
}
@ -312,7 +335,7 @@ export const backupRouter = createTRPCRouter({
} catch (error) {
const message =
error instanceof Error
? error.message
? redactRcloneCredentials(error.message)
: "Error running manual Postgres backup ";
throw new TRPCError({
code: "BAD_REQUEST",
@ -479,13 +502,10 @@ export const backupRouter = createTRPCRouter({
)
.query(async ({ input, ctx }) => {
try {
const destination = await findDestinationById(input.destinationId);
if (destination.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You don't have access to this destination.",
});
}
const destination = await assertDestinationAccess(
input.destinationId,
ctx.session.activeOrganizationId,
);
if (input.serverId) {
const targetServer = await findServerById(input.serverId);
if (
@ -497,8 +517,6 @@ export const backupRouter = createTRPCRouter({
});
}
}
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const lastSlashIndex = input.search.lastIndexOf("/");
const baseDir =
@ -509,8 +527,8 @@ export const backupRouter = createTRPCRouter({
lastSlashIndex !== -1
? input.search.slice(lastSlashIndex + 1)
: input.search;
const searchPath = baseDir ? `${bucketPath}/${baseDir}` : bucketPath;
const { flags: rcloneFlags, path: searchPath } =
await getRclonePathAndFlags(destination, baseDir);
const listCommand = `rclone lsjson ${rcloneFlags.join(" ")} ${quote([searchPath])} --no-mimetype --no-modtime 2>/dev/null`;
let stdout = "";
@ -532,8 +550,6 @@ export const backupRouter = createTRPCRouter({
throw new Error("Failed to parse backup files list");
}
// Limit to first 100 files
const results = baseDir
? files.map((file) => ({
...file,
@ -551,14 +567,15 @@ export const backupRouter = createTRPCRouter({
return results.slice(0, 100);
} catch (error) {
console.error("Error in listBackupFiles:", error);
const safeMessage =
error instanceof Error
? redactRcloneCredentials(error.message)
: "Error listing backup files";
console.error("Error in listBackupFiles:", safeMessage);
throw new TRPCError({
code: "BAD_REQUEST",
message:
error instanceof Error
? error.message
: "Error listing backup files",
cause: error,
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
message: safeMessage,
cause: new Error(safeMessage),
});
}
}),
@ -579,10 +596,13 @@ export const backupRouter = createTRPCRouter({
backup: ["restore"],
});
}
const destination = await findDestinationById(input.destinationId);
const destination = await assertDestinationAccess(
input.destinationId,
ctx.session.activeOrganizationId,
);
const queue: string[] = [];
let done = false;
const onLog = (log: string) => queue.push(log);
const onLog = (log: string) => queue.push(redactRcloneCredentials(log));
const runRestore = async () => {
if (input.backupType === "database") {
if (input.databaseType === "postgres") {

View File

@ -3,11 +3,14 @@ import {
execAsync,
execAsyncRemote,
findDestinationById,
findServerById,
getRclonePathAndFlags,
IS_CLOUD,
removeDestinationById,
updateDestinationById,
} from "@dokploy/server";
import { db } from "@dokploy/server/db";
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
import { TRPCError } from "@trpc/server";
import { desc, eq } from "drizzle-orm";
import { quote } from "shell-quote";
@ -47,37 +50,19 @@ export const destinationRouter = createTRPCRouter({
}),
testConnection: withPermission("destination", "create")
.input(apiCreateDestination)
.mutation(async ({ input }) => {
const {
secretAccessKey,
bucket,
region,
endpoint,
accessKey,
provider,
additionalFlags,
} = input;
.mutation(async ({ input, ctx }) => {
try {
const { flags, path } = await getRclonePathAndFlags(
input as Parameters<typeof getRclonePathAndFlags>[0],
);
const rcloneFlags = [
`--s3-access-key-id=${quote([accessKey])}`,
`--s3-secret-access-key=${quote([secretAccessKey])}`,
`--s3-region=${quote([region])}`,
`--s3-endpoint=${quote([endpoint])}`,
"--s3-no-check-bucket",
"--s3-force-path-style",
...flags,
"--retries 1",
"--low-level-retries 1",
"--timeout 10s",
"--contimeout 5s",
];
if (provider) {
rcloneFlags.unshift(`--s3-provider=${quote([provider])}`);
}
if (additionalFlags?.length) {
rcloneFlags.push(...additionalFlags);
}
const rcloneDestination = `:s3:${bucket}`;
const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} ${quote([rcloneDestination])}`;
const rcloneCommand = `rclone lsd ${rcloneFlags.join(" ")} ${quote([path])}`;
if (IS_CLOUD && !input.serverId) {
throw new TRPCError({
@ -86,19 +71,27 @@ export const destinationRouter = createTRPCRouter({
});
}
if (IS_CLOUD) {
await execAsyncRemote(input.serverId || "", rcloneCommand);
if (input.serverId) {
const server = await findServerById(input.serverId);
if (server.organizationId !== ctx.session.activeOrganizationId) {
throw new TRPCError({
code: "UNAUTHORIZED",
message: "You are not allowed to use this server",
});
}
await execAsyncRemote(input.serverId, rcloneCommand);
} else {
await execAsync(rcloneCommand);
}
} catch (error) {
const safeMessage =
error instanceof Error
? redactRcloneCredentials(error.message)
: "Error connecting to destination";
throw new TRPCError({
code: "BAD_REQUEST",
message:
error instanceof Error
? error?.message
: "Error connecting to bucket",
cause: error,
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
message: safeMessage,
cause: new Error(safeMessage),
});
}
}),
@ -174,8 +167,8 @@ export const destinationRouter = createTRPCRouter({
code: "BAD_REQUEST",
message:
error instanceof Error
? error?.message
: "Error connecting to bucket",
? redactRcloneCredentials(error.message)
: "Error updating destination",
cause: error,
});
}

View File

@ -6,6 +6,7 @@ import { z } from "zod";
import {
ADDITIONAL_FLAG_ERROR,
ADDITIONAL_FLAG_REGEX,
getDestinationValidationIssues,
} from "../validations/destination";
import { organization } from "./account";
import { backups } from "./backups";
@ -54,6 +55,25 @@ const createSchema = createInsertSchema(destinations, {
.default([]),
});
const validateDestination = (
data: {
provider?: string | null;
accessKey?: string;
region?: string;
endpoint?: string;
additionalFlags?: string[] | null;
},
ctx: z.RefinementCtx,
) => {
for (const issue of getDestinationValidationIssues(data)) {
ctx.addIssue({
code: "custom",
path: [issue.field],
message: issue.message,
});
}
};
export const apiCreateDestination = createSchema
.pick({
name: true,
@ -68,7 +88,8 @@ export const apiCreateDestination = createSchema
.required()
.extend({
serverId: z.string().optional(),
});
})
.superRefine(validateDestination);
export const apiFindOneDestination = z.object({
destinationId: z.string().min(1),
@ -95,4 +116,5 @@ export const apiUpdateDestination = createSchema
.required()
.extend({
serverId: z.string().optional(),
});
})
.superRefine(validateDestination);

View File

@ -1,3 +1,183 @@
export const ADDITIONAL_FLAG_REGEX = /^--[a-zA-Z0-9-]+(=[a-zA-Z0-9._:/@-]+)?$/;
export const ADDITIONAL_FLAG_ERROR =
"Invalid flag format. Must start with -- (e.g. --s3-sign-accept-encoding=false)";
export const RCLONE_DESTINATION_PROVIDERS = {
GOOGLE_DRIVE: "GoogleDrive",
ONEDRIVE: "OneDrive",
FTP: "FTP",
SFTP: "SFTP",
REMOTE: "RcloneRemote",
} as const;
export type RcloneDestinationProvider =
(typeof RCLONE_DESTINATION_PROVIDERS)[keyof typeof RCLONE_DESTINATION_PROVIDERS];
const RCLONE_DESTINATION_PROVIDER_VALUES = new Set<string>(
Object.values(RCLONE_DESTINATION_PROVIDERS),
);
export const isRcloneDestinationProvider = (
provider: string | null | undefined,
): provider is RcloneDestinationProvider =>
!!provider && RCLONE_DESTINATION_PROVIDER_VALUES.has(provider);
export const isNamedRcloneDestinationProvider = (
provider: string | null | undefined,
) =>
provider === RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE ||
provider === RCLONE_DESTINATION_PROVIDERS.ONEDRIVE ||
provider === RCLONE_DESTINATION_PROVIDERS.REMOTE;
export const RCLONE_REMOTE_NAME_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$/;
export const RCLONE_REMOTE_NAME_ERROR =
"Invalid rclone remote name. Use only letters, numbers, dots, underscores, and dashes";
export const FTP_TLS_REQUIRED_ERROR =
"FTP destinations must use TLS. Add --ftp-explicit-tls for explicit FTPS (port 21) or --ftp-tls for implicit FTPS (port 990).";
export const FTP_TLS_CONFLICT_ERROR =
"Choose either implicit FTPS or explicit FTPS, not both.";
export const FTP_CERTIFICATE_VERIFICATION_REQUIRED_ERROR =
"FTP TLS certificate verification cannot be disabled.";
export const SFTP_HOST_KEY_REQUIRED_ERROR =
"SFTP destinations must verify the server host key. Add --sftp-known-hosts-file=/path/to/known_hosts.";
const parseBooleanFlagValue = (
flag: string,
flagName: string,
): boolean | undefined => {
if (flag === flagName) return true;
const prefix = `${flagName}=`;
if (!flag.startsWith(prefix)) return undefined;
const value = flag.slice(prefix.length).toLowerCase();
if (["1", "t", "true"].includes(value)) return true;
if (["0", "f", "false"].includes(value)) return false;
return undefined;
};
const getBooleanFlagValues = (flags: readonly string[], flagName: string) =>
flags
.filter((flag) => flag === flagName || flag.startsWith(`${flagName}=`))
.map((flag) => parseBooleanFlagValue(flag, flagName));
const isBooleanFlagEnabled = (
flags: readonly string[],
flagName: string,
): boolean => {
const values = getBooleanFlagValues(flags, flagName);
return values.length > 0 && values.every((value) => value === true);
};
export const getFtpTlsState = (flags: readonly string[] | null | undefined) => {
const values = flags ?? [];
return {
implicitTlsEnabled: isBooleanFlagEnabled(values, "--ftp-tls"),
explicitTlsEnabled: isBooleanFlagEnabled(values, "--ftp-explicit-tls"),
};
};
export const hasDisabledFtpCertificateVerification = (
flags: readonly string[] | null | undefined,
): boolean => {
const values = flags ?? [];
return ["--ftp-no-check-certificate", "--no-check-certificate"].some(
(flagName) => {
const matchingValues = getBooleanFlagValues(values, flagName);
return (
matchingValues.length > 0 &&
matchingValues.some((value) => value !== false)
);
},
);
};
export const hasSftpHostKeyVerification = (
flags: readonly string[] | null | undefined,
): boolean => {
const prefix = "--sftp-known-hosts-file=";
const knownHostsFlags = (flags ?? []).filter((flag) =>
flag.startsWith(prefix),
);
if (knownHostsFlags.length !== 1) return false;
const value = knownHostsFlags[0]?.slice(prefix.length).trim() ?? "";
return value.length > 0 && value !== "none";
};
type DestinationValidationField =
| "endpoint"
| "accessKey"
| "region"
| "additionalFlags";
export interface DestinationValidationIssue {
field: DestinationValidationField;
message: string;
}
export interface DestinationValidationInput {
provider?: string | null;
accessKey?: string;
region?: string;
endpoint?: string;
additionalFlags?: readonly string[] | null;
}
export const getDestinationValidationIssues = (
data: DestinationValidationInput,
): DestinationValidationIssue[] => {
const issues: DestinationValidationIssue[] = [];
const provider = data.provider;
const flags = data.additionalFlags ?? [];
if (isNamedRcloneDestinationProvider(provider)) {
if (!RCLONE_REMOTE_NAME_REGEX.test(data.endpoint?.trim() || "")) {
issues.push({ field: "endpoint", message: RCLONE_REMOTE_NAME_ERROR });
}
return issues;
}
if (
provider !== RCLONE_DESTINATION_PROVIDERS.FTP &&
provider !== RCLONE_DESTINATION_PROVIDERS.SFTP
) {
return issues;
}
if (!data.endpoint?.trim()) {
issues.push({ field: "endpoint", message: "Host is required" });
}
if (!data.accessKey?.trim()) {
issues.push({ field: "accessKey", message: "Username is required" });
}
if (data.region?.trim()) {
const port = Number(data.region);
if (!Number.isInteger(port) || port < 1 || port > 65535) {
issues.push({
field: "region",
message: "Port must be an integer between 1 and 65535",
});
}
}
if (provider === RCLONE_DESTINATION_PROVIDERS.FTP) {
const { implicitTlsEnabled, explicitTlsEnabled } = getFtpTlsState(flags);
if (!implicitTlsEnabled && !explicitTlsEnabled) {
issues.push({ field: "additionalFlags", message: FTP_TLS_REQUIRED_ERROR });
}
if (implicitTlsEnabled && explicitTlsEnabled) {
issues.push({ field: "additionalFlags", message: FTP_TLS_CONFLICT_ERROR });
}
if (hasDisabledFtpCertificateVerification(flags)) {
issues.push({
field: "additionalFlags",
message: FTP_CERTIFICATE_VERIFICATION_REQUIRED_ERROR,
});
}
} else if (!hasSftpHostKeyVerification(flags)) {
issues.push({ field: "additionalFlags", message: SFTP_HOST_KEY_REQUIRED_ERROR });
}
return issues;
};

View File

@ -9,10 +9,11 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getSafeRcloneErrorMessage } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";
@ -35,8 +36,8 @@ export const runComposeBackup = async (
});
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const backupCommand = getBackupCommand(
backup,
rcloneFlags,
@ -62,20 +63,20 @@ export const runComposeBackup = async (
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
console.log(error);
const safeErrorMessage = getSafeRcloneErrorMessage(error);
console.error("Backup error:", safeErrorMessage);
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: getDatabaseType(databaseType),
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: safeErrorMessage,
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(safeErrorMessage);
}
};

View File

@ -6,13 +6,18 @@ import { getAllServers } from "@dokploy/server/services/server";
import { getWebServerSettings } from "@dokploy/server/services/web-server-settings";
import { eq } from "drizzle-orm";
import { scheduleJob } from "node-schedule";
import { quote } from "shell-quote";
import { db } from "../../db/index";
import { startLogCleanup } from "../access-log/handler";
import { cleanupAll } from "../docker/utils";
import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { redactRcloneCredentials } from "./redact";
import { getS3Credentials, normalizeS3Path, scheduleBackup } from "./utils";
import {
getRclonePathAndFlags,
normalizeS3Path,
scheduleBackup,
} from "./utils";
export const initCronJobs = async () => {
console.log("Setting up cron jobs....");
@ -134,17 +139,19 @@ export const keepLatestNBackups = async (
try {
const destination = await findDestinationById(backup.destinationId);
const rcloneFlags = getS3Credentials(destination);
const appName = getServiceAppName(backup);
const backupFilesPath = `:s3:${destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`;
const { flags: rcloneFlags, path: backupFilesPath } =
await getRclonePathAndFlags(
destination,
`${appName}/${normalizeS3Path(backup.prefix)}`,
);
// --include "*.bson.gz" or "*.sql.gz" or "*.zip" ensures nothing else other than the dokploy backup files are touched by rclone
const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${backupFilesPath}`;
const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${quote([backupFilesPath])}`;
// when we pipe the above command with this one, we only get the list of files we want to delete
const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`;
// this command deletes the files
// to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{}
const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`;
const rcloneDelete = `rclone deletefile ${rcloneFlags.join(" ")} ${quote([`${backupFilesPath}/{}`])}`;
const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`;

View File

@ -9,10 +9,11 @@ import type { Libsql } from "@dokploy/server/services/libsql";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getSafeRcloneErrorMessage } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";
@ -34,8 +35,8 @@ export const runLibsqlBackup = async (
const backupFileName = `${getBackupTimestamp()}.sql.gz`;
const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`;
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const backupCommand = getBackupCommand(
backup,
rcloneFlags,
@ -61,19 +62,19 @@ export const runLibsqlBackup = async (
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
const safeErrorMessage = getSafeRcloneErrorMessage(error);
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: "libsql",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: safeErrorMessage,
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(safeErrorMessage);
}
};

View File

@ -9,10 +9,11 @@ import type { Mariadb } from "@dokploy/server/services/mariadb";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getSafeRcloneErrorMessage } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";
@ -33,8 +34,8 @@ export const runMariadbBackup = async (
description: "MariaDB Backup",
});
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const backupCommand = getBackupCommand(
backup,
rcloneFlags,
@ -59,18 +60,18 @@ export const runMariadbBackup = async (
});
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
console.log(error);
const safeErrorMessage = getSafeRcloneErrorMessage(error);
console.error("Backup error:", safeErrorMessage);
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: "mariadb",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: safeErrorMessage,
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(safeErrorMessage);
}
};

View File

@ -9,10 +9,11 @@ import type { Mongo } from "@dokploy/server/services/mongo";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getSafeRcloneErrorMessage } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";
@ -30,8 +31,8 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
description: "MongoDB Backup",
});
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const backupCommand = getBackupCommand(
backup,
rcloneFlags,
@ -57,18 +58,18 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
});
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
console.log(error);
const safeErrorMessage = getSafeRcloneErrorMessage(error);
console.error("Backup error:", safeErrorMessage);
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: "mongodb",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: safeErrorMessage,
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(safeErrorMessage);
}
};

View File

@ -9,10 +9,11 @@ import type { MySql } from "@dokploy/server/services/mysql";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getSafeRcloneErrorMessage } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";
@ -31,8 +32,8 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
});
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const backupCommand = getBackupCommand(
backup,
rcloneFlags,
@ -57,18 +58,18 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
});
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
console.log(error);
const safeErrorMessage = getSafeRcloneErrorMessage(error);
console.error("Backup error:", safeErrorMessage);
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: "mysql",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: safeErrorMessage,
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(safeErrorMessage);
}
};

View File

@ -9,10 +9,11 @@ import type { Postgres } from "@dokploy/server/services/postgres";
import { findProjectById } from "@dokploy/server/services/project";
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getSafeRcloneErrorMessage } from "./redact";
import {
getBackupCommand,
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";
@ -34,8 +35,8 @@ export const runPostgresBackup = async (
const backupFileName = `${getBackupTimestamp()}.sql.gz`;
const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`;
try {
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, bucketDestination);
const backupCommand = getBackupCommand(
backup,
rcloneFlags,
@ -61,20 +62,20 @@ export const runPostgresBackup = async (
await updateDeploymentStatus(deployment.deploymentId, "done");
} catch (error) {
const safeErrorMessage = getSafeRcloneErrorMessage(error);
await sendDatabaseBackupNotifications({
applicationName: name,
projectName: project.name,
databaseType: "postgres",
type: "error",
// @ts-ignore
errorMessage: error?.message || "Error message not provided",
errorMessage: safeErrorMessage,
organizationId: project.organizationId,
databaseName: backup.database,
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(safeErrorMessage);
} finally {
}
};

View File

@ -1,12 +1,16 @@
/**
* Redacts S3 credentials from rclone command strings.
*
* Used to prevent credential leakage in structured logs and error output.
* Matches the flag format produced by `getS3Credentials()`:
* --s3-access-key-id="VALUE" and --s3-secret-access-key="VALUE"
* Redacts credentials from rclone command strings before they reach logs or
* user-facing error output. Handles both the existing S3 flags and the
* provider-specific FTP/SFTP credential flags used by backup destinations.
*/
export const redactRcloneCredentials = (command: string): string => {
return command
.replace(/(--s3-access-key-id=)"[^"]*"/g, '$1"[REDACTED]"')
.replace(/(--s3-secret-access-key=)"[^"]*"/g, '$1"[REDACTED]"');
return command.replace(
/(--(?:s3-access-key-id|s3-secret-access-key|ftp-pass|sftp-pass|sftp-key-file-pass)=)(?:(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\\[^\r\n]|[^\s"'\\])+)/g,
'$1"[REDACTED]"',
);
};
export const getSafeRcloneErrorMessage = (error: unknown): string =>
redactRcloneCredentials(
error instanceof Error ? error.message : String(error),
);

View File

@ -1,8 +1,18 @@
import {
ADDITIONAL_FLAG_ERROR,
ADDITIONAL_FLAG_REGEX,
getDestinationValidationIssues,
getFtpTlsState,
isNamedRcloneDestinationProvider,
isRcloneDestinationProvider,
RCLONE_DESTINATION_PROVIDERS,
} from "@dokploy/server/db/validations/destination";
import { logger } from "@dokploy/server/lib/logger";
import type { BackupSchedule } from "@dokploy/server/services/backup";
import type { Destination } from "@dokploy/server/services/destination";
import { scheduledJobs, scheduleJob } from "node-schedule";
import { quote } from "shell-quote";
import { execFileAsync } from "../process/execAsync";
import { keepLatestNBackups } from ".";
import { runComposeBackup } from "./compose";
import { runLibsqlBackup } from "./libsql";
@ -68,6 +78,16 @@ export const normalizeS3Path = (prefix: string) => {
return normalizedPrefix ? `${normalizedPrefix}/` : "";
};
const getValidatedAdditionalFlags = (destination: Destination): string[] => {
const flags = destination.additionalFlags ?? [];
for (const flag of flags) {
if (!ADDITIONAL_FLAG_REGEX.test(flag)) {
throw new Error(ADDITIONAL_FLAG_ERROR);
}
}
return flags;
};
export const getS3Credentials = (destination: Destination) => {
const { accessKey, secretAccessKey, region, endpoint, provider } =
destination;
@ -84,13 +104,102 @@ export const getS3Credentials = (destination: Destination) => {
rcloneFlags.unshift(`--s3-provider=${quote([provider])}`);
}
if (destination.additionalFlags?.length) {
rcloneFlags.push(...destination.additionalFlags);
}
rcloneFlags.push(...getValidatedAdditionalFlags(destination));
return rcloneFlags;
};
const trimRclonePath = (value: string) =>
value.trim().replace(/^\/+|\/+$/g, "");
const joinRclonePath = (...parts: string[]) =>
parts.map(trimRclonePath).filter(Boolean).join("/");
export const assertSafeRclonePath = (value: string) => {
const normalizedSeparators = value.replace(/\\/g, "/");
if (/[\0\r\n]/.test(normalizedSeparators)) {
throw new Error("Invalid rclone path");
}
const segments = normalizedSeparators.split("/");
if (segments.some((segment) => segment === "." || segment === "..")) {
throw new Error("Invalid rclone path");
}
};
const obscureRclonePassword = async (password: string) => {
if (!password) return "";
const { stdout } = await execFileAsync("rclone", ["obscure", "-"], {
input: password,
});
return stdout.trim();
};
export const getRclonePathAndFlags = async (
destination: Destination,
path = "",
): Promise<{ flags: string[]; path: string }> => {
assertSafeRclonePath(path);
const provider = destination.provider;
const additionalFlags = getValidatedAdditionalFlags(destination);
if (isRcloneDestinationProvider(provider)) {
const [issue] = getDestinationValidationIssues(destination);
if (issue) throw new Error(issue.message);
}
if (isNamedRcloneDestinationProvider(provider)) {
const remotePath = joinRclonePath(destination.bucket, path);
return {
flags: additionalFlags,
path: `${destination.endpoint.trim()}:${remotePath}`,
};
}
if (
provider === RCLONE_DESTINATION_PROVIDERS.FTP ||
provider === RCLONE_DESTINATION_PROVIDERS.SFTP
) {
const backend =
provider === RCLONE_DESTINATION_PROVIDERS.FTP ? "ftp" : "sftp";
const defaultPort =
provider === RCLONE_DESTINATION_PROVIDERS.FTP
? getFtpTlsState(additionalFlags).implicitTlsEnabled
? "990"
: "21"
: "22";
const port = destination.region.trim() || defaultPort;
const flags = [
`--${backend}-host=${quote([destination.endpoint.trim()])}`,
`--${backend}-user=${quote([destination.accessKey])}`,
`--${backend}-port=${quote([port])}`,
];
if (destination.secretAccessKey) {
const obscuredPassword = await obscureRclonePassword(
destination.secretAccessKey,
);
flags.push(`--${backend}-pass=${quote([obscuredPassword])}`);
}
flags.push(...additionalFlags);
if (provider === RCLONE_DESTINATION_PROVIDERS.FTP) {
// CLI options override RCLONE_* environment defaults. Keep TLS
// certificate verification enabled on the execution host.
flags.push(
"--ftp-no-check-certificate=false",
"--no-check-certificate=false",
);
}
return {
flags,
path: `:${backend}:${joinRclonePath(destination.bucket, path)}`,
};
}
return {
flags: getS3Credentials(destination),
path: `:s3:${joinRclonePath(destination.bucket, path)}`,
};
};
// User-controlled values (database name, user, password) are passed to the
// container as environment variables via `docker exec -e VAR=<escaped>` and
// referenced as "$VAR" inside the inner shell, so they never appear in the
@ -265,8 +374,9 @@ export const getBackupCommand = (
) => {
const containerSearch = getContainerSearchCommand(backup);
const backupCommand = generateBackupCommand(backup);
const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
const rcloneTarget = quote([rcloneDestination]);
const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} ${rcloneTarget}`;
const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} ${rcloneTarget}`;
logger.info(
{
@ -290,7 +400,7 @@ export const getBackupCommand = (
fi;
echo "[$(date)] Container Up: $CONTAINER_ID" >> ${logPath};
echo "[$(date)] Starting backup and upload to S3..." >> ${logPath};
echo "[$(date)] Starting backup and upload to destination..." >> ${logPath};
UPLOAD_OUTPUT=$({ ${backupCommand} | ${rcloneCommand}; } 2>&1 >/dev/null) || {
echo "[$(date)] ❌ Error: Backup failed" >> ${logPath};
@ -299,7 +409,7 @@ export const getBackupCommand = (
exit 1;
};
echo "[$(date)] ✅ Backup uploaded to S3 successfully" >> ${logPath};
echo "[$(date)] ✅ Backup uploaded successfully" >> ${logPath};
echo "Backup done ✅" >> ${logPath};
`;
};
};

View File

@ -13,10 +13,15 @@ import {
updateDeploymentStatus,
} from "@dokploy/server/services/deployment";
import { findDestinationById } from "@dokploy/server/services/destination";
import { quote } from "shell-quote";
import { sendDokployBackupNotifications } from "../notifications/dokploy-backup";
import { execAsync } from "../process/execAsync";
import { redactRcloneCredentials } from "./redact";
import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
import { getSafeRcloneErrorMessage, redactRcloneCredentials } from "./redact";
import {
getBackupTimestamp,
getRclonePathAndFlags,
normalizeS3Path,
} from "./utils";
function formatBytes(bytes?: number) {
if (bytes === undefined) return "Unknown size";
@ -41,12 +46,13 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
let computedBackupSize: number | undefined;
try {
const destination = await findDestinationById(backup.destinationId);
const rcloneFlags = getS3Credentials(destination);
const timestamp = getBackupTimestamp();
const { BASE_PATH } = paths();
const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-"));
const backupFileName = `webserver-backup-${timestamp}.zip`;
const s3Path = `:s3:${destination.bucket}/${backup.appName}/${normalizeS3Path(backup.prefix)}${backupFileName}`;
const destinationPath = `${backup.appName}/${normalizeS3Path(backup.prefix)}${backupFileName}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, destinationPath);
try {
await execAsync(`mkdir -p ${tempDir}/filesystem`);
@ -114,10 +120,10 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
// If stat fails, keep undefined
}
const uploadCommand = `rclone copyto ${rcloneFlags.join(" ")} "${zipPath}" "${s3Path}"`;
writeStream.write("Running command to upload backup to S3\n");
const uploadCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([zipPath])} ${quote([rcloneDestination])}`;
writeStream.write("Running command to upload backup to destination\n");
await execAsync(uploadCommand);
writeStream.write("Uploaded backup to S3 ✅\n");
writeStream.write("Uploaded backup to destination ✅\n");
writeStream.end();
await sendDokployBackupNotifications({
type: "success",
@ -136,9 +142,7 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
}
}
} catch (error) {
const safeErrorMessage = redactRcloneCredentials(
error instanceof Error ? error.message : String(error),
);
const safeErrorMessage = getSafeRcloneErrorMessage(error);
console.error("Backup error:", redactRcloneCredentials(String(error)));
writeStream.write("Backup error❌\n");
writeStream.write(`${safeErrorMessage}\n`);
@ -149,6 +153,6 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
backupSize: formatBytes(computedBackupSize),
});
await updateDeploymentStatus(deployment.deploymentId, "error");
throw error;
throw new Error(safeErrorMessage);
}
};

View File

@ -3,7 +3,7 @@ import type { Compose } from "@dokploy/server/services/compose";
import type { Destination } from "@dokploy/server/services/destination";
import { quote } from "shell-quote";
import type { z } from "zod";
import { getS3Credentials } from "../backups/utils";
import { getRclonePathAndFlags } from "../backups/utils";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getRestoreCommand } from "./utils";
@ -24,9 +24,8 @@ export const restoreComposeBackup = async (
}
const { serverId, appName, composeType } = compose;
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
const { flags: rcloneFlags, path: backupPath } =
await getRclonePathAndFlags(destination, backupInput.backupFile);
let rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
if (backupInput.metadata?.mongo) {

View File

@ -3,7 +3,10 @@ import type { Destination } from "@dokploy/server/services/destination";
import type { Libsql } from "@dokploy/server/services/libsql";
import { quote } from "shell-quote";
import type { z } from "zod";
import { getS3Credentials, getServiceContainerCommand } from "../backups/utils";
import {
getRclonePathAndFlags,
getServiceContainerCommand,
} from "../backups/utils";
import { execAsync, execAsyncRemote } from "../process/execAsync";
export const restoreLibsqlBackup = async (
@ -15,11 +18,8 @@ export const restoreLibsqlBackup = async (
try {
const { appName, serverId } = libsql;
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
const { flags: rcloneFlags, path: backupPath } =
await getRclonePathAndFlags(destination, backupInput.backupFile);
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])}`;
const containerSearch = getServiceContainerCommand(appName);

View File

@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
import type { Mariadb } from "@dokploy/server/services/mariadb";
import { quote } from "shell-quote";
import type { z } from "zod";
import { getS3Credentials } from "../backups/utils";
import { getRclonePathAndFlags } from "../backups/utils";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getRestoreCommand } from "./utils";
@ -16,10 +16,8 @@ export const restoreMariadbBackup = async (
try {
const { appName, serverId, databaseUser, databasePassword } = mariadb;
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
const { flags: rcloneFlags, path: backupPath } =
await getRclonePathAndFlags(destination, backupInput.backupFile);
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
const command = getRestoreCommand({

View File

@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
import type { Mongo } from "@dokploy/server/services/mongo";
import { quote } from "shell-quote";
import type { z } from "zod";
import { getS3Credentials } from "../backups/utils";
import { getRclonePathAndFlags } from "../backups/utils";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getRestoreCommand } from "./utils";
@ -16,9 +16,8 @@ export const restoreMongoBackup = async (
try {
const { appName, databasePassword, databaseUser, serverId } = mongo;
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
const { flags: rcloneFlags, path: backupPath } =
await getRclonePathAndFlags(destination, backupInput.backupFile);
const rcloneCommand = `rclone copy ${rcloneFlags.join(" ")} ${quote([backupPath])}`;
const command = getRestoreCommand({

View File

@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
import type { MySql } from "@dokploy/server/services/mysql";
import { quote } from "shell-quote";
import type { z } from "zod";
import { getS3Credentials } from "../backups/utils";
import { getRclonePathAndFlags } from "../backups/utils";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getRestoreCommand } from "./utils";
@ -16,10 +16,8 @@ export const restoreMySqlBackup = async (
try {
const { appName, databaseRootPassword, serverId } = mysql;
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
const { flags: rcloneFlags, path: backupPath } =
await getRclonePathAndFlags(destination, backupInput.backupFile);
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
const command = getRestoreCommand({

View File

@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
import type { Postgres } from "@dokploy/server/services/postgres";
import { quote } from "shell-quote";
import type { z } from "zod";
import { getS3Credentials } from "../backups/utils";
import { getRclonePathAndFlags } from "../backups/utils";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import { getRestoreCommand } from "./utils";
@ -16,11 +16,8 @@ export const restorePostgresBackup = async (
try {
const { appName, databaseUser, serverId } = postgres;
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
const { flags: rcloneFlags, path: backupPath } =
await getRclonePathAndFlags(destination, backupInput.backupFile);
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
const command = getRestoreCommand({

View File

@ -4,7 +4,8 @@ import { join } from "node:path";
import { IS_CLOUD, paths } from "@dokploy/server/constants";
import type { Destination } from "@dokploy/server/services/destination";
import { quote } from "shell-quote";
import { getS3Credentials } from "../backups/utils";
import { getSafeRcloneErrorMessage } from "../backups/redact";
import { getRclonePathAndFlags } from "../backups/utils";
import { execAsync } from "../process/execAsync";
export const restoreWebServerBackup = async (
@ -16,9 +17,8 @@ export const restoreWebServerBackup = async (
return;
}
try {
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupFile}`;
const { flags: rcloneFlags, path: backupPath } =
await getRclonePathAndFlags(destination, backupFile);
const { BASE_PATH } = paths();
// Create a temporary directory outside of BASE_PATH
@ -31,17 +31,19 @@ export const restoreWebServerBackup = async (
// Create temp directory
emit("Creating temporary directory...");
await execAsync(`mkdir -p ${tempDir}`);
await execAsync(`mkdir -p ${quote([tempDir])}`);
// Download backup from S3
emit("Downloading backup from S3...");
// Download backup from configured destination
emit("Downloading backup from destination...");
await execAsync(
`rclone copyto ${rcloneFlags.join(" ")} ${quote([backupPath])} ${quote([`${tempDir}/${backupFile}`])}`,
);
// List files before extraction
emit("Listing files before extraction...");
const { stdout: beforeFiles } = await execAsync(`ls -la ${tempDir}`);
const { stdout: beforeFiles } = await execAsync(
`ls -la ${quote([tempDir])}`,
);
emit(`Files before extraction: ${beforeFiles}`);
// Extract backup
@ -71,16 +73,18 @@ export const restoreWebServerBackup = async (
// Check if database.sql.gz exists and decompress it
const { stdout: hasGzFile } = await execAsync(
`ls ${tempDir}/database.sql.gz || true`,
`ls ${quote([`${tempDir}/database.sql.gz`])} || true`,
);
if (hasGzFile.includes("database.sql.gz")) {
emit("Found compressed database file, decompressing...");
await execAsync(`cd ${tempDir} && gunzip database.sql.gz`);
await execAsync(
`cd ${quote([tempDir])} && gunzip ${quote(["database.sql.gz"])}`,
);
}
// Verify database file exists
const { stdout: hasSqlFile } = await execAsync(
`ls ${tempDir}/database.sql || true`,
`ls ${quote([`${tempDir}/database.sql`])} || true`,
);
if (!hasSqlFile.includes("database.sql")) {
throw new Error("Database file not found after extraction");
@ -115,7 +119,7 @@ export const restoreWebServerBackup = async (
// Copy the backup file into the container
emit("Copying backup file into container...");
await execAsync(
`docker cp ${tempDir}/database.sql ${postgresContainerId}:/tmp/database.sql`,
`docker cp ${quote([`${tempDir}/database.sql`])} ${postgresContainerId}:/tmp/database.sql`,
);
// Verify file in container
@ -140,17 +144,12 @@ export const restoreWebServerBackup = async (
} finally {
// Cleanup
emit("Cleaning up temporary files...");
await execAsync(`rm -rf ${tempDir}`);
await execAsync(`rm -rf ${quote([tempDir])}`);
}
} catch (error) {
console.error(error);
emit(
`Error: ${
error instanceof Error
? error.message
: "Error restoring web server backup"
}`,
);
throw error;
const safeErrorMessage = getSafeRcloneErrorMessage(error);
console.error("Restore error:", safeErrorMessage);
emit(`Error: ${safeErrorMessage}`);
throw new Error(safeErrorMessage);
}
};

View File

@ -3,11 +3,16 @@ import { paths } from "@dokploy/server/constants";
import { findComposeById } from "@dokploy/server/services/compose";
import { findDestinationById } from "@dokploy/server/services/destination";
import type { findVolumeBackupById } from "@dokploy/server/services/volume-backups";
import { quote } from "shell-quote";
import {
getBackupTimestamp,
getS3Credentials,
getRclonePathAndFlags,
normalizeS3Path,
} from "../backups/utils";
import {
normalizeDockerVolumeName,
normalizeVolumeBackupFilePath,
} from "./restore";
interface RestartSafeBackupCommandOptions {
stopCommand: string;
@ -73,36 +78,40 @@ export const backupVolume = async (
const serverId =
volumeBackup.application?.serverId || volumeBackup.compose?.serverId;
const { VOLUME_BACKUPS_PATH, VOLUME_BACKUP_LOCK_PATH } = paths(!!serverId);
const s3AppName = getVolumeServiceAppName(volumeBackup);
const backupFileName = `${volumeName}-${getBackupTimestamp()}.tar`;
const bucketDestination = `${s3AppName}/${normalizeS3Path(prefix || "")}${backupFileName}`;
const rcloneFlags = getS3Credentials(destination);
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
const appName = getVolumeServiceAppName(volumeBackup);
const safeVolumeName = normalizeDockerVolumeName(volumeName);
const backupFileName = normalizeVolumeBackupFilePath(
`${safeVolumeName}-${getBackupTimestamp()}.tar`,
);
const destinationPath = `${appName}/${normalizeS3Path(prefix || "")}${backupFileName}`;
const { flags: rcloneFlags, path: rcloneDestination } =
await getRclonePathAndFlags(destination, destinationPath);
const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeBackup.appName);
const localBackupPath = `${volumeBackupPath}/${backupFileName}`;
const rcloneCommand = `rclone copyto ${rcloneFlags.join(" ")} "${volumeBackupPath}/${backupFileName}" "${rcloneDestination}"`;
const rcloneCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([localBackupPath])} ${quote([rcloneDestination])}`;
const backupCommand = `
set -e
echo "Volume name: ${volumeName}"
echo "Backup file name: ${backupFileName}"
echo "Volume name:" ${quote([safeVolumeName])}
echo "Backup file name:" ${quote([backupFileName])}
echo "Turning off volume backup: ${turnOff ? "Yes" : "No"}"
echo "Starting volume backup"
echo "Dir: ${volumeBackupPath}"
docker run --rm \
-v ${volumeName}:/volume_data \
-v ${volumeBackupPath}:/backup \
-v ${quote([safeVolumeName])}:/volume_data \
-v ${quote([volumeBackupPath])}:/backup \
ubuntu \
bash -c "cd /volume_data && tar cvf /backup/${backupFileName} ."
bash -c 'cd /volume_data && tar cvf "/backup/$1" .' -- ${quote([backupFileName])}
echo "Volume backup done ✅"
`;
const uploadCommand = `
echo "Starting upload to S3..."
echo "Starting upload to destination..."
${rcloneCommand}
echo "Upload to S3 done ✅"
echo "Upload to destination done ✅"
echo "Cleaning up local backup file..."
rm "${volumeBackupPath}/${backupFileName}"
rm ${quote([localBackupPath])}
echo "Local backup file cleaned up ✅"
`;
@ -123,7 +132,7 @@ export const backupVolume = async (
const lockWrapper = (body: string) => `
set -e
LOCK_PATH="${lockPath}"
LOCK_PATH=${quote([lockPath])}
echo "Waiting for volume backup lock: $LOCK_PATH"

View File

@ -4,10 +4,39 @@ import {
findApplicationById,
findComposeById,
findDestinationById,
getS3Credentials,
getRclonePathAndFlags,
paths,
} from "../..";
const UNSAFE_BACKUP_PATH_CHARS = /[\0\r\n;&|`$<>]/;
export const normalizeDockerVolumeName = (value: string) => {
const normalized = value.trim();
if (!normalized || !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(normalized)) {
throw new Error("Invalid docker volume name");
}
return normalized;
};
export const normalizeVolumeBackupFilePath = (value: string) => {
const normalized = value.trim().replace(/\\/g, "/");
if (
!normalized ||
normalized.startsWith("/") ||
normalized.endsWith("/") ||
UNSAFE_BACKUP_PATH_CHARS.test(normalized)
) {
throw new Error("Invalid volume backup file path");
}
const segments = normalized.split("/");
if (
segments.some((segment) => !segment || segment === "." || segment === "..")
) {
throw new Error("Invalid volume backup file path");
}
return segments.join("/");
};
export const restoreVolume = async (
id: string,
destinationId: string,
@ -18,37 +47,45 @@ export const restoreVolume = async (
) => {
const destination = await findDestinationById(destinationId);
const { VOLUME_BACKUPS_PATH } = paths(!!serverId);
const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeName);
const rcloneFlags = getS3Credentials(destination);
const bucketPath = `:s3:${destination.bucket}`;
const backupPath = `${bucketPath}/${backupFileName}`;
const safeVolumeName = normalizeDockerVolumeName(volumeName);
const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, safeVolumeName);
const safeBackupFileName = normalizeVolumeBackupFilePath(backupFileName);
const { flags: rcloneFlags, path: backupPath } = await getRclonePathAndFlags(
destination,
safeBackupFileName,
);
const localBackupPath = path.join(
volumeBackupPath,
...safeBackupFileName.split("/"),
);
const localBackupDirectory = path.dirname(localBackupPath);
// Command to download backup file from S3
const downloadCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([backupPath])} ${quote([`${volumeBackupPath}/${backupFileName}`])}`;
// Command to download backup file from the configured destination
const downloadCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([backupPath])} ${quote([localBackupPath])}`;
// Base restore command that creates the volume and restores data
const baseRestoreCommand = `
set -e
echo "Volume name: ${volumeName}"
echo "Backup file name: ${backupFileName}"
echo "Volume name: ${safeVolumeName}"
echo "Backup file name:" ${quote([safeBackupFileName])}
echo "Volume backup path: ${volumeBackupPath}"
echo "Downloading backup from S3..."
mkdir -p ${volumeBackupPath}
echo "Downloading backup from destination..."
mkdir -p ${quote([localBackupDirectory])}
${downloadCommand}
echo "Download completed ✅"
echo "Creating new volume and restoring data..."
docker run --rm \
-v ${volumeName}:/volume_data \
-v ${volumeBackupPath}:/backup \
-v ${quote([safeVolumeName])}:/volume_data \
-v ${quote([volumeBackupPath])}:/backup \
ubuntu \
bash -c "cd /volume_data && tar xvf /backup/${quote([backupFileName])} ."
bash -c 'cd /volume_data && tar xvf "/backup/$1" .' -- ${quote([safeBackupFileName])}
echo "Volume restore completed ✅"
`;
// Function to check if volume exists and get containers using it
const checkVolumeCommand = `
# Check if volume exists
VOLUME_EXISTS=$(docker volume ls -q --filter name="^${volumeName}$" | wc -l)
VOLUME_EXISTS=$(docker volume ls -q --filter name="^${safeVolumeName}$" | wc -l)
echo "Volume exists: $VOLUME_EXISTS"
if [ "$VOLUME_EXISTS" = "0" ]; then
@ -58,18 +95,18 @@ export const restoreVolume = async (
echo "Volume exists, checking for containers using it (including stopped ones)..."
# Get ALL containers (running and stopped) using this volume - much simpler with native filter!
CONTAINERS_USING_VOLUME=$(docker ps -a --filter "volume=${volumeName}" --format "{{.ID}}|{{.Names}}|{{.State}}|{{.Labels}}")
CONTAINERS_USING_VOLUME=$(docker ps -a --filter "volume=${safeVolumeName}" --format "{{.ID}}|{{.Names}}|{{.State}}|{{.Labels}}")
if [ -z "$CONTAINERS_USING_VOLUME" ]; then
echo "Volume exists but no containers are using it"
echo "Removing existing volume and proceeding with restore"
docker volume rm ${volumeName} --force
docker volume rm ${quote([safeVolumeName])} --force
${baseRestoreCommand}
else
echo ""
echo "⚠️ WARNING: Cannot restore volume as it is currently in use!"
echo ""
echo "📋 The following containers are using volume '${volumeName}':"
echo "📋 The following containers are using volume '${safeVolumeName}':"
echo ""
echo "$CONTAINERS_USING_VOLUME" | while IFS='|' read container_id container_name container_state labels; do
@ -92,7 +129,7 @@ export const restoreVolume = async (
echo ""
echo "🔧 To restore this volume, please:"
echo " 1. Stop all containers/services using this volume"
echo " 2. Remove the existing volume: docker volume rm ${volumeName}"
echo " 2. Remove the existing volume: docker volume rm ${safeVolumeName}"
echo " 3. Run the restore operation again"
echo ""
echo "❌ Volume restore aborted - volume is in use"

View File

@ -11,7 +11,9 @@ import {
execAsyncRemote,
} from "@dokploy/server/utils/process/execAsync";
import { scheduledJobs, scheduleJob } from "node-schedule";
import { getS3Credentials, normalizeS3Path } from "../backups/utils";
import { quote } from "shell-quote";
import { getSafeRcloneErrorMessage } from "../backups/redact";
import { getRclonePathAndFlags, normalizeS3Path } from "../backups/utils";
import { sendVolumeBackupNotifications } from "../notifications/volume-backup";
import { backupVolume, getVolumeServiceAppName } from "./backup";
@ -84,12 +86,16 @@ const cleanupOldVolumeBackups = async (
if (!keepLatestCount) return;
try {
const rcloneFlags = getS3Credentials(destination);
const s3AppName = getVolumeServiceAppName(volumeBackup);
const backupFilesPath = `:s3:${destination.bucket}/${s3AppName}/${normalizeS3Path(prefix || "")}`;
const listCommand = `rclone lsf ${rcloneFlags.join(" ")} --include \"${volumeName}-*.tar\" ${backupFilesPath}`;
const appName = getVolumeServiceAppName(volumeBackup);
const { flags: rcloneFlags, path: backupFilesPath } =
await getRclonePathAndFlags(
destination,
`${appName}/${normalizeS3Path(prefix || "")}`,
);
const includePattern = `${volumeName}-*.tar`;
const listCommand = `rclone lsf ${rcloneFlags.join(" ")} --include ${quote([includePattern])} ${quote([backupFilesPath])}`;
const sortAndPick = `sort -r | tail -n +$((${keepLatestCount}+1)) | xargs -I{}`;
const deleteCommand = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`;
const deleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} ${quote([`${backupFilesPath}/{}`])}`;
const fullCommand = `${listCommand} | ${sortAndPick} ${deleteCommand}`;
if (serverId) {
@ -98,7 +104,10 @@ const cleanupOldVolumeBackups = async (
await execAsync(fullCommand);
}
} catch (error) {
console.error("Volume backup retention error", error);
console.error(
"Volume backup retention error",
getSafeRcloneErrorMessage(error),
);
}
};
@ -151,13 +160,15 @@ export const runVolumeBackup = async (volumeBackupId: string) => {
);
}
} catch (error) {
const safeErrorMessage = getSafeRcloneErrorMessage(error);
console.error("Volume backup error:", safeErrorMessage);
const { VOLUME_BACKUPS_PATH } = paths(!!serverId);
const volumeBackupPath = path.join(
VOLUME_BACKUPS_PATH,
volumeBackup.appName,
);
// delete all the .tar files
const command = `rm -rf ${volumeBackupPath}/*.tar`;
// delete all the .tar files while keeping wildcard expansion intact
const command = `rm -rf ${quote([volumeBackupPath])}/*.tar`;
if (serverId) {
await execAsyncRemote(serverId, command);
} else {
@ -179,7 +190,7 @@ export const runVolumeBackup = async (volumeBackupId: string) => {
serviceType: mappedServiceType,
type: "error",
organizationId,
errorMessage: error instanceof Error ? error.message : String(error),
errorMessage: safeErrorMessage,
});
} catch (notificationError) {
console.error(