mirror of
https://github.com/Dokploy/dokploy.git
synced 2026-09-12 19:51:00 +05:00
Merge 259f551fc3 into 853ca33659
This commit is contained in:
commit
746044367d
@ -1,4 +1,8 @@
|
||||
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
|
||||
import {
|
||||
getSafeRcloneErrorMessage,
|
||||
redactRcloneCredentials,
|
||||
} from "@dokploy/server/utils/backups/redact";
|
||||
import { quote } from "shell-quote";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
describe("redactRcloneCredentials (#4621)", () => {
|
||||
@ -27,6 +31,23 @@ describe("redactRcloneCredentials (#4621)", () => {
|
||||
expect(redacted).toContain('--s3-region="us-east-1"');
|
||||
});
|
||||
|
||||
it("should redact FTP and SFTP obscured password flags", () => {
|
||||
for (const backend of ["ftp", "sftp"] as const) {
|
||||
const cmd = `rclone lsf --${backend}-host=storage.example.com --${backend}-pass='obscured-secret' :${backend}:backups`;
|
||||
const redacted = redactRcloneCredentials(cmd);
|
||||
expect(redacted).not.toContain("obscured-secret");
|
||||
expect(redacted).toContain(`--${backend}-pass="[REDACTED]"`);
|
||||
}
|
||||
});
|
||||
|
||||
it("should redact unquoted FTP and SFTP password flags", () => {
|
||||
const cmd =
|
||||
"rclone lsf --ftp-pass=ftp-secret --sftp-pass=sftp-secret :ftp:backups";
|
||||
const redacted = redactRcloneCredentials(cmd);
|
||||
expect(redacted).not.toContain("ftp-secret");
|
||||
expect(redacted).not.toContain("sftp-secret");
|
||||
});
|
||||
|
||||
it("should not modify non-credential flags", () => {
|
||||
const cmd =
|
||||
'rclone rcat --s3-region="eu-west-1" --s3-endpoint="https://s3.example.com" --s3-no-check-bucket :s3:bucket/file.gz';
|
||||
@ -47,4 +68,32 @@ describe("redactRcloneCredentials (#4621)", () => {
|
||||
expect(redacted).not.toContain("MYSECRET");
|
||||
expect(redacted).toContain("[REDACTED]");
|
||||
});
|
||||
it("should sanitize rclone credentials from propagated backup errors", () => {
|
||||
const error = new Error(
|
||||
"Command failed: rclone rcat --s3-access-key-id=ACCESS_VALUE_9X --s3-secret-access-key=S3_VALUE_LEAK_9X --ftp-pass=FTP_VALUE_LEAK_9X --sftp-pass=SFTP_VALUE_LEAK_9X --sftp-key-file-pass=KEY_VALUE_LEAK_9X :ftp:backups/file.gz",
|
||||
);
|
||||
const safe = getSafeRcloneErrorMessage(error);
|
||||
|
||||
for (const secret of [
|
||||
"ACCESS_VALUE_9X",
|
||||
"S3_VALUE_LEAK_9X",
|
||||
"FTP_VALUE_LEAK_9X",
|
||||
"SFTP_VALUE_LEAK_9X",
|
||||
"KEY_VALUE_LEAK_9X",
|
||||
]) {
|
||||
expect(safe).not.toContain(secret);
|
||||
}
|
||||
expect(safe.match(/\[REDACTED\]/g)?.length).toBe(5);
|
||||
});
|
||||
it("should fully redact shell-quote output with embedded quotes and whitespace", () => {
|
||||
const secret = "PART_A' PART_B\" $PART_C;\\PART_D";
|
||||
const cmd = `rclone lsf --s3-secret-access-key=${quote([secret])} --s3-region=us-east-1 :s3:bucket`;
|
||||
const redacted = redactRcloneCredentials(cmd);
|
||||
|
||||
for (const fragment of ["PART_A", "PART_B", "PART_C", "PART_D"]) {
|
||||
expect(redacted).not.toContain(fragment);
|
||||
}
|
||||
expect(redacted).toContain('--s3-secret-access-key="[REDACTED]"');
|
||||
expect(redacted).toContain("--s3-region=us-east-1");
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,4 +1,10 @@
|
||||
import { normalizeS3Path } from "@dokploy/server/utils/backups/utils";
|
||||
import { apiCreateDestination } from "@dokploy/server/db/schema/destination";
|
||||
import { RCLONE_DESTINATION_PROVIDERS } from "@dokploy/server/db/validations/destination";
|
||||
import {
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "@dokploy/server/utils/backups/utils";
|
||||
import { normalizeVolumeBackupFilePath } from "@dokploy/server/utils/volume-backups/restore";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
describe("normalizeS3Path", () => {
|
||||
@ -59,3 +65,363 @@ describe("normalizeS3Path", () => {
|
||||
expect(normalizeS3Path("instance-backups")).toBe("instance-backups/");
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeVolumeBackupFilePath", () => {
|
||||
test("accepts a destination-relative backup path", () => {
|
||||
expect(normalizeVolumeBackupFilePath("app/prefix/volume-2026.tar")).toBe(
|
||||
"app/prefix/volume-2026.tar",
|
||||
);
|
||||
});
|
||||
|
||||
test.each([
|
||||
"/absolute/backup.tar",
|
||||
"../backup.tar",
|
||||
"app/../backup.tar",
|
||||
"backup.tar; touch /tmp/pwned",
|
||||
"backup.tar$(touch /tmp/pwned)",
|
||||
"backup.tar`touch /tmp/pwned`",
|
||||
"backup.tar\nmalicious-command",
|
||||
])("rejects unsafe restore path %s", (value) => {
|
||||
expect(() => normalizeVolumeBackupFilePath(value)).toThrow(
|
||||
"Invalid volume backup file path",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
const destination = (overrides: Record<string, unknown> = {}) =>
|
||||
({
|
||||
destinationId: "destination-id",
|
||||
name: "Test",
|
||||
provider: "AWS",
|
||||
accessKey: "access",
|
||||
secretAccessKey: "secret",
|
||||
bucket: "bucket",
|
||||
region: "us-east-1",
|
||||
endpoint: "https://s3.example.com",
|
||||
additionalFlags: [],
|
||||
organizationId: "organization-id",
|
||||
createdAt: new Date(0),
|
||||
...overrides,
|
||||
}) as any;
|
||||
|
||||
describe("FTP destination validation", () => {
|
||||
const input = {
|
||||
name: "FTP backups",
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "secret",
|
||||
bucket: "backups",
|
||||
region: "",
|
||||
endpoint: "storage.example.com",
|
||||
};
|
||||
|
||||
test("rejects plaintext FTP", () => {
|
||||
expect(
|
||||
apiCreateDestination.safeParse({ ...input, additionalFlags: [] }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test.each(["--ftp-explicit-tls", "--ftp-tls"])(
|
||||
"accepts encrypted FTP with %s",
|
||||
(flag) => {
|
||||
const result = apiCreateDestination.safeParse({
|
||||
...input,
|
||||
additionalFlags: [flag],
|
||||
});
|
||||
expect(
|
||||
result.success,
|
||||
result.success ? undefined : JSON.stringify(result.error.issues),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
test("rejects conflicting implicit and explicit FTPS", () => {
|
||||
expect(
|
||||
apiCreateDestination.safeParse({
|
||||
...input,
|
||||
additionalFlags: ["--ftp-explicit-tls", "--ftp-tls"],
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SFTP destination validation", () => {
|
||||
const input = {
|
||||
name: "SFTP backups",
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
bucket: "backups",
|
||||
region: "",
|
||||
endpoint: "storage.example.com",
|
||||
};
|
||||
|
||||
test("rejects SFTP without host-key verification", () => {
|
||||
expect(
|
||||
apiCreateDestination.safeParse({ ...input, additionalFlags: [] }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("rejects explicitly disabled SFTP host-key verification", () => {
|
||||
expect(
|
||||
apiCreateDestination.safeParse({
|
||||
...input,
|
||||
additionalFlags: ["--sftp-known-hosts-file=none"],
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("accepts SFTP with a known-hosts file", () => {
|
||||
const result = apiCreateDestination.safeParse({
|
||||
...input,
|
||||
additionalFlags: ["--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts"],
|
||||
});
|
||||
expect(
|
||||
result.success,
|
||||
result.success ? undefined : JSON.stringify(result.error.issues),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getRclonePathAndFlags", () => {
|
||||
test("preserves the existing S3 destination behavior", async () => {
|
||||
const result = await getRclonePathAndFlags(
|
||||
destination(),
|
||||
"service/prefix/backup.sql.gz",
|
||||
);
|
||||
|
||||
expect(result.path).toBe(":s3:bucket/service/prefix/backup.sql.gz");
|
||||
expect(result.flags).toContain("--s3-provider=AWS");
|
||||
expect(result.flags).toContain("--s3-access-key-id=access");
|
||||
expect(result.flags).toContain("--s3-secret-access-key=secret");
|
||||
});
|
||||
|
||||
test.each([
|
||||
RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
|
||||
RCLONE_DESTINATION_PROVIDERS.ONEDRIVE,
|
||||
RCLONE_DESTINATION_PROVIDERS.REMOTE,
|
||||
])("builds a safe named rclone remote for %s", async (provider) => {
|
||||
const result = await getRclonePathAndFlags(
|
||||
destination({
|
||||
provider,
|
||||
endpoint: "team-drive",
|
||||
bucket: "/dokploy/",
|
||||
accessKey: "",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
additionalFlags: ["--transfers=2"],
|
||||
}),
|
||||
"/service/backup.sql.gz",
|
||||
);
|
||||
|
||||
expect(result.path).toBe("team-drive:dokploy/service/backup.sql.gz");
|
||||
expect(result.flags).toEqual(["--transfers=2"]);
|
||||
});
|
||||
|
||||
test("rejects unsafe named remote names", async () => {
|
||||
await expect(
|
||||
getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
|
||||
endpoint: "drive;touch /tmp/pwned",
|
||||
bucket: "",
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("Invalid rclone remote name");
|
||||
});
|
||||
|
||||
test("rejects unsafe stored additional flags at runtime", async () => {
|
||||
await expect(
|
||||
getRclonePathAndFlags(
|
||||
destination({ additionalFlags: ["--transfers=2;touch /tmp/pwned"] }),
|
||||
),
|
||||
).rejects.toThrow("Invalid flag format");
|
||||
});
|
||||
|
||||
test.each([
|
||||
["--ftp-explicit-tls", "21"],
|
||||
["--ftp-tls", "990"],
|
||||
] as const)(
|
||||
"uses secure FTP mode %s with default port %s",
|
||||
async (tlsFlag, defaultPort) => {
|
||||
const result = await getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
endpoint: "storage.example.com",
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
bucket: "/backups/",
|
||||
additionalFlags: [tlsFlag],
|
||||
}),
|
||||
"service/backup.tar",
|
||||
);
|
||||
|
||||
expect(result.path).toBe(":ftp:backups/service/backup.tar");
|
||||
expect(result.flags).toContain("--ftp-host=storage.example.com");
|
||||
expect(result.flags).toContain("--ftp-user=backup-user");
|
||||
expect(result.flags).toContain(`--ftp-port=${defaultPort}`);
|
||||
expect(result.flags).toContain(tlsFlag);
|
||||
},
|
||||
);
|
||||
|
||||
test("rejects plaintext FTP at runtime", async () => {
|
||||
await expect(
|
||||
getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
endpoint: "storage.example.com",
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
bucket: "/backups/",
|
||||
additionalFlags: [],
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("FTP destinations must use TLS");
|
||||
});
|
||||
|
||||
test("builds SFTP flags with host-key verification and port 22", async () => {
|
||||
const result = await getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
|
||||
endpoint: "storage.example.com",
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
bucket: "/backups/",
|
||||
additionalFlags: ["--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts"],
|
||||
}),
|
||||
"service/backup.tar",
|
||||
);
|
||||
|
||||
expect(result.path).toBe(":sftp:backups/service/backup.tar");
|
||||
expect(result.flags).toContain("--sftp-host=storage.example.com");
|
||||
expect(result.flags).toContain("--sftp-user=backup-user");
|
||||
expect(result.flags).toContain("--sftp-port=22");
|
||||
expect(result.flags).toContain(
|
||||
"--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts",
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects SFTP without host-key verification at runtime", async () => {
|
||||
await expect(
|
||||
getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
|
||||
endpoint: "storage.example.com",
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
bucket: "/backups/",
|
||||
additionalFlags: [],
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("SFTP destinations must verify the server host key");
|
||||
});
|
||||
});
|
||||
|
||||
describe("FTP TLS certificate verification", () => {
|
||||
const input = {
|
||||
name: "FTP backups",
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "secret",
|
||||
bucket: "backups",
|
||||
region: "",
|
||||
endpoint: "storage.example.com",
|
||||
};
|
||||
|
||||
test.each([
|
||||
"--ftp-no-check-certificate",
|
||||
"--ftp-no-check-certificate=true",
|
||||
"--ftp-no-check-certificate=TRUE",
|
||||
"--ftp-no-check-certificate=1",
|
||||
"--ftp-no-check-certificate=t",
|
||||
"--no-check-certificate",
|
||||
"--no-check-certificate=true",
|
||||
"--no-check-certificate=TRUE",
|
||||
"--no-check-certificate=1",
|
||||
"--no-check-certificate=t",
|
||||
])(
|
||||
"rejects certificate-verification bypass %s at schema validation",
|
||||
(flag) => {
|
||||
expect(
|
||||
apiCreateDestination.safeParse({
|
||||
...input,
|
||||
additionalFlags: ["--ftp-explicit-tls", flag],
|
||||
}).success,
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
test.each([
|
||||
"--ftp-no-check-certificate",
|
||||
"--ftp-no-check-certificate=TRUE",
|
||||
"--ftp-no-check-certificate=1",
|
||||
"--no-check-certificate",
|
||||
"--no-check-certificate=TRUE",
|
||||
"--no-check-certificate=1",
|
||||
])("rejects certificate-verification bypass %s at runtime", async (flag) => {
|
||||
await expect(
|
||||
getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
endpoint: "storage.example.com",
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
bucket: "backups",
|
||||
additionalFlags: ["--ftp-explicit-tls", flag],
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("FTP TLS certificate verification cannot be disabled");
|
||||
});
|
||||
|
||||
test.each([
|
||||
"--ftp-no-check-certificate=false",
|
||||
"--ftp-no-check-certificate=FALSE",
|
||||
"--ftp-no-check-certificate=0",
|
||||
"--ftp-no-check-certificate=f",
|
||||
"--no-check-certificate=false",
|
||||
"--no-check-certificate=FALSE",
|
||||
"--no-check-certificate=0",
|
||||
"--no-check-certificate=f",
|
||||
])("allows explicitly safe certificate flag %s", (flag) => {
|
||||
expect(
|
||||
apiCreateDestination.safeParse({
|
||||
...input,
|
||||
additionalFlags: ["--ftp-explicit-tls", flag],
|
||||
}).success,
|
||||
).toBe(true);
|
||||
});
|
||||
test.each([
|
||||
"--ftp-explicit-tls=TRUE",
|
||||
"--ftp-explicit-tls=1",
|
||||
"--ftp-explicit-tls=t",
|
||||
])("accepts pflag-compatible TLS true value %s", (flag) => {
|
||||
expect(
|
||||
apiCreateDestination.safeParse({
|
||||
...input,
|
||||
additionalFlags: [flag],
|
||||
}).success,
|
||||
).toBe(true);
|
||||
});
|
||||
test("forces certificate verification on after user flags", async () => {
|
||||
const result = await getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
endpoint: "storage.example.com",
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
bucket: "backups",
|
||||
additionalFlags: ["--ftp-explicit-tls"],
|
||||
}),
|
||||
);
|
||||
expect(result.flags.slice(-2)).toEqual([
|
||||
"--ftp-no-check-certificate=false",
|
||||
"--no-check-certificate=false",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
137
apps/dokploy/__test__/utils/issue-416-path-safety.test.ts
Normal file
137
apps/dokploy/__test__/utils/issue-416-path-safety.test.ts
Normal file
@ -0,0 +1,137 @@
|
||||
import { apiCreateDestination } from "@dokploy/server/db/schema/destination";
|
||||
import { RCLONE_DESTINATION_PROVIDERS } from "@dokploy/server/db/validations/destination";
|
||||
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
|
||||
import {
|
||||
assertSafeRclonePath,
|
||||
getRclonePathAndFlags,
|
||||
} from "@dokploy/server/utils/backups/utils";
|
||||
import {
|
||||
normalizeDockerVolumeName,
|
||||
normalizeVolumeBackupFilePath,
|
||||
} from "@dokploy/server/utils/volume-backups/restore";
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
const destination = (overrides: Record<string, unknown> = {}) =>
|
||||
({
|
||||
destinationId: "destination-id",
|
||||
name: "Test",
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
|
||||
accessKey: "",
|
||||
secretAccessKey: "",
|
||||
bucket: "dokploy",
|
||||
region: "",
|
||||
endpoint: "team-drive",
|
||||
additionalFlags: [],
|
||||
organizationId: "organization-id",
|
||||
createdAt: new Date(0),
|
||||
...overrides,
|
||||
}) as any;
|
||||
|
||||
describe("issue #416 rclone path safety", () => {
|
||||
test.each([
|
||||
"../backup.sql.gz",
|
||||
"app/../backup.sql.gz",
|
||||
"app/./backup.sql.gz",
|
||||
"..\\backup.sql.gz",
|
||||
"app\\..\\backup.sql.gz",
|
||||
"backup.sql.gz\nother",
|
||||
"backup.sql.gz\rboom",
|
||||
"backup.sql.gz\0boom",
|
||||
])("rejects unsafe destination-relative path %s", (value) => {
|
||||
expect(() => assertSafeRclonePath(value)).toThrow("Invalid rclone path");
|
||||
});
|
||||
|
||||
test.each([
|
||||
"service/backup.sql.gz",
|
||||
"/service/backup.sql.gz",
|
||||
"service/backup..sql.gz",
|
||||
"service/file name.sql.gz",
|
||||
])("keeps valid backup path %s", (value) => {
|
||||
expect(() => assertSafeRclonePath(value)).not.toThrow();
|
||||
});
|
||||
|
||||
test.each([
|
||||
RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
|
||||
RCLONE_DESTINATION_PROVIDERS.ONEDRIVE,
|
||||
RCLONE_DESTINATION_PROVIDERS.REMOTE,
|
||||
RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
RCLONE_DESTINATION_PROVIDERS.SFTP,
|
||||
"AWS",
|
||||
])("blocks traversal before building a %s target", async (provider) => {
|
||||
await expect(
|
||||
getRclonePathAndFlags(destination({ provider }), "app/../outside.sql.gz"),
|
||||
).rejects.toThrow("Invalid rclone path");
|
||||
});
|
||||
});
|
||||
|
||||
describe("issue #416 SFTP host-key safety", () => {
|
||||
const conflictingKnownHostsFlags = [
|
||||
"--sftp-known-hosts-file=/etc/ssh/ssh_known_hosts",
|
||||
"--sftp-known-hosts-file=none",
|
||||
];
|
||||
|
||||
test("rejects conflicting host-key flags during destination validation", () => {
|
||||
const result = apiCreateDestination.safeParse({
|
||||
name: "SFTP backups",
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
bucket: "backups",
|
||||
region: "",
|
||||
endpoint: "storage.example.com",
|
||||
additionalFlags: conflictingKnownHostsFlags,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
test("rejects conflicting host-key flags at runtime", async () => {
|
||||
await expect(
|
||||
getRclonePathAndFlags(
|
||||
destination({
|
||||
provider: RCLONE_DESTINATION_PROVIDERS.SFTP,
|
||||
endpoint: "storage.example.com",
|
||||
accessKey: "backup-user",
|
||||
secretAccessKey: "",
|
||||
region: "",
|
||||
bucket: "backups",
|
||||
additionalFlags: conflictingKnownHostsFlags,
|
||||
}),
|
||||
),
|
||||
).rejects.toThrow("SFTP destinations must verify the server host key");
|
||||
});
|
||||
});
|
||||
|
||||
describe("issue #416 credential redaction", () => {
|
||||
test("redacts an SFTP private-key passphrase flag", () => {
|
||||
const command =
|
||||
"rclone lsf --sftp-key-file=/root/.ssh/id_rsa --sftp-key-file-pass=obscured-secret :sftp:backups";
|
||||
const redacted = redactRcloneCredentials(command);
|
||||
|
||||
expect(redacted).not.toContain("obscured-secret");
|
||||
expect(redacted).toContain('--sftp-key-file-pass="[REDACTED]"');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe("issue #416 volume name and backup path shell safety", () => {
|
||||
test("accepts valid docker volume names", () => {
|
||||
expect(normalizeDockerVolumeName("app_data")).toBe("app_data");
|
||||
expect(normalizeDockerVolumeName("App.Data-1")).toBe("App.Data-1");
|
||||
});
|
||||
|
||||
test.each(["", "../evil", "vol;rm -rf /", "vol$(id)", "-sneaky"])(
|
||||
"rejects unsafe docker volume name %s",
|
||||
(value) => {
|
||||
expect(() => normalizeDockerVolumeName(value)).toThrow(
|
||||
"Invalid docker volume name",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test("normalizes generated backup file names", () => {
|
||||
expect(
|
||||
normalizeVolumeBackupFilePath("app_data-2026-01-01T00-00-00.tar"),
|
||||
).toBe("app_data-2026-01-01T00-00-00.tar");
|
||||
});
|
||||
});
|
||||
@ -1,3 +1,5 @@
|
||||
import { RCLONE_DESTINATION_PROVIDERS } from "@dokploy/server/db/validations/destination";
|
||||
|
||||
export const S3_PROVIDERS: Array<{
|
||||
key: string;
|
||||
name: string;
|
||||
@ -131,3 +133,30 @@ export const S3_PROVIDERS: Array<{
|
||||
name: "Any other S3 compatible provider",
|
||||
},
|
||||
];
|
||||
|
||||
export const DESTINATION_PROVIDERS: Array<{
|
||||
key: string;
|
||||
name: string;
|
||||
}> = [
|
||||
{
|
||||
key: RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE,
|
||||
name: "Google Drive (configured rclone remote)",
|
||||
},
|
||||
{
|
||||
key: RCLONE_DESTINATION_PROVIDERS.ONEDRIVE,
|
||||
name: "Microsoft OneDrive (configured rclone remote)",
|
||||
},
|
||||
{
|
||||
key: RCLONE_DESTINATION_PROVIDERS.SFTP,
|
||||
name: "SFTP",
|
||||
},
|
||||
{
|
||||
key: RCLONE_DESTINATION_PROVIDERS.FTP,
|
||||
name: "FTP",
|
||||
},
|
||||
{
|
||||
key: RCLONE_DESTINATION_PROVIDERS.REMOTE,
|
||||
name: "Other configured rclone remote",
|
||||
},
|
||||
...S3_PROVIDERS,
|
||||
];
|
||||
|
||||
@ -1,6 +1,11 @@
|
||||
import {
|
||||
ADDITIONAL_FLAG_ERROR,
|
||||
ADDITIONAL_FLAG_REGEX,
|
||||
getDestinationValidationIssues,
|
||||
getFtpTlsState,
|
||||
isNamedRcloneDestinationProvider,
|
||||
isRcloneDestinationProvider,
|
||||
RCLONE_DESTINATION_PROVIDERS,
|
||||
} from "@dokploy/server/db/validations/destination";
|
||||
import { standardSchemaResolver as zodResolver } from "@hookform/resolvers/standard-schema";
|
||||
import { PenBoxIcon, PlusIcon, Trash2 } from "lucide-react";
|
||||
@ -39,28 +44,62 @@ import {
|
||||
} from "@/components/ui/select";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { api } from "@/utils/api";
|
||||
import { S3_PROVIDERS } from "./constants";
|
||||
import { DESTINATION_PROVIDERS } from "./constants";
|
||||
|
||||
const addDestination = z.object({
|
||||
name: z.string().min(1, "Name is required"),
|
||||
provider: z.string().min(1, "Provider is required"),
|
||||
accessKeyId: z.string().min(1, "Access Key Id is required"),
|
||||
secretAccessKey: z.string().min(1, "Secret Access Key is required"),
|
||||
bucket: z.string().min(1, "Bucket is required"),
|
||||
region: z.string(),
|
||||
endpoint: z.string().min(1, "Endpoint is required"),
|
||||
serverId: z.string().optional(),
|
||||
additionalFlags: z
|
||||
.array(
|
||||
z.object({
|
||||
value: z
|
||||
.string()
|
||||
.min(1, "Flag cannot be empty")
|
||||
.regex(ADDITIONAL_FLAG_REGEX, ADDITIONAL_FLAG_ERROR),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
});
|
||||
const addDestination = z
|
||||
.object({
|
||||
name: z.string().min(1, "Name is required"),
|
||||
provider: z.string().min(1, "Provider is required"),
|
||||
accessKeyId: z.string(),
|
||||
secretAccessKey: z.string(),
|
||||
bucket: z.string(),
|
||||
region: z.string(),
|
||||
endpoint: z.string(),
|
||||
serverId: z.string().optional(),
|
||||
additionalFlags: z
|
||||
.array(
|
||||
z.object({
|
||||
value: z
|
||||
.string()
|
||||
.min(1, "Flag cannot be empty")
|
||||
.regex(ADDITIONAL_FLAG_REGEX, ADDITIONAL_FLAG_ERROR),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
.superRefine((data, ctx) => {
|
||||
const additionalFlags = data.additionalFlags?.map((flag) => flag.value) ?? [];
|
||||
for (const issue of getDestinationValidationIssues({
|
||||
provider: data.provider,
|
||||
accessKey: data.accessKeyId,
|
||||
region: data.region,
|
||||
endpoint: data.endpoint,
|
||||
additionalFlags,
|
||||
})) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: [issue.field === "accessKey" ? "accessKeyId" : issue.field],
|
||||
message: issue.message,
|
||||
});
|
||||
}
|
||||
|
||||
if (isRcloneDestinationProvider(data.provider)) return;
|
||||
|
||||
for (const [field, label] of [
|
||||
["accessKeyId", "Access Key Id"],
|
||||
["secretAccessKey", "Secret Access Key"],
|
||||
["bucket", "Bucket"],
|
||||
["endpoint", "Endpoint"],
|
||||
] as const) {
|
||||
if (!data[field].trim()) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: [field],
|
||||
message: `${label} is required`,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
type AddDestination = z.infer<typeof addDestination>;
|
||||
|
||||
@ -108,6 +147,19 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
resolver: zodResolver(addDestination),
|
||||
});
|
||||
|
||||
const currentProvider = form.watch("provider");
|
||||
const currentAdditionalFlags =
|
||||
form.watch("additionalFlags")?.map((flag) => flag.value) ?? [];
|
||||
const { implicitTlsEnabled: implicitFtpTlsEnabled } = getFtpTlsState(
|
||||
currentAdditionalFlags,
|
||||
);
|
||||
const isNamedRemote = isNamedRcloneDestinationProvider(currentProvider);
|
||||
const isFileTransfer =
|
||||
currentProvider === RCLONE_DESTINATION_PROVIDERS.FTP ||
|
||||
currentProvider === RCLONE_DESTINATION_PROVIDERS.SFTP;
|
||||
const hasRemoteServers = (servers?.length ?? 0) > 0;
|
||||
const showServerSelector = Boolean(isCloud) || hasRemoteServers;
|
||||
|
||||
const { fields, append, remove } = useFieldArray({
|
||||
control: form.control,
|
||||
name: "additionalFlags",
|
||||
@ -131,17 +183,25 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
}
|
||||
}, [form, form.reset, form.formState.isSubmitSuccessful, destination]);
|
||||
|
||||
const getPayload = (data: AddDestination) => ({
|
||||
provider: data.provider,
|
||||
accessKey: isNamedRcloneDestinationProvider(data.provider)
|
||||
? ""
|
||||
: data.accessKeyId,
|
||||
bucket: data.bucket,
|
||||
endpoint: data.endpoint.trim(),
|
||||
name: data.name,
|
||||
region: isNamedRcloneDestinationProvider(data.provider) ? "" : data.region,
|
||||
secretAccessKey: isNamedRcloneDestinationProvider(data.provider)
|
||||
? ""
|
||||
: data.secretAccessKey,
|
||||
additionalFlags: data.additionalFlags?.map((f) => f.value) ?? [],
|
||||
});
|
||||
|
||||
const onSubmit = async (data: AddDestination) => {
|
||||
await mutateAsync({
|
||||
provider: data.provider || "",
|
||||
accessKey: data.accessKeyId,
|
||||
bucket: data.bucket,
|
||||
endpoint: data.endpoint,
|
||||
name: data.name,
|
||||
region: data.region,
|
||||
secretAccessKey: data.secretAccessKey,
|
||||
...getPayload(data),
|
||||
destinationId: destinationId || "",
|
||||
additionalFlags: data.additionalFlags?.map((f) => f.value) ?? [],
|
||||
})
|
||||
.then(async () => {
|
||||
toast.success(`Destination ${destinationId ? "Updated" : "Created"}`);
|
||||
@ -162,14 +222,7 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
};
|
||||
|
||||
const handleTestConnection = async (serverId?: string) => {
|
||||
const result = await form.trigger([
|
||||
"provider",
|
||||
"accessKeyId",
|
||||
"secretAccessKey",
|
||||
"bucket",
|
||||
"endpoint",
|
||||
"additionalFlags",
|
||||
]);
|
||||
const result = await form.trigger();
|
||||
|
||||
if (!result) {
|
||||
const errors = form.formState.errors;
|
||||
@ -184,38 +237,23 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
return;
|
||||
}
|
||||
|
||||
if (isCloud && !serverId) {
|
||||
const selectedServerId = serverId === "none" ? undefined : serverId;
|
||||
if (isCloud && !selectedServerId) {
|
||||
toast.error("Please select a server");
|
||||
return;
|
||||
}
|
||||
|
||||
const provider = form.getValues("provider");
|
||||
const accessKey = form.getValues("accessKeyId");
|
||||
const secretKey = form.getValues("secretAccessKey");
|
||||
const bucket = form.getValues("bucket");
|
||||
const endpoint = form.getValues("endpoint");
|
||||
const region = form.getValues("region");
|
||||
|
||||
const connectionString = `:s3,provider=${provider},access_key_id=${accessKey},secret_access_key=${secretKey},endpoint=${endpoint}${region ? `,region=${region}` : ""}:${bucket}`;
|
||||
|
||||
await testConnection({
|
||||
provider,
|
||||
accessKey,
|
||||
bucket,
|
||||
endpoint,
|
||||
...getPayload(form.getValues()),
|
||||
name: "Test",
|
||||
region,
|
||||
secretAccessKey: secretKey,
|
||||
serverId,
|
||||
additionalFlags:
|
||||
form.getValues("additionalFlags")?.map((f) => f.value) ?? [],
|
||||
serverId: selectedServerId,
|
||||
})
|
||||
.then(() => {
|
||||
toast.success("Connection Success");
|
||||
})
|
||||
.catch((e) => {
|
||||
toast.error("Error connecting to provider", {
|
||||
description: `${e.message}\n\nTry manually: rclone ls ${connectionString}`,
|
||||
description: e.message,
|
||||
});
|
||||
});
|
||||
};
|
||||
@ -244,9 +282,9 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
{destinationId ? "Update" : "Add"} Destination
|
||||
</DialogTitle>
|
||||
<DialogDescription>
|
||||
In this section, you can configure and add new destinations for your
|
||||
backups. Please ensure that you provide the correct information to
|
||||
guarantee secure and efficient storage.
|
||||
Configure a backup destination. Google Drive, OneDrive, and generic
|
||||
rclone destinations use a named rclone remote configured on the
|
||||
machine that executes the backup.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
{(isError || isErrorConnection) && (
|
||||
@ -264,123 +302,113 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="name"
|
||||
render={({ field }) => {
|
||||
return (
|
||||
<FormItem>
|
||||
<FormLabel>Name</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={"S3 Bucket"} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="provider"
|
||||
render={({ field }) => {
|
||||
return (
|
||||
<FormItem>
|
||||
<FormLabel>Provider</FormLabel>
|
||||
<FormControl>
|
||||
<Select
|
||||
onValueChange={field.onChange}
|
||||
defaultValue={field.value}
|
||||
value={field.value}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Select a S3 Provider" />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{S3_PROVIDERS.map((s3Provider) => (
|
||||
<SelectItem
|
||||
key={s3Provider.key}
|
||||
value={s3Provider.key}
|
||||
>
|
||||
{s3Provider.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="accessKeyId"
|
||||
render={({ field }) => {
|
||||
return (
|
||||
<FormItem>
|
||||
<FormLabel>Access Key Id</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={"xcas41dasde"} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="secretAccessKey"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<div className="space-y-0.5">
|
||||
<FormLabel>Secret Access Key</FormLabel>
|
||||
</div>
|
||||
<FormLabel>Name</FormLabel>
|
||||
<FormControl>
|
||||
<Input placeholder={"asd123asdasw"} {...field} />
|
||||
<Input placeholder="Backups" {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="provider"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Provider</FormLabel>
|
||||
<FormControl>
|
||||
<Select
|
||||
onValueChange={field.onChange}
|
||||
defaultValue={field.value}
|
||||
value={field.value}
|
||||
>
|
||||
<FormControl>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Select a destination provider" />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{DESTINATION_PROVIDERS.map((provider) => (
|
||||
<SelectItem key={provider.key} value={provider.key}>
|
||||
{provider.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{!isNamedRemote && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="accessKeyId"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>
|
||||
{isFileTransfer ? "Username" : "Access Key Id"}
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder={
|
||||
isFileTransfer ? "username" : "Access Key ID"
|
||||
}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
{!isNamedRemote && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="secretAccessKey"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>
|
||||
{isFileTransfer
|
||||
? "Password (Optional)"
|
||||
: "Secret Access Key"}
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
type={isFileTransfer ? "password" : "text"}
|
||||
placeholder={
|
||||
isFileTransfer ? "password" : "Secret Access Key"
|
||||
}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="bucket"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<div className="space-y-0.5">
|
||||
<FormLabel>Bucket</FormLabel>
|
||||
</div>
|
||||
<FormControl>
|
||||
<Input placeholder={"dokploy-bucket"} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="region"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<div className="space-y-0.5">
|
||||
<FormLabel>Region</FormLabel>
|
||||
</div>
|
||||
<FormControl>
|
||||
<Input placeholder={"us-east-1"} {...field} />
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="endpoint"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>Endpoint</FormLabel>
|
||||
<FormLabel>
|
||||
{isNamedRemote
|
||||
? "Remote Base Path (Optional)"
|
||||
: isFileTransfer
|
||||
? "Base Path / Directory (Optional)"
|
||||
: "Bucket"}
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder={"https://us.bucket.aws/s3"}
|
||||
placeholder={
|
||||
isNamedRemote || isFileTransfer
|
||||
? "dokploy-backups"
|
||||
: "dokploy-bucket"
|
||||
}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
@ -388,9 +416,78 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
{!isNamedRemote && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="region"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>{isFileTransfer ? "Port" : "Region"}</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder={
|
||||
isFileTransfer
|
||||
? currentProvider ===
|
||||
RCLONE_DESTINATION_PROVIDERS.SFTP
|
||||
? "22"
|
||||
: implicitFtpTlsEnabled
|
||||
? "990"
|
||||
: "21"
|
||||
: "us-east-1"
|
||||
}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="endpoint"
|
||||
render={({ field }) => (
|
||||
<FormItem>
|
||||
<FormLabel>
|
||||
{isNamedRemote
|
||||
? "Rclone Remote Name"
|
||||
: isFileTransfer
|
||||
? "Host"
|
||||
: "Endpoint"}
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder={
|
||||
isNamedRemote
|
||||
? currentProvider ===
|
||||
RCLONE_DESTINATION_PROVIDERS.ONEDRIVE
|
||||
? "onedrive"
|
||||
: "gdrive"
|
||||
: isFileTransfer
|
||||
? "storage.example.com"
|
||||
: "https://us.bucket.aws/s3"
|
||||
}
|
||||
{...field}
|
||||
/>
|
||||
</FormControl>
|
||||
{isNamedRemote && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Configure this remote with rclone on the machine that runs
|
||||
the backup, then enter only its remote name here (without
|
||||
a colon).
|
||||
</p>
|
||||
)}
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<FormLabel>Additional Flags (Optional)</FormLabel>
|
||||
<FormLabel>
|
||||
{isFileTransfer
|
||||
? "Security / Additional Flags"
|
||||
: "Additional Flags (Optional)"}
|
||||
</FormLabel>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
@ -401,6 +498,18 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
Add Flag
|
||||
</Button>
|
||||
</div>
|
||||
{currentProvider === RCLONE_DESTINATION_PROVIDERS.FTP && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Required: --ftp-explicit-tls for explicit FTPS (port 21) or
|
||||
--ftp-tls for implicit FTPS (port 990).
|
||||
</p>
|
||||
)}
|
||||
{currentProvider === RCLONE_DESTINATION_PROVIDERS.SFTP && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Required: --sftp-known-hosts-file=/path/to/known_hosts to
|
||||
verify the server host key.
|
||||
</p>
|
||||
)}
|
||||
{fields.map((field, index) => (
|
||||
<FormField
|
||||
key={field.id}
|
||||
@ -410,10 +519,7 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
<FormItem>
|
||||
<div className="flex items-center gap-2">
|
||||
<FormControl>
|
||||
<Input
|
||||
placeholder="--s3-sign-accept-encoding=false"
|
||||
{...field}
|
||||
/>
|
||||
<Input placeholder="--flag=value" {...field} />
|
||||
</FormControl>
|
||||
<Button
|
||||
type="button"
|
||||
@ -434,15 +540,15 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
|
||||
<DialogFooter
|
||||
className={cn(
|
||||
isCloud ? "flex-col!" : "flex-row",
|
||||
showServerSelector ? "flex-col!" : "flex-row",
|
||||
"flex w-full justify-between! gap-4",
|
||||
)}
|
||||
>
|
||||
{isCloud ? (
|
||||
{showServerSelector ? (
|
||||
<div className="flex flex-col gap-4 border p-2 rounded-lg">
|
||||
<span className="text-sm text-muted-foreground">
|
||||
Select a server to test the destination. If you don't have a
|
||||
server choose the default one.
|
||||
Select the server that will execute the backup so the
|
||||
destination can be tested from the same environment.
|
||||
</span>
|
||||
<FormField
|
||||
control={form.control}
|
||||
@ -469,19 +575,22 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
{server.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
<SelectItem value={"none"}>None</SelectItem>
|
||||
{!isCloud && (
|
||||
<SelectItem value="none">
|
||||
Dokploy Server (Local)
|
||||
</SelectItem>
|
||||
)}
|
||||
</SelectGroup>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
<Button
|
||||
type="button"
|
||||
variant={"secondary"}
|
||||
variant="secondary"
|
||||
isLoading={isPendingConnection}
|
||||
onClick={async () => {
|
||||
await handleTestConnection(form.getValues("serverId"));
|
||||
@ -515,4 +624,4 @@ export const HandleDestinations = ({ destinationId }: Props) => {
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
};
|
||||
};
|
||||
@ -31,8 +31,9 @@ import {
|
||||
import { findDestinationById } from "@dokploy/server/services/destination";
|
||||
import { checkServicePermissionAndAccess } from "@dokploy/server/services/permission";
|
||||
import { runComposeBackup } from "@dokploy/server/utils/backups/compose";
|
||||
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
|
||||
import {
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "@dokploy/server/utils/backups/utils";
|
||||
import {
|
||||
@ -79,6 +80,20 @@ interface RcloneFile {
|
||||
};
|
||||
}
|
||||
|
||||
const assertDestinationAccess = async (
|
||||
destinationId: string,
|
||||
organizationId: string,
|
||||
) => {
|
||||
const destination = await findDestinationById(destinationId);
|
||||
if (destination.organizationId !== organizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You don't have access to this destination.",
|
||||
});
|
||||
}
|
||||
return destination;
|
||||
};
|
||||
|
||||
export const backupRouter = createTRPCRouter({
|
||||
create: protectedProcedure
|
||||
.input(apiCreateBackup)
|
||||
@ -96,6 +111,10 @@ export const backupRouter = createTRPCRouter({
|
||||
backup: ["create"],
|
||||
});
|
||||
}
|
||||
await assertDestinationAccess(
|
||||
input.destinationId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
|
||||
if (IS_CLOUD) {
|
||||
const dbType = (
|
||||
@ -161,7 +180,7 @@ export const backupRouter = createTRPCRouter({
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
|
||||
message:
|
||||
error instanceof Error
|
||||
? error.message
|
||||
@ -207,6 +226,10 @@ export const backupRouter = createTRPCRouter({
|
||||
backup: ["update"],
|
||||
});
|
||||
}
|
||||
await assertDestinationAccess(
|
||||
input.destinationId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
|
||||
await updateBackupById(input.backupId, input);
|
||||
const backup = await findBackupById(input.backupId);
|
||||
@ -242,7 +265,7 @@ export const backupRouter = createTRPCRouter({
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Error updating this Backup";
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
|
||||
message,
|
||||
});
|
||||
}
|
||||
@ -312,7 +335,7 @@ export const backupRouter = createTRPCRouter({
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error
|
||||
? error.message
|
||||
? redactRcloneCredentials(error.message)
|
||||
: "Error running manual Postgres backup ";
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
@ -479,13 +502,10 @@ export const backupRouter = createTRPCRouter({
|
||||
)
|
||||
.query(async ({ input, ctx }) => {
|
||||
try {
|
||||
const destination = await findDestinationById(input.destinationId);
|
||||
if (destination.organizationId !== ctx.session.activeOrganizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You don't have access to this destination.",
|
||||
});
|
||||
}
|
||||
const destination = await assertDestinationAccess(
|
||||
input.destinationId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
if (input.serverId) {
|
||||
const targetServer = await findServerById(input.serverId);
|
||||
if (
|
||||
@ -497,8 +517,6 @@ export const backupRouter = createTRPCRouter({
|
||||
});
|
||||
}
|
||||
}
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
|
||||
const lastSlashIndex = input.search.lastIndexOf("/");
|
||||
const baseDir =
|
||||
@ -509,8 +527,8 @@ export const backupRouter = createTRPCRouter({
|
||||
lastSlashIndex !== -1
|
||||
? input.search.slice(lastSlashIndex + 1)
|
||||
: input.search;
|
||||
|
||||
const searchPath = baseDir ? `${bucketPath}/${baseDir}` : bucketPath;
|
||||
const { flags: rcloneFlags, path: searchPath } =
|
||||
await getRclonePathAndFlags(destination, baseDir);
|
||||
const listCommand = `rclone lsjson ${rcloneFlags.join(" ")} ${quote([searchPath])} --no-mimetype --no-modtime 2>/dev/null`;
|
||||
|
||||
let stdout = "";
|
||||
@ -532,8 +550,6 @@ export const backupRouter = createTRPCRouter({
|
||||
throw new Error("Failed to parse backup files list");
|
||||
}
|
||||
|
||||
// Limit to first 100 files
|
||||
|
||||
const results = baseDir
|
||||
? files.map((file) => ({
|
||||
...file,
|
||||
@ -551,14 +567,15 @@ export const backupRouter = createTRPCRouter({
|
||||
|
||||
return results.slice(0, 100);
|
||||
} catch (error) {
|
||||
console.error("Error in listBackupFiles:", error);
|
||||
const safeMessage =
|
||||
error instanceof Error
|
||||
? redactRcloneCredentials(error.message)
|
||||
: "Error listing backup files";
|
||||
console.error("Error in listBackupFiles:", safeMessage);
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
message:
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Error listing backup files",
|
||||
cause: error,
|
||||
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
|
||||
message: safeMessage,
|
||||
cause: new Error(safeMessage),
|
||||
});
|
||||
}
|
||||
}),
|
||||
@ -579,10 +596,13 @@ export const backupRouter = createTRPCRouter({
|
||||
backup: ["restore"],
|
||||
});
|
||||
}
|
||||
const destination = await findDestinationById(input.destinationId);
|
||||
const destination = await assertDestinationAccess(
|
||||
input.destinationId,
|
||||
ctx.session.activeOrganizationId,
|
||||
);
|
||||
const queue: string[] = [];
|
||||
let done = false;
|
||||
const onLog = (log: string) => queue.push(log);
|
||||
const onLog = (log: string) => queue.push(redactRcloneCredentials(log));
|
||||
const runRestore = async () => {
|
||||
if (input.backupType === "database") {
|
||||
if (input.databaseType === "postgres") {
|
||||
|
||||
@ -3,11 +3,14 @@ import {
|
||||
execAsync,
|
||||
execAsyncRemote,
|
||||
findDestinationById,
|
||||
findServerById,
|
||||
getRclonePathAndFlags,
|
||||
IS_CLOUD,
|
||||
removeDestinationById,
|
||||
updateDestinationById,
|
||||
} from "@dokploy/server";
|
||||
import { db } from "@dokploy/server/db";
|
||||
import { redactRcloneCredentials } from "@dokploy/server/utils/backups/redact";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { desc, eq } from "drizzle-orm";
|
||||
import { quote } from "shell-quote";
|
||||
@ -47,37 +50,19 @@ export const destinationRouter = createTRPCRouter({
|
||||
}),
|
||||
testConnection: withPermission("destination", "create")
|
||||
.input(apiCreateDestination)
|
||||
.mutation(async ({ input }) => {
|
||||
const {
|
||||
secretAccessKey,
|
||||
bucket,
|
||||
region,
|
||||
endpoint,
|
||||
accessKey,
|
||||
provider,
|
||||
additionalFlags,
|
||||
} = input;
|
||||
.mutation(async ({ input, ctx }) => {
|
||||
try {
|
||||
const { flags, path } = await getRclonePathAndFlags(
|
||||
input as Parameters<typeof getRclonePathAndFlags>[0],
|
||||
);
|
||||
const rcloneFlags = [
|
||||
`--s3-access-key-id=${quote([accessKey])}`,
|
||||
`--s3-secret-access-key=${quote([secretAccessKey])}`,
|
||||
`--s3-region=${quote([region])}`,
|
||||
`--s3-endpoint=${quote([endpoint])}`,
|
||||
"--s3-no-check-bucket",
|
||||
"--s3-force-path-style",
|
||||
...flags,
|
||||
"--retries 1",
|
||||
"--low-level-retries 1",
|
||||
"--timeout 10s",
|
||||
"--contimeout 5s",
|
||||
];
|
||||
if (provider) {
|
||||
rcloneFlags.unshift(`--s3-provider=${quote([provider])}`);
|
||||
}
|
||||
if (additionalFlags?.length) {
|
||||
rcloneFlags.push(...additionalFlags);
|
||||
}
|
||||
const rcloneDestination = `:s3:${bucket}`;
|
||||
const rcloneCommand = `rclone ls ${rcloneFlags.join(" ")} ${quote([rcloneDestination])}`;
|
||||
const rcloneCommand = `rclone lsd ${rcloneFlags.join(" ")} ${quote([path])}`;
|
||||
|
||||
if (IS_CLOUD && !input.serverId) {
|
||||
throw new TRPCError({
|
||||
@ -86,19 +71,27 @@ export const destinationRouter = createTRPCRouter({
|
||||
});
|
||||
}
|
||||
|
||||
if (IS_CLOUD) {
|
||||
await execAsyncRemote(input.serverId || "", rcloneCommand);
|
||||
if (input.serverId) {
|
||||
const server = await findServerById(input.serverId);
|
||||
if (server.organizationId !== ctx.session.activeOrganizationId) {
|
||||
throw new TRPCError({
|
||||
code: "UNAUTHORIZED",
|
||||
message: "You are not allowed to use this server",
|
||||
});
|
||||
}
|
||||
await execAsyncRemote(input.serverId, rcloneCommand);
|
||||
} else {
|
||||
await execAsync(rcloneCommand);
|
||||
}
|
||||
} catch (error) {
|
||||
const safeMessage =
|
||||
error instanceof Error
|
||||
? redactRcloneCredentials(error.message)
|
||||
: "Error connecting to destination";
|
||||
throw new TRPCError({
|
||||
code: "BAD_REQUEST",
|
||||
message:
|
||||
error instanceof Error
|
||||
? error?.message
|
||||
: "Error connecting to bucket",
|
||||
cause: error,
|
||||
code: error instanceof TRPCError ? error.code : "BAD_REQUEST",
|
||||
message: safeMessage,
|
||||
cause: new Error(safeMessage),
|
||||
});
|
||||
}
|
||||
}),
|
||||
@ -174,8 +167,8 @@ export const destinationRouter = createTRPCRouter({
|
||||
code: "BAD_REQUEST",
|
||||
message:
|
||||
error instanceof Error
|
||||
? error?.message
|
||||
: "Error connecting to bucket",
|
||||
? redactRcloneCredentials(error.message)
|
||||
: "Error updating destination",
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ import { z } from "zod";
|
||||
import {
|
||||
ADDITIONAL_FLAG_ERROR,
|
||||
ADDITIONAL_FLAG_REGEX,
|
||||
getDestinationValidationIssues,
|
||||
} from "../validations/destination";
|
||||
import { organization } from "./account";
|
||||
import { backups } from "./backups";
|
||||
@ -54,6 +55,25 @@ const createSchema = createInsertSchema(destinations, {
|
||||
.default([]),
|
||||
});
|
||||
|
||||
const validateDestination = (
|
||||
data: {
|
||||
provider?: string | null;
|
||||
accessKey?: string;
|
||||
region?: string;
|
||||
endpoint?: string;
|
||||
additionalFlags?: string[] | null;
|
||||
},
|
||||
ctx: z.RefinementCtx,
|
||||
) => {
|
||||
for (const issue of getDestinationValidationIssues(data)) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
path: [issue.field],
|
||||
message: issue.message,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const apiCreateDestination = createSchema
|
||||
.pick({
|
||||
name: true,
|
||||
@ -68,7 +88,8 @@ export const apiCreateDestination = createSchema
|
||||
.required()
|
||||
.extend({
|
||||
serverId: z.string().optional(),
|
||||
});
|
||||
})
|
||||
.superRefine(validateDestination);
|
||||
|
||||
export const apiFindOneDestination = z.object({
|
||||
destinationId: z.string().min(1),
|
||||
@ -95,4 +116,5 @@ export const apiUpdateDestination = createSchema
|
||||
.required()
|
||||
.extend({
|
||||
serverId: z.string().optional(),
|
||||
});
|
||||
})
|
||||
.superRefine(validateDestination);
|
||||
|
||||
@ -1,3 +1,183 @@
|
||||
export const ADDITIONAL_FLAG_REGEX = /^--[a-zA-Z0-9-]+(=[a-zA-Z0-9._:/@-]+)?$/;
|
||||
export const ADDITIONAL_FLAG_ERROR =
|
||||
"Invalid flag format. Must start with -- (e.g. --s3-sign-accept-encoding=false)";
|
||||
|
||||
export const RCLONE_DESTINATION_PROVIDERS = {
|
||||
GOOGLE_DRIVE: "GoogleDrive",
|
||||
ONEDRIVE: "OneDrive",
|
||||
FTP: "FTP",
|
||||
SFTP: "SFTP",
|
||||
REMOTE: "RcloneRemote",
|
||||
} as const;
|
||||
|
||||
export type RcloneDestinationProvider =
|
||||
(typeof RCLONE_DESTINATION_PROVIDERS)[keyof typeof RCLONE_DESTINATION_PROVIDERS];
|
||||
|
||||
const RCLONE_DESTINATION_PROVIDER_VALUES = new Set<string>(
|
||||
Object.values(RCLONE_DESTINATION_PROVIDERS),
|
||||
);
|
||||
|
||||
export const isRcloneDestinationProvider = (
|
||||
provider: string | null | undefined,
|
||||
): provider is RcloneDestinationProvider =>
|
||||
!!provider && RCLONE_DESTINATION_PROVIDER_VALUES.has(provider);
|
||||
|
||||
export const isNamedRcloneDestinationProvider = (
|
||||
provider: string | null | undefined,
|
||||
) =>
|
||||
provider === RCLONE_DESTINATION_PROVIDERS.GOOGLE_DRIVE ||
|
||||
provider === RCLONE_DESTINATION_PROVIDERS.ONEDRIVE ||
|
||||
provider === RCLONE_DESTINATION_PROVIDERS.REMOTE;
|
||||
|
||||
export const RCLONE_REMOTE_NAME_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$/;
|
||||
export const RCLONE_REMOTE_NAME_ERROR =
|
||||
"Invalid rclone remote name. Use only letters, numbers, dots, underscores, and dashes";
|
||||
|
||||
export const FTP_TLS_REQUIRED_ERROR =
|
||||
"FTP destinations must use TLS. Add --ftp-explicit-tls for explicit FTPS (port 21) or --ftp-tls for implicit FTPS (port 990).";
|
||||
export const FTP_TLS_CONFLICT_ERROR =
|
||||
"Choose either implicit FTPS or explicit FTPS, not both.";
|
||||
export const FTP_CERTIFICATE_VERIFICATION_REQUIRED_ERROR =
|
||||
"FTP TLS certificate verification cannot be disabled.";
|
||||
export const SFTP_HOST_KEY_REQUIRED_ERROR =
|
||||
"SFTP destinations must verify the server host key. Add --sftp-known-hosts-file=/path/to/known_hosts.";
|
||||
|
||||
const parseBooleanFlagValue = (
|
||||
flag: string,
|
||||
flagName: string,
|
||||
): boolean | undefined => {
|
||||
if (flag === flagName) return true;
|
||||
const prefix = `${flagName}=`;
|
||||
if (!flag.startsWith(prefix)) return undefined;
|
||||
|
||||
const value = flag.slice(prefix.length).toLowerCase();
|
||||
if (["1", "t", "true"].includes(value)) return true;
|
||||
if (["0", "f", "false"].includes(value)) return false;
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const getBooleanFlagValues = (flags: readonly string[], flagName: string) =>
|
||||
flags
|
||||
.filter((flag) => flag === flagName || flag.startsWith(`${flagName}=`))
|
||||
.map((flag) => parseBooleanFlagValue(flag, flagName));
|
||||
|
||||
const isBooleanFlagEnabled = (
|
||||
flags: readonly string[],
|
||||
flagName: string,
|
||||
): boolean => {
|
||||
const values = getBooleanFlagValues(flags, flagName);
|
||||
return values.length > 0 && values.every((value) => value === true);
|
||||
};
|
||||
|
||||
export const getFtpTlsState = (flags: readonly string[] | null | undefined) => {
|
||||
const values = flags ?? [];
|
||||
return {
|
||||
implicitTlsEnabled: isBooleanFlagEnabled(values, "--ftp-tls"),
|
||||
explicitTlsEnabled: isBooleanFlagEnabled(values, "--ftp-explicit-tls"),
|
||||
};
|
||||
};
|
||||
|
||||
export const hasDisabledFtpCertificateVerification = (
|
||||
flags: readonly string[] | null | undefined,
|
||||
): boolean => {
|
||||
const values = flags ?? [];
|
||||
return ["--ftp-no-check-certificate", "--no-check-certificate"].some(
|
||||
(flagName) => {
|
||||
const matchingValues = getBooleanFlagValues(values, flagName);
|
||||
return (
|
||||
matchingValues.length > 0 &&
|
||||
matchingValues.some((value) => value !== false)
|
||||
);
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
export const hasSftpHostKeyVerification = (
|
||||
flags: readonly string[] | null | undefined,
|
||||
): boolean => {
|
||||
const prefix = "--sftp-known-hosts-file=";
|
||||
const knownHostsFlags = (flags ?? []).filter((flag) =>
|
||||
flag.startsWith(prefix),
|
||||
);
|
||||
if (knownHostsFlags.length !== 1) return false;
|
||||
|
||||
const value = knownHostsFlags[0]?.slice(prefix.length).trim() ?? "";
|
||||
return value.length > 0 && value !== "none";
|
||||
};
|
||||
|
||||
type DestinationValidationField =
|
||||
| "endpoint"
|
||||
| "accessKey"
|
||||
| "region"
|
||||
| "additionalFlags";
|
||||
|
||||
export interface DestinationValidationIssue {
|
||||
field: DestinationValidationField;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface DestinationValidationInput {
|
||||
provider?: string | null;
|
||||
accessKey?: string;
|
||||
region?: string;
|
||||
endpoint?: string;
|
||||
additionalFlags?: readonly string[] | null;
|
||||
}
|
||||
|
||||
export const getDestinationValidationIssues = (
|
||||
data: DestinationValidationInput,
|
||||
): DestinationValidationIssue[] => {
|
||||
const issues: DestinationValidationIssue[] = [];
|
||||
const provider = data.provider;
|
||||
const flags = data.additionalFlags ?? [];
|
||||
|
||||
if (isNamedRcloneDestinationProvider(provider)) {
|
||||
if (!RCLONE_REMOTE_NAME_REGEX.test(data.endpoint?.trim() || "")) {
|
||||
issues.push({ field: "endpoint", message: RCLONE_REMOTE_NAME_ERROR });
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
if (
|
||||
provider !== RCLONE_DESTINATION_PROVIDERS.FTP &&
|
||||
provider !== RCLONE_DESTINATION_PROVIDERS.SFTP
|
||||
) {
|
||||
return issues;
|
||||
}
|
||||
|
||||
if (!data.endpoint?.trim()) {
|
||||
issues.push({ field: "endpoint", message: "Host is required" });
|
||||
}
|
||||
if (!data.accessKey?.trim()) {
|
||||
issues.push({ field: "accessKey", message: "Username is required" });
|
||||
}
|
||||
if (data.region?.trim()) {
|
||||
const port = Number(data.region);
|
||||
if (!Number.isInteger(port) || port < 1 || port > 65535) {
|
||||
issues.push({
|
||||
field: "region",
|
||||
message: "Port must be an integer between 1 and 65535",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (provider === RCLONE_DESTINATION_PROVIDERS.FTP) {
|
||||
const { implicitTlsEnabled, explicitTlsEnabled } = getFtpTlsState(flags);
|
||||
if (!implicitTlsEnabled && !explicitTlsEnabled) {
|
||||
issues.push({ field: "additionalFlags", message: FTP_TLS_REQUIRED_ERROR });
|
||||
}
|
||||
if (implicitTlsEnabled && explicitTlsEnabled) {
|
||||
issues.push({ field: "additionalFlags", message: FTP_TLS_CONFLICT_ERROR });
|
||||
}
|
||||
if (hasDisabledFtpCertificateVerification(flags)) {
|
||||
issues.push({
|
||||
field: "additionalFlags",
|
||||
message: FTP_CERTIFICATE_VERIFICATION_REQUIRED_ERROR,
|
||||
});
|
||||
}
|
||||
} else if (!hasSftpHostKeyVerification(flags)) {
|
||||
issues.push({ field: "additionalFlags", message: SFTP_HOST_KEY_REQUIRED_ERROR });
|
||||
}
|
||||
|
||||
return issues;
|
||||
};
|
||||
|
||||
@ -9,10 +9,11 @@ import { findEnvironmentById } from "@dokploy/server/services/environment";
|
||||
import { findProjectById } from "@dokploy/server/services/project";
|
||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getSafeRcloneErrorMessage } from "./redact";
|
||||
import {
|
||||
getBackupCommand,
|
||||
getBackupTimestamp,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "./utils";
|
||||
|
||||
@ -35,8 +36,8 @@ export const runComposeBackup = async (
|
||||
});
|
||||
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, bucketDestination);
|
||||
const backupCommand = getBackupCommand(
|
||||
backup,
|
||||
rcloneFlags,
|
||||
@ -62,20 +63,20 @@ export const runComposeBackup = async (
|
||||
|
||||
await updateDeploymentStatus(deployment.deploymentId, "done");
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
console.error("Backup error:", safeErrorMessage);
|
||||
await sendDatabaseBackupNotifications({
|
||||
applicationName: name,
|
||||
projectName: project.name,
|
||||
databaseType: getDatabaseType(databaseType),
|
||||
type: "error",
|
||||
// @ts-ignore
|
||||
errorMessage: error?.message || "Error message not provided",
|
||||
errorMessage: safeErrorMessage,
|
||||
organizationId: project.organizationId,
|
||||
databaseName: backup.database,
|
||||
});
|
||||
|
||||
await updateDeploymentStatus(deployment.deploymentId, "error");
|
||||
throw error;
|
||||
throw new Error(safeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -6,13 +6,18 @@ import { getAllServers } from "@dokploy/server/services/server";
|
||||
import { getWebServerSettings } from "@dokploy/server/services/web-server-settings";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { scheduleJob } from "node-schedule";
|
||||
import { quote } from "shell-quote";
|
||||
import { db } from "../../db/index";
|
||||
import { startLogCleanup } from "../access-log/handler";
|
||||
import { cleanupAll } from "../docker/utils";
|
||||
import { sendDockerCleanupNotifications } from "../notifications/docker-cleanup";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { redactRcloneCredentials } from "./redact";
|
||||
import { getS3Credentials, normalizeS3Path, scheduleBackup } from "./utils";
|
||||
import {
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
scheduleBackup,
|
||||
} from "./utils";
|
||||
|
||||
export const initCronJobs = async () => {
|
||||
console.log("Setting up cron jobs....");
|
||||
@ -134,17 +139,19 @@ export const keepLatestNBackups = async (
|
||||
|
||||
try {
|
||||
const destination = await findDestinationById(backup.destinationId);
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const appName = getServiceAppName(backup);
|
||||
const backupFilesPath = `:s3:${destination.bucket}/${appName}/${normalizeS3Path(backup.prefix)}`;
|
||||
const { flags: rcloneFlags, path: backupFilesPath } =
|
||||
await getRclonePathAndFlags(
|
||||
destination,
|
||||
`${appName}/${normalizeS3Path(backup.prefix)}`,
|
||||
);
|
||||
|
||||
// --include "*.bson.gz" or "*.sql.gz" or "*.zip" ensures nothing else other than the dokploy backup files are touched by rclone
|
||||
const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${backupFilesPath}`;
|
||||
const rcloneList = `rclone lsf ${rcloneFlags.join(" ")} --include "*${backup.databaseType === "web-server" ? ".zip" : ".{sql.gz,bson.gz}"}" ${quote([backupFilesPath])}`;
|
||||
// when we pipe the above command with this one, we only get the list of files we want to delete
|
||||
const sortAndPickUnwantedBackups = `sort -r | tail -n +$((${backup.keepLatestCount}+1)) | xargs -I{}`;
|
||||
// this command deletes the files
|
||||
// to test the deletion before actually deleting we can add --dry-run before ${backupFilesPath}{}
|
||||
const rcloneDelete = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`;
|
||||
const rcloneDelete = `rclone deletefile ${rcloneFlags.join(" ")} ${quote([`${backupFilesPath}/{}`])}`;
|
||||
|
||||
const rcloneCommand = `${rcloneList} | ${sortAndPickUnwantedBackups} ${rcloneDelete}`;
|
||||
|
||||
|
||||
@ -9,10 +9,11 @@ import type { Libsql } from "@dokploy/server/services/libsql";
|
||||
import { findProjectById } from "@dokploy/server/services/project";
|
||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getSafeRcloneErrorMessage } from "./redact";
|
||||
import {
|
||||
getBackupCommand,
|
||||
getBackupTimestamp,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "./utils";
|
||||
|
||||
@ -34,8 +35,8 @@ export const runLibsqlBackup = async (
|
||||
const backupFileName = `${getBackupTimestamp()}.sql.gz`;
|
||||
const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`;
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, bucketDestination);
|
||||
const backupCommand = getBackupCommand(
|
||||
backup,
|
||||
rcloneFlags,
|
||||
@ -61,19 +62,19 @@ export const runLibsqlBackup = async (
|
||||
|
||||
await updateDeploymentStatus(deployment.deploymentId, "done");
|
||||
} catch (error) {
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
await sendDatabaseBackupNotifications({
|
||||
applicationName: name,
|
||||
projectName: project.name,
|
||||
databaseType: "libsql",
|
||||
type: "error",
|
||||
// @ts-ignore
|
||||
errorMessage: error?.message || "Error message not provided",
|
||||
errorMessage: safeErrorMessage,
|
||||
organizationId: project.organizationId,
|
||||
databaseName: backup.database,
|
||||
});
|
||||
|
||||
await updateDeploymentStatus(deployment.deploymentId, "error");
|
||||
|
||||
throw error;
|
||||
throw new Error(safeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
@ -9,10 +9,11 @@ import type { Mariadb } from "@dokploy/server/services/mariadb";
|
||||
import { findProjectById } from "@dokploy/server/services/project";
|
||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getSafeRcloneErrorMessage } from "./redact";
|
||||
import {
|
||||
getBackupCommand,
|
||||
getBackupTimestamp,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "./utils";
|
||||
|
||||
@ -33,8 +34,8 @@ export const runMariadbBackup = async (
|
||||
description: "MariaDB Backup",
|
||||
});
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, bucketDestination);
|
||||
const backupCommand = getBackupCommand(
|
||||
backup,
|
||||
rcloneFlags,
|
||||
@ -59,18 +60,18 @@ export const runMariadbBackup = async (
|
||||
});
|
||||
await updateDeploymentStatus(deployment.deploymentId, "done");
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
console.error("Backup error:", safeErrorMessage);
|
||||
await sendDatabaseBackupNotifications({
|
||||
applicationName: name,
|
||||
projectName: project.name,
|
||||
databaseType: "mariadb",
|
||||
type: "error",
|
||||
// @ts-ignore
|
||||
errorMessage: error?.message || "Error message not provided",
|
||||
errorMessage: safeErrorMessage,
|
||||
organizationId: project.organizationId,
|
||||
databaseName: backup.database,
|
||||
});
|
||||
await updateDeploymentStatus(deployment.deploymentId, "error");
|
||||
throw error;
|
||||
throw new Error(safeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
@ -9,10 +9,11 @@ import type { Mongo } from "@dokploy/server/services/mongo";
|
||||
import { findProjectById } from "@dokploy/server/services/project";
|
||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getSafeRcloneErrorMessage } from "./redact";
|
||||
import {
|
||||
getBackupCommand,
|
||||
getBackupTimestamp,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "./utils";
|
||||
|
||||
@ -30,8 +31,8 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
|
||||
description: "MongoDB Backup",
|
||||
});
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, bucketDestination);
|
||||
const backupCommand = getBackupCommand(
|
||||
backup,
|
||||
rcloneFlags,
|
||||
@ -57,18 +58,18 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => {
|
||||
});
|
||||
await updateDeploymentStatus(deployment.deploymentId, "done");
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
console.error("Backup error:", safeErrorMessage);
|
||||
await sendDatabaseBackupNotifications({
|
||||
applicationName: name,
|
||||
projectName: project.name,
|
||||
databaseType: "mongodb",
|
||||
type: "error",
|
||||
// @ts-ignore
|
||||
errorMessage: error?.message || "Error message not provided",
|
||||
errorMessage: safeErrorMessage,
|
||||
organizationId: project.organizationId,
|
||||
databaseName: backup.database,
|
||||
});
|
||||
await updateDeploymentStatus(deployment.deploymentId, "error");
|
||||
throw error;
|
||||
throw new Error(safeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
@ -9,10 +9,11 @@ import type { MySql } from "@dokploy/server/services/mysql";
|
||||
import { findProjectById } from "@dokploy/server/services/project";
|
||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getSafeRcloneErrorMessage } from "./redact";
|
||||
import {
|
||||
getBackupCommand,
|
||||
getBackupTimestamp,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "./utils";
|
||||
|
||||
@ -31,8 +32,8 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
|
||||
});
|
||||
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, bucketDestination);
|
||||
const backupCommand = getBackupCommand(
|
||||
backup,
|
||||
rcloneFlags,
|
||||
@ -57,18 +58,18 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => {
|
||||
});
|
||||
await updateDeploymentStatus(deployment.deploymentId, "done");
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
console.error("Backup error:", safeErrorMessage);
|
||||
await sendDatabaseBackupNotifications({
|
||||
applicationName: name,
|
||||
projectName: project.name,
|
||||
databaseType: "mysql",
|
||||
type: "error",
|
||||
// @ts-ignore
|
||||
errorMessage: error?.message || "Error message not provided",
|
||||
errorMessage: safeErrorMessage,
|
||||
organizationId: project.organizationId,
|
||||
databaseName: backup.database,
|
||||
});
|
||||
await updateDeploymentStatus(deployment.deploymentId, "error");
|
||||
throw error;
|
||||
throw new Error(safeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
@ -9,10 +9,11 @@ import type { Postgres } from "@dokploy/server/services/postgres";
|
||||
import { findProjectById } from "@dokploy/server/services/project";
|
||||
import { sendDatabaseBackupNotifications } from "../notifications/database-backup";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getSafeRcloneErrorMessage } from "./redact";
|
||||
import {
|
||||
getBackupCommand,
|
||||
getBackupTimestamp,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "./utils";
|
||||
|
||||
@ -34,8 +35,8 @@ export const runPostgresBackup = async (
|
||||
const backupFileName = `${getBackupTimestamp()}.sql.gz`;
|
||||
const bucketDestination = `${appName}/${normalizeS3Path(prefix)}${backupFileName}`;
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, bucketDestination);
|
||||
const backupCommand = getBackupCommand(
|
||||
backup,
|
||||
rcloneFlags,
|
||||
@ -61,20 +62,20 @@ export const runPostgresBackup = async (
|
||||
|
||||
await updateDeploymentStatus(deployment.deploymentId, "done");
|
||||
} catch (error) {
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
await sendDatabaseBackupNotifications({
|
||||
applicationName: name,
|
||||
projectName: project.name,
|
||||
databaseType: "postgres",
|
||||
type: "error",
|
||||
// @ts-ignore
|
||||
errorMessage: error?.message || "Error message not provided",
|
||||
errorMessage: safeErrorMessage,
|
||||
organizationId: project.organizationId,
|
||||
databaseName: backup.database,
|
||||
});
|
||||
|
||||
await updateDeploymentStatus(deployment.deploymentId, "error");
|
||||
|
||||
throw error;
|
||||
throw new Error(safeErrorMessage);
|
||||
} finally {
|
||||
}
|
||||
};
|
||||
|
||||
@ -1,12 +1,16 @@
|
||||
/**
|
||||
* Redacts S3 credentials from rclone command strings.
|
||||
*
|
||||
* Used to prevent credential leakage in structured logs and error output.
|
||||
* Matches the flag format produced by `getS3Credentials()`:
|
||||
* --s3-access-key-id="VALUE" and --s3-secret-access-key="VALUE"
|
||||
* Redacts credentials from rclone command strings before they reach logs or
|
||||
* user-facing error output. Handles both the existing S3 flags and the
|
||||
* provider-specific FTP/SFTP credential flags used by backup destinations.
|
||||
*/
|
||||
export const redactRcloneCredentials = (command: string): string => {
|
||||
return command
|
||||
.replace(/(--s3-access-key-id=)"[^"]*"/g, '$1"[REDACTED]"')
|
||||
.replace(/(--s3-secret-access-key=)"[^"]*"/g, '$1"[REDACTED]"');
|
||||
return command.replace(
|
||||
/(--(?:s3-access-key-id|s3-secret-access-key|ftp-pass|sftp-pass|sftp-key-file-pass)=)(?:(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\\[^\r\n]|[^\s"'\\])+)/g,
|
||||
'$1"[REDACTED]"',
|
||||
);
|
||||
};
|
||||
|
||||
export const getSafeRcloneErrorMessage = (error: unknown): string =>
|
||||
redactRcloneCredentials(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
|
||||
@ -1,8 +1,18 @@
|
||||
import {
|
||||
ADDITIONAL_FLAG_ERROR,
|
||||
ADDITIONAL_FLAG_REGEX,
|
||||
getDestinationValidationIssues,
|
||||
getFtpTlsState,
|
||||
isNamedRcloneDestinationProvider,
|
||||
isRcloneDestinationProvider,
|
||||
RCLONE_DESTINATION_PROVIDERS,
|
||||
} from "@dokploy/server/db/validations/destination";
|
||||
import { logger } from "@dokploy/server/lib/logger";
|
||||
import type { BackupSchedule } from "@dokploy/server/services/backup";
|
||||
import type { Destination } from "@dokploy/server/services/destination";
|
||||
import { scheduledJobs, scheduleJob } from "node-schedule";
|
||||
import { quote } from "shell-quote";
|
||||
import { execFileAsync } from "../process/execAsync";
|
||||
import { keepLatestNBackups } from ".";
|
||||
import { runComposeBackup } from "./compose";
|
||||
import { runLibsqlBackup } from "./libsql";
|
||||
@ -68,6 +78,16 @@ export const normalizeS3Path = (prefix: string) => {
|
||||
return normalizedPrefix ? `${normalizedPrefix}/` : "";
|
||||
};
|
||||
|
||||
const getValidatedAdditionalFlags = (destination: Destination): string[] => {
|
||||
const flags = destination.additionalFlags ?? [];
|
||||
for (const flag of flags) {
|
||||
if (!ADDITIONAL_FLAG_REGEX.test(flag)) {
|
||||
throw new Error(ADDITIONAL_FLAG_ERROR);
|
||||
}
|
||||
}
|
||||
return flags;
|
||||
};
|
||||
|
||||
export const getS3Credentials = (destination: Destination) => {
|
||||
const { accessKey, secretAccessKey, region, endpoint, provider } =
|
||||
destination;
|
||||
@ -84,13 +104,102 @@ export const getS3Credentials = (destination: Destination) => {
|
||||
rcloneFlags.unshift(`--s3-provider=${quote([provider])}`);
|
||||
}
|
||||
|
||||
if (destination.additionalFlags?.length) {
|
||||
rcloneFlags.push(...destination.additionalFlags);
|
||||
}
|
||||
rcloneFlags.push(...getValidatedAdditionalFlags(destination));
|
||||
|
||||
return rcloneFlags;
|
||||
};
|
||||
|
||||
const trimRclonePath = (value: string) =>
|
||||
value.trim().replace(/^\/+|\/+$/g, "");
|
||||
|
||||
const joinRclonePath = (...parts: string[]) =>
|
||||
parts.map(trimRclonePath).filter(Boolean).join("/");
|
||||
|
||||
export const assertSafeRclonePath = (value: string) => {
|
||||
const normalizedSeparators = value.replace(/\\/g, "/");
|
||||
if (/[\0\r\n]/.test(normalizedSeparators)) {
|
||||
throw new Error("Invalid rclone path");
|
||||
}
|
||||
const segments = normalizedSeparators.split("/");
|
||||
if (segments.some((segment) => segment === "." || segment === "..")) {
|
||||
throw new Error("Invalid rclone path");
|
||||
}
|
||||
};
|
||||
|
||||
const obscureRclonePassword = async (password: string) => {
|
||||
if (!password) return "";
|
||||
const { stdout } = await execFileAsync("rclone", ["obscure", "-"], {
|
||||
input: password,
|
||||
});
|
||||
return stdout.trim();
|
||||
};
|
||||
|
||||
export const getRclonePathAndFlags = async (
|
||||
destination: Destination,
|
||||
path = "",
|
||||
): Promise<{ flags: string[]; path: string }> => {
|
||||
assertSafeRclonePath(path);
|
||||
const provider = destination.provider;
|
||||
const additionalFlags = getValidatedAdditionalFlags(destination);
|
||||
|
||||
if (isRcloneDestinationProvider(provider)) {
|
||||
const [issue] = getDestinationValidationIssues(destination);
|
||||
if (issue) throw new Error(issue.message);
|
||||
}
|
||||
|
||||
if (isNamedRcloneDestinationProvider(provider)) {
|
||||
const remotePath = joinRclonePath(destination.bucket, path);
|
||||
return {
|
||||
flags: additionalFlags,
|
||||
path: `${destination.endpoint.trim()}:${remotePath}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
provider === RCLONE_DESTINATION_PROVIDERS.FTP ||
|
||||
provider === RCLONE_DESTINATION_PROVIDERS.SFTP
|
||||
) {
|
||||
const backend =
|
||||
provider === RCLONE_DESTINATION_PROVIDERS.FTP ? "ftp" : "sftp";
|
||||
const defaultPort =
|
||||
provider === RCLONE_DESTINATION_PROVIDERS.FTP
|
||||
? getFtpTlsState(additionalFlags).implicitTlsEnabled
|
||||
? "990"
|
||||
: "21"
|
||||
: "22";
|
||||
const port = destination.region.trim() || defaultPort;
|
||||
const flags = [
|
||||
`--${backend}-host=${quote([destination.endpoint.trim()])}`,
|
||||
`--${backend}-user=${quote([destination.accessKey])}`,
|
||||
`--${backend}-port=${quote([port])}`,
|
||||
];
|
||||
if (destination.secretAccessKey) {
|
||||
const obscuredPassword = await obscureRclonePassword(
|
||||
destination.secretAccessKey,
|
||||
);
|
||||
flags.push(`--${backend}-pass=${quote([obscuredPassword])}`);
|
||||
}
|
||||
flags.push(...additionalFlags);
|
||||
if (provider === RCLONE_DESTINATION_PROVIDERS.FTP) {
|
||||
// CLI options override RCLONE_* environment defaults. Keep TLS
|
||||
// certificate verification enabled on the execution host.
|
||||
flags.push(
|
||||
"--ftp-no-check-certificate=false",
|
||||
"--no-check-certificate=false",
|
||||
);
|
||||
}
|
||||
return {
|
||||
flags,
|
||||
path: `:${backend}:${joinRclonePath(destination.bucket, path)}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
flags: getS3Credentials(destination),
|
||||
path: `:s3:${joinRclonePath(destination.bucket, path)}`,
|
||||
};
|
||||
};
|
||||
|
||||
// User-controlled values (database name, user, password) are passed to the
|
||||
// container as environment variables via `docker exec -e VAR=<escaped>` and
|
||||
// referenced as "$VAR" inside the inner shell, so they never appear in the
|
||||
@ -265,8 +374,9 @@ export const getBackupCommand = (
|
||||
) => {
|
||||
const containerSearch = getContainerSearchCommand(backup);
|
||||
const backupCommand = generateBackupCommand(backup);
|
||||
const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
|
||||
const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} "${rcloneDestination}"`;
|
||||
const rcloneTarget = quote([rcloneDestination]);
|
||||
const rcloneCommand = `rclone rcat ${rcloneFlags.join(" ")} ${rcloneTarget}`;
|
||||
const rcloneDeleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} ${rcloneTarget}`;
|
||||
|
||||
logger.info(
|
||||
{
|
||||
@ -290,7 +400,7 @@ export const getBackupCommand = (
|
||||
fi;
|
||||
|
||||
echo "[$(date)] Container Up: $CONTAINER_ID" >> ${logPath};
|
||||
echo "[$(date)] Starting backup and upload to S3..." >> ${logPath};
|
||||
echo "[$(date)] Starting backup and upload to destination..." >> ${logPath};
|
||||
|
||||
UPLOAD_OUTPUT=$({ ${backupCommand} | ${rcloneCommand}; } 2>&1 >/dev/null) || {
|
||||
echo "[$(date)] ❌ Error: Backup failed" >> ${logPath};
|
||||
@ -299,7 +409,7 @@ export const getBackupCommand = (
|
||||
exit 1;
|
||||
};
|
||||
|
||||
echo "[$(date)] ✅ Backup uploaded to S3 successfully" >> ${logPath};
|
||||
echo "[$(date)] ✅ Backup uploaded successfully" >> ${logPath};
|
||||
echo "Backup done ✅" >> ${logPath};
|
||||
`;
|
||||
};
|
||||
};
|
||||
@ -13,10 +13,15 @@ import {
|
||||
updateDeploymentStatus,
|
||||
} from "@dokploy/server/services/deployment";
|
||||
import { findDestinationById } from "@dokploy/server/services/destination";
|
||||
import { quote } from "shell-quote";
|
||||
import { sendDokployBackupNotifications } from "../notifications/dokploy-backup";
|
||||
import { execAsync } from "../process/execAsync";
|
||||
import { redactRcloneCredentials } from "./redact";
|
||||
import { getBackupTimestamp, getS3Credentials, normalizeS3Path } from "./utils";
|
||||
import { getSafeRcloneErrorMessage, redactRcloneCredentials } from "./redact";
|
||||
import {
|
||||
getBackupTimestamp,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "./utils";
|
||||
|
||||
function formatBytes(bytes?: number) {
|
||||
if (bytes === undefined) return "Unknown size";
|
||||
@ -41,12 +46,13 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
|
||||
let computedBackupSize: number | undefined;
|
||||
try {
|
||||
const destination = await findDestinationById(backup.destinationId);
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const timestamp = getBackupTimestamp();
|
||||
const { BASE_PATH } = paths();
|
||||
const tempDir = await mkdtemp(join(tmpdir(), "dokploy-backup-"));
|
||||
const backupFileName = `webserver-backup-${timestamp}.zip`;
|
||||
const s3Path = `:s3:${destination.bucket}/${backup.appName}/${normalizeS3Path(backup.prefix)}${backupFileName}`;
|
||||
const destinationPath = `${backup.appName}/${normalizeS3Path(backup.prefix)}${backupFileName}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, destinationPath);
|
||||
|
||||
try {
|
||||
await execAsync(`mkdir -p ${tempDir}/filesystem`);
|
||||
@ -114,10 +120,10 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
|
||||
// If stat fails, keep undefined
|
||||
}
|
||||
|
||||
const uploadCommand = `rclone copyto ${rcloneFlags.join(" ")} "${zipPath}" "${s3Path}"`;
|
||||
writeStream.write("Running command to upload backup to S3\n");
|
||||
const uploadCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([zipPath])} ${quote([rcloneDestination])}`;
|
||||
writeStream.write("Running command to upload backup to destination\n");
|
||||
await execAsync(uploadCommand);
|
||||
writeStream.write("Uploaded backup to S3 ✅\n");
|
||||
writeStream.write("Uploaded backup to destination ✅\n");
|
||||
writeStream.end();
|
||||
await sendDokployBackupNotifications({
|
||||
type: "success",
|
||||
@ -136,9 +142,7 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const safeErrorMessage = redactRcloneCredentials(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
console.error("Backup error:", redactRcloneCredentials(String(error)));
|
||||
writeStream.write("Backup error❌\n");
|
||||
writeStream.write(`${safeErrorMessage}\n`);
|
||||
@ -149,6 +153,6 @@ export const runWebServerBackup = async (backup: BackupSchedule) => {
|
||||
backupSize: formatBytes(computedBackupSize),
|
||||
});
|
||||
await updateDeploymentStatus(deployment.deploymentId, "error");
|
||||
throw error;
|
||||
throw new Error(safeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
@ -3,7 +3,7 @@ import type { Compose } from "@dokploy/server/services/compose";
|
||||
import type { Destination } from "@dokploy/server/services/destination";
|
||||
import { quote } from "shell-quote";
|
||||
import type { z } from "zod";
|
||||
import { getS3Credentials } from "../backups/utils";
|
||||
import { getRclonePathAndFlags } from "../backups/utils";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getRestoreCommand } from "./utils";
|
||||
|
||||
@ -24,9 +24,8 @@ export const restoreComposeBackup = async (
|
||||
}
|
||||
const { serverId, appName, composeType } = compose;
|
||||
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
|
||||
const { flags: rcloneFlags, path: backupPath } =
|
||||
await getRclonePathAndFlags(destination, backupInput.backupFile);
|
||||
let rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
|
||||
|
||||
if (backupInput.metadata?.mongo) {
|
||||
|
||||
@ -3,7 +3,10 @@ import type { Destination } from "@dokploy/server/services/destination";
|
||||
import type { Libsql } from "@dokploy/server/services/libsql";
|
||||
import { quote } from "shell-quote";
|
||||
import type { z } from "zod";
|
||||
import { getS3Credentials, getServiceContainerCommand } from "../backups/utils";
|
||||
import {
|
||||
getRclonePathAndFlags,
|
||||
getServiceContainerCommand,
|
||||
} from "../backups/utils";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
|
||||
export const restoreLibsqlBackup = async (
|
||||
@ -15,11 +18,8 @@ export const restoreLibsqlBackup = async (
|
||||
try {
|
||||
const { appName, serverId } = libsql;
|
||||
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
|
||||
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
|
||||
|
||||
const { flags: rcloneFlags, path: backupPath } =
|
||||
await getRclonePathAndFlags(destination, backupInput.backupFile);
|
||||
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])}`;
|
||||
|
||||
const containerSearch = getServiceContainerCommand(appName);
|
||||
|
||||
@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
|
||||
import type { Mariadb } from "@dokploy/server/services/mariadb";
|
||||
import { quote } from "shell-quote";
|
||||
import type { z } from "zod";
|
||||
import { getS3Credentials } from "../backups/utils";
|
||||
import { getRclonePathAndFlags } from "../backups/utils";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getRestoreCommand } from "./utils";
|
||||
|
||||
@ -16,10 +16,8 @@ export const restoreMariadbBackup = async (
|
||||
try {
|
||||
const { appName, serverId, databaseUser, databasePassword } = mariadb;
|
||||
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
|
||||
|
||||
const { flags: rcloneFlags, path: backupPath } =
|
||||
await getRclonePathAndFlags(destination, backupInput.backupFile);
|
||||
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
|
||||
|
||||
const command = getRestoreCommand({
|
||||
|
||||
@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
|
||||
import type { Mongo } from "@dokploy/server/services/mongo";
|
||||
import { quote } from "shell-quote";
|
||||
import type { z } from "zod";
|
||||
import { getS3Credentials } from "../backups/utils";
|
||||
import { getRclonePathAndFlags } from "../backups/utils";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getRestoreCommand } from "./utils";
|
||||
|
||||
@ -16,9 +16,8 @@ export const restoreMongoBackup = async (
|
||||
try {
|
||||
const { appName, databasePassword, databaseUser, serverId } = mongo;
|
||||
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
|
||||
const { flags: rcloneFlags, path: backupPath } =
|
||||
await getRclonePathAndFlags(destination, backupInput.backupFile);
|
||||
const rcloneCommand = `rclone copy ${rcloneFlags.join(" ")} ${quote([backupPath])}`;
|
||||
|
||||
const command = getRestoreCommand({
|
||||
|
||||
@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
|
||||
import type { MySql } from "@dokploy/server/services/mysql";
|
||||
import { quote } from "shell-quote";
|
||||
import type { z } from "zod";
|
||||
import { getS3Credentials } from "../backups/utils";
|
||||
import { getRclonePathAndFlags } from "../backups/utils";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getRestoreCommand } from "./utils";
|
||||
|
||||
@ -16,10 +16,8 @@ export const restoreMySqlBackup = async (
|
||||
try {
|
||||
const { appName, databaseRootPassword, serverId } = mysql;
|
||||
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
|
||||
|
||||
const { flags: rcloneFlags, path: backupPath } =
|
||||
await getRclonePathAndFlags(destination, backupInput.backupFile);
|
||||
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
|
||||
|
||||
const command = getRestoreCommand({
|
||||
|
||||
@ -3,7 +3,7 @@ import type { Destination } from "@dokploy/server/services/destination";
|
||||
import type { Postgres } from "@dokploy/server/services/postgres";
|
||||
import { quote } from "shell-quote";
|
||||
import type { z } from "zod";
|
||||
import { getS3Credentials } from "../backups/utils";
|
||||
import { getRclonePathAndFlags } from "../backups/utils";
|
||||
import { execAsync, execAsyncRemote } from "../process/execAsync";
|
||||
import { getRestoreCommand } from "./utils";
|
||||
|
||||
@ -16,11 +16,8 @@ export const restorePostgresBackup = async (
|
||||
try {
|
||||
const { appName, databaseUser, serverId } = postgres;
|
||||
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
|
||||
const backupPath = `${bucketPath}/${backupInput.backupFile}`;
|
||||
|
||||
const { flags: rcloneFlags, path: backupPath } =
|
||||
await getRclonePathAndFlags(destination, backupInput.backupFile);
|
||||
const rcloneCommand = `rclone cat ${rcloneFlags.join(" ")} ${quote([backupPath])} | gunzip`;
|
||||
|
||||
const command = getRestoreCommand({
|
||||
|
||||
@ -4,7 +4,8 @@ import { join } from "node:path";
|
||||
import { IS_CLOUD, paths } from "@dokploy/server/constants";
|
||||
import type { Destination } from "@dokploy/server/services/destination";
|
||||
import { quote } from "shell-quote";
|
||||
import { getS3Credentials } from "../backups/utils";
|
||||
import { getSafeRcloneErrorMessage } from "../backups/redact";
|
||||
import { getRclonePathAndFlags } from "../backups/utils";
|
||||
import { execAsync } from "../process/execAsync";
|
||||
|
||||
export const restoreWebServerBackup = async (
|
||||
@ -16,9 +17,8 @@ export const restoreWebServerBackup = async (
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
const backupPath = `${bucketPath}/${backupFile}`;
|
||||
const { flags: rcloneFlags, path: backupPath } =
|
||||
await getRclonePathAndFlags(destination, backupFile);
|
||||
const { BASE_PATH } = paths();
|
||||
|
||||
// Create a temporary directory outside of BASE_PATH
|
||||
@ -31,17 +31,19 @@ export const restoreWebServerBackup = async (
|
||||
|
||||
// Create temp directory
|
||||
emit("Creating temporary directory...");
|
||||
await execAsync(`mkdir -p ${tempDir}`);
|
||||
await execAsync(`mkdir -p ${quote([tempDir])}`);
|
||||
|
||||
// Download backup from S3
|
||||
emit("Downloading backup from S3...");
|
||||
// Download backup from configured destination
|
||||
emit("Downloading backup from destination...");
|
||||
await execAsync(
|
||||
`rclone copyto ${rcloneFlags.join(" ")} ${quote([backupPath])} ${quote([`${tempDir}/${backupFile}`])}`,
|
||||
);
|
||||
|
||||
// List files before extraction
|
||||
emit("Listing files before extraction...");
|
||||
const { stdout: beforeFiles } = await execAsync(`ls -la ${tempDir}`);
|
||||
const { stdout: beforeFiles } = await execAsync(
|
||||
`ls -la ${quote([tempDir])}`,
|
||||
);
|
||||
emit(`Files before extraction: ${beforeFiles}`);
|
||||
|
||||
// Extract backup
|
||||
@ -71,16 +73,18 @@ export const restoreWebServerBackup = async (
|
||||
|
||||
// Check if database.sql.gz exists and decompress it
|
||||
const { stdout: hasGzFile } = await execAsync(
|
||||
`ls ${tempDir}/database.sql.gz || true`,
|
||||
`ls ${quote([`${tempDir}/database.sql.gz`])} || true`,
|
||||
);
|
||||
if (hasGzFile.includes("database.sql.gz")) {
|
||||
emit("Found compressed database file, decompressing...");
|
||||
await execAsync(`cd ${tempDir} && gunzip database.sql.gz`);
|
||||
await execAsync(
|
||||
`cd ${quote([tempDir])} && gunzip ${quote(["database.sql.gz"])}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Verify database file exists
|
||||
const { stdout: hasSqlFile } = await execAsync(
|
||||
`ls ${tempDir}/database.sql || true`,
|
||||
`ls ${quote([`${tempDir}/database.sql`])} || true`,
|
||||
);
|
||||
if (!hasSqlFile.includes("database.sql")) {
|
||||
throw new Error("Database file not found after extraction");
|
||||
@ -115,7 +119,7 @@ export const restoreWebServerBackup = async (
|
||||
// Copy the backup file into the container
|
||||
emit("Copying backup file into container...");
|
||||
await execAsync(
|
||||
`docker cp ${tempDir}/database.sql ${postgresContainerId}:/tmp/database.sql`,
|
||||
`docker cp ${quote([`${tempDir}/database.sql`])} ${postgresContainerId}:/tmp/database.sql`,
|
||||
);
|
||||
|
||||
// Verify file in container
|
||||
@ -140,17 +144,12 @@ export const restoreWebServerBackup = async (
|
||||
} finally {
|
||||
// Cleanup
|
||||
emit("Cleaning up temporary files...");
|
||||
await execAsync(`rm -rf ${tempDir}`);
|
||||
await execAsync(`rm -rf ${quote([tempDir])}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
emit(
|
||||
`Error: ${
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: "Error restoring web server backup"
|
||||
}`,
|
||||
);
|
||||
throw error;
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
console.error("Restore error:", safeErrorMessage);
|
||||
emit(`Error: ${safeErrorMessage}`);
|
||||
throw new Error(safeErrorMessage);
|
||||
}
|
||||
};
|
||||
|
||||
@ -3,11 +3,16 @@ import { paths } from "@dokploy/server/constants";
|
||||
import { findComposeById } from "@dokploy/server/services/compose";
|
||||
import { findDestinationById } from "@dokploy/server/services/destination";
|
||||
import type { findVolumeBackupById } from "@dokploy/server/services/volume-backups";
|
||||
import { quote } from "shell-quote";
|
||||
import {
|
||||
getBackupTimestamp,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
normalizeS3Path,
|
||||
} from "../backups/utils";
|
||||
import {
|
||||
normalizeDockerVolumeName,
|
||||
normalizeVolumeBackupFilePath,
|
||||
} from "./restore";
|
||||
|
||||
interface RestartSafeBackupCommandOptions {
|
||||
stopCommand: string;
|
||||
@ -73,36 +78,40 @@ export const backupVolume = async (
|
||||
const serverId =
|
||||
volumeBackup.application?.serverId || volumeBackup.compose?.serverId;
|
||||
const { VOLUME_BACKUPS_PATH, VOLUME_BACKUP_LOCK_PATH } = paths(!!serverId);
|
||||
const s3AppName = getVolumeServiceAppName(volumeBackup);
|
||||
const backupFileName = `${volumeName}-${getBackupTimestamp()}.tar`;
|
||||
const bucketDestination = `${s3AppName}/${normalizeS3Path(prefix || "")}${backupFileName}`;
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const rcloneDestination = `:s3:${destination.bucket}/${bucketDestination}`;
|
||||
const appName = getVolumeServiceAppName(volumeBackup);
|
||||
const safeVolumeName = normalizeDockerVolumeName(volumeName);
|
||||
const backupFileName = normalizeVolumeBackupFilePath(
|
||||
`${safeVolumeName}-${getBackupTimestamp()}.tar`,
|
||||
);
|
||||
const destinationPath = `${appName}/${normalizeS3Path(prefix || "")}${backupFileName}`;
|
||||
const { flags: rcloneFlags, path: rcloneDestination } =
|
||||
await getRclonePathAndFlags(destination, destinationPath);
|
||||
const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeBackup.appName);
|
||||
const localBackupPath = `${volumeBackupPath}/${backupFileName}`;
|
||||
|
||||
const rcloneCommand = `rclone copyto ${rcloneFlags.join(" ")} "${volumeBackupPath}/${backupFileName}" "${rcloneDestination}"`;
|
||||
const rcloneCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([localBackupPath])} ${quote([rcloneDestination])}`;
|
||||
|
||||
const backupCommand = `
|
||||
set -e
|
||||
echo "Volume name: ${volumeName}"
|
||||
echo "Backup file name: ${backupFileName}"
|
||||
echo "Volume name:" ${quote([safeVolumeName])}
|
||||
echo "Backup file name:" ${quote([backupFileName])}
|
||||
echo "Turning off volume backup: ${turnOff ? "Yes" : "No"}"
|
||||
echo "Starting volume backup"
|
||||
echo "Dir: ${volumeBackupPath}"
|
||||
docker run --rm \
|
||||
-v ${volumeName}:/volume_data \
|
||||
-v ${volumeBackupPath}:/backup \
|
||||
-v ${quote([safeVolumeName])}:/volume_data \
|
||||
-v ${quote([volumeBackupPath])}:/backup \
|
||||
ubuntu \
|
||||
bash -c "cd /volume_data && tar cvf /backup/${backupFileName} ."
|
||||
bash -c 'cd /volume_data && tar cvf "/backup/$1" .' -- ${quote([backupFileName])}
|
||||
echo "Volume backup done ✅"
|
||||
`;
|
||||
|
||||
const uploadCommand = `
|
||||
echo "Starting upload to S3..."
|
||||
echo "Starting upload to destination..."
|
||||
${rcloneCommand}
|
||||
echo "Upload to S3 done ✅"
|
||||
echo "Upload to destination done ✅"
|
||||
echo "Cleaning up local backup file..."
|
||||
rm "${volumeBackupPath}/${backupFileName}"
|
||||
rm ${quote([localBackupPath])}
|
||||
echo "Local backup file cleaned up ✅"
|
||||
`;
|
||||
|
||||
@ -123,7 +132,7 @@ export const backupVolume = async (
|
||||
const lockWrapper = (body: string) => `
|
||||
set -e
|
||||
|
||||
LOCK_PATH="${lockPath}"
|
||||
LOCK_PATH=${quote([lockPath])}
|
||||
|
||||
echo "Waiting for volume backup lock: $LOCK_PATH"
|
||||
|
||||
|
||||
@ -4,10 +4,39 @@ import {
|
||||
findApplicationById,
|
||||
findComposeById,
|
||||
findDestinationById,
|
||||
getS3Credentials,
|
||||
getRclonePathAndFlags,
|
||||
paths,
|
||||
} from "../..";
|
||||
|
||||
const UNSAFE_BACKUP_PATH_CHARS = /[\0\r\n;&|`$<>]/;
|
||||
|
||||
export const normalizeDockerVolumeName = (value: string) => {
|
||||
const normalized = value.trim();
|
||||
if (!normalized || !/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(normalized)) {
|
||||
throw new Error("Invalid docker volume name");
|
||||
}
|
||||
return normalized;
|
||||
};
|
||||
|
||||
export const normalizeVolumeBackupFilePath = (value: string) => {
|
||||
const normalized = value.trim().replace(/\\/g, "/");
|
||||
if (
|
||||
!normalized ||
|
||||
normalized.startsWith("/") ||
|
||||
normalized.endsWith("/") ||
|
||||
UNSAFE_BACKUP_PATH_CHARS.test(normalized)
|
||||
) {
|
||||
throw new Error("Invalid volume backup file path");
|
||||
}
|
||||
const segments = normalized.split("/");
|
||||
if (
|
||||
segments.some((segment) => !segment || segment === "." || segment === "..")
|
||||
) {
|
||||
throw new Error("Invalid volume backup file path");
|
||||
}
|
||||
return segments.join("/");
|
||||
};
|
||||
|
||||
export const restoreVolume = async (
|
||||
id: string,
|
||||
destinationId: string,
|
||||
@ -18,37 +47,45 @@ export const restoreVolume = async (
|
||||
) => {
|
||||
const destination = await findDestinationById(destinationId);
|
||||
const { VOLUME_BACKUPS_PATH } = paths(!!serverId);
|
||||
const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, volumeName);
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const bucketPath = `:s3:${destination.bucket}`;
|
||||
const backupPath = `${bucketPath}/${backupFileName}`;
|
||||
const safeVolumeName = normalizeDockerVolumeName(volumeName);
|
||||
const volumeBackupPath = path.join(VOLUME_BACKUPS_PATH, safeVolumeName);
|
||||
const safeBackupFileName = normalizeVolumeBackupFilePath(backupFileName);
|
||||
const { flags: rcloneFlags, path: backupPath } = await getRclonePathAndFlags(
|
||||
destination,
|
||||
safeBackupFileName,
|
||||
);
|
||||
const localBackupPath = path.join(
|
||||
volumeBackupPath,
|
||||
...safeBackupFileName.split("/"),
|
||||
);
|
||||
const localBackupDirectory = path.dirname(localBackupPath);
|
||||
|
||||
// Command to download backup file from S3
|
||||
const downloadCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([backupPath])} ${quote([`${volumeBackupPath}/${backupFileName}`])}`;
|
||||
// Command to download backup file from the configured destination
|
||||
const downloadCommand = `rclone copyto ${rcloneFlags.join(" ")} ${quote([backupPath])} ${quote([localBackupPath])}`;
|
||||
|
||||
// Base restore command that creates the volume and restores data
|
||||
const baseRestoreCommand = `
|
||||
set -e
|
||||
echo "Volume name: ${volumeName}"
|
||||
echo "Backup file name: ${backupFileName}"
|
||||
echo "Volume name: ${safeVolumeName}"
|
||||
echo "Backup file name:" ${quote([safeBackupFileName])}
|
||||
echo "Volume backup path: ${volumeBackupPath}"
|
||||
echo "Downloading backup from S3..."
|
||||
mkdir -p ${volumeBackupPath}
|
||||
echo "Downloading backup from destination..."
|
||||
mkdir -p ${quote([localBackupDirectory])}
|
||||
${downloadCommand}
|
||||
echo "Download completed ✅"
|
||||
echo "Creating new volume and restoring data..."
|
||||
docker run --rm \
|
||||
-v ${volumeName}:/volume_data \
|
||||
-v ${volumeBackupPath}:/backup \
|
||||
-v ${quote([safeVolumeName])}:/volume_data \
|
||||
-v ${quote([volumeBackupPath])}:/backup \
|
||||
ubuntu \
|
||||
bash -c "cd /volume_data && tar xvf /backup/${quote([backupFileName])} ."
|
||||
bash -c 'cd /volume_data && tar xvf "/backup/$1" .' -- ${quote([safeBackupFileName])}
|
||||
echo "Volume restore completed ✅"
|
||||
`;
|
||||
|
||||
// Function to check if volume exists and get containers using it
|
||||
const checkVolumeCommand = `
|
||||
# Check if volume exists
|
||||
VOLUME_EXISTS=$(docker volume ls -q --filter name="^${volumeName}$" | wc -l)
|
||||
VOLUME_EXISTS=$(docker volume ls -q --filter name="^${safeVolumeName}$" | wc -l)
|
||||
echo "Volume exists: $VOLUME_EXISTS"
|
||||
|
||||
if [ "$VOLUME_EXISTS" = "0" ]; then
|
||||
@ -58,18 +95,18 @@ export const restoreVolume = async (
|
||||
echo "Volume exists, checking for containers using it (including stopped ones)..."
|
||||
|
||||
# Get ALL containers (running and stopped) using this volume - much simpler with native filter!
|
||||
CONTAINERS_USING_VOLUME=$(docker ps -a --filter "volume=${volumeName}" --format "{{.ID}}|{{.Names}}|{{.State}}|{{.Labels}}")
|
||||
CONTAINERS_USING_VOLUME=$(docker ps -a --filter "volume=${safeVolumeName}" --format "{{.ID}}|{{.Names}}|{{.State}}|{{.Labels}}")
|
||||
|
||||
if [ -z "$CONTAINERS_USING_VOLUME" ]; then
|
||||
echo "Volume exists but no containers are using it"
|
||||
echo "Removing existing volume and proceeding with restore"
|
||||
docker volume rm ${volumeName} --force
|
||||
docker volume rm ${quote([safeVolumeName])} --force
|
||||
${baseRestoreCommand}
|
||||
else
|
||||
echo ""
|
||||
echo "⚠️ WARNING: Cannot restore volume as it is currently in use!"
|
||||
echo ""
|
||||
echo "📋 The following containers are using volume '${volumeName}':"
|
||||
echo "📋 The following containers are using volume '${safeVolumeName}':"
|
||||
echo ""
|
||||
|
||||
echo "$CONTAINERS_USING_VOLUME" | while IFS='|' read container_id container_name container_state labels; do
|
||||
@ -92,7 +129,7 @@ export const restoreVolume = async (
|
||||
echo ""
|
||||
echo "🔧 To restore this volume, please:"
|
||||
echo " 1. Stop all containers/services using this volume"
|
||||
echo " 2. Remove the existing volume: docker volume rm ${volumeName}"
|
||||
echo " 2. Remove the existing volume: docker volume rm ${safeVolumeName}"
|
||||
echo " 3. Run the restore operation again"
|
||||
echo ""
|
||||
echo "❌ Volume restore aborted - volume is in use"
|
||||
|
||||
@ -11,7 +11,9 @@ import {
|
||||
execAsyncRemote,
|
||||
} from "@dokploy/server/utils/process/execAsync";
|
||||
import { scheduledJobs, scheduleJob } from "node-schedule";
|
||||
import { getS3Credentials, normalizeS3Path } from "../backups/utils";
|
||||
import { quote } from "shell-quote";
|
||||
import { getSafeRcloneErrorMessage } from "../backups/redact";
|
||||
import { getRclonePathAndFlags, normalizeS3Path } from "../backups/utils";
|
||||
import { sendVolumeBackupNotifications } from "../notifications/volume-backup";
|
||||
import { backupVolume, getVolumeServiceAppName } from "./backup";
|
||||
|
||||
@ -84,12 +86,16 @@ const cleanupOldVolumeBackups = async (
|
||||
if (!keepLatestCount) return;
|
||||
|
||||
try {
|
||||
const rcloneFlags = getS3Credentials(destination);
|
||||
const s3AppName = getVolumeServiceAppName(volumeBackup);
|
||||
const backupFilesPath = `:s3:${destination.bucket}/${s3AppName}/${normalizeS3Path(prefix || "")}`;
|
||||
const listCommand = `rclone lsf ${rcloneFlags.join(" ")} --include \"${volumeName}-*.tar\" ${backupFilesPath}`;
|
||||
const appName = getVolumeServiceAppName(volumeBackup);
|
||||
const { flags: rcloneFlags, path: backupFilesPath } =
|
||||
await getRclonePathAndFlags(
|
||||
destination,
|
||||
`${appName}/${normalizeS3Path(prefix || "")}`,
|
||||
);
|
||||
const includePattern = `${volumeName}-*.tar`;
|
||||
const listCommand = `rclone lsf ${rcloneFlags.join(" ")} --include ${quote([includePattern])} ${quote([backupFilesPath])}`;
|
||||
const sortAndPick = `sort -r | tail -n +$((${keepLatestCount}+1)) | xargs -I{}`;
|
||||
const deleteCommand = `rclone delete ${rcloneFlags.join(" ")} ${backupFilesPath}{}`;
|
||||
const deleteCommand = `rclone deletefile ${rcloneFlags.join(" ")} ${quote([`${backupFilesPath}/{}`])}`;
|
||||
const fullCommand = `${listCommand} | ${sortAndPick} ${deleteCommand}`;
|
||||
|
||||
if (serverId) {
|
||||
@ -98,7 +104,10 @@ const cleanupOldVolumeBackups = async (
|
||||
await execAsync(fullCommand);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Volume backup retention error", error);
|
||||
console.error(
|
||||
"Volume backup retention error",
|
||||
getSafeRcloneErrorMessage(error),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@ -151,13 +160,15 @@ export const runVolumeBackup = async (volumeBackupId: string) => {
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
const safeErrorMessage = getSafeRcloneErrorMessage(error);
|
||||
console.error("Volume backup error:", safeErrorMessage);
|
||||
const { VOLUME_BACKUPS_PATH } = paths(!!serverId);
|
||||
const volumeBackupPath = path.join(
|
||||
VOLUME_BACKUPS_PATH,
|
||||
volumeBackup.appName,
|
||||
);
|
||||
// delete all the .tar files
|
||||
const command = `rm -rf ${volumeBackupPath}/*.tar`;
|
||||
// delete all the .tar files while keeping wildcard expansion intact
|
||||
const command = `rm -rf ${quote([volumeBackupPath])}/*.tar`;
|
||||
if (serverId) {
|
||||
await execAsyncRemote(serverId, command);
|
||||
} else {
|
||||
@ -179,7 +190,7 @@ export const runVolumeBackup = async (volumeBackupId: string) => {
|
||||
serviceType: mappedServiceType,
|
||||
type: "error",
|
||||
organizationId,
|
||||
errorMessage: error instanceof Error ? error.message : String(error),
|
||||
errorMessage: safeErrorMessage,
|
||||
});
|
||||
} catch (notificationError) {
|
||||
console.error(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user