feat/implemented manual backups to s3 storage

package.json

@@ -16,6 +16,7 @@
     "test": "jest"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.717.0",
     "@hookform/resolvers": "^3.9.0",
     "@kubernetes/client-node": "^0.22.2",
     "@next-auth/prisma-adapter": "^1.0.7",

src/app/project/app/[appId]/volumes/actions.ts

@@ -78,6 +78,9 @@ export const deleteFileMount = async (fileMountId: string) =>
 export const saveBackupVolume = async (prevState: any, inputData: VolumeBackupEditModel) =>
     saveFormAction(inputData, volumeBackupEditZodModel, async (validatedData) => {
         await getAuthUserSession();
+        if (validatedData.retention < 1) {
+            throw new ServiceException('Retention must be at least 1');
+        }
         await volumeBackupService.save({
             ...validatedData,
             id: validatedData.id ?? undefined,
@@ -90,4 +93,11 @@ export const deleteBackupVolume = async (backupVolumeId: string) =>
         await getAuthUserSession();
         await volumeBackupService.deleteById(backupVolumeId);
         return new SuccessActionResult(undefined, 'Successfully deleted backup schedule');
+    });
+
+export const runBackupVolumeSchedule = async (backupVolumeId: string) =>
+    simpleAction(async () => {
+        await getAuthUserSession();
+        await volumeBackupService.createBackupForVolume(backupVolumeId);
+        return new SuccessActionResult(undefined, 'Backup created and uploaded successfully');
     });

src/app/project/app/[appId]/volumes/volume-backup-edit-overlay.tsx

@@ -86,7 +86,7 @@ export default function VolumeBackupEditDialog({
           <Form {...form}>
             <form action={(e) => form.handleSubmit((data) => {
               return formAction(data);
-            })()}>
+            }, console.error)()}>
               <div className="space-y-4">
                 <FormField
                   control={form.control}

src/app/project/app/[appId]/volumes/volume-backup.tsx

@@ -4,11 +4,11 @@ import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle }
 import { AppExtendedModel } from "@/shared/model/app-extended.model";
 import { Table, TableBody, TableCaption, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
 import { Button } from "@/components/ui/button";
-import { EditIcon, TrashIcon } from "lucide-react";
+import { EditIcon, Play, TrashIcon } from "lucide-react";
 import { Toast } from "@/frontend/utils/toast.utils";
-import { deleteBackupVolume, deleteVolume } from "./actions";
+import { deleteBackupVolume, runBackupVolumeSchedule } from "./actions";
 import { useConfirmDialog } from "@/frontend/states/zustand.states";
-import { S3Target, VolumeBackup } from "@prisma/client";
+import { S3Target } from "@prisma/client";
 import React from "react";
 import { formatDateTime } from "@/frontend/utils/format.utils";
 import VolumeBackupEditDialog from "./volume-backup-edit-overlay";
@@ -25,6 +25,7 @@ export default function VolumeBackupList({
 }) {
 
     const { openConfirmDialog: openDialog } = useConfirmDialog();
+    const [isLoading, setIsLoading] = React.useState(false);
 
     const asyncDeleteBackupVolume = async (volumeId: string) => {
         const confirm = await openDialog({
@@ -37,6 +38,22 @@ export default function VolumeBackupList({
         }
     };
 
+    const asyncRunBackupVolumeSchedule = async (volumeId: string) => {
+        const confirm = await openDialog({
+            title: "Create Backup",
+            description: "Are you sure you want to create a backup now?",
+            okButton: "Create Backup"
+        });
+        setIsLoading(true);
+        try {
+            if (confirm) {
+                await Toast.fromAction(() => runBackupVolumeSchedule(volumeId), undefined, 'Creating backup...');
+            }
+        } finally {
+            setIsLoading(false);
+        }
+    };
+
     return <>
         <Card>
             <CardHeader>
@@ -63,11 +80,14 @@ export default function VolumeBackupList({
                                 <TableCell className="font-medium">{volumeBackup.target.name}</TableCell>
                                 <TableCell className="font-medium">{formatDateTime(volumeBackup.createdAt)}</TableCell>
                                 <TableCell className="font-medium flex gap-2">
+                                    <Button disabled={isLoading} variant="ghost" onClick={() => asyncRunBackupVolumeSchedule(volumeBackup.id)}>
+                                        <Play />
+                                    </Button>
                                     <VolumeBackupEditDialog volumeBackup={volumeBackup}
                                         s3Targets={s3Targets} volumes={app.appVolumes}>
-                                        <Button variant="ghost"><EditIcon /></Button>
+                                        <Button disabled={isLoading} variant="ghost"><EditIcon /></Button>
                                     </VolumeBackupEditDialog>
-                                    <Button variant="ghost" onClick={() => asyncDeleteBackupVolume(volumeBackup.id)}>
+                                    <Button disabled={isLoading} variant="ghost" onClick={() => asyncDeleteBackupVolume(volumeBackup.id)}>
                                         <TrashIcon />
                                     </Button>
                                 </TableCell>

src/app/settings/s3-targets/actions.ts

@@ -4,10 +4,21 @@ import { SuccessActionResult } from "@/shared/model/server-action-error-return.m
 import { getAuthUserSession, saveFormAction, simpleAction } from "@/server/utils/action-wrapper.utils";
 import { S3TargetEditModel, s3TargetEditZodModel } from "@/shared/model/s3-target-edit.model";
 import s3TargetService from "@/server/services/s3-target.service";
+import s3Service from "@/server/services/aws-s3.service";
+import { S3Target } from "@prisma/client";
+import { ServiceException } from "@/shared/model/service.exception.model";
 
 export const saveS3Target = async (prevState: any, inputData: S3TargetEditModel) =>
     saveFormAction(inputData, s3TargetEditZodModel, async (validatedData) => {
         await getAuthUserSession();
+
+        const url = new URL(validatedData.endpoint.includes('://') ? validatedData.endpoint : `https://${validatedData.endpoint}`);
+        validatedData.endpoint = url.hostname;
+
+        if (!await s3Service.testConnection(validatedData as S3Target)) {
+            throw new ServiceException('Could not connect to S3 Target, please check your credentials and try again');
+        }
+
         await s3TargetService.save({
             ...validatedData,
             id: validatedData.id ?? undefined,

src/frontend/utils/toast.utils.ts

@@ -2,7 +2,9 @@ import { ServerActionResult } from "@/shared/model/server-action-error-return.mo
 import { toast } from "sonner";
 
 export class Toast {
-    static async fromAction<A, B>(action: () => Promise<ServerActionResult<A, B>>, defaultSuccessMessage = 'Operation successful') {
+    static async fromAction<A, B>(action: () => Promise<ServerActionResult<A, B>>,
+        defaultSuccessMessage = 'Operation successful',
+        defaultLoadingMessage = 'loading...') {
 
         return new Promise<ServerActionResult<A, B>>(async (resolve, reject) => {
             toast.promise(async () => {
@@ -12,7 +14,7 @@ export class Toast {
                 }
                 return retVal;
             }, {
-                loading: 'loading...',
+                loading: defaultLoadingMessage,
                 success: (result: ServerActionResult<A, B>) => {
                     resolve(result);
                     return result.message ?? defaultSuccessMessage;

src/server/adapter/aws-s3.adapter.ts

@@ -0,0 +1,21 @@
+import { DeleteObjectCommand, HeadBucketCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+
+import { S3Target } from "@prisma/client";
+import { createReadStream } from "fs";
+
+class AwsS3Adapter {
+
+    getS3Client(s3Target: S3Target) {
+        return new S3Client({
+            region: s3Target.region,
+            credentials: {
+                accessKeyId: s3Target.accessKeyId,
+                secretAccessKey: s3Target.secretKey,
+            },
+            endpoint: `https://${s3Target.endpoint}`
+        });
+    }
+}
+
+const s3Adapter = new AwsS3Adapter();
+export default s3Adapter;

src/server/services/aws-s3.service.ts

@@ -0,0 +1,72 @@
+import { DeleteObjectCommand, HeadBucketCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { S3Target } from "@prisma/client";
+import s3Adapter from "../adapter/aws-s3.adapter";
+import { randomUUID } from "crypto";
+import { createReadStream } from "fs";
+
+
+export class S3Service {
+
+    async testConnection(target: S3Target) {
+        try {
+            const client = s3Adapter.getS3Client(target);
+            const output = await client.send(new HeadBucketCommand({ Bucket: target.bucketName }));
+            return output.$metadata.httpStatusCode === 200;
+        } catch (e) {
+            console.log('Error while testing connection to S3 Target', target, e);
+            return false;
+        }
+    }
+
+    async listFiles(s3Target: S3Target) {
+        const client = s3Adapter.getS3Client(s3Target);
+        const command = new ListObjectsV2Command({
+            Bucket: s3Target.bucketName,
+        });
+        const output = await client.send(command);
+        return output.Contents ?? [];
+    }
+
+    async deleteFile(s3Target: S3Target, fileName: string) {
+        const client = s3Adapter.getS3Client(s3Target);
+        const command = new DeleteObjectCommand({
+            Bucket: s3Target.bucketName,
+            Key: fileName,
+        });
+        await client.send(command);
+    }
+
+    async uploadFile(s3Target: S3Target,
+        inputFilePath: string,
+        fileName: string,
+        mimeType: string,
+        encoding: string) {
+
+        const client = s3Adapter.getS3Client(s3Target);
+
+        let fileEnding = fileName.split('.').pop();
+        if (!fileEnding) {
+            throw new Error(`Filename ${fileName} is invalid`);
+        }
+
+        const objectStorageFile = {
+            originalFilename: fileName,
+            mimeType,
+            encoding,
+            fileEnding,
+            fileName: fileName
+        };
+
+        const command = new PutObjectCommand({
+            Bucket: s3Target.bucketName, // todo: use bucket from env
+            Key: objectStorageFile.fileName,
+            Body: createReadStream(inputFilePath),
+            //ContentDisposition: 'inline',
+            ContentType: mimeType,
+        });
+        await client.send(command);
+    }
+}
+
+const s3Service = new S3Service();
+export default s3Service;

src/server/services/volume-backup.service.ts

@@ -3,9 +3,81 @@ import dataAccess from "../adapter/db.client";
 import { Tags } from "../utils/cache-tag-generator.utils";
 import { Prisma, VolumeBackup } from "@prisma/client";
 import { VolumeBackupExtendedModel } from "@/shared/model/volume-backup-extended.model";
+import podService from "./pod.service";
+import { ServiceException } from "@/shared/model/service.exception.model";
+import { PathUtils } from "../utils/path.utils";
+import { FsUtils } from "../utils/fs.utils";
+import s3Service from "./aws-s3.service";
 
 class VolumeBackupService {
 
+    async createBackupForVolume(backupVolumeId: string) {
+
+        const backupVolume = await dataAccess.client.volumeBackup.findFirstOrThrow({
+            where: {
+                id: backupVolumeId
+            },
+            include: {
+                volume: {
+                    include: {
+                        app: true
+                    }
+                },
+                target: true
+            }
+        });
+
+        const projectId = backupVolume.volume.app.projectId;
+        const appId = backupVolume.volume.app.id;
+        const volume = backupVolume.volume;
+
+        const pod = await podService.getPodsForApp(projectId, appId);
+        if (pod.length === 0) {
+            throw new ServiceException(`There are no running pods for volume id ${volume.id} in app ${volume.app.id}. Make sure the app is running.`);
+        }
+        const firstPod = pod[0];
+
+        // zipping and saving backup data in quickstack pod
+        const downloadPath = PathUtils.backupVolumeDownloadZipPath(backupVolume.id);
+        await FsUtils.createDirIfNotExistsAsync(PathUtils.tempBackupDataFolder, true);
+
+        try {
+            console.log(`Downloading data from pod ${firstPod.podName} ${volume.containerMountPath} to ${downloadPath}`);
+            await podService.cpFromPod(projectId, firstPod.podName, firstPod.containerName, volume.containerMountPath, downloadPath);
+
+            // uploac backup
+            console.log(`Uploading backup to S3`);
+            const now = new Date();
+            const nowString = now.toISOString();
+            await s3Service.uploadFile(backupVolume.target, downloadPath,
+                `${appId}/${volume.id}/${nowString}.tar.gz`, 'application/gzip', 'binary');
+
+
+            // delete files wich are nod needed anymore (by retention)
+            console.log(`Deleting old backups`);
+            const files = await s3Service.listFiles(backupVolume.target);
+
+            const filesFromThisBackup = files.filter(f => f.Key?.startsWith(`${appId}/${volume.id}/`)).map(f => ({
+                date: new Date((f.Key ?? '')
+                    .replace(`${appId}/${volume.id}/`, '')
+                    .replace('.tar.gz', '')),
+                key: f.Key
+            })).filter(f => !isNaN(f.date.getTime()) && !!f.key);
+            console.log(filesFromThisBackup)
+
+            filesFromThisBackup.sort((a, b) => a.date.getTime() - b.date.getTime());
+
+            const filesToDelete = filesFromThisBackup.slice(0, -backupVolume.retention);
+            for (const file of filesToDelete) {
+                console.log(`Deleting backup ${file.key}`);
+                await s3Service.deleteFile(backupVolume.target, file.key!);
+            }
+            console.log(`Backup finished for volume ${volume.id}`);
+        } finally {
+            await FsUtils.deleteFileIfExists(downloadPath);
+        }
+    }
+
     async getAll(): Promise<VolumeBackupExtendedModel[]> {
         return await unstable_cache(() => dataAccess.client.volumeBackup.findMany({
             orderBy: {

src/server/utils/fs.utils.ts

@@ -11,6 +11,14 @@ export class FsUtils {
         }
     }
 
+    static async deleteFileIfExists(pathName: string) {
+        try {
+            await fs.promises.unlink(pathName);
+        } catch (ex) {
+
+        }
+    }
+
     static directoryExists(pathName: string) {
         try {
             return fs.existsSync(pathName);

src/server/utils/path.utils.ts

@@ -19,6 +19,10 @@ export class PathUtils {
         return path.join(this.tempDataRoot, 'volume-downloads');
     }
 
+    static get tempBackupDataFolder() {
+        return path.join(this.tempDataRoot, 'backup-data');
+    }
+
     static gitRootPathForApp(appId: string): string {
         return path.join(PathUtils.gitRootPath, this.convertIdToFolderFriendlyName(appId));
     }
@@ -39,6 +43,10 @@ export class PathUtils {
         return path.join(this.tempVolumeDownloadPath, `${volumeId}.tar.gz`);
     }
 
+    static backupVolumeDownloadZipPath(backupVolumeId: string): string {
+        return path.join(this.tempBackupDataFolder, `${backupVolumeId}.tar.gz`);
+    }
+
     static splitPath(relativePath: string): { folderPath: string | undefined; filePath: string } {
         if (!relativePath.includes('/')) {
             return { folderPath: undefined, filePath: relativePath };

src/shared/model/backup-volume-edit.model.ts

@@ -2,7 +2,7 @@ import { stringToNumber } from "@/shared/utils/zod.utils";
 import { z } from "zod";
 
 export const volumeBackupEditZodModel = z.object({
-  id: z.string().nullable(),
+  id: z.string().nullish(),
   volumeId: z.string(),
   targetId: z.string(),
   cron: z.string().trim().regex(/^ *(\*|[0-5]?\d) *(\*|[01]?\d) *(\*|[0-2]?\d) *(\*|[0-6]?\d) *(\*|[0-6]?\d) *$/),