mirror of
https://github.com/biersoeckli/QuickStack.git
synced 2026-02-10 21:49:19 -06:00
feat/add backup scheduling functionality
This commit is contained in:
@@ -40,6 +40,7 @@
|
||||
"@radix-ui/react-tooltip": "^1.1.4",
|
||||
"@tanstack/react-table": "^8.20.5",
|
||||
"@types/bcrypt": "^5.0.2",
|
||||
"@types/node-schedule": "^2.1.7",
|
||||
"@types/qrcode": "^1.5.5",
|
||||
"@types/ws": "^8.5.13",
|
||||
"@xterm/xterm": "^5.5.0",
|
||||
@@ -55,6 +56,7 @@
|
||||
"next": "14.2.15",
|
||||
"next-auth": "^4.24.8",
|
||||
"next-themes": "^0.3.0",
|
||||
"node-schedule": "^2.1.1",
|
||||
"otpauth": "^9.3.4",
|
||||
"prisma": "^5.21.1",
|
||||
"qrcode": "^1.5.4",
|
||||
|
||||
21
src/app/api/print-schedules-jobs/route.ts
Normal file
21
src/app/api/print-schedules-jobs/route.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import k3s from "@/server/adapter/kubernetes-api.adapter";
|
||||
import appService from "@/server/services/app.service";
|
||||
import deploymentService from "@/server/services/deployment.service";
|
||||
import scheduleService from "@/server/services/standalone-services/schedule.service";
|
||||
import { getAuthUserSession, simpleRoute } from "@/server/utils/action-wrapper.utils";
|
||||
import { Informer, V1Pod } from "@kubernetes/client-node";
|
||||
import { NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
|
||||
// Prevents this route's response from being cached
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
|
||||
export async function GET(request: Request) {
|
||||
return simpleRoute(async () => {
|
||||
scheduleService.printScheduledJobs();
|
||||
return NextResponse.json({
|
||||
status: "success"
|
||||
});
|
||||
})
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import pvcService from "@/server/services/pvc.service";
|
||||
import { fileMountEditZodModel } from "@/shared/model/file-mount-edit.model";
|
||||
import { VolumeBackupEditModel, volumeBackupEditZodModel } from "@/shared/model/backup-volume-edit.model";
|
||||
import volumeBackupService from "@/server/services/volume-backup.service";
|
||||
import backupService from "@/server/services/standalone-services/backup.service";
|
||||
|
||||
const actionAppVolumeEditZodModel = appVolumeEditZodModel.merge(z.object({
|
||||
appId: z.string(),
|
||||
@@ -81,16 +82,21 @@ export const saveBackupVolume = async (prevState: any, inputData: VolumeBackupEd
|
||||
if (validatedData.retention < 1) {
|
||||
throw new ServiceException('Retention must be at least 1');
|
||||
}
|
||||
await volumeBackupService.save({
|
||||
if (validatedData.id) {
|
||||
await backupService.unregisterBackupJob(validatedData.id);
|
||||
}
|
||||
const savedVolumeBackup = await volumeBackupService.save({
|
||||
...validatedData,
|
||||
id: validatedData.id ?? undefined,
|
||||
});
|
||||
await backupService.registerBackupJob(savedVolumeBackup);
|
||||
return new SuccessActionResult();
|
||||
});
|
||||
|
||||
export const deleteBackupVolume = async (backupVolumeId: string) =>
|
||||
simpleAction(async () => {
|
||||
await getAuthUserSession();
|
||||
await backupService.unregisterBackupJob(backupVolumeId);
|
||||
await volumeBackupService.deleteById(backupVolumeId);
|
||||
return new SuccessActionResult(undefined, 'Successfully deleted backup schedule');
|
||||
});
|
||||
@@ -98,6 +104,6 @@ export const deleteBackupVolume = async (backupVolumeId: string) =>
|
||||
export const runBackupVolumeSchedule = async (backupVolumeId: string) =>
|
||||
simpleAction(async () => {
|
||||
await getAuthUserSession();
|
||||
await volumeBackupService.createBackupForVolume(backupVolumeId);
|
||||
await backupService.runBackupForVolume(backupVolumeId);
|
||||
return new SuccessActionResult(undefined, 'Backup created and uploaded successfully');
|
||||
});
|
||||
@@ -68,7 +68,7 @@ export default function VolumeBackupEditDialog({
|
||||
|
||||
useEffect(() => {
|
||||
form.reset(volumeBackup);
|
||||
}, [volumeBackup]);
|
||||
}, [volumeBackup, volumes, s3Targets]);
|
||||
|
||||
return (
|
||||
<>
|
||||
|
||||
@@ -44,9 +44,7 @@ export default function S3TargetEditOverlay({ children, target }: { children: Re
|
||||
useEffect(() => {
|
||||
if (state.status === 'success') {
|
||||
form.reset();
|
||||
toast.success('Volume saved successfully', {
|
||||
description: "Click \"deploy\" to apply the changes to your app.",
|
||||
});
|
||||
toast.success('S3 Target saved successfully');
|
||||
setIsOpen(false);
|
||||
}
|
||||
FormUtils.mapValidationErrorsToForm<typeof s3TargetEditZodModel>(state, form);
|
||||
|
||||
13
src/backup.server.ts
Normal file
13
src/backup.server.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import dataAccess from "./server/adapter/db.client";
|
||||
import backupService from "./server/services/standalone-services/backup.service";
|
||||
|
||||
|
||||
export default async function registreAllBackupSchedules() {
|
||||
|
||||
const volumeBackups = await dataAccess.client.volumeBackup.findMany();
|
||||
console.log(`Registering ${volumeBackups.length} backup schedules...`);
|
||||
for (const volumeBackup of volumeBackups) {
|
||||
await backupService.registerBackupJob(volumeBackup);
|
||||
}
|
||||
console.log('Backup schedules registered.');
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { CommandExecutorUtils } from './server/utils/command-executor.utils'
|
||||
import dataAccess from './server/adapter/db.client'
|
||||
import { FancyConsoleUtils } from './shared/utils/fancy-console.utils'
|
||||
import { Constants } from './shared/utils/constants'
|
||||
import registreAllBackupSchedules from './backup.server'
|
||||
|
||||
// Source: https://nextjs.org/docs/app/building-your-application/configuring/custom-server
|
||||
|
||||
@@ -50,6 +51,8 @@ async function initializeNextJs() {
|
||||
}
|
||||
}
|
||||
|
||||
await registreAllBackupSchedules();
|
||||
|
||||
const app = next({ dev })
|
||||
const handle = app.getRequestHandler()
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import k3s from "../adapter/kubernetes-api.adapter";
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import setupPodService from "./setup-services/setup-pod.service";
|
||||
import standalonePodService from "./standalone-services/standalone-pod.service";
|
||||
import clusterService from "./node.service";
|
||||
import { PodsResourceInfoModel } from "@/shared/model/pods-resource-info.model";
|
||||
import { KubernetesSizeConverter } from "../utils/kubernetes-size-converter.utils";
|
||||
@@ -8,7 +8,7 @@ import { KubernetesSizeConverter } from "../utils/kubernetes-size-converter.util
|
||||
class MonitorAppService {
|
||||
async getMonitoringForApp(projectId: string, appId: string): Promise<PodsResourceInfoModel> {
|
||||
const metricsClient = new k8s.Metrics(k3s.getKubeConfig());
|
||||
const podsFromApp = await setupPodService.getPodsForApp(projectId, appId);
|
||||
const podsFromApp = await standalonePodService.getPodsForApp(projectId, appId);
|
||||
const topPods = await k8s.topPods(k3s.core, metricsClient, projectId);
|
||||
|
||||
const filteredTopPods = topPods.filter((topPod) =>
|
||||
|
||||
@@ -1,15 +1,12 @@
|
||||
import { PodsInfoModel } from "@/shared/model/pods-info.model";
|
||||
import k3s from "../adapter/kubernetes-api.adapter";
|
||||
import { ServiceException } from "@/shared/model/service.exception.model";
|
||||
import setupPodService from "./setup-services/setup-pod.service";
|
||||
import fs from 'fs';
|
||||
import stream from 'stream';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import standalonePodService from "./standalone-services/standalone-pod.service";
|
||||
|
||||
class PodService {
|
||||
|
||||
async waitUntilPodIsRunningFailedOrSucceded(projectId: string, podName: string) {
|
||||
const isPodRunnning = await setupPodService.waitUntilPodIsRunningFailedOrSucceded(projectId, podName);
|
||||
const isPodRunnning = await standalonePodService.waitUntilPodIsRunningFailedOrSucceded(projectId, podName);
|
||||
if (!isPodRunnning) {
|
||||
throw new ServiceException(`Pod ${podName} did not become ready in time (timeout).`);
|
||||
}
|
||||
@@ -24,7 +21,7 @@ class PodService {
|
||||
}
|
||||
|
||||
async getPodsForApp(projectId: string, appId: string): Promise<PodsInfoModel[]> {
|
||||
return setupPodService.getPodsForApp(projectId, appId);
|
||||
return standalonePodService.getPodsForApp(projectId, appId);
|
||||
}
|
||||
|
||||
public async runCommandInPod(
|
||||
@@ -33,54 +30,9 @@ class PodService {
|
||||
containerName: string,
|
||||
command: string[],
|
||||
): Promise<void> {
|
||||
const writerStream = new stream.PassThrough();
|
||||
const stderrStream = new stream.PassThrough();
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
|
||||
const exec = new k8s.Exec(k3s.getKubeConfig());
|
||||
exec
|
||||
.exec(
|
||||
namespace,
|
||||
podName,
|
||||
containerName,
|
||||
command,
|
||||
writerStream,
|
||||
stderrStream,
|
||||
null,
|
||||
false,
|
||||
async ({ status }) => {
|
||||
try {
|
||||
console.log(`Output for command "${command.join(' ')}": \n ${writerStream.read().toString()}`);
|
||||
if (status === 'Failure') {
|
||||
return reject(
|
||||
new Error(
|
||||
`Error while running command "${command.join(' ')}": \n ${stderrStream.read().toString()}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
},
|
||||
)
|
||||
.catch(reject);
|
||||
});
|
||||
return await standalonePodService.runCommandInPod(namespace, podName, containerName, command);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copied out of Kubernetes SDK because for whatever reason
|
||||
* cp.fromPod is not working using the sdk bcause of an error with some buffer
|
||||
* Source: https://github.com/kubernetes-client/javascript/blob/master/src/cp.ts
|
||||
*
|
||||
*
|
||||
* @param {string} namespace - The namespace of the pod to exec the command inside.
|
||||
* @param {string} podName - The name of the pod to exec the command inside.
|
||||
* @param {string} containerName - The name of the container in the pod to exec the command inside.
|
||||
* @param {string} srcPath - The source path in the pod
|
||||
* @param {string} zipOutputPath - The target path in local
|
||||
* @param {string} [cwd] - The directory that is used as the parent in the pod when downloading
|
||||
*/
|
||||
public async cpFromPod(
|
||||
namespace: string,
|
||||
podName: string,
|
||||
@@ -89,45 +41,9 @@ class PodService {
|
||||
zipOutputPath: string,
|
||||
cwd?: string,
|
||||
): Promise<void> {
|
||||
const command = ['tar', 'zcf', '-'];
|
||||
if (cwd) {
|
||||
command.push('-C', cwd);
|
||||
}
|
||||
command.push(srcPath);
|
||||
const writerStream = fs.createWriteStream(zipOutputPath);
|
||||
const stderrStream = new stream.PassThrough();
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
|
||||
const exec = new k8s.Exec(k3s.getKubeConfig());
|
||||
exec
|
||||
.exec(
|
||||
namespace,
|
||||
podName,
|
||||
containerName,
|
||||
command,
|
||||
writerStream,
|
||||
stderrStream,
|
||||
null,
|
||||
false,
|
||||
async ({ status }) => {
|
||||
try {
|
||||
writerStream.close();
|
||||
if (status === 'Failure') {
|
||||
return reject(
|
||||
new Error(
|
||||
`Error from cpFromPod - details: \n ${stderrStream.read().toString()}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
},
|
||||
)
|
||||
.catch(reject);
|
||||
});
|
||||
return await standalonePodService.cpFromPod(namespace, podName, containerName, srcPath, zipOutputPath, cwd);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
const podService = new PodService();
|
||||
|
||||
@@ -4,7 +4,7 @@ import namespaceService from "./namespace.service";
|
||||
import { KubeObjectNameUtils } from "../utils/kube-object-name.utils";
|
||||
import crypto from "crypto";
|
||||
import { FancyConsoleUtils } from "../../shared/utils/fancy-console.utils";
|
||||
import setupPodService from "./setup-services/setup-pod.service";
|
||||
import standalonePodService from "./standalone-services/standalone-pod.service";
|
||||
import ingressSetupService from "./setup-services/ingress-setup.service";
|
||||
|
||||
class QuickStackService {
|
||||
@@ -43,14 +43,14 @@ class QuickStackService {
|
||||
async waitUntilQuickstackIsRunning() {
|
||||
console.log('Waiting for QuickStack to be running...');
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
const pods = await setupPodService.getPodsForApp(this.QUICKSTACK_NAMESPACE, this.QUICKSTACK_DEPLOYMENT_NAME);
|
||||
const pods = await standalonePodService.getPodsForApp(this.QUICKSTACK_NAMESPACE, this.QUICKSTACK_DEPLOYMENT_NAME);
|
||||
const quickStackPod = pods.find(p => p);
|
||||
if (!quickStackPod) {
|
||||
console.error('[ERROR] QuickStack pod was not found');
|
||||
return;
|
||||
}
|
||||
await setupPodService.waitUntilPodIsRunningFailedOrSucceded(this.QUICKSTACK_NAMESPACE, quickStackPod.podName);
|
||||
if (setupPodService) {
|
||||
await standalonePodService.waitUntilPodIsRunningFailedOrSucceded(this.QUICKSTACK_NAMESPACE, quickStackPod.podName);
|
||||
if (standalonePodService) {
|
||||
console.log('QuickStack is now running');
|
||||
} else {
|
||||
console.warn('Could not verify if QuickStack is running, please check manually.');
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
import k3s from "../../adapter/kubernetes-api.adapter";
|
||||
|
||||
class SetupPodService {
|
||||
|
||||
async waitUntilPodIsRunningFailedOrSucceded(projectId: string, podName: string) {
|
||||
const timeout = 120000;
|
||||
const interval = 1000;
|
||||
const maxTries = timeout / interval;
|
||||
let tries = 0;
|
||||
|
||||
while (tries < maxTries) {
|
||||
const pod = await this.getPodOrUndefined(projectId, podName);
|
||||
if (pod && ['Running', 'Failed', 'Succeeded'].includes(pod.status?.phase!)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, interval));
|
||||
tries++;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async getPodOrUndefined(projectId: string, podName: string) {
|
||||
const res = await k3s.core.listNamespacedPod(projectId);
|
||||
return res.body.items.find((item) => item.metadata?.name === podName);
|
||||
}
|
||||
|
||||
async getPodsForApp(projectId: string, appId: string): Promise<{
|
||||
podName: string;
|
||||
containerName: string;
|
||||
uid?: string;
|
||||
}[]> {
|
||||
const res = await k3s.core.listNamespacedPod(projectId, undefined, undefined, undefined, undefined, `app=${appId}`);
|
||||
return res.body.items.map((item) => ({
|
||||
podName: item.metadata?.name!,
|
||||
containerName: item.spec?.containers?.[0].name!,
|
||||
uid: item.metadata?.uid,
|
||||
})).filter((item) => !!item.podName && !!item.containerName);
|
||||
}
|
||||
}
|
||||
|
||||
const setupPodService = new SetupPodService();
|
||||
export default setupPodService;
|
||||
3
src/server/services/standalone-services/00_info.md
Normal file
3
src/server/services/standalone-services/00_info.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# What are Standalone Services
|
||||
|
||||
Standalone services are service classes wich can be used within a Next.JS request context or in a standalone context (for example at application startup, without any Next.JS specific features).
|
||||
98
src/server/services/standalone-services/backup.service.ts
Normal file
98
src/server/services/standalone-services/backup.service.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import dataAccess from "../../adapter/db.client";
|
||||
import { ServiceException } from "../../../shared/model/service.exception.model";
|
||||
import { PathUtils } from "../../utils/path.utils";
|
||||
import { FsUtils } from "../../utils/fs.utils";
|
||||
import s3Service from "../aws-s3.service";
|
||||
import { VolumeBackup } from "@prisma/client";
|
||||
import scheduleService from "./schedule.service";
|
||||
import standalonePodService from "./standalone-pod.service";
|
||||
|
||||
class BackupService {
|
||||
|
||||
async registerBackupJob(volumeBackup: VolumeBackup) {
|
||||
const cron = volumeBackup.cron;
|
||||
const jobName = `backup-volume-${volumeBackup.id}`;
|
||||
scheduleService.scheduleJob(jobName, cron, async () => {
|
||||
try {
|
||||
await this.runBackupForVolume(volumeBackup.id);
|
||||
} catch (e) {
|
||||
console.error(`Error during backup for volume ${volumeBackup.id}`);
|
||||
console.error(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async unregisterBackupJob(volumeBackupId: string) {
|
||||
const jobName = `backup-volume-${volumeBackupId}`;
|
||||
scheduleService.cancelJob(jobName);
|
||||
}
|
||||
|
||||
async runBackupForVolume(backupVolumeId: string) {
|
||||
|
||||
const backupVolume = await dataAccess.client.volumeBackup.findFirstOrThrow({
|
||||
where: {
|
||||
id: backupVolumeId
|
||||
},
|
||||
include: {
|
||||
volume: {
|
||||
include: {
|
||||
app: true
|
||||
}
|
||||
},
|
||||
target: true
|
||||
}
|
||||
});
|
||||
|
||||
const projectId = backupVolume.volume.app.projectId;
|
||||
const appId = backupVolume.volume.app.id;
|
||||
const volume = backupVolume.volume;
|
||||
|
||||
const pod = await standalonePodService.getPodsForApp(projectId, appId);
|
||||
if (pod.length === 0) {
|
||||
throw new ServiceException(`There are no running pods for volume id ${volume.id} in app ${volume.app.id}. Make sure the app is running.`);
|
||||
}
|
||||
const firstPod = pod[0];
|
||||
|
||||
// zipping and saving backup data in quickstack pod
|
||||
const downloadPath = PathUtils.backupVolumeDownloadZipPath(backupVolume.id);
|
||||
await FsUtils.createDirIfNotExistsAsync(PathUtils.tempBackupDataFolder, true);
|
||||
|
||||
try {
|
||||
console.log(`Downloading data from pod ${firstPod.podName} ${volume.containerMountPath} to ${downloadPath}`);
|
||||
await standalonePodService.cpFromPod(projectId, firstPod.podName, firstPod.containerName, volume.containerMountPath, downloadPath);
|
||||
|
||||
// uploac backup
|
||||
console.log(`Uploading backup to S3`);
|
||||
const now = new Date();
|
||||
const nowString = now.toISOString();
|
||||
await s3Service.uploadFile(backupVolume.target, downloadPath,
|
||||
`${appId}/${backupVolumeId}/${nowString}.tar.gz`, 'application/gzip', 'binary');
|
||||
|
||||
|
||||
// delete files wich are nod needed anymore (by retention)
|
||||
console.log(`Deleting old backups`);
|
||||
const files = await s3Service.listFiles(backupVolume.target);
|
||||
|
||||
const filesFromThisBackup = files.filter(f => f.Key?.startsWith(`${appId}/${backupVolumeId}/`)).map(f => ({
|
||||
date: new Date((f.Key ?? '')
|
||||
.replace(`${appId}/${backupVolumeId}/`, '')
|
||||
.replace('.tar.gz', '')),
|
||||
key: f.Key
|
||||
})).filter(f => !isNaN(f.date.getTime()) && !!f.key);
|
||||
|
||||
filesFromThisBackup.sort((a, b) => a.date.getTime() - b.date.getTime());
|
||||
|
||||
const filesToDelete = filesFromThisBackup.slice(0, -backupVolume.retention);
|
||||
for (const file of filesToDelete) {
|
||||
console.log(`Deleting backup ${file.key}`);
|
||||
await s3Service.deleteFile(backupVolume.target, file.key!);
|
||||
}
|
||||
console.log(`Backup finished for volume ${volume.id} and backup ${backupVolume.id}`);
|
||||
} finally {
|
||||
await FsUtils.deleteFileIfExists(downloadPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const backupService = new BackupService();
|
||||
export default backupService;
|
||||
41
src/server/services/standalone-services/schedule.service.ts
Normal file
41
src/server/services/standalone-services/schedule.service.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import * as schedule from 'node-schedule';
|
||||
|
||||
|
||||
const globalScheduleInstance = () => {
|
||||
return schedule
|
||||
}
|
||||
|
||||
declare const globalThis: {
|
||||
globalSchedule: ReturnType<typeof globalScheduleInstance>;
|
||||
} & typeof global;
|
||||
|
||||
const scheduleInstance = globalThis.globalSchedule ?? globalScheduleInstance()
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') globalThis.globalSchedule = scheduleInstance
|
||||
|
||||
|
||||
class ScheduleService {
|
||||
|
||||
schedule = globalThis.globalSchedule;
|
||||
|
||||
scheduleJob(jobName: string, cronExpression: string, callback: schedule.JobCallback) {
|
||||
const job = new this.schedule.Job(jobName, callback);
|
||||
job.schedule(cronExpression);
|
||||
console.log(`[${ScheduleService.name}] Job ${jobName} scheduled with cron ${cronExpression}`);
|
||||
}
|
||||
|
||||
cancelJob(jobName: string) {
|
||||
const job = this.schedule.scheduledJobs[jobName];
|
||||
if (job) {
|
||||
job.cancel();
|
||||
console.log(`[${ScheduleService.name}] Job ${jobName} cancelled`);
|
||||
}
|
||||
}
|
||||
|
||||
printScheduledJobs() {
|
||||
console.log(`[${ScheduleService.name}] Scheduled jobs: \n- ${Object.keys(this.schedule.scheduledJobs).join('\n- ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
const scheduleService = new ScheduleService();
|
||||
export default scheduleService;
|
||||
@@ -0,0 +1,149 @@
|
||||
import k3s from "../../adapter/kubernetes-api.adapter";
|
||||
import fs from 'fs';
|
||||
import stream from 'stream';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
|
||||
class SetupPodService {
|
||||
|
||||
async waitUntilPodIsRunningFailedOrSucceded(projectId: string, podName: string) {
|
||||
const timeout = 120000;
|
||||
const interval = 1000;
|
||||
const maxTries = timeout / interval;
|
||||
let tries = 0;
|
||||
|
||||
while (tries < maxTries) {
|
||||
const pod = await this.getPodOrUndefined(projectId, podName);
|
||||
if (pod && ['Running', 'Failed', 'Succeeded'].includes(pod.status?.phase!)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, interval));
|
||||
tries++;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async getPodOrUndefined(projectId: string, podName: string) {
|
||||
const res = await k3s.core.listNamespacedPod(projectId);
|
||||
return res.body.items.find((item) => item.metadata?.name === podName);
|
||||
}
|
||||
|
||||
async getPodsForApp(projectId: string, appId: string): Promise<{
|
||||
podName: string;
|
||||
containerName: string;
|
||||
uid?: string;
|
||||
}[]> {
|
||||
const res = await k3s.core.listNamespacedPod(projectId, undefined, undefined, undefined, undefined, `app=${appId}`);
|
||||
return res.body.items.map((item) => ({
|
||||
podName: item.metadata?.name!,
|
||||
containerName: item.spec?.containers?.[0].name!,
|
||||
uid: item.metadata?.uid,
|
||||
})).filter((item) => !!item.podName && !!item.containerName);
|
||||
}
|
||||
|
||||
public async runCommandInPod(
|
||||
namespace: string,
|
||||
podName: string,
|
||||
containerName: string,
|
||||
command: string[],
|
||||
): Promise<void> {
|
||||
const writerStream = new stream.PassThrough();
|
||||
const stderrStream = new stream.PassThrough();
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
|
||||
const exec = new k8s.Exec(k3s.getKubeConfig());
|
||||
exec
|
||||
.exec(
|
||||
namespace,
|
||||
podName,
|
||||
containerName,
|
||||
command,
|
||||
writerStream,
|
||||
stderrStream,
|
||||
null,
|
||||
false,
|
||||
async ({ status }) => {
|
||||
try {
|
||||
console.log(`Output for command "${command.join(' ')}": \n ${writerStream.read().toString()}`);
|
||||
if (status === 'Failure') {
|
||||
return reject(
|
||||
new Error(
|
||||
`Error while running command "${command.join(' ')}": \n ${stderrStream.read().toString()}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
},
|
||||
)
|
||||
.catch(reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Copied out of Kubernetes SDK because for whatever reason
|
||||
* cp.fromPod is not working using the sdk bcause of an error with some buffer
|
||||
* Source: https://github.com/kubernetes-client/javascript/blob/master/src/cp.ts
|
||||
*
|
||||
*
|
||||
* @param {string} namespace - The namespace of the pod to exec the command inside.
|
||||
* @param {string} podName - The name of the pod to exec the command inside.
|
||||
* @param {string} containerName - The name of the container in the pod to exec the command inside.
|
||||
* @param {string} srcPath - The source path in the pod
|
||||
* @param {string} zipOutputPath - The target path in local
|
||||
* @param {string} [cwd] - The directory that is used as the parent in the pod when downloading
|
||||
*/
|
||||
public async cpFromPod(
|
||||
namespace: string,
|
||||
podName: string,
|
||||
containerName: string,
|
||||
srcPath: string,
|
||||
zipOutputPath: string,
|
||||
cwd?: string,
|
||||
): Promise<void> {
|
||||
const command = ['tar', 'zcf', '-'];
|
||||
if (cwd) {
|
||||
command.push('-C', cwd);
|
||||
}
|
||||
command.push(srcPath);
|
||||
const writerStream = fs.createWriteStream(zipOutputPath);
|
||||
const stderrStream = new stream.PassThrough();
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
|
||||
const exec = new k8s.Exec(k3s.getKubeConfig());
|
||||
exec
|
||||
.exec(
|
||||
namespace,
|
||||
podName,
|
||||
containerName,
|
||||
command,
|
||||
writerStream,
|
||||
stderrStream,
|
||||
null,
|
||||
false,
|
||||
async ({ status }) => {
|
||||
try {
|
||||
writerStream.close();
|
||||
if (status === 'Failure') {
|
||||
return reject(
|
||||
new Error(
|
||||
`Error from cpFromPod - details: \n ${stderrStream.read().toString()}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
resolve();
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
},
|
||||
)
|
||||
.catch(reject);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const standalonePodService = new SetupPodService();
|
||||
export default standalonePodService;
|
||||
@@ -5,7 +5,7 @@ import * as k8s from '@kubernetes/client-node';
|
||||
import stream from 'stream';
|
||||
import { StreamUtils } from "../../shared/utils/stream.utils";
|
||||
import WebSocket from "ws";
|
||||
import setupPodService from "./setup-services/setup-pod.service";
|
||||
import standalonePodService from "./standalone-services/standalone-pod.service";
|
||||
|
||||
interface TerminalStrean {
|
||||
stdoutStream: stream.PassThrough;
|
||||
@@ -34,7 +34,7 @@ export class TerminalService {
|
||||
const streamInputKey = StreamUtils.getInputStreamName(terminalInfo);
|
||||
const streamOutputKey = StreamUtils.getOutputStreamName(terminalInfo);
|
||||
|
||||
const podReachable = await setupPodService.waitUntilPodIsRunningFailedOrSucceded(terminalInfo.namespace, terminalInfo.podName);
|
||||
const podReachable = await standalonePodService.waitUntilPodIsRunningFailedOrSucceded(terminalInfo.namespace, terminalInfo.podName);
|
||||
if (!podReachable) {
|
||||
socket.emit(streamOutputKey, 'Pod is not reachable.');
|
||||
return;
|
||||
|
||||
@@ -3,81 +3,9 @@ import dataAccess from "../adapter/db.client";
|
||||
import { Tags } from "../utils/cache-tag-generator.utils";
|
||||
import { Prisma, VolumeBackup } from "@prisma/client";
|
||||
import { VolumeBackupExtendedModel } from "@/shared/model/volume-backup-extended.model";
|
||||
import podService from "./pod.service";
|
||||
import { ServiceException } from "@/shared/model/service.exception.model";
|
||||
import { PathUtils } from "../utils/path.utils";
|
||||
import { FsUtils } from "../utils/fs.utils";
|
||||
import s3Service from "./aws-s3.service";
|
||||
|
||||
class VolumeBackupService {
|
||||
|
||||
async createBackupForVolume(backupVolumeId: string) {
|
||||
|
||||
const backupVolume = await dataAccess.client.volumeBackup.findFirstOrThrow({
|
||||
where: {
|
||||
id: backupVolumeId
|
||||
},
|
||||
include: {
|
||||
volume: {
|
||||
include: {
|
||||
app: true
|
||||
}
|
||||
},
|
||||
target: true
|
||||
}
|
||||
});
|
||||
|
||||
const projectId = backupVolume.volume.app.projectId;
|
||||
const appId = backupVolume.volume.app.id;
|
||||
const volume = backupVolume.volume;
|
||||
|
||||
const pod = await podService.getPodsForApp(projectId, appId);
|
||||
if (pod.length === 0) {
|
||||
throw new ServiceException(`There are no running pods for volume id ${volume.id} in app ${volume.app.id}. Make sure the app is running.`);
|
||||
}
|
||||
const firstPod = pod[0];
|
||||
|
||||
// zipping and saving backup data in quickstack pod
|
||||
const downloadPath = PathUtils.backupVolumeDownloadZipPath(backupVolume.id);
|
||||
await FsUtils.createDirIfNotExistsAsync(PathUtils.tempBackupDataFolder, true);
|
||||
|
||||
try {
|
||||
console.log(`Downloading data from pod ${firstPod.podName} ${volume.containerMountPath} to ${downloadPath}`);
|
||||
await podService.cpFromPod(projectId, firstPod.podName, firstPod.containerName, volume.containerMountPath, downloadPath);
|
||||
|
||||
// uploac backup
|
||||
console.log(`Uploading backup to S3`);
|
||||
const now = new Date();
|
||||
const nowString = now.toISOString();
|
||||
await s3Service.uploadFile(backupVolume.target, downloadPath,
|
||||
`${appId}/${volume.id}/${nowString}.tar.gz`, 'application/gzip', 'binary');
|
||||
|
||||
|
||||
// delete files wich are nod needed anymore (by retention)
|
||||
console.log(`Deleting old backups`);
|
||||
const files = await s3Service.listFiles(backupVolume.target);
|
||||
|
||||
const filesFromThisBackup = files.filter(f => f.Key?.startsWith(`${appId}/${volume.id}/`)).map(f => ({
|
||||
date: new Date((f.Key ?? '')
|
||||
.replace(`${appId}/${volume.id}/`, '')
|
||||
.replace('.tar.gz', '')),
|
||||
key: f.Key
|
||||
})).filter(f => !isNaN(f.date.getTime()) && !!f.key);
|
||||
console.log(filesFromThisBackup)
|
||||
|
||||
filesFromThisBackup.sort((a, b) => a.date.getTime() - b.date.getTime());
|
||||
|
||||
const filesToDelete = filesFromThisBackup.slice(0, -backupVolume.retention);
|
||||
for (const file of filesToDelete) {
|
||||
console.log(`Deleting backup ${file.key}`);
|
||||
await s3Service.deleteFile(backupVolume.target, file.key!);
|
||||
}
|
||||
console.log(`Backup finished for volume ${volume.id}`);
|
||||
} finally {
|
||||
await FsUtils.deleteFileIfExists(downloadPath);
|
||||
}
|
||||
}
|
||||
|
||||
async getAll(): Promise<VolumeBackupExtendedModel[]> {
|
||||
return await unstable_cache(() => dataAccess.client.volumeBackup.findMany({
|
||||
orderBy: {
|
||||
|
||||
@@ -5,7 +5,7 @@ export const volumeBackupEditZodModel = z.object({
|
||||
id: z.string().nullish(),
|
||||
volumeId: z.string(),
|
||||
targetId: z.string(),
|
||||
cron: z.string().trim().regex(/^ *(\*|[0-5]?\d) *(\*|[01]?\d) *(\*|[0-2]?\d) *(\*|[0-6]?\d) *(\*|[0-6]?\d) *$/),
|
||||
cron: z.string().trim().regex(/(@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*) ?){5,7})/),
|
||||
//cron: z.string().trim().min(1),
|
||||
retention: stringToNumber,
|
||||
});
|
||||
|
||||
38
yarn.lock
38
yarn.lock
@@ -2920,6 +2920,13 @@
|
||||
resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.34.tgz#10964ba0dee6ac4cd462e2795b6bebd407303433"
|
||||
integrity sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==
|
||||
|
||||
"@types/node-schedule@^2.1.7":
|
||||
version "2.1.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/node-schedule/-/node-schedule-2.1.7.tgz#79a1e61adc7bbf8d8eaabcef307e07d76cb40d82"
|
||||
integrity sha512-G7Z3R9H7r3TowoH6D2pkzUHPhcJrDF4Jz1JOQ80AX0K2DWTHoN9VC94XzFAPNMdbW9TBzMZ3LjpFi7RYdbxtXA==
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/node@*", "@types/node@>=10.0.0", "@types/node@^22.7.9":
|
||||
version "22.10.1"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-22.10.1.tgz#41ffeee127b8975a05f8c4f83fb89bcb2987d766"
|
||||
@@ -3776,6 +3783,13 @@ create-require@^1.1.0:
|
||||
resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
|
||||
integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==
|
||||
|
||||
cron-parser@^4.2.0:
|
||||
version "4.9.0"
|
||||
resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5"
|
||||
integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==
|
||||
dependencies:
|
||||
luxon "^3.2.1"
|
||||
|
||||
cross-env@^7.0.3:
|
||||
version "7.0.3"
|
||||
resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf"
|
||||
@@ -6028,6 +6042,11 @@ lodash@^4.17.21:
|
||||
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
|
||||
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
|
||||
|
||||
long-timeout@0.1.1:
|
||||
version "0.1.1"
|
||||
resolved "https://registry.yarnpkg.com/long-timeout/-/long-timeout-0.1.1.tgz#9721d788b47e0bcb5a24c2e2bee1a0da55dab514"
|
||||
integrity sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w==
|
||||
|
||||
loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
|
||||
@@ -6059,6 +6078,11 @@ lucide-react@^0.465.0:
|
||||
resolved "https://registry.yarnpkg.com/lucide-react/-/lucide-react-0.465.0.tgz#3f98d40f7b7ac5266c055aaf582c303b07f84de2"
|
||||
integrity sha512-uV7WEqbwaCcc+QjAxIhAvkAr3kgwkkYID3XptCHll72/F7NZlk6ONmJYpk+Xqx5Q0r/8wiOjz73H1BYbl8Z8iw==
|
||||
|
||||
luxon@^3.2.1:
|
||||
version "3.5.0"
|
||||
resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.5.0.tgz#6b6f65c5cd1d61d1fd19dbf07ee87a50bf4b8e20"
|
||||
integrity sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==
|
||||
|
||||
lz-string@^1.5.0:
|
||||
version "1.5.0"
|
||||
resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941"
|
||||
@@ -6291,6 +6315,15 @@ node-releases@^2.0.18:
|
||||
resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.19.tgz#9e445a52950951ec4d177d843af370b411caf314"
|
||||
integrity sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==
|
||||
|
||||
node-schedule@^2.1.1:
|
||||
version "2.1.1"
|
||||
resolved "https://registry.yarnpkg.com/node-schedule/-/node-schedule-2.1.1.tgz#6958b2c5af8834954f69bb0a7a97c62b97185de3"
|
||||
integrity sha512-OXdegQq03OmXEjt2hZP33W2YPs/E5BcFQks46+G2gAxs4gHOIVD1u7EqlYLYSKsaIpyKCK9Gbk0ta1/gjRSMRQ==
|
||||
dependencies:
|
||||
cron-parser "^4.2.0"
|
||||
long-timeout "0.1.1"
|
||||
sorted-array-functions "^1.3.0"
|
||||
|
||||
nopt@^5.0.0:
|
||||
version "5.0.0"
|
||||
resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88"
|
||||
@@ -7261,6 +7294,11 @@ sonner@^1.5.0:
|
||||
resolved "https://registry.yarnpkg.com/sonner/-/sonner-1.7.0.tgz#f59a2a70e049a179b6fbd1bb1bf2619d5ced07c0"
|
||||
integrity sha512-W6dH7m5MujEPyug3lpI2l3TC3Pp1+LTgK0Efg+IHDrBbtEjyCmCHHo6yfNBOsf1tFZ6zf+jceWwB38baC8yO9g==
|
||||
|
||||
sorted-array-functions@^1.3.0:
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz#8605695563294dffb2c9796d602bd8459f7a0dd5"
|
||||
integrity sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==
|
||||
|
||||
source-map-js@^1.0.2, source-map-js@^1.2.1:
|
||||
version "1.2.1"
|
||||
resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.1.tgz#1ce5650fddd87abc099eda37dcff024c2667ae46"
|
||||
|
||||
Reference in New Issue
Block a user