fix: only build app if new source code available

.dockerignore

@@ -7,4 +7,5 @@ README.md
 !.next/static
 !.next/standalone
 .git
-db
+db
+internal

.gitignore

@@ -41,3 +41,4 @@ kube-config.config
 kube-config.config_clusteradmin
 kube-config.config_old
 kube-config.config_restricted
+internal

package.json

@@ -54,6 +54,7 @@
     "react-dom": "^18.3.1",
     "react-hook-form": "^7.53.1",
     "reflect-metadata": "^0.2.2",
+    "simple-git": "^3.27.0",
     "socket.io": "^4.8.1",
     "socket.io-client": "^4.8.1",
     "sonner": "^1.5.0",

src/app/project/app/[tabName]/action.ts

@@ -6,10 +6,10 @@ import deploymentService from "@/server/services/deployment.service";
 import { getAuthUserSession, simpleAction } from "@/server/utils/action-wrapper.utils";
 
 
-export const deploy = async (appId: string) =>
+export const deploy = async (appId: string, forceBuild = false) =>
     simpleAction(async () => {
         await getAuthUserSession();
-        await appService.buildAndDeploy(appId);
+        await appService.buildAndDeploy(appId, forceBuild);
         return new SuccessActionResult(undefined, 'Successfully started deployment.');
     });
 

src/app/project/app/[tabName]/app-action-buttons.tsx

@@ -6,6 +6,7 @@ import { deploy, startApp, stopApp } from "./action";
 import { AppExtendedModel } from "@/model/app-extended.model";
 import { Toast } from "@/lib/toast.utils";
 import AppStatus from "./app-status";
+import { Hammer, Pause, Play, Rocket } from "lucide-react";
 
 export default function AppActionButtons({
     app
@@ -15,10 +16,10 @@ export default function AppActionButtons({
     return <Card>
         <CardContent className="p-4 flex gap-4">
             <div className="self-center"><AppStatus appId={app.id} /></div>
-            <Button onClick={() => Toast.fromAction(() => deploy(app.id))}>Deploy</Button>
-            <Button onClick={() => Toast.fromAction(() => startApp(app.id))} variant="secondary">Start</Button>
-            <Button onClick={() => Toast.fromAction(() => stopApp(app.id))} variant="secondary">Stop</Button>
-            <Button variant="secondary">Rebuild</Button>
+            <Button onClick={() => Toast.fromAction(() => deploy(app.id))}><Rocket /> Deploy</Button>
+            <Button onClick={() => Toast.fromAction(() => deploy(app.id, true))} variant="secondary"><Hammer /> Rebuild</Button>
+            <Button onClick={() => Toast.fromAction(() => startApp(app.id))} variant="secondary"><Play />Start</Button>
+            <Button onClick={() => Toast.fromAction(() => stopApp(app.id))} variant="secondary"><Pause /> Stop</Button>
         </CardContent>
     </Card >;
 }

src/app/project/app/[tabName]/overview/deployments.tsx

@@ -83,8 +83,10 @@ export default function BuildsTab({
                     <SimpleDataTable columns={[
                         ['replicasetName', 'Deployment Name', false],
                         ['buildJobName', 'Build Job Name', false],
+                        ['buildJobName', 'Build Job Name', false],
                         ['status', 'Status', true, (item) => <DeploymentStatusBadge>{item.status}</DeploymentStatusBadge>],
                         ["startTime", "Started At", true, (item) => formatDateTime(item.createdAt)],
+                        ['gitCommit', 'Git Commit', true],
                     ]}
                         data={appBuilds}
                         hideSearchBar={true}

src/model/build-job.ts

@@ -1,3 +1,4 @@
+import { GitCommit } from "lucide-react";
 import { z } from "zod";
 
 export const buildJobStatusEnumZod = z.union([z.literal('UNKNOWN'), z.literal('RUNNING'), z.literal('FAILED'), z.literal('SUCCEEDED')]);
@@ -5,7 +6,8 @@ export const buildJobStatusEnumZod = z.union([z.literal('UNKNOWN'), z.literal('R
 export const buildJobSchemaZod = z.object({
     name: z.string(),
     startTime: z.date(),
-    status:  buildJobStatusEnumZod
+    status:  buildJobStatusEnumZod,
+    gitCommit: z.string(),
 });
 
 export type BuildJobModel = z.infer<typeof buildJobSchemaZod>;

src/model/deployment-info.model.ts

@@ -14,7 +14,8 @@ export const deploymentInfoZodModel = z.object({
     replicasetName: z.string().optional(),
     buildJobName: z.string().optional(),
     createdAt: z.date(),
-    status: deploymentStatusEnumZod
+    status: deploymentStatusEnumZod,
+    gitCommit: z.string().optional(),
 });
 
 export type DeploymentInfoModel = z.infer<typeof deploymentInfoZodModel>;

src/server/services/app.service.ts

@@ -15,15 +15,14 @@ import svcService from "./svc.service";
 
 class AppService {
 
-    async buildAndDeploy(appId: string) {
+    async buildAndDeploy(appId: string, forceBuild: boolean = false) {
         const app = await this.getExtendedById(appId);
         if (app.sourceType === 'GIT') {
             // first make build
-            await namespaceService.createNamespaceIfNotExists(BUILD_NAMESPACE);
-            const [buildJobName, buildPromise] = await buildService.buildApp(app);
+            const [buildJobName, gitCommitHash, buildPromise] = await buildService.buildApp(app, forceBuild);
             buildPromise.then(async () => {
                 console.warn('Build job finished, deploying...');
-                await deploymentService.createDeployment(app, buildJobName);
+                await deploymentService.createDeployment(app, buildJobName, gitCommitHash);
             });
         } else {
             // only deploy

src/server/services/build.service.ts

@@ -7,6 +7,9 @@ import { ServiceException } from "@/model/service.exception.model";
 import { PodsInfoModel } from "@/model/pods-info.model";
 import namespaceService from "./namespace.service";
 import { Constants } from "../utils/constants";
+import gitService from "./git.service";
+import deploymentService from "./deployment.service";
+import deploymentLogService from "./deployment-logs.service";
 
 const kanikoImage = "gcr.io/kaniko-project/executor:latest";
 const REGISTRY_NODE_PORT = 30100;
@@ -20,13 +23,28 @@ export const REGISTRY_URL_INTERNAL = `${REGISTRY_SVC_NAME}.${BUILD_NAMESPACE}.sv
 class BuildService {
 
 
-    async buildApp(app: AppExtendedModel): Promise<[string, Promise<void>]> {
+    async buildApp(app: AppExtendedModel, forceBuild: boolean = false): Promise<[string, string, Promise<void>]> {
+        await namespaceService.createNamespaceIfNotExists(BUILD_NAMESPACE);
         await this.deployRegistryIfNotExists();
-        const runningJobsForApp = await this.getBuildsForApp(app.id);
-        if (runningJobsForApp.some((job) => job.status === 'RUNNING')) {
+        const buildsForApp = await this.getBuildsForApp(app.id);
+        if (buildsForApp.some((job) => job.status === 'RUNNING')) {
             throw new ServiceException("A build job is already running for this app.");
         }
 
+        // Check if last build is already up to date with data in git repo
+        const latestSuccessfulBuld = buildsForApp.find(x => x.status === 'SUCCEEDED');
+        const latestRemoteGitHash = await gitService.getLatestRemoteCommitHash(app);
+        if (!forceBuild && latestSuccessfulBuld?.gitCommit && latestRemoteGitHash &&
+            latestSuccessfulBuld?.gitCommit === latestRemoteGitHash) {
+            console.log(`Last build is already up to date with data in git repo for app ${app.id}`);
+            // todo check if the container is still in registry
+            return [latestSuccessfulBuld.name, latestRemoteGitHash, Promise.resolve()];
+        }
+        return await this.createAndStartBuildJob(app, latestRemoteGitHash);
+    }
+
+    private async createAndStartBuildJob(app: AppExtendedModel, latestRemoteGitHash: string): Promise<[string, string, Promise<void>]> {
+
         const buildName = StringUtils.addRandomSuffix(StringUtils.toJobName(app.id));
         const jobDefinition: V1Job = {
             apiVersion: "batch/v1",
@@ -37,6 +55,7 @@ class BuildService {
                 annotations: {
                     [Constants.QS_ANNOTATION_APP_ID]: app.id,
                     [Constants.QS_ANNOTATION_PROJECT_ID]: app.projectId,
+                    [Constants.QS_ANNOTATION_GIT_COMMIT]: latestRemoteGitHash,
                 }
             },
             spec: {
@@ -51,7 +70,7 @@ class BuildService {
                                     `--dockerfile=${app.dockerfilePath}`,
                                     `--insecure`,
                                     `--log-format=text`,
-                                    `--context=${app.gitUrl!.replace("https://", "git://")}#refs/heads/${app.gitBranch}`,
+                                    `--context=${app.gitUrl!.replace("https://", "git://")}#refs/heads/${app.gitBranch}`, // todo change to shared folder
                                     `--destination=${this.createInternalContainerRegistryUrlForAppId(app.id)}`
                                 ]
                             },
@@ -79,7 +98,7 @@ class BuildService {
 
         const buildJobPromise = this.waitForJobCompletion(jobDefinition.metadata!.name!)
 
-        return [buildName, buildJobPromise];
+        return [buildName, latestRemoteGitHash, buildJobPromise];
     }
 
     createInternalContainerRegistryUrlForAppId(appId?: string) {
@@ -128,6 +147,7 @@ class BuildService {
                 name: job.metadata?.name,
                 startTime: job.status?.startTime,
                 status: this.getJobStatusString(job.status),
+                gitCommit: job.metadata?.annotations?.[Constants.QS_ANNOTATION_GIT_COMMIT],
             } as BuildJobModel;
         });
         builds.sort((a, b) => {

src/server/services/deployment-logs.service.ts

@@ -0,0 +1,82 @@
+import fsPromises from 'fs/promises';
+import fs from 'fs';
+import { PathUtils } from '../utils/path.utils';
+import { FsUtils } from '../utils/fs.utils';
+
+class DeploymentLogService {
+
+    async writeLogs(deploymentId: string, logs: string) {
+        try {
+            await FsUtils.createDirIfNotExistsAsync(PathUtils.deploymentLogsPath, true);
+            const logFilePath = PathUtils.appDeploymentLogFile(deploymentId);
+            await fsPromises.appendFile(logFilePath, logs, {
+                encoding: 'utf-8'
+            });
+        } catch (ex) {
+            console.error(`Error writing logs for deployment ${deploymentId}: ${ex}`);
+        }
+    }
+
+    catchErrosAndLog<TReturnType>(appId: string, deploymentId: string, fn: (logFunc: (logData: string) => void) => TReturnType): TReturnType {
+        try {
+            return fn((logData: string) => {
+                this.writeLogs(deploymentId, logData);
+            });
+        } catch (ex) {
+            this.writeLogs(deploymentId, `[Error]: ${(ex as any)?.message}`);
+            throw ex;
+        }
+    }
+
+
+    async getLogsStream(appId: string, deploymentId: string, streamedData: (data: string) => void) {
+        await FsUtils.createDirIfNotExistsAsync(PathUtils.deploymentLogsPath, true);
+        const logFilePath = PathUtils.appDeploymentLogFile(deploymentId);
+
+        if (!await FsUtils.fileExists(logFilePath)) {
+            return undefined;
+        }
+        // Create a read stream
+        let fileStream = fs.createReadStream(logFilePath, {
+            encoding: 'utf8',
+            start: 0,
+            flags: 'r'
+        });
+
+        // Watch for changes in the file and read new lines when the file is updated
+        const watcher = fs.watch(logFilePath, (eventType) => {
+            if (eventType === 'change') {
+                // Create a new read stream starting from the current end of the file
+                const newStream = fs.createReadStream(logFilePath, {
+                    encoding: 'utf8',
+                    start: fileStream.bytesRead,
+                    flags: 'r'
+                });
+
+                newStream.on('data', (chunk: string) => {
+                    streamedData(chunk);
+                });
+
+                // Update the read stream pointer
+                newStream.on('end', () => {
+                    fileStream.bytesRead += newStream.readableLength; // Buffer.byteLength(); // todo check if this works
+                    newStream.close();
+                });
+            }
+        });
+
+        return () => {
+            watcher.close();
+            fileStream.close();
+        }
+    }
+
+}
+
+const deploymentLogService = new DeploymentLogService();
+export default deploymentLogService;
+
+
+export const dlog = (deploymentId: string, data: string) => {
+    deploymentLogService.writeLogs(deploymentId, data);
+}

src/server/services/deployment.service.ts

@@ -40,7 +40,7 @@ class DeploymentService {
         }
     }
 
-    async createDeployment(app: AppExtendedModel, buildJobName?: string) {
+    async createDeployment(app: AppExtendedModel, buildJobName?: string, gitCommitHash?: string) {
         await this.validateDeployment(app);
         await namespaceService.createNamespaceIfNotExists(app.projectId);
         const appHasPvcChanges = await pvcService.doesAppConfigurationIncreaseAnyPvcSize(app)
@@ -95,6 +95,10 @@ class DeploymentService {
             body.spec!.template!.metadata!.annotations!.buildJobName = buildJobName; // add buildJobName to deployment
         }
 
+        if (gitCommitHash) {
+            body.spec!.template!.metadata!.annotations![Constants.QS_ANNOTATION_GIT_COMMIT] = gitCommitHash; // add gitCommitHash to deployment
+        }
+
         if (!appHasPvcChanges && app.appVolumes.length === 0 || app.appVolumes.every(vol => vol.accessMode === 'ReadWriteMany')) {
             body.spec!.strategy = {
                 type: 'RollingUpdate',
@@ -163,7 +167,8 @@ class DeploymentService {
                     replicasetName: undefined,
                     createdAt: build.startTime!,
                     buildJobName: build.name!,
-                    status: this.mapBuildStatusToDeploymentStatus(build.status)
+                    status: this.mapBuildStatusToDeploymentStatus(build.status),
+                    gitCommit: build.gitCommit,
                 }
             });
         replicasetRevisions.push(...runningOrFailedBuilds);
@@ -196,6 +201,7 @@ class DeploymentService {
                 replicasetName: rs.metadata?.name!,
                 createdAt: rs.metadata?.creationTimestamp!,
                 buildJobName: rs.spec?.template?.metadata?.annotations?.buildJobName!,
+                gitCommit: rs.spec?.template?.metadata?.annotations?.[Constants.QS_ANNOTATION_GIT_COMMIT],
                 status: status
             }
         });

src/server/services/git.service.ts

@@ -0,0 +1,88 @@
+import { ServiceException } from "@/model/service.exception.model";
+import { AppExtendedModel } from "@/model/app-extended.model";
+import simpleGit from "simple-git";
+import { PathUtils } from "../utils/path.utils";
+import { FsUtils } from "../utils/fs.utils";
+
+class GitService {
+
+    async getLatestRemoteCommitHash(app: AppExtendedModel) {
+        try {
+            const git = await this.pullLatestChangesFromRepo(app);
+
+            // Get the latest commit hash on the default branch (e.g., 'origin/main')
+            const log = await git.log(['origin/' + app.gitBranch]); // Replace 'main' with your branch name if needed
+
+            if (log.latest) {
+                return log.latest.hash;
+            } else {
+                throw new ServiceException("The git repository is empty.");
+            }
+        } catch (error) {
+            console.error('Error while connecting to the git repository:', error);
+            throw new ServiceException("Error while connecting to the git repository.");
+        } finally {
+            await this.cleanupLocalGitDataForApp(app);
+        }
+    }
+
+    async cleanupLocalGitDataForApp(app: AppExtendedModel) {
+        const gitPath = PathUtils.gitRootPathForApp(app.id);
+        await FsUtils.deleteDirIfExistsAsync(gitPath, true);
+    }
+
+    async pullLatestChangesFromRepo(app: AppExtendedModel) {
+        console.log(`Pulling latest source for app ${app.id}...`);
+        const gitPath = PathUtils.gitRootPathForApp(app.id);
+
+        await FsUtils.deleteDirIfExistsAsync(gitPath, true);
+        await FsUtils.createDirIfNotExistsAsync(gitPath, true);
+
+        const git = simpleGit(gitPath);
+        const gitUrl = this.getGitUrl(app);
+
+        // initial clone
+        console.log(await git.clone(gitUrl, gitPath));
+        console.log(await git.checkout(app.gitBranch ?? 'main'));
+        console.log(`Source for app ${app.id} has been cloned successfully.`);
+
+        return git;
+    }
+
+
+    async checkIfLocalRepoIsUpToDate(app: AppExtendedModel) {
+        const gitPath = PathUtils.gitRootPathForApp(app.id);
+        if (!FsUtils.directoryExists(gitPath)) {
+            return false;
+        }
+
+        if (await FsUtils.isFolderEmpty(gitPath)) {
+            return false;
+        }
+
+        const git = simpleGit(gitPath);
+        await git.fetch();
+
+        const status = await git.status();
+        if (status.behind > 0) {
+            console.log(`The local repository is behind by ${status.behind} commits and needs to be updated.`);
+            return false;
+        } else if (status.ahead > 0) {
+            throw new Error(`The local repository is ahead by ${status.ahead} commits. This should not happen.`);
+        }
+
+        // The local repository is up to date
+        return true
+    }
+
+
+    private getGitUrl(app: AppExtendedModel) {
+        if (app.gitUsername && app.gitToken) {
+            return app.gitUrl!.replace('https://', `https://${app.gitUsername}:${app.gitToken}@`);
+        }
+        return app.gitUrl!;
+    }
+}
+
+const gitService = new GitService();
+export default gitService;

src/server/services/pvc.service.ts

@@ -9,6 +9,8 @@ import { MemoryCalcUtils } from "../utils/memory-caluclation.utils";
 
 class PvcService {
 
+    static readonly SHARED_PVC_NAME = 'qs-shared-pvc';
+
     async doesAppConfigurationIncreaseAnyPvcSize(app: AppExtendedModel) {
         const existingPvcs = await this.getAllPvcForApp(app.projectId, app.id);
 

src/server/utils/constants.ts

@@ -1,4 +1,6 @@
 export class Constants {
     static readonly QS_ANNOTATION_APP_ID = 'qs-app-id';
     static readonly QS_ANNOTATION_PROJECT_ID = 'qs-project-id';
+    static readonly QS_ANNOTATION_DEPLOYMENT_ID = 'qs-deplyoment-id';
+    static readonly QS_ANNOTATION_GIT_COMMIT = 'qs-git-commit';
 }

src/server/utils/fs.utils.ts

@@ -0,0 +1,67 @@
+import fs from "fs"
+import fsPromises from "fs/promises"
+
+export class FsUtils {
+
+    static async fileExists(pathName: string) {
+        try {
+            await fsPromises.access(pathName, fs.constants.F_OK);
+            return true;
+        } catch (ex) {
+            return false;
+        }
+    }
+
+    static directoryExists(pathName: string) {
+        try {
+            return fs.existsSync(pathName);
+        } catch (ex) {
+            return false;
+        }
+    }
+
+    static async isFolderEmpty(pathName: string) {
+        try {
+            const files = await fsPromises.readdir(pathName);
+            return files.length === 0;
+        } catch (ex) {
+            return true;
+        }
+    }
+
+    static createDirIfNotExists(pathName: string, recursive = false) {
+        if (!this.directoryExists(pathName)) {
+            fs.mkdirSync(pathName, {
+                recursive
+            });
+        }
+    }
+
+    static async createDirIfNotExistsAsync(pathName: string, recursive = false) {
+        let exists = false;
+        try {
+            exists = fs.existsSync(pathName);
+        } catch (ex) {
+
+        }
+        if (!exists) {
+            await fsPromises.mkdir(pathName, {
+                recursive
+            });
+        }
+    }
+    static async deleteDirIfExistsAsync(pathName: string, recursive = false) {
+        let exists = false;
+        try {
+            exists = fs.existsSync(pathName);
+        } catch (ex) {
+
+        }
+        if (!exists) {
+            return;
+        }
+        await fsPromises.rm(pathName, {
+            recursive
+        });
+    }
+}

src/server/utils/path.utils.ts

@@ -0,0 +1,26 @@
+import path from 'path';
+
+export class PathUtils {
+
+    static internalDataRoot = process.env.NODE_ENV === 'production' ? '/mnt/internal' : '/workspace/internal';
+
+    static get gitRootPath() {
+        return path.join(this.internalDataRoot, 'git');
+    }
+
+    static get deploymentLogsPath() {
+        return path.join(this.internalDataRoot, 'deployment-logs');
+    }
+
+    static appDeploymentLogFile(deploymentId: string): string {
+        return path.join(this.deploymentLogsPath, `${deploymentId}.log`);
+    }
+
+    static gitRootPathForApp(appId: string): string {
+        return path.join(PathUtils.gitRootPath, this.convertIdToFolderFreindlyName(appId));
+    }
+
+    private static convertIdToFolderFreindlyName(id: string): string {
+        return id.replace(/-/g, '_');
+    }
+}