added deployment logs persisted on filesystem

This commit is contained in:
biersoeckli
2024-11-28 14:29:58 +00:00
parent 4323c17c7b
commit f9d0d40183
11 changed files with 313 additions and 61 deletions
+50
View File
@@ -0,0 +1,50 @@
import { z } from "zod";
import { simpleRoute } from "@/server/utils/action-wrapper.utils";
import deploymentLogService from "@/server/services/deployment-logs.service";
// Prevents this route's response from being cached
export const dynamic = "force-dynamic";
const zodInputModel = z.object({
deploymentId: z.string(),
});
export async function POST(request: Request) {
return simpleRoute(async () => {
const input = await request.json();
const inputInfo = zodInputModel.parse(input);
let { deploymentId } = inputInfo;
let closeListenerFunc: (() => void) | undefined;
const encoder = new TextEncoder();
const customReadable = new ReadableStream({
start(controller) {
const innerFunc = async () => {
console.log(`[CONNECT] Client joined build log stream for deployment ${deploymentId}`);
controller.enqueue(encoder.encode('Stream opened, loading build logs...\n'));
closeListenerFunc = await deploymentLogService.getLogsStream(deploymentId, (chunk) => {
controller.enqueue(encoder.encode(chunk));
});
};
innerFunc();
},
cancel() {
console.log(`[DISCONNECTED] Client disconnected build log stream for deployment ${deploymentId}`);
closeListenerFunc?.();
},
})
return new Response(customReadable, {
// Set the headers for Server-Sent Events (SSE)
headers: {
Connection: "keep-alive",
"Content-Encoding": "none",
"Cache-Control": "no-cache, no-transform",
"Content-Type": "text/event-stream; charset=utf-8",
},
})
});
}
@@ -15,6 +15,7 @@ import { set } from "date-fns";
import { DeploymentInfoModel } from "@/shared/model/deployment-info.model";
import LogsStreamed from "../../../../../components/custom/logs-streamed";
import { formatDateTime } from "@/frontend/utils/format.utils";
import BuildLogsStreamed from "@/components/custom/build-logs-streamed";
export function BuildLogsDialog({
deploymentInfo,
@@ -34,14 +35,14 @@ export function BuildLogsDialog({
}}>
<DialogContent className="sm:max-w-[1300px]">
<DialogHeader>
<DialogTitle>Build Logs</DialogTitle>
<DialogTitle>Deployment Logs</DialogTitle>
<DialogDescription>
View the build logs for the selected deployment {formatDateTime(deploymentInfo.createdAt)}.
View the logs for the selected deployment {formatDateTime(deploymentInfo.createdAt)}.
</DialogDescription>
</DialogHeader>
<div >
{!deploymentInfo.buildJobName && 'For this build is no log available'}
{deploymentInfo.buildJobName && <LogsStreamed buildJobName={deploymentInfo.buildJobName} />}
{!deploymentInfo.deploymentId && 'For this build is no log available'}
{deploymentInfo.deploymentId && <BuildLogsStreamed deploymentId={deploymentInfo.deploymentId} />}
</div>
</DialogContent>
</Dialog>
@@ -77,7 +77,7 @@ export default function BuildsTab({
<SimpleDataTable columns={[
['replicasetName', 'Deployment Name', false],
['buildJobName', 'Build Job Name', false],
['buildJobName', 'Build Job Name', false],
['deploymentId', 'Deployment Id', false],
['status', 'Status', true, (item) => <DeploymentStatusBadge>{item.status}</DeploymentStatusBadge>],
["startTime", "Started At", true, (item) => formatDateTime(item.createdAt)],
['gitCommit', 'Git Commit', true, (item) => <ShortCommitHash>{item.gitCommit}</ShortCommitHash>],
@@ -88,7 +88,7 @@ export default function BuildsTab({
return <>
<div className="flex gap-4">
<div className="flex-1"></div>
{item.buildJobName && <Button variant="secondary" onClick={() => setSelectedDeploymentForLogs(item)}>Show Logs</Button>}
{item.deploymentId && <Button variant="secondary" onClick={() => setSelectedDeploymentForLogs(item)}>Show Logs</Button>}
{item.buildJobName && item.status === 'BUILDING' && <Button variant="destructive" onClick={() => deleteBuildClick(item.buildJobName!)}>Stop Build</Button>}
</div>
</>
@@ -0,0 +1,95 @@
import { useEffect, useRef, useState } from "react";
import { Textarea } from "@/components/ui/textarea";
import React from "react";
import {
HoverCard,
HoverCardContent,
HoverCardTrigger,
} from "@/components/ui/hover-card"
export default function BuildLogsStreamed({
deploymentId,
fullHeight = false,
}: {
deploymentId?: string;
fullHeight?: boolean;
}) {
const [isConnected, setIsConnected] = useState(false);
const [logs, setLogs] = useState<string>('');
const textAreaRef = useRef<HTMLTextAreaElement>(null);
const initializeConnection = async (controller: AbortController) => {
// Initiate the first call to connect to SSE API
setLogs('Loading...');
const signal = controller.signal;
const apiResponse = await fetch('/api/build-logs', {
method: "POST",
headers: {
"Content-Type": "text/event-stream",
},
body: JSON.stringify({ deploymentId }),
signal: signal,
});
if (!apiResponse.ok) return;
if (!apiResponse.body) return;
setIsConnected(true);
// To decode incoming data as a string
const reader = apiResponse.body
.pipeThrough(new TextDecoderStream())
.getReader();
setLogs('');
while (true) {
const { value, done } = await reader.read();
if (done) {
setIsConnected(false);
break;
}
if (value) {
setLogs((prevLogs) => prevLogs + value);
}
}
}
useEffect(() => {
if (!deploymentId) {
return;
}
const controller = new AbortController();
initializeConnection(controller);
return () => {
console.log('Disconnecting from logs');
setLogs('');
controller.abort();
};
}, [deploymentId]);
useEffect(() => {
if (textAreaRef.current) {
// Scroll to the bottom every time logs change
textAreaRef.current.scrollTop = textAreaRef.current.scrollHeight;
}
}, [logs]);
return <>
<div className="space-y-4">
<Textarea ref={textAreaRef} value={logs} readOnly className={(fullHeight ? "h-[80vh]" : "h-[400px]") + " bg-slate-900 text-white"} />
<div className="w-fit">
<HoverCard>
<HoverCardTrigger>
{isConnected ? <div className="w-3 h-3 rounded-full bg-green-500"></div> : <div className="w-3 h-3 rounded-full bg-slate-500"></div>}
</HoverCardTrigger>
<HoverCardContent className="text-sm">
{isConnected ? 'Connected to Logstream' : 'Disconnected from Logstream'}
</HoverCardContent>
</HoverCard>
</div>
</div>
</>;
}
+1 -2
View File
@@ -6,7 +6,6 @@ import {
HoverCardContent,
HoverCardTrigger,
} from "@/components/ui/hover-card"
import { Input } from "../ui/input";
export default function LogsStreamed({
@@ -92,7 +91,7 @@ export default function LogsStreamed({
<div className="w-fit">
<HoverCard>
<HoverCardTrigger>
{isConnected ? <div className="w-3 h-3 rounded-full bg-green-500"></div> : <div className="w-3 h-3 rounded-full bg-green-500"></div>}
{isConnected ? <div className="w-3 h-3 rounded-full bg-green-500"></div> : <div className="w-3 h-3 rounded-full bg-slate-500"></div>}
</HoverCardTrigger>
<HoverCardContent className="text-sm">
{isConnected ? 'Connected to Logstream' : 'Disconnected from Logstream'}
+26 -12
View File
@@ -12,22 +12,36 @@ import namespaceService from "./namespace.service";
import ingressService from "./ingress.service";
import pvcService from "./pvc.service";
import svcService from "./svc.service";
import deploymentLogService, { dlog } from "./deployment-logs.service";
import crypto from "crypto";
class AppService {
async buildAndDeploy(appId: string, forceBuild: boolean = false) {
const app = await this.getExtendedById(appId);
if (app.sourceType === 'GIT') {
// first make build
const [buildJobName, gitCommitHash, buildPromise] = await buildService.buildApp(app, forceBuild);
buildPromise.then(async () => {
console.warn('Build job finished, deploying...');
await deploymentService.createDeployment(app, buildJobName, gitCommitHash);
});
} else {
// only deploy
await deploymentService.createDeployment(app);
}
const deploymentId = crypto.randomUUID();
return await deploymentLogService.catchErrosAndLog(deploymentId, async () => {
const app = await this.getExtendedById(appId);
await dlog(deploymentId, `
-----------------------------------------------
Deployment: ${deploymentId}
App: ${app.id}
Project: ${app.projectId}
-----------------------------------------------`, false);
if (app.sourceType === 'GIT') {
// first make build
const [buildJobName, gitCommitHash, buildPromise] = await buildService.buildApp(deploymentId, app, forceBuild);
buildPromise.then(async () => {
console.log('Build job finished, deploying...');
dlog(deploymentId, `Build job ${buildJobName} completed successfully`);
await deploymentService.createDeployment(deploymentId, app, buildJobName, gitCommitHash);
});
} else {
// only deploy
await deploymentService.createDeployment(deploymentId, app);
}
});
}
async deleteById(id: string) {
+52 -4
View File
@@ -9,8 +9,9 @@ import namespaceService from "./namespace.service";
import { Constants } from "../../shared/utils/constants";
import gitService from "./git.service";
import deploymentService from "./deployment.service";
import deploymentLogService from "./deployment-logs.service";
import deploymentLogService, { dlog } from "./deployment-logs.service";
import podService from "./pod.service";
import stream from "stream";
const kanikoImage = "gcr.io/kaniko-project/executor:latest";
const REGISTRY_NODE_PORT = 30100;
@@ -24,7 +25,7 @@ export const REGISTRY_URL_INTERNAL = `${REGISTRY_SVC_NAME}.${BUILD_NAMESPACE}.sv
class BuildService {
async buildApp(app: AppExtendedModel, forceBuild: boolean = false): Promise<[string, string, Promise<void>]> {
async buildApp(deploymentId: string, app: AppExtendedModel, forceBuild: boolean = false): Promise<[string, string, Promise<void>]> {
await namespaceService.createNamespaceIfNotExists(BUILD_NAMESPACE);
await this.deployRegistryIfNotExists();
const buildsForApp = await this.getBuildsForApp(app.id);
@@ -32,21 +33,32 @@ class BuildService {
throw new ServiceException("A build job is already running for this app.");
}
dlog(deploymentId, `Initialized app build...`);
dlog(deploymentId, `Trying to clone repository...`);
// Check if last build is already up to date with data in git repo
const latestSuccessfulBuld = buildsForApp.find(x => x.status === 'SUCCEEDED');
const latestRemoteGitHash = await gitService.getLatestRemoteCommitHash(app);
dlog(deploymentId, `Cloned repository successfully`);
dlog(deploymentId, `Latest remote git hash: ${latestRemoteGitHash}`);
if (!forceBuild && latestSuccessfulBuld?.gitCommit && latestRemoteGitHash &&
latestSuccessfulBuld?.gitCommit === latestRemoteGitHash) {
await dlog(deploymentId, `Latest build is already up to date with git repository, using container from last build.`);
console.log(`Last build is already up to date with data in git repo for app ${app.id}`);
// todo check if the container is still in registry
return [latestSuccessfulBuld.name, latestRemoteGitHash, Promise.resolve()];
}
return await this.createAndStartBuildJob(app, latestRemoteGitHash);
return await this.createAndStartBuildJob(deploymentId, app, latestRemoteGitHash);
}
private async createAndStartBuildJob(app: AppExtendedModel, latestRemoteGitHash: string): Promise<[string, string, Promise<void>]> {
private async createAndStartBuildJob(deploymentId: string, app: AppExtendedModel, latestRemoteGitHash: string): Promise<[string, string, Promise<void>]> {
const buildName = KubeObjectNameUtils.addRandomSuffix(KubeObjectNameUtils.toJobName(app.id));
dlog(deploymentId, `Creating build job with name: ${buildName}`);
const jobDefinition: V1Job = {
apiVersion: "batch/v1",
kind: "Job",
@@ -57,6 +69,7 @@ class BuildService {
[Constants.QS_ANNOTATION_APP_ID]: app.id,
[Constants.QS_ANNOTATION_PROJECT_ID]: app.projectId,
[Constants.QS_ANNOTATION_GIT_COMMIT]: latestRemoteGitHash,
[Constants.QS_ANNOTATION_DEPLOYMENT_ID]: deploymentId,
}
},
spec: {
@@ -97,11 +110,45 @@ class BuildService {
}
await k3s.batch.createNamespacedJob(BUILD_NAMESPACE, jobDefinition);
await dlog(deploymentId, `Build job ${buildName} started successfully`);
await this.logBuildOutput(deploymentId, buildName);
const buildJobPromise = this.waitForJobCompletion(jobDefinition.metadata!.name!)
return [buildName, latestRemoteGitHash, buildJobPromise];
}
async logBuildOutput(deploymentId: string, buildName: string) {
const pod = await this.getPodForJob(buildName);
await podService.waitUntilPodIsRunningFailedOrSucceded(BUILD_NAMESPACE, pod.podName);
const logStream = new stream.PassThrough();
const k3sStreamRequest = await k3s.log.log(BUILD_NAMESPACE, pod.podName, pod.containerName, logStream, {
follow: true,
tailLines: undefined,
timestamps: true,
pretty: false,
previous: false
});
logStream.on('data', async (chunk) => {
await dlog(deploymentId, chunk.toString(), false, false);
});
logStream.on('error', async (error) => {
console.error("Error in build log stream for deployment " + deploymentId, error);
await dlog(deploymentId, '[ERROR] An unexpected error occurred while streaming logs.');
});
logStream.on('end', async () => {
console.log(`[END] Log stream ended for build process: ${buildName}`);
await dlog(deploymentId, `[END] Log stream ended for build process: ${buildName}`);
});
}
createInternalContainerRegistryUrlForAppId(appId?: string) {
if (!appId) {
return undefined;
@@ -149,6 +196,7 @@ class BuildService {
startTime: job.status?.startTime,
status: this.getJobStatusString(job.status),
gitCommit: job.metadata?.annotations?.[Constants.QS_ANNOTATION_GIT_COMMIT],
deploymentId: job.metadata?.annotations?.[Constants.QS_ANNOTATION_DEPLOYMENT_ID],
} as BuildJobModel;
});
builds.sort((a, b) => {
+60 -34
View File
@@ -1,73 +1,99 @@
import fsPromises from 'fs/promises';
import fs from 'fs';
import fs, { read } from 'fs';
import { PathUtils } from '../utils/path.utils';
import { FsUtils } from '../utils/fs.utils';
class DeploymentLogService {
async writeLogs(deploymentId: string, logs: string) {
async writeLogs(deploymentId: string, logMessage: string, addDate = true, addNewLine = true) {
try {
await FsUtils.createDirIfNotExistsAsync(PathUtils.deploymentLogsPath, true);
const now = new Date();
const logFilePath = PathUtils.appDeploymentLogFile(deploymentId);
await fsPromises.appendFile(logFilePath, logs, {
const logText = [];
if (addDate) {
logText.push(`[${now.toISOString()}]: `);
}
logText.push(logMessage);
if (addNewLine) {
logText.push('\n');
}
await fsPromises.appendFile(logFilePath, logText.join(''), {
encoding: 'utf-8'
});
} catch (ex) {
console.error(`Error writing logs for deployment ${deploymentId}: ${ex}`);
}
}
catchErrosAndLog<TReturnType>(appId: string, deploymentId: string, fn: (logFunc: (logData: string) => void) => TReturnType): TReturnType {
async catchErrosAndLog<TReturnType>(deploymentId: string, fn: () => Promise<TReturnType>): Promise<TReturnType> {
try {
return fn((logData: string) => {
this.writeLogs(deploymentId, logData);
});
await FsUtils.createDirIfNotExistsAsync(PathUtils.deploymentLogsPath, true);
return await fn();
} catch (ex) {
console.error(`Error in deployment ${deploymentId}: ${(ex as any)?.message}`, ex);
this.writeLogs(deploymentId, `[Error]: ${(ex as any)?.message}`);
throw ex;
}
}
async getLogsStream(appId: string, deploymentId: string, streamedData: (data: string) => void) {
async getLogsStream(deploymentId: string, streamedData: (data: string) => void) {
await FsUtils.createDirIfNotExistsAsync(PathUtils.deploymentLogsPath, true);
const logFilePath = PathUtils.appDeploymentLogFile(deploymentId);
if (!await FsUtils.fileExists(logFilePath)) {
streamedData(`The log file for deployment ${deploymentId} does not exist.`);
console.error(`Build Log file ${logFilePath} does not exist`);
return undefined;
}
// Create a read stream
let fileStream = fs.createReadStream(logFilePath, {
encoding: 'utf8',
start: 0,
flags: 'r'
let bytesRead = 0;
const readFileFromLastCheckpoint = () => new Promise<void>((resolve) => {
// Create a new read stream starting from the current end of the file
const newStream = fs.createReadStream(logFilePath, {
encoding: 'utf8',
start: bytesRead,
flags: 'r'
});
newStream.on('data', (chunk: string) => {
streamedData(chunk);
});
// Update the read stream pointer
newStream.on('end', () => {
bytesRead += newStream.bytesRead;
newStream.close();
resolve();
});
newStream.on('error', (err) => {
console.error(`Error reading log file ${logFilePath}: ${err}`, err);
newStream.close();
resolve();
});
});
const readerQueue: Promise<void>[] = [readFileFromLastCheckpoint()];
// Watch for changes in the file and read new lines when the file is updated
const watcher = fs.watch(logFilePath, (eventType) => {
const watcher = fs.watch(logFilePath, async (eventType) => {
if (eventType === 'change') {
// Create a new read stream starting from the current end of the file
const newStream = fs.createReadStream(logFilePath, {
encoding: 'utf8',
start: fileStream.bytesRead,
flags: 'r'
});
// wait for all the previous read operations to finish
await Promise.all([
...readerQueue
]);
newStream.on('data', (chunk: string) => {
streamedData(chunk);
});
// Update the read stream pointer
newStream.on('end', () => {
fileStream.bytesRead += newStream.readableLength; // Buffer.byteLength(); // todo check if this works
newStream.close();
});
const promise = readFileFromLastCheckpoint();
readerQueue.push(promise);
}
});
return () => {
watcher.close();
fileStream.close();
}
}
@@ -77,6 +103,6 @@ const deploymentLogService = new DeploymentLogService();
export default deploymentLogService;
export const dlog = (deploymentId: string, data: string) => {
deploymentLogService.writeLogs(deploymentId, data);
export const dlog = async (deploymentId: string, data: string, addDate = true, addNewLine = true) => {
await deploymentLogService.writeLogs(deploymentId, data, addDate, addNewLine);
}
+20 -3
View File
@@ -13,6 +13,7 @@ import ingressService from "./ingress.service";
import namespaceService from "./namespace.service";
import { Constants } from "../../shared/utils/constants";
import svcService from "./svc.service";
import { dlog } from "./deployment-logs.service";
class DeploymentService {
@@ -40,17 +41,25 @@ class DeploymentService {
}
}
async createDeployment(app: AppExtendedModel, buildJobName?: string, gitCommitHash?: string) {
async createDeployment(deplyomentId: string, app: AppExtendedModel, buildJobName?: string, gitCommitHash?: string) {
await this.validateDeployment(app);
dlog(deplyomentId, `Starting deployment of containter...`);
await namespaceService.createNamespaceIfNotExists(app.projectId);
const appHasPvcChanges = await pvcService.doesAppConfigurationIncreaseAnyPvcSize(app)
const appHasPvcChanges = await pvcService.doesAppConfigurationIncreaseAnyPvcSize(app);
if (appHasPvcChanges) {
dlog(deplyomentId, `Configuring Storage Volumes...`);
await this.setReplicasForDeployment(app.projectId, app.id, 0); // update of PVCs is only possible if deployment is scaled down
await new Promise(resolve => setTimeout(resolve, 5000));
}
const { volumes, volumeMounts } = await pvcService.createOrUpdatePvc(app);
if (volumes && volumes.length > 0) {
dlog(deplyomentId, `Configured ${volumes.length} Storage Volumes.`);
}
const envVars = this.parseEnvVariables(app);
dlog(deplyomentId, `Configured ${envVars.length} Env Variables.`);
const existingDeployment = await this.getDeployment(app.projectId, app.id);
const body: V1Deployment = {
@@ -72,6 +81,7 @@ class DeploymentService {
annotations: {
[Constants.QS_ANNOTATION_APP_ID]: app.id,
[Constants.QS_ANNOTATION_PROJECT_ID]: app.projectId,
[Constants.QS_ANNOTATION_DEPLOYMENT_ID]: deplyomentId,
deploymentTimestamp: new Date().getTime() + "",
"kubernetes.io/change-cause": `Deployment ${new Date().toISOString()}`
}
@@ -114,13 +124,18 @@ class DeploymentService {
}
if (existingDeployment) {
dlog(deplyomentId, `Replacing existing deployment...`);
const res = await k3s.apps.replaceNamespacedDeployment(app.id, app.projectId, body);
} else {
dlog(deplyomentId, `Creating deployment...`);
const res = await k3s.apps.createNamespacedDeployment(app.projectId, body);
}
await pvcService.deleteUnusedPvcOfApp(app);
dlog(deplyomentId, `Updating service...`);
await svcService.createOrUpdateService(app);
dlog(deplyomentId, `Updating ingress...`);
await ingressService.createOrUpdateIngressForApp(app);
dlog(deplyomentId, `Deployment finished.`);
}
private parseEnvVariables(app: { id: string; name: string; projectId: string; sourceType: string; dockerfilePath: string; replicas: number; envVars: string; defaultPort: number; createdAt: Date; updatedAt: Date; project: { id: string; name: string; createdAt: Date; updatedAt: Date; }; appDomains: { id: string; createdAt: Date; updatedAt: Date; hostname: string; port: number; useSsl: boolean; redirectHttps: boolean; appId: string; }[]; appVolumes: { id: string; createdAt: Date; updatedAt: Date; appId: string; containerMountPath: string; size: number; accessMode: string; }[]; containerImageSource?: string | null | undefined; gitUrl?: string | null | undefined; gitBranch?: string | null | undefined; gitUsername?: string | null | undefined; gitToken?: string | null | undefined; memoryReservation?: number | null | undefined; memoryLimit?: number | null | undefined; cpuReservation?: number | null | undefined; cpuLimit?: number | null | undefined; }) {
@@ -169,6 +184,7 @@ class DeploymentService {
buildJobName: build.name!,
status: this.mapBuildStatusToDeploymentStatus(build.status),
gitCommit: build.gitCommit,
deploymentId: build.deploymentId
}
});
replicasetRevisions.push(...runningOrFailedBuilds);
@@ -202,7 +218,8 @@ class DeploymentService {
createdAt: rs.metadata?.creationTimestamp!,
buildJobName: rs.spec?.template?.metadata?.annotations?.buildJobName!,
gitCommit: rs.spec?.template?.metadata?.annotations?.[Constants.QS_ANNOTATION_GIT_COMMIT],
status: status
status: status,
deploymentId: rs.spec?.template?.metadata?.annotations?.[Constants.QS_ANNOTATION_DEPLOYMENT_ID]!
}
});
return ListUtils.sortByDate(revisions, (i) => i.createdAt!, true);
+1
View File
@@ -8,6 +8,7 @@ export const buildJobSchemaZod = z.object({
startTime: z.date(),
status: buildJobStatusEnumZod,
gitCommit: z.string(),
deploymentId: z.string(),
});
export type BuildJobModel = z.infer<typeof buildJobSchemaZod>;
@@ -16,6 +16,7 @@ export const deploymentInfoZodModel = z.object({
createdAt: z.date(),
status: deploymentStatusEnumZod,
gitCommit: z.string().optional(),
deploymentId: z.string(),
});
export type DeploymentInfoModel = z.infer<typeof deploymentInfoZodModel>;