Merge pull request #3292 from bluewave-labs/fix/incident-reason

fix/incident reason

server/src/config/services.ts

@@ -151,11 +151,14 @@ export const initializeServices = async ({
 	});
 	const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger);
 
+	const notificationMessageBuilder = new NotificationMessageBuilder();
+
 	const incidentService = new IncidentService({
 		logger,
 		incidentsRepository,
 		monitorsRepository,
 		usersRepository,
+		notificationMessageBuilder,
 	});
 
 	const checkService = new CheckService({
@@ -175,8 +178,6 @@ export const initializeServices = async ({
 	const pagerDutyProvider = new PagerDutyProvider(logger);
 	const matrixProvider = new MatrixProvider(logger);
 
-	const notificationMessageBuilder = new NotificationMessageBuilder();
-
 	const notificationsService = new NotificationsService(
 		notificationsRepository,
 		monitorsRepository,

server/src/service/business/incidentService.ts

@@ -1,10 +1,12 @@
 const SERVICE_NAME = "incidentService";
 import type { Monitor } from "@/types/monitor.js";
+import type { MonitorStatusResponse } from "@/types/network.js";
 import { AppError } from "@/utils/AppError.js";
 import { ParseBoolean } from "@/utils/utils.js";
 import type { IIncidentsRepository, IMonitorsRepository, IUsersRepository } from "@/repositories/index.js";
 import type { Incident } from "@/types/index.js";
 import type { MonitorActionDecision } from "@/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.js";
+import type { INotificationMessageBuilder } from "@/service/infrastructure/notificationMessageBuilder.js";
 
 const dateRangeLookup: Record<string, Date | undefined> = {
 	recent: new Date(new Date().setHours(new Date().getHours() - 2)),
@@ -22,29 +24,38 @@ class IncidentService {
 	private incidentsRepository: IIncidentsRepository;
 	private monitorsRepository: IMonitorsRepository;
 	private usersRepository: IUsersRepository;
+	private notificationMessageBuilder: INotificationMessageBuilder;
 
 	constructor({
 		logger,
 		incidentsRepository,
 		monitorsRepository,
 		usersRepository,
+		notificationMessageBuilder,
 	}: {
 		logger: any;
 		incidentsRepository: IIncidentsRepository;
 		monitorsRepository: IMonitorsRepository;
 		usersRepository: IUsersRepository;
+		notificationMessageBuilder: INotificationMessageBuilder;
 	}) {
 		this.logger = logger;
 		this.incidentsRepository = incidentsRepository;
 		this.monitorsRepository = monitorsRepository;
 		this.usersRepository = usersRepository;
+		this.notificationMessageBuilder = notificationMessageBuilder;
 	}
 
 	get serviceName() {
 		return IncidentService.SERVICE_NAME;
 	}
 
-	handleIncident = async (monitor: Monitor, code: number, decision: MonitorActionDecision): Promise<Incident | null> => {
+	handleIncident = async (
+		monitor: Monitor,
+		code: number,
+		decision: MonitorActionDecision,
+		monitorStatusResponse?: MonitorStatusResponse
+	): Promise<Incident | null> => {
 		if (!decision.shouldCreateIncident && !decision.shouldResolveIncident) {
 			return null;
 		}
@@ -55,12 +66,22 @@ class IncidentService {
 			if (activeIncident) {
 				return activeIncident;
 			} else {
+				let statusCode = code;
+				let message: string | undefined;
+
+				// For threshold breaches, use 9999 status code and build descriptive message
+				if (decision.incidentReason === "threshold_breach") {
+					statusCode = 9999;
+					message = this.buildThresholdBreachMessage(monitor, monitorStatusResponse);
+				}
+
 				const incident = {
 					monitorId: monitor.id,
 					teamId: monitor.teamId,
 					startTime: Date.now().toString(),
 					status: true,
-					statusCode: code,
+					statusCode,
+					message,
 				};
 				return await this.incidentsRepository.create(incident);
 			}
@@ -79,6 +100,20 @@ class IncidentService {
 		return null;
 	};
 
+	private buildThresholdBreachMessage(monitor: Monitor, monitorStatusResponse?: MonitorStatusResponse): string {
+		if (!monitorStatusResponse) {
+			return "Threshold breach detected";
+		}
+
+		const breaches = this.notificationMessageBuilder.extractThresholdBreaches(monitor, monitorStatusResponse);
+
+		if (breaches.length === 0) {
+			return "Threshold breach detected";
+		}
+
+		return breaches.map((b) => `${b.metric.toUpperCase()}: ${b.formattedValue} (threshold: ${b.threshold}${b.unit})`).join(", ");
+	}
+
 	resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string, userEmail?: string) => {
 		try {
 			if (!incidentId) {

server/src/service/infrastructure/SuperSimpleQueue/SuperSimpleQueueHelper.ts

@@ -132,7 +132,7 @@ class SuperSimpleQueueHelper {
 				}
 
 				// Step 7. Handle incidents (best effort, don't wait)
-				this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision).catch((error: any) => {
+				this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code, decision, status).catch((error: any) => {
 					this.logger.warn({
 						message: error.message,
 						service: SERVICE_NAME,
@@ -195,7 +195,6 @@ class SuperSimpleQueueHelper {
 			notificationReason: null,
 		};
 
-		// Simplified logic: Just check status changes
 		if (!statusChanged) {
 			return decision;
 		}

server/src/service/infrastructure/notificationMessageBuilder.ts

@@ -15,6 +15,7 @@ export interface INotificationMessageBuilder {
 		decision: MonitorActionDecision,
 		clientHost: string
 	): NotificationMessage;
+	extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[];
 }
 
 const SERVICE_NAME = "NotificationMessageBuilder";
@@ -192,7 +193,7 @@ export class NotificationMessageBuilder implements INotificationMessageBuilder {
 		};
 	}
 
-	private extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[] {
+	public extractThresholdBreaches(monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): ThresholdBreach[] {
 		const breaches: ThresholdBreach[] = [];
 
 		// Check if this is a hardware monitor with threshold data