mirror of
https://github.com/bluewave-labs/Checkmate.git
synced 2026-05-20 08:28:48 -05:00
Generated
+1
-1
@@ -6203,4 +6203,4 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,11 @@ export class EmailProvider implements INotificationProvider {
|
||||
};
|
||||
|
||||
async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
|
||||
const subject = `Monitor ${monitor.name} infrastructure alerts`;
|
||||
// For grouped notifications (identified by ":" in name), customize subject to indicate multiple services.
|
||||
// Example: "2 services: Service A, Service B" becomes "Alert: 2 services are down"
|
||||
const isGroupedNotification = monitor.name.includes(":");
|
||||
const subject = isGroupedNotification ? `Alert: ${monitor.name} are down` : `Monitor ${monitor.name} is down`;
|
||||
|
||||
let html;
|
||||
if (monitor.type === "hardware") {
|
||||
html = this.buildHardwareEmail(monitor, monitorStatusResponse);
|
||||
|
||||
@@ -34,6 +34,18 @@ export class NotificationsService implements INotificationsService {
|
||||
private matrixProvider: INotificationProvider;
|
||||
private logger: any;
|
||||
|
||||
// Email grouping (batching) configuration
|
||||
private emailGroupingWindowMs: number;
|
||||
private pendingEmailGroups: Map<
|
||||
string,
|
||||
{
|
||||
monitors: Monitor[];
|
||||
statusResponses: MonitorStatusResponse[];
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
createdAt: number;
|
||||
}
|
||||
>;
|
||||
|
||||
constructor(
|
||||
notificationsRepository: INotificationsRepository,
|
||||
monitorsRepository: IMonitorsRepository,
|
||||
@@ -54,6 +66,22 @@ export class NotificationsService implements INotificationsService {
|
||||
this.pagerDutyProvider = pagerDutyProvider;
|
||||
this.matrixProvider = matrixProvider;
|
||||
this.logger = logger;
|
||||
|
||||
// Configure email grouping window (in milliseconds).
|
||||
// When > 0, multiple DOWN events for monitors that share the same
|
||||
// email notification within this window will be batched into a single email.
|
||||
const rawGroupingWindow = process.env.NOTIFICATION_GROUP_WINDOW_MS ?? process.env.NOTIFICATION_GROUP_WINDOW_SECONDS;
|
||||
let groupingWindowMs = 0;
|
||||
if (rawGroupingWindow) {
|
||||
const parsed = Number(rawGroupingWindow);
|
||||
if (!Number.isNaN(parsed) && parsed > 0) {
|
||||
// If value looks like seconds (small number), convert to ms.
|
||||
// This allows either milliseconds (e.g. 60000) or seconds (e.g. 60).
|
||||
groupingWindowMs = parsed <= 300 ? parsed * 1000 : parsed;
|
||||
}
|
||||
}
|
||||
this.emailGroupingWindowMs = groupingWindowMs;
|
||||
this.pendingEmailGroups = new Map();
|
||||
}
|
||||
|
||||
private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
|
||||
@@ -78,7 +106,17 @@ export class NotificationsService implements INotificationsService {
|
||||
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
|
||||
const notificationIds = monitor.notifications ?? [];
|
||||
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
|
||||
const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse));
|
||||
|
||||
const tasks = notifications.map((notification) => {
|
||||
// Only group emails, only for DOWN transitions, and only if a window is configured.
|
||||
if (notification.type === "email" && this.emailGroupingWindowMs > 0 && monitorStatusResponse.status === false) {
|
||||
return this.queueGroupedEmailNotification(notification, monitor, monitorStatusResponse);
|
||||
}
|
||||
|
||||
// For all other cases (UP notifications or non-email channels), send immediately.
|
||||
return this.send(notification, monitor, monitorStatusResponse);
|
||||
});
|
||||
|
||||
const outcomes = await Promise.all(tasks);
|
||||
const succeeded = outcomes.filter(Boolean).length;
|
||||
const failed = outcomes.length - succeeded;
|
||||
@@ -93,6 +131,103 @@ export class NotificationsService implements INotificationsService {
|
||||
return succeeded === notifications.length;
|
||||
};
|
||||
|
||||
/**
|
||||
* Queue a DOWN email notification to be potentially grouped with other
|
||||
* DOWN events for the same email notification within the configured window.
|
||||
*
|
||||
* This method returns immediately; the actual email is sent asynchronously
|
||||
* when the grouping window expires.
|
||||
*/
|
||||
private queueGroupedEmailNotification = async (
|
||||
notification: Notification,
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse
|
||||
): Promise<boolean> => {
|
||||
// If grouping is disabled, fallback to immediate send.
|
||||
if (this.emailGroupingWindowMs <= 0) {
|
||||
return await this.send(notification, monitor, monitorStatusResponse);
|
||||
}
|
||||
|
||||
const key = notification.id;
|
||||
const now = Date.now();
|
||||
const existingGroup = this.pendingEmailGroups.get(key);
|
||||
|
||||
if (!existingGroup) {
|
||||
// Create a new group and schedule a flush after the window expires.
|
||||
const timer = setTimeout(async () => {
|
||||
const group = this.pendingEmailGroups.get(key);
|
||||
if (!group) return;
|
||||
|
||||
this.pendingEmailGroups.delete(key);
|
||||
|
||||
try {
|
||||
await this.flushEmailGroup(notification, group.monitors, group.statusResponses);
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
message: error?.message,
|
||||
service: SERVICE_NAME,
|
||||
method: "flushEmailGroup",
|
||||
stack: error?.stack,
|
||||
});
|
||||
}
|
||||
}, this.emailGroupingWindowMs);
|
||||
|
||||
this.pendingEmailGroups.set(key, {
|
||||
monitors: [monitor],
|
||||
statusResponses: [monitorStatusResponse],
|
||||
timer,
|
||||
createdAt: now,
|
||||
});
|
||||
} else {
|
||||
// Append to existing group.
|
||||
existingGroup.monitors.push(monitor);
|
||||
existingGroup.statusResponses.push(monitorStatusResponse);
|
||||
}
|
||||
|
||||
// Consider queueing as "succeeded" from the caller's perspective.
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Flush a grouped set of DOWN events into a single email.
|
||||
*
|
||||
* To avoid changing email templates, we construct a synthetic Monitor
|
||||
* whose name concisely lists all affected services. The existing
|
||||
* `serverIsDownTemplate` is then reused.
|
||||
*
|
||||
* @param notification The email notification to send to
|
||||
* @param monitors Array of monitors that went down
|
||||
* @param statusResponses Array of status responses (parallel to monitors)
|
||||
* @returns true if email was sent successfully, false otherwise
|
||||
*/
|
||||
private flushEmailGroup = async (notification: Notification, monitors: Monitor[], statusResponses: MonitorStatusResponse[]): Promise<boolean> => {
|
||||
if (!monitors.length || !statusResponses.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Build a combined monitor name listing all affected services.
|
||||
// Example: "Service A, Service B" (2 services) or "Service A" (1 service)
|
||||
const uniqueNames = Array.from(new Set(monitors.map((m) => m.name)));
|
||||
const servicesCount = uniqueNames.length;
|
||||
const servicesList = uniqueNames.join(", ");
|
||||
|
||||
const combinedName = servicesCount === 1 ? servicesList : `${servicesCount} services: ${servicesList}`;
|
||||
|
||||
// Use the first monitor as a base for URL and other fields.
|
||||
const baseMonitor = monitors[0]!;
|
||||
const baseStatus = statusResponses[0]!;
|
||||
|
||||
// Create a shallow clone so we don't mutate the original entity.
|
||||
// This preserves monitor properties while overriding the name for grouped display.
|
||||
const syntheticMonitor: Monitor = {
|
||||
...baseMonitor,
|
||||
name: combinedName,
|
||||
};
|
||||
|
||||
// Reuse existing email provider to send grouped notification.
|
||||
return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus);
|
||||
};
|
||||
|
||||
handleNotifications = async (
|
||||
monitor: Monitor,
|
||||
monitorStatusResponse: MonitorStatusResponse,
|
||||
|
||||
Reference in New Issue
Block a user