Merge pull request #3213 from sukritraj02/#3164

#3164
This commit is contained in:
Alexander Holliday
2026-02-03 13:07:14 -08:00
committed by GitHub
3 changed files with 142 additions and 3 deletions
+1 -1
View File
@@ -6203,4 +6203,4 @@
}
}
}
}
}
@@ -19,7 +19,11 @@ export class EmailProvider implements INotificationProvider {
};
async sendAlert(notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> {
const subject = `Monitor ${monitor.name} infrastructure alerts`;
// For grouped notifications (identified by ":" in name), customize subject to indicate multiple services.
// Example: "2 services: Service A, Service B" becomes "Alert: 2 services are down"
const isGroupedNotification = monitor.name.includes(":");
const subject = isGroupedNotification ? `Alert: ${monitor.name} are down` : `Monitor ${monitor.name} is down`;
let html;
if (monitor.type === "hardware") {
html = this.buildHardwareEmail(monitor, monitorStatusResponse);
@@ -34,6 +34,18 @@ export class NotificationsService implements INotificationsService {
private matrixProvider: INotificationProvider;
private logger: any;
// Email grouping (batching) configuration
private emailGroupingWindowMs: number;
private pendingEmailGroups: Map<
string,
{
monitors: Monitor[];
statusResponses: MonitorStatusResponse[];
timer: ReturnType<typeof setTimeout>;
createdAt: number;
}
>;
constructor(
notificationsRepository: INotificationsRepository,
monitorsRepository: IMonitorsRepository,
@@ -54,6 +66,22 @@ export class NotificationsService implements INotificationsService {
this.pagerDutyProvider = pagerDutyProvider;
this.matrixProvider = matrixProvider;
this.logger = logger;
// Configure email grouping window (in milliseconds).
// When > 0, multiple DOWN events for monitors that share the same
// email notification within this window will be batched into a single email.
const rawGroupingWindow = process.env.NOTIFICATION_GROUP_WINDOW_MS ?? process.env.NOTIFICATION_GROUP_WINDOW_SECONDS;
let groupingWindowMs = 0;
if (rawGroupingWindow) {
const parsed = Number(rawGroupingWindow);
if (!Number.isNaN(parsed) && parsed > 0) {
// If value looks like seconds (small number), convert to ms.
// This allows either milliseconds (e.g. 60000) or seconds (e.g. 60).
groupingWindowMs = parsed <= 300 ? parsed * 1000 : parsed;
}
}
this.emailGroupingWindowMs = groupingWindowMs;
this.pendingEmailGroups = new Map();
}
private send = async (notification: Notification, monitor: Monitor, monitorStatusResponse: MonitorStatusResponse): Promise<boolean> => {
@@ -78,7 +106,17 @@ export class NotificationsService implements INotificationsService {
private sendNotifications = async (monitor: Monitor, monitorStatusResponse: MonitorStatusResponse) => {
const notificationIds = monitor.notifications ?? [];
const notifications = await this.notificationsRepository.findNotificationsByIds(notificationIds);
const tasks = notifications.map((notification) => this.send(notification, monitor, monitorStatusResponse));
const tasks = notifications.map((notification) => {
// Only group emails, only for DOWN transitions, and only if a window is configured.
if (notification.type === "email" && this.emailGroupingWindowMs > 0 && monitorStatusResponse.status === false) {
return this.queueGroupedEmailNotification(notification, monitor, monitorStatusResponse);
}
// For all other cases (UP notifications or non-email channels), send immediately.
return this.send(notification, monitor, monitorStatusResponse);
});
const outcomes = await Promise.all(tasks);
const succeeded = outcomes.filter(Boolean).length;
const failed = outcomes.length - succeeded;
@@ -93,6 +131,103 @@ export class NotificationsService implements INotificationsService {
return succeeded === notifications.length;
};
/**
* Queue a DOWN email notification to be potentially grouped with other
* DOWN events for the same email notification within the configured window.
*
* This method returns immediately; the actual email is sent asynchronously
* when the grouping window expires.
*/
private queueGroupedEmailNotification = async (
notification: Notification,
monitor: Monitor,
monitorStatusResponse: MonitorStatusResponse
): Promise<boolean> => {
// If grouping is disabled, fallback to immediate send.
if (this.emailGroupingWindowMs <= 0) {
return await this.send(notification, monitor, monitorStatusResponse);
}
const key = notification.id;
const now = Date.now();
const existingGroup = this.pendingEmailGroups.get(key);
if (!existingGroup) {
// Create a new group and schedule a flush after the window expires.
const timer = setTimeout(async () => {
const group = this.pendingEmailGroups.get(key);
if (!group) return;
this.pendingEmailGroups.delete(key);
try {
await this.flushEmailGroup(notification, group.monitors, group.statusResponses);
} catch (error: any) {
this.logger.error({
message: error?.message,
service: SERVICE_NAME,
method: "flushEmailGroup",
stack: error?.stack,
});
}
}, this.emailGroupingWindowMs);
this.pendingEmailGroups.set(key, {
monitors: [monitor],
statusResponses: [monitorStatusResponse],
timer,
createdAt: now,
});
} else {
// Append to existing group.
existingGroup.monitors.push(monitor);
existingGroup.statusResponses.push(monitorStatusResponse);
}
// Consider queueing as "succeeded" from the caller's perspective.
return true;
};
/**
* Flush a grouped set of DOWN events into a single email.
*
* To avoid changing email templates, we construct a synthetic Monitor
* whose name concisely lists all affected services. The existing
* `serverIsDownTemplate` is then reused.
*
* @param notification The email notification to send to
* @param monitors Array of monitors that went down
* @param statusResponses Array of status responses (parallel to monitors)
* @returns true if email was sent successfully, false otherwise
*/
private flushEmailGroup = async (notification: Notification, monitors: Monitor[], statusResponses: MonitorStatusResponse[]): Promise<boolean> => {
if (!monitors.length || !statusResponses.length) {
return false;
}
// Build a combined monitor name listing all affected services.
// Example: "Service A, Service B" (2 services) or "Service A" (1 service)
const uniqueNames = Array.from(new Set(monitors.map((m) => m.name)));
const servicesCount = uniqueNames.length;
const servicesList = uniqueNames.join(", ");
const combinedName = servicesCount === 1 ? servicesList : `${servicesCount} services: ${servicesList}`;
// Use the first monitor as a base for URL and other fields.
const baseMonitor = monitors[0]!;
const baseStatus = statusResponses[0]!;
// Create a shallow clone so we don't mutate the original entity.
// This preserves monitor properties while overriding the name for grouped display.
const syntheticMonitor: Monitor = {
...baseMonitor,
name: combinedName,
};
// Reuse existing email provider to send grouped notification.
return await this.emailProvider.sendAlert(notification, syntheticMonitor, baseStatus);
};
handleNotifications = async (
monitor: Monitor,
monitorStatusResponse: MonitorStatusResponse,