diff --git a/Client/src/Pages/Infrastructure/CreateMonitor/index.jsx b/Client/src/Pages/Infrastructure/CreateMonitor/index.jsx index 2171c3c19..bbcb7b05d 100644 --- a/Client/src/Pages/Infrastructure/CreateMonitor/index.jsx +++ b/Client/src/Pages/Infrastructure/CreateMonitor/index.jsx @@ -132,7 +132,7 @@ const CreateInfrastructureMonitor = () => { Object.keys(form) .filter((k) => k.startsWith(THRESHOLD_FIELD_PREFIX)) .map((k) => { - if (form[k]) thresholds[k] = form[k]; + if (form[k]) thresholds[k] = form[k] / 100; delete form[k]; delete form[k.substring(THRESHOLD_FIELD_PREFIX.length)]; }); @@ -158,6 +158,7 @@ const CreateInfrastructureMonitor = () => { : infrastructureMonitor.name, interval: infrastructureMonitor.interval * MS_PER_MINUTE, }; + delete form.notifications; if (hasValidationErrors(form, infrastructureMonitorValidation, setErrors)) { return; diff --git a/Server/db/models/Notification.js b/Server/db/models/Notification.js index 9edb0187a..868324cc2 100644 --- a/Server/db/models/Notification.js +++ b/Server/db/models/Notification.js @@ -16,6 +16,28 @@ const NotificationSchema = mongoose.Schema( phone: { type: String, }, + alertThreshold: { + type: Number, + default: 5, + }, + cpuAlertThreshold: { + type: Number, + default: function () { + return this.alertThreshold; + }, + }, + memoryAlertThreshold: { + type: Number, + default: function () { + return this.alertThreshold; + }, + }, + diskAlertThreshold: { + type: Number, + default: function () { + return this.alertThreshold; + }, + }, }, { timestamps: true, diff --git a/Server/service/emailService.js b/Server/service/emailService.js index 5977048ef..846b7c6c8 100644 --- a/Server/service/emailService.js +++ b/Server/service/emailService.js @@ -65,7 +65,7 @@ class EmailService { serverIsDownTemplate: this.loadTemplate("serverIsDown"), serverIsUpTemplate: this.loadTemplate("serverIsUp"), passwordResetTemplate: this.loadTemplate("passwordReset"), - thresholdViolatedTemplate: this.loadTemplate("thresholdViolated"), + hardwareIncidentTemplate: this.loadTemplate("hardwareIncident"), }; /** diff --git a/Server/service/jobQueue.js b/Server/service/jobQueue.js index eedeb1a5c..182f0235a 100644 --- a/Server/service/jobQueue.js +++ b/Server/service/jobQueue.js @@ -165,17 +165,12 @@ class JobQueue { // Handle status change const { monitor, statusChanged, prevStatus } = await this.statusService.updateStatus(networkResponse); - - //If status hasn't changed, we're done - if (statusChanged === false) return; - - // if prevStatus is undefined, monitor is resuming, we're done - if (prevStatus === undefined) return; - + // Handle notifications this.notificationService.handleNotifications({ ...networkResponse, monitor, prevStatus, + statusChanged, }); } catch (error) { this.logger.error({ diff --git a/Server/service/notificationService.js b/Server/service/notificationService.js index f464c08a7..25bf5bfa6 100644 --- a/Server/service/notificationService.js +++ b/Server/service/notificationService.js @@ -14,15 +14,34 @@ class NotificationService { } /** - * Sends an email notification based on the network response. + * Sends an email notification for hardware infrastructure alerts * - * @param {Object} networkResponse - The response from the network monitor. - * @param {Object} networkResponse.monitor - The monitor object containing details about the monitored service. - * @param {string} networkResponse.monitor.name - The name of the monitor. - * @param {string} networkResponse.monitor.url - The URL of the monitor. - * @param {boolean} networkResponse.status - The current status of the monitor (true for up, false for down). - * @param {boolean} networkResponse.prevStatus - The previous status of the monitor (true for up, false for down). - * @param {string} address - The email address to send the notification to. + * @async + * @function sendHardwareEmail + * @param {Object} networkResponse - Response object containing monitor information + * @param {string} address - Email address to send the notification to + * @param {Array} [alerts=[]] - List of hardware alerts to include in the email + * @returns {Promise} - Indicates whether email was sent successfully + * @throws {Error} + */ + async sendHardwareEmail(networkResponse, address, alerts = []) { + if (alerts.length === 0) return false; + const { monitor, status, prevStatus } = networkResponse; + const template = "hardwareIncidentTemplate"; + const context = { monitor: monitor.name, url: monitor.url, alerts }; + const subject = `Monitor ${monitor.name} infrastructure alerts`; + this.emailService.buildAndSendEmail(template, context, address, subject); + return true; + } + + /** + * Sends an email notification about monitor status change + * + * @async + * @function sendEmail + * @param {Object} networkResponse - Response object containing monitor status information + * @param {string} address - Email address to send the notification to + * @returns {Promise} - Indicates email was sent successfully */ async sendEmail(networkResponse, address) { const { monitor, status, prevStatus } = networkResponse; @@ -30,25 +49,133 @@ class NotificationService { const context = { monitor: monitor.name, url: monitor.url }; const subject = `Monitor ${monitor.name} is ${status === true ? "up" : "down"}`; this.emailService.buildAndSendEmail(template, context, address, subject); + return true; } - /** - * Handles notifications based on the network response. - * - * @param {Object} networkResponse - The response from the network monitor. - * @param {string} networkResponse.monitorId - The ID of the monitor. - */ - async handleNotifications(networkResponse) { + async handleStatusNotifications(networkResponse) { try { + //If status hasn't changed, we're done + if (networkResponse.statusChanged === false) return false; + + // if prevStatus is undefined, monitor is resuming, we're done + if (networkResponse.prevStatus === undefined) return false; const notifications = await this.db.getNotificationsByMonitorId( networkResponse.monitorId ); + for (const notification of notifications) { if (notification.type === "email") { this.sendEmail(networkResponse, notification.address); } // Handle other types of notifications here } + return true; + } catch (error) { + this.logger.warn({ + message: error.message, + service: this.SERVICE_NAME, + method: "handleNotifications", + stack: error.stack, + }); + } + } + /** + * Handles status change notifications for a monitor + * + * @async + * @function handleStatusNotifications + * @param {Object} networkResponse - Response object containing monitor status information + * @returns {Promise} - Indicates whether notifications were processed + * @throws {Error} + */ + async handleHardwareNotifications(networkResponse) { + const thresholds = networkResponse?.monitor?.thresholds; + if (thresholds === undefined) return false; // No thresholds set, we're done + + // Get thresholds from monitor + const { + usage_cpu: cpuThreshold = -1, + usage_memory: memoryThreshold = -1, + usage_disk: diskThreshold = -1, + } = thresholds; + + // Get metrics from response + const metrics = networkResponse?.payload?.data ?? null; + if (metrics === null) return false; + + const { + cpu: { usage_percent: cpuUsage = -1 } = {}, + memory: { usage_percent: memoryUsage = -1 } = {}, + disk = [], + } = metrics; + + const alerts = { + cpu: cpuThreshold !== -1 && cpuUsage > cpuThreshold ? true : false, + memory: memoryThreshold !== -1 && memoryUsage > memoryThreshold ? true : false, + disk: disk.some((d) => diskThreshold !== -1 && d.usage_percent > diskThreshold) + ? true + : false, + }; + + const notifications = await this.db.getNotificationsByMonitorId( + networkResponse.monitorId + ); + for (const notification of notifications) { + const alertsToSend = []; + const alertTypes = ["cpu", "memory", "disk"]; + + for (const type of alertTypes) { + // Iterate over each alert type to see if any need to be decremented + if (alerts[type] === true) { + notification[`${type}AlertThreshold`]--; // Decrement threshold if an alert is triggered + + if (notification[`${type}AlertThreshold`] <= 0) { + // If threshold drops below 0, reset and send notification + notification[`${type}AlertThreshold`] = notification.alertThreshold; + + const formatAlert = { + cpu: () => + `Your current CPU usage (${(cpuUsage * 100).toFixed(0)}%) is above your threshold (${(cpuThreshold * 100).toFixed(0)}%)`, + memory: () => + `Your current memory usage (${(memoryUsage * 100).toFixed(0)}%) is above your threshold (${(memoryThreshold * 100).toFixed(0)}%)`, + disk: () => + `Your current disk usage: ${disk + .map((d, idx) => `(Disk${idx}: ${(d.usage_percent * 100).toFixed(0)}%)`) + .join( + ", " + )} is above your threshold (${(diskThreshold * 100).toFixed(0)}%)`, + }; + alertsToSend.push(formatAlert[type]()); + } + } + } + + await notification.save(); + + if (alertsToSend.length === 0) continue; // No alerts to send, we're done + + if (notification.type === "email") { + this.sendHardwareEmail(networkResponse, notification.address, alertsToSend); + } + } + return true; + } + + /** + * Handles notifications for different monitor types + * + * @async + * @function handleNotifications + * @param {Object} networkResponse - Response object containing monitor information + * @returns {Promise} - Indicates whether notifications were processed successfully + */ + async handleNotifications(networkResponse) { + try { + if (networkResponse.monitor.type === "hardware") { + this.handleHardwareNotifications(networkResponse); + } + this.handleStatusNotifications(networkResponse); + return true; } catch (error) { this.logger.warn({ message: error.message, diff --git a/Server/service/statusService.js b/Server/service/statusService.js index 4f7c1a4ce..442329620 100644 --- a/Server/service/statusService.js +++ b/Server/service/statusService.js @@ -29,7 +29,8 @@ class StatusService { const { monitorId, status } = networkResponse; const monitor = await this.db.getMonitorById(monitorId); // No change in monitor status, return early - if (monitor.status === status) return { statusChanged: false }; + if (monitor.status === status) + return { monitor, statusChanged: false, prevStatus: monitor.status }; // Monitor status changed, save prev status and update monitor this.logger.info({ @@ -103,7 +104,7 @@ class StatusService { if (type === "hardware") { const { cpu, memory, disk, host } = payload?.data ?? {}; - const { errors } = payload; + const { errors } = payload?.errors ?? []; check.cpu = cpu ?? {}; check.memory = memory ?? {}; check.disk = disk ?? {}; diff --git a/Server/templates/hardwareIncident.mjml b/Server/templates/hardwareIncident.mjml new file mode 100644 index 000000000..e27933fb9 --- /dev/null +++ b/Server/templates/hardwareIncident.mjml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + Message from BlueWave Infrastructure Monitoring + + + + + Infrastructure Alerts + + + + + + + +

Hello {{name}}!

+

{{monitor}} at {{url}} has the following infrastructure alerts:

+ {{#each alerts}} +

• {{this}}

+ {{/each}} + + + + + View Infrastructure Details + +

This email was sent by BlueWave Infrastructure Monitoring.

+ + + + + \ No newline at end of file diff --git a/Server/templates/thresholdViolated.mjml b/Server/templates/thresholdViolated.mjml deleted file mode 100644 index a3d4344e4..000000000 --- a/Server/templates/thresholdViolated.mjml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - Message from BlueWave Uptime Service - - - - - {{message}} - - - {{#if cpu}} - {{cpu}} - {{/if}} - - - {{#if disk}} - {{disk}} - {{/if}} - - - {{#if memory}} - {{memory}} - {{/if}} - - - - - - \ No newline at end of file diff --git a/Server/tests/services/notificationService.test.js b/Server/tests/services/notificationService.test.js index b33bd21ed..ccfdd22ae 100644 --- a/Server/tests/services/notificationService.test.js +++ b/Server/tests/services/notificationService.test.js @@ -78,11 +78,23 @@ describe("NotificationService", () => { describe("handleNotifications", async () => { it("should handle notifications based on the network response", async () => { notificationService.sendEmail = sinon.stub(); - notificationService.db.getNotificationsByMonitorId.resolves([ - { type: "email", address: "www.google.com" }, - ]); - await notificationService.handleNotifications({ monitorId: "123" }); - expect(notificationService.sendEmail.calledOnce).to.be.true; + const res = await notificationService.handleNotifications({ + monitor: { + type: "email", + address: "www.google.com", + }, + }); + expect(res).to.be.true; + }); + it("should handle hardware notifications", async () => { + notificationService.sendEmail = sinon.stub(); + const res = await notificationService.handleNotifications({ + monitor: { + type: "hardware", + address: "www.google.com", + }, + }); + expect(res).to.be.true; }); it("should handle an error when getting notifications", async () => { @@ -92,4 +104,184 @@ describe("NotificationService", () => { expect(notificationService.logger.warn.calledOnce).to.be.true; }); }); + + describe("sendHardwareEmail", async () => { + let networkResponse, address, alerts; + beforeEach(() => { + networkResponse = { + monitor: { + name: "Test Monitor", + url: "http://test.com", + }, + status: true, + prevStatus: false, + }; + address = "test@test.com"; + alerts = ["test"]; + }); + + afterEach(() => { + sinon.restore(); + }); + it("should send an email notification with Hardware Template", async () => { + emailService.buildAndSendEmail.resolves(true); + const res = await notificationService.sendHardwareEmail( + networkResponse, + address, + alerts + ); + expect(res).to.be.true; + }); + it("should return false if no alerts are provided", async () => { + alerts = []; + emailService.buildAndSendEmail.resolves(true); + const res = await notificationService.sendHardwareEmail( + networkResponse, + address, + alerts + ); + expect(res).to.be.false; + }); + }); + describe("handleStatusNotifications", async () => { + let networkResponse; + beforeEach(() => { + networkResponse = { + monitor: { + name: "Test Monitor", + url: "http://test.com", + }, + statusChanged: true, + status: true, + prevStatus: false, + }; + }); + + afterEach(() => { + sinon.restore(); + }); + + it("should handle status notifications", async () => { + db.getNotificationsByMonitorId.resolves([ + { type: "email", address: "test@test.com" }, + ]); + const res = await notificationService.handleStatusNotifications(networkResponse); + expect(res).to.be.true; + }); + it("should return false if status hasn't changed", async () => { + networkResponse.statusChanged = false; + const res = await notificationService.handleStatusNotifications(networkResponse); + expect(res).to.be.false; + }); + it("should return false if prevStatus is undefined", async () => { + networkResponse.prevStatus = undefined; + const res = await notificationService.handleStatusNotifications(networkResponse); + expect(res).to.be.false; + }); + it("should handle an error", async () => { + const testError = new Error("Test Error"); + db.getNotificationsByMonitorId.rejects(testError); + try { + await notificationService.handleStatusNotifications(networkResponse); + } catch (error) { + expect(error).to.be.an.instanceOf(Error); + expect(error.message).to.equal("Test Error"); + } + }); + }); + + describe("handleHardwareNotifications", async () => { + let networkResponse; + beforeEach(() => { + networkResponse = { + monitor: { + name: "Test Monitor", + url: "http://test.com", + thresholds: { + usage_cpu: 1, + usage_memory: 1, + usage_disk: 1, + }, + }, + payload: { + data: { + cpu: { + usage_percent: 0.655, + }, + memory: { + usage_percent: 0.783, + }, + disk: [ + { + name: "/dev/sda1", + usage_percent: 0.452, + }, + { + name: "/dev/sdb1", + usage_percent: 0.627, + }, + ], + }, + }, + }; + }); + afterEach(() => { + sinon.restore(); + }); + + describe("it should return false if no thresholds are set", () => { + it("should return false if no thresholds are set", async () => { + networkResponse.monitor.thresholds = undefined; + const res = + await notificationService.handleHardwareNotifications(networkResponse); + expect(res).to.be.false; + }); + + it("should return false if metrics are null", async () => { + networkResponse.payload.data = null; + const res = + await notificationService.handleHardwareNotifications(networkResponse); + expect(res).to.be.false; + }); + + it("should return true if request is well formed and thresholds > 0", async () => { + db.getNotificationsByMonitorId.resolves([ + { + type: "email", + address: "test@test.com", + alertThreshold: 1, + cpuAlertThreshold: 1, + memoryAlertThreshold: 1, + diskAlertThreshold: 1, + save: sinon.stub().resolves(), + }, + ]); + const res = + await notificationService.handleHardwareNotifications(networkResponse); + expect(res).to.be.true; + }); + + it("should return true if thresholds are exceeded", async () => { + db.getNotificationsByMonitorId.resolves([ + { + type: "email", + address: "test@test.com", + alertThreshold: 1, + cpuAlertThreshold: 1, + memoryAlertThreshold: 1, + diskAlertThreshold: 1, + save: sinon.stub().resolves(), + }, + ]); + networkResponse.monitor.thresholds = { + usage_cpu: 0.01, + usage_memory: 0.01, + usage_disk: 0.01, + }; + const res = + await notificationService.handleHardwareNotifications(networkResponse); + expect(res).to.be.true; + }); + }); + }); });