refactor: centralize monitoring data insertion logic

Introduces a unified function for monitoring data insertion,
adds support for group monitor status updates, and ensures
consistency by routing all monitoring data writes through the
new logic. Removes duplicated validation and streamlines
group monitor handling for improved reliability.

Relates to #123

src/lib/server/controllers/controller.js

@@ -9,6 +9,7 @@ import {
   IsValidNav,
   IsValidURL,
 } from "./validators.js";
+import { siteDataKeys } from "./siteDataKeys.js";
 import db from "../db/db.js";
 import bcrypt from "bcrypt";
 import jwt from "jsonwebtoken";
@@ -17,7 +18,7 @@ import Queue from "queue";
 
 import crypto from "crypto";
 import { addMonths, format, startOfMonth, subMonths } from "date-fns";
-import { DEGRADED, DOWN, NO_DATA, SIGNAL, UP } from "../constants.js";
+import { DEGRADED, DOWN, NO_DATA, SIGNAL, UP, REALTIME } from "../constants.js";
 import {
   GetMinuteStartNowTimestampUTC,
   GetMinuteStartTimestampUTC,
@@ -42,208 +43,11 @@ const eventQueue = new Queue({
   autostart: true, // Automatically start the queue (optional)
 });
 
-const siteDataKeys = [
-  {
-    key: "title",
-    isValid: (value) => typeof value === "string" && value.trim().length > 0,
-    data_type: "string",
-  },
-  {
-    key: "siteName",
-    isValid: (value) => typeof value === "string" && value.trim().length > 0,
-    data_type: "string",
-  },
-  {
-    key: "siteURL",
-    isValid: IsValidURL,
-    data_type: "string",
-  },
-  {
-    key: "home",
-    isValid: (value) => typeof value === "string" && value.trim().length > 0,
-    data_type: "string",
-  },
-  {
-    key: "favicon",
-    isValid: (value) => typeof value === "string" && value.trim().length > 0,
-    data_type: "string",
-  },
-  {
-    key: "logo",
-    isValid: (value) => typeof value === "string" && value.trim().length > 0,
-    data_type: "string",
-  },
-  {
-    key: "metaTags",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "nav",
-    isValid: IsValidNav,
-    data_type: "object",
-  },
-  {
-    key: "hero",
-    isValid: IsValidHero,
-    data_type: "object",
-  },
-  {
-    key: "footerHTML",
-    isValid: (value) => typeof value === "string",
-    data_type: "string",
-  },
-  {
-    key: "kenerTheme",
-    isValid: (value) => typeof value === "string",
-    data_type: "string",
-  },
-  {
-    key: "customCSS",
-    isValid: (value) => typeof value === "string",
-    data_type: "string",
-  },
-  {
-    key: "i18n",
-    isValid: IsValidI18n,
-    data_type: "object",
-  },
-  {
-    key: "pattern",
-    //string dots or squares or circle
-    isValid: (value) =>
-      typeof value === "string" &&
-      [
-        "dots",
-        "squares",
-        "tiles",
-        "none",
-        "radial-blue",
-        "radial-mono",
-        "radial-midnight",
-        "circle-mono",
-        "carbon-fibre",
-        "texture-sky",
-        "angular-mono",
-        "angular-spring",
-        "angular-bloom",
-        "pets",
-      ].includes(value),
-    data_type: "string",
-  },
-  {
-    key: "analytics",
-    isValid: IsValidAnalytics,
-    data_type: "object",
-  },
-  {
-    key: "theme",
-    //light dark system none
-    isValid: (value) => typeof value === "string" && ["light", "dark", "system", "none"].includes(value),
-    data_type: "string",
-  },
-  {
-    key: "themeToggle",
-    //boolean
-    isValid: (value) => typeof value === "string",
-    data_type: "string",
-  },
-  {
-    key: "tzToggle",
-    //boolean
-    isValid: (value) => typeof value === "string",
-    data_type: "string",
-  },
-  {
-    key: "showSiteStatus",
-    //boolean
-    isValid: (value) => typeof value === "string",
-    data_type: "string",
-  },
-  {
-    key: "barStyle",
-    //PARTIAL or FULL
-    isValid: (value) => typeof value === "string" && ["PARTIAL", "FULL"].includes(value),
-    data_type: "string",
-  },
-  {
-    key: "barRoundness",
-    //SHARP or ROUNDED
-    isValid: (value) => typeof value === "string" && ["SHARP", "ROUNDED"].includes(value),
-    data_type: "string",
-  },
-  {
-    key: "summaryStyle",
-    //CURRENT or DAY
-    isValid: (value) => typeof value === "string" && ["CURRENT", "DAY"].includes(value),
-    data_type: "string",
-  },
-  {
-    key: "colors",
-    isValid: IsValidColors,
-    data_type: "object",
-  },
-  {
-    key: "font",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "monitorSort",
-    isValid: IsValidJSONArray,
-    data_type: "object",
-  },
-  {
-    key: "categories",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "homeIncidentCount",
-    isValid: (value) => parseInt(value) >= 0,
-    data_type: "string",
-  },
-  {
-    key: "homeIncidentStartTimeWithin",
-    isValid: (value) => parseInt(value) >= 1,
-    data_type: "string",
-  },
-  {
-    key: "incidentGroupView",
-    isValid: (value) => typeof value === "string" && value.trim().length > 0,
-    data_type: "string",
-  },
-  {
-    key: "analytics.googleTagManager",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "analytics.plausible",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "analytics.mixpanel",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "analytics.amplitude",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "analytics.clarity",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-  {
-    key: "analytics.umami",
-    isValid: IsValidJSONString,
-    data_type: "object",
-  },
-];
+const insertStatusQueue = new Queue({
+  concurrency: 1, // Number of tasks that can run concurrently
+  timeout: 10000, // Timeout in ms after which a task will be considered as failed (optional)
+  autostart: true, // Automatically start the queue (optional)
+});
 
 export const PushDataToQueue = async (eventID, eventName, eventData) => {
   //fetch subscription trigger config from db of email type
@@ -589,6 +393,100 @@ export const GetLastHeartbeat = async (monitor_tag) => {
   return await db.getLastHeartbeat(monitor_tag);
 };
 
+export const ProcessGroupUpdate = async (data) => {
+  //find all active monitor that are of type group
+  let groupActiveMonitors = await db.getMonitors({ status: "ACTIVE", monitor_type: "GROUP" });
+  let validMonitorTags = [];
+
+  for (let i = 0; i < groupActiveMonitors.length; i++) {
+    let groupActiveMonitor = groupActiveMonitors[i];
+    let typeData = JSON.parse(groupActiveMonitor.type_data);
+    let monitorsInGroup = typeData.monitors;
+    let selectedMonitorTags = monitorsInGroup
+      .filter((monitor) => {
+        if (!!monitor.selected) {
+          return monitor.tag;
+        }
+      })
+      .map((monitor) => monitor.tag);
+    validMonitorTags.push({
+      groupTag: groupActiveMonitor.tag,
+      selectedMonitorTags: selectedMonitorTags,
+    });
+  }
+
+  for (let i = 0; i < validMonitorTags.length; i++) {
+    let groupActiveMonitor = validMonitorTags[i];
+    if (groupActiveMonitor.selectedMonitorTags.indexOf(data.monitor_tag) !== -1) {
+      //do db insert
+      //get last status by tag for the group tag
+      let updateData = {};
+      let lastStatus = await db.getMonitoringDataAt(groupActiveMonitor.groupTag, data.timestamp);
+      if (!!lastStatus) {
+        let status = lastStatus.status;
+        let timestamp = lastStatus.timestamp;
+        let receivedStatus = data.status;
+        let receivedTimestamp = data.timestamp;
+        if (receivedStatus === DOWN) {
+          updateData = {
+            monitor_tag: groupActiveMonitor.groupTag,
+            timestamp: receivedTimestamp,
+            status: DOWN,
+            type: REALTIME,
+            latency: data.latency,
+          };
+        } else if (receivedStatus === DEGRADED && status !== DOWN) {
+          updateData = {
+            monitor_tag: groupActiveMonitor.groupTag,
+            timestamp: receivedTimestamp,
+            status: DEGRADED,
+            type: REALTIME,
+            latency: data.latency,
+          };
+        } else if (receivedStatus === UP && status !== DOWN && status !== DEGRADED) {
+          updateData = {
+            monitor_tag: groupActiveMonitor.groupTag,
+            timestamp: receivedTimestamp,
+            status: UP,
+            type: REALTIME,
+            latency: data.latency,
+          };
+        }
+      } else {
+        //if no last status then insert the new status
+        updateData = {
+          monitor_tag: groupActiveMonitor.groupTag,
+          timestamp: data.timestamp,
+          status: data.status,
+          type: REALTIME,
+          latency: data.latency,
+        };
+      }
+      if (!!updateData.status) {
+        await db.insertMonitoringData(updateData);
+      }
+    }
+  }
+};
+
+export const InsertMonitoringData = async (data) => {
+  //do validation if present all fields below
+  if (!data.monitor_tag || !data.timestamp || !data.status || !data.type) {
+    throw new Error("Invalid data");
+  }
+  insertStatusQueue.push(async (cb) => {
+    await ProcessGroupUpdate(data);
+    cb();
+  });
+  return await db.insertMonitoringData({
+    monitor_tag: data.monitor_tag,
+    timestamp: data.timestamp,
+    status: data.status,
+    latency: data.latency || 0,
+    type: data.type,
+  });
+};
+
 export const RegisterHeartbeat = async (tag, secret) => {
   let monitor = await db.getMonitorByTag(tag);
   if (!monitor) {
@@ -602,7 +500,7 @@ export const RegisterHeartbeat = async (tag, secret) => {
     let heartbeatConfig = JSON.parse(typeData);
     let heartbeatSecret = heartbeatConfig.secretString;
     if (heartbeatSecret === secret) {
-      return await db.insertMonitoringData({
+      return InsertMonitoringData({
         monitor_tag: monitor.tag,
         timestamp: GetMinuteStartNowTimestampUTC(GetNowTimestampUTC()),
         status: UP,

src/lib/server/controllers/siteDataKeys.js

@@ -0,0 +1,214 @@
+// @ts-nocheck
+import {
+  IsValidAnalytics,
+  IsValidColors,
+  IsValidHero,
+  IsValidI18n,
+  IsValidJSONArray,
+  IsValidJSONString,
+  IsValidNav,
+  IsValidURL,
+} from "./validators.js";
+
+export const siteDataKeys = [
+  {
+    key: "title",
+    isValid: (value) => typeof value === "string" && value.trim().length > 0,
+    data_type: "string",
+  },
+  {
+    key: "siteName",
+    isValid: (value) => typeof value === "string" && value.trim().length > 0,
+    data_type: "string",
+  },
+  {
+    key: "siteURL",
+    isValid: IsValidURL,
+    data_type: "string",
+  },
+  {
+    key: "home",
+    isValid: (value) => typeof value === "string" && value.trim().length > 0,
+    data_type: "string",
+  },
+  {
+    key: "favicon",
+    isValid: (value) => typeof value === "string" && value.trim().length > 0,
+    data_type: "string",
+  },
+  {
+    key: "logo",
+    isValid: (value) => typeof value === "string" && value.trim().length > 0,
+    data_type: "string",
+  },
+  {
+    key: "metaTags",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "nav",
+    isValid: IsValidNav,
+    data_type: "object",
+  },
+  {
+    key: "hero",
+    isValid: IsValidHero,
+    data_type: "object",
+  },
+  {
+    key: "footerHTML",
+    isValid: (value) => typeof value === "string",
+    data_type: "string",
+  },
+  {
+    key: "kenerTheme",
+    isValid: (value) => typeof value === "string",
+    data_type: "string",
+  },
+  {
+    key: "customCSS",
+    isValid: (value) => typeof value === "string",
+    data_type: "string",
+  },
+  {
+    key: "i18n",
+    isValid: IsValidI18n,
+    data_type: "object",
+  },
+  {
+    key: "pattern",
+    //string dots or squares or circle
+    isValid: (value) =>
+      typeof value === "string" &&
+      [
+        "dots",
+        "squares",
+        "tiles",
+        "none",
+        "radial-blue",
+        "radial-mono",
+        "radial-midnight",
+        "circle-mono",
+        "carbon-fibre",
+        "texture-sky",
+        "angular-mono",
+        "angular-spring",
+        "angular-bloom",
+        "pets",
+      ].includes(value),
+    data_type: "string",
+  },
+  {
+    key: "analytics",
+    isValid: IsValidAnalytics,
+    data_type: "object",
+  },
+  {
+    key: "theme",
+    //light dark system none
+    isValid: (value) => typeof value === "string" && ["light", "dark", "system", "none"].includes(value),
+    data_type: "string",
+  },
+  {
+    key: "themeToggle",
+    //boolean
+    isValid: (value) => typeof value === "string",
+    data_type: "string",
+  },
+  {
+    key: "tzToggle",
+    //boolean
+    isValid: (value) => typeof value === "string",
+    data_type: "string",
+  },
+  {
+    key: "showSiteStatus",
+    //boolean
+    isValid: (value) => typeof value === "string",
+    data_type: "string",
+  },
+  {
+    key: "barStyle",
+    //PARTIAL or FULL
+    isValid: (value) => typeof value === "string" && ["PARTIAL", "FULL"].includes(value),
+    data_type: "string",
+  },
+  {
+    key: "barRoundness",
+    //SHARP or ROUNDED
+    isValid: (value) => typeof value === "string" && ["SHARP", "ROUNDED"].includes(value),
+    data_type: "string",
+  },
+  {
+    key: "summaryStyle",
+    //CURRENT or DAY
+    isValid: (value) => typeof value === "string" && ["CURRENT", "DAY"].includes(value),
+    data_type: "string",
+  },
+  {
+    key: "colors",
+    isValid: IsValidColors,
+    data_type: "object",
+  },
+  {
+    key: "font",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "monitorSort",
+    isValid: IsValidJSONArray,
+    data_type: "object",
+  },
+  {
+    key: "categories",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "homeIncidentCount",
+    isValid: (value) => parseInt(value) >= 0,
+    data_type: "string",
+  },
+  {
+    key: "homeIncidentStartTimeWithin",
+    isValid: (value) => parseInt(value) >= 1,
+    data_type: "string",
+  },
+  {
+    key: "incidentGroupView",
+    isValid: (value) => typeof value === "string" && value.trim().length > 0,
+    data_type: "string",
+  },
+  {
+    key: "analytics.googleTagManager",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "analytics.plausible",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "analytics.mixpanel",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "analytics.amplitude",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "analytics.clarity",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+  {
+    key: "analytics.umami",
+    isValid: IsValidJSONString,
+    data_type: "object",
+  },
+];

src/lib/server/cron-minute.js

@@ -12,6 +12,7 @@ import path from "path";
 import db from "./db/db.js";
 import notification from "./notification/notif.js";
 import DNSResolver from "./dns.js";
+import { InsertMonitoringData } from "./controllers/controller.js";
 
 dotenv.config();
 
@@ -83,7 +84,7 @@ const Minuter = async (monitor) => {
           await Wait(500); //wait for 500ms
           console.log("Retrying api call for " + monitor.name + " at " + startOfMinute + " due to timeout");
           serviceClient.execute().then(async (data) => {
-            await db.insertMonitoringData({
+            await InsertMonitoringData({
               monitor_tag: monitor.tag,
               timestamp: startOfMinute,
               status: data.status,
@@ -101,7 +102,8 @@ const Minuter = async (monitor) => {
     } else if (monitor.monitor_type === "DNS") {
       realTimeData[startOfMinute] = await serviceClient.execute();
     } else if (monitor.monitor_type === "GROUP") {
-      realTimeData[startOfMinute] = await serviceClient.execute(startOfMinute);
+      // realTimeData[startOfMinute] = await serviceClient.execute(startOfMinute);
+      realTimeData[startOfMinute] = null;
     } else if (monitor.monitor_type === "SSL") {
       realTimeData[startOfMinute] = await serviceClient.execute();
     } else if (monitor.monitor_type === "SQL") {
@@ -139,7 +141,7 @@ const Minuter = async (monitor) => {
 
     for (const timestamp in mergedData) {
       const element = mergedData[timestamp];
-      db.insertMonitoringData({
+      await InsertMonitoringData({
         monitor_tag: monitor.tag,
         timestamp: parseInt(timestamp),
         status: element.status,

src/lib/server/db/dbimpl.js

@@ -48,6 +48,16 @@ class DbImpl {
       .first();
   }
 
+  //get data at time stamp
+  async getMonitoringDataAt(monitor_tag, timestamp) {
+    return await this.knex("monitoring_data")
+      .where("monitor_tag", monitor_tag)
+      .where("timestamp", timestamp)
+      .orderBy("timestamp", "desc")
+      .limit(1)
+      .first();
+  }
+
   //get latest data for all active monitors
   async getLatestMonitoringDataAllActive(monitor_tags) {
     // Find the latest timestamp for each provided monitor tag
@@ -478,6 +488,10 @@ class DbImpl {
     if (!!data.id) {
       query = query.andWhere("id", data.id);
     }
+    //monitor_type
+    if (!!data.monitor_type) {
+      query = query.andWhere("monitor_type", data.monitor_type);
+    }
     if (!!data.tag) {
       query = query.andWhere("tag", data.tag);
     }

src/lib/server/services/groupCall.js

@@ -4,27 +4,16 @@ import { GetRequiredSecrets, ReplaceAllOccurrences, Wait, GetMinuteStartNowTimes
 import { UP, DOWN, DEGRADED, REALTIME, TIMEOUT, ERROR, MANUAL } from "../constants.js";
 import db from "../db/db.js";
 
-async function waitForDataAndReturn(arr, ts, d, maxTime) {
-  await Wait(d);
-
-  let data = await db.getLastStatusBeforeCombined(arr, ts);
-  if (data) {
-    data.type = REALTIME;
-    return data;
-  } else if (d > maxTime) {
-    data = await db.getLastStatusBeforeCombined(arr, ts, ts - 86400);
-    if (data) {
-      data.type = REALTIME;
-      return data;
-    }
+async function waitForDataAndReturn(tag) {
+  let res = await db.getLatestMonitoringData(tag);
+  if (!!res) {
     return {
-      status: DOWN,
-      latency: 0,
-      type: TIMEOUT,
+      status: res.status,
+      latency: res.latency,
+      type: REALTIME,
     };
-  } else {
-    return await waitForDataAndReturn(arr, ts, d + 500, maxTime);
   }
+  return null;
 }
 
 class GroupCall {
@@ -39,7 +28,7 @@ class GroupCall {
       startOfMinute = GetMinuteStartNowTimestampUTC();
     }
     let tagArr = this.monitor.type_data.monitors.map((m) => m.tag);
-    return await waitForDataAndReturn(tagArr, startOfMinute, 500, this.monitor.type_data.timeout);
+    return await waitForDataAndReturn(this.monitor.tag);
   }
 }
 

src/lib/server/webhook.js

@@ -9,7 +9,7 @@ import {
 import db from "./db/db.js";
 import { WEBHOOK } from "./constants.js";
 
-import { GetMonitors, VerifyAPIKey } from "./controllers/controller.js";
+import { GetMonitors, VerifyAPIKey, InsertMonitoringData } from "./controllers/controller.js";
 
 const GetAllTags = async function () {
   let tags = [];
@@ -94,7 +94,7 @@ const store = async function (data) {
 
   //get the monitor object matching the tag
 
-  await db.insertMonitoringData({
+  await InsertMonitoringData({
     monitor_tag: tag,
     timestamp: data.timestampInSeconds,
     status: resp.status,