changed data to objects

2026-01-06 09:30:21 -06:00 · 2023-12-09 20:22:47 +05:30
parent 4668a4165a
commit 1fdbb0d405
12 changed files with 181 additions and 240 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
 node_modules
 static/kener
 build/client/kener
+config/monitors.yaml
+config/site.yaml
 /.svelte-kit
 /src/lib/.kener
 /package
--- a/config/monitors.bk.yaml
+++ b/config/monitors.bk.yaml
@@ -1,47 +0,0 @@
- name: Google Search
-  description: Google Search
-  tag: "create-order"
-  method: POST
-  url: https://sandbox.cashfree.com/pg/orders
-  headers:
-    Content-Type: application/json
-    x-client-id: $X_CLIENT_ID
-    x-client-secret: $X_CLIENT_SECRET
-    x-api-version: '2022-09-01'
-  body: '{"order_amount":22222.1,"order_currency":"INR","customer_details":{"customer_id":"4091284194","customer_email":"statuspage@example.com","customer_phone":"9876543210"}}'
- name: GET Order
-  description: Some description
-  tag: "get-order"
-  method: POST
-  url: https://sandbox.cashfree.com/pg/orders/$ORDER_ID
-  headers:
-    Content-Type: application/json
-    x-client-id: $X_CLIENT_ID
-    x-client-secret: $X_CLIENT_SECRET
-    x-api-version: '2022-09-01'
-  eval: |
-    (function(statusCode, responseTime, responseDataBase64){
-      const resp = JSON.parse(atob(responseDataBase64));
-      return {
-        status: statusCode == 200 ? 'UP':'DOWN',
-        latency: latency,
-      }
-    })
- name: GET Order Payments
-  description: Some description
-  method: POST
-  tag: "get-order-payments"
-  url: https://sandbox.cashfree.com/pg/orders/$ORDER_ID/payments
-  headers:
-    Content-Type: application/json
-    x-client-id: $X_CLIENT_ID
-    x-client-secret: $X_CLIENT_SECRET
-    x-api-version: '2022-09-01'
-  eval: |
-    (function(statusCode, responseTime, responseDataBase64){
-      const resp = JSON.parse(atob(responseDataBase64));
-      return {
-        status: statusCode == 200 ? 'UP':'DOWN',
-        latency: latency,
-      }
-    })
--- a/config/monitors.example.yaml
+++ b/config/monitors.example.yaml
--- a/config/site.example.yaml
+++ b/config/site.example.yaml
--- a/package.json
+++ b/package.json
@@ -33,6 +33,7 @@
        "check": "svelte-kit sync && svelte-check --tsconfig ./jsconfig.json",
        "check:watch": "svelte-kit sync && svelte-check --tsconfig ./jsconfig.json --watch",
        "kener:dev": "cross-env PUBLIC_KENER_FOLDER=./static/kener tz=UTC concurrently \"node dev.js\" \"vite dev\"",
+        "kener:dev-monitor": "cross-env PUBLIC_KENER_FOLDER=./static/kener tz=UTC node dev.js",
        "kener:build": "cross-env PUBLIC_KENER_FOLDER=./static/kener tz=UTC vite build",
        "kener": "cross-env PUBLIC_KENER_FOLDER=./build/client/kener tz=UTC node prod.js"
    },
--- a/scripts/constants.js
+++ b/scripts/constants.js
@@ -9,16 +9,6 @@ const UP = "UP";
 const DOWN = "DOWN";
 const DEGRADED = "DEGRADED";
 const APP_HTML = "./src/app.html";
-
+const API_TIMEOUT = 10 * 1000; // 5 seconds
 // Export the constants
-export {
-    FOLDER,
-    FOLDER_MONITOR,
-	FOLDER_SITE,
-    MONITOR,
-	UP,
-	DOWN,
-	APP_HTML,
-	SITE,
-	DEGRADED
-};
+export { FOLDER, FOLDER_MONITOR, FOLDER_SITE, MONITOR, UP, DOWN, APP_HTML, SITE, DEGRADED, API_TIMEOUT };
--- a/scripts/cron-minute.js
+++ b/scripts/cron-minute.js
@@ -4,57 +4,19 @@ import { UP, DOWN, DEGRADED } from "./constants.js";
 import moment from "moment";


-const cruncDataForToday = (data, day) => {
-    let ups = 0;
-    let downs = 0;
-    let degraded = 0;
-    let latency = 0;
-
-    for (let i = 0; i < data.length; i++) {
-        const element = data[i];
-        latency = latency + element.latency;
-        if (element.status == UP) {
-            ups++;
-        } else if (element.status == DOWN) {
-            downs++;
-        } else if (element.status == DEGRADED) {
-            degraded++;
-        }
-    }
-
-    const resp = {
-        latency: parseInt(latency / data.length),
-    };
-    resp[UP] = ups;
-    resp[DOWN] = downs;
-    resp[DEGRADED] = degraded;
-    resp.timestamp = day;
-    return resp;
-};
 function replaceAllOccurrences(originalString, searchString, replacement) {
    const regex = new RegExp(`\\${searchString}`, "g");
    const replacedString = originalString.replace(regex, replacement);
    return replacedString;
 }

-function addOrReplaceDate(list, data, timestamp) {
-    let found = false;
-    for (let i = 0; i < list.length; i++) {
-        const element = list[i];
-        if (element.timestamp === timestamp) {
-            list[i] = data;
-            found = true;
-            break;
-        }
-    }
-    if (!found) {
-        list.unshift(data);
-    }
-    return list;
-}
 const OneMinuteFetch = async (envSecrets, url, method, headers, body, timeout, cb, out) => {
+	
+	let axiosHeaders = {};
+    axiosHeaders["User-Agent"] = "Kener/0.0.1";
+    axiosHeaders["Accept"] = "*/*";
    const start = Date.now();
-
+	const startOfMinute = moment(start).startOf("minute").toISOString();
    //replace all secrets
    for (let i = 0; i < envSecrets.length; i++) {
        const secret = envSecrets[i];
@@ -70,6 +32,7 @@ const OneMinuteFetch = async (envSecrets, url, method, headers, body, timeout, c
    }
 	if (!!headers){
 		headers = JSON.parse(headers);
+		axiosHeaders = {...axiosHeaders, ...headers};
 	}
    

@@ -94,7 +57,7 @@ const OneMinuteFetch = async (envSecrets, url, method, headers, body, timeout, c
        statusCode = data.status;
        resp = data.data;
    } catch (err) {
-		console.log(err.message);
+		console.error("API call error: " + err.message);
        if (err.response !== undefined && err.response.status !== undefined) {
            statusCode = err.response.status;
        }
@@ -106,64 +69,64 @@ const OneMinuteFetch = async (envSecrets, url, method, headers, body, timeout, c
        latency = end - start;
    }
    resp = Buffer.from(resp).toString("base64");
-    let toWrite = eval(cb + `(${statusCode}, ${latency}, "${resp}")`);
-    if (toWrite === undefined || toWrite === null) {
-        toWrite = {
+    let evalResp = eval(cb + `(${statusCode}, ${latency}, "${resp}")`);
+    if (evalResp === undefined || evalResp === null) {
+        evalResp = {
            status: DOWN,
            latency: latency,
            type: "error",
        };
-    } else if (toWrite.status === undefined || toWrite.status === null || [UP, DOWN, DEGRADED].indexOf(toWrite.status) === -1) {
-        toWrite.status = DOWN;
-    }
-    toWrite.timestamp = moment(start).startOf("minute").toISOString();
-    toWrite.type = "realtime";
-    let originalData = [];
-    let finalData = [];
+    } else if (evalResp.status === undefined || evalResp.status === null || [UP, DOWN, DEGRADED].indexOf(evalResp.status) === -1) {
+        evalResp = {
+            status: DOWN,
+            latency: latency,
+            type: "error",
+        };
+    } else {
+		evalResp.type = "realtime";
+	}
+
+	let toWrite = {
+        status: DOWN,
+        latency: latency,
+        type: "error",
+    };
+	if(evalResp.status !== undefined && evalResp.status !== null){
+		toWrite.status = evalResp.status;
+	}
+	if(evalResp.latency !== undefined && evalResp.latency !== null){
+		toWrite.latency = evalResp.latency;
+	}
+	if(evalResp.type !== undefined && evalResp.type !== null){
+		toWrite.type = evalResp.type;
+	}
+	let objectToWrite = {};
+	objectToWrite[startOfMinute] = toWrite;
+
+    let originalData = {};

    //read outfile out is the minute file
    try {
        let fd = fs.readFileSync(out, "utf8");
        originalData = JSON.parse(fd);
-        //insert toWrite to the front of the array
    } catch (error) {
        fs.ensureFileSync(out);
-        fs.writeFileSync(out, JSON.stringify([]));
+        fs.writeFileSync(out, JSON.stringify({}));
    }

-
+	originalData[startOfMinute] = toWrite;
+	
+	//sort the keys
+	let keys = Object.keys(originalData);
+	keys.sort((a,b) => {
+		return moment(a).isBefore(moment(b)) ? -1 : 1;
+	});
    
-    finalData[0] = toWrite;
-
-    //get the first item backfill data if needed, take the first item as the reference
-    if (originalData.length > 0) {
-        //timestamp of the first item
-        let firstItem = originalData[0];
-        //compare timestamp, if towrite.timestamp - firstItem.timestamp > 1 minute, insert a new item
-        let diff = moment(toWrite.timestamp).diff(moment(firstItem.timestamp), "minutes");
-
-        if (diff > 1 && firstItem.type === "today") {
-            for (let i = diff - 1; i >= 1; i--) {
-                let newTimestamp = moment(firstItem.timestamp).add(i, "minutes").toISOString();
-                finalData.push({
-                    timestamp: newTimestamp,
-                    status: toWrite.status,
-                    latency: toWrite.latency,
-                    type: "backfill",
-                });
-            }
-        }
-    }
-
-    //copy data from originalData to finalData
-    for (let i = 0; i < originalData.length; i++) {
-        
-        finalData.push(originalData[i]);
-    }
-
-     
-    //write outfile
-    fs.writeFileSync(out, JSON.stringify(finalData, null, 4));
+    try {
+		fs.writeFileSync(out, JSON.stringify(originalData, null, 2));
+	} catch (error) {
+		console.error(error);
+	}

 };

--- a/scripts/startup.js
+++ b/scripts/startup.js
@@ -8,14 +8,15 @@ name of each of these objects need to be unique
 import fs from "fs-extra";
 import yaml from "js-yaml";
 import { Cron } from "croner";
-import { MONITOR, SITE, APP_HTML, FOLDER, FOLDER_MONITOR, FOLDER_SITE } from "./constants.js";
+import { MONITOR, SITE, FOLDER, FOLDER_MONITOR, FOLDER_SITE, API_TIMEOUT } from "./constants.js";
+import { IsValidURL, IsValidHTTPMethod } from "./tool.js";
 import { OneMinuteFetch } from "./cron-minute.js";
 let monitors = [];
 let site = {};
 const envSecrets = [];
 const defaultEval = `(function (statusCode, responseTime, responseData) {
 	let statusCodeShort = Math.floor(statusCode/100);
-    if(statusCodeShort >=2 && statusCodeShort <= 3) {
+    if(statusCode == 429 || (statusCodeShort >=2 && statusCodeShort <= 3)) {
        return {
 			status: 'UP',
 			latency: responseTime,
@@ -49,6 +50,10 @@ if (!fs.existsSync(FOLDER_MONITOR)) {
    fs.writeFileSync(FOLDER_MONITOR, JSON.stringify([]));
    console.log("monitors.json file created successfully!");
 }
+if (!fs.existsSync(FOLDER_MONITOR)) {
+    fs.writeFileSync(FOLDER_MONITOR, JSON.stringify([]));
+    console.log("monitors.json file created successfully!");
+}



@@ -73,14 +78,33 @@ const Startup = async () => {
        let name = monitor.name;
        let url = monitor.url;
        let method = monitor.method;
-
+		let hasAPI = false;
        let folderName = name.replace(/[^a-z0-9]/gi, "-").toLowerCase();
        monitors[i].folderName = folderName;

-        if (!name || !url || !method) {
+        if (!name) {
            console.log("name, url, method are required");
            process.exit(1);
        }
+
+		if (!!url) {
+			if (!IsValidURL(url)) {
+				console.log("url is not valid");
+				process.exit(1);	
+			}
+			hasAPI = true;
+        } 
+
+		if (!!method && hasAPI) {
+			if (!IsValidHTTPMethod(method)) {
+				console.log("method is not valid");
+				process.exit(1);	
+			}
+			method = method.toUpperCase();
+        } else {
+			method = "GET";
+		}
+
        if (monitor.eval === undefined || monitor.eval === null) {
            monitors[i].eval = defaultEval;
        }
@@ -91,10 +115,11 @@ const Startup = async () => {
            monitors[i].body = undefined;
        }
        if (monitor.timeout === undefined || monitor.timeout === null) {
-            monitors[i].timeout = 1000 * 5;
+            monitors[i].timeout = API_TIMEOUT;
        }

-        monitors[i].path0Day = `${FOLDER}/${folderName}-day.json`;
+        monitors[i].path0Day = `${FOLDER}/${folderName}-day-json.json`;
+        monitors[i].hasAPI = hasAPI;

        //secrets can be in url/body/headers
        //match in monitor.url if a words starts with $, get the word
@@ -131,6 +156,9 @@ const Startup = async () => {
    // init monitors
    for (let i = 0; i < monitors.length; i++) {
        const monitor = monitors[i];
+		if (!monitor.hasAPI) {
+			continue;
+		}
        console.log("Staring One Minute Cron for ", monitor.path0Day);
        await OneMinuteFetch(envSecrets, monitor.url, monitor.method, JSON.stringify(monitor.headers), monitor.body, monitor.timeout, monitor.eval, monitor.path0Day);
    }
@@ -139,7 +167,9 @@ const Startup = async () => {

    for (let i = 0; i < monitors.length; i++) {
        const monitor = monitors[i];
-        
+        if (!monitor.hasAPI) {
+			continue;
+		}
        let cronExpession = "* * * * *";
        if (monitor.cron !== undefined && monitor.cron !== null) {
            cronExpession = monitor.cron;
--- a/scripts/tool.js
+++ b/scripts/tool.js
@@ -0,0 +1,7 @@
+const IsValidURL = function (url) {
+    return /^(http|https):\/\/[^ "]+$/.test(url);
+};
+const IsValidHTTPMethod = function (method) {
+	return /^(GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH)$/.test(method);
+}
+export { IsValidURL, IsValidHTTPMethod };
--- a/src/lib/components/monitor.svelte
+++ b/src/lib/components/monitor.svelte
@@ -24,7 +24,8 @@
 		var date = new Date();
 		var hours = date.getHours();
 		var minutes = date.getMinutes();
-		var totalMinutes = (hours*60) + minutes 
+		var totalMinutes = (hours*60) + minutes ;
+		return totalMinutes
 	}

    export let monitor;
@@ -174,7 +175,8 @@
                        Midnight
                        <ArrowDown size="{16}" />
                    </div>
-                    {/if} {#if bar.index == minuteFromMidnightTillNow}
+                    {/if} 
+					{#if bar.index == minuteFromMidnightTillNow}
                    <div class="arrow end text-sm">
                        <ArrowUp size="{16}" />
                        Now
--- a/src/routes/api/today/+server.js
+++ b/src/routes/api/today/+server.js
@@ -9,7 +9,6 @@ let statusObj = {
    DOWN: "api-down",
    NO_DATA: "api-nodata",
 };
- 

 function parseUptime(up, all) {
    if (up == all) {
@@ -17,28 +16,28 @@ function parseUptime(up, all) {
    }
    return String(((up / all) * parseFloat(100)).toFixed(4));
 }
-function parsePercentage(n){
-	if(n == 0){
-		return "0";
-	}
-	if(n == 100){
-		return "100";
-	}
-	return n.toFixed(4);
+function parsePercentage(n) {
+    if (n == 0) {
+        return "0";
+    }
+    if (n == 100) {
+        return "100";
+    }
+    return n.toFixed(4);
 }
 export async function POST({ request }) {
-	 const payload = await request.json();
-     const tz = payload.tz;
+    const payload = await request.json();
+    const tz = payload.tz;

-	let _0Day = {};
+    let _0Day = {};
    let _90Day = {};
    let uptime0Day = "0";
    let dailyUps = 0;
    let dailyDown = 0;
-	let percentage90DaysBuildUp = [];
-	let latency90DaysBuildUp = [];
+    let percentage90DaysBuildUp = [];
+    let latency90DaysBuildUp = [];
    let dailyDegraded = 0;
-	let dailyLatencyBuildUp = [];
+    let dailyLatencyBuildUp = [];
    const now = moment.tz(tz);
    let minuteFromMidnightTillNow = now.diff(now.clone().startOf("day"), "minutes");
    for (let i = 0; i <= minuteFromMidnightTillNow; i++) {
@@ -51,87 +50,81 @@ export async function POST({ request }) {
            index: i,
        };
    }
-	
-   
-	let day0 = JSON.parse(fs.readFileSync(payload.day0, "utf8"));

-	for (let i = 0; i < day0.length; i++) {
-        let min = moment.tz(day0[i].timestamp, tz).format("YYYY-MM-DD HH:mm:00");
-		
-		
-        let day = moment.tz(day0[i].timestamp, tz).format("YYYY-MM-DD");
-        let status = day0[i].status;
-        let latency = day0[i].latency;
+    let day0 = JSON.parse(fs.readFileSync(payload.day0, "utf8"));

-        if (_90Day[day] === undefined) {
-            _90Day[day] = {
-                timestamp: day,
-                UP: status == "UP" ? 1 : 0,
-                DEGRADED: status == "DEGRADED" ? 1 : 0,
-                DOWN: status == "DOWN" ? 1 : 0,
-                latency: latency,
-                avgLatency: latency,
-            };
-        } else {
-            let d = _90Day[day];
-            _90Day[day] = {
-                timestamp: day,
-                UP: status == "UP" ? d.UP + 1 : d.UP,
-                DEGRADED: status == "DEGRADED" ? d.DEGRADED + 1 : d.DEGRADED,
-                DOWN: status == "DOWN" ? d.DOWN + 1 : d.DOWN,
-                latency: d.latency + latency,
-                avgLatency: ((d.latency + latency) / (d.UP + d.DEGRADED + d.DOWN + 1)).toFixed(0),
-            };
-        }
-        _90Day[day].uptimePercentage = parseUptime(_90Day[day].UP + _90Day[day].DEGRADED, _90Day[day].UP + _90Day[day].DEGRADED + _90Day[day].DOWN);
-		//percentage90DaysBuildUp = percentage90DaysBuildUp + parseFloat(_90Day[day].uptimePercentage);
-		
+	//loop day0 as object
+	for (const timestampISO in day0) {
+		if (Object.hasOwnProperty.call(day0, timestampISO)) {
+			const element = day0[timestampISO];
+			let min = moment.tz(timestampISO, tz).format("YYYY-MM-DD HH:mm:00");
+			let day = moment.tz(timestampISO, tz).format("YYYY-MM-DD");
+			let status = element.status;
+            let latency = element.latency;

-        let cssClass = statusObj.UP;
-        let message = "0 Issues";
+			//90 Day data
+			if (_90Day[day] === undefined) {
+                _90Day[day] = {
+                    timestamp: day,
+                    UP: status == "UP" ? 1 : 0,
+                    DEGRADED: status == "DEGRADED" ? 1 : 0,
+                    DOWN: status == "DOWN" ? 1 : 0,
+                    latency: latency,
+                    avgLatency: latency,
+                };
+            } else {
+                let d = _90Day[day];
+                _90Day[day] = {
+                    timestamp: day,
+                    UP: status == "UP" ? d.UP + 1 : d.UP,
+                    DEGRADED: status == "DEGRADED" ? d.DEGRADED + 1 : d.DEGRADED,
+                    DOWN: status == "DOWN" ? d.DOWN + 1 : d.DOWN,
+                    latency: d.latency + latency,
+                    avgLatency: ((d.latency + latency) / (d.UP + d.DEGRADED + d.DOWN + 1)).toFixed(0),
+                };
+            }
+			_90Day[day].uptimePercentage = parseUptime(_90Day[day].UP + _90Day[day].DEGRADED, _90Day[day].UP + _90Day[day].DEGRADED + _90Day[day].DOWN);
+			
+			let cssClass = statusObj.UP;
+            let message = "0 Issues";

-        if (_90Day[day].DEGRADED > 0) {
-            cssClass = statusObj.DEGRADED;
-            message = "Degraded for " + _90Day[day].DEGRADED + " minutes";
-        }
+            if (_90Day[day].DEGRADED > 0) {
+                cssClass = statusObj.DEGRADED;
+                message = "Degraded for " + _90Day[day].DEGRADED + " minutes";
+            }

-        if (_90Day[day].DOWN > 0) {
-            cssClass = statusObj.DOWN;
-            message = "Down for " + _90Day[day].DOWN + " minutes";
-        }
+            if (_90Day[day].DOWN > 0) {
+                cssClass = statusObj.DOWN;
+                message = "Down for " + _90Day[day].DOWN + " minutes";
+            }

-        _90Day[day].cssClass = cssClass;
-        _90Day[day].message = message;
+            _90Day[day].cssClass = cssClass;
+            _90Day[day].message = message;

-        _90Day = Object.keys(_90Day)
-            .sort()
-            .reduce((obj, key) => {
-                obj[key] = _90Day[key];
-                return obj;
-            }, {});
+			//0 Day data
+			if (_0Day[min] !== undefined) {
+                _0Day[min].status = status;
+                _0Day[min].cssClass = statusObj[status];
+                _0Day[min].latency = latency;

-        if (_0Day[min] !== undefined) {
-            _0Day[min].status = status;
-            _0Day[min].cssClass = statusObj[status];
-            _0Day[min].latency = latency;
+                dailyUps = status == "UP" ? dailyUps + 1 : dailyUps;
+                dailyDown = status == "DOWN" ? dailyDown + 1 : dailyDown;
+                dailyDegraded = status == "DEGRADED" ? dailyDegraded + 1 : dailyDegraded;
+                dailyLatencyBuildUp.push(latency);
+            }
+		}
+	}

-            dailyUps = status == "UP" ? dailyUps + 1 : dailyUps;
-            dailyDown = status == "DOWN" ? dailyDown + 1 : dailyDown;
-            dailyDegraded = status == "DEGRADED" ? dailyDegraded + 1 : dailyDegraded;
-			dailyLatencyBuildUp.push(latency);
-        }
-
-        
-    }
-	for (const key in _90Day) {
+    
+    for (const key in _90Day) {
        if (Object.hasOwnProperty.call(_90Day, key)) {
            const element = _90Day[key];
-			percentage90DaysBuildUp.push(parseFloat(element.uptimePercentage));
-			latency90DaysBuildUp.push(parseFloat(element.avgLatency));
+            percentage90DaysBuildUp.push(parseFloat(element.uptimePercentage));
+            latency90DaysBuildUp.push(parseFloat(element.avgLatency));
        }
    }
-	uptime0Day = parseUptime(dailyUps + dailyDegraded, dailyUps + dailyDown + dailyDegraded);
-	return json({
+    uptime0Day = parseUptime(dailyUps + dailyDegraded, dailyUps + dailyDown + dailyDegraded);
+    return json({
        _0Day: _0Day,
        _90Day: _90Day,
        uptime0Day,
--- a/src/routes/incident/[id]/+page.server.js
+++ b/src/routes/incident/[id]/+page.server.js
@@ -53,4 +53,4 @@ export async function load({ params, route, url, parent }) {
        activeIncidents: await Promise.all(gitHubActiveIssues.map(mapper, { github })),
        pastIncidents: await Promise.all(gitHubPastIssues.map(mapper, { github })),
    };
-}
+}