initial commit

This commit is contained in:
Alex Holliday
2026-01-19 23:41:24 +00:00
parent 33dfedb4d3
commit e91a54b75c
12 changed files with 295 additions and 242 deletions
+7
View File
@@ -84,6 +84,7 @@ import {
MongoRecoveryTokensRepository,
MongoSettingsRepository,
MongoNotificationsRepository,
MongoIncidentRepository,
IMonitorsRepository,
IChecksRepository,
IMonitorStatsRepository,
@@ -93,6 +94,7 @@ import {
IRecoveryTokensRepository,
ISettingsRepository,
INotificationsRepository,
IIncidentsRepository,
} from "@/repositories/index.js";
export type InitializedSerivces = {
@@ -127,6 +129,7 @@ export type InitializedSerivces = {
recoveryTokensRepository: IRecoveryTokensRepository;
settingsRepository: ISettingsRepository;
notificationsRepository: INotificationsRepository;
incidentsRepository: IIncidentsRepository;
};
export const initializeServices = async ({
@@ -183,6 +186,7 @@ export const initializeServices = async ({
const recoveryTokensRepository = new MongoRecoveryTokensRepository();
const settingsRepository = new MongoSettingsRepository();
const notificationsRepository = new MongoNotificationsRepository();
const incidentsRepository = new MongoIncidentRepository();
const networkService = new NetworkService({
axios,
got,
@@ -205,6 +209,7 @@ export const initializeServices = async ({
logger,
errorService,
stringService,
incidentsRepository,
});
const checkService = new CheckService({
@@ -247,6 +252,7 @@ export const initializeServices = async ({
notificationsService,
checkService,
buffer: bufferService,
incidentService,
});
const superSimpleQueue = await SuperSimpleQueue.create({
@@ -332,6 +338,7 @@ export const initializeServices = async ({
recoveryTokensRepository,
settingsRepository,
notificationsRepository,
incidentsRepository,
};
Object.values(services).forEach((service) => {
+6 -6
View File
@@ -67,12 +67,12 @@ class IncidentController {
resolveIncidentManually = async (req: Request, res: Response, next: NextFunction) => {
try {
const resolvedIncident = await this.incidentService.resolveIncidentManually({
incidentId: req?.params?.incidentId,
userId: req?.user?.id,
teamId: req?.user?.teamId,
comment: req?.body?.comment,
});
const resolvedIncident = await this.incidentService.resolveIncident(
req?.params?.incidentId,
req?.user?.id,
req?.user?.teamId,
req?.body?.comment
);
return res.status(200).json({
success: true,
@@ -1,16 +1,31 @@
import mongoose from "mongoose";
import { Schema, model, type Types } from "mongoose";
import { IncidentResolutionTypes, type Incident } from "@/types/incident.js";
const IncidentSchema = mongoose.Schema(
type IncidentDocumentBase = Omit<Incident, "id" | "monitorId" | "teamId" | "resolvedBy" | "startTime" | "endTime" | "createdAt" | "updatedAt"> & {
monitorId: Types.ObjectId;
teamId: Types.ObjectId;
resolvedBy?: Types.ObjectId | null;
startTime: Date;
endTime: Date | null;
createdAt: Date;
updatedAt: Date;
};
export interface IncidentDocument extends IncidentDocumentBase {
_id: Types.ObjectId;
}
const IncidentSchema = new Schema<IncidentDocument>(
{
monitorId: {
type: mongoose.Schema.Types.ObjectId,
type: Schema.Types.ObjectId,
ref: "Monitor",
required: true,
immutable: true,
index: true,
},
teamId: {
type: mongoose.Schema.Types.ObjectId,
type: Schema.Types.ObjectId,
ref: "Team",
required: true,
immutable: true,
@@ -18,8 +33,8 @@ const IncidentSchema = mongoose.Schema(
},
startTime: {
type: Date,
required: true,
immutable: true,
required: true,
},
endTime: {
type: Date,
@@ -36,16 +51,16 @@ const IncidentSchema = mongoose.Schema(
},
statusCode: {
type: Number,
index: true,
default: null,
index: true,
},
resolutionType: {
type: String,
enum: ["automatic", "manual"],
enum: IncidentResolutionTypes,
default: null,
},
resolvedBy: {
type: mongoose.Schema.Types.ObjectId,
type: Schema.Types.ObjectId,
ref: "User",
default: null,
},
@@ -53,12 +68,6 @@ const IncidentSchema = mongoose.Schema(
type: String,
default: null,
},
checks: [
{
type: mongoose.Schema.Types.ObjectId,
ref: "Check",
},
],
},
{ timestamps: true }
);
@@ -71,6 +80,7 @@ IncidentSchema.index({ resolutionType: 1, status: 1 });
IncidentSchema.index({ resolvedBy: 1, status: 1 });
IncidentSchema.index({ createdAt: -1 });
const Incident = mongoose.model("Incident", IncidentSchema);
const IncidentModel = model<IncidentDocument>("Incident", IncidentSchema);
export default Incident;
export { IncidentModel };
export default IncidentModel;
+3
View File
@@ -24,3 +24,6 @@ export { default as RecoveryTokenModel } from "@/db/models/RecoveryToken.js";
export * from "@/db/models/Notification.js";
export { default as NotificationModel } from "@/db/models/Notification.js";
export * from "@/db/models/Incident.js";
export { default as IncidentModel } from "@/db/models/Incident.js";
@@ -0,0 +1,22 @@
import type { Incident } from "@/types/index.js";
export interface IIncidentsRepository {
// create
create(incident: Partial<Incident>): Promise<Incident>;
// fetch
findActiveByIncidentId(incidentId: string, teamId: string): Promise<Incident | null>;
findActiveByMonitorId(monitorId: string, teamId: string): Promise<Incident | null>;
findByTeamId(
teamId: string,
dateRange: string,
page: number,
rowsPerPage: number,
sortOrder?: string,
status?: string,
monitorId?: string,
resolutionType?: string
): Promise<Incident[]>;
// update
updateById(incidentId: string, teamId: string, updateData: Partial<Incident>): Promise<Incident>;
// delete
}
@@ -0,0 +1,100 @@
import { IncidentModel } from "@/db/models/index.js";
import type { IncidentDocument } from "@/db/models/Incident.js";
import type { Incident } from "@/types/index.js";
import type { IIncidentsRepository } from "@/repositories/index.js";
import mongoose from "mongoose";
import { AppError } from "@/utils/AppError.js";
class MongoIncidentRepository implements IIncidentsRepository {
private toStringId = (value?: mongoose.Types.ObjectId | string | null): string => {
if (!value) {
return "";
}
return value instanceof mongoose.Types.ObjectId ? value.toString() : String(value);
};
private toDateString = (value?: Date | string | null): string => {
if (!value) {
return new Date(0).toISOString();
}
return value instanceof Date ? value.toISOString() : new Date(value).toISOString();
};
protected toEntity = (doc: IncidentDocument): Incident => {
return {
id: this.toStringId(doc._id),
monitorId: this.toStringId(doc.monitorId),
teamId: this.toStringId(doc.teamId),
startTime: this.toDateString(doc.startTime),
endTime: doc.endTime ? this.toDateString(doc.endTime) : null,
status: doc.status,
message: doc.message ?? null,
statusCode: doc.statusCode ?? null,
resolutionType: doc.resolutionType ?? null,
resolvedBy: doc.resolvedBy ? this.toStringId(doc.resolvedBy) : null,
comment: doc.comment ?? null,
createdAt: this.toDateString(doc.createdAt),
updatedAt: this.toDateString(doc.updatedAt),
};
};
protected mapDocuments = (documents: IncidentDocument[] | IncidentDocument | null): Incident[] => {
if (!documents) {
return [];
}
if (Array.isArray(documents)) {
return documents.map((doc) => this.toEntity(doc));
}
return [this.toEntity(documents)];
};
async create(incident: Partial<Incident>): Promise<Incident> {
const newIncident = await IncidentModel.create(incident);
return this.toEntity(newIncident);
}
findActiveByIncidentId = async (incidentId: string, teamId: string): Promise<Incident | null> => {
const incident = await IncidentModel.findOne({
_id: new mongoose.Types.ObjectId(incidentId),
teamId: new mongoose.Types.ObjectId(teamId),
status: true,
});
if (!incident) {
return null;
}
return this.toEntity(incident);
};
findActiveByMonitorId = async (monitorId: string, teamId: string): Promise<Incident | null> => {
const incident = await IncidentModel.findOne({
monitorId: new mongoose.Types.ObjectId(monitorId),
teamId: new mongoose.Types.ObjectId(teamId),
status: true,
});
if (!incident) {
return null;
}
return this.toEntity(incident);
};
findByTeamId = async (teamId: string): Promise<Incident[]> => {
throw new Error("Method not implemented.");
};
updateById = async (incidentId: string, teamId: string, patch: Partial<Incident>) => {
const updatedIncident = await IncidentModel.findOneAndUpdate(
{ _id: new mongoose.Types.ObjectId(incidentId), teamId: new mongoose.Types.ObjectId(teamId) },
{
$set: {
...patch,
},
},
{ new: true, runValidators: true }
);
if (!updatedIncident) {
throw new AppError({ message: `Failed to update incident with id ${incidentId}`, status: 500 });
}
return this.toEntity(updatedIncident);
};
}
export default MongoIncidentRepository;
+3
View File
@@ -24,3 +24,6 @@ export { default as MongoSettingsRepository } from "@/repositories/settings/Mong
export * from "@/repositories/notifications/INotificationsRepository.js";
export { default as MongoNotificationsRepository } from "@/repositories/notifications/MongoNotificationsRepository.js";
export * from "@/repositories/incidents/IIncidentsRepository.js";
export { default as MongoIncidentRepository } from "@/repositories/incidents/MongoIncidentRepository.js";
@@ -1,183 +1,123 @@
const SERVICE_NAME = "incidentService";
import type { Monitor } from "@/types/monitor.js";
import { AppError } from "@/utils/AppError.js";
import type { IIncidentsRepository } from "@/repositories/index.js";
import type { Incident } from "@/types/index.js";
class IncidentService {
static SERVICE_NAME = SERVICE_NAME;
constructor({ db, logger, errorService, stringService }) {
private db: any;
private logger: any;
private errorService: any;
private stringService: any;
private incidentsRepository: IIncidentsRepository;
constructor({
db,
logger,
errorService,
stringService,
incidentsRepository,
}: {
db: any;
logger: any;
errorService: any;
stringService: any;
incidentsRepository: IIncidentsRepository;
}) {
this.db = db;
this.logger = logger;
this.errorService = errorService;
this.stringService = stringService;
this.incidentsRepository = incidentsRepository;
}
get serviceName() {
return IncidentService.SERVICE_NAME;
}
createIncident = async (monitor, check) => {
try {
if (!monitor || !monitor._id) {
throw this.errorService.createBadRequestError("Monitor is required");
}
if (!check || !check._id) {
throw this.errorService.createBadRequestError("Check is required");
}
const activeIncident = await this.db.incidentModule.getActiveIncidentByMonitor(monitor._id);
handleIncident = async (monitor: Monitor, code: number): Promise<Incident | null> => {
const activeIncident = await this.incidentsRepository.findActiveByMonitorId(monitor.id, monitor.teamId);
// Monitor is down, create an incident
if (monitor.status === false) {
if (activeIncident) {
await this.db.incidentModule.addCheckToIncident(activeIncident._id, check._id);
this.logger.info({
service: this.SERVICE_NAME,
method: "createIncident",
message: `Check added to existing active incident for monitor ${monitor.name}`,
incidentId: activeIncident._id,
monitorId: monitor._id,
});
return activeIncident;
} else {
const incident = {
monitorId: monitor.id,
teamId: monitor.teamId,
startTime: Date.now().toString(),
status: true,
statusCode: code,
};
return await this.incidentsRepository.create(incident);
}
const incidentData = {
monitorId: monitor._id,
teamId: monitor.teamId,
type: monitor.type,
startTime: new Date(),
status: true,
message: check.message || null,
statusCode: check.statusCode || null,
checks: [check._id],
};
const incident = await this.db.incidentModule.createIncident(incidentData);
this.logger.info({
service: this.SERVICE_NAME,
method: "createIncident",
message: `New incident created for monitor ${monitor.name}`,
incidentId: incident._id,
monitorId: monitor._id,
});
return incident;
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "createIncident",
message: error.message,
monitorId: monitor?._id,
error: error.stack,
});
throw error;
}
// Monitor is up, resolve active incidents
if (!activeIncident) {
return null;
}
activeIncident.status = false;
activeIncident.endTime = Date.now().toString();
activeIncident.resolutionType = "automatic";
return await this.incidentsRepository.updateById(activeIncident.id, activeIncident.teamId, activeIncident);
};
resolveIncident = async (monitor, check) => {
try {
if (!monitor || !monitor._id) {
throw this.errorService.createBadRequestError("Monitor is required");
}
const activeIncident = await this.db.incidentModule.getActiveIncidentByMonitor(monitor._id);
if (!activeIncident) {
this.logger.info({
service: this.SERVICE_NAME,
method: "resolveIncident",
message: `No active incident found for monitor ${monitor.name}`,
monitorId: monitor._id,
});
return null;
}
await this.db.incidentModule.addCheckToIncident(activeIncident._id, check._id);
const resolvedIncident = await this.db.incidentModule.resolveIncident(activeIncident._id, {
resolutionType: "automatic",
resolvedBy: null,
endTime: new Date(),
});
this.logger.info({
service: this.SERVICE_NAME,
method: "resolveIncident",
message: `Incident automatically resolved for monitor ${monitor.name}`,
incidentId: resolvedIncident._id,
monitorId: monitor._id,
});
return resolvedIncident;
} catch (error) {
this.logger.error({
service: this.SERVICE_NAME,
method: "resolveIncident",
message: error.message,
monitorId: monitor?._id,
error: error.stack,
});
throw error;
}
};
resolveIncidentManually = async ({ incidentId, userId, teamId, comment }) => {
resolveIncident = async (incidentId: string, userId: string, teamId: string, comment?: string) => {
try {
if (!incidentId) {
throw this.errorService.createBadRequestError("No incident ID in request");
throw new AppError({ message: "No incident ID in request", service: SERVICE_NAME, method: "resolveIncident" });
}
if (!userId) {
throw this.errorService.createBadRequestError("No user ID in request");
throw new AppError({ message: "No user ID in request", service: SERVICE_NAME, method: "resolveIncident" });
}
if (!teamId) {
throw this.errorService.createBadRequestError("No team ID in request");
throw new AppError({ message: "No team ID in request", service: SERVICE_NAME, method: "resolveIncident" });
}
const incident = await this.db.incidentModule.getIncidentById(incidentId);
const incident = await this.incidentsRepository.findActiveByIncidentId(incidentId, teamId);
if (!incident) {
throw this.errorService.createNotFoundError("Incident not found");
}
if (!incident.teamId.equals(teamId)) {
throw this.errorService.createAuthorizationError();
throw new AppError({ message: "Incident not found", service: SERVICE_NAME, method: "resolveIncident" });
}
if (incident.status === false) {
throw this.errorService.createBadRequestError("Incident is already resolved");
throw new AppError({ message: "Incident is already resolved", service: SERVICE_NAME, method: "resolveIncident" });
}
const resolvedIncident = await this.db.incidentModule.resolveIncident(incidentId, {
resolutionType: "manual",
resolvedBy: userId,
comment: comment || null,
endTime: new Date(),
});
incident.resolutionType = "manual";
incident.status = false;
incident.resolvedBy = userId;
incident.comment = comment || null;
incident.endTime = Date.now().toString();
this.logger.info({
service: this.SERVICE_NAME,
const resolvedIncident = await this.incidentsRepository.updateById(incident.id, teamId, incident);
this.logger.debug({
service: SERVICE_NAME,
method: "resolveIncidentManually",
message: `Incident manually resolved by user`,
incidentId: resolvedIncident._id,
userId,
details: resolvedIncident.id,
});
return resolvedIncident;
} catch (error) {
} catch (error: any) {
this.logger.error({
service: this.SERVICE_NAME,
method: "resolveIncidentManually",
service: SERVICE_NAME,
method: "resolveIncident",
message: error.message,
incidentId,
error: error.stack,
details: incidentId,
stack: error.stack,
});
throw error;
}
};
getIncidentsByTeam = async ({ teamId, query }) => {
getIncidentsByTeam = async ({ teamId, query }: { teamId: string; query?: any }) => {
try {
if (!teamId) {
throw this.errorService.createBadRequestError("No team ID in request");
@@ -197,19 +137,19 @@ class IncidentService {
});
return result;
} catch (error) {
} catch (error: any) {
this.logger.error({
service: this.SERVICE_NAME,
service: SERVICE_NAME,
method: "getIncidentsByTeam",
message: error.message,
teamId,
error: error.stack,
details: teamId,
stack: error.stack,
});
throw error;
}
};
getIncidentSummary = async ({ teamId, query }) => {
getIncidentSummary = async ({ teamId, query }: { teamId: string; query?: any }) => {
try {
if (!teamId) {
throw this.errorService.createBadRequestError("No team ID in request");
@@ -223,19 +163,19 @@ class IncidentService {
});
return summary;
} catch (error) {
} catch (error: any) {
this.logger.error({
service: this.SERVICE_NAME,
service: SERVICE_NAME,
method: "getIncidentSummary",
message: error.message,
teamId,
error: error.stack,
details: teamId,
stack: error.stack,
});
throw error;
}
};
getIncidentById = async ({ incidentId, teamId }) => {
getIncidentById = async ({ incidentId, teamId }: { incidentId: string; teamId: string }) => {
try {
if (!incidentId) {
throw this.errorService.createBadRequestError("No incident ID in request");
@@ -256,19 +196,19 @@ class IncidentService {
}
return incident;
} catch (error) {
} catch (error: any) {
this.logger.error({
service: this.SERVICE_NAME,
service: SERVICE_NAME,
method: "getIncidentById",
message: error.message,
incidentId,
error: error.stack,
details: incidentId,
stack: error.stack,
});
throw error;
}
};
processIncidentsFromBuffer = async (incidentBufferItems) => {
processIncidentsFromBuffer = async (incidentBufferItems: any[]) => {
try {
if (!incidentBufferItems || incidentBufferItems.length === 0) {
return;
@@ -285,16 +225,16 @@ class IncidentService {
}
}
for (const item of resolveItems) {
for (const item of resolveItems as any[]) {
try {
await this.resolveIncident(item.monitor, item.check);
} catch (error) {
await this.resolveIncident(item.monitor);
} catch (error: any) {
this.logger.error({
service: this.SERVICE_NAME,
service: SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: `Failed to resolve incident from buffer: ${error.message}`,
monitorId: item.monitor?._id,
error: error.stack,
stack: error.stack,
});
}
}
@@ -303,11 +243,11 @@ class IncidentService {
return;
}
const groupedByMonitor = {};
const groupedByMonitor: Record<string, any[]> = {};
for (const item of createItems) {
if (!item.monitor || !item.monitor.id || !item.check || !item.check.id) {
this.logger.warn({
service: this.SERVICE_NAME,
service: SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: "Skipping item with missing monitor or check data",
item,
@@ -329,7 +269,7 @@ class IncidentService {
const activeIncidents = await this.db.incidentModule.getActiveIncidentsByMonitors(monitorIds);
const incidentsCreatedInFlush = {};
const incidentsCreatedInFlush: Record<string, any> = {};
const checksToAddToIncidents = [];
const newIncidentsToCreate = [];
@@ -392,19 +332,19 @@ class IncidentService {
}
this.logger.info({
service: this.SERVICE_NAME,
service: SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: `Processed ${incidentBufferItems.length} incident buffer items`,
created: newIncidentsToCreate.length,
checksAdded: checksToAddToIncidents.length,
resolved: resolveItems.length,
});
} catch (error) {
} catch (error: any) {
this.logger.error({
service: this.SERVICE_NAME,
service: SERVICE_NAME,
method: "processIncidentsFromBuffer",
message: error.message,
error: error.stack,
stack: error.stack,
});
throw error;
}
@@ -2,6 +2,8 @@ const SERVICE_NAME = "JobQueueHelper";
import type { Monitor } from "@/types/monitor.js";
import { AppError } from "@/utils/AppError.js";
import { INetworkService, INotificationsService, IStatusService } from "@/service/index.js";
import IncidentService from "@/service/business/incidentService.js";
class SuperSimpleQueueHelper {
static SERVICE_NAME = SERVICE_NAME;
@@ -12,6 +14,7 @@ class SuperSimpleQueueHelper {
private notificationsService: INotificationsService;
private checkService: any;
private buffer: any;
private incidentService: IncidentService;
constructor({
db,
@@ -21,6 +24,7 @@ class SuperSimpleQueueHelper {
notificationsService,
checkService,
buffer,
incidentService,
}: {
db: any;
logger: any;
@@ -29,6 +33,7 @@ class SuperSimpleQueueHelper {
notificationsService: INotificationsService;
checkService: any;
buffer: any;
incidentService: IncidentService;
}) {
this.db = db;
this.logger = logger;
@@ -37,6 +42,7 @@ class SuperSimpleQueueHelper {
this.checkService = checkService;
this.buffer = buffer;
this.notificationsService = notificationsService;
this.incidentService = incidentService;
}
get serviceName() {
@@ -90,6 +96,11 @@ class SuperSimpleQueueHelper {
stack: error.stack,
});
});
// Step 6. Handle incidents
if (statusChangeResult.statusChanged) {
await this.incidentService.handleIncident(statusChangeResult.monitor, statusChangeResult.code);
}
} catch (error: any) {
this.logger.warn({
message: error.message,
@@ -218,70 +218,6 @@ export class StatusService implements IStatusService {
statusChanged = true;
}
if (statusChanged) {
this.logger.info({
service: SERVICE_NAME,
message: `${monitor.name} went from ${this.getStatusString(prevStatus)} to ${this.getStatusString(newStatus)}`,
prevStatus,
newStatus,
});
if (newStatus === false) {
await this.handleIncidentForCheck(check, monitor, "create", "status change to down");
} else if (prevStatus === false) {
await this.handleIncidentForCheck(check, monitor, "resolve", "status change to up");
}
}
if (monitor.status === false && !statusChanged) {
try {
const lastManuallyResolvedIncident = await this.db.incidentModule.getLastManuallyResolvedIncident(monitor.id);
let calculatedFailureRate = failureRate;
if (lastManuallyResolvedIncident && lastManuallyResolvedIncident.endTime) {
try {
const checksAfterResolution = await CheckModel.find({
monitorId: monitor.id,
createdAt: { $gt: lastManuallyResolvedIncident.endTime },
})
.sort({ createdAt: 1 })
.limit(monitor.statusWindowSize)
.select("status")
.lean();
if (checksAfterResolution.length > 0) {
const checksStatuses = checksAfterResolution.map((c) => c.status);
const failuresAfterResolution = checksStatuses.filter((s) => s === false).length;
calculatedFailureRate = (failuresAfterResolution / monitor.statusWindow.length) * 100;
} else {
calculatedFailureRate = 0;
}
} catch (checkQueryError: any) {
this.logger.error({
service: SERVICE_NAME,
method: "updateStatus",
message: `Failed to query checks after manual resolution: ${checkQueryError.message}`,
monitorId: monitor.id,
stack: checkQueryError.stack,
});
}
}
if (calculatedFailureRate >= monitor.statusWindowThreshold) {
await this.handleIncidentForCheck(check, monitor, "create", "threshold check without status change");
}
} catch (error: any) {
this.logger.error({
service: SERVICE_NAME,
method: "updateStatus",
message: `Error handling threshold check without status change: ${error.message}`,
monitorId: monitor.id,
stack: error.stack,
});
}
}
monitor.status = newStatus;
const updated = await this.monitorsRepository.updateById(monitor.id, monitor.teamId, monitor);
+20
View File
@@ -0,0 +1,20 @@
// export type IncidentResolutionType = "automatic" | "manual" | null;
export const IncidentResolutionTypes = ["automatic", "manual", null] as const;
export type IncidentResolutionType = (typeof IncidentResolutionTypes)[number];
export interface Incident {
id: string;
monitorId: string;
teamId: string;
startTime: string;
endTime: string | null;
status: boolean;
message?: string | null;
statusCode?: number | null;
resolutionType: IncidentResolutionType;
resolvedBy?: string | null;
comment?: string | null;
createdAt: string;
updatedAt: string;
}
+1
View File
@@ -9,3 +9,4 @@ export * from "@/types/recoveryToken.js";
export * from "@/types/settings.js";
export * from "@/types/notification.js";
export * from "@/types/alert.js";
export * from "@/types/incident.js";