mirror/dble
1
0
Fork 0
mirror of https://github.com/actiontech/dble.git synced 2026-02-14 01:29:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

[inner-1181] add master-slave delay alarm (#3323)

Browse Source
This commit is contained in:
wenyh
2022-07-29 15:00:03 +08:00
committed by GitHub
parent 85d304bb8e
commit 7e1df75160
3 changed files with 16 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/com/actiontech/dble/alarm/AlarmCode.java
View File

@@ -39,4 +39,5 @@ public final class AlarmCode {
public static final String HEARTBEAT_FAIL = "DBLE_HEARTBEAT_FAIL"; //Resolve by trigger
public static final String SHARDING_NODE_LACK = "DBLE_SHARDING_NODE_LACK"; //Resolve by trigger
public static final String DB_INSTANCE_LOWER_CASE_ERROR = "DBLE_DB_INSTANCE_LOWER_CASE_ERROR"; //Resolve by trigger
public static final String DB_SLAVE_INSTANCE_DELAY = "DBLE_DB_SLAVE_INSTANCE_DELAY"; //Resolve by trigger
}

1
src/main/java/com/actiontech/dble/alarm/ToResolveContainer.java
View File

@@ -22,4 +22,5 @@ public final class ToResolveContainer {
public static final Set<String> REACH_MAX_CON = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
public static final Set<String> XA_WRITE_CHECK_POINT_FAIL = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
public static final Set<String> DB_INSTANCE_LOWER_CASE_ERROR = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
public static final Set<String> DB_SLAVE_INSTANCE_DELAY = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
}

16
src/main/java/com/actiontech/dble/backend/heartbeat/MySQLDetector.java
View File

@@ -187,8 +187,20 @@ public class MySQLDetector implements SQLQueryResultListener<SQLQueryResult<Map<
if (null != secondsBehindMaster && !"".equals(secondsBehindMaster) && !"NULL".equalsIgnoreCase(secondsBehindMaster)) {
int behindMaster = Integer.parseInt(secondsBehindMaster);
int delayThreshold = source.getDbGroupConfig().getDelayThreshold();
if (delayThreshold > 0 && behindMaster > delayThreshold) {
MySQLHeartbeat.LOGGER.warn("found MySQL master/slave Replication delay !!! " + heartbeat.getSource().getConfig() + ", binlog sync time delay: " + behindMaster + "s");
if (delayThreshold > 0) {
String alertKey = source.getDbGroupConfig().getName() + "-" + source.getConfig().getInstanceName();
if (behindMaster > delayThreshold) {
Map<String, String> labels = AlertUtil.genSingleLabel("dbInstance", alertKey);
String errMsg = "found MySQL master/slave Replication delay !!! " + source.getConfig() + ", binlog sync time delay: " + behindMaster + "s";
MySQLHeartbeat.LOGGER.warn(errMsg);
AlertUtil.alert(AlarmCode.DB_SLAVE_INSTANCE_DELAY, Alert.AlertLevel.WARN, errMsg, "mysql", source.getConfig().getId(), labels);
ToResolveContainer.DB_SLAVE_INSTANCE_DELAY.add(alertKey);
} else {
if (ToResolveContainer.DB_SLAVE_INSTANCE_DELAY.contains(alertKey)) {
Map<String, String> labels = AlertUtil.genSingleLabel("dbInstance", alertKey);
AlertUtil.alertResolve(AlarmCode.DB_SLAVE_INSTANCE_DELAY, Alert.AlertLevel.WARN, "mysql", source.getConfig().getId(), labels, ToResolveContainer.DB_SLAVE_INSTANCE_DELAY, alertKey);
}
}
}
heartbeat.setSlaveBehindMaster(behindMaster);
} else {