mirror/dble
1
0
Fork 0
mirror of https://github.com/actiontech/dble.git synced 2026-05-02 12:30:31 -05:00
Code Issues Packages Projects Releases Wiki Activity

alert

Browse Source
This commit is contained in:
yanhuqing
2018-06-29 18:13:02 +08:00
parent c47f34c5a0
commit 71dacf4baf
37 changed files with 3427 additions and 3172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
findbugs-exclude.xml
+1 -1
View File
@@ -23,7 +23,7 @@
<Class name="com.actiontech.dble.meta.protocol.StructureMeta$TableMeta$Builder"/>
</Match>
<Match>
<Package name="com.actiontech.dble.log.alarm"/>
<Package name="com.actiontech.dble.alarm"/>
</Match>
<Match>
<Class name="com.actiontech.dble.log.AlarmAppender"/>
src/main/assembly/conf/log4j2.xml
+1 -3
View File
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" monitorInterval="30" packages="com.actiontech.dble.log">
<Configuration status="WARN" monitorInterval="30">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d [%-5p][%t] %m %throwable{full} (%C:%F:%L) %n"/>
@@ -28,13 +28,11 @@
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<AlarmAppender name="grpcAppender" />
</Appenders>
<Loggers>
<asyncRoot level="info" includeLocation="true">
<!--<AppenderRef ref="Console" />-->
<AppenderRef ref="RollingFile"/>
<AppenderRef ref="grpcAppender"/>
</asyncRoot>
</Loggers>
</Configuration>
src/main/java/com/actiontech/dble/DbleServer.java
+5 -2
View File
@@ -24,7 +24,9 @@ import com.actiontech.dble.config.model.SystemConfig;
import com.actiontech.dble.config.model.TableConfig;
import com.actiontech.dble.config.util.ConfigUtil;
import com.actiontech.dble.config.util.DnPropertyUtil;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.log.transaction.TxnLogProcessor;
import com.actiontech.dble.manager.ManagerConnectionFactory;
import com.actiontech.dble.memory.unsafe.Platform;
@@ -672,7 +674,8 @@ public final class DbleServer {
}
}
} catch (Exception e) {
LOGGER.warn(AlarmCode.WRITE_NODE_INDEX_FAIL + "saveDataNodeIndex err:", e);
LOGGER.warn("saveDataNodeIndex err:", e);
AlertUtil.alertSelf(AlarmCode.WRITE_NODE_INDEX_FAIL, Alert.AlertLevel.WARN, "saveDataNodeIndex err:" + e.getMessage(), null);
} finally {
if (fileOut != null) {
try {
src/main/java/com/actiontech/dble/log/alarm/AlarmCode.java → src/main/java/com/actiontech/dble/alarm/AlarmCode.java
+20 -26
View File
@@ -3,7 +3,7 @@
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.log.alarm;
package com.actiontech.dble.alarm;
/**
@@ -12,31 +12,25 @@ package com.actiontech.dble.log.alarm;
public final class AlarmCode {
private AlarmCode() {
}
public static final String ALARM_SPLIT = "::";
public static final String WRITE_NODE_INDEX_FAIL = "DBLE_WRITE_NODE_INDEX_FAIL" + ALARM_SPLIT;
public static final String WRITE_TEMP_RESULT_FAIL = "DBLE_WRITE_TEMP_RESULT_FAIL" + ALARM_SPLIT;
public static final String XA_READ_IO_FAIL = "DBLE_XA_READ_IO_FAIL" + ALARM_SPLIT;
public static final String XA_READ_XA_STREAM_FAIL = "DBLE_XA_READ_XA_STREAM_FAIL" + ALARM_SPLIT;
public static final String XA_READ_DECODE_FAIL = "DBLE_XA_READ_DECODE_FAIL" + ALARM_SPLIT;
public static final String XA_WRITE_CHECK_POINT_FAIL = "DBLE_XA_WRITE_CHECK_POINT_FAIL" + ALARM_SPLIT;
public static final String REACH_MAX_CON = "DBLE_REACH_MAX_CON" + ALARM_SPLIT;
public static final String TABLE_NOT_CONSISTENT_IN_DATAHOSTS = "DBLE_TABLE_NOT_CONSISTENT_IN_DATAHOSTS" + ALARM_SPLIT;
public static final String TABLE_NOT_CONSISTENT_IN_MEMORY = "DBLE_TABLE_NOT_CONSISTENT_IN_MEMORY" + ALARM_SPLIT;
public static final String GLOBAL_TABLE_COLUMN_LOST = "DBLE_GLOBAL_TABLE_COLUMN_LOST" + ALARM_SPLIT;
public static final String CREATE_CONN_FAIL = "DBLE_CREATE_CONN_FAIL" + ALARM_SPLIT;
public static final String DATA_HOST_CAN_NOT_REACH = "DBLE_DATA_HOST_CAN_NOT_REACH" + ALARM_SPLIT;
public static final String TABLE_LACK = "DBLE_TABLE_LACK" + ALARM_SPLIT;
public static final String GET_TABLE_META_FAIL = "DBLE_GET_TABLE_META_FAIL" + ALARM_SPLIT;
public static final String NIOREACTOR_UNKNOWN_EXCEPTION = "DBLE_NIOREACTOR_UNKNOWN_EXCEPTION" + ALARM_SPLIT;
public static final String NIOREACTOR_UNKNOWN_THROWABLE = "DBLE_NIOREACTOR_UNKNOWN_THROWABLE" + ALARM_SPLIT;
public static final String NIOCONNECTOR_UNKNOWN_EXCEPTION = "DBLE_NIOCONNECTOR_UNKNOWN_EXCEPTION" + ALARM_SPLIT;
public static final String TEST_CONN_FAIL = "DBLE_TEST_CONN_FAIL" + ALARM_SPLIT;
public static final String KILL_BACKEND_CONN_FAIL = "DBLE_KILL_BACKEND_CONN_FAIL" + ALARM_SPLIT;
public static final String WRITE_NODE_INDEX_FAIL = "DBLE_WRITE_NODE_INDEX_FAIL";
public static final String WRITE_TEMP_RESULT_FAIL = "DBLE_WRITE_TEMP_RESULT_FAIL";
public static final String XA_READ_IO_FAIL = "DBLE_XA_READ_IO_FAIL";
public static final String XA_READ_XA_STREAM_FAIL = "DBLE_XA_READ_XA_STREAM_FAIL";
public static final String XA_READ_DECODE_FAIL = "DBLE_XA_READ_DECODE_FAIL";
public static final String XA_WRITE_CHECK_POINT_FAIL = "DBLE_XA_WRITE_CHECK_POINT_FAIL"; //Resolve by trigger
public static final String REACH_MAX_CON = "DBLE_REACH_MAX_CON"; //Resolve by trigger
public static final String TABLE_NOT_CONSISTENT_IN_DATAHOSTS = "DBLE_TABLE_NOT_CONSISTENT_IN_DATAHOSTS"; //Resolve by trigger
public static final String TABLE_NOT_CONSISTENT_IN_MEMORY = "DBLE_TABLE_NOT_CONSISTENT_IN_MEMORY"; //Resolve by trigger
public static final String GLOBAL_TABLE_COLUMN_LOST = "DBLE_GLOBAL_TABLE_COLUMN_LOST"; //Resolve by trigger
public static final String CREATE_CONN_FAIL = "DBLE_CREATE_CONN_FAIL"; //Resolve by trigger
public static final String DATA_HOST_CAN_NOT_REACH = "DBLE_DATA_HOST_CAN_NOT_REACH";
public static final String TABLE_LACK = "DBLE_TABLE_LACK"; //Resolve by trigger
public static final String GET_TABLE_META_FAIL = "DBLE_GET_TABLE_META_FAIL";
public static final String NIOREACTOR_UNKNOWN_EXCEPTION = "DBLE_NIOREACTOR_UNKNOWN_EXCEPTION";
public static final String NIOREACTOR_UNKNOWN_THROWABLE = "DBLE_NIOREACTOR_UNKNOWN_THROWABLE";
public static final String NIOCONNECTOR_UNKNOWN_EXCEPTION = "DBLE_NIOCONNECTOR_UNKNOWN_EXCEPTION";
public static final String TEST_CONN_FAIL = "DBLE_TEST_CONN_FAIL";
public static final String KILL_BACKEND_CONN_FAIL = "DBLE_KILL_BACKEND_CONN_FAIL";
}
src/main/java/com/actiontech/dble/alarm/Alert.java
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright (C) 2016-2018 ActionTech.
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.alarm;
import java.util.Map;
public interface Alert {
enum AlertLevel {
NOTICE, WARN, CRITICAL
}
void alertSelfWithTarget(String code, AlertLevel level, String desc, String alertComponentId, Map<String, String> labels);
void alertSelf(String code, AlertLevel level, String desc, Map<String, String> labels);
void alert(String code, AlertLevel level, String desc, String alertComponentType, String alertComponentId, Map<String, String> labels);
boolean alertResolve(String code, AlertLevel level, String alertComponentType, String alertComponentId, Map<String, String> labels);
boolean alertSelfResolve(String code, AlertLevel level, Map<String, String> labels);
boolean alertSelfWithTargetResolve(String code, Alert.AlertLevel level, String alertComponentId, Map<String, String> labels);
}
src/main/java/com/actiontech/dble/alarm/AlertUtil.java
+50
View File
@@ -0,0 +1,50 @@
/*
* Copyright (C) 2016-2018 ActionTech.
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.alarm;
import com.actiontech.dble.cluster.ClusterParamCfg;
import com.actiontech.dble.config.loader.ucoreprocess.UcoreConfig;
import java.util.Map;
public final class AlertUtil {
private AlertUtil() {
}
private static Alert alert;
static {
if (UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_CFG_MYID) != null) {
alert = UcoreAlert.getInstance();
} else {
alert = new NoAlert();
}
}
public static void alertSelfWithTarget(String code, Alert.AlertLevel level, String desc, String alertComponentId, Map<String, String> labels) {
alert.alertSelfWithTarget(code, level, desc, alertComponentId, labels);
}
public static void alertSelf(String code, Alert.AlertLevel level, String desc, Map<String, String> labels) {
alert.alertSelf(code, level, desc, labels);
}
public static void alert(String code, Alert.AlertLevel level, String desc, String alertComponentType, String alertComponentId, Map<String, String> labels) {
alert.alert(code, level, desc, alertComponentType, alertComponentId, labels);
}
public static boolean alertResolve(String code, Alert.AlertLevel level, String alertComponentType, String alertComponentId, Map<String, String> labels) {
return alert.alertResolve(code, level, alertComponentType, alertComponentId, labels);
}
public static boolean alertSelfResolve(String code, Alert.AlertLevel level, Map<String, String> labels) {
return alert.alertSelfResolve(code, level, labels);
}
public static boolean alertSelfWithTargetResolve(String code, Alert.AlertLevel level, String alertComponentId, Map<String, String> labels) {
return alert.alertSelfWithTargetResolve(code, level, alertComponentId, labels);
}
}
src/main/java/com/actiontech/dble/alarm/NoAlert.java
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright (C) 2016-2018 ActionTech.
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.alarm;
import java.util.Map;
public class NoAlert implements Alert {
@Override
public void alertSelfWithTarget(String code, AlertLevel level, String desc, String alertComponentId, Map<String, String> labels) {
}
@Override
public void alertSelf(String code, AlertLevel level, String desc, Map<String, String> labels) {
}
@Override
public void alert(String code, AlertLevel level, String desc, String alertComponentType, String alertComponentId, Map<String, String> labels) {
}
@Override
public boolean alertResolve(String code, AlertLevel level, String alertComponentType, String alertComponentId, Map<String, String> labels) {
return true;
}
@Override
public boolean alertSelfResolve(String code, AlertLevel level, Map<String, String> labels) {
return true;
}
@Override
public boolean alertSelfWithTargetResolve(String code, AlertLevel level, String alertComponentId, Map<String, String> labels) {
return true;
}
}
src/main/java/com/actiontech/dble/alarm/ToResolveContainer.java
+20
View File
@@ -0,0 +1,20 @@
/*
* Copyright (C) 2016-2018 ActionTech.
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.alarm;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
public final class ToResolveContainer {
private ToResolveContainer() {
}
public static final Set<String> TABLE_NOT_CONSISTENT_IN_DATAHOSTS = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
public static final Set<String> TABLE_NOT_CONSISTENT_IN_MEMORY = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
public static final Set<String> TABLE_LACK = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
public static final Set<String> GLOBAL_TABLE_COLUMN_LOST = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
}
src/main/java/com/actiontech/dble/alarm/UcoreAlert.java
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright (C) 2016-2018 ActionTech.
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.alarm;
import com.actiontech.dble.cluster.ClusterParamCfg;
import com.actiontech.dble.config.loader.ucoreprocess.ClusterUcoreSender;
import com.actiontech.dble.config.loader.ucoreprocess.UcoreConfig;
import java.util.Map;
public final class UcoreAlert implements Alert {
private static final String SOURCE_COMPONENT_TYPE = "dble";
private static final String SERVER_ID = UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_CFG_SERVER_ID);
private static final String SOURCE_COMPONENT_ID = UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_CFG_MYID);
private static final UcoreAlert INSTANCE = new UcoreAlert();
public static UcoreAlert getInstance() {
return INSTANCE;
}
private UcoreAlert() {
}
@Override
public void alertSelfWithTarget(String code, AlertLevel level, String desc, String alertComponentId, Map<String, String> labels) {
alert(code, level, desc, SOURCE_COMPONENT_TYPE, alertComponentId, labels);
}
@Override
public void alertSelf(String code, AlertLevel level, String desc, Map<String, String> labels) {
alert(code, level, desc, SOURCE_COMPONENT_TYPE, SOURCE_COMPONENT_ID, labels);
}
@Override
public void alert(String code, AlertLevel level, String desc, String alertComponentType, String alertComponentId, Map<String, String> labels) {
UcoreInterface.AlertInput.Builder builder = UcoreInterface.AlertInput.newBuilder().
setCode(code).
setDesc(desc).
setLevel(level.toString()).
setSourceComponentType(SOURCE_COMPONENT_TYPE).
setSourceComponentId(SOURCE_COMPONENT_ID).
setAlertComponentId(alertComponentId).
setAlertComponentType(alertComponentType).
setServerId(SERVER_ID).
setTimestampUnix(System.currentTimeMillis() * 1000000);
if (labels != null) {
builder.putAllLabels(labels);
}
UcoreInterface.AlertInput input = builder.build();
ClusterUcoreSender.alert(input);
}
@Override
public boolean alertResolve(String code, AlertLevel level, String alertComponentType, String alertComponentId, Map<String, String> labels) {
UcoreInterface.AlertInput.Builder builder = UcoreInterface.AlertInput.newBuilder().
setCode(code).
setDesc("").
setLevel(level.toString()).
setSourceComponentType(SOURCE_COMPONENT_TYPE).
setSourceComponentId(SOURCE_COMPONENT_ID).
setAlertComponentId(alertComponentId).
setAlertComponentType(alertComponentType).
setServerId(SERVER_ID).
setResolveTimestampUnix(System.currentTimeMillis() * 1000000);
if (labels != null) {
builder.putAllLabels(labels);
}
UcoreInterface.AlertInput input = builder.build();
return ClusterUcoreSender.alertResolve(input);
}
@Override
public boolean alertSelfResolve(String code, AlertLevel level, Map<String, String> labels) {
return alertResolve(code, level, SOURCE_COMPONENT_TYPE, SOURCE_COMPONENT_ID, labels);
}
@Override
public boolean alertSelfWithTargetResolve(String code, AlertLevel level, String alertComponentId, Map<String, String> labels) {
return alertResolve(code, level, SOURCE_COMPONENT_TYPE, alertComponentId, labels);
}
}
src/main/java/com/actiontech/dble/log/alarm/UcoreGrpc.java → src/main/java/com/actiontech/dble/alarm/UcoreGrpc.java
+642 -649
View File
File diff suppressed because it is too large Load Diff
src/main/java/com/actiontech/dble/log/alarm/UcoreInterface.java → src/main/java/com/actiontech/dble/alarm/UcoreInterface.java
+2283 -2283
View File
File diff suppressed because it is too large Load Diff
src/main/java/com/actiontech/dble/log/alarm/ucore.proto → src/main/java/com/actiontech/dble/alarm/ucore.proto
+1 -1
View File
@@ -1,6 +1,6 @@
syntax = "proto3";
option java_multiple_files = false;
option java_package = "com.actiontech.dble.log.alarm";
option java_package = "com.actiontech.dble.alarm";
option java_outer_classname = "UcoreInterface";
option objc_class_prefix = "ucore";
src/main/java/com/actiontech/dble/backend/datasource/PhysicalDBNode.java
+9 -2
View File
@@ -8,12 +8,16 @@ package com.actiontech.dble.backend.datasource;
import com.actiontech.dble.DbleServer;
import com.actiontech.dble.backend.BackendConnection;
import com.actiontech.dble.backend.mysql.nio.handler.ResponseHandler;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.route.RouteResultsetNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class PhysicalDBNode {
protected static final Logger LOGGER = LoggerFactory.getLogger(PhysicalDBNode.class);
@@ -110,7 +114,10 @@ public class PhysicalDBNode {
PhysicalDatasource readSource = dbPool.getRWBalanceNode();
if (!readSource.isAlive()) {
String heartbeatError = "the data source[" + readSource.getConfig().getUrl() + "] can't reached, please check the dataHost";
LOGGER.warn(AlarmCode.DATA_HOST_CAN_NOT_REACH + heartbeatError);
LOGGER.warn(heartbeatError);
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", readSource.getHostConfig().getName() + "-" + readSource.getConfig().getHostName());
AlertUtil.alert(AlarmCode.DATA_HOST_CAN_NOT_REACH, Alert.AlertLevel.WARN, heartbeatError, "mysql", readSource.getConfig().getId(), labels);
throw new IOException(heartbeatError);
}
return readSource.getConnection(schema, autoCommit);
src/main/java/com/actiontech/dble/backend/datasource/PhysicalDBPool.java
+7 -2
View File
@@ -11,7 +11,9 @@ import com.actiontech.dble.backend.heartbeat.DBHeartbeat;
import com.actiontech.dble.backend.mysql.nio.handler.GetConnectionHandler;
import com.actiontech.dble.backend.mysql.nio.handler.ResponseHandler;
import com.actiontech.dble.config.model.DataHostConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -512,7 +514,10 @@ public class PhysicalDBPool {
}
if (!theNode.isAlive()) {
String heartbeatError = "the data source[" + theNode.getConfig().getUrl() + "] can't reached, please check the dataHost";
LOGGER.warn(AlarmCode.DATA_HOST_CAN_NOT_REACH + heartbeatError);
LOGGER.warn(heartbeatError);
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", theNode.getHostConfig().getName() + "-" + theNode.getConfig().getHostName());
AlertUtil.alert(AlarmCode.DATA_HOST_CAN_NOT_REACH, Alert.AlertLevel.WARN, heartbeatError, "mysql", theNode.getConfig().getId(), labels);
throw new IOException(heartbeatError);
}
theNode.getConnection(schema, autocommit, handler, attachment);
src/main/java/com/actiontech/dble/backend/datasource/PhysicalDatasource.java
+56 -11
View File
@@ -6,6 +6,9 @@
package com.actiontech.dble.backend.datasource;
import com.actiontech.dble.DbleServer;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.backend.BackendConnection;
import com.actiontech.dble.backend.ConMap;
import com.actiontech.dble.backend.ConQueue;
@@ -17,7 +20,6 @@ import com.actiontech.dble.backend.mysql.nio.handler.NewConnectionRespHandler;
import com.actiontech.dble.backend.mysql.nio.handler.ResponseHandler;
import com.actiontech.dble.config.model.DBHostConfig;
import com.actiontech.dble.config.model.DataHostConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.util.StringUtil;
import com.actiontech.dble.util.TimeUtil;
import org.slf4j.Logger;
@@ -25,7 +27,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -43,6 +47,8 @@ public abstract class PhysicalDatasource {
private volatile long heartbeatRecoveryTime;
private final DataHostConfig hostConfig;
private PhysicalDBPool dbPool;
private volatile boolean maxConnAlert = false;
private volatile boolean createConnAlert = false;
private AtomicBoolean dying = new AtomicBoolean(false);
private AtomicLong readCount = new AtomicLong(0);
@@ -282,8 +288,18 @@ public abstract class PhysicalDatasource {
// creat new connection
this.createNewConnection(simpleHandler, null, schemas[i % schemas.length]);
simpleHandler.getBackConn().release();
if (createConnAlert) {
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
createConnAlert = !AlertUtil.alertResolve(AlarmCode.CREATE_CONN_FAIL, Alert.AlertLevel.WARN, "mysql", this.getConfig().getId(), labels);
}
} catch (IOException e) {
LOGGER.warn(AlarmCode.CREATE_CONN_FAIL + "create connection err ", e);
String errMsg = "create connection err ";
LOGGER.warn(errMsg, e);
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
AlertUtil.alert(AlarmCode.CREATE_CONN_FAIL, Alert.AlertLevel.WARN, errMsg + e.getMessage(), "mysql", this.getConfig().getId(), labels);
createConnAlert = true;
}
}
}
@@ -385,11 +401,19 @@ public abstract class PhysicalDatasource {
int activeCons = this.getActiveCount();
if (activeCons + 1 > size) {
String maxConError = "the max active Connections size can not be max than maxCon for data host[" + this.getHostConfig().getName() + "." + this.getName() + "]";
LOGGER.warn(AlarmCode.REACH_MAX_CON + maxConError);
LOGGER.warn(maxConError);
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
AlertUtil.alert(AlarmCode.REACH_MAX_CON, Alert.AlertLevel.WARN, maxConError, "dble", this.getConfig().getId(), labels);
maxConnAlert = true;
throw new IOException(maxConError);
} else { // create connection
LOGGER.info("no idle connection in pool,create new connection for " +
this.name + " of schema " + schema);
if (maxConnAlert) {
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
maxConnAlert = !AlertUtil.alertResolve(AlarmCode.REACH_MAX_CON, Alert.AlertLevel.WARN, "dble", this.getConfig().getId(), labels);
}
LOGGER.info("no idle connection in pool,create new connection for " + this.name + " of schema " + schema);
createNewConnection(handler, attachment, schema);
}
}
@@ -401,14 +425,35 @@ public abstract class PhysicalDatasource {
int activeCons = this.getActiveCount(); // the max active
if (activeCons + 1 > size) {
String maxConError = "the max active Connections size can not be max than maxCon data host[" + this.getHostConfig().getName() + "." + this.getName() + "]";
LOGGER.warn(AlarmCode.REACH_MAX_CON + maxConError);
LOGGER.warn(maxConError);
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
AlertUtil.alert(AlarmCode.REACH_MAX_CON, Alert.AlertLevel.WARN, maxConError, "dble", this.getConfig().getId(), labels);
maxConnAlert = true;
throw new IOException(maxConError);
} else { // create connection
LOGGER.info(
"no ilde connection in pool,create new connection for " + this.name + " of schema " + schema);
NewConnectionRespHandler simpleHandler = new NewConnectionRespHandler();
this.createNewConnection(simpleHandler, schema);
con = simpleHandler.getBackConn();
if (maxConnAlert) {
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
maxConnAlert = !AlertUtil.alertResolve(AlarmCode.REACH_MAX_CON, Alert.AlertLevel.WARN, "dble", this.getConfig().getId(), labels);
}
LOGGER.info("no ilde connection in pool,create new connection for " + this.name + " of schema " + schema);
try {
NewConnectionRespHandler simpleHandler = new NewConnectionRespHandler();
this.createNewConnection(simpleHandler, schema);
con = simpleHandler.getBackConn();
if (createConnAlert) {
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
createConnAlert = !AlertUtil.alertResolve(AlarmCode.CREATE_CONN_FAIL, Alert.AlertLevel.WARN, "mysql", this.getConfig().getId(), labels);
}
} catch (IOException e) {
Map<String, String> labels = new HashMap<>(1);
labels.put("data_host", this.getHostConfig().getName() + "-" + this.getConfig().getHostName());
AlertUtil.alert(AlarmCode.CREATE_CONN_FAIL, Alert.AlertLevel.WARN, "createNewConn Error" + e.getMessage(), "mysql", this.getConfig().getId(), labels);
createConnAlert = true;
throw e;
}
}
}
return takeCon(con, schema);
src/main/java/com/actiontech/dble/backend/mysql/store/FileStore.java
+5 -2
View File
@@ -9,7 +9,9 @@ import com.actiontech.dble.DbleServer;
import com.actiontech.dble.backend.mysql.store.fs.FilePath;
import com.actiontech.dble.backend.mysql.store.fs.FileUtils;
import com.actiontech.dble.config.ErrorCode;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.util.exception.TmpFileException;
import org.apache.log4j.Logger;
@@ -238,7 +240,8 @@ public class FileStore {
this.files.add(path.open(mode));
this.fileNames.add(path.toString());
} else {
logger.warn(AlarmCode.WRITE_TEMP_RESULT_FAIL + "create file error :", e);
logger.warn("create file error :", e);
AlertUtil.alertSelf(AlarmCode.WRITE_TEMP_RESULT_FAIL, Alert.AlertLevel.WARN, "create file error:" + e.getMessage(), null);
throw e;
}
}
src/main/java/com/actiontech/dble/backend/mysql/xa/XAStateLog.java
+10
View File
@@ -50,6 +50,7 @@ public final class XAStateLog {
private static AtomicInteger batchNum = new AtomicInteger(0);
private static Set<Long> waitSet = new CopyOnWriteArraySet<>();
private static ConcurrentMap<Long, Boolean> mapResult = new ConcurrentHashMap<>();
private static volatile boolean writeAlert = false;
public static boolean saveXARecoveryLog(String xaTxId, TxState sessionState) {
CoordinatorLogEntry coordinatorLogEntry = IN_MEMORY_REPOSITORY.get(xaTxId);
@@ -200,4 +201,13 @@ public final class XAStateLog {
}
}
}
public static void setWriteAlert(boolean writeAlert) {
XAStateLog.writeAlert = writeAlert;
}
public static boolean isWriteAlert() {
return writeAlert;
}
}
src/main/java/com/actiontech/dble/backend/mysql/xa/recovery/impl/FileSystemRepository.java
+20 -11
View File
@@ -6,14 +6,13 @@
package com.actiontech.dble.backend.mysql.xa.recovery.impl;
import com.actiontech.dble.DbleServer;
import com.actiontech.dble.backend.mysql.xa.CoordinatorLogEntry;
import com.actiontech.dble.backend.mysql.xa.Deserializer;
import com.actiontech.dble.backend.mysql.xa.Serializer;
import com.actiontech.dble.backend.mysql.xa.VersionedFile;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.backend.mysql.xa.*;
import com.actiontech.dble.backend.mysql.xa.recovery.DeserializationException;
import com.actiontech.dble.backend.mysql.xa.recovery.Repository;
import com.actiontech.dble.config.model.SystemConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +115,9 @@ public class FileSystemRepository implements Repository {
br = new BufferedReader(isr);
coordinatorLogEntries = readContent(br);
} catch (Exception e) {
LOGGER.warn(AlarmCode.XA_READ_IO_FAIL + "Error in recover", e);
LOGGER.warn("Error in recover", e);
AlertUtil.alertSelf(AlarmCode.XA_READ_IO_FAIL, Alert.AlertLevel.WARN, "Error in recover:" + e.getMessage(), null);
} finally {
closeSilently(br);
}
@@ -141,12 +142,14 @@ public class FileSystemRepository implements Repository {
unexpectedEOF);
// merely return what was read so far...
} catch (ObjectStreamException unexpectedEOF) {
LOGGER.warn(AlarmCode.XA_READ_XA_STREAM_FAIL +
"Unexpected EOF - logfile not closed properly last time?",
unexpectedEOF);
LOGGER.warn("Unexpected EOF - logfile not closed properly last time?", unexpectedEOF);
AlertUtil.alertSelf(AlarmCode.XA_READ_XA_STREAM_FAIL, Alert.AlertLevel.WARN,
"Unexpected EOF - logfile not closed properly last time?" + unexpectedEOF.getMessage(), null);
// merely return what was read so far...
} catch (DeserializationException unexpectedEOF) {
LOGGER.warn(AlarmCode.XA_READ_DECODE_FAIL + "Unexpected EOF - logfile not closed properly last time? " + unexpectedEOF);
LOGGER.warn("DeserializationException - logfile not closed properly last time? ", unexpectedEOF);
AlertUtil.alertSelf(AlarmCode.XA_READ_DECODE_FAIL, Alert.AlertLevel.WARN,
"DeserializationException - logfile not closed properly last time? " + unexpectedEOF.getMessage(), null);
}
return coordinatorLogEntries;
}
@@ -196,9 +199,15 @@ public class FileSystemRepository implements Repository {
}
rwChannel.force(false);
file.discardBackupVersion();
if (XAStateLog.isWriteAlert()) {
boolean resolved = AlertUtil.alertSelfResolve(AlarmCode.XA_WRITE_CHECK_POINT_FAIL, Alert.AlertLevel.WARN, null);
XAStateLog.setWriteAlert(resolved);
}
return true;
} catch (Exception e) {
LOGGER.warn(AlarmCode.XA_WRITE_CHECK_POINT_FAIL + "Failed to write checkpoint", e);
LOGGER.warn("Failed to write checkpoint", e);
AlertUtil.alertSelf(AlarmCode.XA_WRITE_CHECK_POINT_FAIL, Alert.AlertLevel.WARN, "Failed to write checkpoint" + e.getMessage(), null);
XAStateLog.setWriteAlert(true);
return false;
}
}
src/main/java/com/actiontech/dble/config/ServerConfig.java
+7 -3
View File
@@ -12,7 +12,9 @@ import com.actiontech.dble.backend.datasource.PhysicalDatasource;
import com.actiontech.dble.config.model.*;
import com.actiontech.dble.config.util.ConfigException;
import com.actiontech.dble.config.util.ConfigUtil;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.route.sequence.handler.DistributedSequenceHandler;
import com.actiontech.dble.route.sequence.handler.IncrSequenceMySQLHandler;
import com.actiontech.dble.route.sequence.handler.IncrSequenceTimeHandler;
@@ -83,7 +85,8 @@ public class ServerConfig {
try {
confInit.testConnection(true);
} catch (ConfigException e) {
LOGGER.warn(AlarmCode.TEST_CONN_FAIL, e);
LOGGER.warn("TestConnection fail", e);
AlertUtil.alertSelf(AlarmCode.TEST_CONN_FAIL, Alert.AlertLevel.WARN, "TestConnection fail:" + e.getMessage(), null);
}
}
@@ -109,7 +112,8 @@ public class ServerConfig {
try {
confInit.testConnection(true);
} catch (ConfigException e) {
LOGGER.warn(AlarmCode.TEST_CONN_FAIL, e);
LOGGER.warn("TestConnection fail", e);
AlertUtil.alertSelf(AlarmCode.TEST_CONN_FAIL, Alert.AlertLevel.WARN, "TestConnection fail:" + e.getMessage(), null);
}
}
src/main/java/com/actiontech/dble/config/loader/ucoreprocess/ClusterUcoreSender.java
+30 -8
View File
@@ -3,8 +3,8 @@ package com.actiontech.dble.config.loader.ucoreprocess;
import com.actiontech.dble.cluster.ClusterParamCfg;
import com.actiontech.dble.config.loader.ucoreprocess.KVtoXml.UcoreToXml;
import com.actiontech.dble.config.loader.ucoreprocess.bean.UKvBean;
import com.actiontech.dble.log.alarm.UcoreGrpc;
import com.actiontech.dble.log.alarm.UcoreInterface;
import com.actiontech.dble.alarm.UcoreGrpc;
import com.actiontech.dble.alarm.UcoreInterface;
import io.grpc.Channel;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
@@ -324,20 +324,19 @@ public final class ClusterUcoreSender {
}
public static void alert(UcoreInterface.AlertInput inpurt) throws IOException {
public static void alert(UcoreInterface.AlertInput input) {
try {
stub.withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS).alert(inpurt);
stub.withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS).alert(input);
} catch (Exception e) {
for (String ip : UcoreConfig.getInstance().getIpList()) {
ManagedChannel channel = null;
try {
channel = ManagedChannelBuilder.forAddress(ip,
Integer.parseInt(UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_PLUGINS_PORT))).usePlaintext(true).build();
channel = ManagedChannelBuilder.forAddress(ip, Integer.parseInt(UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_PLUGINS_PORT))).usePlaintext(true).build();
stub = UcoreGrpc.newBlockingStub(channel).withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS);
stub.withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS).alert(inpurt);
stub.withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS).alert(input);
return;
} catch (Exception e2) {
LOGGER.info("connect to ucore error ", e2);
LOGGER.info("alert to ucore error ", e2);
if (channel != null) {
channel.shutdownNow();
}
@@ -346,6 +345,29 @@ public final class ClusterUcoreSender {
}
}
public static boolean alertResolve(UcoreInterface.AlertInput input) {
try {
stub.withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS).alertResolve(input);
return true;
} catch (Exception e) {
for (String ip : UcoreConfig.getInstance().getIpList()) {
ManagedChannel channel = null;
try {
channel = ManagedChannelBuilder.forAddress(ip, Integer.parseInt(UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_PLUGINS_PORT))).usePlaintext(true).build();
stub = UcoreGrpc.newBlockingStub(channel).withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS);
stub.withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS).alertResolve(input);
return true;
} catch (Exception e2) {
LOGGER.info("alertResolve to ucore error ", e2);
if (channel != null) {
channel.shutdownNow();
}
return false;
}
}
return false;
}
}
public static String waitingForAllTheNode(String checkString, String path) {
Map<String, String> expectedMap = UcoreToXml.getOnlineMap();
src/main/java/com/actiontech/dble/config/loader/ucoreprocess/listen/UOffLineListener.java
+1 -1
View File
@@ -9,7 +9,7 @@ import com.actiontech.dble.config.loader.ucoreprocess.bean.UKvBean;
import com.actiontech.dble.config.loader.ucoreprocess.loader.UDdlChildResponse;
import com.actiontech.dble.config.loader.zkprocess.zookeeper.process.BinlogPause;
import com.actiontech.dble.config.loader.zkprocess.zookeeper.process.PauseInfo;
import com.actiontech.dble.log.alarm.UcoreInterface;
import com.actiontech.dble.alarm.UcoreInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
src/main/java/com/actiontech/dble/config/loader/ucoreprocess/listen/UcoreClearKeyListener.java
+1 -1
View File
@@ -4,7 +4,7 @@ import com.actiontech.dble.config.loader.ucoreprocess.ClusterUcoreSender;
import com.actiontech.dble.config.loader.ucoreprocess.UcorePathUtil;
import com.actiontech.dble.config.loader.ucoreprocess.UcoreXmlLoader;
import com.actiontech.dble.config.loader.ucoreprocess.bean.UKvBean;
import com.actiontech.dble.log.alarm.UcoreInterface;
import com.actiontech.dble.alarm.UcoreInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
src/main/java/com/actiontech/dble/config/loader/ucoreprocess/listen/UcoreNodesListener.java
+1 -1
View File
@@ -3,7 +3,7 @@ package com.actiontech.dble.config.loader.ucoreprocess.listen;
import com.actiontech.dble.config.loader.ucoreprocess.ClusterUcoreSender;
import com.actiontech.dble.config.loader.ucoreprocess.UcoreConfig;
import com.actiontech.dble.log.alarm.UcoreInterface;
import com.actiontech.dble.alarm.UcoreInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
src/main/java/com/actiontech/dble/config/loader/ucoreprocess/listen/UcoreSingleKeyListener.java
+1 -1
View File
@@ -3,7 +3,7 @@ package com.actiontech.dble.config.loader.ucoreprocess.listen;
import com.actiontech.dble.config.loader.ucoreprocess.ClusterUcoreSender;
import com.actiontech.dble.config.loader.ucoreprocess.UcoreXmlLoader;
import com.actiontech.dble.config.loader.ucoreprocess.bean.UKvBean;
import com.actiontech.dble.log.alarm.UcoreInterface;
import com.actiontech.dble.alarm.UcoreInterface;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
src/main/java/com/actiontech/dble/config/loader/xml/XMLSchemaLoader.java
+8 -5
View File
@@ -497,8 +497,6 @@ public class XMLSchemaLoader implements SchemaLoader {
String nodeUrl = node.getAttribute("url");
String user = node.getAttribute("user");
String password = node.getAttribute("password");
String usingDecrypt = node.getAttribute("usingDecrypt");
String weightStr = node.getAttribute("weight");
if (empty(nodeHost) || empty(nodeUrl) || empty(user)) {
throw new ConfigException(
@@ -506,19 +504,24 @@ public class XMLSchemaLoader implements SchemaLoader {
" define error,some attributes of this element is empty: " +
nodeHost);
}
int colonIndex = nodeUrl.indexOf(':');
String ip = nodeUrl.substring(0, colonIndex).trim();
int port = Integer.parseInt(nodeUrl.substring(colonIndex + 1).trim());
String usingDecrypt = node.getAttribute("usingDecrypt");
String passwordEncryty = DecryptUtil.dbHostDecrypt(usingDecrypt, nodeHost, user, password);
String weightStr = node.getAttribute("weight");
int weight = "".equals(weightStr) ? PhysicalDBPool.WEIGHT : Integer.parseInt(weightStr);
DBHostConfig conf = new DBHostConfig(nodeHost, ip, port, nodeUrl, user, passwordEncryty);
conf.setMaxCon(maxCon);
conf.setMinCon(minCon);
conf.setWeight(weight);
String id = node.getAttribute("id");
if (!"".equals(id)) {
conf.setId(id);
} else {
conf.setId(nodeHost);
}
return conf;
}
src/main/java/com/actiontech/dble/config/loader/zkprocess/entity/schema/datahost/ReadHost.java
+1 -4
View File
@@ -38,10 +38,7 @@ public class ReadHost extends WriteHost {
@Override
public String toString() {
String builder = "ReadHost [weight=" +
weight +
"]";
return builder;
return "ReadHost [host=" + host + ", url=" + url + ", password=" + password + ", user=" + user + ", usingDecrypt=" + usingDecrypt + ", weight=" + weight + "]";
}
}
src/main/java/com/actiontech/dble/config/loader/zkprocess/entity/schema/datahost/WriteHost.java
+12 -3
View File
@@ -31,9 +31,10 @@ public class WriteHost {
protected String password;
@XmlAttribute(required = true)
protected String user;
@XmlAttribute
protected String usingDecrypt;
@XmlAttribute
protected String id;
private List<ReadHost> readHost;
@@ -76,6 +77,15 @@ public class WriteHost {
public void setUsingDecrypt(String usingDecrypt) {
this.usingDecrypt = usingDecrypt;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public List<ReadHost> getReadHost() {
if (this.readHost == null) {
readHost = new ArrayList<>();
@@ -89,7 +99,7 @@ public class WriteHost {
@Override
public String toString() {
String builder = "WriteHost [host=" +
return "WriteHost [host=" +
host +
", url=" +
url +
@@ -102,7 +112,6 @@ public class WriteHost {
", readHost=" +
readHost +
"]";
return builder;
}
}
src/main/java/com/actiontech/dble/config/model/DBHostConfig.java
+9
View File
@@ -17,6 +17,7 @@ public class DBHostConfig {
private int maxCon;
private int minCon;
private int weight;
private String id;
private boolean fake = false;
public DBHostConfig(String hostName, String ip, int port, String url,
@@ -94,6 +95,14 @@ public class DBHostConfig {
this.fake = fake;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
@Override
public String toString() {
return "DBHostConfig [hostName=" + hostName + ", url=" + url + "]";
src/main/java/com/actiontech/dble/log/AlarmAppender.java
-125
View File
@@ -1,125 +0,0 @@
/*
* Copyright (C) 2016-2018 ActionTech.
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.log;
import com.actiontech.dble.DbleServer;
import com.actiontech.dble.DbleStartup;
import com.actiontech.dble.cluster.ClusterParamCfg;
import com.actiontech.dble.config.loader.ucoreprocess.ClusterUcoreSender;
import com.actiontech.dble.config.loader.ucoreprocess.UcoreConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.log.alarm.UcoreGrpc;
import com.actiontech.dble.log.alarm.UcoreInterface;
import io.grpc.Channel;
import io.grpc.ManagedChannelBuilder;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
import org.apache.logging.log4j.core.layout.PatternLayout;
import org.apache.logging.log4j.spi.StandardLevel;
import java.io.Serializable;
import java.util.concurrent.TimeUnit;
import static com.actiontech.dble.cluster.ClusterController.GENERAL_GRPC_TIMEOUT;
/**
* Created by szf on 2017/12/4.
*/
@Plugin(name = "AlarmAppender", category = "Core", elementType = "appender", printObject = true)
public final class AlarmAppender extends AbstractAppender {
private static String serverId = "";
private static String alertComponentId = "";
private static final String USHARD_CODE = "ushard";
private static UcoreGrpc.UcoreBlockingStub stub = null;
/**
* method to init the whole appender
*
* @param name name
* @param layout layout
*/
private AlarmAppender(String name,
Layout<? extends Serializable> layout) {
super(name, null, layout, true);
}
@Override
public void append(LogEvent event) {
if (stub == null && DbleStartup.isInitZKend()) {
//only if the dbleserver init config file finished than the config can be use for alert
try {
if (DbleServer.getInstance().isUseUcore()) {
serverId = UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_CFG_SERVER_ID);
alertComponentId = UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_CFG_MYID);
Channel channel = ManagedChannelBuilder.forAddress(UcoreConfig.getInstance().getIpList().get(0),
Integer.parseInt(UcoreConfig.getInstance().getValue(ClusterParamCfg.CLUSTER_PLUGINS_PORT))).
usePlaintext(true).build();
stub = UcoreGrpc.newBlockingStub(channel).withDeadlineAfter(GENERAL_GRPC_TIMEOUT, TimeUnit.SECONDS);
}
} catch (Exception e) {
//config not ready yet
return;
}
}
if (stub != null) {
send(event);
}
}
public void send(LogEvent event) {
String level;
int eventLevel = event.getLevel().intLevel();
if (eventLevel == StandardLevel.WARN.intLevel()) {
level = "WARN";
} else if (eventLevel == StandardLevel.ERROR.intLevel()) {
level = "CRITICAL";
} else if (eventLevel == StandardLevel.INFO.intLevel()) {
level = "NOTICE";
} else {
return;
}
String data = new String(getLayout().toByteArray(event));
String[] d = data.split(AlarmCode.ALARM_SPLIT);
if (d.length >= 2) {
UcoreInterface.AlertInput inpurt = UcoreInterface.AlertInput.newBuilder().
setCode(d[0]).
setDesc(d[1]).
setLevel(level).
setSourceComponentType(USHARD_CODE).
setSourceComponentId(alertComponentId).
setAlertComponentId(alertComponentId).
setAlertComponentType(USHARD_CODE).
setServerId(serverId).
setTimestampUnix(System.currentTimeMillis() * 1000000).
build();
try {
ClusterUcoreSender.alert(inpurt);
} catch (Exception e1) {
LOGGER.info("connect to ucore error ", e1);
}
}
}
@PluginFactory
public static AlarmAppender createAppender(@PluginAttribute("name") String name) {
if (name == null) {
return null;
}
Layout<? extends Serializable> layout = PatternLayout.createDefaultLayout();
return new AlarmAppender(name, layout);
}
}
src/main/java/com/actiontech/dble/meta/table/AbstractTableMetaHandler.java
+22 -4
View File
@@ -5,8 +5,11 @@
package com.actiontech.dble.meta.table;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.alarm.ToResolveContainer;
import com.actiontech.dble.config.model.TableConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.meta.protocol.StructureMeta;
import com.actiontech.dble.sqlengine.OneRawSQLQueryResultHandler;
import com.actiontech.dble.sqlengine.SQLJob;
@@ -75,15 +78,22 @@ public abstract class AbstractTableMetaHandler {
@Override
public void onResult(SQLQueryResult<Map<String, String>> result) {
String alertComponentId = "DataNode[" + dataNode + "]:Table[" + tableName + "]";
if (!result.isSuccess()) {
//not thread safe
LOGGER.warn(AlarmCode.TABLE_LACK + "Can't get table " + tableName + "'s config from DataNode:" + dataNode + "! Maybe the table is not initialized!");
String warnMsg = "Can't get table " + tableName + "'s config from DataNode:" + dataNode + "! Maybe the table is not initialized!";
LOGGER.warn(warnMsg);
AlertUtil.alertSelfWithTarget(AlarmCode.TABLE_LACK, Alert.AlertLevel.WARN, warnMsg, alertComponentId, null);
ToResolveContainer.TABLE_LACK.add(alertComponentId);
if (nodesNumber.decrementAndGet() == 0) {
StructureMeta.TableMeta tableMeta = genTableMeta();
handlerTable(tableMeta);
countdown();
}
return;
} else if (ToResolveContainer.TABLE_LACK.contains(alertComponentId) &&
AlertUtil.alertSelfWithTargetResolve(AlarmCode.TABLE_LACK, Alert.AlertLevel.WARN, alertComponentId, null)) {
ToResolveContainer.TABLE_LACK.remove(alertComponentId);
}
String currentSql = result.getResult().get(MYSQL_SHOW_CREATE_TABLE_COLS[1]);
if (dataNodeTableStructureSQLMap.containsKey(currentSql)) {
@@ -112,8 +122,12 @@ public abstract class AbstractTableMetaHandler {
tableMeta = initTableMeta(tableName, sql, version);
tableMetas.add(tableMeta);
}
String alertComponentId = schema + "." + tableName;
if (tableMetas.size() > 1) {
consistentWarning();
} else if (ToResolveContainer.TABLE_NOT_CONSISTENT_IN_DATAHOSTS.contains(alertComponentId) &&
AlertUtil.alertSelfWithTargetResolve(AlarmCode.TABLE_NOT_CONSISTENT_IN_DATAHOSTS, Alert.AlertLevel.WARN, alertComponentId, null)) {
ToResolveContainer.TABLE_NOT_CONSISTENT_IN_DATAHOSTS.remove(alertComponentId);
}
tableMetas.clear();
} else if (dataNodeTableStructureSQLMap.size() == 1) {
@@ -123,7 +137,10 @@ public abstract class AbstractTableMetaHandler {
}
private void consistentWarning() {
LOGGER.warn(AlarmCode.TABLE_NOT_CONSISTENT_IN_DATAHOSTS + "Table [" + tableName + "] structure are not consistent!");
String errorMsg = "Table [" + tableName + "] structure are not consistent in different data node!";
LOGGER.warn(errorMsg);
AlertUtil.alertSelfWithTarget(AlarmCode.TABLE_NOT_CONSISTENT_IN_DATAHOSTS, Alert.AlertLevel.WARN, errorMsg, schema + "." + tableName, null);
ToResolveContainer.TABLE_NOT_CONSISTENT_IN_DATAHOSTS.add(schema + "." + tableName);
LOGGER.info("Currently detected: ");
for (Map.Entry<String, List<String>> entry : dataNodeTableStructureSQLMap.entrySet()) {
StringBuilder stringBuilder = new StringBuilder();
@@ -142,7 +159,8 @@ public abstract class AbstractTableMetaHandler {
return MetaHelper.initTableMeta(table, createStatement, timeStamp);
} catch (Exception e) {
LOGGER.warn(AlarmCode.GET_TABLE_META_FAIL + "sql[" + sql + "] parser error:", e);
LOGGER.warn("sql[" + sql + "] parser error:", e);
AlertUtil.alertSelf(AlarmCode.GET_TABLE_META_FAIL, Alert.AlertLevel.WARN, "sql[" + sql + "] parser error:" + e.getMessage(), null);
return null;
}
}
src/main/java/com/actiontech/dble/meta/table/TableMetaCheckHandler.java
+12 -2
View File
@@ -5,8 +5,11 @@
package com.actiontech.dble.meta.table;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.alarm.ToResolveContainer;
import com.actiontech.dble.config.model.TableConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.meta.ProxyMetaManager;
import com.actiontech.dble.meta.protocol.StructureMeta;
@@ -28,8 +31,15 @@ public class TableMetaCheckHandler extends AbstractTableMetaHandler {
@Override
protected void handlerTable(StructureMeta.TableMeta tableMeta) {
if (tableMeta != null) {
String alertComponentId = schema + "." + tableMeta.getTableName();
if (isTableModify(schema, tableMeta)) {
LOGGER.warn(AlarmCode.TABLE_NOT_CONSISTENT_IN_MEMORY + "Table [" + tableMeta.getTableName() + "] are modified by other,Please Check IT!");
String errorMsg = "Table [" + tableMeta.getTableName() + "] are modified by other,Please Check IT!";
LOGGER.warn(errorMsg);
AlertUtil.alertSelfWithTarget(AlarmCode.TABLE_NOT_CONSISTENT_IN_MEMORY, Alert.AlertLevel.WARN, errorMsg, alertComponentId, null);
ToResolveContainer.TABLE_NOT_CONSISTENT_IN_MEMORY.add(alertComponentId);
} else if (ToResolveContainer.TABLE_NOT_CONSISTENT_IN_MEMORY.contains(alertComponentId) &&
AlertUtil.alertSelfWithTargetResolve(AlarmCode.TABLE_NOT_CONSISTENT_IN_MEMORY, Alert.AlertLevel.WARN, alertComponentId, null)) {
ToResolveContainer.TABLE_NOT_CONSISTENT_IN_MEMORY.remove(alertComponentId);
}
LOGGER.debug("checking table Table [" + tableMeta.getTableName() + "]");
}
src/main/java/com/actiontech/dble/net/NIOConnector.java
+5 -2
View File
@@ -6,7 +6,9 @@
package com.actiontech.dble.net;
import com.actiontech.dble.DbleServer;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +69,8 @@ public final class NIOConnector extends Thread implements SocketConnector {
keys.clear();
}
} catch (Exception e) {
LOGGER.warn(AlarmCode.NIOCONNECTOR_UNKNOWN_EXCEPTION + name, e);
LOGGER.warn(name, e);
AlertUtil.alertSelf(AlarmCode.NIOCONNECTOR_UNKNOWN_EXCEPTION, Alert.AlertLevel.WARN, name + e.getMessage(), null);
}
}
}
src/main/java/com/actiontech/dble/net/NIOReactor.java
+7 -3
View File
@@ -6,7 +6,9 @@
package com.actiontech.dble.net;
import com.actiontech.dble.DbleServer;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.statistic.stat.ThreadWorkUsage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -135,7 +137,8 @@ public final class NIOReactor {
LOGGER.debug(con + " socket key canceled");
}
} catch (Exception e) {
LOGGER.warn(AlarmCode.NIOREACTOR_UNKNOWN_EXCEPTION + con, e);
LOGGER.warn("caught err:" + con, e);
AlertUtil.alertSelf(AlarmCode.NIOREACTOR_UNKNOWN_EXCEPTION, Alert.AlertLevel.WARN, "caught err:" + con + e.getMessage(), null);
} catch (final Throwable e) {
// Catch exceptions such as OOM and close connection if exists
//so that the reactor can keep running!
@@ -144,7 +147,8 @@ public final class NIOReactor {
if (con != null) {
con.close("Bad: " + e);
}
LOGGER.warn(AlarmCode.NIOREACTOR_UNKNOWN_THROWABLE + "caught err: ", e);
LOGGER.warn("caught err: ", e);
AlertUtil.alertSelf(AlarmCode.NIOREACTOR_UNKNOWN_THROWABLE, Alert.AlertLevel.WARN, "caught err:" + e.getMessage(), null);
}
}
}
src/main/java/com/actiontech/dble/server/NonBlockingSession.java
+5 -2
View File
@@ -23,7 +23,9 @@ import com.actiontech.dble.backend.mysql.xa.TxState;
import com.actiontech.dble.btrace.provider.CostTimeProvider;
import com.actiontech.dble.config.ErrorCode;
import com.actiontech.dble.config.ServerConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.net.handler.FrontendCommandHandler;
import com.actiontech.dble.net.mysql.EOFPacket;
import com.actiontech.dble.net.mysql.MySQLPacket;
@@ -711,7 +713,8 @@ public class NonBlockingSession implements Session {
dn.getConnectionFromSameSource(en.getValue().getSchema(), true, en.getValue(),
kill, en.getKey());
} catch (Exception e) {
LOGGER.info(AlarmCode.KILL_BACKEND_CONN_FAIL + "get killer connection failed for " + en.getKey(), e);
LOGGER.info("get killer connection failed for " + en.getKey(), e);
AlertUtil.alertSelf(AlarmCode.KILL_BACKEND_CONN_FAIL, Alert.AlertLevel.NOTICE, "get killer connection " + en.getKey() + " failed:" + e.getMessage(), null);
kill.connectionError(e, null);
}
}
src/main/java/com/actiontech/dble/server/util/GlobalTableUtil.java
+24 -9
View File
@@ -6,6 +6,7 @@
package com.actiontech.dble.server.util;
import com.actiontech.dble.DbleServer;
import com.actiontech.dble.alarm.ToResolveContainer;
import com.actiontech.dble.backend.datasource.PhysicalDBNode;
import com.actiontech.dble.backend.datasource.PhysicalDBPool;
import com.actiontech.dble.backend.datasource.PhysicalDatasource;
@@ -15,7 +16,9 @@ import com.actiontech.dble.config.ServerConfig;
import com.actiontech.dble.config.model.SchemaConfig;
import com.actiontech.dble.config.model.SystemConfig;
import com.actiontech.dble.config.model.TableConfig;
import com.actiontech.dble.log.alarm.AlarmCode;
import com.actiontech.dble.alarm.AlarmCode;
import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.meta.protocol.StructureMeta;
import com.actiontech.dble.sqlengine.SQLQueryResult;
import com.actiontech.dble.util.StringUtil;
@@ -72,15 +75,22 @@ public final class GlobalTableUtil {
}
public static boolean isInnerColExist(SchemaUtil.SchemaInfo schemaInfo, StructureMeta.TableMeta orgTbMeta) {
String alertComponentId = schemaInfo.getSchema() + "." + schemaInfo.getTable();
for (int i = 0; i < orgTbMeta.getColumnsList().size(); i++) {
String column = orgTbMeta.getColumnsList().get(i).getName();
if (column.equalsIgnoreCase(GLOBAL_TABLE_CHECK_COLUMN))
if (column.equalsIgnoreCase(GLOBAL_TABLE_CHECK_COLUMN)) {
if (ToResolveContainer.GLOBAL_TABLE_COLUMN_LOST.contains(alertComponentId) &&
AlertUtil.alertSelfWithTargetResolve(AlarmCode.GLOBAL_TABLE_COLUMN_LOST, Alert.AlertLevel.WARN, alertComponentId, null)) {
ToResolveContainer.GLOBAL_TABLE_COLUMN_LOST.remove(alertComponentId);
}
return true;
}
}
String warnStr = AlarmCode.GLOBAL_TABLE_COLUMN_LOST + schemaInfo.getSchema() + "." + schemaInfo.getTable() +
" inner column: " + GLOBAL_TABLE_CHECK_COLUMN + " is not exist.";
String warnStr = alertComponentId + " inner column: " + GLOBAL_TABLE_CHECK_COLUMN + " is not exist.";
LOGGER.warn(warnStr);
return false; // tableName witout inner column
AlertUtil.alertSelfWithTarget(AlarmCode.GLOBAL_TABLE_COLUMN_LOST, Alert.AlertLevel.WARN, warnStr, alertComponentId, null);
ToResolveContainer.GLOBAL_TABLE_COLUMN_LOST.add(alertComponentId);
return false; // tableName without inner column
}
private static void getGlobalTable() {
@@ -201,11 +211,16 @@ public final class GlobalTableUtil {
} catch (Exception e) {
LOGGER.info(row.get(GlobalTableUtil.INNER_COLUMN) + ", " + e.getMessage());
} finally {
if (columnsList == null ||
!columnsList.contains(GlobalTableUtil.GLOBAL_TABLE_CHECK_COLUMN)) {
LOGGER.warn(AlarmCode.GLOBAL_TABLE_COLUMN_LOST + map.getDataNode() + "." + map.getTableName() +
" inner column: " + GlobalTableUtil.GLOBAL_TABLE_CHECK_COLUMN + " is not exist.");
String alertComponentId = map.getDataNode() + "." + map.getTableName();
if (columnsList == null || !columnsList.contains(GlobalTableUtil.GLOBAL_TABLE_CHECK_COLUMN)) {
String warnMsg = alertComponentId + " inner column: " + GlobalTableUtil.GLOBAL_TABLE_CHECK_COLUMN + " is not exist.";
LOGGER.warn(warnMsg);
AlertUtil.alertSelfWithTarget(AlarmCode.GLOBAL_TABLE_COLUMN_LOST, Alert.AlertLevel.WARN, warnMsg, alertComponentId, null);
} else {
if (ToResolveContainer.GLOBAL_TABLE_COLUMN_LOST.contains(alertComponentId) &&
AlertUtil.alertSelfWithTargetResolve(AlarmCode.GLOBAL_TABLE_COLUMN_LOST, Alert.AlertLevel.WARN, alertComponentId, null)) {
ToResolveContainer.GLOBAL_TABLE_COLUMN_LOST.remove(alertComponentId);
}
LOGGER.debug("columnsList: " + columnsList);
}
}
src/main/resources/log4j2.xml
+1 -3
View File
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" packages="com.actiontech.dble.log">
<Configuration status="WARN">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d [%-5p][%t] %m %throwable{full} (%C:%F:%L) %n"/>
@@ -28,14 +28,12 @@
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
<AlarmAppender name="grpcAppender" />
</Appenders>
<Loggers>
<asyncRoot level="debug" includeLocation="true">
<AppenderRef ref="Console"/>
<AppenderRef ref="RollingFile"/>
<AppenderRef ref="grpcAppender"/>
</asyncRoot>
</Loggers>
</Configuration>
src/main/resources/schema.dtd
+4 -2
View File
@@ -51,8 +51,8 @@
maxCon CDATA #REQUIRED
minCon CDATA #REQUIRED
name NMTOKEN #REQUIRED
switchType CDATA #IMPLIED
slaveThreshold CDATA #IMPLIED
switchType CDATA #IMPLIED
slaveThreshold CDATA #IMPLIED
tempReadHostAvailable CDATA #IMPLIED
keepOrig CDATA #IMPLIED>
@@ -62,6 +62,7 @@
host NMTOKEN #REQUIRED
password CDATA #REQUIRED
user CDATA #REQUIRED
id CDATA #IMPLIED
usingDecrypt CDATA #IMPLIED>
<!ELEMENT heartbeat (#PCDATA)>
@@ -73,6 +74,7 @@
password CDATA #REQUIRED
user NMTOKEN #REQUIRED
weight CDATA #IMPLIED
id CDATA #IMPLIED
usingDecrypt CDATA #IMPLIED>