mirror/dble
1
0
Fork 0
mirror of https://github.com/actiontech/dble.git synced 2026-01-06 04:40:17 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix: use rw-split, mysql version and dble fakeMySQLVersion keep the same range

Browse Source
This commit is contained in:
guoaomen
2023-03-22 13:43:05 +08:00
parent b6a8f0e658
commit 80c5753d08
6 changed files with 195 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/com/actiontech/dble/backend/heartbeat/MySQLDetector.java
View File

@@ -11,10 +11,10 @@ import com.actiontech.dble.alarm.Alert;
import com.actiontech.dble.alarm.AlertUtil;
import com.actiontech.dble.alarm.ToResolveContainer;
import com.actiontech.dble.backend.datasource.PhysicalDbInstance;
import com.actiontech.dble.backend.mysql.VersionUtil;
import com.actiontech.dble.config.helper.GetAndSyncDbInstanceKeyVariables;
import com.actiontech.dble.config.helper.KeyVariables;
import com.actiontech.dble.config.model.SystemConfig;
import com.actiontech.dble.config.util.ConfigUtil;
import com.actiontech.dble.sqlengine.OneRawSQLQueryResultHandler;
import com.actiontech.dble.sqlengine.SQLQueryResult;
import com.actiontech.dble.sqlengine.SQLQueryResultListener;
@@ -140,10 +140,11 @@ public class MySQLDetector implements SQLQueryResultListener<SQLQueryResult<Map<
}
GetAndSyncDbInstanceKeyVariables task = new GetAndSyncDbInstanceKeyVariables(source, true);
KeyVariables variables = task.call();
boolean versionMismatch = false;
if (variables == null ||
variables.isLowerCase() != DbleServer.getInstance().getSystemVariables().isLowerCaseTableNames() ||
variables.getMaxPacketSize() < SystemConfig.getInstance().getMaxPacketSize() ||
VersionUtil.getMajorVersion(variables.getVersion()) < VersionUtil.getMajorVersion(SystemConfig.getInstance().getFakeMySQLVersion())) {
(versionMismatch = !ConfigUtil.checkMysqlVersion(variables.getVersion(), source, false))) {
String url = heartbeat.getSource().getConfig().getUrl();
Map<String, String> labels = AlertUtil.genSingleLabel("dbInstance", url);
String errMsg;
@@ -151,7 +152,7 @@ public class MySQLDetector implements SQLQueryResultListener<SQLQueryResult<Map<
errMsg = "GetAndSyncDbInstanceKeyVariables failed";
} else if (variables.isLowerCase() != DbleServer.getInstance().getSystemVariables().isLowerCaseTableNames()) {
errMsg = "this dbInstance[=" + url + "]'s lower_case is wrong";
} else if (VersionUtil.getMajorVersion(variables.getVersion()) < VersionUtil.getMajorVersion(SystemConfig.getInstance().getFakeMySQLVersion())) {
} else if (versionMismatch) {
errMsg = "this dbInstance[=" + url + "]'s version[=" + variables.getVersion() + "] cannot be lower than the dble version[=" + SystemConfig.getInstance().getFakeMySQLVersion() + "]";
} else {
errMsg = "this dbInstance[=" + url + "]'s max_allowed_packet is " + variables.getMaxPacketSize() + ", but dble's is " + SystemConfig.getInstance().getMaxPacketSize();

87
src/main/java/com/actiontech/dble/backend/mysql/VersionUtil.java
View File

@@ -5,9 +5,15 @@
package com.actiontech.dble.backend.mysql;
import com.actiontech.dble.config.model.MysqlVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
public final class VersionUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(VersionUtil.class);
private static final String VERSION_8 = "8.";
private static final String VERSION_5 = "5.";
@@ -23,12 +29,11 @@ public final class VersionUtil {
if (version == null) {
return null;
} else {
final int versionNumber = getMajorVersion(version);
if (versionNumber == 8) {
return TRANSACTION_ISOLATION;
} else {
return TX_ISOLATION;
}
MysqlVersion mysqlVersion = VersionUtil.parseVersion(version);
//refer to:com.mysql.jdbc.ConnectionImpl.getTransactionIsolation
boolean isMatch = (VersionUtil.versionMeetsMinimum(5, 7, 20, mysqlVersion) && !VersionUtil.versionMeetsMinimum(8, 0, 0, mysqlVersion)) ||
VersionUtil.versionMeetsMinimum(8, 0, 3, mysqlVersion);
return isMatch ? TRANSACTION_ISOLATION : TX_ISOLATION;
}
}
@@ -60,4 +65,74 @@ public final class VersionUtil {
return versionNumber == 8;
}
/**
* Does the version of the MySQL server we are connected to meet the given
* minimums?
*/
public static boolean versionMeetsMinimum(int major, int minor, int subminor, MysqlVersion mysqlVersion) {
if (mysqlVersion.getServerMajorVersion() >= major) {
if (mysqlVersion.getServerMajorVersion() == major) {
if (mysqlVersion.getServerMinorVersion() >= minor) {
if (mysqlVersion.getServerMinorVersion() == minor) {
return mysqlVersion.getServerSubMinorVersion() >= subminor;
}
// newer than major.minor
return true;
}
// older than major.minor
return false;
}
// newer than major
return true;
}
return false;
}
public static MysqlVersion parseVersion(String version) {
// Parse the server version into major/minor/subminor
int point = version.indexOf('.');
MysqlVersion mysqlVersion = new MysqlVersion();
if (point != -1) {
try {
mysqlVersion.setServerMajorVersion(Integer.parseInt(version.substring(0, point)));
} catch (NumberFormatException e) {
// ignore
LOGGER.warn("version[{}] format is wrong", version);
}
String remaining = version.substring(point + 1);
point = remaining.indexOf('.');
if (point != -1) {
try {
mysqlVersion.setServerMinorVersion(Integer.parseInt(remaining.substring(0, point)));
} catch (NumberFormatException e) {
// ignore
LOGGER.warn("version[{}] format is wrong", version);
}
remaining = remaining.substring(point + 1);
int pos = 0;
while (pos < remaining.length()) {
if ((remaining.charAt(pos) < '0') || (remaining.charAt(pos) > '9')) {
break;
}
pos++;
}
try {
mysqlVersion.setServerSubMinorVersion(Integer.parseInt(remaining.substring(0, pos)));
} catch (NumberFormatException e) {
// ignore
LOGGER.warn("version[{}] format is wrong", version);
}
}
}
return mysqlVersion;
}
}

34
src/main/java/com/actiontech/dble/config/model/MysqlVersion.java Normal file
View File

@@ -0,0 +1,34 @@
package com.actiontech.dble.config.model;
public class MysqlVersion {
private int serverMajorVersion = 0;
private int serverMinorVersion = 0;
private int serverSubMinorVersion = 0;
public MysqlVersion() {
}
public int getServerMajorVersion() {
return serverMajorVersion;
}
public void setServerMajorVersion(int serverMajorVersion) {
this.serverMajorVersion = serverMajorVersion;
}
public int getServerMinorVersion() {
return serverMinorVersion;
}
public void setServerMinorVersion(int serverMinorVersion) {
this.serverMinorVersion = serverMinorVersion;
}
public int getServerSubMinorVersion() {
return serverSubMinorVersion;
}
public void setServerSubMinorVersion(int serverSubMinorVersion) {
this.serverSubMinorVersion = serverSubMinorVersion;
}
}

12
src/main/java/com/actiontech/dble/config/model/SystemConfig.java
View File

@@ -6,6 +6,7 @@
package com.actiontech.dble.config.model;
import com.actiontech.dble.backend.mysql.CharsetUtil;
import com.actiontech.dble.backend.mysql.VersionUtil;
import com.actiontech.dble.config.Isolations;
import com.actiontech.dble.config.ProblemReporter;
import com.actiontech.dble.config.converter.DBConverter;
@@ -209,6 +210,7 @@ public final class SystemConfig {
private String traceSamplerParam = null;
private String fakeMySQLVersion = "5.7.21";
private MysqlVersion mysqlVersion;
private int enableRoutePenetration = 0;
private String routePenetrationRules = "";
@@ -1671,8 +1673,14 @@ public final class SystemConfig {
}
@SuppressWarnings("unused")
public void setFakeMySQLVersion(String mysqlVersion) {
this.fakeMySQLVersion = mysqlVersion;
public void setFakeMySQLVersion(String version) {
this.fakeMySQLVersion = version;
if (this.fakeMySQLVersion == null) return;
this.mysqlVersion = VersionUtil.parseVersion(this.fakeMySQLVersion);
}
public MysqlVersion getMysqlVersion() {
return mysqlVersion;
}
public String getTraceEndPoint() {

87
src/main/java/com/actiontech/dble/config/util/ConfigUtil.java
View File

@@ -13,12 +13,14 @@ import com.actiontech.dble.backend.mysql.VersionUtil;
import com.actiontech.dble.config.DbleTempConfig;
import com.actiontech.dble.config.helper.GetAndSyncDbInstanceKeyVariables;
import com.actiontech.dble.config.helper.KeyVariables;
import com.actiontech.dble.config.model.MysqlVersion;
import com.actiontech.dble.config.model.SystemConfig;
import com.actiontech.dble.config.model.db.type.DataBaseType;
import com.actiontech.dble.services.manager.response.ChangeItem;
import com.actiontech.dble.services.manager.response.ChangeItemType;
import com.actiontech.dble.services.manager.response.ChangeType;
import com.actiontech.dble.singleton.TraceManager;
import com.actiontech.dble.util.StringUtil;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.util.Strings;
@@ -106,7 +108,6 @@ public final class ConfigUtil {
Set<String> diffGroup = new HashSet<>();
int minNodePacketSize = Integer.MAX_VALUE;
int minVersion = Integer.parseInt(SystemConfig.getInstance().getFakeMySQLVersion().substring(0, 1));
Boolean lowerCase = DbleServer.getInstance().getConfig().isLowerCase();
for (Map.Entry<VariableMapKey, Future<KeyVariables>> entry : keyVariablesTaskMap.entrySet()) {
VariableMapKey variableMapKey = entry.getKey();
@@ -119,8 +120,10 @@ public final class ConfigUtil {
diffGroup.add(variableMapKey.getDataSourceName());
}
minNodePacketSize = Math.min(minNodePacketSize, keyVariables.getMaxPacketSize());
int version = Integer.parseInt(keyVariables.getVersion().substring(0, 1));
minVersion = Math.min(minVersion, version);
PhysicalDbInstance instance = variableMapKey.getDbInstance();
//check mysql version
checkMysqlVersion(keyVariables.getVersion(), instance, true);
}
}
if (minNodePacketSize < SystemConfig.getInstance().getMaxPacketSize() + KeyVariables.MARGIN_PACKET_SIZE) {
@@ -128,9 +131,7 @@ public final class ConfigUtil {
msg = "dble's maxPacketSize will be set to (the min of all dbGroup's max_allowed_packet) - " + KeyVariables.MARGIN_PACKET_SIZE + ":" + (minNodePacketSize - KeyVariables.MARGIN_PACKET_SIZE);
LOGGER.warn(msg);
}
if (minVersion < Integer.parseInt(SystemConfig.getInstance().getFakeMySQLVersion().substring(0, 1))) {
throw new ConfigException("the dble version[=" + SystemConfig.getInstance().getFakeMySQLVersion() + "] cannot be higher than the minimum version of the backend mysql node,pls check the backend mysql node.");
}
if (diffGroup.size() != 0) {
// if all datasoure's lower case are not equal, throw exception
StringBuilder sb = new StringBuilder("The values of lower_case_table_names for backend MySQLs are different.");
@@ -192,7 +193,6 @@ public final class ConfigUtil {
Set<String> firstGroup = new HashSet<>();
Set<String> secondGroup = new HashSet<>();
int minNodePacketSize = Integer.MAX_VALUE;
int minVersion = VersionUtil.getMajorVersion(SystemConfig.getInstance().getFakeMySQLVersion());
for (Map.Entry<VariableMapKey, Future<KeyVariables>> entry : keyVariablesTaskMap.entrySet()) {
VariableMapKey variableMapKey = entry.getKey();
Future<KeyVariables> future = entry.getValue();
@@ -206,13 +206,10 @@ public final class ConfigUtil {
secondGroup.add(variableMapKey.getDataSourceName());
}
minNodePacketSize = Math.min(minNodePacketSize, keyVariables.getMaxPacketSize());
Integer majorVersion = VersionUtil.getMajorVersionWithoutDefaultValue(keyVariables.getVersion());
if (majorVersion == null) {
LOGGER.warn("the backend mysql server version [{}] is unrecognized, we will treat as default official mysql version 5.*. ", keyVariables.getVersion());
majorVersion = 5;
}
minVersion = Math.min(minVersion, majorVersion);
PhysicalDbInstance instance = variableMapKey.getDbInstance();
//check mysql version
checkMysqlVersion(keyVariables.getVersion(), instance, true);
// The back_log value indicates how many requests can be stacked during this short time before MySQL momentarily stops answering new requests
int minCon = instance.getConfig().getMinCon();
int backLog = keyVariables.getBackLog();
@@ -228,9 +225,6 @@ public final class ConfigUtil {
list.add(msg);
LOGGER.warn(msg);
}
if (minVersion < VersionUtil.getMajorVersion(SystemConfig.getInstance().getFakeMySQLVersion())) {
throw new ConfigException("the dble version[=" + SystemConfig.getInstance().getFakeMySQLVersion() + "] cannot be higher than the minimum version of the backend mysql node,pls check the backend mysql node.");
}
if (secondGroup.size() != 0) {
// if all datasoure's lower case are not equal, throw exception
StringBuilder sb = new StringBuilder("The values of lower_case_table_names for backend MySQLs are different.");
@@ -257,6 +251,53 @@ public final class ConfigUtil {
return list;
}
public static boolean checkMysqlVersion(String version, PhysicalDbInstance instance, boolean isThrowException) {
String type = instance.getDbGroupConfig().instanceDatabaseType().toString();
Integer majorVersion = VersionUtil.getMajorVersionWithoutDefaultValue(version);
if (majorVersion == null) {
LOGGER.warn("the backend {} server version [{}] is unrecognized, we will treat as default official {} version 5.*. ", type, type, version);
majorVersion = 5;
}
if (!instance.getDbGroup().isRwSplitUseless()) {
//rw-split
return checkVersionWithRwSplit(version, instance, isThrowException, type);
} else {
boolean isMatch = majorVersion >= VersionUtil.getMajorVersion(SystemConfig.getInstance().getFakeMySQLVersion());
if (!isMatch && isThrowException) {
throw new ConfigException("the dble version[=" + SystemConfig.getInstance().getFakeMySQLVersion() + "] cannot be higher than the minimum version of the backend " + type + " node,pls check the backend " + type + " node.");
}
return isMatch;
}
}
/**
* check dble-mysql version<p>
* 1.transaction_isolation/transaction_read_only(com.mysql.jdbc.ConnectionImpl.getTransactionIsolation): 5.7.20 <= version <= 8.0.0 || version >= 8.0.3 <p>
* 2.query_cache(com.mysql.jdbc.ConnectionImpl.loadServerVariables): version < 8.0.3 <p>
* dble and mysql versions meet the above requirements, such as:<p>
* ✔: dble-version：5.7.20 mysql-version：8.0.0<p>
* ✘: dble-version：5.7.20 mysql-version：8.0.3<p>
* ✔: dble-version：8.0.3 mysql-version：8.0.23<p>
* ✘: dble-version：8.0.3 mysql-version：8.0.1<p>
* ✔: dble-version：5.7.15 mysql-version：8.0.1<p>
* ✘: dble-version：5.7.15 mysql-version：5.7.25
*/
private static boolean checkVersionWithRwSplit(String version, PhysicalDbInstance instance, boolean isThrowException, String type) {
if (StringUtil.isBlank(version)) return false;
LOGGER.debug("check version: dble-version[{}], mysql-version[{}]", SystemConfig.getInstance().getFakeMySQLVersion(), version);
MysqlVersion mysqlVersion = VersionUtil.parseVersion(version);
MysqlVersion dbleVersion = SystemConfig.getInstance().getMysqlVersion();
boolean mysqlFlag0 = VersionUtil.versionMeetsMinimum(5, 7, 20, mysqlVersion) && !VersionUtil.versionMeetsMinimum(8, 0, 0, mysqlVersion);
boolean mysqlFlag1 = VersionUtil.versionMeetsMinimum(8, 0, 3, mysqlVersion);
boolean dbleFlag0 = VersionUtil.versionMeetsMinimum(5, 7, 20, dbleVersion) && !VersionUtil.versionMeetsMinimum(8, 0, 0, dbleVersion);
boolean dbleFlag1 = VersionUtil.versionMeetsMinimum(8, 0, 3, dbleVersion);
boolean isMatch = mysqlFlag0 == dbleFlag0 && mysqlFlag1 == dbleFlag1;
if (!isMatch && isThrowException) {
throw new ConfigException("the dble version[=" + SystemConfig.getInstance().getFakeMySQLVersion() + "] and " + type + "[" + instance.getConfig().getUrl() + "] version[=" + version + "] not match, Please check the version.");
}
return isMatch;
}
@Nullable
private static List<String> getClickHouseSyncKeyVariables(Map<String, PhysicalDbGroup> dbGroups, boolean needSync) throws InterruptedException, ExecutionException, IOException {
String msg = null;
@@ -274,7 +315,6 @@ public final class ConfigUtil {
Set<String> firstGroup = new HashSet<>();
Set<String> secondGroup = new HashSet<>();
int minNodePacketSize = Integer.MAX_VALUE;
int minVersion = VersionUtil.getMajorVersion(SystemConfig.getInstance().getFakeMySQLVersion());
for (Map.Entry<VariableMapKey, Future<KeyVariables>> entry : keyVariablesTaskMap.entrySet()) {
VariableMapKey variableMapKey = entry.getKey();
Future<KeyVariables> future = entry.getValue();
@@ -288,12 +328,10 @@ public final class ConfigUtil {
secondGroup.add(variableMapKey.getDataSourceName());
}
minNodePacketSize = Math.min(minNodePacketSize, keyVariables.getMaxPacketSize());
Integer majorVersion = VersionUtil.getMajorVersionWithoutDefaultValue(keyVariables.getVersion());
if (majorVersion == null) {
LOGGER.warn("the backend clickhouse server version [{}] is unrecognized, we will treat as default official clickhouse version 5.*. ", keyVariables.getVersion());
majorVersion = 5;
}
minVersion = Math.min(minVersion, majorVersion);
PhysicalDbInstance instance = variableMapKey.getDbInstance();
//check mysql version
checkMysqlVersion(keyVariables.getVersion(), instance, true);
}
}
if (minNodePacketSize < SystemConfig.getInstance().getMaxPacketSize() + KeyVariables.MARGIN_PACKET_SIZE) {
@@ -302,9 +340,6 @@ public final class ConfigUtil {
list.add(msg);
LOGGER.warn(msg);
}
if (minVersion < VersionUtil.getMajorVersion(SystemConfig.getInstance().getFakeMySQLVersion())) {
throw new ConfigException("the dble version[=" + SystemConfig.getInstance().getFakeMySQLVersion() + "] cannot be higher than the minimum version of the backend clickHouse node,pls check the backend clickHouse node.");
}
if (secondGroup.size() != 0) {
// if all datasoure's lower case are not equal, throw exception
StringBuilder sb = new StringBuilder("The values of lower_case_table_names for backend clickHouse are different.");

27
src/main/java/com/actiontech/dble/services/rwsplit/handle/RwSplitSelectHandler.java
View File

@@ -6,36 +6,19 @@
package com.actiontech.dble.services.rwsplit.handle;
import com.actiontech.dble.server.parser.RwSplitServerParseSelect;
import com.actiontech.dble.server.response.SelectVariables;
import com.actiontech.dble.services.rwsplit.RWSplitService;
/**
* @author mycat
*/
public final class RwSplitSelectHandler {
private RwSplitSelectHandler() {
}
public static void handle(String stmt, RWSplitService service, int offset) {
switch (RwSplitServerParseSelect.parse(stmt, offset)) {
case RwSplitServerParseSelect.SELECT_VAR_ALL:
SelectVariables.execute(service, stmt);
break;
default: {
int rs2 = RwSplitServerParseSelect.parseSpecial(stmt);
switch (rs2) {
case RwSplitServerParseSelect.LOCK_READ:
service.getSession2().execute(true, null, false, true);
break;
default:
service.getSession2().execute(null, null, false, true);
break;
}
break;
}
int rs2 = RwSplitServerParseSelect.parseSpecial(stmt);
if (rs2 == RwSplitServerParseSelect.LOCK_READ) {
service.getSession2().execute(true, null, false, true);
} else {
service.getSession2().execute(null, null, false, true);
}
}