local read support inner 1752 (#3263)

* local read support inner 1752

fix

* fix

fix
This commit is contained in:
ylinzhu
2022-05-30 20:34:51 +08:00
committed by GitHub
parent dbba9488ab
commit 7312090801
13 changed files with 317 additions and 29 deletions
@@ -0,0 +1,85 @@
/*
* Copyright (C) 2016-2022 ActionTech.
* License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
*/
package com.actiontech.dble.backend.datasource;
import com.actiontech.dble.config.model.SystemConfig;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
public class LocalReadLoadBalancer extends AbstractLoadBalancer {
private final ThreadLocalRandom random = ThreadLocalRandom.current();
public LocalReadLoadBalancer() {
}
@Override
protected PhysicalDbInstance doSelect(List<PhysicalDbInstance> okSources) {
List<PhysicalDbInstance> matchSources = matchSources(okSources);
if (matchSources.isEmpty()) {
matchSources = okSources;
}
int length = matchSources.size();
int totalWeight = 0;
boolean sameWeight = true;
for (int i = 0; i < length; i++) {
int readWeight = matchSources.get(i).getConfig().getReadWeight();
totalWeight += readWeight;
if (sameWeight && i > 0 && readWeight != matchSources.get(i - 1).getConfig().getReadWeight()) {
sameWeight = false;
}
}
if (totalWeight > 0 && !sameWeight) {
// random by different weight
int offset = random.nextInt(totalWeight);
for (PhysicalDbInstance okSource : matchSources) {
offset -= okSource.getConfig().getReadWeight();
if (offset < 0) {
return okSource;
}
}
}
return matchSources.get(random.nextInt(length));
}
private List<PhysicalDbInstance> matchSources(List<PhysicalDbInstance> okSources) {
String district = SystemConfig.getInstance().getDistrict();
String dataCenter = SystemConfig.getInstance().getDataCenter();
if (Strings.isNullOrEmpty(district)) {
return okSources;
}
List<PhysicalDbInstance> firstSources = Lists.newArrayList();
List<PhysicalDbInstance> secondSources = Lists.newArrayList();
List<PhysicalDbInstance> otherSources;
otherSources = okSources.stream().filter(okSource -> okSource.isAlive()).collect(Collectors.toList());
for (PhysicalDbInstance okSource : otherSources) {
String dbDistrict = okSource.getConfig().getDbDistrict();
String dbDataCenter = okSource.getConfig().getDbDataCenter();
if (StringUtils.equals(district, dbDistrict)) {
secondSources.add(okSource);
if (StringUtils.equals(dataCenter, dbDataCenter)) {
firstSources.add(okSource);
}
}
}
if (!firstSources.isEmpty()) {
return firstSources;
}
if (!secondSources.isEmpty()) {
return secondSources;
}
return otherSources;
}
}
@@ -58,6 +58,7 @@ public class PhysicalDbGroup {
private final int rwSplitMode;
protected String[] schemas;
private final LoadBalancer loadBalancer = new RandomLoadBalancer();
private final LocalReadLoadBalancer localReadLoadBalancer = new LocalReadLoadBalancer();
private final ReentrantReadWriteLock adjustLock = new ReentrantReadWriteLock();
private boolean shardingUseless = true;
@@ -293,15 +294,28 @@ public class PhysicalDbGroup {
* rwsplit user
*
* @param master
* @param write
* @param writeStatistical
* @return
* @throws IOException
*/
public PhysicalDbInstance rwSelect(Boolean master, Boolean write) throws IOException {
if (Objects.nonNull(write)) {
return select(master, false, write);
public PhysicalDbInstance rwSelect(Boolean master, Boolean writeStatistical) throws IOException {
return rwSelect(master, writeStatistical, false);
}
/**
* rwsplit user
*
* @param master
* @param writeStatistical
* @param localRead only the SELECT and show statements attempt to localRead
* @return
* @throws IOException
*/
public PhysicalDbInstance rwSelect(Boolean master, Boolean writeStatistical, boolean localRead) throws IOException {
if (Objects.nonNull(writeStatistical)) {
return select(master, false, writeStatistical, localRead);
}
return select(master, false, Objects.nonNull(master) ? master : false);
return select(master, false, Objects.nonNull(master) ? master : false, localRead);
}
/**
@@ -319,7 +333,22 @@ public class PhysicalDbGroup {
return select(master, isForUpdate, false);
}
public PhysicalDbInstance select(Boolean master, boolean isForUpdate, boolean write) throws IOException {
public PhysicalDbInstance select(Boolean master, boolean isForUpdate, boolean writeStatistical) throws IOException {
return select(master, isForUpdate, writeStatistical, false);
}
/**
* Select an instance
*
* @param master
* @param isForUpdate
* @param writeStatistical
* @param localRead only the SELECT and show statements attempt to localRead
* @return
* @throws IOException
*/
public PhysicalDbInstance select(Boolean master, boolean isForUpdate, boolean writeStatistical, boolean localRead) throws IOException {
if (rwSplitMode == RW_SPLIT_OFF && (master != null && !master)) {
LOGGER.warn("force slave,but the dbGroup[{}] doesn't contains active slave dbInstance", groupName);
throw new IOException("force slave,but the dbGroup[" + groupName + "] doesn't contain active slave dbInstance");
@@ -330,7 +359,7 @@ public class PhysicalDbGroup {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("select write {}", writeDbInstance);
}
if (write) {
if (writeStatistical) {
writeDbInstance.incrementWriteCount();
} else {
writeDbInstance.incrementReadCount();
@@ -345,6 +374,20 @@ public class PhysicalDbGroup {
if (instances.size() == 0) {
throw new IOException("the dbGroup[" + groupName + "] doesn't contain active dbInstance.");
}
if (localRead) {
PhysicalDbInstance selectInstance = localReadLoadBalancer.select(instances);
selectInstance.incrementReadCount();
if (selectInstance.isAlive()) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("select {}", selectInstance);
}
return selectInstance;
} else {
reportError(selectInstance);
return selectInstance;
}
}
PhysicalDbInstance selectInstance = loadBalancer.select(instances);
selectInstance.incrementReadCount();
if (selectInstance.isAlive()) {
@@ -7,6 +7,7 @@ package com.actiontech.dble.cluster.zkprocess.entity.dbGroups;
import com.actiontech.dble.cluster.zkprocess.entity.Propertied;
import com.actiontech.dble.cluster.zkprocess.entity.Property;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
@@ -54,6 +55,12 @@ public class DBInstance implements Propertied {
@XmlAttribute
protected String databaseType;
@XmlAttribute
protected String dbDistrict;
@XmlAttribute
protected String dbDataCenter;
protected List<Property> property;
protected transient String dbGroup;
@@ -62,7 +69,7 @@ public class DBInstance implements Propertied {
}
public DBInstance(String name, String url, String password, String user, Integer maxCon, Integer minCon, String disabled, String id, String readWeight,
Boolean primary, List<Property> property, String usingDecrypt, String databaseType) {
Boolean primary, List<Property> property, String usingDecrypt, String databaseType, String dbDistrict, String dbDataCenter) {
this.name = name;
this.url = url;
this.password = password;
@@ -76,6 +83,8 @@ public class DBInstance implements Propertied {
this.property = property;
this.usingDecrypt = usingDecrypt;
this.databaseType = databaseType;
this.dbDistrict = dbDistrict;
this.dbDataCenter = dbDataCenter;
}
@Override
@@ -202,6 +211,23 @@ public class DBInstance implements Propertied {
this.databaseType = databaseType;
}
public String getDbDistrict() {
return dbDistrict;
}
public void setDbDistrict(String dbDistrict) {
this.dbDistrict = dbDistrict;
}
public String getDbDataCenter() {
return dbDataCenter;
}
public void setDbDataCenter(String dbDataCenter) {
this.dbDataCenter = dbDataCenter;
}
@Override
public String toString() {
return "dbInstance [name=" +
@@ -226,6 +252,10 @@ public class DBInstance implements Propertied {
readWeight +
", databaseType=" +
databaseType +
", dbDistrict=" +
dbDistrict +
", dbDataCenter=" +
dbDataCenter +
",property=" +
property +
"]";
@@ -30,6 +30,7 @@ import com.actiontech.dble.config.util.ParameterMapping;
import com.actiontech.dble.util.DecryptUtil;
import com.actiontech.dble.util.LongUtil;
import com.actiontech.dble.util.StringUtil;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
@@ -37,6 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.Charset;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -255,6 +257,13 @@ public class DBConverter {
conf.setMaxCon(maxCon);
conf.setMinCon(minCon);
conf.setReadWeight(readWeight);
String dbDistrict = dbInstance.getDbDistrict();
checkChineseAndRules(dbDistrict, "dbDistrict");
String dbDataCenter = dbInstance.getDbDataCenter();
checkChineseAndRules(dbDataCenter, "dbDataCenter");
conf.setDbDistrict(dbDistrict);
conf.setDbDataCenter(dbDataCenter);
// id
String id = dbInstance.getId();
if (StringUtil.isEmpty(id)) {
@@ -280,6 +289,18 @@ public class DBConverter {
return dataBaseType;
}
private void checkChineseAndRules(String val, String name) {
if (Objects.nonNull(val)) {
String chinese = val.replaceAll(PATTERN_DB.toString(), "");
if (Strings.isNullOrEmpty(chinese)) {
return;
}
if (!StringUtil.isChinese(chinese)) {
throw new ConfigException("properties of system may not recognized:" + val + "the " + Charset.defaultCharset().name() + " encoding is recommended, dbInstance name " + name + " show be use u4E00-u9FA5a-zA-Z_0-9\\-\\.");
}
}
}
private void checkProperty(List<String> errorMsgList, Property property) {
String value = property.getValue();
if (StringUtil.isBlank(value)) {
@@ -8,13 +8,19 @@ package com.actiontech.dble.config.model;
import com.actiontech.dble.backend.mysql.CharsetUtil;
import com.actiontech.dble.config.Isolations;
import com.actiontech.dble.config.ProblemReporter;
import com.actiontech.dble.config.converter.DBConverter;
import com.actiontech.dble.config.util.ParameterMapping;
import com.actiontech.dble.config.util.StartProblemReporter;
import com.actiontech.dble.memory.unsafe.Platform;
import com.actiontech.dble.util.NetUtil;
import com.actiontech.dble.util.StringUtil;
import com.google.common.base.Strings;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.Objects;
import static com.actiontech.dble.config.model.db.PoolConfig.DEFAULT_IDLE_TIMEOUT;
@@ -209,6 +215,9 @@ public final class SystemConfig {
// For rwSplitUser, Implement stickiness for read and write instances, the default value is 1000ms
private long rwStickyTime = 1000;
private String district = null;
private String dataCenter = null;
private boolean closeHeartBeatRecord = false;
@@ -1517,6 +1526,25 @@ public final class SystemConfig {
routePenetrationRules = sqlPenetrationRegexesTmp;
}
public String getDistrict() {
return district;
}
@SuppressWarnings("unused")
public void setDistrict(String district) throws UnsupportedEncodingException {
checkChineseProperty(district, "district");
this.district = district;
}
public String getDataCenter() {
return dataCenter;
}
@SuppressWarnings("unused")
public void setDataCenter(String dataCenter) {
checkChineseProperty(dataCenter, "dataCenter");
this.dataCenter = dataCenter;
}
@Override
public String toString() {
@@ -1618,6 +1646,8 @@ public final class SystemConfig {
", closeHeartBeatRecord=" + closeHeartBeatRecord +
", enableRoutePenetration=" + enableRoutePenetration +
", routePenetrationRules='" + routePenetrationRules + '\'' +
", district='" + district +
", dataCenter='" + dataCenter +
"]";
}
@@ -1628,4 +1658,21 @@ public final class SystemConfig {
public void setCloseHeartBeatRecord(boolean closeHeartBeatRecord) {
this.closeHeartBeatRecord = closeHeartBeatRecord;
}
private void checkChineseProperty(String val, String name) {
if (Objects.nonNull(val)) {
int length = 11;
if (val.length() > length) {
problemReporter.warn("Property [ " + name + " ] " + val + " in bootstrap.cnf is illegalthe value contains a maximum of " + length + " characters");
}
String chinese = val.replaceAll(DBConverter.PATTERN_DB.toString(), "");
if (Strings.isNullOrEmpty(chinese)) {
return;
}
if (!StringUtil.isChinese(chinese)) {
problemReporter.warn("Property [ " + name + " ] " + val + " in bootstrap.cnf is illegalthe " + Charset.defaultCharset().name() + " encoding is recommended, Property [ " + name + " ] show be use u4E00-u9FA5a-zA-Z_0-9\\-\\.");
}
}
}
}
@@ -24,6 +24,8 @@ public class DbInstanceConfig {
private volatile PoolConfig poolConfig;
private final boolean usingDecrypt;
private DataBaseType dataBaseType;
private String dbDistrict;
private String dbDataCenter;
public DbInstanceConfig(String instanceName, String ip, int port, String url,
String user, String password, boolean disabled, boolean primary, boolean usingDecrypt, DataBaseType dataBaseType) {
@@ -149,6 +151,21 @@ public class DbInstanceConfig {
return false;
}
public String getDbDistrict() {
return dbDistrict;
}
public void setDbDistrict(String dbDistrict) {
this.dbDistrict = dbDistrict;
}
public String getDbDataCenter() {
return dbDataCenter;
}
public void setDbDataCenter(String dbDataCenter) {
this.dbDataCenter = dbDataCenter;
}
@Override
public String toString() {
@@ -107,31 +107,39 @@ public class RWSplitNonBlockingSession extends Session {
execute(master, null, callback);
}
public void execute(Boolean master, Callback callback, boolean write) {
execute(master, null, callback, write);
public void execute(Boolean master, Callback callback, boolean writeStatistical) {
execute(master, null, callback, writeStatistical, false);
}
/**
* @param master
* @param callback
* @param writeStatistical
* @param localRead only the SELECT and show statements attempt to localRead
*/
public void execute(Boolean master, Callback callback, boolean writeStatistical, boolean localRead) {
execute(master, null, callback, writeStatistical, localRead && !rwGroup.isRwSplitUseless());
}
public void execute(Boolean master, byte[] originPacket, Callback callback) {
execute(master, originPacket, callback, false, false);
}
public void execute(Boolean master, byte[] originPacket, Callback callback, boolean writeStatistical) {
execute(master, originPacket, callback, writeStatistical, false);
}
public void execute(Boolean master, byte[] originPacket, Callback callback, boolean writeStatistical, boolean localRead) {
try {
RWSplitHandler handler = getRwSplitHandler(originPacket, callback);
if (handler == null) return;
getConnection(handler, master, null);
getConnection(handler, master, isWriteStatistical(writeStatistical), localRead);
} catch (SQLSyntaxErrorException | IOException se) {
rwSplitService.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, se.getMessage());
}
}
public void execute(Boolean master, byte[] originPacket, Callback callback, boolean write) {
try {
RWSplitHandler handler = getRwSplitHandler(originPacket, callback);
if (handler == null) return;
getConnection(handler, master, isWrite(write));
} catch (SQLSyntaxErrorException | IOException se) {
rwSplitService.writeErrMessage(ErrorCode.ER_UNKNOWN_ERROR, se.getMessage());
}
}
public void getConnection(RWSplitHandler handler, Boolean master, Boolean write) {
public void getConnection(RWSplitHandler handler, Boolean master, Boolean writeStatistical, boolean localRead) {
try {
Boolean isMaster = canRunOnMaster(master); // first
boolean firstValue = isMaster == null ? false : isMaster;
@@ -146,7 +154,7 @@ public class RWSplitNonBlockingSession extends Session {
resetLastSqlResponseTime();
}
}
PhysicalDbInstance instance = reSelectRWDbGroup(rwGroup).rwSelect(isMaster, write); // second
PhysicalDbInstance instance = reSelectRWDbGroup(rwGroup).rwSelect(isMaster, writeStatistical, localRead); // second
boolean isWrite = !instance.isReadInstance();
this.setPreSendIsWrite(isWrite && firstValue); // ensure that the first and second results are write instances
checkDest(isWrite);
@@ -189,11 +197,11 @@ public class RWSplitNonBlockingSession extends Session {
return handler;
}
private boolean isWrite(boolean write) {
private boolean isWriteStatistical(boolean writeStatistical) {
if (!rwSplitService.isAutocommit() || rwSplitService.isTxStart() || rwSplitService.isUsingTmpTable()) {
return true;
}
return write;
return writeStatistical;
}
private Boolean canRunOnMaster(Boolean master) {
@@ -73,6 +73,10 @@ public class DbleDbInstance extends ManagerWritableTable {
private static final String COLUMN_DATABASE_TYPE = "database_type";
private static final String COLUMN_DB_DISTRICT = "db_district";
private static final String COLUMN_DB_DATA_CENTER = "db_data_center";
private static final String COLUMN_LAST_HEARTBEAT_ACK_TIMESTAMP = "last_heartbeat_ack_timestamp";
private static final String COLUMN_LAST_HEARTBEAT_ACK = "last_heartbeat_ack";
@@ -114,7 +118,7 @@ public class DbleDbInstance extends ManagerWritableTable {
private static final String COLUMN_FLOW_LOW_LEVEL = "flow_low_level";
public DbleDbInstance() {
super(TABLE_NAME, 34);
super(TABLE_NAME, 36);
setNotWritableColumnSet(COLUMN_ACTIVE_CONN_COUNT, COLUMN_IDLE_CONN_COUNT, COLUMN_READ_CONN_REQUEST, COLUMN_WRITE_CONN_REQUEST,
COLUMN_LAST_HEARTBEAT_ACK_TIMESTAMP, COLUMN_LAST_HEARTBEAT_ACK, COLUMN_HEARTBEAT_STATUS, COLUMN_HEARTBEAT_FAILURE_IN_LAST_5MIN);
@@ -168,6 +172,12 @@ public class DbleDbInstance extends ManagerWritableTable {
columns.put(COLUMN_DATABASE_TYPE, new ColumnMeta(COLUMN_DATABASE_TYPE, "varchar(11)", true, "mysql"));
columnsType.put(COLUMN_DATABASE_TYPE, Fields.FIELD_TYPE_VAR_STRING);
columns.put(COLUMN_DB_DISTRICT, new ColumnMeta(COLUMN_DB_DISTRICT, "varchar(11)", true, null));
columnsType.put(COLUMN_DB_DISTRICT, Fields.FIELD_TYPE_VAR_STRING);
columns.put(COLUMN_DB_DATA_CENTER, new ColumnMeta(COLUMN_DB_DATA_CENTER, "varchar(11)", true, null));
columnsType.put(COLUMN_DB_DATA_CENTER, Fields.FIELD_TYPE_VAR_STRING);
columns.put(COLUMN_LAST_HEARTBEAT_ACK_TIMESTAMP, new ColumnMeta(COLUMN_LAST_HEARTBEAT_ACK_TIMESTAMP, "varchar(64)", true));
columnsType.put(COLUMN_LAST_HEARTBEAT_ACK_TIMESTAMP, Fields.FIELD_TYPE_VAR_STRING);
@@ -255,6 +265,8 @@ public class DbleDbInstance extends ManagerWritableTable {
map.put(COLUMN_WRITE_CONN_REQUEST, String.valueOf(dbInstance.getCount(false)));
map.put(COLUMN_DISABLED, String.valueOf(dbInstance.isDisabled()));
map.put(COLUMN_DATABASE_TYPE, String.valueOf(dbInstanceConfig.getDataBaseType()).toLowerCase());
map.put(COLUMN_DB_DISTRICT, Optional.ofNullable(dbInstanceConfig.getDbDistrict()).orElse("").toLowerCase());
map.put(COLUMN_DB_DATA_CENTER, Optional.ofNullable(dbInstanceConfig.getDbDataCenter()).orElse("").toLowerCase());
map.put(COLUMN_LAST_HEARTBEAT_ACK_TIMESTAMP, heartbeat.getLastActiveTime());
map.put(COLUMN_LAST_HEARTBEAT_ACK, heartbeat.getStatusStr());
map.put(COLUMN_HEARTBEAT_STATUS, heartbeat.isChecking() ? MySQLHeartbeat.CHECK_STATUS_CHECKING : MySQLHeartbeat.CHECK_STATUS_IDLE);
@@ -415,6 +427,12 @@ public class DbleDbInstance extends ManagerWritableTable {
case COLUMN_DATABASE_TYPE:
dbInstance.setDatabaseType(entry.getValue());
break;
case COLUMN_DB_DISTRICT:
dbInstance.setDbDistrict(entry.getValue());
break;
case COLUMN_DB_DATA_CENTER:
dbInstance.setDbDataCenter(entry.getValue());
break;
case COLUMN_MIN_CONN_COUNT:
if (!StringUtil.isBlank(entry.getValue())) {
dbInstance.setMinCon(IntegerUtil.parseInt(entry.getValue()));
@@ -70,7 +70,7 @@ public class RWSplitQueryHandler implements FrontendQueryHandler {
session.execute(true, (isSuccess, resp, rwSplitService) -> rwSplitService.setSchema(schema));
break;
case RwSplitServerParse.SHOW:
session.execute(true, null, false);
session.execute(true, null, false, true);
break;
case RwSplitServerParse.SELECT:
RwSplitSelectHandler.handle(sql, session.getService(), rs >>> 8);
@@ -26,10 +26,10 @@ public final class RwSplitSelectHandler {
int rs2 = RwSplitServerParseSelect.parseSpecial(stmt);
switch (rs2) {
case RwSplitServerParseSelect.LOCK_READ:
service.getSession2().execute(true, null, false);
service.getSession2().execute(true, null, false, true);
break;
default:
service.getSession2().execute(null, null, false);
service.getSession2().execute(null, null, false, true);
break;
}
break;
@@ -139,6 +139,8 @@ public final class SystemParams {
readOnlyParams.add(new ParamInfo("closeHeartBeatRecord", sysConfig.isCloseHeartBeatRecord() + "", "close heartbeat record. if closed, `show @@dbinstance.synstatus`,`show @@dbinstance.syndetail`,`show @@heartbeat.detail` will be empty and `show @@heartbeat`'s EXECUTE_TIME will be '-' .The default value is false"));
readOnlyParams.add(new ParamInfo("enableRoutePenetration", sysConfig.isEnableRoutePenetration() + "", "Whether enable route penetration.The default value is 0"));
readOnlyParams.add(new ParamInfo("routePenetrationRules", sysConfig.getRoutePenetrationRules() + "", "The config of route penetration.The default value is ''"));
readOnlyParams.add(new ParamInfo("district", sysConfig.getDistrict() + "", "The location of the DBLE"));
readOnlyParams.add(new ParamInfo("dataCenter", sysConfig.getDataCenter() + "", "The data center where the DBLE resides"));
}
@@ -372,6 +372,7 @@ public final class StringUtil {
/**
* option to ignore case depending on the condition
*
* @param str1
* @param str2
* @param ignoreCase
@@ -609,4 +610,18 @@ public final class StringUtil {
}
return orgStr;
}
/**
* src: https://stackoverflow.com/questions/26357938/detect-chinese-character-in-java/26357985
* <p>
* Now Character.isIdeographic(int codepoint) would tell wether the codepoint is a C (Chinese) ideograph.
* Nearer is using Character.UnicodeScript.HAN.
*
* @param val
* @return
*/
public static boolean isChinese(String val) {
return val.codePoints().allMatch(codepoint -> Character.UnicodeScript.of(codepoint) == Character.UnicodeScript.HAN);
}
}
+2
View File
@@ -44,6 +44,8 @@
<xs:attribute name="usingDecrypt" type="xs:boolean"/>
<xs:attribute name="disabled" type="xs:boolean"/>
<xs:attribute name="databaseType" type="xs:string"/>
<xs:attribute name="dbDistrict" type="xs:string"/>
<xs:attribute name="dbDataCenter" type="xs:string"/>
</xs:complexType>
</xs:element>
</xs:sequence>