From a31a8df368dfcf9d01ee5719317059b64dc7a8c3 Mon Sep 17 00:00:00 2001 From: Rico Date: Wed, 16 Feb 2022 16:21:51 +0800 Subject: [PATCH] inner-1632&1560&1565&1632:charset merge (#3131) * Revert "inner/1630:charset issue in union (#3127)" This reverts commit d45d45e4 Signed-off-by: dcy * Revert "field charset (#3037)" This reverts commit 2d873d85 Signed-off-by: dcy * inner-1632:charset merge in union Signed-off-by: dcy * inner-1632-supplement:charset merge in union Signed-off-by: dcy --- .../nio/handler/query/impl/UnionHandler.java | 8 ++++ .../mysql/nio/handler/util/HandlerTool.java | 13 +++-- .../dble/net/mysql/FieldPacket.java | 9 ++++ .../dble/plan/common/field/Field.java | 10 ++++ .../dble/plan/common/field/FieldUtil.java | 47 +++++++++++++++++++ .../dble/plan/common/item/Item.java | 8 ++++ .../dble/plan/common/item/ItemField.java | 2 +- .../actiontech/dble/plan/node/JoinNode.java | 10 ++-- .../dble/plan/node/ManagerTableNode.java | 2 +- .../actiontech/dble/plan/node/MergeNode.java | 3 +- .../actiontech/dble/plan/node/PlanNode.java | 1 - 11 files changed, 99 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/query/impl/UnionHandler.java b/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/query/impl/UnionHandler.java index 6ba2f65c0..d464eee77 100644 --- a/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/query/impl/UnionHandler.java +++ b/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/query/impl/UnionHandler.java @@ -6,6 +6,7 @@ package com.actiontech.dble.backend.mysql.nio.handler.query.impl; +import com.actiontech.dble.backend.mysql.CharsetUtil; import com.actiontech.dble.backend.mysql.nio.handler.query.BaseDMLHandler; import com.actiontech.dble.net.Session; import com.actiontech.dble.net.mysql.FieldPacket; @@ -129,6 +130,13 @@ public class UnionHandler extends BaseDMLHandler { FieldTypes fieldType1 = FieldTypes.valueOf(fp1.getType()); FieldTypes fieldType2 = FieldTypes.valueOf(fp2.getType()); FieldTypes mergeFieldType = FieldUtil.fieldTypeMerge(fieldType1, fieldType2); + if (FieldUtil.resultMergeType(mergeFieldType) == FieldUtil.ItemResult.STRING_RESULT) { + final int binary = CharsetUtil.getCollationIndex("binary"); + if (fp1.getCharsetIndex() == binary || fp2.getCharsetIndex() == binary) { + union.setCharsetIndex(binary); + union.setCharsetPriority(-1); + } + } union.setType(mergeFieldType.numberValue()); return union; } diff --git a/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/util/HandlerTool.java b/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/util/HandlerTool.java index e71f84127..a210fa83a 100644 --- a/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/util/HandlerTool.java +++ b/src/main/java/com/actiontech/dble/backend/mysql/nio/handler/util/HandlerTool.java @@ -62,8 +62,10 @@ public final class HandlerTool { } public static Field createField(FieldPacket fp) { - return Field.getFieldItem(fp.getName(), fp.getDb(), fp.getTable(), fp.getOrgTable(), fp.getType(), + final Field fieldItem = Field.getFieldItem(fp.getName(), fp.getDb(), fp.getTable(), fp.getOrgTable(), fp.getType(), fp.getCharsetIndex(), (int) fp.getLength(), fp.getDecimals(), fp.getFlags()); + fieldItem.setCharsetPriority(fp.getCharsetPriority()); + return fieldItem; } public static List createFields(List fps) { @@ -291,9 +293,12 @@ public final class HandlerTool { throw new MySQLOutPutException(ErrorCode.ER_QUERYHANDLER, "", "field not found:" + col); Field field = fields.get(index); // if org col contains chinese, but push down's use alias col - field.setCharsetIndex(col.getCharsetIndex()); - ItemField ret = new ItemField(field.getDbName(), field.getTable(), field.getName(), field.getCharsetIndex()); - // ItemField ret = new ItemField(field.getDbName(), field.getTable(), field.getName()); + int charsetIndex = field.getCharsetIndex(); + if (col.getCharsetPriority() < field.getCharsetPriority()) { + charsetIndex = col.getCharsetIndex(); + } + field.setCharsetIndex(charsetIndex); + ItemField ret = new ItemField(field.getDbName(), field.getTable(), field.getName(), charsetIndex); ret.setField(fields, index); ret.setItemName(col.getPushDownName() == null ? col.getItemName() : col.getPushDownName()); return ret; diff --git a/src/main/java/com/actiontech/dble/net/mysql/FieldPacket.java b/src/main/java/com/actiontech/dble/net/mysql/FieldPacket.java index 616192350..7eda6d2f5 100644 --- a/src/main/java/com/actiontech/dble/net/mysql/FieldPacket.java +++ b/src/main/java/com/actiontech/dble/net/mysql/FieldPacket.java @@ -50,6 +50,7 @@ public class FieldPacket extends MySQLPacket { private byte[] name; private byte[] orgName; private int charsetIndex; + protected int charsetPriority = 0; private long length; private int type; private int flags; @@ -249,4 +250,12 @@ public class FieldPacket extends MySQLPacket { public boolean isEndOfQuery() { return false; } + + public int getCharsetPriority() { + return charsetPriority; + } + + public void setCharsetPriority(int charsetPriority) { + this.charsetPriority = charsetPriority; + } } diff --git a/src/main/java/com/actiontech/dble/plan/common/field/Field.java b/src/main/java/com/actiontech/dble/plan/common/field/Field.java index 9c3165e51..55b172620 100644 --- a/src/main/java/com/actiontech/dble/plan/common/field/Field.java +++ b/src/main/java/com/actiontech/dble/plan/common/field/Field.java @@ -104,6 +104,7 @@ public abstract class Field { protected String orgTable; protected String dbName; protected int charsetIndex; + protected int charsetPriority = 0; protected String javaCharsetName; protected long flags; protected byte[] ptr; @@ -307,4 +308,13 @@ public abstract class Field { public void setDecimals(int decimals) { this.decimals = decimals; } + + + public int getCharsetPriority() { + return charsetPriority; + } + + public void setCharsetPriority(int charsetPriority) { + this.charsetPriority = charsetPriority; + } } diff --git a/src/main/java/com/actiontech/dble/plan/common/field/FieldUtil.java b/src/main/java/com/actiontech/dble/plan/common/field/FieldUtil.java index 4298a763f..23579ffaa 100644 --- a/src/main/java/com/actiontech/dble/plan/common/field/FieldUtil.java +++ b/src/main/java/com/actiontech/dble/plan/common/field/FieldUtil.java @@ -10,6 +10,9 @@ import com.actiontech.dble.plan.common.item.FieldTypes; import java.util.List; +import static com.actiontech.dble.plan.common.field.FieldUtil.ItemResult.*; + + public class FieldUtil { public static final int NOT_NULL_FLAG = 1; /* Field can't be NULL */ public static final int PRI_KEY_FLAG = 2; /* Field is part of a primary key */ @@ -180,6 +183,50 @@ public class FieldUtil { return FIELD_TYPES_MERGE_RULES[fieldType2Index(a)][fieldType2Index(b)]; } + public static ItemResult resultMergeType(FieldTypes fieldType) { + return FIELD_TYPES_RESULT_TYPE[fieldType2Index(fieldType)]; + } + + public enum ItemResult { + STRING_RESULT, REAL_RESULT, INT_RESULT, ROW_RESULT, + DECIMAL_RESULT + } + + ; + + private static final ItemResult[] FIELD_TYPES_RESULT_TYPE = new ItemResult[]{ + //MYSQL_TYPE_DECIMAL MYSQL_TYPE_TINY + DECIMAL_RESULT, INT_RESULT, + //MYSQL_TYPE_SHORT MYSQL_TYPE_LONG + INT_RESULT, INT_RESULT, + //MYSQL_TYPE_FLOAT MYSQL_TYPE_DOUBLE + REAL_RESULT, REAL_RESULT, + //MYSQL_TYPE_NULL MYSQL_TYPE_TIMESTAMP + STRING_RESULT, STRING_RESULT, + //MYSQL_TYPE_LONGLONG MYSQL_TYPE_INT24 + INT_RESULT, INT_RESULT, + //MYSQL_TYPE_DATE MYSQL_TYPE_TIME + STRING_RESULT, STRING_RESULT, + //MYSQL_TYPE_DATETIME MYSQL_TYPE_YEAR + STRING_RESULT, INT_RESULT, + //MYSQL_TYPE_NEWDATE MYSQL_TYPE_VARCHAR + STRING_RESULT, STRING_RESULT, + //MYSQL_TYPE_BIT <16>-<244> + STRING_RESULT, + //MYSQL_TYPE_JSON + STRING_RESULT, + //MYSQL_TYPE_NEWDECIMAL MYSQL_TYPE_ENUM + DECIMAL_RESULT, STRING_RESULT, + //MYSQL_TYPE_SET MYSQL_TYPE_TINY_BLOB + STRING_RESULT, STRING_RESULT, + //MYSQL_TYPE_MEDIUM_BLOB MYSQL_TYPE_LONG_BLOB + STRING_RESULT, STRING_RESULT, + //MYSQL_TYPE_BLOB MYSQL_TYPE_VAR_STRING + STRING_RESULT, STRING_RESULT, + //MYSQL_TYPE_STRING MYSQL_TYPE_GEOMETRY + STRING_RESULT, STRING_RESULT, + }; + private static final FieldTypes[][] FIELD_TYPES_MERGE_RULES = new FieldTypes[][]{ /* enum_field_types.MYSQL_TYPE_DECIMAL -> */ { diff --git a/src/main/java/com/actiontech/dble/plan/common/item/Item.java b/src/main/java/com/actiontech/dble/plan/common/item/Item.java index d1e32dcd4..af0b9bfa7 100644 --- a/src/main/java/com/actiontech/dble/plan/common/item/Item.java +++ b/src/main/java/com/actiontech/dble/plan/common/item/Item.java @@ -135,6 +135,13 @@ public abstract class Item { this.charsetIndex = charsetIndex; } + public int getCharsetPriority() { + return charsetPriority; + } + + public void setCharsetPriority(int charsetPriority) { + this.charsetPriority = charsetPriority; + } public enum ItemResult { STRING_RESULT, REAL_RESULT, INT_RESULT, ROW_RESULT, DECIMAL_RESULT @@ -160,6 +167,7 @@ public abstract class Item { protected ItemResult cmpContext; /* the default charset index is my_charset_bin */ protected int charsetIndex = 63; + protected int charsetPriority = 0; private HashSet referTables; public boolean fixFields() { diff --git a/src/main/java/com/actiontech/dble/plan/common/item/ItemField.java b/src/main/java/com/actiontech/dble/plan/common/item/ItemField.java index f765591c3..418da2c58 100644 --- a/src/main/java/com/actiontech/dble/plan/common/item/ItemField.java +++ b/src/main/java/com/actiontech/dble/plan/common/item/ItemField.java @@ -45,7 +45,7 @@ public class ItemField extends ItemIdent { } public ItemField(Field field) { - this(null, field.getTable(), field.getName(), field.getCharsetIndex()); + super(null, field.getTable(), field.getName()); setField(field); } diff --git a/src/main/java/com/actiontech/dble/plan/node/JoinNode.java b/src/main/java/com/actiontech/dble/plan/node/JoinNode.java index 1ab90fb01..1ea8c1575 100644 --- a/src/main/java/com/actiontech/dble/plan/node/JoinNode.java +++ b/src/main/java/com/actiontech/dble/plan/node/JoinNode.java @@ -252,7 +252,7 @@ public class JoinNode extends PlanNode { String name = field.getName(); for (String fieldName : usingFields) { if (name.equals(fieldName)) { - ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName, field.getCharsetIndex()); + ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName); newSels.add(col); } } @@ -262,13 +262,13 @@ public class JoinNode extends PlanNode { if (usingFields.contains(field.getName())) { continue; } - ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName(), field.getCharsetIndex()); + ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName()); newSels.add(col); } // add Remaining innerFields for (NamedField field : innerFields.keySet()) { - ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName(), field.getCharsetIndex()); + ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName()); boolean contians = false; for (Item f : newSels) { if (f instanceof ItemField) { @@ -285,7 +285,7 @@ public class JoinNode extends PlanNode { } else { for (String fieldName : usingFields) { - ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName); // fixme?: charsetIndex + ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName); newSels.add(col); } @@ -293,7 +293,7 @@ public class JoinNode extends PlanNode { if (usingFields.contains(field.getName())) { continue; } - ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName(), field.getCharsetIndex()); + ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName()); newSels.add(col); } } diff --git a/src/main/java/com/actiontech/dble/plan/node/ManagerTableNode.java b/src/main/java/com/actiontech/dble/plan/node/ManagerTableNode.java index 709316c4f..9bbec8de5 100644 --- a/src/main/java/com/actiontech/dble/plan/node/ManagerTableNode.java +++ b/src/main/java/com/actiontech/dble/plan/node/ManagerTableNode.java @@ -66,7 +66,7 @@ public class ManagerTableNode extends PlanNode { newSelects.add(sel); else { for (NamedField innerField : innerFields.keySet()) { - ItemField col = new ItemField(null, sel.getTableName(), innerField.getName(), innerField.getCharsetIndex()); + ItemField col = new ItemField(null, sel.getTableName(), innerField.getName()); newSelects.add(col); } } diff --git a/src/main/java/com/actiontech/dble/plan/node/MergeNode.java b/src/main/java/com/actiontech/dble/plan/node/MergeNode.java index 19abc10dd..7a47e1a60 100644 --- a/src/main/java/com/actiontech/dble/plan/node/MergeNode.java +++ b/src/main/java/com/actiontech/dble/plan/node/MergeNode.java @@ -105,11 +105,10 @@ public class MergeNode extends PlanNode { throw new MySQLOutPutException(ErrorCode.ER_DUP_FIELDNAME, "", "Duplicate column name " + coutField.getName()); } checkDup.add(testDupField); - ItemField column = new ItemField(null, coutField.getTable(), coutField.getName(), coutField.getCharsetIndex()); + ItemField column = new ItemField(null, coutField.getTable(), coutField.getName()); column.getReferTables().clear(); column.getReferTables().add(coutField.planNode); NamedField tmpField = new NamedField(null, coutField.getTable(), coutField.getName(), this); - tmpField.setCharsetIndex(coutField.getCharsetIndex()); outerFields.put(tmpField, column); getColumnsSelected().add(column); } diff --git a/src/main/java/com/actiontech/dble/plan/node/PlanNode.java b/src/main/java/com/actiontech/dble/plan/node/PlanNode.java index ec057e296..9b0e46bfd 100644 --- a/src/main/java/com/actiontech/dble/plan/node/PlanNode.java +++ b/src/main/java/com/actiontech/dble/plan/node/PlanNode.java @@ -358,7 +358,6 @@ public abstract class PlanNode { ((ItemFuncGroupConcat) sel).fixOrders(nameContext); } NamedField field = makeOutNamedField(sel); - field.setCharsetIndex(sel.getCharsetIndex()); if (outerFields.containsKey(field) && isDuplicateField(this)) throw new MySQLOutPutException(ErrorCode.ER_OPTIMIZER, "", "duplicate field"); outerFields.put(field, sel);