inner-1632&1560&1565&1632:charset merge (#3131)

* Revert "inner/1630:charset issue in union (#3127)"

This reverts commit d45d45e4

Signed-off-by: dcy <dcy10000@gmail.com>

* Revert "field charset (#3037)"

This reverts commit 2d873d85

Signed-off-by: dcy <dcy10000@gmail.com>

* inner-1632:charset merge in union

Signed-off-by: dcy <dcy10000@gmail.com>

* inner-1632-supplement:charset merge in union

Signed-off-by: dcy <dcy10000@gmail.com>

src/main/java/com/actiontech/dble/backend/mysql/nio/handler/query/impl/UnionHandler.java

@@ -6,6 +6,7 @@
 package com.actiontech.dble.backend.mysql.nio.handler.query.impl;
 
 
+import com.actiontech.dble.backend.mysql.CharsetUtil;
 import com.actiontech.dble.backend.mysql.nio.handler.query.BaseDMLHandler;
 import com.actiontech.dble.net.Session;
 import com.actiontech.dble.net.mysql.FieldPacket;
@@ -129,6 +130,13 @@ public class UnionHandler extends BaseDMLHandler {
         FieldTypes fieldType1 = FieldTypes.valueOf(fp1.getType());
         FieldTypes fieldType2 = FieldTypes.valueOf(fp2.getType());
         FieldTypes mergeFieldType = FieldUtil.fieldTypeMerge(fieldType1, fieldType2);
+        if (FieldUtil.resultMergeType(mergeFieldType) == FieldUtil.ItemResult.STRING_RESULT) {
+            final int binary = CharsetUtil.getCollationIndex("binary");
+            if (fp1.getCharsetIndex() == binary || fp2.getCharsetIndex() == binary) {
+                union.setCharsetIndex(binary);
+                union.setCharsetPriority(-1);
+            }
+        }
         union.setType(mergeFieldType.numberValue());
         return union;
     }

src/main/java/com/actiontech/dble/backend/mysql/nio/handler/util/HandlerTool.java

@@ -62,8 +62,10 @@ public final class HandlerTool {
     }
 
     public static Field createField(FieldPacket fp) {
-        return Field.getFieldItem(fp.getName(), fp.getDb(), fp.getTable(), fp.getOrgTable(), fp.getType(),
+        final Field fieldItem = Field.getFieldItem(fp.getName(), fp.getDb(), fp.getTable(), fp.getOrgTable(), fp.getType(),
                 fp.getCharsetIndex(), (int) fp.getLength(), fp.getDecimals(), fp.getFlags());
+        fieldItem.setCharsetPriority(fp.getCharsetPriority());
+        return fieldItem;
     }
 
     public static List<Field> createFields(List<FieldPacket> fps) {
@@ -291,9 +293,12 @@ public final class HandlerTool {
             throw new MySQLOutPutException(ErrorCode.ER_QUERYHANDLER, "", "field not found:" + col);
         Field field = fields.get(index);
         // if org col contains chinese, but push down's use alias col
-        field.setCharsetIndex(col.getCharsetIndex());
-        ItemField ret = new ItemField(field.getDbName(), field.getTable(), field.getName(), field.getCharsetIndex());
-        // ItemField ret = new ItemField(field.getDbName(), field.getTable(), field.getName());
+        int charsetIndex = field.getCharsetIndex();
+        if (col.getCharsetPriority() < field.getCharsetPriority()) {
+            charsetIndex = col.getCharsetIndex();
+        }
+        field.setCharsetIndex(charsetIndex);
+        ItemField ret = new ItemField(field.getDbName(), field.getTable(), field.getName(), charsetIndex);
         ret.setField(fields, index);
         ret.setItemName(col.getPushDownName() == null ? col.getItemName() : col.getPushDownName());
         return ret;

src/main/java/com/actiontech/dble/net/mysql/FieldPacket.java

@@ -50,6 +50,7 @@ public class FieldPacket extends MySQLPacket {
     private byte[] name;
     private byte[] orgName;
     private int charsetIndex;
+    protected int charsetPriority = 0;
     private long length;
     private int type;
     private int flags;
@@ -249,4 +250,12 @@ public class FieldPacket extends MySQLPacket {
     public boolean isEndOfQuery() {
         return false;
     }
+
+    public int getCharsetPriority() {
+        return charsetPriority;
+    }
+
+    public void setCharsetPriority(int charsetPriority) {
+        this.charsetPriority = charsetPriority;
+    }
 }

src/main/java/com/actiontech/dble/plan/common/field/Field.java

@@ -104,6 +104,7 @@ public abstract class Field {
     protected String orgTable;
     protected String dbName;
     protected int charsetIndex;
+    protected int charsetPriority = 0;
     protected String javaCharsetName;
     protected long flags;
     protected byte[] ptr;
@@ -307,4 +308,13 @@ public abstract class Field {
     public void setDecimals(int decimals) {
         this.decimals = decimals;
     }
+
+
+    public int getCharsetPriority() {
+        return charsetPriority;
+    }
+
+    public void setCharsetPriority(int charsetPriority) {
+        this.charsetPriority = charsetPriority;
+    }
 }

src/main/java/com/actiontech/dble/plan/common/field/FieldUtil.java

@@ -10,6 +10,9 @@ import com.actiontech.dble.plan.common.item.FieldTypes;
 
 import java.util.List;
 
+import static com.actiontech.dble.plan.common.field.FieldUtil.ItemResult.*;
+
+
 public class FieldUtil {
     public static final int NOT_NULL_FLAG = 1; /* Field can't be NULL */
     public static final int PRI_KEY_FLAG = 2; /* Field is part of a primary key */
@@ -180,6 +183,50 @@ public class FieldUtil {
         return FIELD_TYPES_MERGE_RULES[fieldType2Index(a)][fieldType2Index(b)];
     }
 
+    public static ItemResult resultMergeType(FieldTypes fieldType) {
+        return FIELD_TYPES_RESULT_TYPE[fieldType2Index(fieldType)];
+    }
+
+    public enum ItemResult {
+        STRING_RESULT, REAL_RESULT, INT_RESULT, ROW_RESULT,
+        DECIMAL_RESULT
+    }
+
+    ;
+
+    private static final ItemResult[] FIELD_TYPES_RESULT_TYPE = new ItemResult[]{
+            //MYSQL_TYPE_DECIMAL      MYSQL_TYPE_TINY
+            DECIMAL_RESULT, INT_RESULT,
+            //MYSQL_TYPE_SHORT        MYSQL_TYPE_LONG
+            INT_RESULT, INT_RESULT,
+            //MYSQL_TYPE_FLOAT        MYSQL_TYPE_DOUBLE
+            REAL_RESULT, REAL_RESULT,
+            //MYSQL_TYPE_NULL         MYSQL_TYPE_TIMESTAMP
+            STRING_RESULT, STRING_RESULT,
+            //MYSQL_TYPE_LONGLONG     MYSQL_TYPE_INT24
+            INT_RESULT, INT_RESULT,
+            //MYSQL_TYPE_DATE         MYSQL_TYPE_TIME
+            STRING_RESULT, STRING_RESULT,
+            //MYSQL_TYPE_DATETIME     MYSQL_TYPE_YEAR
+            STRING_RESULT, INT_RESULT,
+            //MYSQL_TYPE_NEWDATE      MYSQL_TYPE_VARCHAR
+            STRING_RESULT, STRING_RESULT,
+            //MYSQL_TYPE_BIT          <16>-<244>
+            STRING_RESULT,
+            //MYSQL_TYPE_JSON
+            STRING_RESULT,
+            //MYSQL_TYPE_NEWDECIMAL   MYSQL_TYPE_ENUM
+            DECIMAL_RESULT, STRING_RESULT,
+            //MYSQL_TYPE_SET          MYSQL_TYPE_TINY_BLOB
+            STRING_RESULT, STRING_RESULT,
+            //MYSQL_TYPE_MEDIUM_BLOB  MYSQL_TYPE_LONG_BLOB
+            STRING_RESULT, STRING_RESULT,
+            //MYSQL_TYPE_BLOB         MYSQL_TYPE_VAR_STRING
+            STRING_RESULT, STRING_RESULT,
+            //MYSQL_TYPE_STRING       MYSQL_TYPE_GEOMETRY
+            STRING_RESULT, STRING_RESULT,
+    };
+
     private static final FieldTypes[][] FIELD_TYPES_MERGE_RULES = new FieldTypes[][]{
             /* enum_field_types.MYSQL_TYPE_DECIMAL -> */
             {

src/main/java/com/actiontech/dble/plan/common/item/Item.java

@@ -135,6 +135,13 @@ public abstract class Item {
         this.charsetIndex = charsetIndex;
     }
 
+    public int getCharsetPriority() {
+        return charsetPriority;
+    }
+
+    public void setCharsetPriority(int charsetPriority) {
+        this.charsetPriority = charsetPriority;
+    }
 
     public enum ItemResult {
         STRING_RESULT, REAL_RESULT, INT_RESULT, ROW_RESULT, DECIMAL_RESULT
@@ -160,6 +167,7 @@ public abstract class Item {
     protected ItemResult cmpContext;
     /* the default charset index is my_charset_bin */
     protected int charsetIndex = 63;
+    protected int charsetPriority = 0;
     private HashSet<PlanNode> referTables;
 
     public boolean fixFields() {

src/main/java/com/actiontech/dble/plan/common/item/ItemField.java

@@ -45,7 +45,7 @@ public class ItemField extends ItemIdent {
     }
 
     public ItemField(Field field) {
-        this(null, field.getTable(), field.getName(), field.getCharsetIndex());
+        super(null, field.getTable(), field.getName());
         setField(field);
     }
 

src/main/java/com/actiontech/dble/plan/node/JoinNode.java

@@ -252,7 +252,7 @@ public class JoinNode extends PlanNode {
                     String name = field.getName();
                     for (String fieldName : usingFields) {
                         if (name.equals(fieldName)) {
-                            ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName, field.getCharsetIndex());
+                            ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName);
                             newSels.add(col);
                         }
                     }
@@ -262,13 +262,13 @@ public class JoinNode extends PlanNode {
                     if (usingFields.contains(field.getName())) {
                         continue;
                     }
-                    ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName(), field.getCharsetIndex());
+                    ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName());
                     newSels.add(col);
                 }
 
                 // add Remaining innerFields
                 for (NamedField field : innerFields.keySet()) {
-                    ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName(), field.getCharsetIndex());
+                    ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName());
                     boolean contians = false;
                     for (Item f : newSels) {
                         if (f instanceof ItemField) {
@@ -285,7 +285,7 @@ public class JoinNode extends PlanNode {
 
             } else {
                 for (String fieldName : usingFields) {
-                    ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName); // fixme?: charsetIndex
+                    ItemField col = new ItemField(tableInfo.getKey(), tableInfo.getValue(), fieldName);
                     newSels.add(col);
                 }
 
@@ -293,7 +293,7 @@ public class JoinNode extends PlanNode {
                     if (usingFields.contains(field.getName())) {
                         continue;
                     }
-                    ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName(), field.getCharsetIndex());
+                    ItemField col = new ItemField(field.getSchema(), field.getTable(), field.getName());
                     newSels.add(col);
                 }
             }

src/main/java/com/actiontech/dble/plan/node/ManagerTableNode.java

@@ -66,7 +66,7 @@ public class ManagerTableNode extends PlanNode {
                 newSelects.add(sel);
             else {
                 for (NamedField innerField : innerFields.keySet()) {
-                    ItemField col = new ItemField(null, sel.getTableName(), innerField.getName(), innerField.getCharsetIndex());
+                    ItemField col = new ItemField(null, sel.getTableName(), innerField.getName());
                     newSelects.add(col);
                 }
             }

src/main/java/com/actiontech/dble/plan/node/MergeNode.java

@@ -105,11 +105,10 @@ public class MergeNode extends PlanNode {
                 throw new MySQLOutPutException(ErrorCode.ER_DUP_FIELDNAME, "", "Duplicate column name " + coutField.getName());
             }
             checkDup.add(testDupField);
-            ItemField column = new ItemField(null, coutField.getTable(), coutField.getName(), coutField.getCharsetIndex());
+            ItemField column = new ItemField(null, coutField.getTable(), coutField.getName());
             column.getReferTables().clear();
             column.getReferTables().add(coutField.planNode);
             NamedField tmpField = new NamedField(null, coutField.getTable(), coutField.getName(), this);
-            tmpField.setCharsetIndex(coutField.getCharsetIndex());
             outerFields.put(tmpField, column);
             getColumnsSelected().add(column);
         }

src/main/java/com/actiontech/dble/plan/node/PlanNode.java

@@ -358,7 +358,6 @@ public abstract class PlanNode {
                 ((ItemFuncGroupConcat) sel).fixOrders(nameContext);
             }
             NamedField field = makeOutNamedField(sel);
-            field.setCharsetIndex(sel.getCharsetIndex());
             if (outerFields.containsKey(field) && isDuplicateField(this))
                 throw new MySQLOutPutException(ErrorCode.ER_OPTIMIZER, "", "duplicate field");
             outerFields.put(field, sel);