[no-release-notes] fix for building and modifying secondary indexes for keyless tables (#3930)

* fix for building and modifying secondary indexes for keyless tables

Also includes some changes to fix engine test validation of secondary indexes. Instead of validating the latest commit of all branches, we validate the working root of all branches.

Also, all byte like tuple fields are now debug printed as hex encoded strings for human readability and easier debugging in IDEs

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* fix mapping tests

Co-authored-by: druvv <druvv@users.noreply.github.com>

go/go.mod

@@ -58,7 +58,7 @@ require (
 )
 
 require (
-	github.com/dolthub/go-mysql-server v0.12.1-0.20220726021412-60dcca3a24be
+	github.com/dolthub/go-mysql-server v0.12.1-0.20220726201900-e9a3055c5c24
 	github.com/google/flatbuffers v2.0.6+incompatible
 	github.com/gosuri/uilive v0.0.4
 	github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
@@ -140,8 +140,4 @@ replace (
 	github.com/oliveagle/jsonpath => github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474
 )
 
-// replace github.com/dolthub/vitess => /Users/taylor/go/src/github.com/dolthub/vitess
-
-// replace github.com/dolthub/go-mysql-server => /Users/taylor/go/src/github.com/dolthub/go-mysql-server
-
 go 1.18

go/go.sum

@@ -173,8 +173,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
-github.com/dolthub/go-mysql-server v0.12.1-0.20220726021412-60dcca3a24be h1:0nkOM5NJjD7GGVSJm+EtWghopFRKURsy0ahLv8nA84Q=
-github.com/dolthub/go-mysql-server v0.12.1-0.20220726021412-60dcca3a24be/go.mod h1:JgB3WpY0RMgyAda3YG5VHVncH2B8i1N9Mx9LOp41lIs=
+github.com/dolthub/go-mysql-server v0.12.1-0.20220726201900-e9a3055c5c24 h1:dXdcHIsfIlt1fVW81irh3h65HwoK0Tw9i4A37khZYaE=
+github.com/dolthub/go-mysql-server v0.12.1-0.20220726201900-e9a3055c5c24/go.mod h1:JgB3WpY0RMgyAda3YG5VHVncH2B8i1N9Mx9LOp41lIs=
 github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g=
 github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms=
 github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=

go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go

@@ -229,7 +229,7 @@ func TestSingleScriptPrepared(t *testing.T) {
 	//sch3, rows3 := enginetest.MustQuery(ctx, e, rawQuery)
 	//fmt.Println(sch3, rows3)
 
-	enginetest.TestQueryWithContext(t, ctx, e, tt.Query, tt.Expected, tt.ExpectedColumns, tt.Bindings)
+	enginetest.TestQueryWithContext(t, ctx, e, harness, tt.Query, tt.Expected, tt.ExpectedColumns, tt.Bindings)
 }
 
 func TestVersionedQueries(t *testing.T) {
@@ -413,7 +413,7 @@ func TestDoltUserPrivileges(t *testing.T) {
 					})
 				} else {
 					t.Run(assertion.Query, func(t *testing.T) {
-						enginetest.TestQueryWithContext(t, ctx, engine, assertion.Query, assertion.Expected, nil, nil)
+						enginetest.TestQueryWithContext(t, ctx, engine, harness, assertion.Query, assertion.Expected, nil, nil)
 					})
 				}
 			}

go/libraries/doltcore/sqle/enginetest/validation.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
+	"github.com/dolthub/dolt/go/libraries/doltcore/ref"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	"github.com/dolthub/dolt/go/store/prolly"
@@ -181,7 +182,7 @@ func validateKeylessIndex(ctx context.Context, sch schema.Schema, def schema.Ind
 			return err
 		}
 		if !ok {
-			return fmt.Errorf("index key %v not found in index %s", k, def.Name())
+			return fmt.Errorf("index key %s not found in index %s", builder.Desc.Format(k), def.Name())
 		}
 	}
 }
@@ -273,18 +274,18 @@ func iterDatabaseTables(
 		return err
 	}
 
-	for i := range branches {
-		var c *doltdb.Commit
-		var r *doltdb.RootValue
+	for _, branchRef := range branches {
+		wsRef, err := ref.WorkingSetRefForHead(branchRef)
+		if err != nil {
+			return err
+		}
+		ws, err := ddb.ResolveWorkingSet(ctx, wsRef)
+		if err != nil {
+			return err
+		}
+
+		r := ws.WorkingRoot()
 
-		c, err = ddb.ResolveCommitRef(ctx, branches[i])
-		if err != nil {
-			return err
-		}
-		r, err = c.GetRootValue(ctx)
-		if err != nil {
-			return err
-		}
 		if err = r.IterTables(ctx, cb); err != nil {
 			return err
 		}

go/libraries/doltcore/sqle/writer/prolly_index_writer.go

@@ -241,11 +241,10 @@ type prollySecondaryIndexWriter struct {
 	mut    prolly.MutableMap
 	unique bool
 
-	keyBld      *val.TupleBuilder
-	prefixBld   *val.TupleBuilder
-	suffixBld   *val.TupleBuilder
-	numUniqCols int
-	keyMap      val.OrdinalMapping
+	keyBld    *val.TupleBuilder
+	prefixBld *val.TupleBuilder
+	suffixBld *val.TupleBuilder
+	keyMap    val.OrdinalMapping
 }
 
 var _ indexWriter = prollySecondaryIndexWriter{}
@@ -264,7 +263,7 @@ func (m prollySecondaryIndexWriter) Insert(ctx context.Context, sqlRow sql.Row)
 		if err := index.PutField(ctx, m.mut.NodeStore(), m.keyBld, to, sqlRow[from]); err != nil {
 			return err
 		}
-		if to < m.numUniqCols {
+		if to < m.prefixBld.Desc.Count() {
 			if err := index.PutField(ctx, m.mut.NodeStore(), m.prefixBld, to, sqlRow[from]); err != nil {
 				return err
 			}
@@ -299,8 +298,8 @@ func (m prollySecondaryIndexWriter) checkForUniqueKeyErr(ctx context.Context, pr
 		return err
 	}
 	if err == nil {
-		for i := m.numUniqCols; i < existingK.Count(); i++ {
-			j := i - m.numUniqCols
+		for i := m.prefixBld.Desc.Count(); i < existingK.Count(); i++ {
+			j := i - m.prefixBld.Desc.Count()
 			m.suffixBld.PutRaw(j, existingK.GetField(i))
 		}
 		suffixK := m.suffixBld.Build(sharePool)
@@ -341,7 +340,7 @@ func (m prollySecondaryIndexWriter) Update(ctx context.Context, oldRow sql.Row,
 		if err := index.PutField(ctx, m.mut.NodeStore(), m.keyBld, to, newRow[from]); err != nil {
 			return err
 		}
-		if to < m.numUniqCols {
+		if to < m.prefixBld.Desc.Count() {
 			if err := index.PutField(ctx, m.mut.NodeStore(), m.prefixBld, to, newRow[from]); err != nil {
 				return err
 			}

go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go

@@ -209,10 +209,13 @@ func (writer prollyKeylessSecondaryWriter) Insert(ctx context.Context, sqlRow sq
 		if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, sqlRow[from]); err != nil {
 			return err
 		}
-		if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, sqlRow[from]); err != nil {
-			return err
+		if to < writer.prefixBld.Desc.Count() {
+			if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, sqlRow[from]); err != nil {
+				return err
+			}
 		}
 	}
+
 	hashId, _, err := writer.primary.tuplesFromRow(ctx, sqlRow)
 	if err != nil {
 		return err

go/libraries/doltcore/sqle/writer/prolly_table_writer.go

@@ -75,14 +75,13 @@ func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch
 		keyDesc, _ := m.Descriptors()
 
 		writers[defName] = prollySecondaryIndexWriter{
-			name:        defName,
-			mut:         m.Mutate(),
-			unique:      def.IsUnique(),
-			keyBld:      val.NewTupleBuilder(keyDesc),
-			prefixBld:   val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())),
-			suffixBld:   val.NewTupleBuilder(keyDesc.SuffixDesc(keyDesc.Count() - def.Count())),
-			numUniqCols: def.Count(),
-			keyMap:      keyMap,
+			name:      defName,
+			mut:       m.Mutate(),
+			unique:    def.IsUnique(),
+			keyBld:    val.NewTupleBuilder(keyDesc),
+			prefixBld: val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())),
+			suffixBld: val.NewTupleBuilder(keyDesc.SuffixDesc(keyDesc.Count() - def.Count())),
+			keyMap:    keyMap,
 		}
 	}
 

go/libraries/doltcore/table/editor/creation/index.go

@@ -377,7 +377,9 @@ type rangeIterator interface {
 func GetIndexKeyMapping(sch schema.Schema, idx schema.Index) (keyLen int, m val.OrdinalMapping) {
 	m = make(val.OrdinalMapping, len(idx.AllTags()))
 
-	if schema.IsKeyless(sch) {
+	keyless := schema.IsKeyless(sch)
+
+	if keyless {
 		// the only key is the hash of the values
 		keyLen = 1
 	} else {
@@ -387,8 +389,12 @@ func GetIndexKeyMapping(sch schema.Schema, idx schema.Index) (keyLen int, m val.
 	for i, tag := range idx.AllTags() {
 		j, ok := sch.GetPKCols().TagToIdx[tag]
 		if !ok {
-			j = sch.GetNonPKCols().TagToIdx[tag]
-			j += keyLen
+			if keyless {
+				// Skip cardinality column
+				j = keyLen + 1 + sch.GetNonPKCols().TagToIdx[tag]
+			} else {
+				j = keyLen + sch.GetNonPKCols().TagToIdx[tag]
+			}
 		}
 		m[i] = j
 	}

go/libraries/doltcore/table/editor/creation/index_test.go

@@ -104,7 +104,8 @@ func TestGetIndexKeyMapping(t *testing.T) {
 			},
 			IdxCols: []string{"col1"},
 			KeyLen:  1,
-			Mapping: []int{1, 0},
+			// Mapping should skip over cardinality
+			Mapping: []int{2, 0},
 		},
 		{
 			Name: "keyless other",
@@ -114,7 +115,8 @@ func TestGetIndexKeyMapping(t *testing.T) {
 			},
 			IdxCols: []string{"col2"},
 			KeyLen:  1,
-			Mapping: []int{2, 0},
+			// Mapping should skip over cardinality
+			Mapping: []int{3, 0},
 		},
 		{
 			Name: "compound keyless",
@@ -125,7 +127,8 @@ func TestGetIndexKeyMapping(t *testing.T) {
 			},
 			IdxCols: []string{"col2", "col3"},
 			KeyLen:  1,
-			Mapping: []int{2, 3, 0},
+			// Mapping should skip over cardinality
+			Mapping: []int{3, 4, 0},
 		},
 		{
 			Name: "compound keyless reverse",
@@ -136,7 +139,8 @@ func TestGetIndexKeyMapping(t *testing.T) {
 			},
 			IdxCols: []string{"col3", "col2"},
 			KeyLen:  1,
-			Mapping: []int{3, 2, 0},
+			// Mapping should skip over cardinality
+			Mapping: []int{4, 3, 0},
 		},
 	}
 

go/store/val/tuple_descriptor.go

@@ -15,6 +15,7 @@
 package val
 
 import (
+	"encoding/hex"
 	"fmt"
 	"strconv"
 	"strings"
@@ -538,13 +539,13 @@ func formatValue(enc Encoding, value []byte) string {
 	case StringEnc:
 		return readString(value)
 	case ByteStringEnc:
-		return string(value)
+		return hex.EncodeToString(value)
 	case Hash128Enc:
-		return string(value)
+		return hex.EncodeToString(value)
 	case BytesAddrEnc:
-		return string(value)
+		return hex.EncodeToString(value)
 	case CommitAddrEnc:
-		return string(value)
+		return hex.EncodeToString(value)
 	default:
 		return string(value)
 	}