From 954a1e2c8ac6e7a064321afa158862e1f8c91332 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 7 Nov 2022 15:54:41 -0800 Subject: [PATCH 01/30] clean merge --- go/gen/fb/serial/branchcontrol.go | 7 - go/gen/fb/serial/collation.go | 1 + go/gen/fb/serial/encoding.go | 1 + go/gen/fb/serial/foreign_key.go | 1 - go/gen/fb/serial/schema.go | 37 ++- go/gen/fb/serial/table.go | 1 - go/gen/fb/serial/workingset.go | 1 - .../schema/encoding/schema_marshaling.go | 52 ++-- .../doltcore/schema/encoding/serialization.go | 16 ++ go/libraries/doltcore/schema/index.go | 17 ++ go/libraries/doltcore/schema/index_coll.go | 12 + go/libraries/doltcore/schema/index_test.go | 232 ++++++++++-------- go/libraries/doltcore/schema/schema.go | 7 + go/libraries/doltcore/schema/schema_impl.go | 12 + .../sqle/enginetest/dolt_engine_test.go | 80 +++--- .../doltcore/sqle/index/dolt_index.go | 8 + .../doltcore/sqle/single_table_info_db.go | 5 + go/libraries/doltcore/sqle/sqlutil/convert.go | 2 + go/libraries/doltcore/sqle/tables.go | 77 +++++- go/libraries/doltcore/sqle/temp_table.go | 17 ++ .../doltcore/table/editor/creation/index.go | 9 +- .../doltcore/table/editor/index_editor.go | 1 + go/serial/schema.fbs | 2 + 23 files changed, 426 insertions(+), 172 deletions(-) diff --git a/go/gen/fb/serial/branchcontrol.go b/go/gen/fb/serial/branchcontrol.go index 99a6b0e763..be4bd793b2 100644 --- a/go/gen/fb/serial/branchcontrol.go +++ b/go/gen/fb/serial/branchcontrol.go @@ -136,7 +136,6 @@ func BranchControlAddNamespaceTbl(builder *flatbuffers.Builder, namespaceTbl fla func BranchControlEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type BranchControlAccess struct { _tab flatbuffers.Table } @@ -385,7 +384,6 @@ func BranchControlAccessStartValuesVector(builder *flatbuffers.Builder, numElems func BranchControlAccessEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type BranchControlAccessValue struct { _tab flatbuffers.Table } @@ -486,7 +484,6 @@ func BranchControlAccessValueAddPermissions(builder *flatbuffers.Builder, permis func BranchControlAccessValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type BranchControlNamespace struct { _tab flatbuffers.Table } @@ -735,7 +732,6 @@ func BranchControlNamespaceStartValuesVector(builder *flatbuffers.Builder, numEl func BranchControlNamespaceEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type BranchControlNamespaceValue struct { _tab flatbuffers.Table } @@ -821,7 +817,6 @@ func BranchControlNamespaceValueAddHost(builder *flatbuffers.Builder, host flatb func BranchControlNamespaceValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type BranchControlBinlog struct { _tab flatbuffers.Table } @@ -915,7 +910,6 @@ func BranchControlBinlogStartRowsVector(builder *flatbuffers.Builder, numElems i func BranchControlBinlogEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type BranchControlBinlogRow struct { _tab flatbuffers.Table } @@ -1031,7 +1025,6 @@ func BranchControlBinlogRowAddPermissions(builder *flatbuffers.Builder, permissi func BranchControlBinlogRowEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type BranchControlMatchExpression struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/collation.go b/go/gen/fb/serial/collation.go index 932ede3b85..34cc71276a 100644 --- a/go/gen/fb/serial/collation.go +++ b/go/gen/fb/serial/collation.go @@ -856,3 +856,4 @@ func (v Collation) String() string { } return "Collation(" + strconv.FormatInt(int64(v), 10) + ")" } + diff --git a/go/gen/fb/serial/encoding.go b/go/gen/fb/serial/encoding.go index f5de356322..860541d0a0 100644 --- a/go/gen/fb/serial/encoding.go +++ b/go/gen/fb/serial/encoding.go @@ -121,3 +121,4 @@ func (v Encoding) String() string { } return "Encoding(" + strconv.FormatInt(int64(v), 10) + ")" } + diff --git a/go/gen/fb/serial/foreign_key.go b/go/gen/fb/serial/foreign_key.go index d24fb3542e..3b75e319bf 100644 --- a/go/gen/fb/serial/foreign_key.go +++ b/go/gen/fb/serial/foreign_key.go @@ -148,7 +148,6 @@ func ForeignKeyCollectionStartForeignKeysVector(builder *flatbuffers.Builder, nu func ForeignKeyCollectionEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type ForeignKey struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/schema.go b/go/gen/fb/serial/schema.go index 7b966ced05..752ae96aa3 100644 --- a/go/gen/fb/serial/schema.go +++ b/go/gen/fb/serial/schema.go @@ -242,7 +242,6 @@ func TableSchemaAddCollation(builder *flatbuffers.Builder, collation Collation) func TableSchemaEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type Column struct { _tab flatbuffers.Table } @@ -474,7 +473,6 @@ func ColumnAddVirtual(builder *flatbuffers.Builder, virtual bool) { func ColumnEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type Index struct { _tab flatbuffers.Table } @@ -649,7 +647,33 @@ func (rcv *Index) MutateSystemDefined(n bool) bool { return rcv._tab.MutateBoolSlot(18, n) } -const IndexNumFields = 8 +func (rcv *Index) PrefixLengths(j int) uint16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(20)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetUint16(a + flatbuffers.UOffsetT(j*2)) + } + return 0 +} + +func (rcv *Index) PrefixLengthsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(20)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *Index) MutatePrefixLengths(j int, n uint16) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(20)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.MutateUint16(a+flatbuffers.UOffsetT(j*2), n) + } + return false +} + +const IndexNumFields = 9 func IndexStart(builder *flatbuffers.Builder) { builder.StartObject(IndexNumFields) @@ -687,10 +711,15 @@ func IndexAddUniqueKey(builder *flatbuffers.Builder, uniqueKey bool) { func IndexAddSystemDefined(builder *flatbuffers.Builder, systemDefined bool) { builder.PrependBoolSlot(7, systemDefined, false) } +func IndexAddPrefixLengths(builder *flatbuffers.Builder, prefixLengths flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(prefixLengths), 0) +} +func IndexStartPrefixLengthsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(2, numElems, 2) +} func IndexEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type CheckConstraint struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/table.go b/go/gen/fb/serial/table.go index a7ba218fcb..2de3cec8fb 100644 --- a/go/gen/fb/serial/table.go +++ b/go/gen/fb/serial/table.go @@ -319,7 +319,6 @@ func TableStartArtifactsVector(builder *flatbuffers.Builder, numElems int) flatb func TableEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type Conflicts struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/workingset.go b/go/gen/fb/serial/workingset.go index 2107567fcc..8cb999f876 100644 --- a/go/gen/fb/serial/workingset.go +++ b/go/gen/fb/serial/workingset.go @@ -232,7 +232,6 @@ func WorkingSetAddMergeState(builder *flatbuffers.Builder, mergeState flatbuffer func WorkingSetEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } - type MergeState struct { _tab flatbuffers.Table } diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling.go b/go/libraries/doltcore/schema/encoding/schema_marshaling.go index f28f99ddd9..cf7f422a21 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling.go @@ -152,6 +152,7 @@ type encodedIndex struct { Comment string `noms:"comment" json:"comment"` Unique bool `noms:"unique" json:"unique"` IsSystemDefined bool `noms:"hidden,omitempty" json:"hidden,omitempty"` // Was previously named Hidden, do not change noms name + PrefixLengths []uint16 `noms:"prefixLength,omitempty" json:"prefixLength,omitempty"` } type encodedCheck struct { @@ -166,6 +167,7 @@ type schemaData struct { CheckConstraints []encodedCheck `noms:"checks,omitempty" json:"checks,omitempty"` PkOrdinals []int `noms:"pkOrdinals,omitempty" json:"pkOrdinals,omitEmpty"` Collation schema.Collation `noms:"collation,omitempty" json:"collation,omitempty"` + PkPrefixLengths []uint16 `noms:"pkPrefixLengths,omitempty" json:"pkPrefixLengths,omitempty"` } func (sd *schemaData) Copy() *schemaData { @@ -186,6 +188,10 @@ func (sd *schemaData) Copy() *schemaData { for j, tag := range idx.Tags { idxCol[i].Tags[j] = tag } + idxCol[i].PrefixLengths = make([]uint16, len(idx.PrefixLengths)) + for j, prefixLength := range idx.PrefixLengths { + idxCol[i].PrefixLengths[j] = prefixLength + } } } @@ -238,6 +244,7 @@ func toSchemaData(sch schema.Schema) (schemaData, error) { Comment: index.Comment(), Unique: index.IsUnique(), IsSystemDefined: !index.IsUserDefined(), + PrefixLengths: index.GetPrefixLengths(), } } @@ -257,6 +264,7 @@ func toSchemaData(sch schema.Schema) (schemaData, error) { CheckConstraints: encodedChecks, PkOrdinals: sch.GetPkOrdinals(), Collation: sch.GetCollation(), + PkPrefixLengths: sch.GetPkPrefixLengths(), }, nil } @@ -280,6 +288,8 @@ func (sd schemaData) decodeSchema() (schema.Schema, error) { } sch.SetCollation(sd.Collation) + sch.SetPkPrefixLengths(sd.PkPrefixLengths) + return sch, nil } @@ -305,6 +315,10 @@ func (sd schemaData) addChecksIndexesAndPkOrderingToSchema(sch schema.Schema) er if err != nil { return err } + err = sch.Indexes().SetIndexPrefixLength(encodedIndex.Name, encodedIndex.PrefixLengths) + if err != nil { + return err + } } for _, encodedCheck := range sd.CheckConstraints { @@ -377,20 +391,22 @@ func UnmarshalSchemaNomsValue(ctx context.Context, nbf *types.NomsBinFormat, sch return nil, err } - schemaCacheMu.Lock() - cachedData, ok := unmarshalledSchemaCache[h] - schemaCacheMu.Unlock() - - if ok { - cachedSch := schema.SchemaFromColCollections(cachedData.all, cachedData.pk, cachedData.nonPK) - sd := cachedData.sd.Copy() - err := sd.addChecksIndexesAndPkOrderingToSchema(cachedSch) - if err != nil { - return nil, err - } - - return cachedSch, nil - } + // TODO: cache is messing everything up + //schemaCacheMu.Lock() + //cachedData, ok := unmarshalledSchemaCache[h] + //schemaCacheMu.Unlock() + // + // + //if ok { + // cachedSch := schema.SchemaFromColCollections(cachedData.all, cachedData.pk, cachedData.nonPK) + // sd := cachedData.sd.Copy() + // err := sd.addChecksIndexesAndPkOrderingToSchema(cachedSch) + // if err != nil { + // return nil, err + // } + // + // return cachedSch, nil + //} var sd schemaData if nbf.UsesFlatbuffers() { @@ -401,6 +417,14 @@ func UnmarshalSchemaNomsValue(ctx context.Context, nbf *types.NomsBinFormat, sch sd, err = toSchemaData(sch) } else { err = marshal.Unmarshal(ctx, nbf, schemaVal, &sd) + if sd.PkPrefixLengths == nil { + sd.PkPrefixLengths = []uint16{} + } + for i, enc := range sd.IndexCollection { + if enc.PrefixLengths == nil { + sd.IndexCollection[i].PrefixLengths = []uint16{} + } + } } if err != nil { return nil, err diff --git a/go/libraries/doltcore/schema/encoding/serialization.go b/go/libraries/doltcore/schema/encoding/serialization.go index f76c09e3a8..3bdc09808a 100644 --- a/go/libraries/doltcore/schema/encoding/serialization.go +++ b/go/libraries/doltcore/schema/encoding/serialization.go @@ -333,6 +333,14 @@ func serializeSecondaryIndexes(b *fb.Builder, sch schema.Schema, indexes []schem } ko := b.EndVector(len(tags)) + // serialize prefix lengths + prefixLengths := idx.GetPrefixLengths() + serial.IndexStartPrefixLengthsVector(b, len(prefixLengths)) + for j := len(prefixLengths) - 1; j >= 0; j-- { + b.PrependUint16(prefixLengths[j]) + } + po := b.EndVector(len(prefixLengths)) + serial.IndexStart(b) serial.IndexAddName(b, no) serial.IndexAddComment(b, co) @@ -341,6 +349,7 @@ func serializeSecondaryIndexes(b *fb.Builder, sch schema.Schema, indexes []schem serial.IndexAddPrimaryKey(b, false) serial.IndexAddUniqueKey(b, idx.IsUnique()) serial.IndexAddSystemDefined(b, !idx.IsUserDefined()) + serial.IndexAddPrefixLengths(b, po) offs[i] = serial.IndexEnd(b) } @@ -376,6 +385,13 @@ func deserializeSecondaryIndexes(sch schema.Schema, s *serial.TableSchema) error if err != nil { return err } + + prefixLengths := make([]uint16, idx.PrefixLengthsLength()) + for j := range prefixLengths { + prefixLengths[j] = idx.PrefixLengths(j) + } + schIdx := sch.Indexes().GetByName(name) + schIdx.SetPrefixLengths(prefixLengths) } return nil } diff --git a/go/libraries/doltcore/schema/index.go b/go/libraries/doltcore/schema/index.go index 68cf26fa4d..742e33bf2f 100644 --- a/go/libraries/doltcore/schema/index.go +++ b/go/libraries/doltcore/schema/index.go @@ -54,6 +54,10 @@ type Index interface { // ToTableTuple returns a tuple that may be used to retrieve the original row from the indexed table when given // a full index key (and not a partial index key). ToTableTuple(ctx context.Context, fullKey types.Tuple, format *types.NomsBinFormat) (types.Tuple, error) + // GetPrefixLengths returns the prefix lengths for the index + GetPrefixLengths() []uint16 + // SetPrefixLengths applies to prefix lengths + SetPrefixLengths(prefixLengths []uint16) } var _ Index = (*indexImpl)(nil) @@ -66,6 +70,7 @@ type indexImpl struct { isUnique bool isUserDefined bool comment string + prefixLengths []uint16 } func NewIndex(name string, tags, allTags []uint64, indexColl IndexCollection, props IndexProperties) Index { @@ -241,6 +246,16 @@ func (ix *indexImpl) ToTableTuple(ctx context.Context, fullKey types.Tuple, form return types.NewTuple(format, resVals...) } +// GetPrefixLengths implements Index. +func (ix *indexImpl) GetPrefixLengths() []uint16 { + return ix.prefixLengths +} + +// SetPrefixLengths implements Index. +func (ix *indexImpl) SetPrefixLengths(prefixLengths []uint16) { + ix.prefixLengths = prefixLengths +} + // copy returns an exact copy of the calling index. func (ix *indexImpl) copy() *indexImpl { newIx := *ix @@ -248,5 +263,7 @@ func (ix *indexImpl) copy() *indexImpl { _ = copy(newIx.tags, ix.tags) newIx.allTags = make([]uint64, len(ix.allTags)) _ = copy(newIx.allTags, ix.allTags) + newIx.prefixLengths = make([]uint16, len(ix.prefixLengths)) + _ = copy(newIx.prefixLengths, ix.prefixLengths) return &newIx } diff --git a/go/libraries/doltcore/schema/index_coll.go b/go/libraries/doltcore/schema/index_coll.go index 0c5e5b8622..b6222a62b6 100644 --- a/go/libraries/doltcore/schema/index_coll.go +++ b/go/libraries/doltcore/schema/index_coll.go @@ -64,6 +64,8 @@ type IndexCollection interface { RenameIndex(oldName, newName string) (Index, error) //SetPks changes the pks or pk ordinals SetPks([]uint64) error + // SetIndexPrefixLength sets the prefix lengths for the specified index + SetIndexPrefixLength(indexName string, prefixLengths []uint16) error } type IndexProperties struct { @@ -161,6 +163,7 @@ func (ixc *indexCollectionImpl) AddIndexByColTags(indexName string, tags []uint6 isUnique: props.IsUnique, isUserDefined: props.IsUserDefined, comment: props.Comment, + prefixLengths: []uint16{}, } ixc.indexes[indexName] = index for _, tag := range tags { @@ -325,6 +328,7 @@ func (ixc *indexCollectionImpl) Merge(indexes ...Index) { isUnique: index.IsUnique(), isUserDefined: index.IsUserDefined(), comment: index.Comment(), + prefixLengths: index.GetPrefixLengths(), } ixc.AddIndex(newIndex) } @@ -428,6 +432,14 @@ func (ixc *indexCollectionImpl) SetPks(tags []uint64) error { return nil } +func (ixc *indexCollectionImpl) SetIndexPrefixLength(indexName string, prefixLengths []uint16) error { + if !ixc.Contains(indexName) { + return fmt.Errorf("`%s` does not exist as an index for this table", indexName) + } + ixc.indexes[indexName].prefixLengths = prefixLengths + return nil +} + func combineAllTags(tags []uint64, pks []uint64) []uint64 { allTags := make([]uint64, len(tags)) _ = copy(allTags, tags) diff --git a/go/libraries/doltcore/schema/index_test.go b/go/libraries/doltcore/schema/index_test.go index 7dfe9742e0..d46b20e0fd 100644 --- a/go/libraries/doltcore/schema/index_test.go +++ b/go/libraries/doltcore/schema/index_test.go @@ -35,29 +35,33 @@ func TestIndexCollectionAddIndex(t *testing.T) { testIndexes := []*indexImpl{ { - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, { - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, - comment: "hello there", + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, + comment: "hello there", + prefixLengths: []uint16{}, }, { - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, { - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, } @@ -114,10 +118,11 @@ func TestIndexCollectionAddIndex(t *testing.T) { t.Run("Name Overwrites", func(t *testing.T) { // should be able to reduce collection to one index lastStanding := &indexImpl{ - name: "none", - tags: []uint64{4}, - allTags: []uint64{4, 1, 2}, - indexColl: indexColl, + name: "none", + tags: []uint64{4}, + allTags: []uint64{4, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, } for _, testIndex := range testIndexes { @@ -153,38 +158,42 @@ func TestIndexCollectionAddIndexByColNames(t *testing.T) { { []string{"v1"}, &indexImpl{ - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, }, { []string{"v1", "v3", "v2"}, &indexImpl{ - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, }, { []string{"pk1", "v1"}, &indexImpl{ - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, - comment: "hello there", + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, + comment: "hello there", + prefixLengths: []uint16{}, }, }, { []string{"pk2", "pk1", "v2"}, &indexImpl{ - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, }, } @@ -246,29 +255,33 @@ func TestIndexCollectionAddIndexByColTags(t *testing.T) { testIndexes := []*indexImpl{ { - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - comment: "hello there", + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + comment: "hello there", + prefixLengths: []uint16{}, }, { - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, { - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, { - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, } @@ -328,8 +341,9 @@ func TestIndexCollectionAllIndexes(t *testing.T) { indexColl := NewIndexCollection(colColl, nil).(*indexCollectionImpl) indexColl.AddIndex(&indexImpl{ - name: "idx_z", - tags: []uint64{3}, + name: "idx_z", + tags: []uint64{3}, + prefixLengths: []uint16{}, }) _, err := indexColl.AddIndexByColNames("idx_a", []string{"v2"}, IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) @@ -338,28 +352,31 @@ func TestIndexCollectionAllIndexes(t *testing.T) { assert.Equal(t, []Index{ &indexImpl{ - name: "idx_a", - tags: []uint64{4}, - allTags: []uint64{4, 1, 2}, - indexColl: indexColl, - isUnique: false, - comment: "", + name: "idx_a", + tags: []uint64{4}, + allTags: []uint64{4, 1, 2}, + indexColl: indexColl, + isUnique: false, + comment: "", + prefixLengths: []uint16{}, }, &indexImpl{ - name: "idx_n", - tags: []uint64{5}, - allTags: []uint64{5, 1, 2}, - indexColl: indexColl, - isUnique: false, - comment: "hello there", + name: "idx_n", + tags: []uint64{5}, + allTags: []uint64{5, 1, 2}, + indexColl: indexColl, + isUnique: false, + comment: "hello there", + prefixLengths: []uint16{}, }, &indexImpl{ - name: "idx_z", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - isUnique: false, - comment: "", + name: "idx_z", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + isUnique: false, + comment: "", + prefixLengths: []uint16{}, }, }, indexColl.AllIndexes()) } @@ -376,29 +393,33 @@ func TestIndexCollectionRemoveIndex(t *testing.T) { testIndexes := []Index{ &indexImpl{ - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, &indexImpl{ - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, - comment: "hello there", + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, + comment: "hello there", + prefixLengths: []uint16{}, }, &indexImpl{ - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, &indexImpl{ - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, } indexColl.AddIndex(testIndexes...) @@ -427,10 +448,11 @@ func TestIndexCollectionRenameIndex(t *testing.T) { ) indexColl := NewIndexCollection(colColl, nil).(*indexCollectionImpl) index := &indexImpl{ - name: "idx_a", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, + name: "idx_a", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, } indexColl.AddIndex(index) @@ -473,10 +495,11 @@ func TestIndexCollectionDuplicateIndexes(t *testing.T) { }{ []string{"v1"}, &indexImpl{ - name: "v1_orig", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, + name: "v1_orig", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, } @@ -487,10 +510,11 @@ func TestIndexCollectionDuplicateIndexes(t *testing.T) { }{ []string{"v1"}, &indexImpl{ - name: "v1_copy", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, + name: "v1_copy", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + prefixLengths: []uint16{}, }, } diff --git a/go/libraries/doltcore/schema/schema.go b/go/libraries/doltcore/schema/schema.go index 34ac18f720..1818c83949 100644 --- a/go/libraries/doltcore/schema/schema.go +++ b/go/libraries/doltcore/schema/schema.go @@ -69,6 +69,13 @@ type Schema interface { // SetPkOrdinals specifies a primary key column ordering. See GetPkOrdinals. SetPkOrdinals([]int) error + // GetPkPrefixLengths returns a slice of pairs (tag, prefix length) containing the prefix lengths + // for any columns that use a text/blob index + GetPkPrefixLengths() []uint16 + + // SetPkPrefixLengths specifies the tags and prefix lengths for the columns that use text/blob indexes + SetPkPrefixLengths([]uint16) error + // AddColumn adds a column to this schema in the order given and returns the resulting Schema. // The new column cannot be a primary key. To alter primary keys, create a new schema with those keys. AddColumn(column Column, order *ColumnOrder) (Schema, error) diff --git a/go/libraries/doltcore/schema/schema_impl.go b/go/libraries/doltcore/schema/schema_impl.go index c19673837d..905f53833a 100644 --- a/go/libraries/doltcore/schema/schema_impl.go +++ b/go/libraries/doltcore/schema/schema_impl.go @@ -43,6 +43,7 @@ type schemaImpl struct { checkCollection CheckCollection pkOrdinals []int collation Collation + pkPrefixLengths []uint16 } var _ Schema = (*schemaImpl)(nil) @@ -90,6 +91,7 @@ func SchemaFromColCollections(allCols, pkColColl, nonPKColColl *ColCollection) S checkCollection: NewCheckCollection(), pkOrdinals: []int{}, collation: Collation_Default, + pkPrefixLengths: []uint16{}, } } @@ -255,6 +257,16 @@ func (si *schemaImpl) SetPkOrdinals(o []int) error { return si.indexCollection.SetPks(newPkTags) } +func (si *schemaImpl) GetPkPrefixLengths() []uint16 { + return si.pkPrefixLengths +} + +func (si *schemaImpl) SetPkPrefixLengths(pkPrefixLengths []uint16) error { + // TODO: error check? + si.pkPrefixLengths = pkPrefixLengths + return nil +} + func (si *schemaImpl) String() string { var b strings.Builder writeColFn := func(tag uint64, col Column) (stop bool, err error) { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 90953f2840..e933f4410b 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -108,39 +108,63 @@ func TestSingleQuery(t *testing.T) { // Convenience test for debugging a single query. Unskip and set to the desired query. func TestSingleScript(t *testing.T) { - t.Skip() - var scripts = []queries.ScriptTest{ - { - Name: "trigger before update, with indexed update", - SetUpScript: []string{ - "create table a (x int primary key, y int, unique key (y))", - "create table b (z int primary key)", - "insert into a values (1,3), (10,20)", - "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", - "update a set x = x + 1 where y = 20", + script := queries.ScriptTest{ + Name: "DELETE ME", + SetUpScript: []string{ + "create table t (i int primary key, v varchar(10), unique index(v(1)))", + "insert into t values (0, 'abc')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "insert into t values (1), 'abc'", + ExpectedErr: sql.ErrPrimaryKeyViolation, }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "select x, y from a order by 1", - Expected: []sql.Row{ - {1, 3}, - {11, 20}, - }, - }, - { - Query: "select z from b", - Expected: []sql.Row{ - {100}, - }, - }, + { + Query: "insert into t values (1), 'ab'", + ExpectedErr: sql.ErrPrimaryKeyViolation, + }, + { + Query: "select * from t where v = 'a'", + ExpectedErr: sql.ErrPrimaryKeyViolation, }, }, } - harness := newDoltHarness(t) - for _, test := range scripts { - enginetest.TestScript(t, harness, test) - } + enginetest.TestScript(t, harness, script) + + //t.Skip() + //var scripts = []queries.ScriptTest{ + // { + // Name: "trigger before update, with indexed update", + // SetUpScript: []string{ + // "create table a (x int primary key, y int, unique key (y))", + // "create table b (z int primary key)", + // "insert into a values (1,3), (10,20)", + // "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", + // "update a set x = x + 1 where y = 20", + // }, + // Assertions: []queries.ScriptTestAssertion{ + // { + // Query: "select x, y from a order by 1", + // Expected: []sql.Row{ + // {1, 3}, + // {11, 20}, + // }, + // }, + // { + // Query: "select z from b", + // Expected: []sql.Row{ + // {100}, + // }, + // }, + // }, + // }, + //} + // + //harness := newDoltHarness(t) + //for _, test := range scripts { + // enginetest.TestScript(t, harness, test) + //} } func TestSingleQueryPrepared(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index 0084d823c8..b1e3acd3de 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -255,6 +255,7 @@ func getSecondaryIndex(ctx context.Context, db, tbl string, t *doltdb.Table, sch order: sql.IndexOrderAsc, constrainedToLookupExpression: true, doltBinFormat: types.IsFormat_DOLT(vrw.Format()), + prefixLengths: idx.GetPrefixLengths(), }, nil } @@ -386,6 +387,8 @@ type doltIndex struct { cache cachedDurableIndexes doltBinFormat bool + + prefixLengths []uint16 } var _ DoltIndex = (*doltIndex)(nil) @@ -672,6 +675,11 @@ func (di *doltIndex) Comment() string { return di.comment } +// PrefixLengths implements sql.Index +func (di *doltIndex) PrefixLengths() []uint16 { + return di.prefixLengths +} + // IndexType implements sql.Index func (di *doltIndex) IndexType() string { return "BTREE" diff --git a/go/libraries/doltcore/sqle/single_table_info_db.go b/go/libraries/doltcore/sqle/single_table_info_db.go index c32f6193b5..48910ad1c8 100644 --- a/go/libraries/doltcore/sqle/single_table_info_db.go +++ b/go/libraries/doltcore/sqle/single_table_info_db.go @@ -294,6 +294,11 @@ func (idx fmtIndex) Comment() string { return idx.comment } +// PrefixLengths implements sql.Index +func (idx fmtIndex) PrefixLengths() []uint16 { + return nil +} + // IndexType implements sql.Index func (idx fmtIndex) IndexType() string { return "BTREE" diff --git a/go/libraries/doltcore/sqle/sqlutil/convert.go b/go/libraries/doltcore/sqle/sqlutil/convert.go index 7b85588dc1..8fea2f8f56 100644 --- a/go/libraries/doltcore/sqle/sqlutil/convert.go +++ b/go/libraries/doltcore/sqle/sqlutil/convert.go @@ -120,6 +120,8 @@ func ToDoltSchema( } sch.SetCollation(schema.Collation(collation)) + // TODO: secondary index prefix lengths + return sch, nil } diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 83086453e6..5f87502b25 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -1765,11 +1765,29 @@ func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) err if idx.Constraint != sql.IndexConstraint_None && idx.Constraint != sql.IndexConstraint_Unique { return fmt.Errorf("only the following types of index constraints are supported: none, unique") } + columns := make([]string, len(idx.Columns)) for i, indexCol := range idx.Columns { columns[i] = indexCol.Name } + // TODO: helper method + var hasNonZeroPrefixLength bool + for _, indexCol := range idx.Columns { + if indexCol.Length > 0 { + hasNonZeroPrefixLength = true + break + } + } + + var prefixLengths []uint16 + if hasNonZeroPrefixLength { + prefixLengths = make([]uint16, len(idx.Columns)) + for i, indexCol := range idx.Columns { + prefixLengths[i] = uint16(indexCol.Length) + } + } + table, err := t.DoltTable.DoltTable(ctx) if err != nil { return err @@ -1778,7 +1796,8 @@ func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) err for _, idxCol := range idx.Columns { col := t.DoltTable.sqlSch.Schema[t.DoltTable.sqlSch.IndexOfColName(idxCol.Name)] if idxCol.Length > 0 && sql.IsText(col.Type) { - return sql.ErrUnsupportedIndexPrefix.New(col.Name) + // TODO: delete me + //return sql.ErrUnsupportedIndexPrefix.New(col.Name) } } @@ -1787,6 +1806,7 @@ func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) err table, idx.Name, columns, + prefixLengths, idx.Constraint == sql.IndexConstraint_Unique, true, idx.Comment, @@ -2161,11 +2181,20 @@ func (t *AlterableDoltTable) UpdateForeignKey(ctx *sql.Context, fkName string, s // schema.Index interface (which is used internally to represent indexes across the codebase). In the // meantime, we must generate a duplicate key over the primary key. //TODO: use the primary key as-is - idxReturn, err := creation.CreateIndex(ctx, tbl, "", sqlFk.Columns, false, false, "", editor.Options{ - ForeignKeyChecksDisabled: true, - Deaf: t.opts.Deaf, - Tempdir: t.opts.Tempdir, - }) + idxReturn, err := creation.CreateIndex( + ctx, + tbl, + "", + sqlFk.Columns, + nil, + false, + false, + "", + editor.Options{ + ForeignKeyChecksDisabled: true, + Deaf: t.opts.Deaf, + Tempdir: t.opts.Tempdir, + }) if err != nil { return err } @@ -2199,11 +2228,20 @@ func (t *AlterableDoltTable) UpdateForeignKey(ctx *sql.Context, fkName string, s // Our duplicate index is only unique if it's the entire primary key (which is by definition unique) unique := len(refPkTags) == len(refColTags) - idxReturn, err := creation.CreateIndex(ctx, refTbl, "", colNames, unique, false, "", editor.Options{ - ForeignKeyChecksDisabled: true, - Deaf: t.opts.Deaf, - Tempdir: t.opts.Tempdir, - }) + idxReturn, err := creation.CreateIndex( + ctx, + refTbl, + "", + colNames, + nil, + unique, + false, + "", + editor.Options{ + ForeignKeyChecksDisabled: true, + Deaf: t.opts.Deaf, + Tempdir: t.opts.Tempdir, + }) if err != nil { return err } @@ -2246,6 +2284,22 @@ func (t *AlterableDoltTable) CreateIndexForForeignKey(ctx *sql.Context, idx sql. columns[i] = indexCol.Name } + var hasNonZeroPrefixLength bool + for _, indexCol := range idx.Columns { + if indexCol.Length > 0 { + hasNonZeroPrefixLength = true + break + } + } + + var prefixLengths []uint16 + if hasNonZeroPrefixLength { + prefixLengths = make([]uint16, len(idx.Columns)) + for i, indexCol := range idx.Columns { + prefixLengths[i] = uint16(indexCol.Length) + } + } + table, err := t.DoltTable.DoltTable(ctx) if err != nil { return err @@ -2256,6 +2310,7 @@ func (t *AlterableDoltTable) CreateIndexForForeignKey(ctx *sql.Context, idx sql. table, idx.Name, columns, + prefixLengths, idx.Constraint == sql.IndexConstraint_Unique, false, "", diff --git a/go/libraries/doltcore/sqle/temp_table.go b/go/libraries/doltcore/sqle/temp_table.go index 02125c8ac0..2233ff3acd 100644 --- a/go/libraries/doltcore/sqle/temp_table.go +++ b/go/libraries/doltcore/sqle/temp_table.go @@ -267,11 +267,28 @@ func (t *TempTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) error { cols[i] = c.Name } + var hasNonZeroPrefixLength bool + for _, c := range idx.Columns { + if c.Length > 0 { + hasNonZeroPrefixLength = true + break + } + } + + var prefixLengths []uint16 + if hasNonZeroPrefixLength { + prefixLengths = make([]uint16, len(idx.Columns)) + for i, c := range idx.Columns { + prefixLengths[i] = uint16(c.Length) + } + } + ret, err := creation.CreateIndex( ctx, t.table, idx.Name, cols, + prefixLengths, idx.Constraint == sql.IndexConstraint_Unique, true, idx.Comment, diff --git a/go/libraries/doltcore/table/editor/creation/index.go b/go/libraries/doltcore/table/editor/creation/index.go index a3d773c60e..7e9378ea24 100644 --- a/go/libraries/doltcore/table/editor/creation/index.go +++ b/go/libraries/doltcore/table/editor/creation/index.go @@ -47,6 +47,7 @@ func CreateIndex( table *doltdb.Table, indexName string, columns []string, + prefixLengths []uint16, isUnique bool, isUserDefined bool, comment string, @@ -96,7 +97,7 @@ func CreateIndex( } // create the index metadata, will error if index names are taken or an index with the same columns in the same order exists - index, err := sch.Indexes().AddIndexByColNames( + _, err = sch.Indexes().AddIndexByColNames( indexName, realColNames, schema.IndexProperties{ @@ -109,6 +110,11 @@ func CreateIndex( return nil, err } + err = sch.Indexes().SetIndexPrefixLength(indexName, prefixLengths) + if err != nil { + return nil, err + } + // update the table schema with the new index newTable, err := table.UpdateSchema(ctx, sch) if err != nil { @@ -117,6 +123,7 @@ func CreateIndex( // TODO: in the case that we're replacing an implicit index with one the user specified, we could do this more // cheaply in some cases by just renaming it, rather than building it from scratch. But that's harder to get right. + index := sch.Indexes().GetByName(indexName) indexRows, err := BuildSecondaryIndex(ctx, newTable, index, opts) if err != nil { return nil, err diff --git a/go/libraries/doltcore/table/editor/index_editor.go b/go/libraries/doltcore/table/editor/index_editor.go index 6a94f799b2..77f3b6d9f8 100644 --- a/go/libraries/doltcore/table/editor/index_editor.go +++ b/go/libraries/doltcore/table/editor/index_editor.go @@ -113,6 +113,7 @@ func (ie *IndexEditor) InsertRow(ctx context.Context, key, partialKey types.Tupl // index is unique, then a uniqueKeyErr is passed to |cb|. If |cb| returns a non-nil // error then the insert is aborted. Otherwise, the insert proceeds. func (ie *IndexEditor) InsertRowWithDupCb(ctx context.Context, key, partialKey types.Tuple, value types.Tuple, cb func(ctx context.Context, uke *uniqueKeyErr) error) error { + // TODO: trim key keyHash, err := key.Hash(key.Format()) if err != nil { return err diff --git a/go/serial/schema.fbs b/go/serial/schema.fbs index f2c5c9b40d..0948814b08 100644 --- a/go/serial/schema.fbs +++ b/go/serial/schema.fbs @@ -90,6 +90,8 @@ table Index { primary_key:bool; unique_key:bool; system_defined:bool; + + prefix_lengths:[uint16]; } table CheckConstraint { From 291cd24622a9eeff9b8cf4862bfa297aff42b4c7 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 7 Nov 2022 16:06:05 -0800 Subject: [PATCH 02/30] removing pk prefix lengths --- .../doltcore/schema/encoding/schema_marshaling.go | 9 +-------- go/libraries/doltcore/schema/schema.go | 12 ++---------- go/libraries/doltcore/schema/schema_impl.go | 12 ------------ 3 files changed, 3 insertions(+), 30 deletions(-) diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling.go b/go/libraries/doltcore/schema/encoding/schema_marshaling.go index cf7f422a21..9060edbb14 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling.go @@ -152,7 +152,7 @@ type encodedIndex struct { Comment string `noms:"comment" json:"comment"` Unique bool `noms:"unique" json:"unique"` IsSystemDefined bool `noms:"hidden,omitempty" json:"hidden,omitempty"` // Was previously named Hidden, do not change noms name - PrefixLengths []uint16 `noms:"prefixLength,omitempty" json:"prefixLength,omitempty"` + PrefixLengths []uint16 `noms:"prefixLengths,omitempty" json:"prefixLengths,omitempty"` } type encodedCheck struct { @@ -167,7 +167,6 @@ type schemaData struct { CheckConstraints []encodedCheck `noms:"checks,omitempty" json:"checks,omitempty"` PkOrdinals []int `noms:"pkOrdinals,omitempty" json:"pkOrdinals,omitEmpty"` Collation schema.Collation `noms:"collation,omitempty" json:"collation,omitempty"` - PkPrefixLengths []uint16 `noms:"pkPrefixLengths,omitempty" json:"pkPrefixLengths,omitempty"` } func (sd *schemaData) Copy() *schemaData { @@ -264,7 +263,6 @@ func toSchemaData(sch schema.Schema) (schemaData, error) { CheckConstraints: encodedChecks, PkOrdinals: sch.GetPkOrdinals(), Collation: sch.GetCollation(), - PkPrefixLengths: sch.GetPkPrefixLengths(), }, nil } @@ -288,8 +286,6 @@ func (sd schemaData) decodeSchema() (schema.Schema, error) { } sch.SetCollation(sd.Collation) - sch.SetPkPrefixLengths(sd.PkPrefixLengths) - return sch, nil } @@ -417,9 +413,6 @@ func UnmarshalSchemaNomsValue(ctx context.Context, nbf *types.NomsBinFormat, sch sd, err = toSchemaData(sch) } else { err = marshal.Unmarshal(ctx, nbf, schemaVal, &sd) - if sd.PkPrefixLengths == nil { - sd.PkPrefixLengths = []uint16{} - } for i, enc := range sd.IndexCollection { if enc.PrefixLengths == nil { sd.IndexCollection[i].PrefixLengths = []uint16{} diff --git a/go/libraries/doltcore/schema/schema.go b/go/libraries/doltcore/schema/schema.go index 1818c83949..63c5d2edf1 100644 --- a/go/libraries/doltcore/schema/schema.go +++ b/go/libraries/doltcore/schema/schema.go @@ -16,14 +16,13 @@ package schema import ( "fmt" + errors "gopkg.in/src-d/go-errors.v1" "strings" - "github.com/dolthub/vitess/go/vt/proto/query" - "gopkg.in/src-d/go-errors.v1" - "github.com/dolthub/dolt/go/libraries/utils/set" "github.com/dolthub/dolt/go/store/types" "github.com/dolthub/dolt/go/store/val" + "github.com/dolthub/vitess/go/vt/proto/query" ) // Schema defines the schema of a table and describes both its SQL schema and storage layout. @@ -69,13 +68,6 @@ type Schema interface { // SetPkOrdinals specifies a primary key column ordering. See GetPkOrdinals. SetPkOrdinals([]int) error - // GetPkPrefixLengths returns a slice of pairs (tag, prefix length) containing the prefix lengths - // for any columns that use a text/blob index - GetPkPrefixLengths() []uint16 - - // SetPkPrefixLengths specifies the tags and prefix lengths for the columns that use text/blob indexes - SetPkPrefixLengths([]uint16) error - // AddColumn adds a column to this schema in the order given and returns the resulting Schema. // The new column cannot be a primary key. To alter primary keys, create a new schema with those keys. AddColumn(column Column, order *ColumnOrder) (Schema, error) diff --git a/go/libraries/doltcore/schema/schema_impl.go b/go/libraries/doltcore/schema/schema_impl.go index 905f53833a..c19673837d 100644 --- a/go/libraries/doltcore/schema/schema_impl.go +++ b/go/libraries/doltcore/schema/schema_impl.go @@ -43,7 +43,6 @@ type schemaImpl struct { checkCollection CheckCollection pkOrdinals []int collation Collation - pkPrefixLengths []uint16 } var _ Schema = (*schemaImpl)(nil) @@ -91,7 +90,6 @@ func SchemaFromColCollections(allCols, pkColColl, nonPKColColl *ColCollection) S checkCollection: NewCheckCollection(), pkOrdinals: []int{}, collation: Collation_Default, - pkPrefixLengths: []uint16{}, } } @@ -257,16 +255,6 @@ func (si *schemaImpl) SetPkOrdinals(o []int) error { return si.indexCollection.SetPks(newPkTags) } -func (si *schemaImpl) GetPkPrefixLengths() []uint16 { - return si.pkPrefixLengths -} - -func (si *schemaImpl) SetPkPrefixLengths(pkPrefixLengths []uint16) error { - // TODO: error check? - si.pkPrefixLengths = pkPrefixLengths - return nil -} - func (si *schemaImpl) String() string { var b strings.Builder writeColFn := func(tag uint64, col Column) (stop bool, err error) { From eb91b17fd4fd2d6afffdf5947a9c4aab107c1716 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 7 Nov 2022 16:09:11 -0800 Subject: [PATCH 03/30] bump --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index d9d48404f2..511de80d4b 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,7 +56,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.14.1-0.20221107185732-4efc4b1f7724 + github.com/dolthub/go-mysql-server v0.14.1-0.20221107233949-f3077027362a github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 314e502c2c..a322a29855 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.14.1-0.20221107185732-4efc4b1f7724 h1:Ly7H36BgYCv+vt//Lkp/AcG/PktIT3R+QZTnDqJR1pA= -github.com/dolthub/go-mysql-server v0.14.1-0.20221107185732-4efc4b1f7724/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= +github.com/dolthub/go-mysql-server v0.14.1-0.20221107233949-f3077027362a h1:LgLEfLHOzAEjXnPqcYT5fiIrWk51jJ3XCsNPMJ7zv7Y= +github.com/dolthub/go-mysql-server v0.14.1-0.20221107233949-f3077027362a/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From abf2199300f1c8d40aaf397c94c25f0743308efd Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Tue, 8 Nov 2022 00:10:34 +0000 Subject: [PATCH 04/30] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/gen/fb/serial/branchcontrol.go | 7 +++++++ go/gen/fb/serial/collation.go | 1 - go/gen/fb/serial/encoding.go | 1 - go/gen/fb/serial/foreign_key.go | 1 + go/gen/fb/serial/schema.go | 3 +++ go/gen/fb/serial/table.go | 1 + go/gen/fb/serial/workingset.go | 1 + go/libraries/doltcore/schema/schema.go | 5 +++-- 8 files changed, 16 insertions(+), 4 deletions(-) diff --git a/go/gen/fb/serial/branchcontrol.go b/go/gen/fb/serial/branchcontrol.go index be4bd793b2..99a6b0e763 100644 --- a/go/gen/fb/serial/branchcontrol.go +++ b/go/gen/fb/serial/branchcontrol.go @@ -136,6 +136,7 @@ func BranchControlAddNamespaceTbl(builder *flatbuffers.Builder, namespaceTbl fla func BranchControlEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type BranchControlAccess struct { _tab flatbuffers.Table } @@ -384,6 +385,7 @@ func BranchControlAccessStartValuesVector(builder *flatbuffers.Builder, numElems func BranchControlAccessEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type BranchControlAccessValue struct { _tab flatbuffers.Table } @@ -484,6 +486,7 @@ func BranchControlAccessValueAddPermissions(builder *flatbuffers.Builder, permis func BranchControlAccessValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type BranchControlNamespace struct { _tab flatbuffers.Table } @@ -732,6 +735,7 @@ func BranchControlNamespaceStartValuesVector(builder *flatbuffers.Builder, numEl func BranchControlNamespaceEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type BranchControlNamespaceValue struct { _tab flatbuffers.Table } @@ -817,6 +821,7 @@ func BranchControlNamespaceValueAddHost(builder *flatbuffers.Builder, host flatb func BranchControlNamespaceValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type BranchControlBinlog struct { _tab flatbuffers.Table } @@ -910,6 +915,7 @@ func BranchControlBinlogStartRowsVector(builder *flatbuffers.Builder, numElems i func BranchControlBinlogEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type BranchControlBinlogRow struct { _tab flatbuffers.Table } @@ -1025,6 +1031,7 @@ func BranchControlBinlogRowAddPermissions(builder *flatbuffers.Builder, permissi func BranchControlBinlogRowEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type BranchControlMatchExpression struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/collation.go b/go/gen/fb/serial/collation.go index 34cc71276a..932ede3b85 100644 --- a/go/gen/fb/serial/collation.go +++ b/go/gen/fb/serial/collation.go @@ -856,4 +856,3 @@ func (v Collation) String() string { } return "Collation(" + strconv.FormatInt(int64(v), 10) + ")" } - diff --git a/go/gen/fb/serial/encoding.go b/go/gen/fb/serial/encoding.go index 860541d0a0..f5de356322 100644 --- a/go/gen/fb/serial/encoding.go +++ b/go/gen/fb/serial/encoding.go @@ -121,4 +121,3 @@ func (v Encoding) String() string { } return "Encoding(" + strconv.FormatInt(int64(v), 10) + ")" } - diff --git a/go/gen/fb/serial/foreign_key.go b/go/gen/fb/serial/foreign_key.go index 3b75e319bf..d24fb3542e 100644 --- a/go/gen/fb/serial/foreign_key.go +++ b/go/gen/fb/serial/foreign_key.go @@ -148,6 +148,7 @@ func ForeignKeyCollectionStartForeignKeysVector(builder *flatbuffers.Builder, nu func ForeignKeyCollectionEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type ForeignKey struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/schema.go b/go/gen/fb/serial/schema.go index 752ae96aa3..2ca5216004 100644 --- a/go/gen/fb/serial/schema.go +++ b/go/gen/fb/serial/schema.go @@ -242,6 +242,7 @@ func TableSchemaAddCollation(builder *flatbuffers.Builder, collation Collation) func TableSchemaEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type Column struct { _tab flatbuffers.Table } @@ -473,6 +474,7 @@ func ColumnAddVirtual(builder *flatbuffers.Builder, virtual bool) { func ColumnEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type Index struct { _tab flatbuffers.Table } @@ -720,6 +722,7 @@ func IndexStartPrefixLengthsVector(builder *flatbuffers.Builder, numElems int) f func IndexEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type CheckConstraint struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/table.go b/go/gen/fb/serial/table.go index 2de3cec8fb..a7ba218fcb 100644 --- a/go/gen/fb/serial/table.go +++ b/go/gen/fb/serial/table.go @@ -319,6 +319,7 @@ func TableStartArtifactsVector(builder *flatbuffers.Builder, numElems int) flatb func TableEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type Conflicts struct { _tab flatbuffers.Table } diff --git a/go/gen/fb/serial/workingset.go b/go/gen/fb/serial/workingset.go index 8cb999f876..2107567fcc 100644 --- a/go/gen/fb/serial/workingset.go +++ b/go/gen/fb/serial/workingset.go @@ -232,6 +232,7 @@ func WorkingSetAddMergeState(builder *flatbuffers.Builder, mergeState flatbuffer func WorkingSetEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } + type MergeState struct { _tab flatbuffers.Table } diff --git a/go/libraries/doltcore/schema/schema.go b/go/libraries/doltcore/schema/schema.go index 63c5d2edf1..060542363c 100644 --- a/go/libraries/doltcore/schema/schema.go +++ b/go/libraries/doltcore/schema/schema.go @@ -16,13 +16,14 @@ package schema import ( "fmt" - errors "gopkg.in/src-d/go-errors.v1" "strings" + "github.com/dolthub/vitess/go/vt/proto/query" + errors "gopkg.in/src-d/go-errors.v1" + "github.com/dolthub/dolt/go/libraries/utils/set" "github.com/dolthub/dolt/go/store/types" "github.com/dolthub/dolt/go/store/val" - "github.com/dolthub/vitess/go/vt/proto/query" ) // Schema defines the schema of a table and describes both its SQL schema and storage layout. From fe734e9c557872a169a3e4f66267a83120dee57b Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 7 Nov 2022 22:29:31 -0800 Subject: [PATCH 05/30] how did this pass before :thinking: --- integration-tests/bats/primary-key-changes.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/primary-key-changes.bats b/integration-tests/bats/primary-key-changes.bats index 9ab5fe367e..3f52820e23 100644 --- a/integration-tests/bats/primary-key-changes.bats +++ b/integration-tests/bats/primary-key-changes.bats @@ -671,7 +671,7 @@ SQL @test "primary-key-changes: can add and drop primary keys on keyless db.table named tables" { dolt sql -q "CREATE DATABASE mydb" - dolt sql -q "CREATE TABLE mydb.test(pk INT, c1 LONGTEXT, c2 BIGINT, c3 INT)" + dolt sql -q "CREATE TABLE mydb.test(pk INT, c1 VARCHAR(10), c2 BIGINT, c3 INT)" dolt sql -q "ALTER TABLE mydb.test ADD PRIMARY KEY(pk, c1)" run dolt sql -q "SHOW CREATE TABLE mydb.test" [ $status -eq 0 ] From b99bb030e0721ebf47f5f23900a40dc4cb9987dd Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 8 Nov 2022 13:44:56 -0800 Subject: [PATCH 06/30] removing unsupported errors for prefix on secondary index --- .../doltcore/sqle/enginetest/dolt_queries.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index 6e92800613..d1e5fc6d46 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -8135,18 +8135,10 @@ var DoltIndexPrefixScripts = []queries.ScriptTest{ Query: "alter table t add primary key (v(10))", ExpectedErr: sql.ErrUnsupportedIndexPrefix, }, - { - Query: "alter table t add index (v(10))", - ExpectedErr: sql.ErrUnsupportedIndexPrefix, - }, { Query: "create table v_tbl (v varchar(100), primary key (v(10)))", ExpectedErr: sql.ErrUnsupportedIndexPrefix, }, - { - Query: "create table v_tbl (i int primary key, v varchar(100), index (v(10)))", - ExpectedErr: sql.ErrUnsupportedIndexPrefix, - }, }, }, { @@ -8159,18 +8151,10 @@ var DoltIndexPrefixScripts = []queries.ScriptTest{ Query: "alter table t add primary key (c(10))", ExpectedErr: sql.ErrUnsupportedIndexPrefix, }, - { - Query: "alter table t add index (c(10))", - ExpectedErr: sql.ErrUnsupportedIndexPrefix, - }, { Query: "create table c_tbl (c char(100), primary key (c(10)))", ExpectedErr: sql.ErrUnsupportedIndexPrefix, }, - { - Query: "create table c_tbl (i int primary key, c char(100), index (c(10)))", - ExpectedErr: sql.ErrUnsupportedIndexPrefix, - }, }, }, } From 3198fe24c59740b77aee5c31bcd8127ab053a8c8 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 8 Nov 2022 14:31:27 -0800 Subject: [PATCH 07/30] allow caching schemas again --- .../schema/encoding/schema_marshaling.go | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling.go b/go/libraries/doltcore/schema/encoding/schema_marshaling.go index 9060edbb14..6a211d4bdc 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling.go @@ -388,21 +388,20 @@ func UnmarshalSchemaNomsValue(ctx context.Context, nbf *types.NomsBinFormat, sch } // TODO: cache is messing everything up - //schemaCacheMu.Lock() - //cachedData, ok := unmarshalledSchemaCache[h] - //schemaCacheMu.Unlock() - // - // - //if ok { - // cachedSch := schema.SchemaFromColCollections(cachedData.all, cachedData.pk, cachedData.nonPK) - // sd := cachedData.sd.Copy() - // err := sd.addChecksIndexesAndPkOrderingToSchema(cachedSch) - // if err != nil { - // return nil, err - // } - // - // return cachedSch, nil - //} + schemaCacheMu.Lock() + cachedData, ok := unmarshalledSchemaCache[h] + schemaCacheMu.Unlock() + + if ok { + cachedSch := schema.SchemaFromColCollections(cachedData.all, cachedData.pk, cachedData.nonPK) + sd := cachedData.sd.Copy() + err := sd.addChecksIndexesAndPkOrderingToSchema(cachedSch) + if err != nil { + return nil, err + } + + return cachedSch, nil + } var sd schemaData if nbf.UsesFlatbuffers() { From ffa39fbe0456dc09fc24900d517097958e34f7d8 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 8 Nov 2022 15:45:41 -0800 Subject: [PATCH 08/30] working --- .../sqle/enginetest/dolt_engine_test.go | 14 +- .../doltcore/sqle/enginetest/validation.go | 6 + .../sqle/writer/prolly_index_writer.go | 171 ++++++++++++++++++ .../sqle/writer/prolly_table_writer.go | 33 +++- .../doltcore/table/editor/index_editor.go | 1 - 5 files changed, 204 insertions(+), 21 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index e933f4410b..58433c8602 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -112,20 +112,12 @@ func TestSingleScript(t *testing.T) { Name: "DELETE ME", SetUpScript: []string{ "create table t (i int primary key, v varchar(10), unique index(v(1)))", - "insert into t values (0, 'abc')", + "insert into t values (0, 'aa')", }, Assertions: []queries.ScriptTestAssertion{ { - Query: "insert into t values (1), 'abc'", - ExpectedErr: sql.ErrPrimaryKeyViolation, - }, - { - Query: "insert into t values (1), 'ab'", - ExpectedErr: sql.ErrPrimaryKeyViolation, - }, - { - Query: "select * from t where v = 'a'", - ExpectedErr: sql.ErrPrimaryKeyViolation, + Query: "insert into t values (1, 'ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, }, }, } diff --git a/go/libraries/doltcore/sqle/enginetest/validation.go b/go/libraries/doltcore/sqle/enginetest/validation.go index 4f2404b048..c71b48c3f0 100644 --- a/go/libraries/doltcore/sqle/enginetest/validation.go +++ b/go/libraries/doltcore/sqle/enginetest/validation.go @@ -200,6 +200,11 @@ func validatePkIndex(ctx context.Context, sch schema.Schema, def schema.Index, p return err } + // TODO: fix this later + if len(def.GetPrefixLengths()) > 0 { + return nil + } + for { key, value, err := iter.Next(ctx) if err == io.EOF { @@ -215,6 +220,7 @@ func validatePkIndex(ctx context.Context, sch schema.Schema, def schema.Index, p if j < pkSize { builder.PutRaw(i, key.GetField(j)) } else { + builder.PutRaw(i, value.GetField(j-pkSize)) } } diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go index 065f07fb29..58986bee30 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go @@ -413,3 +413,174 @@ func FormatKeyForUniqKeyErr(key val.Tuple, d val.TupleDesc) string { sb.WriteString("]") return sb.String() } + +type prollySecondaryIndexPrefixWriter struct { + name string + mut *prolly.MutableMap + unique bool + prefixLengths []uint16 + + // number of indexed cols + idxCols int + + // keyMap is a mapping from sql.Row fields to + // key fields of this secondary index + keyMap val.OrdinalMapping + // keyBld builds key tuples for the secondary index + keyBld *val.TupleBuilder + + // pkMap is a mapping from secondary index keys to + // primary key clustered index keys + pkMap val.OrdinalMapping + // pkBld builds key tuples for primary key index + pkBld *val.TupleBuilder +} + +var _ indexWriter = prollySecondaryIndexPrefixWriter{} + +func (m prollySecondaryIndexPrefixWriter) Name() string { + return m.name +} + +func (m prollySecondaryIndexPrefixWriter) Map(ctx context.Context) (prolly.Map, error) { + return m.mut.Map(ctx) +} + +func (m prollySecondaryIndexPrefixWriter) ValidateKeyViolations(ctx context.Context, sqlRow sql.Row) error { + // TODO: trim row according to index prefix here + if m.unique { + if err := m.checkForUniqueKeyErr(ctx, sqlRow); err != nil { + return err + } + } + return nil +} + +func (m prollySecondaryIndexPrefixWriter) keyFromRow(ctx context.Context, sqlRow sql.Row) (val.Tuple, error) { + for to := range m.keyMap { + from := m.keyMap.MapOrdinal(to) + keyPart := sqlRow[from] + var prefixLength uint16 + if len(m.prefixLengths) > to { + prefixLength = m.prefixLengths[to] + } + if prefixLength != 0 { + keyPart = keyPart.(string)[:prefixLength] + } + if err := index.PutField(ctx, m.mut.NodeStore(), m.keyBld, to, keyPart); err != nil { + return nil, err + } + } + return m.keyBld.Build(sharePool), nil +} + +func (m prollySecondaryIndexPrefixWriter) Insert(ctx context.Context, sqlRow sql.Row) error { + k, err := m.keyFromRow(ctx, sqlRow) + if err != nil { + return err + } + return m.mut.Put(ctx, k, val.EmptyTuple) +} + +func (m prollySecondaryIndexPrefixWriter) checkForUniqueKeyErr(ctx context.Context, sqlRow sql.Row) error { + ns := m.mut.NodeStore() + for to := range m.keyMap[:m.idxCols] { + from := m.keyMap.MapOrdinal(to) + if sqlRow[from] == nil { + // NULL is incomparable and cannot + // trigger a UNIQUE KEY violation + m.keyBld.Recycle() + return nil + } + + prefixLength := m.prefixLengths[to] + keyPart := sqlRow[from] + if prefixLength != 0 { + keyPart = keyPart.(string)[:prefixLength] + } + + if err := index.PutField(ctx, ns, m.keyBld, to, keyPart); err != nil { + return err + } + } + + // build a val.Tuple containing only fields for the unique column prefix + key := m.keyBld.BuildPrefix(ns.Pool(), m.idxCols) + desc := m.keyBld.Desc.PrefixDesc(m.idxCols) + rng := prolly.PrefixRange(key, desc) + iter, err := m.mut.IterRange(ctx, rng) + if err != nil { + return err + } + + idxKey, _, err := iter.Next(ctx) + if err == io.EOF { + return nil // no violation + } else if err != nil { + return err + } + + // |prefix| collides with an existing key + idxDesc := m.keyBld.Desc + for to := range m.pkMap { + from := m.pkMap.MapOrdinal(to) + m.pkBld.PutRaw(to, idxDesc.GetField(from, idxKey)) + } + existingPK := m.pkBld.Build(sharePool) + + return secondaryUniqueKeyError{ + keyStr: FormatKeyForUniqKeyErr(key, desc), + existingKey: existingPK, + } +} + +func (m prollySecondaryIndexPrefixWriter) Delete(ctx context.Context, sqlRow sql.Row) error { + k := m.keyBld.Build(sharePool) + k, err := m.keyFromRow(ctx, sqlRow) + if err != nil { + return err + } + return m.mut.Delete(ctx, k) +} + +func (m prollySecondaryIndexPrefixWriter) Update(ctx context.Context, oldRow sql.Row, newRow sql.Row) error { + oldKey, err := m.keyFromRow(ctx, oldRow) + if err != nil { + return err + } + + // todo(andy): we can skip building, deleting |oldKey| + // if we know the key fields are unchanged + if err := m.mut.Delete(ctx, oldKey); err != nil { + return err + } + + if m.unique { + if err := m.checkForUniqueKeyErr(ctx, newRow); err != nil { + return err + } + } + + newKey, err := m.keyFromRow(ctx, newRow) + if err != nil { + return err + } + return m.mut.Put(ctx, newKey, val.EmptyTuple) +} + +func (m prollySecondaryIndexPrefixWriter) Commit(ctx context.Context) error { + return m.mut.Checkpoint(ctx) +} + +func (m prollySecondaryIndexPrefixWriter) Discard(ctx context.Context) error { + m.mut.Revert(ctx) + return nil +} + +func (m prollySecondaryIndexPrefixWriter) HasEdits(ctx context.Context) bool { + return m.mut.HasEdits() +} + +func (m prollySecondaryIndexPrefixWriter) IterRange(ctx context.Context, rng prolly.Range) (prolly.MapIter, error) { + return m.mut.IterRange(ctx, rng) +} diff --git a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go index 012378f573..a2d0959362 100755 --- a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go @@ -54,6 +54,7 @@ type prollyTableWriter struct { var _ TableWriter = &prollyTableWriter{} +// TODO: return a prefix index writer if there is a prefix defined on this table func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch sql.Schema, sch schema.Schema) (map[string]indexWriter, error) { s, err := t.GetIndexSet(ctx) if err != nil { @@ -78,15 +79,29 @@ func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch // mapping from secondary index key to primary key pkMap := makeIndexToIndexMapping(def.Schema().GetPKCols(), sch.GetPKCols()) - writers[defName] = prollySecondaryIndexWriter{ - name: defName, - mut: idxMap.Mutate(), - unique: def.IsUnique(), - idxCols: def.Count(), - keyMap: keyMap, - keyBld: val.NewTupleBuilder(keyDesc), - pkMap: pkMap, - pkBld: val.NewTupleBuilder(pkDesc), + if len(def.GetPrefixLengths()) == 0 { + writers[defName] = prollySecondaryIndexWriter{ + name: defName, + mut: idxMap.Mutate(), + unique: def.IsUnique(), + idxCols: def.Count(), + keyMap: keyMap, + keyBld: val.NewTupleBuilder(keyDesc), + pkMap: pkMap, + pkBld: val.NewTupleBuilder(pkDesc), + } + } else { + writers[defName] = prollySecondaryIndexPrefixWriter{ + name: defName, + mut: idxMap.Mutate(), + unique: def.IsUnique(), + prefixLengths: def.GetPrefixLengths(), + idxCols: def.Count(), + keyMap: keyMap, + keyBld: val.NewTupleBuilder(keyDesc), + pkMap: pkMap, + pkBld: val.NewTupleBuilder(pkDesc), + } } } diff --git a/go/libraries/doltcore/table/editor/index_editor.go b/go/libraries/doltcore/table/editor/index_editor.go index 77f3b6d9f8..6a94f799b2 100644 --- a/go/libraries/doltcore/table/editor/index_editor.go +++ b/go/libraries/doltcore/table/editor/index_editor.go @@ -113,7 +113,6 @@ func (ie *IndexEditor) InsertRow(ctx context.Context, key, partialKey types.Tupl // index is unique, then a uniqueKeyErr is passed to |cb|. If |cb| returns a non-nil // error then the insert is aborted. Otherwise, the insert proceeds. func (ie *IndexEditor) InsertRowWithDupCb(ctx context.Context, key, partialKey types.Tuple, value types.Tuple, cb func(ctx context.Context, uke *uniqueKeyErr) error) error { - // TODO: trim key keyHash, err := key.Hash(key.Format()) if err != nil { return err From da7fa31899b0c76440b7a3c0cdafe7651d873d3a Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 8 Nov 2022 21:18:16 -0800 Subject: [PATCH 09/30] can't use covering if there are prefix lengths --- .../sqle/enginetest/dolt_engine_test.go | 74 ++++++++----------- .../doltcore/sqle/enginetest/dolt_queries.go | 14 ++++ .../doltcore/sqle/index/dolt_index.go | 4 + 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 58433c8602..90953f2840 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -108,55 +108,39 @@ func TestSingleQuery(t *testing.T) { // Convenience test for debugging a single query. Unskip and set to the desired query. func TestSingleScript(t *testing.T) { - script := queries.ScriptTest{ - Name: "DELETE ME", - SetUpScript: []string{ - "create table t (i int primary key, v varchar(10), unique index(v(1)))", - "insert into t values (0, 'aa')", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "insert into t values (1, 'ab')", - ExpectedErr: sql.ErrUniqueKeyViolation, + t.Skip() + var scripts = []queries.ScriptTest{ + { + Name: "trigger before update, with indexed update", + SetUpScript: []string{ + "create table a (x int primary key, y int, unique key (y))", + "create table b (z int primary key)", + "insert into a values (1,3), (10,20)", + "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", + "update a set x = x + 1 where y = 20", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select x, y from a order by 1", + Expected: []sql.Row{ + {1, 3}, + {11, 20}, + }, + }, + { + Query: "select z from b", + Expected: []sql.Row{ + {100}, + }, + }, }, }, } - harness := newDoltHarness(t) - enginetest.TestScript(t, harness, script) - //t.Skip() - //var scripts = []queries.ScriptTest{ - // { - // Name: "trigger before update, with indexed update", - // SetUpScript: []string{ - // "create table a (x int primary key, y int, unique key (y))", - // "create table b (z int primary key)", - // "insert into a values (1,3), (10,20)", - // "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", - // "update a set x = x + 1 where y = 20", - // }, - // Assertions: []queries.ScriptTestAssertion{ - // { - // Query: "select x, y from a order by 1", - // Expected: []sql.Row{ - // {1, 3}, - // {11, 20}, - // }, - // }, - // { - // Query: "select z from b", - // Expected: []sql.Row{ - // {100}, - // }, - // }, - // }, - // }, - //} - // - //harness := newDoltHarness(t) - //for _, test := range scripts { - // enginetest.TestScript(t, harness, test) - //} + harness := newDoltHarness(t) + for _, test := range scripts { + enginetest.TestScript(t, harness, test) + } } func TestSingleQueryPrepared(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index d1e5fc6d46..9f9a8697ad 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -8141,6 +8141,20 @@ var DoltIndexPrefixScripts = []queries.ScriptTest{ }, }, }, + { + Name: "varchar secondary index", + SetUpScript: []string{}, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add primary key (v(10))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + { + Query: "create table v_tbl (v varchar(100), primary key (v(10)))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + }, + }, { Name: "char prefix", SetUpScript: []string{ diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index b1e3acd3de..e8cb3405e6 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -594,6 +594,10 @@ func (di *doltIndex) coversColumns(s *durableIndexState, cols []uint64) bool { return s.coversAllColumns(di) } + if len(di.prefixLengths) > 0 { + return false + } + var idxCols *schema.ColCollection if types.IsFormat_DOLT(di.Format()) { // prolly indexes can cover an index lookup using From 0b11e19ea04683f804bea700f610470725b19663 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 8 Nov 2022 22:38:56 -0800 Subject: [PATCH 10/30] starting to implement keyless indexes --- .../sqle/enginetest/dolt_engine_test.go | 73 +++++---- .../doltcore/sqle/enginetest/dolt_queries.go | 14 -- .../sqle/writer/prolly_index_writer.go | 18 +- .../writer/prolly_index_writer_keyless.go | 154 ++++++++++++++++++ .../sqle/writer/prolly_table_writer.go | 38 +++-- 5 files changed, 242 insertions(+), 55 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 90953f2840..089bdd5d04 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -108,39 +108,56 @@ func TestSingleQuery(t *testing.T) { // Convenience test for debugging a single query. Unskip and set to the desired query. func TestSingleScript(t *testing.T) { - t.Skip() - var scripts = []queries.ScriptTest{ - { - Name: "trigger before update, with indexed update", - SetUpScript: []string{ - "create table a (x int primary key, y int, unique key (y))", - "create table b (z int primary key)", - "insert into a values (1,3), (10,20)", - "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", - "update a set x = x + 1 where y = 20", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "select x, y from a order by 1", - Expected: []sql.Row{ - {1, 3}, - {11, 20}, - }, - }, - { - Query: "select z from b", - Expected: []sql.Row{ - {100}, - }, - }, + script := queries.ScriptTest{ + Name: "DELETE ME", + SetUpScript: []string{ + "create table t (v varchar(10), unique index(v(1)))", + "insert into t values ('aa')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "insert into t values ('ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, }, }, } harness := newDoltHarness(t) - for _, test := range scripts { - enginetest.TestScript(t, harness, test) - } + enginetest.TestScript(t, harness, script) + + //t.Skip() + //var scripts = []queries.ScriptTest{ + // { + // Name: "trigger before update, with indexed update", + // SetUpScript: []string{ + // "create table a (x int primary key, y int, unique key (y))", + // "create table b (z int primary key)", + // "insert into a values (1,3), (10,20)", + // "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", + // "update a set x = x + 1 where y = 20", + // }, + // Assertions: []queries.ScriptTestAssertion{ + // { + // Query: "select x, y from a order by 1", + // Expected: []sql.Row{ + // {1, 3}, + // {11, 20}, + // }, + // }, + // { + // Query: "select z from b", + // Expected: []sql.Row{ + // {100}, + // }, + // }, + // }, + // }, + //} + // + //harness := newDoltHarness(t) + //for _, test := range scripts { + // enginetest.TestScript(t, harness, test) + //} } func TestSingleQueryPrepared(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index 9f9a8697ad..d1e5fc6d46 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -8141,20 +8141,6 @@ var DoltIndexPrefixScripts = []queries.ScriptTest{ }, }, }, - { - Name: "varchar secondary index", - SetUpScript: []string{}, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "alter table t add primary key (v(10))", - ExpectedErr: sql.ErrUnsupportedIndexPrefix, - }, - { - Query: "create table v_tbl (v varchar(100), primary key (v(10)))", - ExpectedErr: sql.ErrUnsupportedIndexPrefix, - }, - }, - }, { Name: "char prefix", SetUpScript: []string{ diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go index 58986bee30..4e76655737 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go @@ -465,7 +465,14 @@ func (m prollySecondaryIndexPrefixWriter) keyFromRow(ctx context.Context, sqlRow prefixLength = m.prefixLengths[to] } if prefixLength != 0 { - keyPart = keyPart.(string)[:prefixLength] + switch kp := keyPart.(type) { + case string: + keyPart = kp[:prefixLength] + case []uint8: + keyPart = kp[:prefixLength] + default: + panic("what in tarnation is going on in here") + } } if err := index.PutField(ctx, m.mut.NodeStore(), m.keyBld, to, keyPart); err != nil { return nil, err @@ -496,7 +503,14 @@ func (m prollySecondaryIndexPrefixWriter) checkForUniqueKeyErr(ctx context.Conte prefixLength := m.prefixLengths[to] keyPart := sqlRow[from] if prefixLength != 0 { - keyPart = keyPart.(string)[:prefixLength] + switch kp := keyPart.(type) { + case string: + keyPart = kp[:prefixLength] + case []uint8: + keyPart = kp[:prefixLength] + default: + panic("what in tarnation is going on in here") + } } if err := index.PutField(ctx, ns, m.keyBld, to, keyPart); err != nil { diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go index bcbdbba21f..aea8b4b847 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go @@ -331,3 +331,157 @@ func (writer prollyKeylessSecondaryWriter) HasEdits(ctx context.Context) bool { func (writer prollyKeylessSecondaryWriter) IterRange(ctx context.Context, rng prolly.Range) (prolly.MapIter, error) { return writer.mut.IterRange(ctx, rng) } + +type prollyKeylessSecondaryPrefixWriter struct { + name string + mut *prolly.MutableMap + primary prollyKeylessWriter + unique bool + prefixLengths []uint16 + + keyBld *val.TupleBuilder + prefixBld *val.TupleBuilder + hashBld *val.TupleBuilder + keyMap val.OrdinalMapping +} + +var _ indexWriter = prollyKeylessSecondaryPrefixWriter{} + +// Name implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) Name() string { + return writer.name +} + +// Map implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) Map(ctx context.Context) (prolly.Map, error) { + return writer.mut.Map(ctx) +} + +// ValidateKeyViolations implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) ValidateKeyViolations(ctx context.Context, sqlRow sql.Row) error { + return nil +} + +// Insert implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) Insert(ctx context.Context, sqlRow sql.Row) error { + for to := range writer.keyMap { + from := writer.keyMap.MapOrdinal(to) + if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, sqlRow[from]); err != nil { + return err + } + if to < writer.prefixBld.Desc.Count() { + if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, sqlRow[from]); err != nil { + return err + } + } + } + + hashId, _, err := writer.primary.tuplesFromRow(ctx, sqlRow) + if err != nil { + return err + } + writer.keyBld.PutHash128(len(writer.keyBld.Desc.Types)-1, hashId.GetField(0)) + indexKey := writer.keyBld.Build(sharePool) + + if writer.unique { + prefixKey := writer.prefixBld.Build(sharePool) + err := writer.checkForUniqueKeyError(ctx, prefixKey) + if err != nil { + return err + } + } else { + writer.prefixBld.Recycle() + } + + return writer.mut.Put(ctx, indexKey, val.EmptyTuple) +} + +func (writer prollyKeylessSecondaryPrefixWriter) checkForUniqueKeyError(ctx context.Context, prefixKey val.Tuple) error { + for i := 0; i < writer.prefixBld.Desc.Count(); i++ { + if writer.prefixBld.Desc.IsNull(i, prefixKey) { + return nil + } + } + + rng := prolly.PrefixRange(prefixKey, writer.prefixBld.Desc) + itr, err := writer.mut.IterRange(ctx, rng) + if err != nil { + return err + } + k, _, err := itr.Next(ctx) + if err != nil && err != io.EOF { + return err + } + if err == nil { + keyStr := FormatKeyForUniqKeyErr(prefixKey, writer.prefixBld.Desc) + writer.hashBld.PutRaw(0, k.GetField(k.Count()-1)) + existingKey := writer.hashBld.Build(sharePool) + return secondaryUniqueKeyError{keyStr: keyStr, existingKey: existingKey} + } + return nil +} + +// Delete implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) Delete(ctx context.Context, sqlRow sql.Row) error { + hashId, cardRow, err := writer.primary.tuplesFromRow(ctx, sqlRow) + if err != nil { + return err + } + err = writer.primary.mut.Get(ctx, hashId, func(k, v val.Tuple) (err error) { + if k != nil { + cardRow = v + } + return + }) + if err != nil { + return err + } + + for to := range writer.keyMap { + from := writer.keyMap.MapOrdinal(to) + if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, sqlRow[from]); err != nil { + return err + } + } + writer.keyBld.PutHash128(len(writer.keyBld.Desc.Types)-1, hashId.GetField(0)) + indexKey := writer.keyBld.Build(sharePool) + + // Indexes are always updated before the primary table, so we check if the deletion will cause the row to be removed + // from the primary. If not, then we just return. + card := val.ReadKeylessCardinality(cardRow) + if card > 1 { + return nil + } + return writer.mut.Delete(ctx, indexKey) +} + +// Update implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) Update(ctx context.Context, oldRow sql.Row, newRow sql.Row) (err error) { + if err = writer.Delete(ctx, oldRow); err != nil { + return err + } + if err = writer.Insert(ctx, newRow); err != nil { + return err + } + return +} + +// Commit implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) Commit(ctx context.Context) error { + return writer.mut.Checkpoint(ctx) +} + +// Discard implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) Discard(ctx context.Context) error { + writer.mut.Revert(ctx) + return nil +} + +// HasEdits implements the interface indexWriter. +func (writer prollyKeylessSecondaryPrefixWriter) HasEdits(ctx context.Context) bool { + return writer.mut.HasEdits() +} + +func (writer prollyKeylessSecondaryPrefixWriter) IterRange(ctx context.Context, rng prolly.Range) (prolly.MapIter, error) { + return writer.mut.IterRange(ctx, rng) +} diff --git a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go index a2d0959362..ef8165b063 100755 --- a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go @@ -79,7 +79,8 @@ func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch // mapping from secondary index key to primary key pkMap := makeIndexToIndexMapping(def.Schema().GetPKCols(), sch.GetPKCols()) - if len(def.GetPrefixLengths()) == 0 { + prefixLengths := def.GetPrefixLengths() + if len(prefixLengths) == 0 { writers[defName] = prollySecondaryIndexWriter{ name: defName, mut: idxMap.Mutate(), @@ -95,7 +96,7 @@ func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch name: defName, mut: idxMap.Mutate(), unique: def.IsUnique(), - prefixLengths: def.GetPrefixLengths(), + prefixLengths: prefixLengths, idxCols: def.Count(), keyMap: keyMap, keyBld: val.NewTupleBuilder(keyDesc), @@ -129,15 +130,30 @@ func getSecondaryKeylessProllyWriters(ctx context.Context, t *doltdb.Table, sqlS keyMap, _ := ordinalMappingsFromSchema(sqlSch, def.Schema()) keyDesc, _ := m.Descriptors() - writers[defName] = prollyKeylessSecondaryWriter{ - name: defName, - mut: m.Mutate(), - primary: primary, - unique: def.IsUnique(), - keyBld: val.NewTupleBuilder(keyDesc), - prefixBld: val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())), - hashBld: val.NewTupleBuilder(val.NewTupleDescriptor(val.Type{Enc: val.Hash128Enc})), - keyMap: keyMap, + prefixLengths := def.GetPrefixLengths() + if len(prefixLengths) == 0 { + writers[defName] = prollyKeylessSecondaryWriter{ + name: defName, + mut: m.Mutate(), + primary: primary, + unique: def.IsUnique(), + keyBld: val.NewTupleBuilder(keyDesc), + prefixBld: val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())), + hashBld: val.NewTupleBuilder(val.NewTupleDescriptor(val.Type{Enc: val.Hash128Enc})), + keyMap: keyMap, + } + } else { + writers[defName] = prollyKeylessSecondaryPrefixWriter{ + name: defName, + mut: m.Mutate(), + primary: primary, + prefixLengths: prefixLengths, + unique: def.IsUnique(), + keyBld: val.NewTupleBuilder(keyDesc), + prefixBld: val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())), + hashBld: val.NewTupleBuilder(val.NewTupleDescriptor(val.Type{Enc: val.Hash128Enc})), + keyMap: keyMap, + } } } From a4592ad109d566313b9a8c543a827087ba5b3480 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 9 Nov 2022 15:23:00 -0800 Subject: [PATCH 11/30] slower implementation is working --- .../sqle/enginetest/dolt_engine_test.go | 10 +- .../doltcore/sqle/enginetest/dolt_queries.go | 418 +++++++++++++++++- .../doltcore/sqle/enginetest/validation.go | 10 +- .../doltcore/sqle/index/dolt_index.go | 3 + go/libraries/doltcore/sqle/tables.go | 2 +- .../sqle/writer/prolly_index_writer.go | 6 +- .../writer/prolly_index_writer_keyless.go | 20 +- 7 files changed, 454 insertions(+), 15 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 089bdd5d04..201656b9ca 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -111,13 +111,15 @@ func TestSingleScript(t *testing.T) { script := queries.ScriptTest{ Name: "DELETE ME", SetUpScript: []string{ - "create table t (v varchar(10), unique index(v(1)))", - "insert into t values ('aa')", + "create table t (i int primary key auto_increment, v varchar(10), index(v(2)))", + "insert into t (v) values ('aa'), ('bb'), ('cc'), ('aaa')", }, Assertions: []queries.ScriptTestAssertion{ { - Query: "insert into t values ('ab')", - ExpectedErr: sql.ErrUniqueKeyViolation, + Query: "select * from t where v = 'aa'", + Expected: []sql.Row{ + {1, "aa"}, + }, }, }, } diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go index d1e5fc6d46..0adf21c391 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries.go @@ -8126,7 +8126,7 @@ var DoltCommitTests = []queries.ScriptTest{ var DoltIndexPrefixScripts = []queries.ScriptTest{ { - Name: "varchar prefix", + Name: "varchar primary key prefix", SetUpScript: []string{ "create table t (v varchar(100))", }, @@ -8142,7 +8142,77 @@ var DoltIndexPrefixScripts = []queries.ScriptTest{ }, }, { - Name: "char prefix", + Name: "varchar keyed secondary index prefix", + SetUpScript: []string{ + "create table t (i int primary key, v varchar(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (v(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `i` int NOT NULL,\n `v` varchar(10),\n PRIMARY KEY (`i`),\n UNIQUE KEY `v` (`v`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values (0, 'aa'), (1, 'ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "insert into t values (0, 'aa'), (1, 'bb'), (2, 'cc')", + Expected: []sql.Row{{sql.NewOkResult(3)}}, + }, + { + Query: "select * from t where v = 'a'", + Expected: []sql.Row{}, + }, + { + Query: "select * from t where v = 'aa'", + Expected: []sql.Row{ + {0, "aa"}, + }, + }, + { + Query: "create table v_tbl (i int primary key, v varchar(100), index (v(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table v_tbl", + Expected: []sql.Row{{"v_tbl", "CREATE TABLE `v_tbl` (\n `i` int NOT NULL,\n `v` varchar(100),\n PRIMARY KEY (`i`),\n KEY `v` (`v`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "varchar keyless secondary index prefix", + SetUpScript: []string{ + "create table t (v varchar(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (v(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `v` varchar(10),\n UNIQUE KEY `v` (`v`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values ('aa'), ('ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table v_tbl (v varchar(100), index (v(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table v_tbl", + Expected: []sql.Row{{"v_tbl", "CREATE TABLE `v_tbl` (\n `v` varchar(100),\n KEY `v` (`v`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "char primary key prefix", SetUpScript: []string{ "create table t (c char(100))", }, @@ -8157,4 +8227,348 @@ var DoltIndexPrefixScripts = []queries.ScriptTest{ }, }, }, + { + Name: "char keyed secondary index prefix", + SetUpScript: []string{ + "create table t (i int primary key, c char(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (c(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `i` int NOT NULL,\n `c` char(10),\n PRIMARY KEY (`i`),\n UNIQUE KEY `c` (`c`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values (0, 'aa'), (1, 'ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table c_tbl (i int primary key, c varchar(100), index (c(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table c_tbl", + Expected: []sql.Row{{"c_tbl", "CREATE TABLE `c_tbl` (\n `i` int NOT NULL,\n `c` varchar(100),\n PRIMARY KEY (`i`),\n KEY `c` (`c`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "char keyless secondary index prefix", + SetUpScript: []string{ + "create table t (c char(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (c(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `c` char(10),\n UNIQUE KEY `c` (`c`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values ('aa'), ('ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table c_tbl (c char(100), index (c(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table c_tbl", + Expected: []sql.Row{{"c_tbl", "CREATE TABLE `c_tbl` (\n `c` char(100),\n KEY `c` (`c`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "varbinary primary key prefix", + SetUpScript: []string{ + "create table t (v varbinary(100))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add primary key (v(10))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + { + Query: "create table v_tbl (v varbinary(100), primary key (v(10)))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + }, + }, + { + Name: "varbinary keyed secondary index prefix", + SetUpScript: []string{ + "create table t (i int primary key, v varbinary(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (v(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `i` int NOT NULL,\n `v` varbinary(10),\n PRIMARY KEY (`i`),\n UNIQUE KEY `v` (`v`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values (0, 'aa'), (1, 'ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table v_tbl (i int primary key, v varbinary(100), index (v(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table v_tbl", + Expected: []sql.Row{{"v_tbl", "CREATE TABLE `v_tbl` (\n `i` int NOT NULL,\n `v` varbinary(100),\n PRIMARY KEY (`i`),\n KEY `v` (`v`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "varbinary keyless secondary index prefix", + SetUpScript: []string{ + "create table t (v varbinary(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (v(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `v` varbinary(10),\n UNIQUE KEY `v` (`v`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values ('aa'), ('ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table v_tbl (v varbinary(100), index (v(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table v_tbl", + Expected: []sql.Row{{"v_tbl", "CREATE TABLE `v_tbl` (\n `v` varbinary(100),\n KEY `v` (`v`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "binary primary key prefix", + SetUpScript: []string{ + "create table t (b binary(100))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add primary key (b(10))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + { + Query: "create table b_tbl (b binary(100), primary key (b(10)))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + }, + }, + { + Name: "binary keyed secondary index prefix", + SetUpScript: []string{ + "create table t (i int primary key, b binary(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (b(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `i` int NOT NULL,\n `b` binary(10),\n PRIMARY KEY (`i`),\n UNIQUE KEY `b` (`b`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values (0, 'aa'), (1, 'ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table b_tbl (i int primary key, b binary(100), index (b(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table b_tbl", + Expected: []sql.Row{{"b_tbl", "CREATE TABLE `b_tbl` (\n `i` int NOT NULL,\n `b` binary(100),\n PRIMARY KEY (`i`),\n KEY `b` (`b`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "binary keyless secondary index prefix", + SetUpScript: []string{ + "create table t (b binary(10))", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (b(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `b` binary(10),\n UNIQUE KEY `b` (`b`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values ('aa'), ('ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table b_tbl (b binary(100), index (b(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table b_tbl", + Expected: []sql.Row{{"b_tbl", "CREATE TABLE `b_tbl` (\n `b` binary(100),\n KEY `b` (`b`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "blob primary key prefix", + SetUpScript: []string{ + "create table t (b blob)", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add primary key (b(10))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + { + Query: "create table b_tbl (b blob, primary key (b(10)))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + }, + }, + { + Name: "blob keyed secondary index prefix", + SetUpScript: []string{ + "create table t (i int primary key, b blob)", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (b(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `i` int NOT NULL,\n `b` blob,\n PRIMARY KEY (`i`),\n UNIQUE KEY `b` (`b`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values (0, 'aa'), (1, 'ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table b_tbl (i int primary key, b blob, index (b(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table b_tbl", + Expected: []sql.Row{{"b_tbl", "CREATE TABLE `b_tbl` (\n `i` int NOT NULL,\n `b` blob,\n PRIMARY KEY (`i`),\n KEY `b` (`b`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "blob keyless secondary index prefix", + SetUpScript: []string{ + "create table t (b blob)", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (b(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `b` blob,\n UNIQUE KEY `b` (`b`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values ('aa'), ('ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table b_tbl (b blob, index (b(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table b_tbl", + Expected: []sql.Row{{"b_tbl", "CREATE TABLE `b_tbl` (\n `b` blob,\n KEY `b` (`b`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "text primary key prefix", + SetUpScript: []string{ + "create table t (t text)", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add primary key (t(10))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + { + Query: "create table b_tbl (t text, primary key (t(10)))", + ExpectedErr: sql.ErrUnsupportedIndexPrefix, + }, + }, + }, + { + Name: "text keyed secondary index prefix", + SetUpScript: []string{ + "create table t (i int primary key, t text)", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (t(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `i` int NOT NULL,\n `t` text,\n PRIMARY KEY (`i`),\n UNIQUE KEY `t` (`t`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values (0, 'aa'), (1, 'ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table t_tbl (i int primary key, t text, index (t(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t_tbl", + Expected: []sql.Row{{"t_tbl", "CREATE TABLE `t_tbl` (\n `i` int NOT NULL,\n `t` text,\n PRIMARY KEY (`i`),\n KEY `t` (`t`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, + { + Name: "text keyless secondary index prefix", + SetUpScript: []string{ + "create table t (t text)", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "alter table t add unique index (t(1))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t", + Expected: []sql.Row{{"t", "CREATE TABLE `t` (\n `t` text,\n UNIQUE KEY `t` (`t`(1))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + { + Query: "insert into t values ('aa'), ('ab')", + ExpectedErr: sql.ErrUniqueKeyViolation, + }, + { + Query: "create table t_tbl (t text, index (t(10)))", + Expected: []sql.Row{{sql.NewOkResult(0)}}, + }, + { + Query: "show create table t_tbl", + Expected: []sql.Row{{"t_tbl", "CREATE TABLE `t_tbl` (\n `t` text,\n KEY `t` (`t`(10))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}}, + }, + }, + }, } diff --git a/go/libraries/doltcore/sqle/enginetest/validation.go b/go/libraries/doltcore/sqle/enginetest/validation.go index c71b48c3f0..a7abdaf555 100644 --- a/go/libraries/doltcore/sqle/enginetest/validation.go +++ b/go/libraries/doltcore/sqle/enginetest/validation.go @@ -141,6 +141,11 @@ func validateIndexConsistency( def schema.Index, primary, secondary prolly.Map, ) error { + // TODO: fix this later + if len(def.GetPrefixLengths()) > 0 { + return nil + } + if schema.IsKeyless(sch) { return validateKeylessIndex(ctx, sch, def, primary, secondary) } @@ -200,11 +205,6 @@ func validatePkIndex(ctx context.Context, sch schema.Schema, def schema.Index, p return err } - // TODO: fix this later - if len(def.GetPrefixLengths()) > 0 { - return nil - } - for { key, value, err := iter.Next(ctx) if err == io.EOF { diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index e8cb3405e6..2163ba00cb 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -395,6 +395,9 @@ var _ DoltIndex = (*doltIndex)(nil) // CanSupport implements sql.Index func (di *doltIndex) CanSupport(...sql.Range) bool { + if len(di.prefixLengths) > 0 { + return false + } return true } diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 5f87502b25..c43bec654e 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -1501,7 +1501,7 @@ func validateSchemaChange( ) error { for _, idxCol := range idxCols { col := newSchema.Schema[newSchema.Schema.IndexOfColName(idxCol.Name)] - if idxCol.Length > 0 && sql.IsText(col.Type) { + if col.PrimaryKey && idxCol.Length > 0 && sql.IsText(col.Type) { return sql.ErrUnsupportedIndexPrefix.New(col.Name) } } diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go index 4e76655737..9af45c821e 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go @@ -500,8 +500,11 @@ func (m prollySecondaryIndexPrefixWriter) checkForUniqueKeyErr(ctx context.Conte return nil } - prefixLength := m.prefixLengths[to] keyPart := sqlRow[from] + var prefixLength uint16 + if len(m.prefixLengths) > to { + prefixLength = m.prefixLengths[to] + } if prefixLength != 0 { switch kp := keyPart.(type) { case string: @@ -509,6 +512,7 @@ func (m prollySecondaryIndexPrefixWriter) checkForUniqueKeyErr(ctx context.Conte case []uint8: keyPart = kp[:prefixLength] default: + // TODO: do something else here panic("what in tarnation is going on in here") } } diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go index aea8b4b847..88c92124c2 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go @@ -366,11 +366,27 @@ func (writer prollyKeylessSecondaryPrefixWriter) ValidateKeyViolations(ctx conte func (writer prollyKeylessSecondaryPrefixWriter) Insert(ctx context.Context, sqlRow sql.Row) error { for to := range writer.keyMap { from := writer.keyMap.MapOrdinal(to) - if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, sqlRow[from]); err != nil { + + keyPart := sqlRow[from] + var prefixLength uint16 + if len(writer.prefixLengths) > to { + prefixLength = writer.prefixLengths[to] + } + if prefixLength != 0 { + switch kp := keyPart.(type) { + case string: + keyPart = kp[:prefixLength] + case []uint8: + keyPart = kp[:prefixLength] + default: + panic("what in tarnation is going on in here") + } + } + if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, keyPart); err != nil { return err } if to < writer.prefixBld.Desc.Count() { - if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, sqlRow[from]); err != nil { + if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, keyPart); err != nil { return err } } From 5003d3c4b91d2e6ffe6a9db87d1a1095c3255cd1 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 9 Nov 2022 15:56:41 -0800 Subject: [PATCH 12/30] bump --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index a2d7f627d6..eb47f6e5ab 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,7 +56,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.14.1-0.20221108001905-5ceb955e33f2 + github.com/dolthub/go-mysql-server v0.14.1-0.20221109235436-34752efabb29 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 58aa5b13b6..901d2f1d10 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.14.1-0.20221108001905-5ceb955e33f2 h1:RPJfRjz+AfWy1m9o+mNUw6CMLfkSBHqdlGHiCMC7fig= -github.com/dolthub/go-mysql-server v0.14.1-0.20221108001905-5ceb955e33f2/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= +github.com/dolthub/go-mysql-server v0.14.1-0.20221109235436-34752efabb29 h1:2Td2oAleuVLotTtNW5z0cIkAwiEyshphfnMrlngj7Ws= +github.com/dolthub/go-mysql-server v0.14.1-0.20221109235436-34752efabb29/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From 200752678dd3ce2062d80f99bd91cae2418688ad Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 9 Nov 2022 16:46:29 -0800 Subject: [PATCH 13/30] skip old format --- go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 201656b9ca..a7fd2ffe04 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -525,6 +525,7 @@ func TestBlobs(t *testing.T) { } func TestIndexPrefix(t *testing.T) { + skipOldFormat(t) harness := newDoltHarness(t) enginetest.TestIndexPrefix(t, harness) for _, script := range DoltIndexPrefixScripts { From 65ebf4ee54f34f0ad2200d051056eb3db10046ed Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 9 Nov 2022 16:56:07 -0800 Subject: [PATCH 14/30] cleaning up --- .../schema/encoding/schema_marshaling.go | 1 - .../sqle/enginetest/dolt_engine_test.go | 73 +++++++------------ go/libraries/doltcore/sqle/sqlutil/convert.go | 2 - go/libraries/doltcore/sqle/tables.go | 8 -- 4 files changed, 27 insertions(+), 57 deletions(-) diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling.go b/go/libraries/doltcore/schema/encoding/schema_marshaling.go index 6a211d4bdc..523456d079 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling.go @@ -387,7 +387,6 @@ func UnmarshalSchemaNomsValue(ctx context.Context, nbf *types.NomsBinFormat, sch return nil, err } - // TODO: cache is messing everything up schemaCacheMu.Lock() cachedData, ok := unmarshalledSchemaCache[h] schemaCacheMu.Unlock() diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index a7fd2ffe04..6408bf6aff 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -108,58 +108,39 @@ func TestSingleQuery(t *testing.T) { // Convenience test for debugging a single query. Unskip and set to the desired query. func TestSingleScript(t *testing.T) { - script := queries.ScriptTest{ - Name: "DELETE ME", - SetUpScript: []string{ - "create table t (i int primary key auto_increment, v varchar(10), index(v(2)))", - "insert into t (v) values ('aa'), ('bb'), ('cc'), ('aaa')", - }, - Assertions: []queries.ScriptTestAssertion{ - { - Query: "select * from t where v = 'aa'", - Expected: []sql.Row{ - {1, "aa"}, + t.Skip() + var scripts = []queries.ScriptTest{ + { + Name: "trigger before update, with indexed update", + SetUpScript: []string{ + "create table a (x int primary key, y int, unique key (y))", + "create table b (z int primary key)", + "insert into a values (1,3), (10,20)", + "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", + "update a set x = x + 1 where y = 20", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "select x, y from a order by 1", + Expected: []sql.Row{ + {1, 3}, + {11, 20}, + }, + }, + { + Query: "select z from b", + Expected: []sql.Row{ + {100}, + }, }, }, }, } harness := newDoltHarness(t) - enginetest.TestScript(t, harness, script) - - //t.Skip() - //var scripts = []queries.ScriptTest{ - // { - // Name: "trigger before update, with indexed update", - // SetUpScript: []string{ - // "create table a (x int primary key, y int, unique key (y))", - // "create table b (z int primary key)", - // "insert into a values (1,3), (10,20)", - // "create trigger insert_b before update on a for each row insert into b values (old.x * 10)", - // "update a set x = x + 1 where y = 20", - // }, - // Assertions: []queries.ScriptTestAssertion{ - // { - // Query: "select x, y from a order by 1", - // Expected: []sql.Row{ - // {1, 3}, - // {11, 20}, - // }, - // }, - // { - // Query: "select z from b", - // Expected: []sql.Row{ - // {100}, - // }, - // }, - // }, - // }, - //} - // - //harness := newDoltHarness(t) - //for _, test := range scripts { - // enginetest.TestScript(t, harness, test) - //} + for _, test := range scripts { + enginetest.TestScript(t, harness, test) + } } func TestSingleQueryPrepared(t *testing.T) { diff --git a/go/libraries/doltcore/sqle/sqlutil/convert.go b/go/libraries/doltcore/sqle/sqlutil/convert.go index 8fea2f8f56..7b85588dc1 100644 --- a/go/libraries/doltcore/sqle/sqlutil/convert.go +++ b/go/libraries/doltcore/sqle/sqlutil/convert.go @@ -120,8 +120,6 @@ func ToDoltSchema( } sch.SetCollation(schema.Collation(collation)) - // TODO: secondary index prefix lengths - return sch, nil } diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index c43bec654e..5e9d7fbea2 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -1793,14 +1793,6 @@ func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) err return err } - for _, idxCol := range idx.Columns { - col := t.DoltTable.sqlSch.Schema[t.DoltTable.sqlSch.IndexOfColName(idxCol.Name)] - if idxCol.Length > 0 && sql.IsText(col.Type) { - // TODO: delete me - //return sql.ErrUnsupportedIndexPrefix.New(col.Name) - } - } - ret, err := creation.CreateIndex( ctx, table, From 50b77a89e8906eff86bb2c1883255fe6766fbc43 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 11:52:52 -0800 Subject: [PATCH 15/30] making helper method --- go/libraries/doltcore/sqle/tables.go | 60 ++++++++++++---------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 5e9d7fbea2..f6971c2fb8 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -1757,6 +1757,29 @@ func (t *AlterableDoltTable) getFirstAutoIncrementValue( return seq, nil } +// hasNonZeroPrefixLength will return true if at least one of the sql.IndexColumns has a Length > 0 +func hasNonZeroPrefixLength(idxCols []sql.IndexColumn) bool { + for _, idxCol := range idxCols { + if idxCol.Length > 0 { + return true + } + } + return false +} + +// allocatePrefixLengths will return a []uint16 populated with the Length field from sql.IndexColumn +// if all the lengths have a value of 0, it will return nil +func allocatePrefixLengths(idxCols []sql.IndexColumn) []uint16 { + if !hasNonZeroPrefixLength(idxCols) { + return nil + } + prefixLengths := make([]uint16, len(idxCols)) + for i, idxCol := range idxCols { + prefixLengths[i] = uint16(idxCol.Length) + } + return prefixLengths +} + // CreateIndex implements sql.IndexAlterableTable func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) error { if err := branch_control.CheckAccess(ctx, branch_control.Permissions_Write); err != nil { @@ -1771,23 +1794,6 @@ func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) err columns[i] = indexCol.Name } - // TODO: helper method - var hasNonZeroPrefixLength bool - for _, indexCol := range idx.Columns { - if indexCol.Length > 0 { - hasNonZeroPrefixLength = true - break - } - } - - var prefixLengths []uint16 - if hasNonZeroPrefixLength { - prefixLengths = make([]uint16, len(idx.Columns)) - for i, indexCol := range idx.Columns { - prefixLengths[i] = uint16(indexCol.Length) - } - } - table, err := t.DoltTable.DoltTable(ctx) if err != nil { return err @@ -1798,7 +1804,7 @@ func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) err table, idx.Name, columns, - prefixLengths, + allocatePrefixLengths(idx.Columns), idx.Constraint == sql.IndexConstraint_Unique, true, idx.Comment, @@ -2276,22 +2282,6 @@ func (t *AlterableDoltTable) CreateIndexForForeignKey(ctx *sql.Context, idx sql. columns[i] = indexCol.Name } - var hasNonZeroPrefixLength bool - for _, indexCol := range idx.Columns { - if indexCol.Length > 0 { - hasNonZeroPrefixLength = true - break - } - } - - var prefixLengths []uint16 - if hasNonZeroPrefixLength { - prefixLengths = make([]uint16, len(idx.Columns)) - for i, indexCol := range idx.Columns { - prefixLengths[i] = uint16(indexCol.Length) - } - } - table, err := t.DoltTable.DoltTable(ctx) if err != nil { return err @@ -2302,7 +2292,7 @@ func (t *AlterableDoltTable) CreateIndexForForeignKey(ctx *sql.Context, idx sql. table, idx.Name, columns, - prefixLengths, + allocatePrefixLengths(idx.Columns), idx.Constraint == sql.IndexConstraint_Unique, false, "", From 795a81e047254bb30c6abaeb76b2e301900f3bf3 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 12:37:02 -0800 Subject: [PATCH 16/30] adding index prefix lengths in creation instead of setting --- go/libraries/doltcore/doltdb/doltdb_test.go | 4 +-- go/libraries/doltcore/dtestutils/data.go | 2 +- go/libraries/doltcore/merge/merge_test.go | 4 +-- .../doltcore/migrate/integration_test.go | 2 +- go/libraries/doltcore/row/noms_row_test.go | 2 +- .../schema/encoding/schema_marshaling.go | 7 ++--- .../schema/encoding/schema_marshaling_test.go | 4 +-- .../doltcore/schema/encoding/serialization.go | 14 ++++----- go/libraries/doltcore/schema/index.go | 13 ++------- go/libraries/doltcore/schema/index_coll.go | 29 +++++++------------ go/libraries/doltcore/sqle/alterschema.go | 14 +++++---- .../doltcore/sqle/enginetest/validation.go | 2 +- .../doltcore/sqle/index/dolt_index.go | 2 +- go/libraries/doltcore/sqle/tables.go | 14 +++++---- .../sqle/writer/prolly_table_writer.go | 4 +-- .../doltcore/table/editor/creation/index.go | 9 ++---- .../table/editor/index_editor_test.go | 24 +++++++-------- .../table/editor/keyless_table_editor_test.go | 6 ++-- .../table/editor/pk_table_editor_test.go | 4 +-- 19 files changed, 71 insertions(+), 89 deletions(-) diff --git a/go/libraries/doltcore/doltdb/doltdb_test.go b/go/libraries/doltcore/doltdb/doltdb_test.go index 82a03b313b..e4e5a97d85 100644 --- a/go/libraries/doltcore/doltdb/doltdb_test.go +++ b/go/libraries/doltcore/doltdb/doltdb_test.go @@ -64,9 +64,9 @@ func createTestSchema(t *testing.T) schema.Schema { ) sch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexName, []uint64{firstTag, lastTag}, schema.IndexProperties{IsUnique: false, Comment: ""}) + _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexName, []uint64{firstTag, lastTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) - _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexAge, []uint64{ageTag}, schema.IndexProperties{IsUnique: false, Comment: ""}) + _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexAge, []uint64{ageTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) return sch } diff --git a/go/libraries/doltcore/dtestutils/data.go b/go/libraries/doltcore/dtestutils/data.go index c03dc235fd..fa80077316 100644 --- a/go/libraries/doltcore/dtestutils/data.go +++ b/go/libraries/doltcore/dtestutils/data.go @@ -55,7 +55,7 @@ func Schema() (schema.Schema, error) { ) sch := schema.MustSchemaFromCols(typedColColl) - _, err := sch.Indexes().AddIndexByColTags(IndexName, []uint64{NameTag}, schema.IndexProperties{IsUnique: false, Comment: ""}) + _, err := sch.Indexes().AddIndexByColTags(IndexName, []uint64{NameTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/merge/merge_test.go b/go/libraries/doltcore/merge/merge_test.go index 12048bf32e..1259fd7b30 100644 --- a/go/libraries/doltcore/merge/merge_test.go +++ b/go/libraries/doltcore/merge/merge_test.go @@ -62,8 +62,8 @@ var indexSchema schema.Index var compositeIndexSchema schema.Index func init() { - indexSchema, _ = sch.Indexes().AddIndexByColTags("idx_col1", []uint64{col1Tag}, schema.IndexProperties{IsUnique: false, Comment: ""}) - compositeIndexSchema, _ = sch.Indexes().AddIndexByColTags("idx_col1_col2", []uint64{col1Tag, col2Tag}, schema.IndexProperties{IsUnique: false, Comment: ""}) + indexSchema, _ = sch.Indexes().AddIndexByColTags("idx_col1", []uint64{col1Tag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) + compositeIndexSchema, _ = sch.Indexes().AddIndexByColTags("idx_col1_col2", []uint64{col1Tag, col2Tag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) } type rowV struct { diff --git a/go/libraries/doltcore/migrate/integration_test.go b/go/libraries/doltcore/migrate/integration_test.go index d3e3ec4513..de8539190d 100644 --- a/go/libraries/doltcore/migrate/integration_test.go +++ b/go/libraries/doltcore/migrate/integration_test.go @@ -154,7 +154,7 @@ func SetupHookRefKeys(ctx context.Context, dEnv *env.DoltEnv) (*env.DoltEnv, err if err != nil { return nil, err } - _, err = sch.Indexes().AddIndexByColNames("blob_idx", []string{"c1"}, schema.IndexProperties{IsUserDefined: true}) + _, err = sch.Indexes().AddIndexByColNames("blob_idx", []string{"c1"}, nil, schema.IndexProperties{IsUserDefined: true}) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/row/noms_row_test.go b/go/libraries/doltcore/row/noms_row_test.go index c43482652c..55fc79bda4 100644 --- a/go/libraries/doltcore/row/noms_row_test.go +++ b/go/libraries/doltcore/row/noms_row_test.go @@ -66,7 +66,7 @@ var sch, _ = schema.SchemaFromPKAndNonPKCols(testKeyColColl, testNonKeyColColl) var index schema.Index func init() { - index, _ = sch.Indexes().AddIndexByColTags(indexName, []uint64{ageColTag}, schema.IndexProperties{IsUnique: false, Comment: ""}) + index, _ = sch.Indexes().AddIndexByColTags(indexName, []uint64{ageColTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) } func newTestRow() (Row, error) { diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling.go b/go/libraries/doltcore/schema/encoding/schema_marshaling.go index 523456d079..344a2f0b96 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling.go @@ -243,7 +243,7 @@ func toSchemaData(sch schema.Schema) (schemaData, error) { Comment: index.Comment(), Unique: index.IsUnique(), IsSystemDefined: !index.IsUserDefined(), - PrefixLengths: index.GetPrefixLengths(), + PrefixLengths: index.PrefixLengths(), } } @@ -302,6 +302,7 @@ func (sd schemaData) addChecksIndexesAndPkOrderingToSchema(sch schema.Schema) er _, err := sch.Indexes().UnsafeAddIndexByColTags( encodedIndex.Name, encodedIndex.Tags, + encodedIndex.PrefixLengths, schema.IndexProperties{ IsUnique: encodedIndex.Unique, IsUserDefined: !encodedIndex.IsSystemDefined, @@ -311,10 +312,6 @@ func (sd schemaData) addChecksIndexesAndPkOrderingToSchema(sch schema.Schema) er if err != nil { return err } - err = sch.Indexes().SetIndexPrefixLength(encodedIndex.Name, encodedIndex.PrefixLengths) - if err != nil { - return err - } } for _, encodedCheck := range sd.CheckConstraints { diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling_test.go b/go/libraries/doltcore/schema/encoding/schema_marshaling_test.go index dc52ce0a1c..6d6ea420b7 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling_test.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling_test.go @@ -46,7 +46,7 @@ func createTestSchema() schema.Schema { colColl := schema.NewColCollection(columns...) sch := schema.MustSchemaFromCols(colColl) - _, _ = sch.Indexes().AddIndexByColTags("idx_age", []uint64{3}, schema.IndexProperties{IsUnique: false, Comment: ""}) + _, _ = sch.Indexes().AddIndexByColTags("idx_age", []uint64{3}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) return sch } @@ -255,7 +255,7 @@ func (tsd testSchemaData) decodeSchema() (schema.Schema, error) { sch.SetCollation(tsd.Collation) for _, encodedIndex := range tsd.IndexCollection { - _, err = sch.Indexes().AddIndexByColTags(encodedIndex.Name, encodedIndex.Tags, schema.IndexProperties{IsUnique: encodedIndex.Unique, Comment: encodedIndex.Comment}) + _, err = sch.Indexes().AddIndexByColTags(encodedIndex.Name, encodedIndex.Tags, nil, schema.IndexProperties{IsUnique: encodedIndex.Unique, Comment: encodedIndex.Comment}) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/schema/encoding/serialization.go b/go/libraries/doltcore/schema/encoding/serialization.go index 3bdc09808a..ee5c983575 100644 --- a/go/libraries/doltcore/schema/encoding/serialization.go +++ b/go/libraries/doltcore/schema/encoding/serialization.go @@ -334,7 +334,7 @@ func serializeSecondaryIndexes(b *fb.Builder, sch schema.Schema, indexes []schem ko := b.EndVector(len(tags)) // serialize prefix lengths - prefixLengths := idx.GetPrefixLengths() + prefixLengths := idx.PrefixLengths() serial.IndexStartPrefixLengthsVector(b, len(prefixLengths)) for j := len(prefixLengths) - 1; j >= 0; j-- { b.PrependUint16(prefixLengths[j]) @@ -381,17 +381,15 @@ func deserializeSecondaryIndexes(sch schema.Schema, s *serial.TableSchema) error tags[j] = col.Tag() } - _, err := sch.Indexes().AddIndexByColTags(name, tags, props) - if err != nil { - return err - } - prefixLengths := make([]uint16, idx.PrefixLengthsLength()) for j := range prefixLengths { prefixLengths[j] = idx.PrefixLengths(j) } - schIdx := sch.Indexes().GetByName(name) - schIdx.SetPrefixLengths(prefixLengths) + + _, err := sch.Indexes().AddIndexByColTags(name, tags, prefixLengths, props) + if err != nil { + return err + } } return nil } diff --git a/go/libraries/doltcore/schema/index.go b/go/libraries/doltcore/schema/index.go index 742e33bf2f..8ce89d62dd 100644 --- a/go/libraries/doltcore/schema/index.go +++ b/go/libraries/doltcore/schema/index.go @@ -54,10 +54,8 @@ type Index interface { // ToTableTuple returns a tuple that may be used to retrieve the original row from the indexed table when given // a full index key (and not a partial index key). ToTableTuple(ctx context.Context, fullKey types.Tuple, format *types.NomsBinFormat) (types.Tuple, error) - // GetPrefixLengths returns the prefix lengths for the index - GetPrefixLengths() []uint16 - // SetPrefixLengths applies to prefix lengths - SetPrefixLengths(prefixLengths []uint16) + // PrefixLengths returns the prefix lengths for the index + PrefixLengths() []uint16 } var _ Index = (*indexImpl)(nil) @@ -247,15 +245,10 @@ func (ix *indexImpl) ToTableTuple(ctx context.Context, fullKey types.Tuple, form } // GetPrefixLengths implements Index. -func (ix *indexImpl) GetPrefixLengths() []uint16 { +func (ix *indexImpl) PrefixLengths() []uint16 { return ix.prefixLengths } -// SetPrefixLengths implements Index. -func (ix *indexImpl) SetPrefixLengths(prefixLengths []uint16) { - ix.prefixLengths = prefixLengths -} - // copy returns an exact copy of the calling index. func (ix *indexImpl) copy() *indexImpl { newIx := *ix diff --git a/go/libraries/doltcore/schema/index_coll.go b/go/libraries/doltcore/schema/index_coll.go index b6222a62b6..704240d9d7 100644 --- a/go/libraries/doltcore/schema/index_coll.go +++ b/go/libraries/doltcore/schema/index_coll.go @@ -25,11 +25,11 @@ type IndexCollection interface { // It does not perform any kind of checking, and is intended for schema modifications. AddIndex(indexes ...Index) // AddIndexByColNames adds an index with the given name and columns (in index order). - AddIndexByColNames(indexName string, cols []string, props IndexProperties) (Index, error) + AddIndexByColNames(indexName string, cols []string, prefixLengths []uint16, props IndexProperties) (Index, error) // AddIndexByColTags adds an index with the given name and column tags (in index order). - AddIndexByColTags(indexName string, tags []uint64, props IndexProperties) (Index, error) + AddIndexByColTags(indexName string, tags []uint64, prefixLengths []uint16, props IndexProperties) (Index, error) // todo: this method is trash, clean up this interface - UnsafeAddIndexByColTags(indexName string, tags []uint64, props IndexProperties) (Index, error) + UnsafeAddIndexByColTags(indexName string, tags []uint64, prefixLengths []uint16, props IndexProperties) (Index, error) // AllIndexes returns a slice containing all of the indexes in this collection. AllIndexes() []Index // Contains returns whether the given index name already exists for this table. @@ -64,8 +64,6 @@ type IndexCollection interface { RenameIndex(oldName, newName string) (Index, error) //SetPks changes the pks or pk ordinals SetPks([]uint64) error - // SetIndexPrefixLength sets the prefix lengths for the specified index - SetIndexPrefixLength(indexName string, prefixLengths []uint16) error } type IndexProperties struct { @@ -127,15 +125,15 @@ func (ixc *indexCollectionImpl) AddIndex(indexes ...Index) { } } -func (ixc *indexCollectionImpl) AddIndexByColNames(indexName string, cols []string, props IndexProperties) (Index, error) { +func (ixc *indexCollectionImpl) AddIndexByColNames(indexName string, cols []string, prefixLengths []uint16, props IndexProperties) (Index, error) { tags, ok := ixc.columnNamesToTags(cols) if !ok { return nil, fmt.Errorf("the table does not contain at least one of the following columns: `%v`", cols) } - return ixc.AddIndexByColTags(indexName, tags, props) + return ixc.AddIndexByColTags(indexName, tags, prefixLengths, props) } -func (ixc *indexCollectionImpl) AddIndexByColTags(indexName string, tags []uint64, props IndexProperties) (Index, error) { +func (ixc *indexCollectionImpl) AddIndexByColTags(indexName string, tags []uint64, prefixLengths []uint16, props IndexProperties) (Index, error) { if strings.HasPrefix(indexName, "dolt_") { return nil, fmt.Errorf("indexes cannot be prefixed with `dolt_`") } @@ -163,7 +161,7 @@ func (ixc *indexCollectionImpl) AddIndexByColTags(indexName string, tags []uint6 isUnique: props.IsUnique, isUserDefined: props.IsUserDefined, comment: props.Comment, - prefixLengths: []uint16{}, + prefixLengths: prefixLengths, } ixc.indexes[indexName] = index for _, tag := range tags { @@ -180,7 +178,7 @@ func validateColumnIndexable(c Column) error { return nil } -func (ixc *indexCollectionImpl) UnsafeAddIndexByColTags(indexName string, tags []uint64, props IndexProperties) (Index, error) { +func (ixc *indexCollectionImpl) UnsafeAddIndexByColTags(indexName string, tags []uint64, prefixLengths []uint16, props IndexProperties) (Index, error) { index := &indexImpl{ indexColl: ixc, name: indexName, @@ -189,6 +187,7 @@ func (ixc *indexCollectionImpl) UnsafeAddIndexByColTags(indexName string, tags [ isUnique: props.IsUnique, isUserDefined: props.IsUserDefined, comment: props.Comment, + prefixLengths: prefixLengths, } ixc.indexes[indexName] = index for _, tag := range tags { @@ -328,7 +327,7 @@ func (ixc *indexCollectionImpl) Merge(indexes ...Index) { isUnique: index.IsUnique(), isUserDefined: index.IsUserDefined(), comment: index.Comment(), - prefixLengths: index.GetPrefixLengths(), + prefixLengths: index.PrefixLengths(), } ixc.AddIndex(newIndex) } @@ -432,14 +431,6 @@ func (ixc *indexCollectionImpl) SetPks(tags []uint64) error { return nil } -func (ixc *indexCollectionImpl) SetIndexPrefixLength(indexName string, prefixLengths []uint16) error { - if !ixc.Contains(indexName) { - return fmt.Errorf("`%s` does not exist as an index for this table", indexName) - } - ixc.indexes[indexName].prefixLengths = prefixLengths - return nil -} - func combineAllTags(tags []uint64, pks []uint64) []uint64 { allTags := make([]uint64, len(tags)) _ = copy(allTags, tags) diff --git a/go/libraries/doltcore/sqle/alterschema.go b/go/libraries/doltcore/sqle/alterschema.go index d251884ecc..88c718483e 100755 --- a/go/libraries/doltcore/sqle/alterschema.go +++ b/go/libraries/doltcore/sqle/alterschema.go @@ -265,11 +265,15 @@ func replaceColumnInSchema(sch schema.Schema, oldCol schema.Column, newCol schem tags[i] = newCol.Tag } } - _, err = newSch.Indexes().AddIndexByColTags(index.Name(), tags, schema.IndexProperties{ - IsUnique: index.IsUnique(), - IsUserDefined: index.IsUserDefined(), - Comment: index.Comment(), - }) + _, err = newSch.Indexes().AddIndexByColTags( + index.Name(), + tags, + index.PrefixLengths(), + schema.IndexProperties{ + IsUnique: index.IsUnique(), + IsUserDefined: index.IsUserDefined(), + Comment: index.Comment(), + }) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/enginetest/validation.go b/go/libraries/doltcore/sqle/enginetest/validation.go index a7abdaf555..aaad93d1bf 100644 --- a/go/libraries/doltcore/sqle/enginetest/validation.go +++ b/go/libraries/doltcore/sqle/enginetest/validation.go @@ -142,7 +142,7 @@ func validateIndexConsistency( primary, secondary prolly.Map, ) error { // TODO: fix this later - if len(def.GetPrefixLengths()) > 0 { + if len(def.PrefixLengths()) > 0 { return nil } diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index 2163ba00cb..36c7ed774a 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -255,7 +255,7 @@ func getSecondaryIndex(ctx context.Context, db, tbl string, t *doltdb.Table, sch order: sql.IndexOrderAsc, constrainedToLookupExpression: true, doltBinFormat: types.IsFormat_DOLT(vrw.Format()), - prefixLengths: idx.GetPrefixLengths(), + prefixLengths: idx.PrefixLengths(), }, nil } diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index f6971c2fb8..12803951b6 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -1342,11 +1342,15 @@ func (t *AlterableDoltTable) RewriteInserter( colNames = append(colNames, colName) } } - newSch.Indexes().AddIndexByColNames(index.Name(), colNames, schema.IndexProperties{ - IsUnique: index.IsUnique(), - IsUserDefined: index.IsUserDefined(), - Comment: index.Comment(), - }) + newSch.Indexes().AddIndexByColNames( + index.Name(), + colNames, + index.PrefixLengths(), + schema.IndexProperties{ + IsUnique: index.IsUnique(), + IsUserDefined: index.IsUserDefined(), + Comment: index.Comment(), + }) } } else { newSch = schema.CopyIndexes(oldSch, newSch) diff --git a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go index ef8165b063..728f37a1c2 100755 --- a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go @@ -79,7 +79,7 @@ func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch // mapping from secondary index key to primary key pkMap := makeIndexToIndexMapping(def.Schema().GetPKCols(), sch.GetPKCols()) - prefixLengths := def.GetPrefixLengths() + prefixLengths := def.PrefixLengths() if len(prefixLengths) == 0 { writers[defName] = prollySecondaryIndexWriter{ name: defName, @@ -130,7 +130,7 @@ func getSecondaryKeylessProllyWriters(ctx context.Context, t *doltdb.Table, sqlS keyMap, _ := ordinalMappingsFromSchema(sqlSch, def.Schema()) keyDesc, _ := m.Descriptors() - prefixLengths := def.GetPrefixLengths() + prefixLengths := def.PrefixLengths() if len(prefixLengths) == 0 { writers[defName] = prollyKeylessSecondaryWriter{ name: defName, diff --git a/go/libraries/doltcore/table/editor/creation/index.go b/go/libraries/doltcore/table/editor/creation/index.go index 7e9378ea24..9885b48fb6 100644 --- a/go/libraries/doltcore/table/editor/creation/index.go +++ b/go/libraries/doltcore/table/editor/creation/index.go @@ -97,9 +97,10 @@ func CreateIndex( } // create the index metadata, will error if index names are taken or an index with the same columns in the same order exists - _, err = sch.Indexes().AddIndexByColNames( + index, err := sch.Indexes().AddIndexByColNames( indexName, realColNames, + prefixLengths, schema.IndexProperties{ IsUnique: isUnique, IsUserDefined: isUserDefined, @@ -110,11 +111,6 @@ func CreateIndex( return nil, err } - err = sch.Indexes().SetIndexPrefixLength(indexName, prefixLengths) - if err != nil { - return nil, err - } - // update the table schema with the new index newTable, err := table.UpdateSchema(ctx, sch) if err != nil { @@ -123,7 +119,6 @@ func CreateIndex( // TODO: in the case that we're replacing an implicit index with one the user specified, we could do this more // cheaply in some cases by just renaming it, rather than building it from scratch. But that's harder to get right. - index := sch.Indexes().GetByName(indexName) indexRows, err := BuildSecondaryIndex(ctx, newTable, index, opts) if err != nil { return nil, err diff --git a/go/libraries/doltcore/table/editor/index_editor_test.go b/go/libraries/doltcore/table/editor/index_editor_test.go index b648787112..10f8c31fb6 100644 --- a/go/libraries/doltcore/table/editor/index_editor_test.go +++ b/go/libraries/doltcore/table/editor/index_editor_test.go @@ -62,7 +62,7 @@ func TestIndexEditorConcurrency(t *testing.T) { schema.NewColumn("v2", 2, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, schema.IndexProperties{IsUnique: false, Comment: ""}) + index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) indexSch := index.Schema() emptyMap, err := types.NewMap(context.Background(), vrw) @@ -158,7 +158,7 @@ func TestIndexEditorConcurrencyPostInsert(t *testing.T) { schema.NewColumn("v2", 2, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, schema.IndexProperties{IsUnique: false, Comment: ""}) + index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) indexSch := index.Schema() emptyMap, err := types.NewMap(context.Background(), vrw) @@ -250,7 +250,7 @@ func TestIndexEditorUniqueMultipleNil(t *testing.T) { schema.NewColumn("v1", 1, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_unique", []string{"v1"}, schema.IndexProperties{IsUnique: true, Comment: ""}) + index, err := tableSch.Indexes().AddIndexByColNames("idx_unique", []string{"v1"}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) require.NoError(t, err) indexSch := index.Schema() emptyMap, err := types.NewMap(context.Background(), vrw) @@ -297,7 +297,7 @@ func TestIndexEditorWriteAfterFlush(t *testing.T) { schema.NewColumn("v2", 2, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, schema.IndexProperties{IsUnique: false, Comment: ""}) + index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) indexSch := index.Schema() emptyMap, err := types.NewMap(context.Background(), vrw) @@ -365,7 +365,7 @@ func TestIndexEditorUniqueErrorDoesntPersist(t *testing.T) { schema.NewColumn("v1", 1, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_unq", []string{"v1"}, schema.IndexProperties{IsUnique: true, Comment: ""}) + index, err := tableSch.Indexes().AddIndexByColNames("idx_unq", []string{"v1"}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) require.NoError(t, err) indexSch := index.Schema() emptyMap, err := types.NewMap(context.Background(), vrw) @@ -618,7 +618,7 @@ func TestIndexRebuildingUniqueSuccessOneCol(t *testing.T) { originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData) require.NoError(t, err) - index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2}, schema.IndexProperties{IsUnique: true, Comment: ""}) + index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) require.NoError(t, err) updatedTable, err := originalTable.UpdateSchema(context.Background(), sch) require.NoError(t, err) @@ -649,7 +649,7 @@ func TestIndexRebuildingUniqueSuccessTwoCol(t *testing.T) { originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData) require.NoError(t, err) - index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2, 3}, schema.IndexProperties{IsUnique: true, Comment: ""}) + index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2, 3}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) require.NoError(t, err) updatedTable, err := originalTable.UpdateSchema(context.Background(), sch) require.NoError(t, err) @@ -680,7 +680,7 @@ func TestIndexRebuildingUniqueFailOneCol(t *testing.T) { originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData) require.NoError(t, err) - index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2}, schema.IndexProperties{IsUnique: true, Comment: ""}) + index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) require.NoError(t, err) updatedTable, err := originalTable.UpdateSchema(context.Background(), sch) require.NoError(t, err) @@ -712,7 +712,7 @@ func TestIndexRebuildingUniqueFailTwoCol(t *testing.T) { originalTable, err := createTableWithoutIndexRebuilding(context.Background(), vrw, ns, sch, rowData) require.NoError(t, err) - index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2, 3}, schema.IndexProperties{IsUnique: true, Comment: ""}) + index, err := sch.Indexes().AddIndexByColTags("idx_v1", []uint64{2, 3}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) require.NoError(t, err) updatedTable, err := originalTable.UpdateSchema(context.Background(), sch) require.NoError(t, err) @@ -738,7 +738,7 @@ func TestIndexEditorCapacityExceeded(t *testing.T) { schema.NewColumn("v1", 1, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_cap", []string{"v1"}, schema.IndexProperties{IsUnique: false, Comment: ""}) + index, err := tableSch.Indexes().AddIndexByColNames("idx_cap", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) indexSch := index.Schema() emptyMap, err := types.NewMap(ctx, vrw) @@ -837,9 +837,9 @@ func createTestSchema(t *testing.T) schema.Schema { ) sch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexName, []uint64{firstTag, lastTag}, schema.IndexProperties{IsUnique: false, Comment: ""}) + _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexName, []uint64{firstTag, lastTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) - _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexAge, []uint64{ageTag}, schema.IndexProperties{IsUnique: false, Comment: ""}) + _, err = sch.Indexes().AddIndexByColTags(testSchemaIndexAge, []uint64{ageTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) return sch } diff --git a/go/libraries/doltcore/table/editor/keyless_table_editor_test.go b/go/libraries/doltcore/table/editor/keyless_table_editor_test.go index fd38a4b06f..2c9928ec80 100644 --- a/go/libraries/doltcore/table/editor/keyless_table_editor_test.go +++ b/go/libraries/doltcore/table/editor/keyless_table_editor_test.go @@ -426,11 +426,11 @@ func TestKeylessTableEditorMultipleIndexErrorHandling(t *testing.T) { schema.NewColumn("v2", 2, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - idxv1, err := tableSch.Indexes().AddIndexByColNames("idx_v1", []string{"v1"}, schema.IndexProperties{ + idxv1, err := tableSch.Indexes().AddIndexByColNames("idx_v1", []string{"v1"}, nil, schema.IndexProperties{ IsUnique: false, }) require.NoError(t, err) - idxv2, err := tableSch.Indexes().AddIndexByColNames("idx_v2", []string{"v2"}, schema.IndexProperties{ + idxv2, err := tableSch.Indexes().AddIndexByColNames("idx_v2", []string{"v2"}, nil, schema.IndexProperties{ IsUnique: false, }) require.NoError(t, err) @@ -583,7 +583,7 @@ func TestKeylessTableEditorIndexCardinality(t *testing.T) { schema.NewColumn("v2", 2, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - idxv1, err := tableSch.Indexes().AddIndexByColNames("idx_v1", []string{"v1"}, schema.IndexProperties{ + idxv1, err := tableSch.Indexes().AddIndexByColNames("idx_v1", []string{"v1"}, nil, schema.IndexProperties{ IsUnique: false, }) require.NoError(t, err) diff --git a/go/libraries/doltcore/table/editor/pk_table_editor_test.go b/go/libraries/doltcore/table/editor/pk_table_editor_test.go index 65639d2093..adbcfd3c2f 100644 --- a/go/libraries/doltcore/table/editor/pk_table_editor_test.go +++ b/go/libraries/doltcore/table/editor/pk_table_editor_test.go @@ -392,11 +392,11 @@ func TestTableEditorMultipleIndexErrorHandling(t *testing.T) { schema.NewColumn("v2", 2, types.IntKind, false)) tableSch, err := schema.SchemaFromCols(colColl) require.NoError(t, err) - idxv1, err := tableSch.Indexes().AddIndexByColNames("idx_v1", []string{"v1"}, schema.IndexProperties{ + idxv1, err := tableSch.Indexes().AddIndexByColNames("idx_v1", []string{"v1"}, nil, schema.IndexProperties{ IsUnique: true, }) require.NoError(t, err) - idxv2, err := tableSch.Indexes().AddIndexByColNames("idx_v2", []string{"v2"}, schema.IndexProperties{ + idxv2, err := tableSch.Indexes().AddIndexByColNames("idx_v2", []string{"v2"}, nil, schema.IndexProperties{ IsUnique: true, }) require.NoError(t, err) From 962ce92593c9d28c00ac4e31a6431711f28b208b Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 12:55:53 -0800 Subject: [PATCH 17/30] bump --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index eb47f6e5ab..4ec4fad82d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,7 +56,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.14.1-0.20221109235436-34752efabb29 + github.com/dolthub/go-mysql-server v0.14.1-0.20221110185417-b4c7095105de github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 901d2f1d10..9b47fd87d2 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.14.1-0.20221109235436-34752efabb29 h1:2Td2oAleuVLotTtNW5z0cIkAwiEyshphfnMrlngj7Ws= -github.com/dolthub/go-mysql-server v0.14.1-0.20221109235436-34752efabb29/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= +github.com/dolthub/go-mysql-server v0.14.1-0.20221110185417-b4c7095105de h1:9A7bN2ivv4Pza3kpujtCckuDEp3IP23h22rWAaPHZQs= +github.com/dolthub/go-mysql-server v0.14.1-0.20221110185417-b4c7095105de/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From 1a3261e7c84eb373b7187a0eccddf7830d0c71bc Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Thu, 10 Nov 2022 21:39:28 +0000 Subject: [PATCH 18/30] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/go.mod | 5 +++-- go/go.sum | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go/go.mod b/go/go.mod index 4ec4fad82d..76e4c91d0b 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,6 +56,8 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 + github.com/cespare/xxhash v1.1.0 + github.com/creasty/defaults v1.6.0 github.com/dolthub/go-mysql-server v0.14.1-0.20221110185417-b4c7095105de github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 @@ -75,6 +77,7 @@ require ( go.opentelemetry.io/otel/trace v1.7.0 golang.org/x/text v0.3.7 gonum.org/v1/plot v0.11.0 + gopkg.in/yaml.v3 v3.0.0 ) require ( @@ -85,7 +88,6 @@ require ( github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-fonts/liberation v0.2.0 // indirect @@ -131,7 +133,6 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2 // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect ) replace ( diff --git a/go/go.sum b/go/go.sum index 9b47fd87d2..417099190f 100644 --- a/go/go.sum +++ b/go/go.sum @@ -163,6 +163,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc= +github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From e218c4577080c731faa27493e42a0726a79ad929 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 14:19:13 -0800 Subject: [PATCH 19/30] removintg extra writers --- .../doltcore/sqle/enginetest/validation.go | 1 - go/libraries/doltcore/sqle/temp_table.go | 18 +- .../sqle/writer/prolly_index_writer.go | 219 ++---------------- .../writer/prolly_index_writer_keyless.go | 201 ++-------------- .../sqle/writer/prolly_table_writer.go | 69 ++---- 5 files changed, 71 insertions(+), 437 deletions(-) diff --git a/go/libraries/doltcore/sqle/enginetest/validation.go b/go/libraries/doltcore/sqle/enginetest/validation.go index aaad93d1bf..4c7fd339ae 100644 --- a/go/libraries/doltcore/sqle/enginetest/validation.go +++ b/go/libraries/doltcore/sqle/enginetest/validation.go @@ -220,7 +220,6 @@ func validatePkIndex(ctx context.Context, sch schema.Schema, def schema.Index, p if j < pkSize { builder.PutRaw(i, key.GetField(j)) } else { - builder.PutRaw(i, value.GetField(j-pkSize)) } } diff --git a/go/libraries/doltcore/sqle/temp_table.go b/go/libraries/doltcore/sqle/temp_table.go index 2233ff3acd..9e4f919076 100644 --- a/go/libraries/doltcore/sqle/temp_table.go +++ b/go/libraries/doltcore/sqle/temp_table.go @@ -267,28 +267,12 @@ func (t *TempTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) error { cols[i] = c.Name } - var hasNonZeroPrefixLength bool - for _, c := range idx.Columns { - if c.Length > 0 { - hasNonZeroPrefixLength = true - break - } - } - - var prefixLengths []uint16 - if hasNonZeroPrefixLength { - prefixLengths = make([]uint16, len(idx.Columns)) - for i, c := range idx.Columns { - prefixLengths[i] = uint16(c.Length) - } - } - ret, err := creation.CreateIndex( ctx, t.table, idx.Name, cols, - prefixLengths, + allocatePrefixLengths(idx.Columns), idx.Constraint == sql.IndexConstraint_Unique, true, idx.Comment, diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go index 9af45c821e..741a8cafde 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer.go @@ -244,9 +244,10 @@ func (m prollyIndexWriter) uniqueKeyError(ctx context.Context, keyStr string, ke } type prollySecondaryIndexWriter struct { - name string - mut *prolly.MutableMap - unique bool + name string + mut *prolly.MutableMap + unique bool + prefixLengths []uint16 // number of indexed cols idxCols int @@ -283,10 +284,28 @@ func (m prollySecondaryIndexWriter) ValidateKeyViolations(ctx context.Context, s return nil } +// trimKeyPart will trim entry into the sql.Row depending on the prefixLengths +func (m prollySecondaryIndexWriter) trimKeyPart(to int, keyPart interface{}) interface{} { + var prefixLength uint16 + if len(m.prefixLengths) > to { + prefixLength = m.prefixLengths[to] + } + if prefixLength != 0 { + switch kp := keyPart.(type) { + case string: + keyPart = kp[:prefixLength] + case []uint8: + keyPart = kp[:prefixLength] + } + } + return keyPart +} + func (m prollySecondaryIndexWriter) keyFromRow(ctx context.Context, sqlRow sql.Row) (val.Tuple, error) { for to := range m.keyMap { from := m.keyMap.MapOrdinal(to) - if err := index.PutField(ctx, m.mut.NodeStore(), m.keyBld, to, sqlRow[from]); err != nil { + keyPart := m.trimKeyPart(to, sqlRow[from]) + if err := index.PutField(ctx, m.mut.NodeStore(), m.keyBld, to, keyPart); err != nil { return nil, err } } @@ -311,7 +330,8 @@ func (m prollySecondaryIndexWriter) checkForUniqueKeyErr(ctx context.Context, sq m.keyBld.Recycle() return nil } - if err := index.PutField(ctx, ns, m.keyBld, to, sqlRow[from]); err != nil { + keyPart := m.trimKeyPart(to, sqlRow[from]) + if err := index.PutField(ctx, ns, m.keyBld, to, keyPart); err != nil { return err } } @@ -413,192 +433,3 @@ func FormatKeyForUniqKeyErr(key val.Tuple, d val.TupleDesc) string { sb.WriteString("]") return sb.String() } - -type prollySecondaryIndexPrefixWriter struct { - name string - mut *prolly.MutableMap - unique bool - prefixLengths []uint16 - - // number of indexed cols - idxCols int - - // keyMap is a mapping from sql.Row fields to - // key fields of this secondary index - keyMap val.OrdinalMapping - // keyBld builds key tuples for the secondary index - keyBld *val.TupleBuilder - - // pkMap is a mapping from secondary index keys to - // primary key clustered index keys - pkMap val.OrdinalMapping - // pkBld builds key tuples for primary key index - pkBld *val.TupleBuilder -} - -var _ indexWriter = prollySecondaryIndexPrefixWriter{} - -func (m prollySecondaryIndexPrefixWriter) Name() string { - return m.name -} - -func (m prollySecondaryIndexPrefixWriter) Map(ctx context.Context) (prolly.Map, error) { - return m.mut.Map(ctx) -} - -func (m prollySecondaryIndexPrefixWriter) ValidateKeyViolations(ctx context.Context, sqlRow sql.Row) error { - // TODO: trim row according to index prefix here - if m.unique { - if err := m.checkForUniqueKeyErr(ctx, sqlRow); err != nil { - return err - } - } - return nil -} - -func (m prollySecondaryIndexPrefixWriter) keyFromRow(ctx context.Context, sqlRow sql.Row) (val.Tuple, error) { - for to := range m.keyMap { - from := m.keyMap.MapOrdinal(to) - keyPart := sqlRow[from] - var prefixLength uint16 - if len(m.prefixLengths) > to { - prefixLength = m.prefixLengths[to] - } - if prefixLength != 0 { - switch kp := keyPart.(type) { - case string: - keyPart = kp[:prefixLength] - case []uint8: - keyPart = kp[:prefixLength] - default: - panic("what in tarnation is going on in here") - } - } - if err := index.PutField(ctx, m.mut.NodeStore(), m.keyBld, to, keyPart); err != nil { - return nil, err - } - } - return m.keyBld.Build(sharePool), nil -} - -func (m prollySecondaryIndexPrefixWriter) Insert(ctx context.Context, sqlRow sql.Row) error { - k, err := m.keyFromRow(ctx, sqlRow) - if err != nil { - return err - } - return m.mut.Put(ctx, k, val.EmptyTuple) -} - -func (m prollySecondaryIndexPrefixWriter) checkForUniqueKeyErr(ctx context.Context, sqlRow sql.Row) error { - ns := m.mut.NodeStore() - for to := range m.keyMap[:m.idxCols] { - from := m.keyMap.MapOrdinal(to) - if sqlRow[from] == nil { - // NULL is incomparable and cannot - // trigger a UNIQUE KEY violation - m.keyBld.Recycle() - return nil - } - - keyPart := sqlRow[from] - var prefixLength uint16 - if len(m.prefixLengths) > to { - prefixLength = m.prefixLengths[to] - } - if prefixLength != 0 { - switch kp := keyPart.(type) { - case string: - keyPart = kp[:prefixLength] - case []uint8: - keyPart = kp[:prefixLength] - default: - // TODO: do something else here - panic("what in tarnation is going on in here") - } - } - - if err := index.PutField(ctx, ns, m.keyBld, to, keyPart); err != nil { - return err - } - } - - // build a val.Tuple containing only fields for the unique column prefix - key := m.keyBld.BuildPrefix(ns.Pool(), m.idxCols) - desc := m.keyBld.Desc.PrefixDesc(m.idxCols) - rng := prolly.PrefixRange(key, desc) - iter, err := m.mut.IterRange(ctx, rng) - if err != nil { - return err - } - - idxKey, _, err := iter.Next(ctx) - if err == io.EOF { - return nil // no violation - } else if err != nil { - return err - } - - // |prefix| collides with an existing key - idxDesc := m.keyBld.Desc - for to := range m.pkMap { - from := m.pkMap.MapOrdinal(to) - m.pkBld.PutRaw(to, idxDesc.GetField(from, idxKey)) - } - existingPK := m.pkBld.Build(sharePool) - - return secondaryUniqueKeyError{ - keyStr: FormatKeyForUniqKeyErr(key, desc), - existingKey: existingPK, - } -} - -func (m prollySecondaryIndexPrefixWriter) Delete(ctx context.Context, sqlRow sql.Row) error { - k := m.keyBld.Build(sharePool) - k, err := m.keyFromRow(ctx, sqlRow) - if err != nil { - return err - } - return m.mut.Delete(ctx, k) -} - -func (m prollySecondaryIndexPrefixWriter) Update(ctx context.Context, oldRow sql.Row, newRow sql.Row) error { - oldKey, err := m.keyFromRow(ctx, oldRow) - if err != nil { - return err - } - - // todo(andy): we can skip building, deleting |oldKey| - // if we know the key fields are unchanged - if err := m.mut.Delete(ctx, oldKey); err != nil { - return err - } - - if m.unique { - if err := m.checkForUniqueKeyErr(ctx, newRow); err != nil { - return err - } - } - - newKey, err := m.keyFromRow(ctx, newRow) - if err != nil { - return err - } - return m.mut.Put(ctx, newKey, val.EmptyTuple) -} - -func (m prollySecondaryIndexPrefixWriter) Commit(ctx context.Context) error { - return m.mut.Checkpoint(ctx) -} - -func (m prollySecondaryIndexPrefixWriter) Discard(ctx context.Context) error { - m.mut.Revert(ctx) - return nil -} - -func (m prollySecondaryIndexPrefixWriter) HasEdits(ctx context.Context) bool { - return m.mut.HasEdits() -} - -func (m prollySecondaryIndexPrefixWriter) IterRange(ctx context.Context, rng prolly.Range) (prolly.MapIter, error) { - return m.mut.IterRange(ctx, rng) -} diff --git a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go index 88c92124c2..cb489c1dd6 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go +++ b/go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go @@ -180,10 +180,11 @@ func (e secondaryUniqueKeyError) Error() string { } type prollyKeylessSecondaryWriter struct { - name string - mut *prolly.MutableMap - primary prollyKeylessWriter - unique bool + name string + mut *prolly.MutableMap + primary prollyKeylessWriter + unique bool + prefixLengths []uint16 keyBld *val.TupleBuilder prefixBld *val.TupleBuilder @@ -208,15 +209,33 @@ func (writer prollyKeylessSecondaryWriter) ValidateKeyViolations(ctx context.Con return nil } +// trimKeyPart will trim entry into the sql.Row depending on the prefixLengths +func (writer prollyKeylessSecondaryWriter) trimKeyPart(to int, keyPart interface{}) interface{} { + var prefixLength uint16 + if len(writer.prefixLengths) > to { + prefixLength = writer.prefixLengths[to] + } + if prefixLength != 0 { + switch kp := keyPart.(type) { + case string: + keyPart = kp[:prefixLength] + case []uint8: + keyPart = kp[:prefixLength] + } + } + return keyPart +} + // Insert implements the interface indexWriter. func (writer prollyKeylessSecondaryWriter) Insert(ctx context.Context, sqlRow sql.Row) error { for to := range writer.keyMap { from := writer.keyMap.MapOrdinal(to) - if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, sqlRow[from]); err != nil { + keyPart := writer.trimKeyPart(to, sqlRow[from]) + if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, keyPart); err != nil { return err } if to < writer.prefixBld.Desc.Count() { - if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, sqlRow[from]); err != nil { + if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, keyPart); err != nil { return err } } @@ -331,173 +350,3 @@ func (writer prollyKeylessSecondaryWriter) HasEdits(ctx context.Context) bool { func (writer prollyKeylessSecondaryWriter) IterRange(ctx context.Context, rng prolly.Range) (prolly.MapIter, error) { return writer.mut.IterRange(ctx, rng) } - -type prollyKeylessSecondaryPrefixWriter struct { - name string - mut *prolly.MutableMap - primary prollyKeylessWriter - unique bool - prefixLengths []uint16 - - keyBld *val.TupleBuilder - prefixBld *val.TupleBuilder - hashBld *val.TupleBuilder - keyMap val.OrdinalMapping -} - -var _ indexWriter = prollyKeylessSecondaryPrefixWriter{} - -// Name implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) Name() string { - return writer.name -} - -// Map implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) Map(ctx context.Context) (prolly.Map, error) { - return writer.mut.Map(ctx) -} - -// ValidateKeyViolations implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) ValidateKeyViolations(ctx context.Context, sqlRow sql.Row) error { - return nil -} - -// Insert implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) Insert(ctx context.Context, sqlRow sql.Row) error { - for to := range writer.keyMap { - from := writer.keyMap.MapOrdinal(to) - - keyPart := sqlRow[from] - var prefixLength uint16 - if len(writer.prefixLengths) > to { - prefixLength = writer.prefixLengths[to] - } - if prefixLength != 0 { - switch kp := keyPart.(type) { - case string: - keyPart = kp[:prefixLength] - case []uint8: - keyPart = kp[:prefixLength] - default: - panic("what in tarnation is going on in here") - } - } - if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, keyPart); err != nil { - return err - } - if to < writer.prefixBld.Desc.Count() { - if err := index.PutField(ctx, writer.mut.NodeStore(), writer.prefixBld, to, keyPart); err != nil { - return err - } - } - } - - hashId, _, err := writer.primary.tuplesFromRow(ctx, sqlRow) - if err != nil { - return err - } - writer.keyBld.PutHash128(len(writer.keyBld.Desc.Types)-1, hashId.GetField(0)) - indexKey := writer.keyBld.Build(sharePool) - - if writer.unique { - prefixKey := writer.prefixBld.Build(sharePool) - err := writer.checkForUniqueKeyError(ctx, prefixKey) - if err != nil { - return err - } - } else { - writer.prefixBld.Recycle() - } - - return writer.mut.Put(ctx, indexKey, val.EmptyTuple) -} - -func (writer prollyKeylessSecondaryPrefixWriter) checkForUniqueKeyError(ctx context.Context, prefixKey val.Tuple) error { - for i := 0; i < writer.prefixBld.Desc.Count(); i++ { - if writer.prefixBld.Desc.IsNull(i, prefixKey) { - return nil - } - } - - rng := prolly.PrefixRange(prefixKey, writer.prefixBld.Desc) - itr, err := writer.mut.IterRange(ctx, rng) - if err != nil { - return err - } - k, _, err := itr.Next(ctx) - if err != nil && err != io.EOF { - return err - } - if err == nil { - keyStr := FormatKeyForUniqKeyErr(prefixKey, writer.prefixBld.Desc) - writer.hashBld.PutRaw(0, k.GetField(k.Count()-1)) - existingKey := writer.hashBld.Build(sharePool) - return secondaryUniqueKeyError{keyStr: keyStr, existingKey: existingKey} - } - return nil -} - -// Delete implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) Delete(ctx context.Context, sqlRow sql.Row) error { - hashId, cardRow, err := writer.primary.tuplesFromRow(ctx, sqlRow) - if err != nil { - return err - } - err = writer.primary.mut.Get(ctx, hashId, func(k, v val.Tuple) (err error) { - if k != nil { - cardRow = v - } - return - }) - if err != nil { - return err - } - - for to := range writer.keyMap { - from := writer.keyMap.MapOrdinal(to) - if err := index.PutField(ctx, writer.mut.NodeStore(), writer.keyBld, to, sqlRow[from]); err != nil { - return err - } - } - writer.keyBld.PutHash128(len(writer.keyBld.Desc.Types)-1, hashId.GetField(0)) - indexKey := writer.keyBld.Build(sharePool) - - // Indexes are always updated before the primary table, so we check if the deletion will cause the row to be removed - // from the primary. If not, then we just return. - card := val.ReadKeylessCardinality(cardRow) - if card > 1 { - return nil - } - return writer.mut.Delete(ctx, indexKey) -} - -// Update implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) Update(ctx context.Context, oldRow sql.Row, newRow sql.Row) (err error) { - if err = writer.Delete(ctx, oldRow); err != nil { - return err - } - if err = writer.Insert(ctx, newRow); err != nil { - return err - } - return -} - -// Commit implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) Commit(ctx context.Context) error { - return writer.mut.Checkpoint(ctx) -} - -// Discard implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) Discard(ctx context.Context) error { - writer.mut.Revert(ctx) - return nil -} - -// HasEdits implements the interface indexWriter. -func (writer prollyKeylessSecondaryPrefixWriter) HasEdits(ctx context.Context) bool { - return writer.mut.HasEdits() -} - -func (writer prollyKeylessSecondaryPrefixWriter) IterRange(ctx context.Context, rng prolly.Range) (prolly.MapIter, error) { - return writer.mut.IterRange(ctx, rng) -} diff --git a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go index 728f37a1c2..fce9d33454 100755 --- a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go @@ -78,31 +78,16 @@ func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch // mapping from secondary index key to primary key pkMap := makeIndexToIndexMapping(def.Schema().GetPKCols(), sch.GetPKCols()) - - prefixLengths := def.PrefixLengths() - if len(prefixLengths) == 0 { - writers[defName] = prollySecondaryIndexWriter{ - name: defName, - mut: idxMap.Mutate(), - unique: def.IsUnique(), - idxCols: def.Count(), - keyMap: keyMap, - keyBld: val.NewTupleBuilder(keyDesc), - pkMap: pkMap, - pkBld: val.NewTupleBuilder(pkDesc), - } - } else { - writers[defName] = prollySecondaryIndexPrefixWriter{ - name: defName, - mut: idxMap.Mutate(), - unique: def.IsUnique(), - prefixLengths: prefixLengths, - idxCols: def.Count(), - keyMap: keyMap, - keyBld: val.NewTupleBuilder(keyDesc), - pkMap: pkMap, - pkBld: val.NewTupleBuilder(pkDesc), - } + writers[defName] = prollySecondaryIndexWriter{ + name: defName, + mut: idxMap.Mutate(), + unique: def.IsUnique(), + prefixLengths: def.PrefixLengths(), + idxCols: def.Count(), + keyMap: keyMap, + keyBld: val.NewTupleBuilder(keyDesc), + pkMap: pkMap, + pkBld: val.NewTupleBuilder(pkDesc), } } @@ -130,30 +115,16 @@ func getSecondaryKeylessProllyWriters(ctx context.Context, t *doltdb.Table, sqlS keyMap, _ := ordinalMappingsFromSchema(sqlSch, def.Schema()) keyDesc, _ := m.Descriptors() - prefixLengths := def.PrefixLengths() - if len(prefixLengths) == 0 { - writers[defName] = prollyKeylessSecondaryWriter{ - name: defName, - mut: m.Mutate(), - primary: primary, - unique: def.IsUnique(), - keyBld: val.NewTupleBuilder(keyDesc), - prefixBld: val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())), - hashBld: val.NewTupleBuilder(val.NewTupleDescriptor(val.Type{Enc: val.Hash128Enc})), - keyMap: keyMap, - } - } else { - writers[defName] = prollyKeylessSecondaryPrefixWriter{ - name: defName, - mut: m.Mutate(), - primary: primary, - prefixLengths: prefixLengths, - unique: def.IsUnique(), - keyBld: val.NewTupleBuilder(keyDesc), - prefixBld: val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())), - hashBld: val.NewTupleBuilder(val.NewTupleDescriptor(val.Type{Enc: val.Hash128Enc})), - keyMap: keyMap, - } + writers[defName] = prollyKeylessSecondaryWriter{ + name: defName, + mut: m.Mutate(), + primary: primary, + unique: def.IsUnique(), + prefixLengths: def.PrefixLengths(), + keyBld: val.NewTupleBuilder(keyDesc), + prefixBld: val.NewTupleBuilder(keyDesc.PrefixDesc(def.Count())), + hashBld: val.NewTupleBuilder(val.NewTupleDescriptor(val.Type{Enc: val.Hash128Enc})), + keyMap: keyMap, } } From 8e95d3fded01f3d7d01487765e2f12d99e87fc2f Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 14:22:12 -0800 Subject: [PATCH 20/30] fix test signatures --- go/libraries/doltcore/schema/index_test.go | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/go/libraries/doltcore/schema/index_test.go b/go/libraries/doltcore/schema/index_test.go index d46b20e0fd..04870e5fc7 100644 --- a/go/libraries/doltcore/schema/index_test.go +++ b/go/libraries/doltcore/schema/index_test.go @@ -205,7 +205,7 @@ func TestIndexCollectionAddIndexByColNames(t *testing.T) { assert.False(t, indexColl.hasIndexOnTags(testIndex.index.IndexedColumnTags()...)) assert.Nil(t, indexColl.GetByName(testIndex.index.Name())) - resIndex, err := indexColl.AddIndexByColNames(testIndex.index.Name(), testIndex.cols, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) + resIndex, err := indexColl.AddIndexByColNames(testIndex.index.Name(), testIndex.cols, nil, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) assert.NoError(t, err) assert.Equal(t, testIndex.index, resIndex) assert.Equal(t, testIndex.index, indexColl.GetByName(resIndex.Name())) @@ -225,20 +225,20 @@ func TestIndexCollectionAddIndexByColNames(t *testing.T) { t.Run("Pre-existing", func(t *testing.T) { for _, testIndex := range testIndexes { - _, err := indexColl.AddIndexByColNames(testIndex.index.Name(), testIndex.cols, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) + _, err := indexColl.AddIndexByColNames(testIndex.index.Name(), testIndex.cols, nil, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) assert.NoError(t, err) - _, err = indexColl.AddIndexByColNames(testIndex.index.Name()+"copy", testIndex.cols, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) + _, err = indexColl.AddIndexByColNames(testIndex.index.Name()+"copy", testIndex.cols, nil, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) assert.NoError(t, err) - _, err = indexColl.AddIndexByColNames(testIndex.index.Name(), []string{"v2"}, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) + _, err = indexColl.AddIndexByColNames(testIndex.index.Name(), []string{"v2"}, nil, IndexProperties{IsUnique: testIndex.index.IsUnique(), Comment: testIndex.index.Comment()}) assert.Error(t, err) } indexColl.clear(t) }) t.Run("Non-existing Columns", func(t *testing.T) { - _, err := indexColl.AddIndexByColNames("nonsense", []string{"v4"}, IndexProperties{IsUnique: false, Comment: ""}) + _, err := indexColl.AddIndexByColNames("nonsense", []string{"v4"}, nil, IndexProperties{IsUnique: false, Comment: ""}) assert.Error(t, err) - _, err = indexColl.AddIndexByColNames("nonsense", []string{"v1", "v2", "pk3"}, IndexProperties{IsUnique: false, Comment: ""}) + _, err = indexColl.AddIndexByColNames("nonsense", []string{"v1", "v2", "pk3"}, nil, IndexProperties{IsUnique: false, Comment: ""}) assert.Error(t, err) }) } @@ -292,7 +292,7 @@ func TestIndexCollectionAddIndexByColTags(t *testing.T) { assert.False(t, indexColl.hasIndexOnTags(testIndex.IndexedColumnTags()...)) assert.Nil(t, indexColl.GetByName(testIndex.Name())) - resIndex, err := indexColl.AddIndexByColTags(testIndex.Name(), testIndex.tags, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) + resIndex, err := indexColl.AddIndexByColTags(testIndex.Name(), testIndex.tags, nil, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) assert.NoError(t, err) assert.Equal(t, testIndex, resIndex) assert.Equal(t, testIndex, indexColl.GetByName(resIndex.Name())) @@ -312,20 +312,20 @@ func TestIndexCollectionAddIndexByColTags(t *testing.T) { t.Run("Pre-existing", func(t *testing.T) { for _, testIndex := range testIndexes { - _, err := indexColl.AddIndexByColTags(testIndex.Name(), testIndex.tags, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) + _, err := indexColl.AddIndexByColTags(testIndex.Name(), testIndex.tags, nil, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) assert.NoError(t, err) - _, err = indexColl.AddIndexByColTags(testIndex.Name()+"copy", testIndex.tags, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) + _, err = indexColl.AddIndexByColTags(testIndex.Name()+"copy", testIndex.tags, nil, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) assert.NoError(t, err) - _, err = indexColl.AddIndexByColTags(testIndex.Name(), []uint64{4}, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) + _, err = indexColl.AddIndexByColTags(testIndex.Name(), []uint64{4}, nil, IndexProperties{IsUnique: testIndex.IsUnique(), Comment: testIndex.Comment()}) assert.Error(t, err) } indexColl.clear(t) }) t.Run("Non-existing Tags", func(t *testing.T) { - _, err := indexColl.AddIndexByColTags("nonsense", []uint64{6}, IndexProperties{IsUnique: false, Comment: ""}) + _, err := indexColl.AddIndexByColTags("nonsense", []uint64{6}, nil, IndexProperties{IsUnique: false, Comment: ""}) assert.Error(t, err) - _, err = indexColl.AddIndexByColTags("nonsense", []uint64{3, 4, 10}, IndexProperties{IsUnique: false, Comment: ""}) + _, err = indexColl.AddIndexByColTags("nonsense", []uint64{3, 4, 10}, nil, IndexProperties{IsUnique: false, Comment: ""}) assert.Error(t, err) }) } @@ -345,9 +345,9 @@ func TestIndexCollectionAllIndexes(t *testing.T) { tags: []uint64{3}, prefixLengths: []uint16{}, }) - _, err := indexColl.AddIndexByColNames("idx_a", []string{"v2"}, IndexProperties{IsUnique: false, Comment: ""}) + _, err := indexColl.AddIndexByColNames("idx_a", []string{"v2"}, nil, IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) - _, err = indexColl.AddIndexByColTags("idx_n", []uint64{5}, IndexProperties{IsUnique: false, Comment: "hello there"}) + _, err = indexColl.AddIndexByColTags("idx_n", []uint64{5}, nil, IndexProperties{IsUnique: false, Comment: "hello there"}) require.NoError(t, err) assert.Equal(t, []Index{ @@ -525,7 +525,7 @@ func TestIndexCollectionDuplicateIndexes(t *testing.T) { assert.Nil(t, indexColl.GetByName(origIndex.index.Name())) // Insert origIndex and see that no errors occur - resOrigIndex, err := indexColl.AddIndexByColNames(origIndex.index.Name(), origIndex.cols, IndexProperties{IsUnique: origIndex.index.IsUnique(), Comment: origIndex.index.Comment()}) + resOrigIndex, err := indexColl.AddIndexByColNames(origIndex.index.Name(), origIndex.cols, nil, IndexProperties{IsUnique: origIndex.index.IsUnique(), Comment: origIndex.index.Comment()}) assert.NoError(t, err) assert.Equal(t, origIndex.index, resOrigIndex) assert.Equal(t, origIndex.index, indexColl.GetByName(resOrigIndex.Name())) @@ -549,7 +549,7 @@ func TestIndexCollectionDuplicateIndexes(t *testing.T) { assert.Nil(t, indexColl.GetByName(copyIndex.index.Name())) // Insert copyIndex and see that no errors occur - resCopyIndex, err := indexColl.AddIndexByColNames(copyIndex.index.Name(), copyIndex.cols, IndexProperties{IsUnique: copyIndex.index.IsUnique(), Comment: copyIndex.index.Comment()}) + resCopyIndex, err := indexColl.AddIndexByColNames(copyIndex.index.Name(), copyIndex.cols, nil, IndexProperties{IsUnique: copyIndex.index.IsUnique(), Comment: copyIndex.index.Comment()}) assert.NoError(t, err) assert.Equal(t, copyIndex.index, resCopyIndex) assert.Equal(t, copyIndex.index, indexColl.GetByName(resCopyIndex.Name())) From f2c9dae1651bba98c6f9f8d3cd9a504749354d3a Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 14:27:23 -0800 Subject: [PATCH 21/30] fix go mod --- go/go.mod | 9 +++--- go/go.sum | 90 +++++-------------------------------------------------- 2 files changed, 12 insertions(+), 87 deletions(-) diff --git a/go/go.mod b/go/go.mod index 76e4c91d0b..6efd5a9ec4 100644 --- a/go/go.mod +++ b/go/go.mod @@ -37,20 +37,20 @@ require ( github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/sirupsen/logrus v1.8.1 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.0 github.com/tealeg/xlsx v1.0.5 github.com/tklauser/go-sysconf v0.3.9 // indirect go.uber.org/zap v1.15.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 + golang.org/x/sys v0.1.0 google.golang.org/api v0.32.0 google.golang.org/grpc v1.49.0 google.golang.org/protobuf v1.27.1 gopkg.in/square/go-jose.v2 v2.5.1 gopkg.in/src-d/go-errors.v1 v1.0.0 - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -77,7 +77,7 @@ require ( go.opentelemetry.io/otel/trace v1.7.0 golang.org/x/text v0.3.7 gonum.org/v1/plot v0.11.0 - gopkg.in/yaml.v3 v3.0.0 + gopkg.in/yaml.v2 v2.3.0 ) require ( @@ -117,7 +117,6 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect - github.com/stretchr/objx v0.2.0 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.22.4 // indirect diff --git a/go/go.sum b/go/go.sum index 417099190f..b6e720ecc3 100644 --- a/go/go.sum +++ b/go/go.sum @@ -692,19 +692,10 @@ github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRk github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= @@ -914,72 +905,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200620081246-981b61492c35/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1218,9 +1145,8 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 02e8bfd06e166bc7061e07591d0c6a07b6513c7c Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Thu, 10 Nov 2022 22:28:33 +0000 Subject: [PATCH 22/30] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/go.sum | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/go/go.sum b/go/go.sum index b6e720ecc3..7c269e433b 100644 --- a/go/go.sum +++ b/go/go.sum @@ -692,8 +692,18 @@ github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRk github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -905,6 +915,70 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200620081246-981b61492c35/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1145,6 +1219,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From f69291f009ad9d2ec7d1cfaaf601a914218942d5 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 14:29:13 -0800 Subject: [PATCH 23/30] fixing --- go/go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/go.mod b/go/go.mod index 6efd5a9ec4..44c199c0db 100644 --- a/go/go.mod +++ b/go/go.mod @@ -50,7 +50,7 @@ require ( google.golang.org/protobuf v1.27.1 gopkg.in/square/go-jose.v2 v2.5.1 gopkg.in/src-d/go-errors.v1 v1.0.0 - gopkg.in/yaml.v3 v3.0.1 + gopkg.in/yaml.v2 v2.3.0 ) require ( @@ -77,7 +77,7 @@ require ( go.opentelemetry.io/otel/trace v1.7.0 golang.org/x/text v0.3.7 gonum.org/v1/plot v0.11.0 - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( From 12b200872e1f17a2a9cee80bd8f32605eee330f3 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 14:42:12 -0800 Subject: [PATCH 24/30] removing extra prefix lengths --- .../schema/encoding/schema_marshaling.go | 13 +- go/libraries/doltcore/schema/index.go | 6 +- go/libraries/doltcore/schema/index_test.go | 232 ++++++++---------- 3 files changed, 113 insertions(+), 138 deletions(-) diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling.go b/go/libraries/doltcore/schema/encoding/schema_marshaling.go index 344a2f0b96..fceb691548 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling.go @@ -187,9 +187,11 @@ func (sd *schemaData) Copy() *schemaData { for j, tag := range idx.Tags { idxCol[i].Tags[j] = tag } - idxCol[i].PrefixLengths = make([]uint16, len(idx.PrefixLengths)) - for j, prefixLength := range idx.PrefixLengths { - idxCol[i].PrefixLengths[j] = prefixLength + if len(idx.PrefixLengths) > 0 { + idxCol[i].PrefixLengths = make([]uint16, len(idx.PrefixLengths)) + for j, prefixLength := range idx.PrefixLengths { + idxCol[i].PrefixLengths[j] = prefixLength + } } } } @@ -408,11 +410,6 @@ func UnmarshalSchemaNomsValue(ctx context.Context, nbf *types.NomsBinFormat, sch sd, err = toSchemaData(sch) } else { err = marshal.Unmarshal(ctx, nbf, schemaVal, &sd) - for i, enc := range sd.IndexCollection { - if enc.PrefixLengths == nil { - sd.IndexCollection[i].PrefixLengths = []uint16{} - } - } } if err != nil { return nil, err diff --git a/go/libraries/doltcore/schema/index.go b/go/libraries/doltcore/schema/index.go index 8ce89d62dd..fdeeb2729a 100644 --- a/go/libraries/doltcore/schema/index.go +++ b/go/libraries/doltcore/schema/index.go @@ -256,7 +256,9 @@ func (ix *indexImpl) copy() *indexImpl { _ = copy(newIx.tags, ix.tags) newIx.allTags = make([]uint64, len(ix.allTags)) _ = copy(newIx.allTags, ix.allTags) - newIx.prefixLengths = make([]uint16, len(ix.prefixLengths)) - _ = copy(newIx.prefixLengths, ix.prefixLengths) + if len(ix.prefixLengths) > 0 { + newIx.prefixLengths = make([]uint16, len(ix.prefixLengths)) + _ = copy(newIx.prefixLengths, ix.prefixLengths) + } return &newIx } diff --git a/go/libraries/doltcore/schema/index_test.go b/go/libraries/doltcore/schema/index_test.go index 04870e5fc7..5c3ad696c9 100644 --- a/go/libraries/doltcore/schema/index_test.go +++ b/go/libraries/doltcore/schema/index_test.go @@ -35,33 +35,29 @@ func TestIndexCollectionAddIndex(t *testing.T) { testIndexes := []*indexImpl{ { - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, }, { - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, - comment: "hello there", - prefixLengths: []uint16{}, + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, + comment: "hello there", }, { - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, }, { - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, }, } @@ -118,11 +114,10 @@ func TestIndexCollectionAddIndex(t *testing.T) { t.Run("Name Overwrites", func(t *testing.T) { // should be able to reduce collection to one index lastStanding := &indexImpl{ - name: "none", - tags: []uint64{4}, - allTags: []uint64{4, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "none", + tags: []uint64{4}, + allTags: []uint64{4, 1, 2}, + indexColl: indexColl, } for _, testIndex := range testIndexes { @@ -158,42 +153,38 @@ func TestIndexCollectionAddIndexByColNames(t *testing.T) { { []string{"v1"}, &indexImpl{ - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, }, }, { []string{"v1", "v3", "v2"}, &indexImpl{ - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, }, }, { []string{"pk1", "v1"}, &indexImpl{ - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, - comment: "hello there", - prefixLengths: []uint16{}, + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, + comment: "hello there", }, }, { []string{"pk2", "pk1", "v2"}, &indexImpl{ - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, }, }, } @@ -255,33 +246,29 @@ func TestIndexCollectionAddIndexByColTags(t *testing.T) { testIndexes := []*indexImpl{ { - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - comment: "hello there", - prefixLengths: []uint16{}, + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + comment: "hello there", }, { - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, }, { - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, }, { - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, }, } @@ -341,9 +328,8 @@ func TestIndexCollectionAllIndexes(t *testing.T) { indexColl := NewIndexCollection(colColl, nil).(*indexCollectionImpl) indexColl.AddIndex(&indexImpl{ - name: "idx_z", - tags: []uint64{3}, - prefixLengths: []uint16{}, + name: "idx_z", + tags: []uint64{3}, }) _, err := indexColl.AddIndexByColNames("idx_a", []string{"v2"}, nil, IndexProperties{IsUnique: false, Comment: ""}) require.NoError(t, err) @@ -352,31 +338,28 @@ func TestIndexCollectionAllIndexes(t *testing.T) { assert.Equal(t, []Index{ &indexImpl{ - name: "idx_a", - tags: []uint64{4}, - allTags: []uint64{4, 1, 2}, - indexColl: indexColl, - isUnique: false, - comment: "", - prefixLengths: []uint16{}, + name: "idx_a", + tags: []uint64{4}, + allTags: []uint64{4, 1, 2}, + indexColl: indexColl, + isUnique: false, + comment: "", }, &indexImpl{ - name: "idx_n", - tags: []uint64{5}, - allTags: []uint64{5, 1, 2}, - indexColl: indexColl, - isUnique: false, - comment: "hello there", - prefixLengths: []uint16{}, + name: "idx_n", + tags: []uint64{5}, + allTags: []uint64{5, 1, 2}, + indexColl: indexColl, + isUnique: false, + comment: "hello there", }, &indexImpl{ - name: "idx_z", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - isUnique: false, - comment: "", - prefixLengths: []uint16{}, + name: "idx_z", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, + isUnique: false, + comment: "", }, }, indexColl.AllIndexes()) } @@ -393,33 +376,29 @@ func TestIndexCollectionRemoveIndex(t *testing.T) { testIndexes := []Index{ &indexImpl{ - name: "idx_v1", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_v1", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, }, &indexImpl{ - name: "idx_v1v3v2", - tags: []uint64{3, 5, 4}, - allTags: []uint64{3, 5, 4, 1, 2}, - indexColl: indexColl, - comment: "hello there", - prefixLengths: []uint16{}, + name: "idx_v1v3v2", + tags: []uint64{3, 5, 4}, + allTags: []uint64{3, 5, 4, 1, 2}, + indexColl: indexColl, + comment: "hello there", }, &indexImpl{ - name: "idx_pk1v1", - tags: []uint64{1, 3}, - allTags: []uint64{1, 3, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_pk1v1", + tags: []uint64{1, 3}, + allTags: []uint64{1, 3, 2}, + indexColl: indexColl, }, &indexImpl{ - name: "idx_pk2pk1v2", - tags: []uint64{2, 1, 4}, - allTags: []uint64{2, 1, 4}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_pk2pk1v2", + tags: []uint64{2, 1, 4}, + allTags: []uint64{2, 1, 4}, + indexColl: indexColl, }, } indexColl.AddIndex(testIndexes...) @@ -448,11 +427,10 @@ func TestIndexCollectionRenameIndex(t *testing.T) { ) indexColl := NewIndexCollection(colColl, nil).(*indexCollectionImpl) index := &indexImpl{ - name: "idx_a", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "idx_a", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, } indexColl.AddIndex(index) @@ -495,11 +473,10 @@ func TestIndexCollectionDuplicateIndexes(t *testing.T) { }{ []string{"v1"}, &indexImpl{ - name: "v1_orig", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "v1_orig", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, }, } @@ -510,11 +487,10 @@ func TestIndexCollectionDuplicateIndexes(t *testing.T) { }{ []string{"v1"}, &indexImpl{ - name: "v1_copy", - tags: []uint64{3}, - allTags: []uint64{3, 1, 2}, - indexColl: indexColl, - prefixLengths: []uint16{}, + name: "v1_copy", + tags: []uint64{3}, + allTags: []uint64{3, 1, 2}, + indexColl: indexColl, }, } From 25cd26fd15ec3f839528473b037aee2e1800a058 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 14:58:34 -0800 Subject: [PATCH 25/30] nils --- go/libraries/doltcore/schema/encoding/serialization.go | 10 +++++++--- go/libraries/doltcore/schema/schema.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/schema/encoding/serialization.go b/go/libraries/doltcore/schema/encoding/serialization.go index ee5c983575..ba1c8f5476 100644 --- a/go/libraries/doltcore/schema/encoding/serialization.go +++ b/go/libraries/doltcore/schema/encoding/serialization.go @@ -381,9 +381,13 @@ func deserializeSecondaryIndexes(sch schema.Schema, s *serial.TableSchema) error tags[j] = col.Tag() } - prefixLengths := make([]uint16, idx.PrefixLengthsLength()) - for j := range prefixLengths { - prefixLengths[j] = idx.PrefixLengths(j) + var prefixLengths []uint16 + prefixLengthsLength := idx.PrefixLengthsLength() + if prefixLengthsLength > 0 { + prefixLengths = make([]uint16, prefixLengthsLength) + for j := range prefixLengths { + prefixLengths[j] = idx.PrefixLengths(j) + } } _, err := sch.Indexes().AddIndexByColTags(name, tags, prefixLengths, props) diff --git a/go/libraries/doltcore/schema/schema.go b/go/libraries/doltcore/schema/schema.go index 060542363c..34ac18f720 100644 --- a/go/libraries/doltcore/schema/schema.go +++ b/go/libraries/doltcore/schema/schema.go @@ -19,7 +19,7 @@ import ( "strings" "github.com/dolthub/vitess/go/vt/proto/query" - errors "gopkg.in/src-d/go-errors.v1" + "gopkg.in/src-d/go-errors.v1" "github.com/dolthub/dolt/go/libraries/utils/set" "github.com/dolthub/dolt/go/store/types" From ab078e87ab5a4f2ee89953d3d52102fbfb46acf1 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 10 Nov 2022 16:49:01 -0800 Subject: [PATCH 26/30] need to change hash? --- integration-tests/bats/migration-integration.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/bats/migration-integration.bats b/integration-tests/bats/migration-integration.bats index e344d8063c..1be12b7b44 100644 --- a/integration-tests/bats/migration-integration.bats +++ b/integration-tests/bats/migration-integration.bats @@ -29,7 +29,7 @@ teardown() { dolt tag -v run dolt tag -v [ "$status" -eq 0 ] - [[ "$output" =~ "tdkt7s7805k1ml4hu37pm688g5i0b8ie" ]] || false + [[ "$output" =~ "7jrvg1ajcdq6t9sevcejv4e9o0fgrmle" ]] || false [[ ! "$output" =~ "r9jv07tf9un3fm1fg72v7ad9er89oeo7" ]] || false # validate TEXT migration @@ -46,7 +46,7 @@ teardown() { dolt tag -v run dolt tag -v [ "$status" -eq 0 ] - [[ "$output" =~ "r9jv07tf9un3fm1fg72v7ad9er89oeo7" ]] || false + [[ "$output" =~ "7jrvg1ajcdq6t9sevcejv4e9o0fgrmle" ]] || false [[ ! "$output" =~ "tdkt7s7805k1ml4hu37pm688g5i0b8ie" ]] || false dolt migrate From 9b3dd8d42f3123b2e4a4ce087b467ee20e45a0a5 Mon Sep 17 00:00:00 2001 From: James Cor Date: Fri, 11 Nov 2022 10:50:38 -0800 Subject: [PATCH 27/30] replacing hashes --- integration-tests/bats/migration-integration.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/migration-integration.bats b/integration-tests/bats/migration-integration.bats index 1be12b7b44..ec53a9e33b 100644 --- a/integration-tests/bats/migration-integration.bats +++ b/integration-tests/bats/migration-integration.bats @@ -55,7 +55,7 @@ teardown() { dolt tag -v run dolt tag -v [ "$status" -eq 0 ] - [[ "$output" =~ "tdkt7s7805k1ml4hu37pm688g5i0b8ie" ]] || false + [[ "$output" =~ "7jrvg1ajcdq6t9sevcejv4e9o0fgrmle" ]] || false [[ ! "$output" =~ "r9jv07tf9un3fm1fg72v7ad9er89oeo7" ]] || false # validate TEXT migration From 68fe875ec866630b021e8b4e193bee0686a9d187 Mon Sep 17 00:00:00 2001 From: James Cor Date: Fri, 11 Nov 2022 11:02:44 -0800 Subject: [PATCH 28/30] removing comment --- go/libraries/doltcore/sqle/writer/prolly_table_writer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go index fce9d33454..2444bacf68 100755 --- a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go @@ -54,7 +54,6 @@ type prollyTableWriter struct { var _ TableWriter = &prollyTableWriter{} -// TODO: return a prefix index writer if there is a prefix defined on this table func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch sql.Schema, sch schema.Schema) (map[string]indexWriter, error) { s, err := t.GetIndexSet(ctx) if err != nil { From aa4c53858540ba5ea7dcdc6e6541ca230a9f5a26 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Fri, 11 Nov 2022 19:41:40 +0000 Subject: [PATCH 29/30] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/go.sum b/go/go.sum index 0df551469b..164cc13422 100644 --- a/go/go.sum +++ b/go/go.sum @@ -180,8 +180,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.14.1-0.20221110233726-0dafee9b8f80 h1:38XwYasYADK4SO+rxwmTD2hE8zngczzKSNo2XmYFe0Q= -github.com/dolthub/go-mysql-server v0.14.1-0.20221110233726-0dafee9b8f80/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= +github.com/dolthub/go-mysql-server v0.14.1-0.20221111192934-cf0c1818d579 h1:rOV6whqkxka2wGMGD/DOgUgX0jWw/gaJwTMqJ1ye2wA= +github.com/dolthub/go-mysql-server v0.14.1-0.20221111192934-cf0c1818d579/go.mod h1:KtpU4Sf7J+SIat/nxoA733QTn3tdL34NtoGxEBFcTsA= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From 3cb03a9b5d7cfc2eb907bcc90bb7047b15501638 Mon Sep 17 00:00:00 2001 From: James Cor Date: Fri, 11 Nov 2022 12:24:52 -0800 Subject: [PATCH 30/30] asdfasdfas --- integration-tests/bats/migration-integration.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/migration-integration.bats b/integration-tests/bats/migration-integration.bats index ec53a9e33b..b417ae3548 100644 --- a/integration-tests/bats/migration-integration.bats +++ b/integration-tests/bats/migration-integration.bats @@ -46,7 +46,7 @@ teardown() { dolt tag -v run dolt tag -v [ "$status" -eq 0 ] - [[ "$output" =~ "7jrvg1ajcdq6t9sevcejv4e9o0fgrmle" ]] || false + [[ "$output" =~ "r9jv07tf9un3fm1fg72v7ad9er89oeo7" ]] || false [[ ! "$output" =~ "tdkt7s7805k1ml4hu37pm688g5i0b8ie" ]] || false dolt migrate