mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-10 18:49:02 -06:00
Code Issues Packages Projects Releases 128 Wiki Activity

Updated the index interface

Browse Source
This commit is contained in:
Daylon Wilkins
2021-11-01 16:33:39 -07:00
committed by Daylon Wilkins
parent d11f5ec41b
commit 280bbb42f9
11 changed files with 272 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go/go.mod
View File

@@ -19,7 +19,7 @@ require (
github.com/denisbrodbeck/machineid v1.0.1
github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20201005193433-3ee972b1d078
github.com/dolthub/fslock v0.0.3
github.com/dolthub/go-mysql-server v0.11.1-0.20211029162420-43a1226b27d3
github.com/dolthub/go-mysql-server v0.11.1-0.20211101232357-2f55f4d5a9e7
github.com/dolthub/ishell v0.0.0-20210205014355-16a4ce758446
github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66
github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81

4
go/go.sum
View File

@@ -144,8 +144,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-mysql-server v0.11.1-0.20211029162420-43a1226b27d3 h1:xS2HsEKXij0yE/I2plvuFpG9/jp1PnlGdRhbbLH6DqM=
github.com/dolthub/go-mysql-server v0.11.1-0.20211029162420-43a1226b27d3/go.mod h1:NXWOVk1RyZI/mR7bghGYU+Zmb58mo37420r91O7aKGk=
github.com/dolthub/go-mysql-server v0.11.1-0.20211101232357-2f55f4d5a9e7 h1:EKqnEVVJc2N5MyZKd72AjWdg5qZsukraOXfeZ/iPQvA=
github.com/dolthub/go-mysql-server v0.11.1-0.20211101232357-2f55f4d5a9e7/go.mod h1:NXWOVk1RyZI/mR7bghGYU+Zmb58mo37420r91O7aKGk=
github.com/dolthub/ishell v0.0.0-20210205014355-16a4ce758446 h1:0ol5pj+QlKUKAtqs1LiPM3ZJKs+rHPgLSsMXmhTrCAM=
github.com/dolthub/ishell v0.0.0-20210205014355-16a4ce758446/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms=
github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=

296
go/libraries/doltcore/sqle/dolt_index.go
View File

@@ -1,4 +1,4 @@
// Copyright 2020 Dolthub, Inc.
// Copyright 2020-2021 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@ package sqle
import (
"context"
"errors"
"fmt"
"github.com/dolthub/go-mysql-server/sql"
@@ -28,14 +29,11 @@ import (
type DoltIndex interface {
sql.Index
sql.AscendIndex
sql.DescendIndex
sql.NegateIndex
Schema() schema.Schema
IndexSchema() schema.Schema
TableData() types.Map
IndexRowData() types.Map
Equals(index DoltIndex) bool
EqualsDoltIndex(index DoltIndex) bool
}
type doltIndex struct {
@@ -53,100 +51,84 @@ type doltIndex struct {
generated bool
}
//TODO: have queries using IS NULL make use of indexes
var _ DoltIndex = (*doltIndex)(nil)
// AscendGreaterOrEqual implements sql.AscendIndex
func (di *doltIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) {
tpl, err := di.keysToTuple(keys)
if err != nil {
return nil, err
// ColumnExpressionTypes implements the interface sql.Index.
func (di *doltIndex) ColumnExpressionTypes(ctx *sql.Context) []sql.ColumnExpressionType {
cets := make([]sql.ColumnExpressionType, len(di.cols))
for i, col := range di.cols {
cets[i] = sql.ColumnExpressionType{
Expression: di.tableName + "." + col.Name,
Type: col.TypeInfo.ToSqlType(),
}
}
return &doltIndexLookup{
idx: di,
ranges: []lookup.Range{
lookup.GreaterOrEqualRange(tpl),
},
}, nil
return cets
}
// AscendLessThan implements sql.AscendIndex
func (di *doltIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) {
tpl, err := di.keysToTuple(keys)
if err != nil {
return nil, err
// NewLookup implements the interface sql.Index.
func (di *doltIndex) NewLookup(ctx *sql.Context, ranges ...sql.Range) (sql.IndexLookup, error) {
if len(ranges) == 0 {
return nil, nil
}
exprs := di.Expressions()
if len(ranges[0]) > len(exprs) {
return nil, nil
}
idx := di
if len(ranges[0]) < len(exprs) {
idx = idx.prefix(len(ranges[0]))
exprs = idx.Expressions()
}
return &doltIndexLookup{
idx: di,
ranges: []lookup.Range{
lookup.LessThanRange(tpl),
},
}, nil
}
// AscendRange implements sql.AscendIndex
// TODO: rename this from AscendRange to BetweenRange or something
func (di *doltIndex) AscendRange(greaterOrEqual, lessThanOrEqual []interface{}) (sql.IndexLookup, error) {
greaterTpl, err := di.keysToTuple(greaterOrEqual)
if err != nil {
return nil, err
}
lessTpl, err := di.keysToTuple(lessThanOrEqual)
if err != nil {
return nil, err
}
r, err := lookup.ClosedRange(greaterTpl, lessTpl)
if err != nil {
return nil, err
}
return &doltIndexLookup{
idx: di,
ranges: []lookup.Range{
r,
},
}, nil
}
var lookupRanges []lookup.Range
// DescendGreater implements sql.DescendIndex
func (di *doltIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) {
tpl, err := di.keysToTuple(keys)
if err != nil {
return nil, err
}
r, err := lookup.GreaterThanRange(tpl)
if err != nil {
return nil, err
}
return &doltIndexLookup{
idx: di,
ranges: []lookup.Range{
r,
},
}, nil
}
for _, rang := range ranges {
// TODO: support mixing range types and lengths, which will allow us to support indexing over mixed operators
// For now, grab a range to have a baseline so that we may enforce parity with all other ranges
var rangeTypes []sql.RangeType
rangeTypes = make([]sql.RangeType, len(rang[0]))
for i, r := range rang[0] {
rangeTypes[i] = r.Type()
}
// DescendLessOrEqual implements sql.DescendIndex
func (di *doltIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) {
tpl, err := di.keysToTuple(keys)
if err != nil {
return nil, err
}
r, err := lookup.LessOrEqualRange(tpl)
if err != nil {
return nil, err
}
return &doltIndexLookup{
idx: di,
ranges: []lookup.Range{
r,
},
}, nil
}
for i, rangeType := range rangeTypes {
var keys1 []interface{} // used if only one bound is set, or if both bounds are set represents the lowerbound
var keys2 []interface{} // used only when both bounds are set, thus will always represent the upperbound
// DescendRange implements sql.DescendIndex
// TODO: fix go-mysql-server to remove this duplicate function
func (di *doltIndex) DescendRange(lessOrEqual, greaterOrEqual []interface{}) (sql.IndexLookup, error) {
return di.AscendRange(greaterOrEqual, lessOrEqual)
for _, rangeColumn := range rang {
if len(rangeTypes) != len(rangeColumn) {
return nil, nil //TODO: support indexes having different range counts
}
rangeColumnExpr := rangeColumn[i]
if rangeColumnExpr.Type() != rangeType {
return nil, nil //TODO: support mixing range types
}
hasLower := rangeColumnExpr.HasLowerBound()
hasUpper := rangeColumnExpr.HasUpperBound()
if hasLower && hasUpper {
keys1 = append(keys1, sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
keys2 = append(keys2, sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
} else if hasLower && !hasUpper {
keys1 = append(keys1, sql.GetRangeCutKey(rangeColumnExpr.LowerBound))
} else if !hasLower && hasUpper {
keys1 = append(keys1, sql.GetRangeCutKey(rangeColumnExpr.UpperBound))
}
}
lookupRange, err := idx.sqlRangeToLookupRange(ctx, rangeType, keys1, keys2)
if err != nil {
return nil, err
}
lookupRanges = append(lookupRanges, lookupRange)
}
}
return &doltIndexLookup{
idx: idx,
ranges: lookupRanges,
sqlRanges: ranges,
}, nil
}
// Database implement sql.Index
@@ -163,49 +145,6 @@ func (di *doltIndex) Expressions() []string {
return strs
}
// Get implements sql.Index
func (di *doltIndex) Get(keys ...interface{}) (sql.IndexLookup, error) {
tpl, err := di.keysToTuple(keys)
if err != nil {
return nil, err
}
r, err := lookup.ClosedRange(tpl, tpl)
if err != nil {
return nil, err
}
return &doltIndexLookup{
idx: di,
ranges: []lookup.Range{
r,
},
}, nil
}
// Not implements sql.NegateIndex
func (di *doltIndex) Not(keys ...interface{}) (sql.IndexLookup, error) {
tpl, err := di.keysToTuple(keys)
if err != nil {
return nil, err
}
r1 := lookup.LessThanRange(tpl)
r2, err := lookup.GreaterThanRange(tpl)
if err != nil {
return nil, err
}
return &doltIndexLookup{
idx: di,
ranges: []lookup.Range{
r1,
r2,
},
}, nil
}
// Has implements sql.Index
func (*doltIndex) Has(partition sql.Partition, key ...interface{}) (bool, error) {
return false, errors.New("unimplemented")
}
// ID implements sql.Index
func (di *doltIndex) ID() string {
return di.id
@@ -236,7 +175,7 @@ func (di *doltIndex) Schema() schema.Schema {
return di.tableSch
}
// Schema returns the dolt index schema.
// IndexSchema returns the dolt index schema.
func (di *doltIndex) IndexSchema() schema.Schema {
return di.indexSch
}
@@ -256,6 +195,95 @@ func (di *doltIndex) IndexRowData() types.Map {
return di.indexRowData
}
// sqlRangeToLookupRange takes a range returned by the sql engine and converts it to the appropriate lookup range used for noms traversal.
func (di *doltIndex) sqlRangeToLookupRange(ctx *sql.Context, rangeType sql.RangeType, keys1, keys2 []interface{}) (lookup.Range, error) {
switch rangeType {
case sql.RangeType_Empty:
return lookup.EmptyRange(), nil
case sql.RangeType_All:
return lookup.AllRange(), nil
case sql.RangeType_GreaterThan:
tpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
return lookup.GreaterThanRange(tpl)
case sql.RangeType_GreaterOrEqual:
tpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
return lookup.GreaterOrEqualRange(tpl), nil
case sql.RangeType_LessThan:
tpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
return lookup.LessThanRange(tpl), nil
case sql.RangeType_LessOrEqual:
tpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
return lookup.LessOrEqualRange(tpl)
case sql.RangeType_ClosedClosed:
lowerTpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
upperTpl, err := di.keysToTuple(keys2)
if err != nil {
return lookup.Range{}, err
}
return lookup.ClosedRange(lowerTpl, upperTpl)
case sql.RangeType_OpenOpen:
lowerTpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
upperTpl, err := di.keysToTuple(keys2)
if err != nil {
return lookup.Range{}, err
}
return lookup.OpenRange(lowerTpl, upperTpl)
case sql.RangeType_OpenClosed:
lowerTpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
upperTpl, err := di.keysToTuple(keys2)
if err != nil {
return lookup.Range{}, err
}
return lookup.CustomRange(lowerTpl, upperTpl, lookup.Open, lookup.Closed)
case sql.RangeType_ClosedOpen:
lowerTpl, err := di.keysToTuple(keys1)
if err != nil {
return lookup.Range{}, err
}
upperTpl, err := di.keysToTuple(keys2)
if err != nil {
return lookup.Range{}, err
}
return lookup.CustomRange(lowerTpl, upperTpl, lookup.Closed, lookup.Open)
}
return lookup.Range{}, sql.ErrInvalidRangeType.New()
}
// prefix returns a copy of this index with only the first n columns. If n is >= the number of columns present, then
// the exact index is returned without copying.
func (di *doltIndex) prefix(n int) *doltIndex {
if n >= len(di.cols) {
return di
}
ndi := *di
ndi.cols = di.cols[:n]
ndi.id = fmt.Sprintf("%s_PREFIX_%d", di.id, n)
ndi.comment = fmt.Sprintf("prefix of %s multi-column index on %d column(s)", di.id, n)
ndi.generated = true
return &ndi
}
func (di *doltIndex) keysToTuple(keys []interface{}) (types.Tuple, error) {
nbf := di.indexRowData.Format()
if len(di.cols) != len(keys) {
@@ -274,7 +302,7 @@ func (di *doltIndex) keysToTuple(keys []interface{}) (types.Tuple, error) {
return types.NewTuple(nbf, vals...)
}
func (di *doltIndex) Equals(oIdx DoltIndex) bool {
func (di *doltIndex) EqualsDoltIndex(oIdx DoltIndex) bool {
if !expressionsAreEquals(di.Expressions(), oIdx.Expressions()) {
return false
}

73
go/libraries/doltcore/sqle/dolt_index_test.go
View File

@@ -34,6 +34,17 @@ import (
"github.com/dolthub/dolt/go/store/types"
)
type indexComp int
const (
indexComp_Eq = iota
indexComp_NEq
indexComp_Gt
indexComp_GtE
indexComp_Lt
indexComp_LtE
)
type doltIndexTestCase struct {
indexName string
keys []interface{}
@@ -284,7 +295,7 @@ func TestDoltIndexEqual(t *testing.T) {
t.Run(fmt.Sprintf("%s|%v", test.indexName, test.keys), func(t *testing.T) {
index, ok := indexMap[test.indexName]
require.True(t, ok)
testDoltIndex(t, test.keys, test.expectedRows, index.Get)
testDoltIndex(t, test.keys, test.expectedRows, index, indexComp_Eq)
})
}
}
@@ -415,7 +426,7 @@ func TestDoltIndexGreaterThan(t *testing.T) {
t.Run(fmt.Sprintf("%s|%v", test.indexName, test.keys), func(t *testing.T) {
index, ok := indexMap[test.indexName]
require.True(t, ok)
testDoltIndex(t, test.keys, test.expectedRows, index.DescendGreater)
testDoltIndex(t, test.keys, test.expectedRows, index, indexComp_Gt)
})
}
}
@@ -552,7 +563,7 @@ func TestDoltIndexGreaterThanOrEqual(t *testing.T) {
t.Run(fmt.Sprintf("%s|%v", test.indexName, test.keys), func(t *testing.T) {
index, ok := indexMap[test.indexName]
require.True(t, ok)
testDoltIndex(t, test.keys, test.expectedRows, index.AscendGreaterOrEqual)
testDoltIndex(t, test.keys, test.expectedRows, index, indexComp_GtE)
})
}
}
@@ -683,7 +694,7 @@ func TestDoltIndexLessThan(t *testing.T) {
t.Run(fmt.Sprintf("%s|%v", test.indexName, test.keys), func(t *testing.T) {
index, ok := indexMap[test.indexName]
require.True(t, ok)
testDoltIndex(t, test.keys, test.expectedRows, index.AscendLessThan)
testDoltIndex(t, test.keys, test.expectedRows, index, indexComp_Lt)
})
}
}
@@ -820,7 +831,7 @@ func TestDoltIndexLessThanOrEqual(t *testing.T) {
t.Run(fmt.Sprintf("%s|%v", test.indexName, test.keys), func(t *testing.T) {
index, ok := indexMap[test.indexName]
require.True(t, ok)
testDoltIndex(t, test.keys, test.expectedRows, index.DescendLessOrEqual)
testDoltIndex(t, test.keys, test.expectedRows, index, indexComp_LtE)
})
}
}
@@ -984,16 +995,22 @@ func TestDoltIndexBetween(t *testing.T) {
for _, test := range tests {
t.Run(fmt.Sprintf("%s|%v%v", test.indexName, test.greaterThanOrEqual, test.lessThanOrEqual), func(t *testing.T) {
ctx := NewTestSQLCtx(context.Background())
index, ok := indexMap[test.indexName]
require.True(t, ok)
expectedRows := convertSqlRowToInt64(test.expectedRows)
indexLookup, err := index.AscendRange(test.greaterThanOrEqual, test.lessThanOrEqual)
exprs := index.Expressions()
sqlIndex := sql.NewIndexBuilder(ctx, index)
for i := range test.greaterThanOrEqual {
sqlIndex = sqlIndex.GreaterOrEqual(ctx, exprs[i], test.greaterThanOrEqual[i]).LessOrEqual(ctx, exprs[i], test.lessThanOrEqual[i])
}
indexLookup, err := sqlIndex.Build(ctx)
require.NoError(t, err)
dil, ok := indexLookup.(*doltIndexLookup)
require.True(t, ok)
indexIter, err := dil.RowIter(NewTestSQLCtx(context.Background()), dil.IndexRowData(), nil)
indexIter, err := dil.RowIter(ctx, dil.IndexRowData(), nil)
require.NoError(t, err)
var readRows []sql.Row
@@ -1004,21 +1021,6 @@ func TestDoltIndexBetween(t *testing.T) {
require.Equal(t, io.EOF, err)
requireUnorderedRowsEqual(t, expectedRows, readRows)
indexLookup, err = index.DescendRange(test.lessThanOrEqual, test.greaterThanOrEqual)
require.NoError(t, err)
dil, ok = indexLookup.(*doltIndexLookup)
require.True(t, ok)
indexIter, err = dil.RowIter(NewTestSQLCtx(context.Background()), dil.IndexRowData(), nil)
require.NoError(t, err)
readRows = nil
for nextRow, err = indexIter.Next(); err == nil; nextRow, err = indexIter.Next() {
readRows = append(readRows, nextRow)
}
require.Equal(t, io.EOF, err)
requireUnorderedRowsEqual(t, expectedRows, readRows)
})
}
}
@@ -1202,12 +1204,33 @@ func requireUnorderedRowsEqual(t *testing.T, rows1, rows2 []sql.Row) {
require.Equal(t, rows1, rows2)
}
func testDoltIndex(t *testing.T, keys []interface{}, expectedRows []sql.Row, indexLookupFn func(keys ...interface{}) (sql.IndexLookup, error)) {
indexLookup, err := indexLookupFn(keys...)
func testDoltIndex(t *testing.T, keys []interface{}, expectedRows []sql.Row, index sql.Index, cmp indexComp) {
ctx := NewTestSQLCtx(context.Background())
exprs := index.Expressions()
builder := sql.NewIndexBuilder(sql.NewEmptyContext(), index)
for i, key := range keys {
switch cmp {
case indexComp_Eq:
builder = builder.Equals(ctx, exprs[i], key)
case indexComp_NEq:
builder = builder.NotEquals(ctx, exprs[i], key)
case indexComp_Gt:
builder = builder.GreaterThan(ctx, exprs[i], key)
case indexComp_GtE:
builder = builder.GreaterOrEqual(ctx, exprs[i], key)
case indexComp_Lt:
builder = builder.LessThan(ctx, exprs[i], key)
case indexComp_LtE:
builder = builder.LessOrEqual(ctx, exprs[i], key)
default:
panic("should not be hit")
}
}
indexLookup, err := builder.Build(ctx)
require.NoError(t, err)
dil, ok := indexLookup.(*doltIndexLookup)
require.True(t, ok)
indexIter, err := dil.RowIter(NewTestSQLCtx(context.Background()), dil.IndexRowData(), nil)
indexIter, err := dil.RowIter(ctx, dil.IndexRowData(), nil)
require.NoError(t, err)
var readRows []sql.Row

96
go/libraries/doltcore/sqle/index_lookup.go
View File

@@ -32,11 +32,12 @@ type IndexLookupKeyIterator interface {
}
type doltIndexLookup struct {
idx DoltIndex
ranges []lookup.Range // The collection of ranges that represent this lookup.
idx DoltIndex
ranges []lookup.Range // The collection of ranges that represent this lookup.
sqlRanges sql.RangeCollection
}
var _ sql.MergeableIndexLookup = (*doltIndexLookup)(nil)
var _ sql.IndexLookup = (*doltIndexLookup)(nil)
func (il *doltIndexLookup) String() string {
// TODO: this could be expanded with additional info (like the expression used to create the index lookup)
@@ -47,91 +48,14 @@ func (il *doltIndexLookup) IndexRowData() types.Map {
return il.idx.IndexRowData()
}
// IsMergeable implements sql.MergeableIndexLookup
func (il *doltIndexLookup) IsMergeable(indexLookup sql.IndexLookup) bool {
otherIl, ok := indexLookup.(*doltIndexLookup)
if !ok {
return false
}
return il.idx.Equals(otherIl.idx)
// Index implements the interface sql.IndexLookup
func (il *doltIndexLookup) Index() sql.Index {
return il.idx
}
// Intersection implements sql.MergeableIndexLookup
func (il *doltIndexLookup) Intersection(indexLookups ...sql.IndexLookup) (sql.IndexLookup, error) {
rangeCombinations := make([][]lookup.Range, len(il.ranges))
for i, ilRange := range il.ranges {
rangeCombinations[i] = []lookup.Range{ilRange}
}
for _, indexLookup := range indexLookups {
otherIl, ok := indexLookup.(*doltIndexLookup)
if !ok {
return nil, fmt.Errorf("failed to intersect sql.IndexLookup with type '%T'", indexLookup)
}
var newRangeCombination [][]lookup.Range
for _, rangeCombination := range rangeCombinations {
for _, ilRange := range otherIl.ranges {
rc := make([]lookup.Range, len(rangeCombination)+1)
copy(rc, rangeCombination)
rc[len(rangeCombination)] = ilRange
newRangeCombination = append(newRangeCombination, rc)
}
}
rangeCombinations = newRangeCombination
}
var newRanges []lookup.Range
var err error
var ok bool
for _, rangeCombination := range rangeCombinations {
intersectedRange := lookup.AllRange()
for _, rangeToIntersect := range rangeCombination {
intersectedRange, ok, err = intersectedRange.TryIntersect(rangeToIntersect)
if err != nil {
return nil, err
}
if !ok {
break
}
}
if !intersectedRange.IsEmpty() {
newRanges = append(newRanges, intersectedRange)
}
}
newRanges, err = lookup.SimplifyRanges(newRanges)
if err != nil {
return nil, err
}
return &doltIndexLookup{
idx: il.idx,
ranges: newRanges,
}, nil
}
// Union implements sql.MergeableIndexLookup
func (il *doltIndexLookup) Union(indexLookups ...sql.IndexLookup) (sql.IndexLookup, error) {
var ranges []lookup.Range
var err error
if len(il.ranges) == 0 {
ranges = []lookup.Range{lookup.EmptyRange()}
} else {
ranges = make([]lookup.Range, len(il.ranges))
copy(ranges, il.ranges)
}
for _, indexLookup := range indexLookups {
otherIl, ok := indexLookup.(*doltIndexLookup)
if !ok {
return nil, fmt.Errorf("failed to union sql.IndexLookup with type '%T'", indexLookup)
}
ranges = append(ranges, otherIl.ranges...)
}
ranges, err = lookup.SimplifyRanges(ranges)
if err != nil {
return nil, err
}
return &doltIndexLookup{
idx: il.idx,
ranges: ranges,
}, nil
// Ranges implements the interface sql.IndexLookup
func (il *doltIndexLookup) Ranges() sql.RangeCollection {
return il.sqlRanges
}
// RowIter returns a row iterator for this index lookup. The iterator will return the single matching row for the index.

2
go/libraries/doltcore/sqle/lookup/below.go
View File

@@ -20,7 +20,7 @@ import (
"github.com/dolthub/dolt/go/store/types"
)
// Above represents the position immediately below the contained key.
// Below represents the position immediately below the contained key.
type Below struct {
key types.Tuple
}

166
go/libraries/doltcore/sqle/mergeable_indexes_setup_test.go
View File

@@ -35,7 +35,7 @@ import (
"github.com/dolthub/dolt/go/store/types"
)
func setupMergeableIndexes(t *testing.T, tableName, insertQuery string) (*sqle.Engine, *env.DoltEnv, *testMergeableIndexDb, []*indexTuple, *doltdb.RootValue) {
func setupIndexes(t *testing.T, tableName, insertQuery string) (*sqle.Engine, *env.DoltEnv, *testIndexDb, []*indexTuple, *doltdb.RootValue) {
dEnv := dtestutils.CreateTestEnv()
root, err := dEnv.WorkingRoot(context.Background())
require.NoError(t, err)
@@ -94,22 +94,22 @@ func setupMergeableIndexes(t *testing.T, tableName, insertQuery string) (*sqle.E
cols: idxv2v1Cols,
}
mergeableDb := &testMergeableIndexDb{
tiDb := &testIndexDb{
t: t,
tbl: tbl,
editOpts: opts,
}
pro := NewDoltDatabaseProvider(dEnv.Config, mergeableDb)
pro := NewDoltDatabaseProvider(dEnv.Config, tiDb)
engine = sqle.NewDefault(pro)
// Get an updated root to use for the rest of the test
ctx := sql.NewEmptyContext()
sess, err := dsess.NewDoltSession(ctx, ctx.Session.(*sql.BaseSession), pro, dEnv.Config, getDbState(t, db, dEnv))
require.NoError(t, err)
roots, ok := sess.GetRoots(ctx, mergeableDb.Name())
roots, ok := sess.GetRoots(ctx, tiDb.Name())
require.True(t, ok)
return engine, dEnv, mergeableDb, []*indexTuple{
return engine, dEnv, tiDb, []*indexTuple{
idxv1ToTuple,
idxv2v1ToTuple,
{
@@ -119,25 +119,25 @@ func setupMergeableIndexes(t *testing.T, tableName, insertQuery string) (*sqle.E
}, roots.Working
}
// Database made to test mergeable indexes while using the full SQL engine.
type testMergeableIndexDb struct {
// Database made to test indexes while using the full SQL engine.
type testIndexDb struct {
t *testing.T
tbl *AlterableDoltTable
finalRanges func([]lookup.Range) // We return the final range set to compare to the expected ranges
editOpts editor.Options
}
func (db *testMergeableIndexDb) EditOptions() editor.Options {
func (db *testIndexDb) EditOptions() editor.Options {
return db.editOpts
}
func (db *testMergeableIndexDb) Name() string {
func (db *testIndexDb) Name() string {
return "dolt"
}
func (db *testMergeableIndexDb) GetTableInsensitive(_ *sql.Context, tblName string) (sql.Table, bool, error) {
func (db *testIndexDb) GetTableInsensitive(_ *sql.Context, tblName string) (sql.Table, bool, error) {
if strings.ToLower(tblName) == strings.ToLower(db.tbl.tableName) {
return &testMergeableIndexTable{
return &testIndexTable{
AlterableDoltTable: db.tbl,
t: db.t,
finalRanges: db.finalRanges,
@@ -145,27 +145,27 @@ func (db *testMergeableIndexDb) GetTableInsensitive(_ *sql.Context, tblName stri
}
return nil, false, nil
}
func (db *testMergeableIndexDb) GetTableNames(_ *sql.Context) ([]string, error) {
func (db *testIndexDb) GetTableNames(_ *sql.Context) ([]string, error) {
return []string{db.tbl.tableName}, nil
}
// Table made to test mergeable indexes by intercepting specific index-related functions.
type testMergeableIndexTable struct {
// Table made to test indexes by intercepting specific index-related functions.
type testIndexTable struct {
*AlterableDoltTable
t *testing.T
il *testMergeableIndexLookup
il *testIndexLookup
finalRanges func([]lookup.Range) // We return the final range set to compare to the expected ranges
}
var _ sql.IndexedTable = (*testMergeableIndexTable)(nil)
var _ sql.IndexedTable = (*testIndexTable)(nil)
func (tbl *testMergeableIndexTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) {
func (tbl *testIndexTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) {
indexes, err := tbl.AlterableDoltTable.GetIndexes(ctx)
if err != nil {
return nil, err
}
for i, index := range indexes {
indexes[i] = &testMergeableDoltIndex{
indexes[i] = &testIndex{
doltIndex: index.(*doltIndex),
t: tbl.t,
finalRanges: tbl.finalRanges,
@@ -174,10 +174,10 @@ func (tbl *testMergeableIndexTable) GetIndexes(ctx *sql.Context) ([]sql.Index, e
return indexes, nil
}
func (tbl *testMergeableIndexTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table {
il, ok := lookup.(*testMergeableIndexLookup)
func (tbl *testIndexTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table {
il, ok := lookup.(*testIndexLookup)
require.True(tbl.t, ok)
return &testMergeableIndexTable{
return &testIndexTable{
AlterableDoltTable: tbl.AlterableDoltTable,
t: tbl.t,
il: il,
@@ -185,137 +185,57 @@ func (tbl *testMergeableIndexTable) WithIndexLookup(lookup sql.IndexLookup) sql.
}
}
type testProjectedMergableIndexTable struct {
*testMergeableIndexTable
type testProjectedIndexTable struct {
*testIndexTable
cols []string
}
func (tbl *testMergeableIndexTable) WithProjection(colNames []string) sql.Table {
return &testProjectedMergableIndexTable{tbl, colNames}
func (tbl *testIndexTable) WithProjection(colNames []string) sql.Table {
return &testProjectedIndexTable{tbl, colNames}
}
func (tbl *testMergeableIndexTable) Partitions(_ *sql.Context) (sql.PartitionIter, error) {
func (tbl *testIndexTable) Partitions(_ *sql.Context) (sql.PartitionIter, error) {
rowData := tbl.il.IndexRowData()
return sqlutil.NewSinglePartitionIter(rowData), nil
}
func (tbl *testMergeableIndexTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
func (tbl *testIndexTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
return tbl.il.RowIter(ctx, part.(sqlutil.SinglePartition).RowData)
}
// Index made to test mergeable indexes by intercepting all calls that return lookups and returning modified lookups.
type testMergeableDoltIndex struct {
// Index made to test indexes by intercepting all calls that return index builders and returning modified builders.
type testIndex struct {
*doltIndex
t *testing.T
finalRanges func([]lookup.Range) // We return the final range set to compare to the expected ranges
}
func (di *testMergeableDoltIndex) Get(keys ...interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.Get(keys...)
return &testMergeableIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
t: di.t,
finalRanges: di.finalRanges,
}, err
}
func (di *testMergeableDoltIndex) Not(keys ...interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.Not(keys...)
return &testMergeableIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
t: di.t,
finalRanges: di.finalRanges,
}, err
}
func (di *testMergeableDoltIndex) AscendGreaterOrEqual(keys ...interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.AscendGreaterOrEqual(keys...)
return &testMergeableIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
t: di.t,
finalRanges: di.finalRanges,
}, err
}
func (di *testMergeableDoltIndex) AscendLessThan(keys ...interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.AscendLessThan(keys...)
return &testMergeableIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
t: di.t,
finalRanges: di.finalRanges,
}, err
}
func (di *testMergeableDoltIndex) AscendRange(greaterOrEqual, lessThanOrEqual []interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.AscendRange(greaterOrEqual, lessThanOrEqual)
return &testMergeableIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
t: di.t,
finalRanges: di.finalRanges,
}, err
}
func (di *testMergeableDoltIndex) DescendGreater(keys ...interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.DescendGreater(keys...)
return &testMergeableIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
t: di.t,
finalRanges: di.finalRanges,
}, err
}
func (di *testMergeableDoltIndex) DescendLessOrEqual(keys ...interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.DescendLessOrEqual(keys...)
return &testMergeableIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
t: di.t,
finalRanges: di.finalRanges,
}, err
}
func (di *testMergeableDoltIndex) DescendRange(lessOrEqual, greaterOrEqual []interface{}) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.DescendRange(lessOrEqual, greaterOrEqual)
return &testMergeableIndexLookup{
var _ sql.Index = (*testIndex)(nil)
func (di *testIndex) NewLookup(ctx *sql.Context, ranges ...sql.Range) (sql.IndexLookup, error) {
indexLookup, err := di.doltIndex.NewLookup(ctx, ranges...)
return &testIndexLookup{
doltIndexLookup: indexLookup.(*doltIndexLookup),
testIdx: di,
t: di.t,
finalRanges: di.finalRanges,
}, err
}
// Lookup made to test mergeable indexes by intercepting the lookup functions and adding tracking for testing.
type testMergeableIndexLookup struct {
// Lookup made to test indexes by intercepting the lookup functions and adding tracking for testing.
type testIndexLookup struct {
*doltIndexLookup
testIdx *testIndex
t *testing.T
finalRanges func([]lookup.Range) // We return the final range set to compare to the expected ranges
}
func (il *testMergeableIndexLookup) IsMergeable(indexLookup sql.IndexLookup) bool {
return il.doltIndexLookup.IsMergeable(indexLookup.(*testMergeableIndexLookup).doltIndexLookup)
var _ sql.IndexLookup = (*testIndexLookup)(nil)
func (il *testIndexLookup) Index() sql.Index {
return il.testIdx
}
func (il *testMergeableIndexLookup) Intersection(indexLookups ...sql.IndexLookup) (sql.IndexLookup, error) {
newLookups := make([]sql.IndexLookup, len(indexLookups))
for i, otherIl := range indexLookups {
newLookups[i] = otherIl.(*testMergeableIndexLookup).doltIndexLookup
}
intersectedIl, err := il.doltIndexLookup.Intersection(newLookups...)
if err != nil {
return nil, err
}
return &testMergeableIndexLookup{
doltIndexLookup: intersectedIl.(*doltIndexLookup),
t: il.t,
finalRanges: il.finalRanges,
}, nil
}
func (il *testMergeableIndexLookup) Union(indexLookups ...sql.IndexLookup) (sql.IndexLookup, error) {
newLookups := make([]sql.IndexLookup, len(indexLookups))
for i, otherIl := range indexLookups {
newLookups[i] = otherIl.(*testMergeableIndexLookup).doltIndexLookup
}
unionedIl, err := il.doltIndexLookup.Union(newLookups...)
if err != nil {
return nil, err
}
return &testMergeableIndexLookup{
doltIndexLookup: unionedIl.(*doltIndexLookup),
t: il.t,
finalRanges: il.finalRanges,
}, nil
}
func (il *testMergeableIndexLookup) RowIter(ctx *sql.Context, rowData types.Map) (sql.RowIter, error) {
func (il *testIndexLookup) RowIter(ctx *sql.Context, rowData types.Map) (sql.RowIter, error) {
il.finalRanges(il.ranges) // this is where the ranges turn into noms.ReadRanges, so we return the final slice here
return il.doltIndexLookup.RowIter(ctx, rowData, nil)
}

4
go/libraries/doltcore/sqle/mergeable_indexes_test.go
View File

@@ -32,7 +32,7 @@ import (
// they're converted into a format that Noms understands to verify that they were handled correctly. Lastly, we ensure
// that the final output is as expected.
func TestMergeableIndexes(t *testing.T) {
engine, denv, db, indexTuples, initialRoot := setupMergeableIndexes(t, "test", `INSERT INTO test VALUES
engine, denv, db, indexTuples, initialRoot := setupIndexes(t, "test", `INSERT INTO test VALUES
(-3, NULL, NULL),
(-2, NULL, NULL),
(-1, NULL, NULL),
@@ -1395,7 +1395,7 @@ func TestMergeableIndexes(t *testing.T) {
// ranges may be incorrect.
// TODO: disassociate NULL ranges from value ranges and fix the intermediate ranges (finalRanges).
func TestMergeableIndexesNulls(t *testing.T) {
engine, denv, db, indexTuples, initialRoot := setupMergeableIndexes(t, "test", `INSERT INTO test VALUES
engine, denv, db, indexTuples, initialRoot := setupIndexes(t, "test", `INSERT INTO test VALUES
(0, 10, 20),
(1, 11, 21),
(2, NULL, NULL),

2
go/libraries/doltcore/sqle/procedures_table.go
View File

@@ -166,7 +166,7 @@ func DoltProceduresGetDetails(ctx *sql.Context, tbl *WritableDoltTable, name str
doltdb.SchemasTableName)
}
indexLookup, err := fragNameIndex.Get(name)
indexLookup, err := sql.NewIndexBuilder(ctx, fragNameIndex).Equals(ctx, fragNameIndex.Expressions()[0], name).Build(ctx)
if err != nil {
return sql.StoredProcedureDetails{}, false, err
}

3
go/libraries/doltcore/sqle/schema_table.go
View File

@@ -210,7 +210,8 @@ func fragFromSchemasTable(ctx *sql.Context, tbl *WritableDoltTable, fragType str
return nil, false, noSchemaIndexDefined
}
indexLookup, err := fragNameIndex.Get(fragType, name)
exprs := fragNameIndex.Expressions()
indexLookup, err := sql.NewIndexBuilder(ctx, fragNameIndex).Equals(ctx, exprs[0], fragType).Equals(ctx, exprs[1], name).Build(ctx)
if err != nil {
return nil, false, err
}

32
go/libraries/doltcore/sqle/tables.go
View File

@@ -213,22 +213,6 @@ func (t *DoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) {
unique: true,
generated: false,
})
for i := 1; i < len(cols); i++ {
sqlIndexes = append(sqlIndexes, &doltIndex{
cols: cols[:i],
db: t.db,
id: fmt.Sprintf("PRIMARY_PARTIAL_%d", i),
indexRowData: rowData,
indexSch: sch,
table: tbl,
tableData: rowData,
tableName: t.Name(),
tableSch: sch,
unique: false,
comment: fmt.Sprintf("partial of PRIMARY multi-column index on %d column(s)", i),
generated: true,
})
}
}
for _, index := range sch.Indexes().AllIndexes() {
@@ -254,22 +238,6 @@ func (t *DoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) {
comment: index.Comment(),
generated: false,
})
for i := 1; i < len(cols); i++ {
sqlIndexes = append(sqlIndexes, &doltIndex{
cols: cols[:i],
db: t.db,
id: fmt.Sprintf("%s_PARTIAL_%d", index.Name(), i),
indexRowData: indexRowData,
indexSch: index.Schema(),
table: tbl,
tableData: rowData,
tableName: t.Name(),
tableSch: sch,
unique: false,
comment: fmt.Sprintf("prefix of %s multi-column index on %d column(s)", index.Name(), i),
generated: true,
})
}
}
return sqlIndexes, nil