mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-29 03:06:35 -05:00
Code Issues Packages Projects Releases 159 Wiki Activity

indexed look ups mostly working, but lower bound always inclusive?

Browse Source
This commit is contained in:
James Cor
2022-06-01 10:00:03 -07:00
parent 094245d591
commit e4238b774a
8 changed files with 38 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/libraries/doltcore/diff/async_differ.go
+6
View File
@@ -32,6 +32,8 @@ type RowDiffer interface {
// Start starts the RowDiffer.
Start(ctx context.Context, from, to types.Map)
StartWithRange(ctx context.Context, from, to types.Map, start types.Value, inRange types.ValueInRange)
// GetDiffs returns the requested number of diff.Differences, or times out.
GetDiffs(numDiffs int, timeout time.Duration) ([]*diff.Difference, bool, error)
@@ -298,6 +300,10 @@ var _ RowDiffer = &EmptyRowDiffer{}
func (e EmptyRowDiffer) Start(ctx context.Context, from, to types.Map) {
}
func (e EmptyRowDiffer) StartWithRange(ctx context.Context, from, to types.Map, start types.Value, inRange types.ValueInRange) {
}
func (e EmptyRowDiffer) GetDiffs(numDiffs int, timeout time.Duration) ([]*diff.Difference, bool, error) {
return nil, false, nil
}
go/libraries/doltcore/sqle/dolt_diff_table_function.go
+1 -1
View File
@@ -397,7 +397,7 @@ func (itr *diffTableFunctionRowIter) Next(ctx *sql.Context) (sql.Row, error) {
if itr.currentRowIter == nil {
dp := (*itr.currentPartition).(dtables.DiffPartition)
rowIter, err := dp.GetRowIter(ctx, itr.ddb, itr.joiner)
rowIter, err := dp.GetRowIter(ctx, itr.ddb, itr.joiner, nil)
if err != nil {
return nil, err
}
go/libraries/doltcore/sqle/dtables/commit_diff_table.go
+1 -1
View File
@@ -288,5 +288,5 @@ func (dt *CommitDiffTable) WithFilters(_ *sql.Context, _ []sql.Expression) sql.T
func (dt *CommitDiffTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
dp := part.(DiffPartition)
return dp.GetRowIter(ctx, dt.ddb, dt.joiner)
return dp.GetRowIter(ctx, dt.ddb, dt.joiner, nil)
}
go/libraries/doltcore/sqle/dtables/diff_iter.go
+18 -2
View File
@@ -17,6 +17,7 @@ package dtables
import (
"context"
"errors"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"io"
"time"
@@ -52,7 +53,7 @@ type commitInfo struct {
dateTag uint64
}
func newNomsDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner, dp DiffPartition) (*diffRowItr, error) {
func newNomsDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner, dp DiffPartition, lookup sql.IndexLookup) (*diffRowItr, error) {
fromData, fromSch, err := tableData(ctx, dp.from, ddb)
if err != nil {
@@ -88,7 +89,22 @@ func newNomsDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joine
rd := diff.NewRowDiffer(ctx, fromSch, toSch, 1024)
// TODO (dhruv) don't cast to noms map
rd.Start(ctx, durable.NomsMapFromIndex(fromData), durable.NomsMapFromIndex(toData))
// Use index lookup if it exists
if lookup == nil {
rd.Start(ctx, durable.NomsMapFromIndex(fromData), durable.NomsMapFromIndex(toData))
} else {
// TODO: this is a testing method
ranges := index.NomsRangesFromIndexLookup(lookup)
rangeFunc := func(ctx context.Context, val types.Value) (bool, error) {
v, ok := val.(types.Tuple)
if !ok {
return false, nil
}
valid, _, err := ranges[0].Check.Check(ctx, v)
return valid, err
}
rd.StartWithRange(ctx, durable.NomsMapFromIndex(fromData), durable.NomsMapFromIndex(toData), ranges[0].Start, rangeFunc)
}
warnFn := func(code int, message string, args ...string) {
ctx.Warn(code, message, args)
go/libraries/doltcore/sqle/dtables/diff_table.go
+3 -3
View File
@@ -205,7 +205,7 @@ func (dt *DiffTable) WithFilters(_ *sql.Context, filters []sql.Expression) sql.T
func (dt *DiffTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
dp := part.(DiffPartition)
return dp.GetRowIter(ctx, dt.ddb, dt.joiner)
return dp.GetRowIter(ctx, dt.ddb, dt.joiner, dt.lookup)
}
// TODO: create new index type or use modified DoltIndexes?
@@ -301,11 +301,11 @@ func (dp DiffPartition) Key() []byte {
return []byte(dp.toName + dp.fromName)
}
func (dp DiffPartition) GetRowIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner) (sql.RowIter, error) {
func (dp DiffPartition) GetRowIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner, lookup sql.IndexLookup) (sql.RowIter, error) {
if types.IsFormat_DOLT_1(ddb.Format()) {
return newProllyDiffIter(ctx, dp, ddb, *dp.fromSch, *dp.toSch)
} else {
return newNomsDiffIter(ctx, ddb, joiner, dp)
return newNomsDiffIter(ctx, ddb, joiner, dp, lookup)
}
}
go/store/diff/diff.go
+1 -1
View File
@@ -275,7 +275,7 @@ func (d differ) diffLists(ctx context.Context, p types.Path, v1, v2 types.List)
}
func (d differ) diffMaps(ctx context.Context, p types.Path, v1, v2 types.Map) error {
trueFunc := func(value types.Value) (bool, error) {
trueFunc := func(ctx context.Context, value types.Value) (bool, error) {
return true, nil
}
go/store/types/map.go
+3 -2
View File
@@ -32,7 +32,8 @@ import (
"github.com/dolthub/dolt/go/store/hash"
)
type ValueInRange func(Value) (bool, error)
//type ValueInRange func(Value) (bool, error)
type ValueInRange func(context.Context, Value) (bool, error)
var ErrKeysNotOrdered = errors.New("streaming map keys not ordered")
@@ -203,7 +204,7 @@ func (m Map) Diff(ctx context.Context, last Map, changes chan<- ValueChanged) er
// streaming approach, optimised for returning results early, but not
// completing quickly.
func (m Map) DiffLeftRight(ctx context.Context, last Map, changes chan<- ValueChanged) error {
trueFunc := func(Value) (bool, error) {
trueFunc := func(context.Context, Value) (bool, error) {
return true, nil
}
return m.DiffLeftRightInRange(ctx, last, nil, trueFunc, changes)
go/store/types/ordered_sequences_diff.go
+5 -5
View File
@@ -52,7 +52,7 @@ func sendChange(ctx context.Context, changes chan<- ValueChanged, change ValueCh
// Streams the diff from |last| to |current| into |changes|, using a left-right approach.
// Left-right immediately descends to the first change and starts streaming changes, but compared to top-down it's serial and much slower to calculate the full diff.
func orderedSequenceDiffLeftRight(ctx context.Context, last orderedSequence, current orderedSequence, changes chan<- ValueChanged) error {
trueFunc := func(Value) (bool, error) {
trueFunc := func(context.Context, Value) (bool, error) {
return true, nil
}
return orderedSequenceDiffLeftRightInRange(ctx, last, current, emptyKey, trueFunc, changes)
@@ -99,7 +99,7 @@ VALIDRANGES:
if isLess, err := currentKey.Less(last.format(), lastKey); err != nil {
return err
} else if isLess {
isInRange, err := inRange(currentKey.v)
isInRange, err := inRange(ctx, currentKey.v)
if err != nil {
return err
} else if !isInRange {
@@ -120,7 +120,7 @@ VALIDRANGES:
return err
}
} else {
isInRange, err := inRange(lastKey.v)
isInRange, err := inRange(ctx, lastKey.v)
if !isInRange {
return err
} else if !isInRange {
@@ -178,7 +178,7 @@ VALIDRANGES:
return err
}
isInRange, err := inRange(lastKey.v)
isInRange, err := inRange(ctx, lastKey.v)
if err != nil {
return err
} else if !isInRange {
@@ -206,7 +206,7 @@ VALIDRANGES:
return err
}
isInRange, err := inRange(currKey.v)
isInRange, err := inRange(ctx, currKey.v)
if err != nil {
return err
} else if !isInRange {