mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-03 11:30:28 -05:00
Code Issues Packages Projects Releases 159 Wiki Activity

working even when adding from columns

Browse Source
This commit is contained in:
James Cor
2022-06-01 11:24:40 -07:00
parent a6c2c23933
commit e5c1aa002a
6 changed files with 90 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/libraries/doltcore/diff/async_differ_test.go
+13 -10
View File
@@ -112,8 +112,8 @@ func TestAsyncDiffer(t *testing.T) {
name: "iter range starting with nil",
createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer {
ad := NewAsyncDiffer(4)
ad.StartWithRange(ctx, m1, m2, nil, func(ctx context.Context, value types.Value) (bool, error) {
return true, nil
ad.StartWithRange(ctx, m1, m2, nil, func(ctx context.Context, value types.Value) (bool, bool, error) {
return true, false, nil
})
return ad
},
@@ -128,8 +128,8 @@ func TestAsyncDiffer(t *testing.T) {
name: "iter range staring with Null Value",
createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer {
ad := NewAsyncDiffer(4)
ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, error) {
return true, nil
ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, bool, error) {
return true, false, nil
})
return ad
},
@@ -145,8 +145,9 @@ func TestAsyncDiffer(t *testing.T) {
createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer {
ad := NewAsyncDiffer(4)
end := types.Uint(27)
ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, error) {
return value.Less(m1.Format(), end)
ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, bool, error) {
valid, err := value.Less(m1.Format(), end)
return valid, false, err
})
return ad
},
@@ -162,8 +163,9 @@ func TestAsyncDiffer(t *testing.T) {
createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer {
ad := NewAsyncDiffer(4)
end := types.Uint(15)
ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, error) {
return value.Less(m1.Format(), end)
ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, bool, error) {
valid, err := value.Less(m1.Format(), end)
return valid, false, err
})
return ad
},
@@ -180,8 +182,9 @@ func TestAsyncDiffer(t *testing.T) {
ad := NewAsyncDiffer(4)
start := types.Uint(10)
end := types.Uint(15)
ad.StartWithRange(ctx, m1, m2, start, func(ctx context.Context, value types.Value) (bool, error) {
return value.Less(m1.Format(), end)
ad.StartWithRange(ctx, m1, m2, start, func(ctx context.Context, value types.Value) (bool, bool, error) {
valid, err := value.Less(m1.Format(), end)
return valid, false, err
})
return ad
},
go/libraries/doltcore/sqle/dtables/diff_iter.go
+4 -4
View File
@@ -94,13 +94,13 @@ func newNomsDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joine
rd.Start(ctx, durable.NomsMapFromIndex(fromData), durable.NomsMapFromIndex(toData))
} else {
ranges := index.NomsRangesFromIndexLookup(lookup) // TODO: this is a testing method
rangeFunc := func(ctx context.Context, val types.Value) (bool, error) {
// TODO: maybe just use Check
rangeFunc := func(ctx context.Context, val types.Value) (bool, bool, error) {
v, ok := val.(types.Tuple)
if !ok {
return false, nil
return false, false, nil
}
valid, _, err := ranges[0].Check.Check(ctx, v)
return valid, err
return ranges[0].Check.Check(ctx, v)
}
rd.StartWithRange(ctx, durable.NomsMapFromIndex(fromData), durable.NomsMapFromIndex(toData), ranges[0].Start, rangeFunc)
}
go/libraries/doltcore/sqle/index/dolt_index.go
+13 -14
View File
@@ -79,23 +79,22 @@ func DoltDiffIndexesFromTable(ctx context.Context, db, tbl string, t *doltdb.Tab
vrw: t.ValueReadWriter(),
keyBld: keyBld,
}
indexes = append(indexes, toIndex)
// TODO: from_ columns
//indexes[1] = doltIndex{
// id: "PRIMARY",
// tblName: doltdb.DoltDiffTablePrefix + tbl,
// dbName: db,
// columns: cols,
// indexSch: sch,
// tableSch: sch,
// unique: true,
// comment: "",
// vrw: t.ValueReadWriter(),
// keyBld: keyBld,
//}
fromIndex := doltIndex{
id: "PRIMARY",
tblName: doltdb.DoltDiffTablePrefix + tbl,
dbName: db,
columns: cols,
indexSch: sch,
tableSch: sch,
unique: true,
comment: "",
vrw: t.ValueReadWriter(),
keyBld: keyBld,
}
return indexes, nil
return append(indexes, toIndex, fromIndex), nil
}
func DoltIndexesFromTable(ctx context.Context, db, tbl string, t *doltdb.Table) (indexes []sql.Index, err error) {
go/store/diff/diff.go
+2 -2
View File
@@ -275,8 +275,8 @@ func (d differ) diffLists(ctx context.Context, p types.Path, v1, v2 types.List)
}
func (d differ) diffMaps(ctx context.Context, p types.Path, v1, v2 types.Map) error {
trueFunc := func(ctx context.Context, value types.Value) (bool, error) {
return true, nil
trueFunc := func(ctx context.Context, value types.Value) (bool, bool, error) {
return true, false, nil
}
return d.diffMapsInRange(ctx, p, v1, v2, nil, trueFunc)
go/store/types/map.go
+3 -3
View File
@@ -33,7 +33,7 @@ import (
)
//type ValueInRange func(Value) (bool, error)
type ValueInRange func(context.Context, Value) (bool, error)
type ValueInRange func(context.Context, Value) (bool, bool, error)
var ErrKeysNotOrdered = errors.New("streaming map keys not ordered")
@@ -204,8 +204,8 @@ func (m Map) Diff(ctx context.Context, last Map, changes chan<- ValueChanged) er
// streaming approach, optimised for returning results early, but not
// completing quickly.
func (m Map) DiffLeftRight(ctx context.Context, last Map, changes chan<- ValueChanged) error {
trueFunc := func(context.Context, Value) (bool, error) {
return true, nil
trueFunc := func(context.Context, Value) (bool, bool, error) {
return true, false, nil
}
return m.DiffLeftRightInRange(ctx, last, nil, trueFunc, changes)
}
go/store/types/ordered_sequences_diff.go
+55 -29
View File
@@ -52,8 +52,8 @@ func sendChange(ctx context.Context, changes chan<- ValueChanged, change ValueCh
// Streams the diff from |last| to |current| into |changes|, using a left-right approach.
// Left-right immediately descends to the first change and starts streaming changes, but compared to top-down it's serial and much slower to calculate the full diff.
func orderedSequenceDiffLeftRight(ctx context.Context, last orderedSequence, current orderedSequence, changes chan<- ValueChanged) error {
trueFunc := func(context.Context, Value) (bool, error) {
return true, nil
trueFunc := func(context.Context, Value) (bool, bool, error) {
return true, false, nil
}
return orderedSequenceDiffLeftRightInRange(ctx, last, current, emptyKey, trueFunc, changes)
}
@@ -99,20 +99,26 @@ VALIDRANGES:
if isLess, err := currentKey.Less(last.format(), lastKey); err != nil {
return err
} else if isLess {
isInRange, err := inRange(ctx, currentKey.v)
valid, skip, err := inRange(ctx, currentKey.v)
if err != nil {
return err
} else if !isInRange {
}
// Out of range
if !valid {
break VALIDRANGES
}
mv, err := getMapValue(currentCur)
if err != nil {
return err
}
// Not skipping this value (want to skip for non-inclusive lower bound range)
if !skip {
mv, err := getMapValue(currentCur)
if err != nil {
return err
}
if err := sendChange(ctx, changes, ValueChanged{DiffChangeAdded, currentKey.v, nil, mv}); err != nil {
return err
if err := sendChange(ctx, changes, ValueChanged{DiffChangeAdded, currentKey.v, nil, mv}); err != nil {
return err
}
}
_, err = currentCur.advance(ctx)
@@ -120,13 +126,25 @@ VALIDRANGES:
return err
}
} else {
isInRange, err := inRange(ctx, lastKey.v)
if !isInRange {
valid, skip, err := inRange(ctx, lastKey.v)
if err != nil {
return err
} else if !isInRange {
}
// Out of range
if !valid {
break VALIDRANGES
}
// Skip this last key
if skip {
_, err = lastCur.advance(ctx)
if err != nil {
return err
}
continue
}
if isLess, err := lastKey.Less(last.format(), currentKey); err != nil {
return err
} else if isLess {
@@ -178,20 +196,24 @@ VALIDRANGES:
return err
}
isInRange, err := inRange(ctx, lastKey.v)
valid, skip, err := inRange(ctx, lastKey.v)
if err != nil {
return err
} else if !isInRange {
}
if !valid {
break
}
mv, err := getMapValue(lastCur)
if err != nil {
return err
}
if !skip {
mv, err := getMapValue(lastCur)
if err != nil {
return err
}
if err := sendChange(ctx, changes, ValueChanged{DiffChangeRemoved, lastKey.v, mv, nil}); err != nil {
return err
if err := sendChange(ctx, changes, ValueChanged{DiffChangeRemoved, lastKey.v, mv, nil}); err != nil {
return err
}
}
_, err = lastCur.advance(ctx)
@@ -206,20 +228,24 @@ VALIDRANGES:
return err
}
isInRange, err := inRange(ctx, currKey.v)
valid, skip, err := inRange(ctx, currKey.v)
if err != nil {
return err
} else if !isInRange {
}
if !valid {
break
}
mv, err := getMapValue(currentCur)
if err != nil {
return err
}
if !skip {
mv, err := getMapValue(currentCur)
if err != nil {
return err
}
if err := sendChange(ctx, changes, ValueChanged{DiffChangeAdded, currKey.v, nil, mv}); err != nil {
return err
if err := sendChange(ctx, changes, ValueChanged{DiffChangeAdded, currKey.v, nil, mv}); err != nil {
return err
}
}
_, err = currentCur.advance(ctx)