mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-03-13 11:09:10 -05:00
Code Issues Packages Projects Releases 146 Wiki Activity

Refactor DiffInfo struct into prolly tree package instead of merge package.

Browse Source
This commit is contained in:
Nick Tobey
2023-11-22 11:35:24 -08:00
parent 4c746771bb
commit 48cfabce0e
5 changed files with 44 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
go/libraries/doltcore/merge/merge_prolly_rows.go
View File

@@ -51,7 +51,7 @@ var ErrUnableToMergeColumnDefaultValue = errorkinds.NewKind("unable to automatic
// table's primary index will also be rewritten. This function merges the table's artifacts (e.g. recorded
// conflicts), migrates any existing table data to the specified |mergedSch|, and merges table data from both
// sides of the merge together.
func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Schema, mergeInfo MergeInfo) (*doltdb.Table, *MergeStats, error) {
func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Schema, diffInfo tree.ThreeWayDiffInfo) (*doltdb.Table, *MergeStats, error) {
mergeTbl, err := mergeTableArtifacts(ctx, tm, tm.leftTbl)
if err != nil {
return nil, nil, err
@@ -77,10 +77,10 @@ func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Sch
}
schemasDifferentSize := len(tm.leftSch.GetAllCols().GetColumns()) != len(mergedSch.GetAllCols().GetColumns())
rebuildPrimaryIndex := mergeInfo.TableRewrite || schemasDifferentSize || !valueMerger.leftMapping.IsIdentityMapping()
rebuildPrimaryIndex := diffInfo.TableRewrite || schemasDifferentSize || !valueMerger.leftMapping.IsIdentityMapping()
var stats *MergeStats
mergeTbl, stats, err = mergeProllyTableData(sqlCtx, tm, mergedSch, mergeTbl, valueMerger, mergeInfo, rebuildPrimaryIndex)
mergeTbl, stats, err = mergeProllyTableData(sqlCtx, tm, mergedSch, mergeTbl, valueMerger, diffInfo, rebuildPrimaryIndex)
if err != nil {
return nil, nil, err
}
@@ -108,8 +108,8 @@ func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Sch
// if not, they are recorded as conflicts in the table's artifacts. If |rebuildIndexes| is set to
// true, then secondary indexes will be rebuilt, instead of being incrementally merged together. This
// is less efficient, but safer, especially when type changes have been applied to a table's schema.
func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Schema, mergeTbl *doltdb.Table, valueMerger *valueMerger, mergeInfo MergeInfo, rebuildPrimaryIndex bool) (*doltdb.Table, *MergeStats, error) {
iter, err := threeWayDiffer(ctx, tm, valueMerger, mergeInfo)
func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Schema, mergeTbl *doltdb.Table, valueMerger *valueMerger, diffInfo tree.ThreeWayDiffInfo, rebuildPrimaryIndex bool) (*doltdb.Table, *MergeStats, error) {
iter, err := threeWayDiffer(ctx, tm, valueMerger, diffInfo)
if err != nil {
return nil, nil, err
}
@@ -285,7 +285,7 @@ func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Sch
return nil, nil, err
}
finalIdxs, err := mergeProllySecondaryIndexes(ctx, tm, leftIdxs, rightIdxs, finalSch, finalRows, conflicts.ae, mergeInfo.TableRewrite)
finalIdxs, err := mergeProllySecondaryIndexes(ctx, tm, leftIdxs, rightIdxs, finalSch, finalRows, conflicts.ae, diffInfo.TableRewrite)
if err != nil {
return nil, nil, err
}
@@ -311,7 +311,7 @@ func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Sch
return finalTbl, s, nil
}
func threeWayDiffer(ctx context.Context, tm *TableMerger, valueMerger *valueMerger, mergeInfo MergeInfo) (*tree.ThreeWayDiffer[val.Tuple, val.TupleDesc], error) {
func threeWayDiffer(ctx context.Context, tm *TableMerger, valueMerger *valueMerger, diffInfo tree.ThreeWayDiffInfo) (*tree.ThreeWayDiffer[val.Tuple, val.TupleDesc], error) {
lr, err := tm.leftTbl.GetRowData(ctx)
if err != nil {
return nil, err
@@ -334,13 +334,11 @@ func threeWayDiffer(ctx context.Context, tm *TableMerger, valueMerger *valueMerg
ctx,
leftRows.NodeStore(),
leftRows.Tuples(),
mergeInfo.LeftSchemaChange,
rightRows.Tuples(),
mergeInfo.RightSchemaChange,
ancRows.Tuples(),
valueMerger.tryMerge,
valueMerger.keyless,
mergeInfo.LeftAndRightSchemasDiffer,
diffInfo,
leftRows.Tuples().Order,
)
}

4
go/libraries/doltcore/merge/merge_rows.go
View File

@@ -126,7 +126,7 @@ func (rm *RootMerger) MergeTable(ctx *sql.Context, tblName string, opts editor.O
}
// Calculate a merge of the schemas, but don't apply it yet
mergeSch, schConflicts, mergeInfo, err := SchemaMerge(ctx, tm.vrw.Format(), tm.leftSch, tm.rightSch, tm.ancSch, tblName)
mergeSch, schConflicts, diffInfo, err := SchemaMerge(ctx, tm.vrw.Format(), tm.leftSch, tm.rightSch, tm.ancSch, tblName)
if err != nil {
return nil, nil, err
}
@@ -148,7 +148,7 @@ func (rm *RootMerger) MergeTable(ctx *sql.Context, tblName string, opts editor.O
var tbl *doltdb.Table
if types.IsFormat_DOLT(tm.vrw.Format()) {
tbl, stats, err = mergeProllyTable(ctx, tm, mergeSch, mergeInfo)
tbl, stats, err = mergeProllyTable(ctx, tm, mergeSch, diffInfo)
} else {
tbl, stats, err = mergeNomsTable(ctx, tm, mergeSch, rm.vrw, opts)
}

48
go/libraries/doltcore/merge/merge_schema.go
View File

@@ -18,6 +18,7 @@ import (
"context"
"errors"
"fmt"
"github.com/dolthub/dolt/go/store/prolly/tree"
"sort"
"strings"
@@ -152,17 +153,10 @@ func (c ChkConflict) String() string {
var ErrMergeWithDifferentPks = errors.New("error: cannot merge two tables with different primary keys")
type MergeInfo struct {
TableRewrite bool
LeftSchemaChange bool
RightSchemaChange bool
LeftAndRightSchemasDiffer bool
}
// SchemaMerge performs a three-way merge of |ourSch|, |theirSch|, and |ancSch|, and returns: the merged schema,
// any schema conflicts identified, whether moving to the new schema requires a full table rewrite, and any
// unexpected error encountered while merging the schemas.
func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch, theirSch, ancSch schema.Schema, tblName string) (sch schema.Schema, sc SchemaConflict, mergeInfo MergeInfo, err error) {
func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch, theirSch, ancSch schema.Schema, tblName string) (sch schema.Schema, sc SchemaConflict, diffInfo tree.ThreeWayDiffInfo, err error) {
// (sch - ancSch) ∪ (mergeSch - ancSch) ∪ (sch ∩ mergeSch)
sc = SchemaConflict{
TableName: tblName,
@@ -171,38 +165,38 @@ func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch,
// TODO: We'll remove this once it's possible to get diff and merge on different primary key sets
// TODO: decide how to merge different orders of PKS
if !schema.ArePrimaryKeySetsDiffable(format, ourSch, theirSch) || !schema.ArePrimaryKeySetsDiffable(format, ourSch, ancSch) {
return nil, SchemaConflict{}, mergeInfo, ErrMergeWithDifferentPks
return nil, SchemaConflict{}, diffInfo, ErrMergeWithDifferentPks
}
var mergedCC *schema.ColCollection
mergedCC, sc.ColConflicts, mergeInfo, err = mergeColumns(tblName, format, ourSch.GetAllCols(), theirSch.GetAllCols(), ancSch.GetAllCols())
mergedCC, sc.ColConflicts, diffInfo, err = mergeColumns(tblName, format, ourSch.GetAllCols(), theirSch.GetAllCols(), ancSch.GetAllCols())
if err != nil {
return nil, SchemaConflict{}, mergeInfo, err
return nil, SchemaConflict{}, diffInfo, err
}
if len(sc.ColConflicts) > 0 {
return nil, sc, mergeInfo, nil
return nil, sc, diffInfo, nil
}
var mergedIdxs schema.IndexCollection
mergedIdxs, sc.IdxConflicts = mergeIndexes(mergedCC, ourSch, theirSch, ancSch)
if len(sc.IdxConflicts) > 0 {
return nil, sc, mergeInfo, nil
return nil, sc, diffInfo, nil
}
sch, err = schema.SchemaFromCols(mergedCC)
if err != nil {
return nil, sc, mergeInfo, err
return nil, sc, diffInfo, err
}
sch, err = mergeTableCollation(ctx, tblName, ancSch, ourSch, theirSch, sch)
if err != nil {
return nil, sc, mergeInfo, err
return nil, sc, diffInfo, err
}
// TODO: Merge conflict should have blocked any primary key ordinal changes
err = sch.SetPkOrdinals(ourSch.GetPkOrdinals())
if err != nil {
return nil, sc, mergeInfo, err
return nil, sc, diffInfo, err
}
_ = mergedIdxs.Iter(func(index schema.Index) (stop bool, err error) {
@@ -214,17 +208,17 @@ func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch,
var mergedChks []schema.Check
mergedChks, sc.ChkConflicts, err = mergeChecks(ctx, ourSch.Checks(), theirSch.Checks(), ancSch.Checks())
if err != nil {
return nil, SchemaConflict{}, mergeInfo, err
return nil, SchemaConflict{}, diffInfo, err
}
if len(sc.ChkConflicts) > 0 {
return nil, sc, mergeInfo, nil
return nil, sc, diffInfo, nil
}
// Look for invalid CHECKs
for _, chk := range mergedChks {
// CONFLICT: a CHECK now references a column that no longer exists in schema
if ok, err := isCheckReferenced(sch, chk); err != nil {
return nil, sc, mergeInfo, err
return nil, sc, diffInfo, err
} else if !ok {
// Append to conflicts
sc.ChkConflicts = append(sc.ChkConflicts, ChkConflict{
@@ -239,7 +233,7 @@ func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch,
sch.Checks().AddCheck(chk.Name(), chk.Expression(), chk.Enforced())
}
return sch, sc, mergeInfo, nil
return sch, sc, diffInfo, nil
}
// ForeignKeysMerge performs a three-way merge of (ourRoot, theirRoot, ancRoot) and using mergeRoot to validate FKs.
@@ -376,20 +370,20 @@ func checkUnmergeableNewColumns(tblName string, columnMappings columnMappings) e
// compatible with the current stored format. The merged columns, any column conflicts, and a boolean value stating if
// a full table rewrite is needed to align the existing table rows with the new, merged schema. If any unexpected error
// occurs, then that error is returned and the other response fields should be ignored.
func mergeColumns(tblName string, format *storetypes.NomsBinFormat, ourCC, theirCC, ancCC *schema.ColCollection) (*schema.ColCollection, []ColConflict, MergeInfo, error) {
func mergeColumns(tblName string, format *storetypes.NomsBinFormat, ourCC, theirCC, ancCC *schema.ColCollection) (*schema.ColCollection, []ColConflict, tree.ThreeWayDiffInfo, error) {
columnMappings, err := mapColumns(ourCC, theirCC, ancCC)
if err != nil {
return nil, nil, MergeInfo{}, err
return nil, nil, tree.ThreeWayDiffInfo{}, err
}
conflicts, err := checkSchemaConflicts(columnMappings)
if err != nil {
return nil, nil, MergeInfo{}, err
return nil, nil, tree.ThreeWayDiffInfo{}, err
}
err = checkUnmergeableNewColumns(tblName, columnMappings)
if err != nil {
return nil, nil, MergeInfo{}, err
return nil, nil, tree.ThreeWayDiffInfo{}, err
}
compatChecker := newTypeCompatabilityCheckerForStorageFormat(format)
@@ -504,16 +498,16 @@ func mergeColumns(tblName string, format *storetypes.NomsBinFormat, ourCC, their
// Check that there are no duplicate column names or tags in the merged column set
conflicts = append(conflicts, checkForColumnConflicts(mergedColumns)...)
if conflicts != nil {
return nil, conflicts, MergeInfo{}, nil
return nil, conflicts, tree.ThreeWayDiffInfo{}, nil
}
mergeInfo := MergeInfo{
diffInfo := tree.ThreeWayDiffInfo{
TableRewrite: tableRewrite,
LeftSchemaChange: ourSchemaChanged,
RightSchemaChange: theirSchemaChanged,
LeftAndRightSchemasDiffer: leftAndRightSchemasDiffer,
}
return schema.NewColCollection(mergedColumns...), nil, mergeInfo, nil
return schema.NewColCollection(mergedColumns...), nil, diffInfo, nil
}
// checkForColumnConflicts iterates over |mergedColumns|, checks for duplicate column names or column tags, and returns

17
go/store/prolly/tree/three_way_differ.go
View File

@@ -41,26 +41,31 @@ type ThreeWayDiffer[K ~[]byte, O Ordering[K]] struct {
type resolveCb func(context.Context, val.Tuple, val.Tuple, val.Tuple) (val.Tuple, bool, error)
type ThreeWayDiffInfo struct {
TableRewrite bool
LeftSchemaChange bool
RightSchemaChange bool
LeftAndRightSchemasDiffer bool
}
func NewThreeWayDiffer[K, V ~[]byte, O Ordering[K]](
ctx context.Context,
ns NodeStore,
left StaticMap[K, V, O],
leftSchemaChanged bool,
right StaticMap[K, V, O],
rightSchemaChanged bool,
base StaticMap[K, V, O],
resolveCb resolveCb,
keyless bool,
leftAndRightSchemasDiffer bool,
diffInfo ThreeWayDiffInfo,
order O,
) (*ThreeWayDiffer[K, O], error) {
// probably compute each of these separately
ld, err := DifferFromRoots[K](ctx, ns, ns, base.Root, left.Root, order, leftSchemaChanged)
ld, err := DifferFromRoots[K](ctx, ns, ns, base.Root, left.Root, order, diffInfo.LeftSchemaChange)
if err != nil {
return nil, err
}
rd, err := DifferFromRoots[K](ctx, ns, ns, base.Root, right.Root, order, rightSchemaChanged)
rd, err := DifferFromRoots[K](ctx, ns, ns, base.Root, right.Root, order, diffInfo.RightSchemaChange)
if err != nil {
return nil, err
}
@@ -70,7 +75,7 @@ func NewThreeWayDiffer[K, V ~[]byte, O Ordering[K]](
rIter: rd,
resolveCb: resolveCb,
keyless: keyless,
leftAndRightSchemasDiffer: leftAndRightSchemasDiffer,
leftAndRightSchemasDiffer: diffInfo.LeftAndRightSchemasDiffer,
}, nil
}

3
go/store/prolly/tree/three_way_differ_test.go
View File

@@ -192,7 +192,8 @@ func TestThreeWayDiffer(t *testing.T) {
left := newTestMap(t, ctx, tt.left, ns, valDesc)
right := newTestMap(t, ctx, tt.right, ns, valDesc)
iter, err := NewThreeWayDiffer(ctx, ns, left, false, right, false, base, testResolver(t, ns, valDesc, val.NewTupleBuilder(valDesc)), false, keyDesc)
var diffInfo ThreeWayDiffInfo
iter, err := NewThreeWayDiffer(ctx, ns, left, right, base, testResolver(t, ns, valDesc, val.NewTupleBuilder(valDesc)), false, diffInfo, keyDesc)
require.NoError(t, err)
var cmp []testDiff