From 48cfabce0e942424465a1ad4e25542e64dbe1b0d Mon Sep 17 00:00:00 2001 From: Nick Tobey Date: Wed, 22 Nov 2023 11:35:24 -0800 Subject: [PATCH] Refactor `DiffInfo` struct into prolly tree package instead of merge package. --- .../doltcore/merge/merge_prolly_rows.go | 18 ++++--- go/libraries/doltcore/merge/merge_rows.go | 4 +- go/libraries/doltcore/merge/merge_schema.go | 48 ++++++++----------- go/store/prolly/tree/three_way_differ.go | 17 ++++--- go/store/prolly/tree/three_way_differ_test.go | 3 +- 5 files changed, 44 insertions(+), 46 deletions(-) diff --git a/go/libraries/doltcore/merge/merge_prolly_rows.go b/go/libraries/doltcore/merge/merge_prolly_rows.go index 2935e19a8e..f36d392d2f 100644 --- a/go/libraries/doltcore/merge/merge_prolly_rows.go +++ b/go/libraries/doltcore/merge/merge_prolly_rows.go @@ -51,7 +51,7 @@ var ErrUnableToMergeColumnDefaultValue = errorkinds.NewKind("unable to automatic // table's primary index will also be rewritten. This function merges the table's artifacts (e.g. recorded // conflicts), migrates any existing table data to the specified |mergedSch|, and merges table data from both // sides of the merge together. -func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Schema, mergeInfo MergeInfo) (*doltdb.Table, *MergeStats, error) { +func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Schema, diffInfo tree.ThreeWayDiffInfo) (*doltdb.Table, *MergeStats, error) { mergeTbl, err := mergeTableArtifacts(ctx, tm, tm.leftTbl) if err != nil { return nil, nil, err @@ -77,10 +77,10 @@ func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Sch } schemasDifferentSize := len(tm.leftSch.GetAllCols().GetColumns()) != len(mergedSch.GetAllCols().GetColumns()) - rebuildPrimaryIndex := mergeInfo.TableRewrite || schemasDifferentSize || !valueMerger.leftMapping.IsIdentityMapping() + rebuildPrimaryIndex := diffInfo.TableRewrite || schemasDifferentSize || !valueMerger.leftMapping.IsIdentityMapping() var stats *MergeStats - mergeTbl, stats, err = mergeProllyTableData(sqlCtx, tm, mergedSch, mergeTbl, valueMerger, mergeInfo, rebuildPrimaryIndex) + mergeTbl, stats, err = mergeProllyTableData(sqlCtx, tm, mergedSch, mergeTbl, valueMerger, diffInfo, rebuildPrimaryIndex) if err != nil { return nil, nil, err } @@ -108,8 +108,8 @@ func mergeProllyTable(ctx context.Context, tm *TableMerger, mergedSch schema.Sch // if not, they are recorded as conflicts in the table's artifacts. If |rebuildIndexes| is set to // true, then secondary indexes will be rebuilt, instead of being incrementally merged together. This // is less efficient, but safer, especially when type changes have been applied to a table's schema. -func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Schema, mergeTbl *doltdb.Table, valueMerger *valueMerger, mergeInfo MergeInfo, rebuildPrimaryIndex bool) (*doltdb.Table, *MergeStats, error) { - iter, err := threeWayDiffer(ctx, tm, valueMerger, mergeInfo) +func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Schema, mergeTbl *doltdb.Table, valueMerger *valueMerger, diffInfo tree.ThreeWayDiffInfo, rebuildPrimaryIndex bool) (*doltdb.Table, *MergeStats, error) { + iter, err := threeWayDiffer(ctx, tm, valueMerger, diffInfo) if err != nil { return nil, nil, err } @@ -285,7 +285,7 @@ func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Sch return nil, nil, err } - finalIdxs, err := mergeProllySecondaryIndexes(ctx, tm, leftIdxs, rightIdxs, finalSch, finalRows, conflicts.ae, mergeInfo.TableRewrite) + finalIdxs, err := mergeProllySecondaryIndexes(ctx, tm, leftIdxs, rightIdxs, finalSch, finalRows, conflicts.ae, diffInfo.TableRewrite) if err != nil { return nil, nil, err } @@ -311,7 +311,7 @@ func mergeProllyTableData(ctx *sql.Context, tm *TableMerger, finalSch schema.Sch return finalTbl, s, nil } -func threeWayDiffer(ctx context.Context, tm *TableMerger, valueMerger *valueMerger, mergeInfo MergeInfo) (*tree.ThreeWayDiffer[val.Tuple, val.TupleDesc], error) { +func threeWayDiffer(ctx context.Context, tm *TableMerger, valueMerger *valueMerger, diffInfo tree.ThreeWayDiffInfo) (*tree.ThreeWayDiffer[val.Tuple, val.TupleDesc], error) { lr, err := tm.leftTbl.GetRowData(ctx) if err != nil { return nil, err @@ -334,13 +334,11 @@ func threeWayDiffer(ctx context.Context, tm *TableMerger, valueMerger *valueMerg ctx, leftRows.NodeStore(), leftRows.Tuples(), - mergeInfo.LeftSchemaChange, rightRows.Tuples(), - mergeInfo.RightSchemaChange, ancRows.Tuples(), valueMerger.tryMerge, valueMerger.keyless, - mergeInfo.LeftAndRightSchemasDiffer, + diffInfo, leftRows.Tuples().Order, ) } diff --git a/go/libraries/doltcore/merge/merge_rows.go b/go/libraries/doltcore/merge/merge_rows.go index 084417d17e..52fa0e72cc 100644 --- a/go/libraries/doltcore/merge/merge_rows.go +++ b/go/libraries/doltcore/merge/merge_rows.go @@ -126,7 +126,7 @@ func (rm *RootMerger) MergeTable(ctx *sql.Context, tblName string, opts editor.O } // Calculate a merge of the schemas, but don't apply it yet - mergeSch, schConflicts, mergeInfo, err := SchemaMerge(ctx, tm.vrw.Format(), tm.leftSch, tm.rightSch, tm.ancSch, tblName) + mergeSch, schConflicts, diffInfo, err := SchemaMerge(ctx, tm.vrw.Format(), tm.leftSch, tm.rightSch, tm.ancSch, tblName) if err != nil { return nil, nil, err } @@ -148,7 +148,7 @@ func (rm *RootMerger) MergeTable(ctx *sql.Context, tblName string, opts editor.O var tbl *doltdb.Table if types.IsFormat_DOLT(tm.vrw.Format()) { - tbl, stats, err = mergeProllyTable(ctx, tm, mergeSch, mergeInfo) + tbl, stats, err = mergeProllyTable(ctx, tm, mergeSch, diffInfo) } else { tbl, stats, err = mergeNomsTable(ctx, tm, mergeSch, rm.vrw, opts) } diff --git a/go/libraries/doltcore/merge/merge_schema.go b/go/libraries/doltcore/merge/merge_schema.go index 9793cddca3..562a4e18fe 100644 --- a/go/libraries/doltcore/merge/merge_schema.go +++ b/go/libraries/doltcore/merge/merge_schema.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "github.com/dolthub/dolt/go/store/prolly/tree" "sort" "strings" @@ -152,17 +153,10 @@ func (c ChkConflict) String() string { var ErrMergeWithDifferentPks = errors.New("error: cannot merge two tables with different primary keys") -type MergeInfo struct { - TableRewrite bool - LeftSchemaChange bool - RightSchemaChange bool - LeftAndRightSchemasDiffer bool -} - // SchemaMerge performs a three-way merge of |ourSch|, |theirSch|, and |ancSch|, and returns: the merged schema, // any schema conflicts identified, whether moving to the new schema requires a full table rewrite, and any // unexpected error encountered while merging the schemas. -func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch, theirSch, ancSch schema.Schema, tblName string) (sch schema.Schema, sc SchemaConflict, mergeInfo MergeInfo, err error) { +func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch, theirSch, ancSch schema.Schema, tblName string) (sch schema.Schema, sc SchemaConflict, diffInfo tree.ThreeWayDiffInfo, err error) { // (sch - ancSch) ∪ (mergeSch - ancSch) ∪ (sch ∩ mergeSch) sc = SchemaConflict{ TableName: tblName, @@ -171,38 +165,38 @@ func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch, // TODO: We'll remove this once it's possible to get diff and merge on different primary key sets // TODO: decide how to merge different orders of PKS if !schema.ArePrimaryKeySetsDiffable(format, ourSch, theirSch) || !schema.ArePrimaryKeySetsDiffable(format, ourSch, ancSch) { - return nil, SchemaConflict{}, mergeInfo, ErrMergeWithDifferentPks + return nil, SchemaConflict{}, diffInfo, ErrMergeWithDifferentPks } var mergedCC *schema.ColCollection - mergedCC, sc.ColConflicts, mergeInfo, err = mergeColumns(tblName, format, ourSch.GetAllCols(), theirSch.GetAllCols(), ancSch.GetAllCols()) + mergedCC, sc.ColConflicts, diffInfo, err = mergeColumns(tblName, format, ourSch.GetAllCols(), theirSch.GetAllCols(), ancSch.GetAllCols()) if err != nil { - return nil, SchemaConflict{}, mergeInfo, err + return nil, SchemaConflict{}, diffInfo, err } if len(sc.ColConflicts) > 0 { - return nil, sc, mergeInfo, nil + return nil, sc, diffInfo, nil } var mergedIdxs schema.IndexCollection mergedIdxs, sc.IdxConflicts = mergeIndexes(mergedCC, ourSch, theirSch, ancSch) if len(sc.IdxConflicts) > 0 { - return nil, sc, mergeInfo, nil + return nil, sc, diffInfo, nil } sch, err = schema.SchemaFromCols(mergedCC) if err != nil { - return nil, sc, mergeInfo, err + return nil, sc, diffInfo, err } sch, err = mergeTableCollation(ctx, tblName, ancSch, ourSch, theirSch, sch) if err != nil { - return nil, sc, mergeInfo, err + return nil, sc, diffInfo, err } // TODO: Merge conflict should have blocked any primary key ordinal changes err = sch.SetPkOrdinals(ourSch.GetPkOrdinals()) if err != nil { - return nil, sc, mergeInfo, err + return nil, sc, diffInfo, err } _ = mergedIdxs.Iter(func(index schema.Index) (stop bool, err error) { @@ -214,17 +208,17 @@ func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch, var mergedChks []schema.Check mergedChks, sc.ChkConflicts, err = mergeChecks(ctx, ourSch.Checks(), theirSch.Checks(), ancSch.Checks()) if err != nil { - return nil, SchemaConflict{}, mergeInfo, err + return nil, SchemaConflict{}, diffInfo, err } if len(sc.ChkConflicts) > 0 { - return nil, sc, mergeInfo, nil + return nil, sc, diffInfo, nil } // Look for invalid CHECKs for _, chk := range mergedChks { // CONFLICT: a CHECK now references a column that no longer exists in schema if ok, err := isCheckReferenced(sch, chk); err != nil { - return nil, sc, mergeInfo, err + return nil, sc, diffInfo, err } else if !ok { // Append to conflicts sc.ChkConflicts = append(sc.ChkConflicts, ChkConflict{ @@ -239,7 +233,7 @@ func SchemaMerge(ctx context.Context, format *storetypes.NomsBinFormat, ourSch, sch.Checks().AddCheck(chk.Name(), chk.Expression(), chk.Enforced()) } - return sch, sc, mergeInfo, nil + return sch, sc, diffInfo, nil } // ForeignKeysMerge performs a three-way merge of (ourRoot, theirRoot, ancRoot) and using mergeRoot to validate FKs. @@ -376,20 +370,20 @@ func checkUnmergeableNewColumns(tblName string, columnMappings columnMappings) e // compatible with the current stored format. The merged columns, any column conflicts, and a boolean value stating if // a full table rewrite is needed to align the existing table rows with the new, merged schema. If any unexpected error // occurs, then that error is returned and the other response fields should be ignored. -func mergeColumns(tblName string, format *storetypes.NomsBinFormat, ourCC, theirCC, ancCC *schema.ColCollection) (*schema.ColCollection, []ColConflict, MergeInfo, error) { +func mergeColumns(tblName string, format *storetypes.NomsBinFormat, ourCC, theirCC, ancCC *schema.ColCollection) (*schema.ColCollection, []ColConflict, tree.ThreeWayDiffInfo, error) { columnMappings, err := mapColumns(ourCC, theirCC, ancCC) if err != nil { - return nil, nil, MergeInfo{}, err + return nil, nil, tree.ThreeWayDiffInfo{}, err } conflicts, err := checkSchemaConflicts(columnMappings) if err != nil { - return nil, nil, MergeInfo{}, err + return nil, nil, tree.ThreeWayDiffInfo{}, err } err = checkUnmergeableNewColumns(tblName, columnMappings) if err != nil { - return nil, nil, MergeInfo{}, err + return nil, nil, tree.ThreeWayDiffInfo{}, err } compatChecker := newTypeCompatabilityCheckerForStorageFormat(format) @@ -504,16 +498,16 @@ func mergeColumns(tblName string, format *storetypes.NomsBinFormat, ourCC, their // Check that there are no duplicate column names or tags in the merged column set conflicts = append(conflicts, checkForColumnConflicts(mergedColumns)...) if conflicts != nil { - return nil, conflicts, MergeInfo{}, nil + return nil, conflicts, tree.ThreeWayDiffInfo{}, nil } - mergeInfo := MergeInfo{ + diffInfo := tree.ThreeWayDiffInfo{ TableRewrite: tableRewrite, LeftSchemaChange: ourSchemaChanged, RightSchemaChange: theirSchemaChanged, LeftAndRightSchemasDiffer: leftAndRightSchemasDiffer, } - return schema.NewColCollection(mergedColumns...), nil, mergeInfo, nil + return schema.NewColCollection(mergedColumns...), nil, diffInfo, nil } // checkForColumnConflicts iterates over |mergedColumns|, checks for duplicate column names or column tags, and returns diff --git a/go/store/prolly/tree/three_way_differ.go b/go/store/prolly/tree/three_way_differ.go index 1740255c48..42a6dc371f 100644 --- a/go/store/prolly/tree/three_way_differ.go +++ b/go/store/prolly/tree/three_way_differ.go @@ -41,26 +41,31 @@ type ThreeWayDiffer[K ~[]byte, O Ordering[K]] struct { type resolveCb func(context.Context, val.Tuple, val.Tuple, val.Tuple) (val.Tuple, bool, error) +type ThreeWayDiffInfo struct { + TableRewrite bool + LeftSchemaChange bool + RightSchemaChange bool + LeftAndRightSchemasDiffer bool +} + func NewThreeWayDiffer[K, V ~[]byte, O Ordering[K]]( ctx context.Context, ns NodeStore, left StaticMap[K, V, O], - leftSchemaChanged bool, right StaticMap[K, V, O], - rightSchemaChanged bool, base StaticMap[K, V, O], resolveCb resolveCb, keyless bool, - leftAndRightSchemasDiffer bool, + diffInfo ThreeWayDiffInfo, order O, ) (*ThreeWayDiffer[K, O], error) { // probably compute each of these separately - ld, err := DifferFromRoots[K](ctx, ns, ns, base.Root, left.Root, order, leftSchemaChanged) + ld, err := DifferFromRoots[K](ctx, ns, ns, base.Root, left.Root, order, diffInfo.LeftSchemaChange) if err != nil { return nil, err } - rd, err := DifferFromRoots[K](ctx, ns, ns, base.Root, right.Root, order, rightSchemaChanged) + rd, err := DifferFromRoots[K](ctx, ns, ns, base.Root, right.Root, order, diffInfo.RightSchemaChange) if err != nil { return nil, err } @@ -70,7 +75,7 @@ func NewThreeWayDiffer[K, V ~[]byte, O Ordering[K]]( rIter: rd, resolveCb: resolveCb, keyless: keyless, - leftAndRightSchemasDiffer: leftAndRightSchemasDiffer, + leftAndRightSchemasDiffer: diffInfo.LeftAndRightSchemasDiffer, }, nil } diff --git a/go/store/prolly/tree/three_way_differ_test.go b/go/store/prolly/tree/three_way_differ_test.go index 2146fd7107..652e294734 100644 --- a/go/store/prolly/tree/three_way_differ_test.go +++ b/go/store/prolly/tree/three_way_differ_test.go @@ -192,7 +192,8 @@ func TestThreeWayDiffer(t *testing.T) { left := newTestMap(t, ctx, tt.left, ns, valDesc) right := newTestMap(t, ctx, tt.right, ns, valDesc) - iter, err := NewThreeWayDiffer(ctx, ns, left, false, right, false, base, testResolver(t, ns, valDesc, val.NewTupleBuilder(valDesc)), false, keyDesc) + var diffInfo ThreeWayDiffInfo + iter, err := NewThreeWayDiffer(ctx, ns, left, right, base, testResolver(t, ns, valDesc, val.NewTupleBuilder(valDesc)), false, diffInfo, keyDesc) require.NoError(t, err) var cmp []testDiff