From 219466973ba0ee80becb1196e65058a46d33be83 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 12 Sep 2022 09:48:13 -0700 Subject: [PATCH 01/58] getting started on conflicts --- go/cmd/dolt/cli/arg_parser_helpers.go | 9 + .../sqle/dfunctions/dolt_conflicts_resolve.go | 174 ++++++++++++++++++ .../dprocedures/dolt_conflicts_resolve.go | 30 +++ .../doltcore/sqle/dprocedures/init.go | 1 + 4 files changed, 214 insertions(+) create mode 100644 go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go create mode 100644 go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index 64f66247e9..2dd2a7092e 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -103,6 +103,8 @@ const ( CommitFlag = "commit" NoCommitFlag = "no-commit" NoEditFlag = "no-edit" + OursFlag = "ours" + TheirsFlag = "theirs" ) const ( @@ -132,6 +134,13 @@ func CreateCommitArgParser() *argparser.ArgParser { return ap } +func CreateConflictsResolveArgParser() *argparser.ArgParser { + ap := argparser.NewArgParser() + ap.SupportsFlag(OursFlag, "", "For all conflicts, take the version from our branch and resolve the conflict") + ap.SupportsFlag(TheirsFlag, "", "For all conflicts, take the version from their branch and resolve the conflict") + return ap +} + func CreateMergeArgParser() *argparser.ArgParser { ap := argparser.NewArgParser() ap.SupportsFlag(NoFFParam, "", "Create a merge commit even when the merge resolves as a fast-forward.") diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go new file mode 100644 index 0000000000..dca5103148 --- /dev/null +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -0,0 +1,174 @@ +// Copyright 2020 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dfunctions + +import ( + "errors" + "fmt" + "strings" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/go-mysql-server/sql" +) + +var ErrConfSchIncompatible = errors.New("the conflict schema's columns are not equal to the current schema's columns, please resolve manually") + +const DoltConflictsResolveFuncName = "dolt_conflicts_resolve" + +// DoltConflictsCatFunc runs a `dolt commit` in the SQL context, committing staged changes to head. +// Deprecated: please use the version in the dprocedures package +type DoltConflictsCatFunc struct { + children []sql.Expression +} + +// NewDoltConflictsResolveFunc creates a new DoltCommitFunc expression whose children represents the args passed in DOLT_CONFLICTS_RESOLVE. +// Deprecated: please use the version in the dprocedures package +func NewDoltConflictsResolveFunc(args ...sql.Expression) (sql.Expression, error) { + return &DoltConflictsCatFunc{children: args}, nil +} + +func (d DoltConflictsCatFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { + args, err := getDoltArgs(ctx, row, d.Children()) + if err != nil { + return 1, err + } + return DoDoltConflictsResolve(ctx, args) +} + +func AutoResolveTables(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, ours bool, tblNames []string) error { + for _, tblName := range tblNames { + tbl, ok, err := root.GetTable(ctx, tblName) + if err != nil { + return err + } + if !ok { + return doltdb.ErrTableNotFound + } + + has, err := tbl.HasConflicts(ctx) + if err != nil { + return err + } + if !has { + continue + } + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return err + } + + _, ourSch, theirSch, err := tbl.GetConflictSchemas(ctx, tblName) + if err != nil { + return err + } + + conflictSchema, conflictIndex, err := tbl.GetConflicts(ctx) + if conflictSchema.Base == nil { + } + if conflictIndex == nil { + } + + if ours && !schema.ColCollsAreEqual(sch.GetAllCols(), ourSch.GetAllCols()) { + return ErrConfSchIncompatible + } else if !ours && !schema.ColCollsAreEqual(sch.GetAllCols(), theirSch.GetAllCols()) { + return ErrConfSchIncompatible + } + + } + return nil +} + +func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { + dbName := ctx.GetCurrentDatabase() + + apr, err := cli.CreateConflictsResolveArgParser().Parse(args) + if err != nil { + return 1, err + } + + dSess := dsess.DSessFromSess(ctx.Session) + roots, ok := dSess.GetRoots(ctx, dbName) + if !ok { + return 1, fmt.Errorf("Could not load database %s", dbName) + } + + ours := apr.Contains(cli.OursFlag) + theirs := apr.Contains(cli.TheirsFlag) + if ours && theirs { + return 1, fmt.Errorf("specify only either --ours or --theirs") + } else if !ours && !theirs { + return 1, fmt.Errorf("--ours or --theirs must be supplied") + } + + if apr.NArg() == 0 { + return 1, fmt.Errorf("specify at least one table to resolve conflicts") + } + + // get all tables in conflict + root := roots.Working + tbls := apr.Args + if len(tbls) == 1 && tbls[0] == "." { + if allTables, err := root.TablesInConflict(ctx); err != nil { + return 1, err + } else { + tbls = allTables + } + } + + err = AutoResolveTables(ctx, dSess, root, ours, tbls) + if err != nil { + return 1, err + } + + return 0, nil +} + +func (d DoltConflictsCatFunc) String() string { + childrenStrings := make([]string, len(d.children)) + + for _, child := range d.children { + childrenStrings = append(childrenStrings, child.String()) + } + return fmt.Sprintf("DOLT_CONFLICTS_RESOLVE(%s)", strings.Join(childrenStrings, ",")) +} + +func (d DoltConflictsCatFunc) Type() sql.Type { + return sql.Text +} + +func (d DoltConflictsCatFunc) IsNullable() bool { + return false +} + +func (d DoltConflictsCatFunc) WithChildren(children ...sql.Expression) (sql.Expression, error) { + return NewDoltConflictsResolveFunc(children...) +} + +func (d DoltConflictsCatFunc) Resolved() bool { + for _, child := range d.Children() { + if !child.Resolved() { + return false + } + } + return true +} + +func (d DoltConflictsCatFunc) Children() []sql.Expression { + return d.children +} diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go new file mode 100644 index 0000000000..f0471359e2 --- /dev/null +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go @@ -0,0 +1,30 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dprocedures + +import ( + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" + + "github.com/dolthub/go-mysql-server/sql" +) + +// doltConflictsResolve is the stored procedure version of the function `dolt conflicts cat`. +func doltConflictsResolve(ctx *sql.Context, args ...string) (sql.RowIter, error) { + res, err := dfunctions.DoDoltConflictsResolve(ctx, args) + if err != nil { + return nil, err + } + return rowToIter(res), nil +} diff --git a/go/libraries/doltcore/sqle/dprocedures/init.go b/go/libraries/doltcore/sqle/dprocedures/init.go index ba709e1fc6..741396f69c 100644 --- a/go/libraries/doltcore/sqle/dprocedures/init.go +++ b/go/libraries/doltcore/sqle/dprocedures/init.go @@ -24,6 +24,7 @@ var DoltProcedures = []sql.ExternalStoredProcedureDetails{ {Name: "dolt_clean", Schema: int64Schema("status"), Function: doltClean}, {Name: "dolt_clone", Schema: int64Schema("status"), Function: doltClone}, {Name: "dolt_commit", Schema: stringSchema("hash"), Function: doltCommit}, + {Name: "dolt_conflicts_resolve", Schema: int64Schema("status"), Function: doltConflictsResolve}, {Name: "dolt_fetch", Schema: int64Schema("success"), Function: doltFetch}, {Name: "dolt_merge", Schema: int64Schema("fast_forward", "conflicts"), Function: doltMerge}, {Name: "dolt_pull", Schema: int64Schema("fast_forward", "conflicts"), Function: doltPull}, From c169437a73e1e4e4dc7d7e663caed1645672e933 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 12 Sep 2022 14:52:39 -0700 Subject: [PATCH 02/58] messy and janky, but working --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 115 ++++++++++++++---- 1 file changed, 89 insertions(+), 26 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index dca5103148..4e291b67e9 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,11 +17,13 @@ package dfunctions import ( "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/merge" + "github.com/dolthub/dolt/go/store/types" + "io" "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/go-mysql-server/sql" ) @@ -50,9 +52,10 @@ func (d DoltConflictsCatFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, return DoDoltConflictsResolve(ctx, args) } -func AutoResolveTables(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, ours bool, tblNames []string) error { +func AutoResolveTables(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { + newRoot := root for _, tblName := range tblNames { - tbl, ok, err := root.GetTable(ctx, tblName) + tbl, ok, err := newRoot.GetTable(ctx, tblName) if err != nil { return err } @@ -60,42 +63,102 @@ func AutoResolveTables(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb. return doltdb.ErrTableNotFound } - has, err := tbl.HasConflicts(ctx) + //has, err := tbl.HasConflicts(ctx) + //if err != nil { + // return err + //} + //if !has { + // continue + //} + // + //sch, err := tbl.GetSchema(ctx) + //if err != nil { + // return err + //} + // + _, _, theirSch, err := tbl.GetConflictSchemas(ctx, tblName) if err != nil { return err } - if !has { - continue + // + //conflictSchema, conflictIndex, err := tbl.GetConflicts(ctx) + //if conflictSchema.Base == nil { + //} + //if conflictIndex == nil { + //} + + cnfReader, err := merge.NewConflictReader(ctx, tbl) + if err != nil { } - sch, err := tbl.GetSchema(ctx) + joiner := cnfReader.GetJoiner() + + var pkVals []types.Value + for { + cnfRow, _, err := cnfReader.NextConflict(ctx) + if err == io.EOF { + break + } + cnfMap, err := joiner.Split(cnfRow) + if err != nil { + return err + } + + vrw := tbl.ValueReadWriter() + var pkVal types.Value + if ours { + row := cnfMap["their"] + k := row.NomsMapKey(theirSch) + v := row.NomsMapValue(theirSch) + + kv, _ := k.Value(ctx) + vv, _ := v.Value(ctx) + + newMap, _ := types.NewMap(ctx, vrw, kv, vv) + + updatedTable, _ := tbl.UpdateNomsRows(ctx, newMap) + pkVals = append(pkVals, kv) + _, _, updatedTbl, _ := updatedTable.ResolveConflicts(ctx, pkVals) + newRoot, _ = newRoot.PutTable(ctx, tblName, updatedTbl) + return dSess.SetRoot(ctx, dbName, newRoot) + } else { + row := cnfMap["their"] + pkVal, err = row.NomsMapKey(theirSch).Value(ctx) + aa, _ := pkVal.Value(ctx) + panic(fmt.Sprintf("%v, %v, %v", pkVal, aa, row)) + } + if err != nil { + return err + } + pkVals = append(pkVals, pkVal) + } + + // TODO: problem is that this always picks ours... + _, _, updatedTbl, err := tbl.ResolveConflicts(ctx, pkVals) + if err != nil { + if errors.Is(err, doltdb.ErrNoConflictsResolved) { + return nil + } + return err + } + + newRoot, err = newRoot.PutTable(ctx, tblName, updatedTbl) if err != nil { return err } - _, ourSch, theirSch, err := tbl.GetConflictSchemas(ctx, tblName) - if err != nil { - return err - } - - conflictSchema, conflictIndex, err := tbl.GetConflicts(ctx) - if conflictSchema.Base == nil { - } - if conflictIndex == nil { - } - - if ours && !schema.ColCollsAreEqual(sch.GetAllCols(), ourSch.GetAllCols()) { - return ErrConfSchIncompatible - } else if !ours && !schema.ColCollsAreEqual(sch.GetAllCols(), theirSch.GetAllCols()) { - return ErrConfSchIncompatible - } - + //if ours && !schema.ColCollsAreEqual(sch.GetAllCols(), ourSch.GetAllCols()) { + // return ErrConfSchIncompatible + //} else if !ours && !schema.ColCollsAreEqual(sch.GetAllCols(), theirSch.GetAllCols()) { + // return ErrConfSchIncompatible + //} } - return nil + return dSess.SetRoot(ctx, dbName, newRoot) } func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { dbName := ctx.GetCurrentDatabase() + fmt.Printf("database name: %s", dbName) apr, err := cli.CreateConflictsResolveArgParser().Parse(args) if err != nil { @@ -131,7 +194,7 @@ func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { } } - err = AutoResolveTables(ctx, dSess, root, ours, tbls) + err = AutoResolveTables(ctx, dSess, root, dbName, ours, tbls) if err != nil { return 1, err } From 4c30ae1ee87892c7a253145cdd3c32e03d466cf6 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Mon, 12 Sep 2022 22:11:31 +0000 Subject: [PATCH 03/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 4e291b67e9..3ac5f1142d 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,15 +17,16 @@ package dfunctions import ( "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/merge" - "github.com/dolthub/dolt/go/store/types" "io" "strings" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/merge" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/store/types" ) var ErrConfSchIncompatible = errors.New("the conflict schema's columns are not equal to the current schema's columns, please resolve manually") From eb385cd175e2a7b7bbef7f15f954881664373c8a Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 12 Sep 2022 16:06:31 -0700 Subject: [PATCH 04/58] working, need lots of tests --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 114 +++++++++--------- 1 file changed, 55 insertions(+), 59 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 4e291b67e9..83db0369a6 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,6 +18,8 @@ import ( "errors" "fmt" "github.com/dolthub/dolt/go/libraries/doltcore/merge" + "github.com/dolthub/dolt/go/libraries/doltcore/row" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/store/types" "io" "strings" @@ -63,37 +65,36 @@ func AutoResolveTables(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb. return doltdb.ErrTableNotFound } - //has, err := tbl.HasConflicts(ctx) - //if err != nil { - // return err - //} - //if !has { - // continue - //} - // - //sch, err := tbl.GetSchema(ctx) - //if err != nil { - // return err - //} - // - _, _, theirSch, err := tbl.GetConflictSchemas(ctx, tblName) + if has, err := tbl.HasConflicts(ctx); err != nil { + return err + } else if !has { + return nil + } + + sch, err := tbl.GetSchema(ctx) if err != nil { return err } - // - //conflictSchema, conflictIndex, err := tbl.GetConflicts(ctx) - //if conflictSchema.Base == nil { - //} - //if conflictIndex == nil { - //} + _, ourSch, theirSch, err := tbl.GetConflictSchemas(ctx, tblName) + if err != nil { + return err + } + + if ours && !schema.ColCollsAreEqual(sch.GetAllCols(), ourSch.GetAllCols()) { + return ErrConfSchIncompatible + } else if !ours && !schema.ColCollsAreEqual(sch.GetAllCols(), theirSch.GetAllCols()) { + return ErrConfSchIncompatible + } cnfReader, err := merge.NewConflictReader(ctx, tbl) if err != nil { + return err } joiner := cnfReader.GetJoiner() - var pkVals []types.Value + var pkTuples []types.Value + vrw := tbl.ValueReadWriter() for { cnfRow, _, err := cnfReader.NextConflict(ctx) if err == io.EOF { @@ -104,54 +105,49 @@ func AutoResolveTables(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb. return err } - vrw := tbl.ValueReadWriter() - var pkVal types.Value + var row row.Row + var k, v types.Value if ours { - row := cnfMap["their"] - k := row.NomsMapKey(theirSch) - v := row.NomsMapValue(theirSch) - - kv, _ := k.Value(ctx) - vv, _ := v.Value(ctx) - - newMap, _ := types.NewMap(ctx, vrw, kv, vv) - - updatedTable, _ := tbl.UpdateNomsRows(ctx, newMap) - pkVals = append(pkVals, kv) - _, _, updatedTbl, _ := updatedTable.ResolveConflicts(ctx, pkVals) - newRoot, _ = newRoot.PutTable(ctx, tblName, updatedTbl) - return dSess.SetRoot(ctx, dbName, newRoot) + row = cnfMap["our"] + k, err = row.NomsMapKey(ourSch).Value(ctx) + if err != nil { + return err + } + v, err = row.NomsMapValue(ourSch).Value(ctx) + if err != nil { + return err + } } else { row := cnfMap["their"] - pkVal, err = row.NomsMapKey(theirSch).Value(ctx) - aa, _ := pkVal.Value(ctx) - panic(fmt.Sprintf("%v, %v, %v", pkVal, aa, row)) + k, err = row.NomsMapKey(theirSch).Value(ctx) + if err != nil { + return err + } + v, err = row.NomsMapValue(theirSch).Value(ctx) + if err != nil { + return err + } } - if err != nil { - return err - } - pkVals = append(pkVals, pkVal) + pkTuples = append(pkTuples, k, v) } - // TODO: problem is that this always picks ours... - _, _, updatedTbl, err := tbl.ResolveConflicts(ctx, pkVals) - if err != nil { - if errors.Is(err, doltdb.ErrNoConflictsResolved) { - return nil - } - return err - } - - newRoot, err = newRoot.PutTable(ctx, tblName, updatedTbl) + newMap, err := types.NewMap(ctx, vrw, pkTuples...) if err != nil { return err } - //if ours && !schema.ColCollsAreEqual(sch.GetAllCols(), ourSch.GetAllCols()) { - // return ErrConfSchIncompatible - //} else if !ours && !schema.ColCollsAreEqual(sch.GetAllCols(), theirSch.GetAllCols()) { - // return ErrConfSchIncompatible - //} + updatedTable, err := tbl.UpdateNomsRows(ctx, newMap) + if err != nil { + return err + } + _, _, newTbl, err := updatedTable.ResolveConflicts(ctx, pkTuples) + if err != nil { + return err + } + newRoot, err = newRoot.PutTable(ctx, tblName, newTbl) + if err != nil { + return err + } } return dSess.SetRoot(ctx, dbName, newRoot) } From 8f99d1414189e01420cb459237ca782cd5263ccb Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 12 Sep 2022 16:49:12 -0700 Subject: [PATCH 05/58] renaming function --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 4cc5d88251..cc3212a1b0 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -55,7 +55,7 @@ func (d DoltConflictsCatFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, return DoDoltConflictsResolve(ctx, args) } -func AutoResolveTables(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { +func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { tbl, ok, err := newRoot.GetTable(ctx, tblName) @@ -191,7 +191,7 @@ func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { } } - err = AutoResolveTables(ctx, dSess, root, dbName, ours, tbls) + err = ResolveConflicts(ctx, dSess, root, dbName, ours, tbls) if err != nil { return 1, err } From 7a56d3a86f974e4be33cca959c06690e176f4ee7 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 12 Sep 2022 16:49:20 -0700 Subject: [PATCH 06/58] adding simple tests --- .../bats/sql-conflicts-resolve.bats | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 integration-tests/bats/sql-conflicts-resolve.bats diff --git a/integration-tests/bats/sql-conflicts-resolve.bats b/integration-tests/bats/sql-conflicts-resolve.bats new file mode 100644 index 0000000000..c8d9197ee4 --- /dev/null +++ b/integration-tests/bats/sql-conflicts-resolve.bats @@ -0,0 +1,153 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash + +setup() { + setup_common +} + +basic_conflict() { + dolt sql -q "create table t (i int primary key, t text)" + dolt add . + dolt commit -am "init commit" + dolt checkout -b other + dolt sql -q "insert into t values (1,'other')" + dolt commit -am "other commit" + dolt checkout main + dolt sql -q "insert into t values (1,'main')" + dolt commit -am "main commit" +} + +teardown() { + assert_feature_version + teardown_common +} + +@test "sql-conflicts-resolve: call with no arguments, errors" { + run dolt sql -q "call dolt_conflicts_resolve()" + [ $status -eq 1 ] + [[ $output =~ "--ours or --theirs must be supplied" ]] || false +} + +@test "sql-conflicts-resolve: call without specifying table, errors" { + run dolt sql -q "call dolt_conflicts_resolve('--theirs')" + [ $status -eq 1 ] + [[ $output =~ "specify at least one table to resolve conflicts" ]] || false +} + +@test "sql-conflicts-resolve: call with non-existent table, errors" { + run dolt sql -q "call dolt_conflicts_resolve('--ours', 'notexists')" + [ $status -eq 1 ] + [[ $output =~ "table not found" ]] || false +} + +@test "sql-conflicts-resolve: no conflicts, no changes" { + basic_conflict + + dolt checkout main + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "main" ]] || false + + dolt checkout other + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "other" ]] || false + + dolt checkout main + run dolt sql -q "CALL dolt_conflicts_resolve('--ours', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "main" ]] || false + + run dolt sql -q "CALL dolt_conflicts_resolve('--theirs', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "main" ]] || false + + dolt checkout other + run dolt sql -q "CALL dolt_conflicts_resolve('--ours', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "other" ]] || false + + run dolt sql -q "CALL dolt_conflicts_resolve('--theirs', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "other" ]] || false +} + +@test "sql-conflicts-resolve: merge other into main, resolve with ours" { + basic_conflict + + dolt checkout main + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "main" ]] || false + + run dolt merge other + [[ $output =~ "Automatic merge failed" ]] || false + + run dolt sql -q "CALL dolt_conflicts_resolve('--ours', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "main" ]] || false +} + +@test "sql-conflicts-resolve: merge other into main, resolve with theirs" { + basic_conflict + + dolt checkout main + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "main" ]] || false + + run dolt merge other + [[ $output =~ "Automatic merge failed" ]] || false + + run dolt sql -q "CALL dolt_conflicts_resolve('--theirs', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "other" ]] || false +} + +@test "sql-conflicts-resolve: merge main into other, resolve with ours" { + basic_conflict + + dolt checkout other + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "other" ]] || false + + run dolt merge main + [[ $output =~ "Automatic merge failed" ]] || false + + run dolt sql -q "CALL dolt_conflicts_resolve('--ours', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "other" ]] || false +} + +@test "sql-conflicts-resolve: merge main into other, resolve with theirs" { + basic_conflict + + dolt checkout other + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "other" ]] || false + + run dolt merge main + [[ $output =~ "Automatic merge failed" ]] || false + + run dolt sql -q "CALL dolt_conflicts_resolve('--theirs', 't')" + [ $status -eq 0 ] + run dolt sql -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "main" ]] || false +} \ No newline at end of file From 90054e6ac97d456e1a094172d4cd2bd54bbdcaaa Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 12 Sep 2022 16:51:23 -0700 Subject: [PATCH 07/58] unsused func name --- go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index cc3212a1b0..8c06f4c27d 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -33,8 +33,6 @@ import ( var ErrConfSchIncompatible = errors.New("the conflict schema's columns are not equal to the current schema's columns, please resolve manually") -const DoltConflictsResolveFuncName = "dolt_conflicts_resolve" - // DoltConflictsCatFunc runs a `dolt commit` in the SQL context, committing staged changes to head. // Deprecated: please use the version in the dprocedures package type DoltConflictsCatFunc struct { From 3252fcdac2fde5ce2d49d61383ee80878a520ee9 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 13 Sep 2022 10:22:00 -0700 Subject: [PATCH 08/58] working better --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 8c06f4c27d..eb3931e864 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -108,26 +108,21 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R var k, v types.Value if ours { row = cnfMap["our"] - k, err = row.NomsMapKey(ourSch).Value(ctx) - if err != nil { - return err - } - v, err = row.NomsMapValue(ourSch).Value(ctx) - if err != nil { - return err - } } else { - row := cnfMap["their"] - k, err = row.NomsMapKey(theirSch).Value(ctx) - if err != nil { - return err - } - v, err = row.NomsMapValue(theirSch).Value(ctx) - if err != nil { - return err - } + row = cnfMap["their"] + } + + if row != nil { + k, err = row.NomsMapKey(sch).Value(ctx) + if err != nil { + return err + } + v, err = row.NomsMapValue(sch).Value(ctx) + if err != nil { + return err + } + pkTuples = append(pkTuples, k, v) } - pkTuples = append(pkTuples, k, v) } newMap, err := types.NewMap(ctx, vrw, pkTuples...) @@ -135,14 +130,16 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R return err } - updatedTable, err := tbl.UpdateNomsRows(ctx, newMap) + newTbl, err := tbl.UpdateNomsRows(ctx, newMap) if err != nil { return err } - _, _, newTbl, err := updatedTable.ResolveConflicts(ctx, pkTuples) + + newTbl, err = newTbl.ClearConflicts(ctx) if err != nil { return err } + newRoot, err = newRoot.PutTable(ctx, tblName, newTbl) if err != nil { return err From b32d5a5b5621fb8669804ebb7dd50c5ea98584c7 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 13 Sep 2022 10:22:14 -0700 Subject: [PATCH 09/58] adding one new test --- .../bats/sql-conflicts-resolve.bats | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/integration-tests/bats/sql-conflicts-resolve.bats b/integration-tests/bats/sql-conflicts-resolve.bats index c8d9197ee4..cfab792ff1 100644 --- a/integration-tests/bats/sql-conflicts-resolve.bats +++ b/integration-tests/bats/sql-conflicts-resolve.bats @@ -150,4 +150,82 @@ teardown() { run dolt sql -q "select * from t" [ $status -eq 0 ] [[ $output =~ "main" ]] || false +} + +@test "sql-conflicts-resolve: two branches, one deletes rows, one modifies those same rows. merge. conflict" { + dolt sql -q 'CREATE TABLE foo (`pk` INT PRIMARY KEY, `col:1` INT);' + dolt sql -q "INSERT INTO foo VALUES (1, 1), (2, 1), (3, 1), (4, 1), (5, 1);" + dolt add foo + dolt commit -m 'initial commit.' + + dolt checkout -b deleter + dolt sql -q 'delete from foo' + dolt add foo + dolt commit -m 'delete commit.' + + dolt checkout -b modifier main + dolt sql -q 'update foo set `col:1` = `col:1` + 1 where pk in (1, 3, 5);' + dolt add foo + dolt commit -m 'modify commit.' + + dolt checkout -b merge-into-modified modifier + run dolt merge deleter -m "merge" + [ "$status" -eq 0 ] + [[ "$output" =~ "CONFLICT" ]] || false + dolt merge --abort + + # Accept theirs deletes all rows. + dolt checkout main + dolt branch -d -f merge-into-modified + dolt checkout -b merge-into-modified modifier + dolt merge deleter -m "merge" + + dolt sql -q "call dolt_conflicts_resolve('--theirs', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 0 |" ]] || false + dolt merge --abort + dolt reset --hard + + # Accept ours deletes two rows. + dolt checkout main + dolt branch -d -f merge-into-modified + dolt checkout -b merge-into-modified modifier + dolt merge deleter -m "merge" + dolt sql -q "call dolt_conflicts_resolve('--ours', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 3 |" ]] || false + dolt merge --abort + dolt reset --hard + + dolt checkout -b merge-into-deleter deleter + run dolt merge modifier -m "merge" + [ "$status" -eq 0 ] + [[ "$output" =~ "CONFLICT" ]] || false + dolt merge --abort + + # Accept ours deletes all rows. + dolt checkout main + dolt branch -d -f merge-into-deleter + dolt checkout -b merge-into-deleter deleter + dolt merge modifier -m "merge" + dolt sql -q "call dolt_conflicts_resolve('--ours', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 0 |" ]] || false + dolt merge --abort + dolt reset --hard + + # Accept theirs adds modified. + dolt checkout main + dolt branch -d -f merge-into-deleter + dolt checkout -b merge-into-deleter deleter + dolt merge modifier -m "merge" + dolt sql -q "call dolt_conflicts_resolve('--theirs', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 3 |" ]] || false + dolt merge --abort + dolt reset --hard } \ No newline at end of file From ad5421d5e76646d59d86afafed2cb928b1df8dfd Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 13 Sep 2022 10:24:43 -0700 Subject: [PATCH 10/58] working, so far --- .../bats/sql-conflicts-resolve.bats | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/integration-tests/bats/sql-conflicts-resolve.bats b/integration-tests/bats/sql-conflicts-resolve.bats index cfab792ff1..67eb277e99 100644 --- a/integration-tests/bats/sql-conflicts-resolve.bats +++ b/integration-tests/bats/sql-conflicts-resolve.bats @@ -228,4 +228,45 @@ teardown() { [[ "$output" =~ "| 3 |" ]] || false dolt merge --abort dolt reset --hard +} + +@test "sql-conflicts-resolve: conflicts table properly cleared on dolt conflicts resolve" { + dolt sql -q "create table test(pk int, c1 int, primary key(pk))" + + run dolt conflicts cat test + [ $status -eq 0 ] + [ "$output" = "" ] + ! [[ "$output" =~ "pk" ]] || false + + dolt add . + dolt commit -m "created table" + dolt branch branch1 + dolt sql -q "insert into test values (0,0)" + dolt add . + dolt commit -m "inserted 0,0" + dolt checkout branch1 + dolt sql -q "insert into test values (0,1)" + dolt add . + dolt commit -m "inserted 0,1" + dolt checkout main + dolt merge branch1 -m "merge" + run dolt sql -q "call dolt_conflicts_resolve('--ours', 'test')" + [ $status -eq 0 ] + + run dolt conflicts cat test + [ $status -eq 0 ] + [ "$output" = "" ] + ! [[ "$output" =~ "pk" ]] || false + + run dolt sql -q "update test set c1=1" + [ $status -eq 0 ] + ! [[ "$output" =~ "unresolved conflicts from the merge" ]] || false + + dolt add . + dolt commit -m "Committing active merge" + + run dolt conflicts cat test + [ $status -eq 0 ] + [ "$output" = "" ] + ! [[ "$output" =~ "pk" ]] || false } \ No newline at end of file From 9398e12e26d2b2e30086a0baf38890c5a2b533e4 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 13 Sep 2022 12:03:23 -0700 Subject: [PATCH 11/58] check status for bats? --- integration-tests/bats/sql-conflicts-resolve.bats | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-tests/bats/sql-conflicts-resolve.bats b/integration-tests/bats/sql-conflicts-resolve.bats index 67eb277e99..861c737ae6 100644 --- a/integration-tests/bats/sql-conflicts-resolve.bats +++ b/integration-tests/bats/sql-conflicts-resolve.bats @@ -89,6 +89,7 @@ teardown() { [[ $output =~ "main" ]] || false run dolt merge other + [ $status -eq 0 ] [[ $output =~ "Automatic merge failed" ]] || false run dolt sql -q "CALL dolt_conflicts_resolve('--ours', 't')" @@ -107,6 +108,7 @@ teardown() { [[ $output =~ "main" ]] || false run dolt merge other + [ $status -eq 0 ] [[ $output =~ "Automatic merge failed" ]] || false run dolt sql -q "CALL dolt_conflicts_resolve('--theirs', 't')" @@ -125,6 +127,7 @@ teardown() { [[ $output =~ "other" ]] || false run dolt merge main + [ $status -eq 0 ] [[ $output =~ "Automatic merge failed" ]] || false run dolt sql -q "CALL dolt_conflicts_resolve('--ours', 't')" @@ -143,6 +146,7 @@ teardown() { [[ $output =~ "other" ]] || false run dolt merge main + [ $status -eq 0 ] [[ $output =~ "Automatic merge failed" ]] || false run dolt sql -q "CALL dolt_conflicts_resolve('--theirs', 't')" From e5f3e38cdf22307cb2d98a9fd641c481af8f2fbb Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 13 Sep 2022 15:12:53 -0700 Subject: [PATCH 12/58] tmp --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index eb3931e864..8fbd2eccc4 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,6 +17,7 @@ package dfunctions import ( "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "io" "strings" @@ -85,6 +86,23 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R return ErrConfSchIncompatible } + if tbl.Format() == types.Format_DOLT { + artifactIdx, err := tbl.GetArtifacts(ctx) + if err != nil { + return err + } + + artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + iter, err := artifactMap.IterAllConflicts(ctx) + if err != nil { + return err + } + + iter.Next(ctx) + + } + + // WORKS FOR OLD FORMAT cnfReader, err := merge.NewConflictReader(ctx, tbl) if err != nil { return err From ab8f63b5e2b2fee611190ae03a699b4bfc6554bf Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 13 Sep 2022 15:36:06 -0700 Subject: [PATCH 13/58] aaaaaaaaaaa --- go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 8fbd2eccc4..ad2d64df22 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -99,7 +99,6 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R } iter.Next(ctx) - } // WORKS FOR OLD FORMAT From 119c14876a1260763d63a2b9ebbe868a6be73c2a Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 14 Sep 2022 09:54:57 -0700 Subject: [PATCH 14/58] blah --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index ad2d64df22..eedd4cb678 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "io" "strings" @@ -87,6 +88,8 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R } if tbl.Format() == types.Format_DOLT { + dtables.NewConflictsTable(ctx, tblName, root, nil) + artifactIdx, err := tbl.GetArtifacts(ctx) if err != nil { return err @@ -98,7 +101,11 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R return err } - iter.Next(ctx) + cnfArt, err := iter.Next(ctx) + + doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) + cnfArt.Metadata.BaseRootIsh + cnfArt.TheirRootIsh } // WORKS FOR OLD FORMAT From 174ac3c8a148b0ab1f20d079ef86616bd0e1cfd0 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Tue, 20 Sep 2022 19:08:20 +0000 Subject: [PATCH 15/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go | 5 +++-- .../dolt/services/remotesapi/v1alpha1/chunkstore_grpc.pb.go | 1 + .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go index 24db1b1319..e64e5a5b5f 100644 --- a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go +++ b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go @@ -21,11 +21,12 @@ package remotesapi import ( + reflect "reflect" + sync "sync" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" - sync "sync" ) const ( diff --git a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore_grpc.pb.go b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore_grpc.pb.go index 48a2391de4..f56a6d51c3 100644 --- a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore_grpc.pb.go +++ b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore_grpc.pb.go @@ -8,6 +8,7 @@ package remotesapi import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index eedd4cb678..e5e753db32 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,8 +17,6 @@ package dfunctions import ( "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "io" "strings" @@ -26,10 +24,12 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/merge" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/dolt/go/store/types" ) From f9736877d57ad5803984d8d82196d7b6d7b60735 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 20 Sep 2022 13:36:46 -0700 Subject: [PATCH 16/58] tmp --- go/libraries/doltcore/merge/conflict_reader.go | 4 ++-- .../sqle/dfunctions/dolt_conflicts_resolve.go | 13 +++++-------- .../doltcore/sqle/dtables/conflicts_tables.go | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go/libraries/doltcore/merge/conflict_reader.go b/go/libraries/doltcore/merge/conflict_reader.go index e8d6235db1..37739a39bf 100644 --- a/go/libraries/doltcore/merge/conflict_reader.go +++ b/go/libraries/doltcore/merge/conflict_reader.go @@ -52,8 +52,8 @@ type ConflictReader struct { } // NewConflictReader returns a new conflict reader for a given table -func NewConflictReader(ctx context.Context, tbl *doltdb.Table) (*ConflictReader, error) { - base, sch, mergeSch, err := tbl.GetConflictSchemas(ctx, "") // tblName unused by old storage format +func NewConflictReader(ctx context.Context, tbl *doltdb.Table, tblName string) (*ConflictReader, error) { + base, sch, mergeSch, err := tbl.GetConflictSchemas(ctx, tblName) // tblName unused by old storage format if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index eedd4cb678..c2dfd12c06 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -1,4 +1,4 @@ -// Copyright 2020 Dolthub, Inc. +// Copyright 2022 Dolthub, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ import ( "errors" "fmt" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "io" "strings" @@ -88,7 +87,7 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R } if tbl.Format() == types.Format_DOLT { - dtables.NewConflictsTable(ctx, tblName, root, nil) + newTbl, err := tbl.ClearConflicts(ctx) artifactIdx, err := tbl.GetArtifacts(ctx) if err != nil { @@ -102,14 +101,12 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R } cnfArt, err := iter.Next(ctx) - doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) - cnfArt.Metadata.BaseRootIsh - cnfArt.TheirRootIsh + } // WORKS FOR OLD FORMAT - cnfReader, err := merge.NewConflictReader(ctx, tbl) + cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) if err != nil { return err } @@ -119,7 +116,7 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R var pkTuples []types.Value vrw := tbl.ValueReadWriter() for { - cnfRow, _, err := cnfReader.NextConflict(ctx) + cnfRow, err := cnfReader.NextConflict(ctx) if err == io.EOF { break } diff --git a/go/libraries/doltcore/sqle/dtables/conflicts_tables.go b/go/libraries/doltcore/sqle/dtables/conflicts_tables.go index 13c3f4927e..c94dc9e4e5 100644 --- a/go/libraries/doltcore/sqle/dtables/conflicts_tables.go +++ b/go/libraries/doltcore/sqle/dtables/conflicts_tables.go @@ -43,7 +43,7 @@ func NewConflictsTable(ctx *sql.Context, tblName string, root *doltdb.RootValue, } func newNomsConflictsTable(ctx *sql.Context, tbl *doltdb.Table, tblName string, root *doltdb.RootValue, rs RootSetter) (sql.Table, error) { - rd, err := merge.NewConflictReader(ctx, tbl) + rd, err := merge.NewConflictReader(ctx, tbl, tblName) if err != nil { return nil, err } @@ -109,7 +109,7 @@ func (ct ConflictsTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) // PartitionRows returns a RowIter for the given partition func (ct ConflictsTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) { // conflict reader must be reset each time partitionRows is called. - rd, err := merge.NewConflictReader(ctx, ct.tbl) + rd, err := merge.NewConflictReader(ctx, ct.tbl, ct.tblName) if err != nil { return nil, err } From 0b917484c9ed6cbdf2e16cb1fc321a58bf2103f0 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 20 Sep 2022 13:42:54 -0700 Subject: [PATCH 17/58] idk wtf git is doing --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 38d3a165a6..994a6c3619 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,10 +17,6 @@ package dfunctions import ( "errors" "fmt" -<<<<<<< HEAD - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" -======= ->>>>>>> 174ac3c8a148b0ab1f20d079ef86616bd0e1cfd0 "io" "strings" @@ -33,7 +29,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables" "github.com/dolthub/dolt/go/store/types" ) From 21d6ccbe0bda7382868c3119943f32b07058c061 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 20 Sep 2022 16:20:23 -0700 Subject: [PATCH 18/58] very sus but seems to work --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 219 +++++++++++++----- 1 file changed, 163 insertions(+), 56 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 994a6c3619..7619a37acd 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,6 +17,9 @@ package dfunctions import ( "errors" "fmt" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/tree" "io" "strings" @@ -54,6 +57,24 @@ func (d DoltConflictsCatFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, return DoDoltConflictsResolve(ctx, args) } +func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeStore, hash hash.Hash, tblName string) (prolly.Map, error) { + rootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, vrw, ns, hash) + tbl, ok, err := rootVal.GetTable(ctx, tblName) + if err != nil { + return prolly.Map{}, err + } + if !ok { + return prolly.Map{}, doltdb.ErrTableNotFound + } + + idx, err := tbl.GetRowData(ctx) + if err != nil { + return prolly.Map{}, err + } + + return durable.ProllyMapFromIndex(idx), nil +} + func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { @@ -86,74 +107,160 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R return ErrConfSchIncompatible } + var newTbl *doltdb.Table if tbl.Format() == types.Format_DOLT { - newTbl, err := tbl.ClearConflicts(ctx) - - artifactIdx, err := tbl.GetArtifacts(ctx) - if err != nil { - return err - } - - artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) - iter, err := artifactMap.IterAllConflicts(ctx) - if err != nil { - return err - } - - cnfArt, err := iter.Next(ctx) - doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) - - } - - // WORKS FOR OLD FORMAT - cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) - if err != nil { - return err - } - - joiner := cnfReader.GetJoiner() - - var pkTuples []types.Value - vrw := tbl.ValueReadWriter() - for { - cnfRow, err := cnfReader.NextConflict(ctx) - if err == io.EOF { - break - } - cnfMap, err := joiner.Split(cnfRow) - if err != nil { - return err - } - - var row row.Row - var k, v types.Value + var idx durable.Index if ours { - row = cnfMap["our"] + idx, err = tbl.GetRowData(ctx) + if err != nil { + return err + } } else { - row = cnfMap["their"] - } - - if row != nil { - k, err = row.NomsMapKey(sch).Value(ctx) + artifactIdx, err := tbl.GetArtifacts(ctx) if err != nil { return err } - v, err = row.NomsMapValue(sch).Value(ctx) + + artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + iter, err := artifactMap.IterAllConflicts(ctx) + if err != nil { + return err + } + + cnfArt, err := iter.Next(ctx) + if err == io.EOF { + break + } + theirRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh) + theirTbl, ok, err := theirRootVal.GetTable(ctx, tblName) + if err != nil { + return err + } + if !ok { + return doltdb.ErrTableNotFound + } + + idx, err = theirTbl.GetRowData(ctx) if err != nil { return err } - pkTuples = append(pkTuples, k, v) } - } - newMap, err := types.NewMap(ctx, vrw, pkTuples...) - if err != nil { - return err - } + newTbl, err = tbl.UpdateRows(ctx, idx) + if err != nil { + return err + } + // TODO: did I need any of this? delete after testing + //artifactIdx, err := tbl.GetArtifacts(ctx) + //if err != nil { + // return err + //} + // + //artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + //iter, err := artifactMap.IterAllConflicts(ctx) + //if err != nil { + // return err + //} + // + //for { + // cnfArt, err := iter.Next(ctx) + // if err == io.EOF { + // break + // } + // + // baseRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) + // if err != nil { + // return err + // } + // + // ourIdx, err := tbl.GetRowData(ctx) + // if err != nil { + // return err + // } + // ourRows := durable.ProllyMapFromIndex(ourIdx) + // + // theirRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) + // if err != nil { + // return err + // } + // + // var bRow val.Tuple + // err = baseRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { + // bRow = v + // return nil + // }) + // + // var oRow val.Tuple + // err = ourRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { + // oRow = v + // return nil + // }) + // + // var tRow val.Tuple + // err = theirRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { + // tRow = v + // return nil + // }) + // + // bRow.Count() + // oRow.Count() + // tRow.Count() + // + // newTbl, err = tbl.UpdateRows(ctx, ourIdx) + // if err != nil { + // return err + // } + //} + } else { + cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) + if err != nil { + return err + } - newTbl, err := tbl.UpdateNomsRows(ctx, newMap) - if err != nil { - return err + joiner := cnfReader.GetJoiner() + + var pkTuples []types.Value + vrw := tbl.ValueReadWriter() + for { + cnfRow, err := cnfReader.NextConflict(ctx) + if err == io.EOF { + break + } + cnfMap, err := joiner.Split(cnfRow) + if err != nil { + return err + } + + var row row.Row + var k, v types.Value + if ours { + row = cnfMap["our"] + } else { + row = cnfMap["their"] + } + + if row != nil { + k, err = row.NomsMapKey(sch).Value(ctx) + if err != nil { + return err + } + v, err = row.NomsMapValue(sch).Value(ctx) + if err != nil { + return err + } + pkTuples = append(pkTuples, k, v) + } + } + + newMap, err := types.NewMap(ctx, vrw, pkTuples...) + if err != nil { + return err + } + + newTbl, err = tbl.UpdateNomsRows(ctx, newMap) + if err != nil { + return err + } } newTbl, err = newTbl.ClearConflicts(ctx) From f504a7d225e66662dfd40462542cb9ee195e6f85 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Tue, 20 Sep 2022 23:34:07 +0000 Subject: [PATCH 19/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 7619a37acd..bfbb7881a5 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,9 +17,6 @@ package dfunctions import ( "errors" "fmt" - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/prolly" - "github.com/dolthub/dolt/go/store/prolly/tree" "io" "strings" @@ -32,6 +29,9 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" ) From 5f5f82bc371f7d8aff088c085ac11e3f0a9d9978 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 21 Sep 2022 09:50:28 -0700 Subject: [PATCH 20/58] moving old to helper method --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 103 +++++++++--------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index bfbb7881a5..9d0a772739 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -75,6 +75,59 @@ func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeS return durable.ProllyMapFromIndex(idx), nil } +func resolveOldFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { + cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) + if err != nil { + return nil, err + } + + joiner := cnfReader.GetJoiner() + + var pkTuples []types.Value + vrw := tbl.ValueReadWriter() + for { + cnfRow, err := cnfReader.NextConflict(ctx) + if err == io.EOF { + break + } + cnfMap, err := joiner.Split(cnfRow) + if err != nil { + return nil, err + } + + var row row.Row + var k, v types.Value + if ours { + row = cnfMap["our"] + } else { + row = cnfMap["their"] + } + + if row != nil { + k, err = row.NomsMapKey(sch).Value(ctx) + if err != nil { + return nil, err + } + v, err = row.NomsMapValue(sch).Value(ctx) + if err != nil { + return nil, err + } + pkTuples = append(pkTuples, k, v) + } + } + + newMap, err := types.NewMap(ctx, vrw, pkTuples...) + if err != nil { + return nil, err + } + + newTbl, err := tbl.UpdateNomsRows(ctx, newMap) + if err != nil { + return nil, err + } + return newTbl, nil +} + func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { @@ -212,55 +265,7 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R // } //} } else { - cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) - if err != nil { - return err - } - - joiner := cnfReader.GetJoiner() - - var pkTuples []types.Value - vrw := tbl.ValueReadWriter() - for { - cnfRow, err := cnfReader.NextConflict(ctx) - if err == io.EOF { - break - } - cnfMap, err := joiner.Split(cnfRow) - if err != nil { - return err - } - - var row row.Row - var k, v types.Value - if ours { - row = cnfMap["our"] - } else { - row = cnfMap["their"] - } - - if row != nil { - k, err = row.NomsMapKey(sch).Value(ctx) - if err != nil { - return err - } - v, err = row.NomsMapValue(sch).Value(ctx) - if err != nil { - return err - } - pkTuples = append(pkTuples, k, v) - } - } - - newMap, err := types.NewMap(ctx, vrw, pkTuples...) - if err != nil { - return err - } - - newTbl, err = tbl.UpdateNomsRows(ctx, newMap) - if err != nil { - return err - } + newTbl, err = resolveOldFormatConflicts(ctx, tbl, tblName, sch, ours) } newTbl, err = newTbl.ClearConflicts(ctx) From 2b9ccf677fdd52a1e47b53a4ada89e221b27d79b Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 21 Sep 2022 10:12:35 -0700 Subject: [PATCH 21/58] seems to be working --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 236 ++++++++++-------- 1 file changed, 134 insertions(+), 102 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 9d0a772739..66f6b06335 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -128,6 +128,139 @@ func resolveOldFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri return newTbl, nil } +func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { + var idx durable.Index + var err error + if ours { + idx, err = tbl.GetRowData(ctx) + if err != nil { + return nil, err + } + } else { + artifactIdx, err := tbl.GetArtifacts(ctx) + if err != nil { + return nil, err + } + + artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + iter, err := artifactMap.IterAllConflicts(ctx) + if err != nil { + return nil, err + } + + cnfArt, err := iter.Next(ctx) + if err == io.EOF { + // no conflicts, should be impossible + return nil, nil + } + + baseRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) + baseTbl, ok, err := baseRootVal.GetTable(ctx, tblName) + if err != nil { + return nil, err + } + if !ok { + return nil, doltdb.ErrTableNotFound + } + + baseIdx, err := baseTbl.GetRowData(ctx) + if err != nil { + return nil, err + } + + theirRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh) + theirTbl, ok, err := theirRootVal.GetTable(ctx, tblName) + if err != nil { + return nil, err + } + if !ok { + return nil, doltdb.ErrTableNotFound + } + + theirIdx, err := theirTbl.GetRowData(ctx) + if err != nil { + return nil, err + } + + baseMap := durable.ProllyMapFromIndex(baseIdx) + theirMap := durable.ProllyMapFromIndex(theirIdx) + merged, err := prolly.MergeMaps(ctx, baseMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { + return right, true + }) + if err != nil { + return nil, err + } + idx = durable.IndexFromProllyMap(merged) + } + + newTbl, err := tbl.UpdateRows(ctx, idx) + if err != nil { + return nil, err + } + return newTbl, nil + // TODO: did I need any of this? delete after testing + //artifactIdx, err := tbl.GetArtifacts(ctx) + //if err != nil { + // return err + //} + // + //artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + //iter, err := artifactMap.IterAllConflicts(ctx) + //if err != nil { + // return err + //} + // + //for { + // cnfArt, err := iter.Next(ctx) + // if err == io.EOF { + // break + // } + // + // baseRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) + // if err != nil { + // return err + // } + // + // ourIdx, err := tbl.GetRowData(ctx) + // if err != nil { + // return err + // } + // ourRows := durable.ProllyMapFromIndex(ourIdx) + // + // theirRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) + // if err != nil { + // return err + // } + // + // var bRow val.Tuple + // err = baseRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { + // bRow = v + // return nil + // }) + // + // var oRow val.Tuple + // err = ourRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { + // oRow = v + // return nil + // }) + // + // var tRow val.Tuple + // err = theirRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { + // tRow = v + // return nil + // }) + // + // bRow.Count() + // oRow.Count() + // tRow.Count() + // + // newTbl, err = tbl.UpdateRows(ctx, ourIdx) + // if err != nil { + // return err + // } + //} +} + func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { @@ -162,108 +295,7 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R var newTbl *doltdb.Table if tbl.Format() == types.Format_DOLT { - var idx durable.Index - if ours { - idx, err = tbl.GetRowData(ctx) - if err != nil { - return err - } - } else { - artifactIdx, err := tbl.GetArtifacts(ctx) - if err != nil { - return err - } - - artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) - iter, err := artifactMap.IterAllConflicts(ctx) - if err != nil { - return err - } - - cnfArt, err := iter.Next(ctx) - if err == io.EOF { - break - } - theirRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh) - theirTbl, ok, err := theirRootVal.GetTable(ctx, tblName) - if err != nil { - return err - } - if !ok { - return doltdb.ErrTableNotFound - } - - idx, err = theirTbl.GetRowData(ctx) - if err != nil { - return err - } - } - - newTbl, err = tbl.UpdateRows(ctx, idx) - if err != nil { - return err - } - // TODO: did I need any of this? delete after testing - //artifactIdx, err := tbl.GetArtifacts(ctx) - //if err != nil { - // return err - //} - // - //artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) - //iter, err := artifactMap.IterAllConflicts(ctx) - //if err != nil { - // return err - //} - // - //for { - // cnfArt, err := iter.Next(ctx) - // if err == io.EOF { - // break - // } - // - // baseRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) - // if err != nil { - // return err - // } - // - // ourIdx, err := tbl.GetRowData(ctx) - // if err != nil { - // return err - // } - // ourRows := durable.ProllyMapFromIndex(ourIdx) - // - // theirRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) - // if err != nil { - // return err - // } - // - // var bRow val.Tuple - // err = baseRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { - // bRow = v - // return nil - // }) - // - // var oRow val.Tuple - // err = ourRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { - // oRow = v - // return nil - // }) - // - // var tRow val.Tuple - // err = theirRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { - // tRow = v - // return nil - // }) - // - // bRow.Count() - // oRow.Count() - // tRow.Count() - // - // newTbl, err = tbl.UpdateRows(ctx, ourIdx) - // if err != nil { - // return err - // } - //} + newTbl, err = resolveNewFormatConflicts(ctx, tbl, tblName, sch, ours) } else { newTbl, err = resolveOldFormatConflicts(ctx, tbl, tblName, sch, ours) } From d8fc123c9f05d6b6b47af3dbc845a17b17cdc88b Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 21 Sep 2022 13:39:16 -0700 Subject: [PATCH 22/58] ok now it's really working --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 222 +++++++----------- 1 file changed, 89 insertions(+), 133 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 66f6b06335..492202c1ec 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -15,6 +15,7 @@ package dfunctions import ( + "bytes" "errors" "fmt" "io" @@ -75,6 +76,94 @@ func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeS return durable.ProllyMapFromIndex(idx), nil } +func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { + var idx durable.Index + var err error + if ours { + // todo: need to actually look at artifacts... + idx, err = tbl.GetRowData(ctx) + if err != nil { + return nil, err + } + + } else { + artifactIdx, err := tbl.GetArtifacts(ctx) + if err != nil { + return nil, err + } + + artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + iter, err := artifactMap.IterAllConflicts(ctx) + if err != nil { + return nil, err + } + + cnfArt, err := iter.Next(ctx) + if err == io.EOF { + // no conflicts, should be impossible + return nil, nil + } + + baseRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) + baseTbl, ok, err := baseRootVal.GetTable(ctx, tblName) + if err != nil { + return nil, err + } + if !ok { + return nil, doltdb.ErrTableNotFound + } + + theirRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh) + theirTbl, ok, err := theirRootVal.GetTable(ctx, tblName) + if err != nil { + return nil, err + } + if !ok { + return nil, doltdb.ErrTableNotFound + } + + baseIdx, err := baseTbl.GetRowData(ctx) + if err != nil { + return nil, err + } + + ourIdx, err := tbl.GetRowData(ctx) + if err != nil { + return nil, err + } + + theirIdx, err := theirTbl.GetRowData(ctx) + if err != nil { + return nil, err + } + + baseMap := durable.ProllyMapFromIndex(baseIdx) + ourMap := durable.ProllyMapFromIndex(ourIdx) + theirMap := durable.ProllyMapFromIndex(theirIdx) + merged, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { + // resolve conflicts with right + if left.From != nil && ((left.To == nil) != (right.To == nil)) { + return right, true + } + if bytes.Compare(left.To, right.To) == 0 { + return left, true + } else { + return right, true + } + }) + if err != nil { + return nil, err + } + idx = durable.IndexFromProllyMap(merged) + } + + newTbl, err := tbl.UpdateRows(ctx, idx) + if err != nil { + return nil, err + } + return newTbl, nil +} + func resolveOldFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) if err != nil { @@ -128,139 +217,6 @@ func resolveOldFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri return newTbl, nil } -func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { - var idx durable.Index - var err error - if ours { - idx, err = tbl.GetRowData(ctx) - if err != nil { - return nil, err - } - } else { - artifactIdx, err := tbl.GetArtifacts(ctx) - if err != nil { - return nil, err - } - - artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) - iter, err := artifactMap.IterAllConflicts(ctx) - if err != nil { - return nil, err - } - - cnfArt, err := iter.Next(ctx) - if err == io.EOF { - // no conflicts, should be impossible - return nil, nil - } - - baseRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) - baseTbl, ok, err := baseRootVal.GetTable(ctx, tblName) - if err != nil { - return nil, err - } - if !ok { - return nil, doltdb.ErrTableNotFound - } - - baseIdx, err := baseTbl.GetRowData(ctx) - if err != nil { - return nil, err - } - - theirRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh) - theirTbl, ok, err := theirRootVal.GetTable(ctx, tblName) - if err != nil { - return nil, err - } - if !ok { - return nil, doltdb.ErrTableNotFound - } - - theirIdx, err := theirTbl.GetRowData(ctx) - if err != nil { - return nil, err - } - - baseMap := durable.ProllyMapFromIndex(baseIdx) - theirMap := durable.ProllyMapFromIndex(theirIdx) - merged, err := prolly.MergeMaps(ctx, baseMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { - return right, true - }) - if err != nil { - return nil, err - } - idx = durable.IndexFromProllyMap(merged) - } - - newTbl, err := tbl.UpdateRows(ctx, idx) - if err != nil { - return nil, err - } - return newTbl, nil - // TODO: did I need any of this? delete after testing - //artifactIdx, err := tbl.GetArtifacts(ctx) - //if err != nil { - // return err - //} - // - //artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) - //iter, err := artifactMap.IterAllConflicts(ctx) - //if err != nil { - // return err - //} - // - //for { - // cnfArt, err := iter.Next(ctx) - // if err == io.EOF { - // break - // } - // - // baseRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) - // if err != nil { - // return err - // } - // - // ourIdx, err := tbl.GetRowData(ctx) - // if err != nil { - // return err - // } - // ourRows := durable.ProllyMapFromIndex(ourIdx) - // - // theirRows, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) - // if err != nil { - // return err - // } - // - // var bRow val.Tuple - // err = baseRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { - // bRow = v - // return nil - // }) - // - // var oRow val.Tuple - // err = ourRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { - // oRow = v - // return nil - // }) - // - // var tRow val.Tuple - // err = theirRows.Get(ctx, cnfArt.Key, func(k, v val.Tuple) error { - // tRow = v - // return nil - // }) - // - // bRow.Count() - // oRow.Count() - // tRow.Count() - // - // newTbl, err = tbl.UpdateRows(ctx, ourIdx) - // if err != nil { - // return err - // } - //} -} - func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { From e00840459c16b265c890d1309a6436ccbc614760 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 21 Sep 2022 13:51:49 -0700 Subject: [PATCH 23/58] ? --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 492202c1ec..eea5bf07cd 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -98,32 +98,12 @@ func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri return nil, err } + // just get first conflict artifact cnfArt, err := iter.Next(ctx) - if err == io.EOF { - // no conflicts, should be impossible - return nil, nil - } - - baseRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) - baseTbl, ok, err := baseRootVal.GetTable(ctx, tblName) - if err != nil { - return nil, err - } - if !ok { - return nil, doltdb.ErrTableNotFound - } - - theirRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh) - theirTbl, ok, err := theirRootVal.GetTable(ctx, tblName) - if err != nil { - return nil, err - } - if !ok { - return nil, doltdb.ErrTableNotFound - } - - baseIdx, err := baseTbl.GetRowData(ctx) if err != nil { + if err == io.EOF { + panic("no conflicts, should be impossible") + } return nil, err } @@ -131,15 +111,18 @@ func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri if err != nil { return nil, err } + ourMap := durable.ProllyMapFromIndex(ourIdx) - theirIdx, err := theirTbl.GetRowData(ctx) + baseMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) + if err != nil { + return nil, err + } + + theirMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) if err != nil { return nil, err } - baseMap := durable.ProllyMapFromIndex(baseIdx) - ourMap := durable.ProllyMapFromIndex(ourIdx) - theirMap := durable.ProllyMapFromIndex(theirIdx) merged, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { // resolve conflicts with right if left.From != nil && ((left.To == nil) != (right.To == nil)) { From 4c48785026ea28b7fcedc53b2525047ef37b6f01 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 21 Sep 2022 14:00:14 -0700 Subject: [PATCH 24/58] compile error from nothing?????? --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 115 +++++++++--------- 1 file changed, 55 insertions(+), 60 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index eea5bf07cd..741c66475c 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -77,69 +77,64 @@ func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeS } func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { - var idx durable.Index var err error - if ours { - // todo: need to actually look at artifacts... - idx, err = tbl.GetRowData(ctx) - if err != nil { - return nil, err - } - - } else { - artifactIdx, err := tbl.GetArtifacts(ctx) - if err != nil { - return nil, err - } - - artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) - iter, err := artifactMap.IterAllConflicts(ctx) - if err != nil { - return nil, err - } - - // just get first conflict artifact - cnfArt, err := iter.Next(ctx) - if err != nil { - if err == io.EOF { - panic("no conflicts, should be impossible") - } - return nil, err - } - - ourIdx, err := tbl.GetRowData(ctx) - if err != nil { - return nil, err - } - ourMap := durable.ProllyMapFromIndex(ourIdx) - - baseMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) - if err != nil { - return nil, err - } - - theirMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) - if err != nil { - return nil, err - } - - merged, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { - // resolve conflicts with right - if left.From != nil && ((left.To == nil) != (right.To == nil)) { - return right, true - } - if bytes.Compare(left.To, right.To) == 0 { - return left, true - } else { - return right, true - } - }) - if err != nil { - return nil, err - } - idx = durable.IndexFromProllyMap(merged) + artifactIdx, err := tbl.GetArtifacts(ctx) + if err != nil { + return nil, err } + artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + iter, err := artifactMap.IterAllConflicts(ctx) + if err != nil { + return nil, err + } + + // just get first conflict artifact + cnfArt, err := iter.Next(ctx) + if err != nil { + if err == io.EOF { + panic("no conflicts, should be impossible") + } + return nil, err + } + + ourIdx, err := tbl.GetRowData(ctx) + if err != nil { + return nil, err + } + ourMap := durable.ProllyMapFromIndex(ourIdx) + + baseMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) + if err != nil { + return nil, err + } + + theirMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) + if err != nil { + return nil, err + } + + // swap the maps when resolving with theirs + if !ours { + ourMap, theirMap = theirMap, ourMap + } + + // resolve conflicts with left + merged, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { + if left.From != nil && ((left.To == nil) != (right.To == nil)) { + return left, true + } + if bytes.Compare(left.To, right.To) == 0 { + return right, true + } else { + return left, true + } + }) + if err != nil { + return nil, err + } + + idx := durable.IndexFromProllyMap(merged) newTbl, err := tbl.UpdateRows(ctx, idx) if err != nil { return nil, err From 3c5180bfde68c3965b22848ecc459e10e3760491 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 21 Sep 2022 14:13:59 -0700 Subject: [PATCH 25/58] idk how I missed this --- go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 741c66475c..c5f6b59c2d 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -120,7 +120,7 @@ func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri } // resolve conflicts with left - merged, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { + merged, _, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { if left.From != nil && ((left.To == nil) != (right.To == nil)) { return left, true } From 4a301966a63817bd34b8156ae735f106fb70a6d6 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 22 Sep 2022 13:06:32 -0700 Subject: [PATCH 26/58] there are lots of bugs, not ready --- integration-tests/bats/1pk5col-ints.bats | 63 ++++++++++ .../bats/conflict-detection-2.bats | 119 ++++++++++++++++++ integration-tests/bats/foreign-keys.bats | 33 +++++ integration-tests/bats/index.bats | 54 ++++++++ integration-tests/bats/keyless.bats | 50 ++++++++ .../bats/sql-conflicts-resolve.bats | 119 ------------------ 6 files changed, 319 insertions(+), 119 deletions(-) diff --git a/integration-tests/bats/1pk5col-ints.bats b/integration-tests/bats/1pk5col-ints.bats index 66d8c0e963..3fd755d124 100755 --- a/integration-tests/bats/1pk5col-ints.bats +++ b/integration-tests/bats/1pk5col-ints.bats @@ -435,6 +435,49 @@ teardown() { [[ "$output" =~ "Merge:" ]] || false } +@test "1pk5col-ints: generate a merge conflict and resolve with ours using stored procedure" { + dolt add test + dolt commit -m "added test table" + dolt branch test-branch + dolt sql -q "insert into test values (0, 1, 2, 3, 4, 5)" + dolt add test + dolt commit -m "added test row" + dolt checkout test-branch + dolt sql -q "insert into test values (0, 1, 2, 3, 4, 6)" + dolt add test + dolt commit -m "added conflicting test row" + dolt checkout main + run dolt merge test-branch --no-commit + [ "$status" -eq 0 ] + [[ "$output" =~ "CONFLICT (content)" ]] + run dolt conflicts cat test + [ "$status" -eq 0 ] + [[ "$output" =~ \+[[:space:]]+\|[[:space:]]+ours[[:space:]] ]] || false + [[ "$output" =~ \+[[:space:]]+\|[[:space:]]+theirs[[:space:]] ]] || false + + EXPECTED=$(echo -e "table,num_conflicts\ntest,1") + run dolt sql -r csv -q 'SELECT * FROM dolt_conflicts' + [ "$status" -eq 0 ] + [[ "$output" =~ "$EXPECTED" ]] || false + + run dolt sql -q "call dolt_conflicts_resolve('--ours', 'test')" + [ "$status" -eq 0 ] + run dolt sql -q "select * from test" + [ "$status" -eq 0 ] + [[ "$output" =~ \|[[:space:]]+5 ]] || false + [[ ! "$output" =~ \|[[:space:]]+6 ]] || false + run dolt conflicts cat test + [[ ! "$output" =~ "ours" ]] || false + [[ ! "$output" =~ "theirs" ]] || false + dolt add test + dolt commit -m "merged and resolved conflict" + run dolt log + [[ "$output" =~ "added test row" ]] || false + [[ "$output" =~ "added conflicting test row" ]] || false + [[ "$output" =~ "merged and resolved conflict" ]] || false + [[ "$output" =~ "Merge:" ]] || false +} + @test "1pk5col-ints: generate a merge conflict and try to roll back using dolt merge --abort" { dolt add test dolt commit -m "added test table" @@ -487,6 +530,26 @@ teardown() { [[ ! "$output" =~ "|5" ]] || false } +@test "1pk5col-ints: generate a merge conflict and resolve with theirs using stored procedure" { + dolt add test + dolt commit -m "added test table" + dolt branch test-branch + dolt sql -q "insert into test values (0, 1, 2, 3, 4, 5)" + dolt add test + dolt commit -m "added test row" + dolt checkout test-branch + dolt sql -q "insert into test values (0, 1, 2, 3, 4, 6)" + dolt add test + dolt commit -m "added conflicting test row" + dolt checkout main + dolt merge test-branch + run dolt sql -q "call dolt_conflicts_resolve('--theirs', 'test')" + [ "$status" -eq 0 ] + run dolt sql -q "select * from test" + [[ "$output" =~ \|[[:space:]]+6 ]] || false + [[ ! "$output" =~ "|5" ]] || false +} + @test "1pk5col-ints: put a row that violates the schema" { run dolt sql -q "insert into test values (0, 1, 2, 3, 4, 'foo')" [ "$status" -ne 0 ] diff --git a/integration-tests/bats/conflict-detection-2.bats b/integration-tests/bats/conflict-detection-2.bats index fa89c83cfa..e909f9ad07 100644 --- a/integration-tests/bats/conflict-detection-2.bats +++ b/integration-tests/bats/conflict-detection-2.bats @@ -459,6 +459,84 @@ SQL dolt reset --hard } +@test "conflict-detection-2: two branches, one deletes rows, one modifies those same rows. merge. conflict. resolve with stored procedure" { + dolt sql -q 'CREATE TABLE foo (`pk` INT PRIMARY KEY, `col:1` INT);' + dolt sql -q "INSERT INTO foo VALUES (1, 1), (2, 1), (3, 1), (4, 1), (5, 1);" + dolt add foo + dolt commit -m 'initial commit.' + + dolt checkout -b deleter + dolt sql -q 'delete from foo' + dolt add foo + dolt commit -m 'delete commit.' + + dolt checkout -b modifier main + dolt sql -q 'update foo set `col:1` = `col:1` + 1 where pk in (1, 3, 5);' + dolt add foo + dolt commit -m 'modify commit.' + + dolt checkout -b merge-into-modified modifier + run dolt merge deleter -m "merge" + [ "$status" -eq 0 ] + [[ "$output" =~ "CONFLICT" ]] || false + dolt merge --abort + + # Accept theirs deletes all rows. + dolt checkout main + dolt branch -d -f merge-into-modified + dolt checkout -b merge-into-modified modifier + dolt merge deleter -m "merge" + + dolt sql -q "call dolt_conflicts_resolve('--theirs', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 0 |" ]] || false + dolt merge --abort + dolt reset --hard + + # Accept ours deletes two rows. + dolt checkout main + dolt branch -d -f merge-into-modified + dolt checkout -b merge-into-modified modifier + dolt merge deleter -m "merge" + dolt sql -q "call dolt_conflicts_resolve('--ours', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 3 |" ]] || false + dolt merge --abort + dolt reset --hard + + dolt checkout -b merge-into-deleter deleter + run dolt merge modifier -m "merge" + [ "$status" -eq 0 ] + [[ "$output" =~ "CONFLICT" ]] || false + dolt merge --abort + + # Accept ours deletes all rows. + dolt checkout main + dolt branch -d -f merge-into-deleter + dolt checkout -b merge-into-deleter deleter + dolt merge modifier -m "merge" + dolt sql -q "call dolt_conflicts_resolve('--ours', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 0 |" ]] || false + dolt merge --abort + dolt reset --hard + + # Accept theirs adds modified. + dolt checkout main + dolt branch -d -f merge-into-deleter + dolt checkout -b merge-into-deleter deleter + dolt merge modifier -m "merge" + dolt sql -q "call dolt_conflicts_resolve('--theirs', 'foo')" + run dolt sql -q 'select count(*) from foo' + [ "$status" -eq 0 ] + [[ "$output" =~ "| 3 |" ]] || false + dolt merge --abort + dolt reset --hard +} + @test "conflict-detection-2: dolt_force_transaction_commit along with dolt_allow_commit_conflicts ignores conflicts" { dolt sql <<"SQL" CREATE TABLE test (pk BIGINT PRIMARY KEY, v1 BIGINT); @@ -528,3 +606,44 @@ SQL [ "$output" = "" ] ! [[ "$output" =~ "pk" ]] || false } + +@test "sql-conflicts-resolve: conflicts table properly cleared on dolt conflicts resolve with stored procedure" { + dolt sql -q "create table test(pk int, c1 int, primary key(pk))" + + run dolt conflicts cat test + [ $status -eq 0 ] + [ "$output" = "" ] + ! [[ "$output" =~ "pk" ]] || false + + dolt add . + dolt commit -m "created table" + dolt branch branch1 + dolt sql -q "insert into test values (0,0)" + dolt add . + dolt commit -m "inserted 0,0" + dolt checkout branch1 + dolt sql -q "insert into test values (0,1)" + dolt add . + dolt commit -m "inserted 0,1" + dolt checkout main + dolt merge branch1 -m "merge" + run dolt sql -q "call dolt_conflicts_resolve('--ours', 'test')" + [ $status -eq 0 ] + + run dolt conflicts cat test + [ $status -eq 0 ] + [ "$output" = "" ] + ! [[ "$output" =~ "pk" ]] || false + + run dolt sql -q "update test set c1=1" + [ $status -eq 0 ] + ! [[ "$output" =~ "unresolved conflicts from the merge" ]] || false + + dolt add . + dolt commit -m "Committing active merge" + + run dolt conflicts cat test + [ $status -eq 0 ] + [ "$output" = "" ] + ! [[ "$output" =~ "pk" ]] || false +} diff --git a/integration-tests/bats/foreign-keys.bats b/integration-tests/bats/foreign-keys.bats index 7e2023a14a..ba82a14c25 100644 --- a/integration-tests/bats/foreign-keys.bats +++ b/integration-tests/bats/foreign-keys.bats @@ -1290,6 +1290,39 @@ SQL [[ "$output" =~ "violation" ]] || false } +@test "foreign-keys: Resolve with stored catches violations" { + dolt sql < Date: Mon, 26 Sep 2022 10:29:45 -0700 Subject: [PATCH 27/58] test format --- integration-tests/bats/conflict-detection-2.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/conflict-detection-2.bats b/integration-tests/bats/conflict-detection-2.bats index e909f9ad07..bc13dfd204 100644 --- a/integration-tests/bats/conflict-detection-2.bats +++ b/integration-tests/bats/conflict-detection-2.bats @@ -607,7 +607,7 @@ SQL ! [[ "$output" =~ "pk" ]] || false } -@test "sql-conflicts-resolve: conflicts table properly cleared on dolt conflicts resolve with stored procedure" { +@test "conflict-detection-2: conflicts table properly cleared on dolt conflicts resolve with stored procedure" { dolt sql -q "create table test(pk int, c1 int, primary key(pk))" run dolt conflicts cat test From f1204c381412fd90e3c2b738f2a125631ca03704 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 26 Sep 2022 16:03:04 -0700 Subject: [PATCH 28/58] keyless conflicts figured out probably, but i have denv anyway --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index c5f6b59c2d..005fe82e3e 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,6 +18,9 @@ import ( "bytes" "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "io" "strings" @@ -195,6 +198,84 @@ func resolveOldFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri return newTbl, nil } +func resolveOldFormatConflicts2(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "0.41.5") + if dEnv == nil { + } + + cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) + if err != nil { + return nil, err + } + + joiner := cnfReader.GetJoiner() + cnfSch := cnfReader.GetSchema() + + // this matters because only keyless tables will have a cardinality + isKeyless := schema.IsKeyless(cnfSch) + if isKeyless { + } + + // Create new table editor + edit, err := editor.NewTableEditor(ctx, tbl, sch, tblName, editor.Options{}) + if err != nil { + return nil, err + } + + // get relevant cardinality tags + tags := cnfSch.GetAllCols().SortedTags + ourCardinalityTag := tags[len(tags)-2] + theirCardinalityTag := tags[len(tags)-1] + + for { + cnfRow, err := cnfReader.NextConflict(ctx) + if err == io.EOF { + break + } + cnfMap, err := joiner.Split(cnfRow) + if err != nil { + return nil, err + } + + // get cardinality + var ourCardinality, theirCardinality uint64 + if ourCardinalityVal, ok := cnfRow.GetColVal(ourCardinalityTag); ok { + ourCardinality = uint64(ourCardinalityVal.(types.Uint)) + } else { + panic("huh") + } + if theirCardinalityVal, ok := cnfRow.GetColVal(theirCardinalityTag); ok { + theirCardinality = uint64(theirCardinalityVal.(types.Uint)) + } else { + panic("huh") + } + + rowDelta := 0 + var row row.Row + if ours { + row = cnfMap["our"] + } else { + row = cnfMap["their"] + rowDelta = int(theirCardinality - ourCardinality) + } + + if rowDelta > 0 { + for i := 0; i < rowDelta; i++ { + edit.InsertRow(ctx, row, func(newKeyString, indexName string, existingKey, existingVal types.Tuple, isPk bool) error { + return nil + }) + } + } else { + rowDelta *= -1 + for i := 0; i < rowDelta; i++ { + edit.DeleteRow(ctx, row) + } + } + } + + return edit.Table(ctx) +} + func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { @@ -231,7 +312,7 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R if tbl.Format() == types.Format_DOLT { newTbl, err = resolveNewFormatConflicts(ctx, tbl, tblName, sch, ours) } else { - newTbl, err = resolveOldFormatConflicts(ctx, tbl, tblName, sch, ours) + newTbl, err = resolveOldFormatConflicts2(ctx, tbl, tblName, sch, ours) } newTbl, err = newTbl.ClearConflicts(ctx) From 84526cdf65aa163c153d1c5c75745ad9be171ddb Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Mon, 26 Sep 2022 23:06:20 +0000 Subject: [PATCH 29/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 005fe82e3e..4fde5fd4c8 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,9 +18,6 @@ import ( "bytes" "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/env" - "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" - "github.com/dolthub/dolt/go/libraries/utils/filesys" "io" "strings" @@ -29,10 +26,13 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/merge" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" From a0f1d6fe327bdd5f25c4f2901546911b78274a16 Mon Sep 17 00:00:00 2001 From: James Cor Date: Mon, 26 Sep 2022 21:06:53 -0700 Subject: [PATCH 30/58] i forget --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 4fde5fd4c8..006c0fc83b 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -199,10 +199,8 @@ func resolveOldFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri } func resolveOldFormatConflicts2(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "0.41.5") - if dEnv == nil { - } - + // TODO: move this somewhere else + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "does this matter?") cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) if err != nil { return nil, err @@ -210,14 +208,15 @@ func resolveOldFormatConflicts2(ctx *sql.Context, tbl *doltdb.Table, tblName str joiner := cnfReader.GetJoiner() cnfSch := cnfReader.GetSchema() - - // this matters because only keyless tables will have a cardinality - isKeyless := schema.IsKeyless(cnfSch) - if isKeyless { - } + isKeyless := schema.IsKeyless(sch) // Create new table editor - edit, err := editor.NewTableEditor(ctx, tbl, sch, tblName, editor.Options{}) + tmpDir, err := dEnv.TempTableFilesDir() + if err != nil { + return nil, err + } + opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} + edit, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) if err != nil { return nil, err } @@ -239,15 +238,15 @@ func resolveOldFormatConflicts2(ctx *sql.Context, tbl *doltdb.Table, tblName str // get cardinality var ourCardinality, theirCardinality uint64 - if ourCardinalityVal, ok := cnfRow.GetColVal(ourCardinalityTag); ok { + if ourCardinalityVal, ok := cnfRow.GetColVal(ourCardinalityTag); ok && isKeyless { ourCardinality = uint64(ourCardinalityVal.(types.Uint)) } else { - panic("huh") + panic("wtf") } - if theirCardinalityVal, ok := cnfRow.GetColVal(theirCardinalityTag); ok { + if theirCardinalityVal, ok := cnfRow.GetColVal(theirCardinalityTag); ok && isKeyless { theirCardinality = uint64(theirCardinalityVal.(types.Uint)) } else { - panic("huh") + panic("wtf") } rowDelta := 0 From 26d0bc6dd165d77cca926b3e240c028374097ed1 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 27 Sep 2022 00:21:58 -0700 Subject: [PATCH 31/58] working for noms, new format tomorrow :)))))) --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 156 ++++++++++-------- 1 file changed, 91 insertions(+), 65 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 006c0fc83b..165fc077e0 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,6 +18,9 @@ import ( "bytes" "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/conflict" + "github.com/dolthub/dolt/go/libraries/doltcore/table" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "io" "strings" @@ -32,7 +35,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" - "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" @@ -145,83 +147,89 @@ func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName stri return newTbl, nil } -func resolveOldFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { - cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) +func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor, ours bool) (*doltdb.Table, error) { + // Get conflicts + _, cnfIdx, err := tbl.GetConflicts(ctx) if err != nil { return nil, err } - joiner := cnfReader.GetJoiner() - - var pkTuples []types.Value - vrw := tbl.ValueReadWriter() - for { - cnfRow, err := cnfReader.NextConflict(ctx) - if err == io.EOF { - break - } - cnfMap, err := joiner.Split(cnfRow) + // Iterate over conflicts, resolve by picking either ours or theirs + conflicts := durable.NomsMapFromConflictIndex(cnfIdx) + err = conflicts.Iter(ctx, func(key, val types.Value) (stop bool, err error) { + cnf, err := conflict.ConflictFromTuple(val.(types.Tuple)) if err != nil { - return nil, err + return true, err } - var row row.Row - var k, v types.Value + k := key.(types.Tuple) + var newVal types.Value if ours { - row = cnfMap["our"] + newVal = cnf.Value } else { - row = cnfMap["their"] + newVal = cnf.MergeValue } - if row != nil { - k, err = row.NomsMapKey(sch).Value(ctx) + // resolve by deleting row + if types.IsNull(newVal) { + baseRow, err := row.FromNoms(sch, k, cnf.Base.(types.Tuple)) if err != nil { - return nil, err + return true, err } - v, err = row.NomsMapValue(sch).Value(ctx) + err = tblEditor.DeleteRow(ctx, baseRow) if err != nil { - return nil, err + return true, err } - pkTuples = append(pkTuples, k, v) + return false, nil } - } - newMap, err := types.NewMap(ctx, vrw, pkTuples...) + newRow, err := row.FromNoms(sch, k, newVal.(types.Tuple)) + if err != nil { + return true, err + } + + if isValid, err := row.IsValid(newRow, sch); err != nil { + return true, err + } else if !isValid { + return true, table.NewBadRow(newRow, "error resolving conflicts", fmt.Sprintf("row with primary key %v in table %s does not match constraints or types of the table's schema.", key, tblName)) + } + + // resolve by inserting new row + if types.IsNull(cnf.Value) { + err = tblEditor.InsertRow(ctx, newRow, nil) + if err != nil { + return true, err + } + return false, nil + } + + // resolve by updating existing row + oldRow, err := row.FromNoms(sch, k, cnf.Value.(types.Tuple)) + if err != nil { + return true, err + } + err = tblEditor.UpdateRow(ctx, oldRow, newRow, nil) + if err != nil { + return true, err + } + return false, nil + }) if err != nil { return nil, err } - - newTbl, err := tbl.UpdateNomsRows(ctx, newMap) - if err != nil { - return nil, err - } - return newTbl, nil + return tblEditor.Table(ctx) } -func resolveOldFormatConflicts2(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { - // TODO: move this somewhere else - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "does this matter?") +func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor, ours bool) (*doltdb.Table, error) { + // Iterate over conflicts, resolve by picking either ours or theirs cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) if err != nil { return nil, err } - joiner := cnfReader.GetJoiner() - cnfSch := cnfReader.GetSchema() - isKeyless := schema.IsKeyless(sch) - - // Create new table editor - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} - edit, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) - if err != nil { - return nil, err - } // get relevant cardinality tags + cnfSch := cnfReader.GetSchema() tags := cnfSch.GetAllCols().SortedTags ourCardinalityTag := tags[len(tags)-2] theirCardinalityTag := tags[len(tags)-1] @@ -236,46 +244,63 @@ func resolveOldFormatConflicts2(ctx *sql.Context, tbl *doltdb.Table, tblName str return nil, err } - // get cardinality + // Get cardinality var ourCardinality, theirCardinality uint64 - if ourCardinalityVal, ok := cnfRow.GetColVal(ourCardinalityTag); ok && isKeyless { + if ourCardinalityVal, ok := cnfRow.GetColVal(ourCardinalityTag); ok { ourCardinality = uint64(ourCardinalityVal.(types.Uint)) } else { - panic("wtf") + panic("shouldn't be possible") } - if theirCardinalityVal, ok := cnfRow.GetColVal(theirCardinalityTag); ok && isKeyless { + if theirCardinalityVal, ok := cnfRow.GetColVal(theirCardinalityTag); ok { theirCardinality = uint64(theirCardinalityVal.(types.Uint)) } else { - panic("wtf") + panic("shouldn't be possible") } rowDelta := 0 - var row row.Row + var newRow row.Row if ours { - row = cnfMap["our"] + newRow = cnfMap["our"] } else { - row = cnfMap["their"] + newRow = cnfMap["their"] rowDelta = int(theirCardinality - ourCardinality) } if rowDelta > 0 { for i := 0; i < rowDelta; i++ { - edit.InsertRow(ctx, row, func(newKeyString, indexName string, existingKey, existingVal types.Tuple, isPk bool) error { - return nil - }) + tblEditor.InsertRow(ctx, newRow, nil) } } else { rowDelta *= -1 for i := 0; i < rowDelta; i++ { - edit.DeleteRow(ctx, row) + tblEditor.DeleteRow(ctx, newRow) } } } - return edit.Table(ctx) + return tblEditor.Table(ctx) } -func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { +func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { + // Create new table editor + tmpDir, err := dEnv.TempTableFilesDir() + if err != nil { + return nil, err + } + opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} + tblEditor, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) + if err != nil { + return nil, err + } + + if schema.IsKeyless(sch) { + return resolveKeylessConflicts(ctx, tbl, tblName, sch, tblEditor, ours) + } else { + return resolvePkConflicts(ctx, tbl, tblName, sch, tblEditor, ours) + } +} + +func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { tbl, ok, err := newRoot.GetTable(ctx, tblName) @@ -311,7 +336,7 @@ func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.R if tbl.Format() == types.Format_DOLT { newTbl, err = resolveNewFormatConflicts(ctx, tbl, tblName, sch, ours) } else { - newTbl, err = resolveOldFormatConflicts2(ctx, tbl, tblName, sch, ours) + newTbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch, ours) } newTbl, err = newTbl.ClearConflicts(ctx) @@ -365,7 +390,8 @@ func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { } } - err = ResolveConflicts(ctx, dSess, root, dbName, ours, tbls) + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "does this matter?") + err = ResolveConflicts(ctx, dEnv, dSess, root, dbName, ours, tbls) if err != nil { return 1, err } From 3d2ea83dc9bb3f2e164db48ecabcad3e6807d54a Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Tue, 27 Sep 2022 07:23:52 +0000 Subject: [PATCH 32/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 165fc077e0..93d8b3a052 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,15 +18,13 @@ import ( "bytes" "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/conflict" - "github.com/dolthub/dolt/go/libraries/doltcore/table" - "github.com/dolthub/dolt/go/libraries/utils/filesys" "io" "strings" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/libraries/doltcore/conflict" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/env" @@ -34,7 +32,9 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/table" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" From bf1de09adf760f61f686c45feb2e258fe47c0a19 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 27 Sep 2022 00:59:28 -0700 Subject: [PATCH 33/58] rename to prolly --- .../doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 165fc077e0..c86f9e39db 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -81,7 +81,7 @@ func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeS return durable.ProllyMapFromIndex(idx), nil } -func resolveNewFormatConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { +func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { var err error artifactIdx, err := tbl.GetArtifacts(ctx) if err != nil { @@ -334,7 +334,7 @@ func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSess var newTbl *doltdb.Table if tbl.Format() == types.Format_DOLT { - newTbl, err = resolveNewFormatConflicts(ctx, tbl, tblName, sch, ours) + newTbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch, ours) } else { newTbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch, ours) } From d000d35f39ea3e474adcfeb839163ad244d48ee4 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 27 Sep 2022 10:04:20 -0700 Subject: [PATCH 34/58] i wrote test bad --- integration-tests/bats/keyless.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/keyless.bats b/integration-tests/bats/keyless.bats index faa0cfe073..f1368d1c7c 100644 --- a/integration-tests/bats/keyless.bats +++ b/integration-tests/bats/keyless.bats @@ -515,7 +515,7 @@ SQL [ $status -eq 0 ] [[ "$output" =~ "CONFLICT" ]] || false - call dolt sql -q "call dolt_conflicts_resolve('--theirs', 'dupe')" + run dolt sql -q "call dolt_conflicts_resolve('--theirs', 'dupe')" [ $status -eq 0 ] dolt commit -am "resolved" run dolt sql -q "select sum(c0), sum(c1) from dupe" -r csv From 6d51206676a14c4d3dcc6382be8c226e36a658c5 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 27 Sep 2022 11:29:20 -0700 Subject: [PATCH 35/58] more tests, and small keyless fix --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 2 +- .../bats/garbage_collection.bats | 18 +++++ integration-tests/bats/json.bats | 50 ++++++++++++ integration-tests/bats/keyless.bats | 80 ++++++++++++++++++- 4 files changed, 148 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index bfd5826e3c..f6f457f056 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -273,7 +273,7 @@ func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string } else { rowDelta *= -1 for i := 0; i < rowDelta; i++ { - tblEditor.DeleteRow(ctx, newRow) + tblEditor.DeleteRow(ctx, cnfMap["our"]) } } } diff --git a/integration-tests/bats/garbage_collection.bats b/integration-tests/bats/garbage_collection.bats index df63a8dbb8..ce29dfdfde 100644 --- a/integration-tests/bats/garbage_collection.bats +++ b/integration-tests/bats/garbage_collection.bats @@ -181,6 +181,24 @@ setup_merge() { [[ "${lines[3]}" =~ "2,12" ]] || false } +@test "garbage_collection: leave merge commit with stored procedure" { + skip_nbf_dolt + setup_merge + dolt merge other -m "merge" + + dolt gc + + dolt sql -q "call dolt_conflicts_resolve('--ours', '.')" + dolt add . + dolt commit -am "resolved conflicts with ours" + + run dolt sql -q "SELECT * FROM test;" -r csv + [ "$status" -eq 0 ] + [[ "${lines[1]}" =~ "0,10" ]] || false + [[ "${lines[2]}" =~ "1,11" ]] || false + [[ "${lines[3]}" =~ "2,12" ]] || false +} + @test "garbage_collection: leave working pre-merge" { setup_merge diff --git a/integration-tests/bats/json.bats b/integration-tests/bats/json.bats index 3b74f9c18f..816cb57861 100644 --- a/integration-tests/bats/json.bats +++ b/integration-tests/bats/json.bats @@ -169,3 +169,53 @@ SQL [ "${lines[1]}" = '1,"{""a"": 1}"' ] [ "${lines[2]}" = '2,"{""b"": 99}"' ] } + +@test "json: merge JSON values with stored procedure" { + dolt sql < 6 order by c0" -r csv + [ $status -eq 0 ] + [[ "${lines[1]}" = "7,7" ]] || false + [[ "${lines[2]}" = "8,8" ]] || false + [[ "${lines[3]}" = "9,9" ]] || false +} + @test "keyless: diff branches with offset mutation history" { dolt branch other @@ -825,6 +852,31 @@ SQL [[ "${lines[4]}" = "9,9" ]] || false } +@test "keyless: merge branches with offset mutation history with stored procedure" { + dolt branch other + + dolt sql -q "INSERT INTO keyless VALUES (7,7),(8,8),(9,9);" + dolt commit -am "inserted on main" + + dolt checkout other + dolt sql -q "INSERT INTO keyless VALUES (7,7),(7,7),(8,8),(9,9);" + dolt commit -am "inserted on other" + + run dolt merge main -m "merge" + [ $status -eq 0 ] + [[ "$output" =~ "CONFLICT" ]] || false + + run dolt sql -q "call dolt_conflicts_resolve('--ours', 'keyless')" + [ $status -eq 0 ] + dolt commit -am "resolved" + run dolt sql -q "select * from keyless where c0 > 6 order by c0" -r csv + [ $status -eq 0 ] + [[ "${lines[1]}" = "7,7" ]] || false + [[ "${lines[2]}" = "7,7" ]] || false + [[ "${lines[3]}" = "8,8" ]] || false + [[ "${lines[4]}" = "9,9" ]] || false +} + @test "keyless: diff delete+add against working" { dolt sql < Date: Tue, 27 Sep 2022 15:19:19 -0700 Subject: [PATCH 36/58] i wonder what this will break --- .../doltcore/merge/merge_prolly_indexes.go | 4 +- .../doltcore/merge/mutable_secondary_index.go | 4 +- .../sqle/dfunctions/dolt_conflicts_resolve.go | 61 ++++++++++++++++++- 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/go/libraries/doltcore/merge/merge_prolly_indexes.go b/go/libraries/doltcore/merge/merge_prolly_indexes.go index c1f20959e5..99ed3499f4 100644 --- a/go/libraries/doltcore/merge/merge_prolly_indexes.go +++ b/go/libraries/doltcore/merge/merge_prolly_indexes.go @@ -253,7 +253,7 @@ func updateProllySecondaryIndexes(ctx context.Context, tm TableMerger, cellWiseE if err != nil { return nil, nil, err } - lm, err := getMutableSecondaryIdxs(ctx, tm.leftSch, ls) + lm, err := GetMutableSecondaryIdxs(ctx, tm.leftSch, ls) if err != nil { return nil, nil, err } @@ -262,7 +262,7 @@ func updateProllySecondaryIndexes(ctx context.Context, tm TableMerger, cellWiseE if err != nil { return nil, nil, err } - rm, err := getMutableSecondaryIdxs(ctx, tm.rightSch, rs) + rm, err := GetMutableSecondaryIdxs(ctx, tm.rightSch, rs) if err != nil { return nil, nil, err } diff --git a/go/libraries/doltcore/merge/mutable_secondary_index.go b/go/libraries/doltcore/merge/mutable_secondary_index.go index 3ca591a9bf..355a365fc2 100644 --- a/go/libraries/doltcore/merge/mutable_secondary_index.go +++ b/go/libraries/doltcore/merge/mutable_secondary_index.go @@ -26,8 +26,8 @@ import ( "github.com/dolthub/dolt/go/store/val" ) -// getMutableSecondaryIdxs returns a MutableSecondaryIdx for each secondary index in |indexes|. -func getMutableSecondaryIdxs(ctx context.Context, sch schema.Schema, indexes durable.IndexSet) ([]MutableSecondaryIdx, error) { +// GetMutableSecondaryIdxs returns a MutableSecondaryIdx for each secondary index in |indexes|. +func GetMutableSecondaryIdxs(ctx context.Context, sch schema.Schema, indexes durable.IndexSet) ([]MutableSecondaryIdx, error) { mods := make([]MutableSecondaryIdx, sch.Indexes().Count()) for i, index := range sch.Indexes().AllIndexes() { idx, err := indexes.GetIndex(ctx, sch, index.Name()) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index f6f457f056..2df41d6c7f 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,6 +18,7 @@ import ( "bytes" "errors" "fmt" + "github.com/dolthub/dolt/go/store/val" "io" "strings" @@ -109,10 +110,19 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, } ourMap := durable.ProllyMapFromIndex(ourIdx) - baseMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh, tblName) + baseRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) + baseTbl, ok, err := baseRootVal.GetTable(ctx, tblName) if err != nil { return nil, err } + if !ok { + return nil, doltdb.ErrTableNotFound + } + baseIdx, err := baseTbl.GetRowData(ctx) + if err != nil { + return nil, err + } + baseMap := durable.ProllyMapFromIndex(baseIdx) theirMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) if err != nil { @@ -125,13 +135,17 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, } // resolve conflicts with left + var indexEdits []tree.Diff merged, _, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { if left.From != nil && ((left.To == nil) != (right.To == nil)) { + indexEdits = append(indexEdits, left) return left, true } if bytes.Compare(left.To, right.To) == 0 { + indexEdits = append(indexEdits, right) return right, true } else { + indexEdits = append(indexEdits, left) return left, true } }) @@ -144,6 +158,51 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, if err != nil { return nil, err } + + // TODO: need to either build from base or delete all the ones that aren't supposed to be there + idxSet, err := baseTbl.GetIndexSet(ctx) + if err != nil { + return nil, err + } + + secondaryIdxs, err := merge.GetMutableSecondaryIdxs(ctx, sch, idxSet) + if err != nil { + return nil, err + } + for _, edit := range indexEdits { + for _, secIdx := range secondaryIdxs { + if len(edit.From) == 0 { + err := secIdx.InsertEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.To)) + if err != nil { + return nil, err + } + } else if len(edit.To) == 0 { + err := secIdx.DeleteEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From)) + if err != nil { + return nil, err + } + } else { + err := secIdx.UpdateEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From), val.Tuple(edit.To)) + if err != nil { + return nil, err + } + } + m, err := secIdx.Map(ctx) + if err != nil { + return nil, err + } + idxSet, err = idxSet.PutIndex(ctx, secIdx.Name, durable.IndexFromProllyMap(m)) + if err != nil { + return nil, err + } + } + } + + newTbl, err = newTbl.SetIndexSet(ctx, idxSet) + if err != nil { + return nil, err + } + return newTbl, nil } From 9760bd26e14d129b6b3eeaa277bf137f6f176913 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Tue, 27 Sep 2022 22:20:47 +0000 Subject: [PATCH 37/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 2df41d6c7f..980a638dc5 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -18,7 +18,6 @@ import ( "bytes" "errors" "fmt" - "github.com/dolthub/dolt/go/store/val" "io" "strings" @@ -40,6 +39,7 @@ import ( "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" + "github.com/dolthub/dolt/go/store/val" ) var ErrConfSchIncompatible = errors.New("the conflict schema's columns are not equal to the current schema's columns, please resolve manually") From 413ca59ec6ed94ad4d9a93070298bf8888e6f942 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 27 Sep 2022 17:15:39 -0700 Subject: [PATCH 38/58] some changes, need to fix prolly conflicts to not remerge --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 131 ++++++++++-------- 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 980a638dc5..7be9cd9d0d 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -82,7 +82,7 @@ func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeS return durable.ProllyMapFromIndex(idx), nil } -func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { +func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { var err error artifactIdx, err := tbl.GetArtifacts(ctx) if err != nil { @@ -95,21 +95,44 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return nil, err } - // just get first conflict artifact - cnfArt, err := iter.Next(ctx) - if err != nil { - if err == io.EOF { - panic("no conflicts, should be impossible") - } - return nil, err - } - ourIdx, err := tbl.GetRowData(ctx) if err != nil { return nil, err } ourMap := durable.ProllyMapFromIndex(ourIdx) + // accumulate index edits + once := true + var theirMap prolly.Map + var indexEdits []tree.Diff + for { + cnfArt, err := iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if once { + theirMap, err = getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) + if err != nil { + return nil, err + } + once = false + } + + var theirRow val.Tuple + theirMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { + theirRow = v + return nil + }) + + // todo: update actual table row + // todo: accumulate index edits + + } + baseRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) baseTbl, ok, err := baseRootVal.GetTable(ctx, tblName) if err != nil { @@ -124,18 +147,8 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, } baseMap := durable.ProllyMapFromIndex(baseIdx) - theirMap, err := getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) - if err != nil { - return nil, err - } - - // swap the maps when resolving with theirs - if !ours { - ourMap, theirMap = theirMap, ourMap - } - // resolve conflicts with left - var indexEdits []tree.Diff + merged, _, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { if left.From != nil && ((left.To == nil) != (right.To == nil)) { indexEdits = append(indexEdits, left) @@ -206,31 +219,24 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return newTbl, nil } -func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor, ours bool) (*doltdb.Table, error) { +func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor) (*doltdb.Table, error) { // Get conflicts _, cnfIdx, err := tbl.GetConflicts(ctx) if err != nil { return nil, err } - // Iterate over conflicts, resolve by picking either ours or theirs + // Iterate over conflicts, resolve with theirs conflicts := durable.NomsMapFromConflictIndex(cnfIdx) err = conflicts.Iter(ctx, func(key, val types.Value) (stop bool, err error) { + k := key.(types.Tuple) cnf, err := conflict.ConflictFromTuple(val.(types.Tuple)) if err != nil { return true, err } - k := key.(types.Tuple) - var newVal types.Value - if ours { - newVal = cnf.Value - } else { - newVal = cnf.MergeValue - } - - // resolve by deleting row - if types.IsNull(newVal) { + // row was removed + if types.IsNull(cnf.MergeValue) { baseRow, err := row.FromNoms(sch, k, cnf.Base.(types.Tuple)) if err != nil { return true, err @@ -242,7 +248,7 @@ func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch return false, nil } - newRow, err := row.FromNoms(sch, k, newVal.(types.Tuple)) + newRow, err := row.FromNoms(sch, k, cnf.MergeValue.(types.Tuple)) if err != nil { return true, err } @@ -253,7 +259,7 @@ func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch return true, table.NewBadRow(newRow, "error resolving conflicts", fmt.Sprintf("row with primary key %v in table %s does not match constraints or types of the table's schema.", key, tblName)) } - // resolve by inserting new row + // row was added if types.IsNull(cnf.Value) { err = tblEditor.InsertRow(ctx, newRow, nil) if err != nil { @@ -262,7 +268,7 @@ func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch return false, nil } - // resolve by updating existing row + // row was modified oldRow, err := row.FromNoms(sch, k, cnf.Value.(types.Tuple)) if err != nil { return true, err @@ -279,8 +285,7 @@ func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch return tblEditor.Table(ctx) } -func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor, ours bool) (*doltdb.Table, error) { - // Iterate over conflicts, resolve by picking either ours or theirs +func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor) (*doltdb.Table, error) { cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) if err != nil { return nil, err @@ -316,14 +321,8 @@ func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string panic("shouldn't be possible") } - rowDelta := 0 - var newRow row.Row - if ours { - newRow = cnfMap["our"] - } else { - newRow = cnfMap["their"] - rowDelta = int(theirCardinality - ourCardinality) - } + rowDelta := int(theirCardinality - ourCardinality) + newRow := cnfMap["their"] if rowDelta > 0 { for i := 0; i < rowDelta; i++ { @@ -340,7 +339,7 @@ func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string return tblEditor.Table(ctx) } -func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema, ours bool) (*doltdb.Table, error) { +func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { // Create new table editor tmpDir, err := dEnv.TempTableFilesDir() if err != nil { @@ -352,13 +351,26 @@ func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table return nil, err } + // TODO: just clear conflicts when resolving with ours; non-conflicting rows are already pulled into ours if schema.IsKeyless(sch) { - return resolveKeylessConflicts(ctx, tbl, tblName, sch, tblEditor, ours) + return resolveKeylessConflicts(ctx, tbl, tblName, sch, tblEditor) } else { - return resolvePkConflicts(ctx, tbl, tblName, sch, tblEditor, ours) + return resolvePkConflicts(ctx, tbl, tblName, sch, tblEditor) } } +func clearTableAndUpdateRoot(ctx *sql.Context, root *doltdb.RootValue, tbl *doltdb.Table, tblName string) (*doltdb.RootValue, error) { + newTbl, err := tbl.ClearConflicts(ctx) + if err != nil { + return nil, err + } + newRoot, err := root.PutTable(ctx, tblName, newTbl) + if err != nil { + return nil, err + } + return newRoot, nil +} + func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { newRoot := root for _, tblName := range tblNames { @@ -391,22 +403,21 @@ func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSess return ErrConfSchIncompatible } + if ours { + newRoot, err = clearTableAndUpdateRoot(ctx, newRoot, tbl, tblName) + if err != nil { + return err + } + } + var newTbl *doltdb.Table if tbl.Format() == types.Format_DOLT { - newTbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch, ours) + newTbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch) } else { - newTbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch, ours) + newTbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch) } - newTbl, err = newTbl.ClearConflicts(ctx) - if err != nil { - return err - } - - newRoot, err = newRoot.PutTable(ctx, tblName, newTbl) - if err != nil { - return err - } + newRoot, err = clearTableAndUpdateRoot(ctx, newRoot, newTbl, tblName) } return dSess.SetRoot(ctx, dbName, newRoot) } From b683820fc6347f955a080da54b69b7a1a05ca324 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 27 Sep 2022 23:31:13 -0700 Subject: [PATCH 39/58] it now do the thing --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 118 ++++++++---------- 1 file changed, 55 insertions(+), 63 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 7be9cd9d0d..0f37408f62 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -15,7 +15,6 @@ package dfunctions import ( - "bytes" "errors" "fmt" "io" @@ -95,11 +94,13 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return nil, err } + // get mutable prolly map ourIdx, err := tbl.GetRowData(ctx) if err != nil { return nil, err } ourMap := durable.ProllyMapFromIndex(ourIdx) + mutMap := ourMap.Mutate() // accumulate index edits once := true @@ -122,89 +123,79 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, once = false } - var theirRow val.Tuple - theirMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { + var ourRow, theirRow val.Tuple + err = ourMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { + ourRow = v + return nil + }) + if err != nil { + return nil, err + } + err = theirMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { theirRow = v return nil }) - - // todo: update actual table row - // todo: accumulate index edits - - } - - baseRootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.Metadata.BaseRootIsh) - baseTbl, ok, err := baseRootVal.GetTable(ctx, tblName) - if err != nil { - return nil, err - } - if !ok { - return nil, doltdb.ErrTableNotFound - } - baseIdx, err := baseTbl.GetRowData(ctx) - if err != nil { - return nil, err - } - baseMap := durable.ProllyMapFromIndex(baseIdx) - - // resolve conflicts with left - - merged, _, err := prolly.MergeMaps(ctx, ourMap, theirMap, baseMap, func(left, right tree.Diff) (tree.Diff, bool) { - if left.From != nil && ((left.To == nil) != (right.To == nil)) { - indexEdits = append(indexEdits, left) - return left, true + if err != nil { + return nil, err } - if bytes.Compare(left.To, right.To) == 0 { - indexEdits = append(indexEdits, right) - return right, true + + edit := tree.Diff{ + Key: tree.Item(cnfArt.Key), + From: tree.Item(ourRow), + To: tree.Item(theirRow), + } + indexEdits = append(indexEdits, edit) + + if theirRow == nil { + err = mutMap.Delete(ctx, cnfArt.Key) } else { - indexEdits = append(indexEdits, left) - return left, true + err = mutMap.Put(ctx, cnfArt.Key, theirRow) } - }) + if err != nil { + return nil, err + } + } + + // Update table + newMap, err := mutMap.Map(ctx) + if err != nil { + return nil, err + } + newIdx := durable.IndexFromProllyMap(newMap) + newTbl, err := tbl.UpdateRows(ctx, newIdx) if err != nil { return nil, err } - idx := durable.IndexFromProllyMap(merged) - newTbl, err := tbl.UpdateRows(ctx, idx) + // get mutable secondary indexes + idxSet, err := tbl.GetIndexSet(ctx) + if err != nil { + return nil, err + } + mutIdxs, err := merge.GetMutableSecondaryIdxs(ctx, sch, idxSet) if err != nil { return nil, err } - // TODO: need to either build from base or delete all the ones that aren't supposed to be there - idxSet, err := baseTbl.GetIndexSet(ctx) - if err != nil { - return nil, err - } - - secondaryIdxs, err := merge.GetMutableSecondaryIdxs(ctx, sch, idxSet) - if err != nil { - return nil, err - } - for _, edit := range indexEdits { - for _, secIdx := range secondaryIdxs { + // TODO: could do this inside the loop while resolving row data + // Update secondary indexes + for _, mutIdx := range mutIdxs { + for _, edit := range indexEdits { if len(edit.From) == 0 { - err := secIdx.InsertEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.To)) - if err != nil { - return nil, err - } + err = mutIdx.InsertEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.To)) } else if len(edit.To) == 0 { - err := secIdx.DeleteEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From)) - if err != nil { - return nil, err - } + err = mutIdx.DeleteEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From)) } else { - err := secIdx.UpdateEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From), val.Tuple(edit.To)) - if err != nil { - return nil, err - } + err = mutIdx.UpdateEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From), val.Tuple(edit.To)) } - m, err := secIdx.Map(ctx) if err != nil { return nil, err } - idxSet, err = idxSet.PutIndex(ctx, secIdx.Name, durable.IndexFromProllyMap(m)) + m, err := mutIdx.Map(ctx) + if err != nil { + return nil, err + } + idxSet, err = idxSet.PutIndex(ctx, mutIdx.Name, durable.IndexFromProllyMap(m)) if err != nil { return nil, err } @@ -408,6 +399,7 @@ func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSess if err != nil { return err } + continue } var newTbl *doltdb.Table From d0546ffcdfa83cf7968fae30342a500b8819dced Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 11:23:28 -0700 Subject: [PATCH 40/58] feedbacK --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 81 +++++++++---------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 0f37408f62..e971f9a8dd 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -102,10 +102,18 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, ourMap := durable.ProllyMapFromIndex(ourIdx) mutMap := ourMap.Mutate() - // accumulate index edits - once := true + // get mutable secondary indexes + idxSet, err := tbl.GetIndexSet(ctx) + if err != nil { + return nil, err + } + mutIdxs, err := merge.GetMutableSecondaryIdxs(ctx, sch, idxSet) + if err != nil { + return nil, err + } + + var theirRoot hash.Hash var theirMap prolly.Map - var indexEdits []tree.Diff for { cnfArt, err := iter.Next(ctx) if err == io.EOF { @@ -115,14 +123,16 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return nil, err } - if once { + // reload if their root hash changes + if theirRoot != cnfArt.TheirRootIsh { theirMap, err = getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) if err != nil { return nil, err } - once = false + theirRoot = cnfArt.TheirRootIsh } + // get row data var ourRow, theirRow val.Tuple err = ourMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { ourRow = v @@ -139,13 +149,7 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return nil, err } - edit := tree.Diff{ - Key: tree.Item(cnfArt.Key), - From: tree.Item(ourRow), - To: tree.Item(theirRow), - } - indexEdits = append(indexEdits, edit) - + // update row data if theirRow == nil { err = mutMap.Delete(ctx, cnfArt.Key) } else { @@ -154,6 +158,20 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, if err != nil { return nil, err } + + // update secondary indexes + for _, mutIdx := range mutIdxs { + if len(ourRow) == 0 { + err = mutIdx.InsertEntry(ctx, cnfArt.Key, theirRow) + } else if len(theirRow) == 0 { + err = mutIdx.DeleteEntry(ctx, cnfArt.Key, ourRow) + } else { + err = mutIdx.UpdateEntry(ctx, cnfArt.Key, ourRow, theirRow) + } + if err != nil { + return nil, err + } + } } // Update table @@ -167,38 +185,15 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return nil, err } - // get mutable secondary indexes - idxSet, err := tbl.GetIndexSet(ctx) - if err != nil { - return nil, err - } - mutIdxs, err := merge.GetMutableSecondaryIdxs(ctx, sch, idxSet) - if err != nil { - return nil, err - } - - // TODO: could do this inside the loop while resolving row data - // Update secondary indexes + // Apply index set changes for _, mutIdx := range mutIdxs { - for _, edit := range indexEdits { - if len(edit.From) == 0 { - err = mutIdx.InsertEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.To)) - } else if len(edit.To) == 0 { - err = mutIdx.DeleteEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From)) - } else { - err = mutIdx.UpdateEntry(ctx, val.Tuple(edit.Key), val.Tuple(edit.From), val.Tuple(edit.To)) - } - if err != nil { - return nil, err - } - m, err := mutIdx.Map(ctx) - if err != nil { - return nil, err - } - idxSet, err = idxSet.PutIndex(ctx, mutIdx.Name, durable.IndexFromProllyMap(m)) - if err != nil { - return nil, err - } + m, err := mutIdx.Map(ctx) + if err != nil { + return nil, err + } + idxSet, err = idxSet.PutIndex(ctx, mutIdx.Name, durable.IndexFromProllyMap(m)) + if err != nil { + return nil, err } } From da8246b9420e82783619835175ba9107eb6fefb5 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 11:35:09 -0700 Subject: [PATCH 41/58] copy pasta and refacta --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 95 ++++++++----------- 1 file changed, 38 insertions(+), 57 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index e971f9a8dd..be02d3e1bd 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -196,7 +196,6 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return nil, err } } - newTbl, err = newTbl.SetIndexSet(ctx, idxSet) if err != nil { return nil, err @@ -205,15 +204,18 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return newTbl, nil } -func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor) (*doltdb.Table, error) { - // Get conflicts - _, cnfIdx, err := tbl.GetConflicts(ctx) +func resolvePkConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema, conflicts types.Map) (*doltdb.Table, error) { + // Create table editor + tmpDir, err := dEnv.TempTableFilesDir() + if err != nil { + return nil, err + } + opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} + tblEditor, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) if err != nil { return nil, err } - // Iterate over conflicts, resolve with theirs - conflicts := durable.NomsMapFromConflictIndex(cnfIdx) err = conflicts.Iter(ctx, func(key, val types.Value) (stop bool, err error) { k := key.(types.Tuple) cnf, err := conflict.ConflictFromTuple(val.(types.Tuple)) @@ -271,78 +273,57 @@ func resolvePkConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch return tblEditor.Table(ctx) } -func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema, tblEditor editor.TableEditor) (*doltdb.Table, error) { - cnfReader, err := merge.NewConflictReader(ctx, tbl, tblName) +func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, conflicts types.Map) (*doltdb.Table, error) { + rowData, err := tbl.GetNomsRowData(ctx) if err != nil { return nil, err } - joiner := cnfReader.GetJoiner() - // get relevant cardinality tags - cnfSch := cnfReader.GetSchema() - tags := cnfSch.GetAllCols().SortedTags - ourCardinalityTag := tags[len(tags)-2] - theirCardinalityTag := tags[len(tags)-1] - - for { - cnfRow, err := cnfReader.NextConflict(ctx) - if err == io.EOF { - break - } - cnfMap, err := joiner.Split(cnfRow) + mapEditor := rowData.Edit() + err = conflicts.Iter(ctx, func(key, value types.Value) (stop bool, err error) { + cnf, err := conflict.ConflictFromTuple(value.(types.Tuple)) if err != nil { - return nil, err + return false, err } - // Get cardinality - var ourCardinality, theirCardinality uint64 - if ourCardinalityVal, ok := cnfRow.GetColVal(ourCardinalityTag); ok { - ourCardinality = uint64(ourCardinalityVal.(types.Uint)) - } else { - panic("shouldn't be possible") - } - if theirCardinalityVal, ok := cnfRow.GetColVal(theirCardinalityTag); ok { - theirCardinality = uint64(theirCardinalityVal.(types.Uint)) - } else { - panic("shouldn't be possible") + resolved := cnf.MergeValue + if err != nil { + return false, err } - rowDelta := int(theirCardinality - ourCardinality) - newRow := cnfMap["their"] - - if rowDelta > 0 { - for i := 0; i < rowDelta; i++ { - tblEditor.InsertRow(ctx, newRow, nil) - } + if types.IsNull(resolved) { + mapEditor.Remove(key) } else { - rowDelta *= -1 - for i := 0; i < rowDelta; i++ { - tblEditor.DeleteRow(ctx, cnfMap["our"]) - } + mapEditor.Set(key, resolved) } + + return false, nil + }) + if err != nil { + return nil, err } - return tblEditor.Table(ctx) + rowData, err = mapEditor.Map(ctx) + if err != nil { + return nil, err + } + + return tbl.UpdateNomsRows(ctx, rowData) } func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { - // Create new table editor - tmpDir, err := dEnv.TempTableFilesDir() + // Get conflicts + _, confIdx, err := tbl.GetConflicts(ctx) if err != nil { return nil, err } - opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} - tblEditor, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) - if err != nil { - return nil, err + conflicts := durable.NomsMapFromConflictIndex(confIdx) + + if schema.IsKeyless(sch) { + return resolveKeylessConflicts(ctx, tbl, conflicts) } - // TODO: just clear conflicts when resolving with ours; non-conflicting rows are already pulled into ours - if schema.IsKeyless(sch) { - return resolveKeylessConflicts(ctx, tbl, tblName, sch, tblEditor) - } else { - return resolvePkConflicts(ctx, tbl, tblName, sch, tblEditor) - } + return resolvePkConflicts(ctx, dEnv, tbl, tblName, sch, conflicts) } func clearTableAndUpdateRoot(ctx *sql.Context, root *doltdb.RootValue, tbl *doltdb.Table, tblName string) (*doltdb.RootValue, error) { From fea7fa876c387af7b6ac196ff1794ccb613aff44 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 11:39:14 -0700 Subject: [PATCH 42/58] more refactoring --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index be02d3e1bd..ee4a0e3d33 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -283,18 +283,13 @@ func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, conflicts type err = conflicts.Iter(ctx, func(key, value types.Value) (stop bool, err error) { cnf, err := conflict.ConflictFromTuple(value.(types.Tuple)) if err != nil { - return false, err + return true, err } - resolved := cnf.MergeValue - if err != nil { - return false, err - } - - if types.IsNull(resolved) { + if types.IsNull(cnf.MergeValue) { mapEditor.Remove(key) } else { - mapEditor.Set(key, resolved) + mapEditor.Set(key, cnf.MergeValue) } return false, nil From dc18ebb1f50db33efb58b4ea8a1c390c673e3458 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 13:12:09 -0700 Subject: [PATCH 43/58] adding foreign key checks --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 50 ++++++++++++++----- integration-tests/bats/foreign-keys.bats | 6 +-- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index ee4a0e3d33..7c25dfada7 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,6 +17,7 @@ package dfunctions import ( "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/utils/set" "io" "strings" @@ -321,6 +322,24 @@ func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table return resolvePkConflicts(ctx, dEnv, tbl, tblName, sch, conflicts) } +func validateConstraintViolations(ctx *sql.Context, before, after *doltdb.RootValue, table string) error { + tables, err := after.GetTableNames(ctx) + if err != nil { + return err + } + + // todo: this is an expensive way to compute this + _, violators, err := merge.AddForeignKeyViolations(ctx, after, before, set.NewStrSet(tables), hash.Of(nil)) + if err != nil { + return err + } + if violators.Size() > 0 { + return fmt.Errorf("resolving conflicts for table %s created foreign key violations", table) + } + + return nil +} + func clearTableAndUpdateRoot(ctx *sql.Context, root *doltdb.RootValue, tbl *doltdb.Table, tblName string) (*doltdb.RootValue, error) { newTbl, err := tbl.ClearConflicts(ctx) if err != nil { @@ -334,9 +353,8 @@ func clearTableAndUpdateRoot(ctx *sql.Context, root *doltdb.RootValue, tbl *dolt } func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { - newRoot := root for _, tblName := range tblNames { - tbl, ok, err := newRoot.GetTable(ctx, tblName) + tbl, ok, err := root.GetTable(ctx, tblName) if err != nil { return err } @@ -365,24 +383,30 @@ func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSess return ErrConfSchIncompatible } - if ours { - newRoot, err = clearTableAndUpdateRoot(ctx, newRoot, tbl, tblName) + if !ours { + if tbl.Format() == types.Format_DOLT { + tbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch) + } else { + tbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch) + } if err != nil { return err } - continue } - var newTbl *doltdb.Table - if tbl.Format() == types.Format_DOLT { - newTbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch) - } else { - newTbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch) + newRoot, err := clearTableAndUpdateRoot(ctx, root, tbl, tblName) + if err != nil { + return err } - newRoot, err = clearTableAndUpdateRoot(ctx, newRoot, newTbl, tblName) + err = validateConstraintViolations(ctx, root, newRoot, tblName) + if err != nil { + return err + } + + root = newRoot } - return dSess.SetRoot(ctx, dbName, newRoot) + return dSess.SetRoot(ctx, dbName, root) } func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { @@ -423,7 +447,7 @@ func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { } } - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "does this matter?") + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "") err = ResolveConflicts(ctx, dEnv, dSess, root, dbName, ours, tbls) if err != nil { return 1, err diff --git a/integration-tests/bats/foreign-keys.bats b/integration-tests/bats/foreign-keys.bats index ba82a14c25..0fdb16163f 100644 --- a/integration-tests/bats/foreign-keys.bats +++ b/integration-tests/bats/foreign-keys.bats @@ -1314,13 +1314,13 @@ SQL dolt commit -m "added 2s" dolt checkout main dolt merge other -m "merge other" - # TODO: it does error and prevent the merge, but not in the same way + dolt sql -q "set @@dolt_allow_commit_conflicts = 1" run dolt sql -q "call dolt_conflicts_resolve('--theirs', 'parent')" [ "$status" -eq 1 ] - [[ "$output" =~ "conflict detected" ]] || false + [[ "$output" =~ "violation" ]] || false run dolt sql -q "call dolt_conflicts_resolve('--theirs', 'child')" [ "$status" -eq 1 ] - [[ "$output" =~ "conflict detected" ]] || false + [[ "$output" =~ "violation" ]] || false } @test "foreign-keys: FKs move with the working set on checkout" { From b3af84be09a186e7c2c13898cccb16a2c07b5a9f Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Wed, 28 Sep 2022 20:13:44 +0000 Subject: [PATCH 44/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index 7c25dfada7..ff1814f5d5 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -17,7 +17,6 @@ package dfunctions import ( "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/utils/set" "io" "strings" @@ -35,6 +34,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/table" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/libraries/utils/set" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" From c555f0b16c95eac040647e5062cbdc6a7ce3b16c Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 13:23:41 -0700 Subject: [PATCH 45/58] one last test, probably --- integration-tests/bats/merge.bats | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/integration-tests/bats/merge.bats b/integration-tests/bats/merge.bats index 08fc2038f0..07cf79325a 100644 --- a/integration-tests/bats/merge.bats +++ b/integration-tests/bats/merge.bats @@ -379,7 +379,6 @@ SQL run dolt merge other --no-commit log_status_eq 0 [[ "$output" =~ "CONFLICT" ]] || false - dolt conflicts resolve --theirs dolt_schemas run dolt conflicts resolve --theirs dolt_schemas log_status_eq 0 run dolt sql -q "select name from dolt_schemas" -r csv @@ -387,6 +386,26 @@ SQL [[ "$output" =~ "c1c1" ]] || false } +@test "merge: Add views on two branches, merge with stored procedure" { + dolt branch other + dolt sql -q "CREATE VIEW pkpk AS SELECT pk*pk FROM test1;" + dolt add . && dolt commit -m "added view on table test1" + + dolt checkout other + dolt sql -q "CREATE VIEW c1c1 AS SELECT c1*c1 FROM test2;" + dolt add . && dolt commit -m "added view on table test2" + + dolt checkout main + run dolt merge other --no-commit + log_status_eq 0 + [[ "$output" =~ "CONFLICT" ]] || false + run dolt sql -q "call dolt_conflicts_resolve('--theirs', 'dolt_schemas')" + log_status_eq 0 + run dolt sql -q "select name from dolt_schemas" -r csv + log_status_eq 0 + [[ "$output" =~ "c1c1" ]] || false +} + @test "merge: Add views on two branches, merge without conflicts" { dolt branch other dolt sql -q "CREATE VIEW pkpk AS SELECT pk*pk FROM test1;" From 16a2ae75c2fbcf80c5805ac8dae598f4ee305b68 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 13:40:00 -0700 Subject: [PATCH 46/58] row data --- go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go index ff1814f5d5..0dea4b326a 100644 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go @@ -151,7 +151,7 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, } // update row data - if theirRow == nil { + if len(theirRow) == 0 { err = mutMap.Delete(ctx, cnfArt.Key) } else { err = mutMap.Put(ctx, cnfArt.Key, theirRow) From 149720fcaa59a55de41eb5c86a3a5c8f7988a246 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 13:50:34 -0700 Subject: [PATCH 47/58] test fix --- integration-tests/bats/foreign-keys.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/foreign-keys.bats b/integration-tests/bats/foreign-keys.bats index 0fdb16163f..134e2db5ef 100644 --- a/integration-tests/bats/foreign-keys.bats +++ b/integration-tests/bats/foreign-keys.bats @@ -1290,7 +1290,7 @@ SQL [[ "$output" =~ "violation" ]] || false } -@test "foreign-keys: Resolve with stored catches violations" { +@test "foreign-keys: Resolve catches violations with stored procedure" { dolt sql < Date: Wed, 28 Sep 2022 14:04:24 -0700 Subject: [PATCH 48/58] improve error message when current branch on session is force deleted (#4395) --- go/go.mod | 2 +- go/go.sum | 4 +- go/libraries/doltcore/sqle/dsess/session.go | 31 ++++++ .../sqle/enginetest/dolt_server_test.go | 98 +++++++++++++++++++ 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index 528917d097..b79c9ada82 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible - github.com/dolthub/go-mysql-server v0.12.1-0.20220927222348-8b178aa50764 + github.com/dolthub/go-mysql-server v0.12.1-0.20220928174432-cd8b451c9bf4 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 9ba3e62341..3f02e10fb4 100644 --- a/go/go.sum +++ b/go/go.sum @@ -177,8 +177,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20220927222348-8b178aa50764 h1:7cJCBFyyHl1pGrNgME77mUkkzlSZ94jo3vZOQb/xan4= -github.com/dolthub/go-mysql-server v0.12.1-0.20220927222348-8b178aa50764/go.mod h1:8zHF9V5MPmb3dWB2kGjeltnHQc15QdynK9GnwSutreA= +github.com/dolthub/go-mysql-server v0.12.1-0.20220928174432-cd8b451c9bf4 h1:27KKhBenaRJy++ZiVeEKcXQv0yJawAc+oPozDGqEMuc= +github.com/dolthub/go-mysql-server v0.12.1-0.20220928174432-cd8b451c9bf4/go.mod h1:8zHF9V5MPmb3dWB2kGjeltnHQc15QdynK9GnwSutreA= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= diff --git a/go/libraries/doltcore/sqle/dsess/session.go b/go/libraries/doltcore/sqle/dsess/session.go index 23d519d53f..6caf876373 100644 --- a/go/libraries/doltcore/sqle/dsess/session.go +++ b/go/libraries/doltcore/sqle/dsess/session.go @@ -47,6 +47,7 @@ const ( var ErrWorkingSetChanges = goerrors.NewKind("Cannot switch working set, session state is dirty. " + "Rollback or commit changes before changing working sets.") var ErrSessionNotPeristable = errors.New("session is not persistable") +var ErrCurrentBranchDeleted = errors.New("current branch has been force deleted. run 'USE /' to checkout a different branch, or reconnect to the server") // DoltSession is the sql.Session implementation used by dolt. It is accessible through a *sql.Context instance type DoltSession struct { @@ -177,6 +178,36 @@ func (d *DoltSession) Flush(ctx *sql.Context, dbName string) error { return d.SetRoot(ctx, dbName, ws.WorkingRoot()) } +// ValidateSession validates a working set if there are a valid sessionState with non-nil working set. +// If there is no sessionState or its current working set not defined, then no need for validation, +// so no error is returned. +func (d *DoltSession) ValidateSession(ctx *sql.Context, dbName string) error { + sessionState, ok, err := d.LookupDbState(ctx, dbName) + if !ok { + return nil + } + if err != nil { + return err + } + if sessionState.WorkingSet == nil { + return nil + } + wsRef := sessionState.WorkingSet.Ref() + _, err = sessionState.dbData.Ddb.ResolveWorkingSet(ctx, wsRef) + if err == doltdb.ErrWorkingSetNotFound { + _, err = d.newWorkingSetForHead(ctx, wsRef, dbName) + // if the current head is not found, the branch was force deleted, so use nil working set. + if errors.Is(err, doltdb.ErrBranchNotFound) { + return ErrCurrentBranchDeleted + } else if err != nil { + return err + } + } else if err != nil { + return err + } + return nil +} + // StartTransaction refreshes the state of this session and starts a new transaction. func (d *DoltSession) StartTransaction(ctx *sql.Context, dbName string, tCharacteristic sql.TransactionCharacteristic) (sql.Transaction, error) { if TransactionsDisabled(ctx) { diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go index 6880297870..eeb2d38c0c 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go @@ -201,6 +201,104 @@ var DoltBranchMultiSessionScriptTests = []queries.ScriptTest{ }, }, }, + { + Name: "Test multi-session behavior for force deleting active branch with autocommit on", + Assertions: []queries.ScriptTestAssertion{ + { + Query: "/* client a */ SET @@autocommit=1;", + Expected: []sql.Row{}, + }, + { + Query: "/* client a */ CALL DOLT_CHECKOUT('-b', 'branch1');", + Expected: []sql.Row{{0}}, + }, + { + Query: "/* client a */ select active_branch();", + Expected: []sql.Row{{"branch1"}}, + }, + { + Query: "/* client b */ select active_branch();", + Expected: []sql.Row{{"main"}}, + }, + { + Query: "/* client b */ select name from dolt_branches order by name;", + Expected: []sql.Row{{"branch1"}, {"main"}}, + }, + { + Query: "/* client b */ CALL DOLT_BRANCH('-D', 'branch1');", + Expected: []sql.Row{{0}}, + }, + { + Query: "/* client b */ select name from dolt_branches;", + Expected: []sql.Row{{"main"}}, + }, + { + Query: "/* client a */ select name from dolt_branches;", + ExpectedErrStr: "Error 1105: current branch has been force deleted. run 'USE /' to checkout a different branch, or reconnect to the server", + }, + { + Query: "/* client a */ CALL DOLT_CHECKOUT('main');", + ExpectedErrStr: "Error 1105: current branch has been force deleted. run 'USE /' to checkout a different branch, or reconnect to the server", + }, + { + Query: "/* client a */ USE dolt/main;", + Expected: []sql.Row{}, + }, + { + Query: "/* client a */ select active_branch();", + Expected: []sql.Row{{"main"}}, + }, + }, + }, + { + Name: "Test multi-session behavior for force deleting active branch with autocommit off", + Assertions: []queries.ScriptTestAssertion{ + { + Query: "/* client a */ SET @@autocommit=0;", + Expected: []sql.Row{}, + }, + { + Query: "/* client a */ CALL DOLT_CHECKOUT('-b', 'branch1');", + Expected: []sql.Row{{0}}, + }, + { + Query: "/* client a */ select active_branch();", + Expected: []sql.Row{{"branch1"}}, + }, + { + Query: "/* client b */ select active_branch();", + Expected: []sql.Row{{"main"}}, + }, + { + Query: "/* client b */ select name from dolt_branches order by name;", + Expected: []sql.Row{{"branch1"}, {"main"}}, + }, + { + Query: "/* client b */ CALL DOLT_BRANCH('-D', 'branch1');", + Expected: []sql.Row{{0}}, + }, + { + Query: "/* client b */ select name from dolt_branches;", + Expected: []sql.Row{{"main"}}, + }, + { + Query: "/* client a */ select name from dolt_branches;", + ExpectedErrStr: "Error 1105: current branch has been force deleted. run 'USE /' to checkout a different branch, or reconnect to the server", + }, + { + Query: "/* client a */ CALL DOLT_CHECKOUT('main');", + ExpectedErrStr: "Error 1105: current branch has been force deleted. run 'USE /' to checkout a different branch, or reconnect to the server", + }, + { + Query: "/* client a */ USE dolt/main;", + Expected: []sql.Row{}, + }, + { + Query: "/* client a */ select active_branch();", + Expected: []sql.Row{{"main"}}, + }, + }, + }, } // TestDoltMultiSessionBehavior runs tests that exercise multi-session logic on a running SQL server. Statements From 786e892f1ce87e16e941ee94ff6dda0575beb8ae Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 14:11:45 -0700 Subject: [PATCH 49/58] fix tests again --- integration-tests/bats/foreign-keys.bats | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/integration-tests/bats/foreign-keys.bats b/integration-tests/bats/foreign-keys.bats index 134e2db5ef..ce8d05f328 100644 --- a/integration-tests/bats/foreign-keys.bats +++ b/integration-tests/bats/foreign-keys.bats @@ -1314,11 +1314,16 @@ SQL dolt commit -m "added 2s" dolt checkout main dolt merge other -m "merge other" - dolt sql -q "set @@dolt_allow_commit_conflicts = 1" - run dolt sql -q "call dolt_conflicts_resolve('--theirs', 'parent')" + run dolt sql < Date: Wed, 28 Sep 2022 15:25:39 -0700 Subject: [PATCH 50/58] only making procedure version --- .../sqle/dfunctions/dolt_conflicts_resolve.go | 491 ------------------ .../dprocedures/dolt_conflicts_resolve.go | 428 ++++++++++++++- 2 files changed, 424 insertions(+), 495 deletions(-) delete mode 100644 go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go diff --git a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go deleted file mode 100644 index 0dea4b326a..0000000000 --- a/go/libraries/doltcore/sqle/dfunctions/dolt_conflicts_resolve.go +++ /dev/null @@ -1,491 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dfunctions - -import ( - "errors" - "fmt" - "io" - "strings" - - "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/cmd/dolt/cli" - "github.com/dolthub/dolt/go/libraries/doltcore/conflict" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" - "github.com/dolthub/dolt/go/libraries/doltcore/env" - "github.com/dolthub/dolt/go/libraries/doltcore/merge" - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" - "github.com/dolthub/dolt/go/libraries/doltcore/table" - "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" - "github.com/dolthub/dolt/go/libraries/utils/filesys" - "github.com/dolthub/dolt/go/libraries/utils/set" - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/prolly" - "github.com/dolthub/dolt/go/store/prolly/tree" - "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/dolt/go/store/val" -) - -var ErrConfSchIncompatible = errors.New("the conflict schema's columns are not equal to the current schema's columns, please resolve manually") - -// DoltConflictsCatFunc runs a `dolt commit` in the SQL context, committing staged changes to head. -// Deprecated: please use the version in the dprocedures package -type DoltConflictsCatFunc struct { - children []sql.Expression -} - -// NewDoltConflictsResolveFunc creates a new DoltCommitFunc expression whose children represents the args passed in DOLT_CONFLICTS_RESOLVE. -// Deprecated: please use the version in the dprocedures package -func NewDoltConflictsResolveFunc(args ...sql.Expression) (sql.Expression, error) { - return &DoltConflictsCatFunc{children: args}, nil -} - -func (d DoltConflictsCatFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) { - args, err := getDoltArgs(ctx, row, d.Children()) - if err != nil { - return 1, err - } - return DoDoltConflictsResolve(ctx, args) -} - -func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeStore, hash hash.Hash, tblName string) (prolly.Map, error) { - rootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, vrw, ns, hash) - tbl, ok, err := rootVal.GetTable(ctx, tblName) - if err != nil { - return prolly.Map{}, err - } - if !ok { - return prolly.Map{}, doltdb.ErrTableNotFound - } - - idx, err := tbl.GetRowData(ctx) - if err != nil { - return prolly.Map{}, err - } - - return durable.ProllyMapFromIndex(idx), nil -} - -func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { - var err error - artifactIdx, err := tbl.GetArtifacts(ctx) - if err != nil { - return nil, err - } - - artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) - iter, err := artifactMap.IterAllConflicts(ctx) - if err != nil { - return nil, err - } - - // get mutable prolly map - ourIdx, err := tbl.GetRowData(ctx) - if err != nil { - return nil, err - } - ourMap := durable.ProllyMapFromIndex(ourIdx) - mutMap := ourMap.Mutate() - - // get mutable secondary indexes - idxSet, err := tbl.GetIndexSet(ctx) - if err != nil { - return nil, err - } - mutIdxs, err := merge.GetMutableSecondaryIdxs(ctx, sch, idxSet) - if err != nil { - return nil, err - } - - var theirRoot hash.Hash - var theirMap prolly.Map - for { - cnfArt, err := iter.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - // reload if their root hash changes - if theirRoot != cnfArt.TheirRootIsh { - theirMap, err = getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) - if err != nil { - return nil, err - } - theirRoot = cnfArt.TheirRootIsh - } - - // get row data - var ourRow, theirRow val.Tuple - err = ourMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { - ourRow = v - return nil - }) - if err != nil { - return nil, err - } - err = theirMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { - theirRow = v - return nil - }) - if err != nil { - return nil, err - } - - // update row data - if len(theirRow) == 0 { - err = mutMap.Delete(ctx, cnfArt.Key) - } else { - err = mutMap.Put(ctx, cnfArt.Key, theirRow) - } - if err != nil { - return nil, err - } - - // update secondary indexes - for _, mutIdx := range mutIdxs { - if len(ourRow) == 0 { - err = mutIdx.InsertEntry(ctx, cnfArt.Key, theirRow) - } else if len(theirRow) == 0 { - err = mutIdx.DeleteEntry(ctx, cnfArt.Key, ourRow) - } else { - err = mutIdx.UpdateEntry(ctx, cnfArt.Key, ourRow, theirRow) - } - if err != nil { - return nil, err - } - } - } - - // Update table - newMap, err := mutMap.Map(ctx) - if err != nil { - return nil, err - } - newIdx := durable.IndexFromProllyMap(newMap) - newTbl, err := tbl.UpdateRows(ctx, newIdx) - if err != nil { - return nil, err - } - - // Apply index set changes - for _, mutIdx := range mutIdxs { - m, err := mutIdx.Map(ctx) - if err != nil { - return nil, err - } - idxSet, err = idxSet.PutIndex(ctx, mutIdx.Name, durable.IndexFromProllyMap(m)) - if err != nil { - return nil, err - } - } - newTbl, err = newTbl.SetIndexSet(ctx, idxSet) - if err != nil { - return nil, err - } - - return newTbl, nil -} - -func resolvePkConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema, conflicts types.Map) (*doltdb.Table, error) { - // Create table editor - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} - tblEditor, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) - if err != nil { - return nil, err - } - - err = conflicts.Iter(ctx, func(key, val types.Value) (stop bool, err error) { - k := key.(types.Tuple) - cnf, err := conflict.ConflictFromTuple(val.(types.Tuple)) - if err != nil { - return true, err - } - - // row was removed - if types.IsNull(cnf.MergeValue) { - baseRow, err := row.FromNoms(sch, k, cnf.Base.(types.Tuple)) - if err != nil { - return true, err - } - err = tblEditor.DeleteRow(ctx, baseRow) - if err != nil { - return true, err - } - return false, nil - } - - newRow, err := row.FromNoms(sch, k, cnf.MergeValue.(types.Tuple)) - if err != nil { - return true, err - } - - if isValid, err := row.IsValid(newRow, sch); err != nil { - return true, err - } else if !isValid { - return true, table.NewBadRow(newRow, "error resolving conflicts", fmt.Sprintf("row with primary key %v in table %s does not match constraints or types of the table's schema.", key, tblName)) - } - - // row was added - if types.IsNull(cnf.Value) { - err = tblEditor.InsertRow(ctx, newRow, nil) - if err != nil { - return true, err - } - return false, nil - } - - // row was modified - oldRow, err := row.FromNoms(sch, k, cnf.Value.(types.Tuple)) - if err != nil { - return true, err - } - err = tblEditor.UpdateRow(ctx, oldRow, newRow, nil) - if err != nil { - return true, err - } - return false, nil - }) - if err != nil { - return nil, err - } - return tblEditor.Table(ctx) -} - -func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, conflicts types.Map) (*doltdb.Table, error) { - rowData, err := tbl.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - mapEditor := rowData.Edit() - err = conflicts.Iter(ctx, func(key, value types.Value) (stop bool, err error) { - cnf, err := conflict.ConflictFromTuple(value.(types.Tuple)) - if err != nil { - return true, err - } - - if types.IsNull(cnf.MergeValue) { - mapEditor.Remove(key) - } else { - mapEditor.Set(key, cnf.MergeValue) - } - - return false, nil - }) - if err != nil { - return nil, err - } - - rowData, err = mapEditor.Map(ctx) - if err != nil { - return nil, err - } - - return tbl.UpdateNomsRows(ctx, rowData) -} - -func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { - // Get conflicts - _, confIdx, err := tbl.GetConflicts(ctx) - if err != nil { - return nil, err - } - conflicts := durable.NomsMapFromConflictIndex(confIdx) - - if schema.IsKeyless(sch) { - return resolveKeylessConflicts(ctx, tbl, conflicts) - } - - return resolvePkConflicts(ctx, dEnv, tbl, tblName, sch, conflicts) -} - -func validateConstraintViolations(ctx *sql.Context, before, after *doltdb.RootValue, table string) error { - tables, err := after.GetTableNames(ctx) - if err != nil { - return err - } - - // todo: this is an expensive way to compute this - _, violators, err := merge.AddForeignKeyViolations(ctx, after, before, set.NewStrSet(tables), hash.Of(nil)) - if err != nil { - return err - } - if violators.Size() > 0 { - return fmt.Errorf("resolving conflicts for table %s created foreign key violations", table) - } - - return nil -} - -func clearTableAndUpdateRoot(ctx *sql.Context, root *doltdb.RootValue, tbl *doltdb.Table, tblName string) (*doltdb.RootValue, error) { - newTbl, err := tbl.ClearConflicts(ctx) - if err != nil { - return nil, err - } - newRoot, err := root.PutTable(ctx, tblName, newTbl) - if err != nil { - return nil, err - } - return newRoot, nil -} - -func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { - for _, tblName := range tblNames { - tbl, ok, err := root.GetTable(ctx, tblName) - if err != nil { - return err - } - if !ok { - return doltdb.ErrTableNotFound - } - - if has, err := tbl.HasConflicts(ctx); err != nil { - return err - } else if !has { - return nil - } - - sch, err := tbl.GetSchema(ctx) - if err != nil { - return err - } - _, ourSch, theirSch, err := tbl.GetConflictSchemas(ctx, tblName) - if err != nil { - return err - } - - if ours && !schema.ColCollsAreEqual(sch.GetAllCols(), ourSch.GetAllCols()) { - return ErrConfSchIncompatible - } else if !ours && !schema.ColCollsAreEqual(sch.GetAllCols(), theirSch.GetAllCols()) { - return ErrConfSchIncompatible - } - - if !ours { - if tbl.Format() == types.Format_DOLT { - tbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch) - } else { - tbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch) - } - if err != nil { - return err - } - } - - newRoot, err := clearTableAndUpdateRoot(ctx, root, tbl, tblName) - if err != nil { - return err - } - - err = validateConstraintViolations(ctx, root, newRoot, tblName) - if err != nil { - return err - } - - root = newRoot - } - return dSess.SetRoot(ctx, dbName, root) -} - -func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { - dbName := ctx.GetCurrentDatabase() - fmt.Printf("database name: %s", dbName) - - apr, err := cli.CreateConflictsResolveArgParser().Parse(args) - if err != nil { - return 1, err - } - - dSess := dsess.DSessFromSess(ctx.Session) - roots, ok := dSess.GetRoots(ctx, dbName) - if !ok { - return 1, fmt.Errorf("Could not load database %s", dbName) - } - - ours := apr.Contains(cli.OursFlag) - theirs := apr.Contains(cli.TheirsFlag) - if ours && theirs { - return 1, fmt.Errorf("specify only either --ours or --theirs") - } else if !ours && !theirs { - return 1, fmt.Errorf("--ours or --theirs must be supplied") - } - - if apr.NArg() == 0 { - return 1, fmt.Errorf("specify at least one table to resolve conflicts") - } - - // get all tables in conflict - root := roots.Working - tbls := apr.Args - if len(tbls) == 1 && tbls[0] == "." { - if allTables, err := root.TablesInConflict(ctx); err != nil { - return 1, err - } else { - tbls = allTables - } - } - - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "") - err = ResolveConflicts(ctx, dEnv, dSess, root, dbName, ours, tbls) - if err != nil { - return 1, err - } - - return 0, nil -} - -func (d DoltConflictsCatFunc) String() string { - childrenStrings := make([]string, len(d.children)) - - for _, child := range d.children { - childrenStrings = append(childrenStrings, child.String()) - } - return fmt.Sprintf("DOLT_CONFLICTS_RESOLVE(%s)", strings.Join(childrenStrings, ",")) -} - -func (d DoltConflictsCatFunc) Type() sql.Type { - return sql.Text -} - -func (d DoltConflictsCatFunc) IsNullable() bool { - return false -} - -func (d DoltConflictsCatFunc) WithChildren(children ...sql.Expression) (sql.Expression, error) { - return NewDoltConflictsResolveFunc(children...) -} - -func (d DoltConflictsCatFunc) Resolved() bool { - for _, child := range d.Children() { - if !child.Resolved() { - return false - } - } - return true -} - -func (d DoltConflictsCatFunc) Children() []sql.Expression { - return d.children -} diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go index f0471359e2..308ab09493 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go @@ -15,16 +15,436 @@ package dprocedures import ( - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" - + "errors" + "fmt" "github.com/dolthub/go-mysql-server/sql" + "io" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/libraries/doltcore/conflict" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/merge" + "github.com/dolthub/dolt/go/libraries/doltcore/row" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" + "github.com/dolthub/dolt/go/libraries/doltcore/table" + "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" + "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/libraries/utils/set" + "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/prolly" + "github.com/dolthub/dolt/go/store/prolly/tree" + "github.com/dolthub/dolt/go/store/types" + "github.com/dolthub/dolt/go/store/val" ) -// doltConflictsResolve is the stored procedure version of the function `dolt conflicts cat`. +var ErrConfSchIncompatible = errors.New("the conflict schema's columns are not equal to the current schema's columns, please resolve manually") + +// doltConflictsResolve is the stored procedure version of the function `dolt conflict resolve`. func doltConflictsResolve(ctx *sql.Context, args ...string) (sql.RowIter, error) { - res, err := dfunctions.DoDoltConflictsResolve(ctx, args) + res, err := DoDoltConflictsResolve(ctx, args) if err != nil { return nil, err } return rowToIter(res), nil } + +// DoltConflictsCatFunc runs a `dolt commit` in the SQL context, committing staged changes to head. +// Deprecated: please use the version in the dprocedures package +type DoltConflictsCatFunc struct { + children []sql.Expression +} + +func getProllyRowMaps(ctx *sql.Context, vrw types.ValueReadWriter, ns tree.NodeStore, hash hash.Hash, tblName string) (prolly.Map, error) { + rootVal, err := doltdb.LoadRootValueFromRootIshAddr(ctx, vrw, ns, hash) + tbl, ok, err := rootVal.GetTable(ctx, tblName) + if err != nil { + return prolly.Map{}, err + } + if !ok { + return prolly.Map{}, doltdb.ErrTableNotFound + } + + idx, err := tbl.GetRowData(ctx) + if err != nil { + return prolly.Map{}, err + } + + return durable.ProllyMapFromIndex(idx), nil +} + +func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { + var err error + artifactIdx, err := tbl.GetArtifacts(ctx) + if err != nil { + return nil, err + } + + artifactMap := durable.ProllyMapFromArtifactIndex(artifactIdx) + iter, err := artifactMap.IterAllConflicts(ctx) + if err != nil { + return nil, err + } + + // get mutable prolly map + ourIdx, err := tbl.GetRowData(ctx) + if err != nil { + return nil, err + } + ourMap := durable.ProllyMapFromIndex(ourIdx) + mutMap := ourMap.Mutate() + + // get mutable secondary indexes + idxSet, err := tbl.GetIndexSet(ctx) + if err != nil { + return nil, err + } + mutIdxs, err := merge.GetMutableSecondaryIdxs(ctx, sch, idxSet) + if err != nil { + return nil, err + } + + var theirRoot hash.Hash + var theirMap prolly.Map + for { + cnfArt, err := iter.Next(ctx) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + // reload if their root hash changes + if theirRoot != cnfArt.TheirRootIsh { + theirMap, err = getProllyRowMaps(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), cnfArt.TheirRootIsh, tblName) + if err != nil { + return nil, err + } + theirRoot = cnfArt.TheirRootIsh + } + + // get row data + var ourRow, theirRow val.Tuple + err = ourMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { + ourRow = v + return nil + }) + if err != nil { + return nil, err + } + err = theirMap.Get(ctx, cnfArt.Key, func(_, v val.Tuple) error { + theirRow = v + return nil + }) + if err != nil { + return nil, err + } + + // update row data + if len(theirRow) == 0 { + err = mutMap.Delete(ctx, cnfArt.Key) + } else { + err = mutMap.Put(ctx, cnfArt.Key, theirRow) + } + if err != nil { + return nil, err + } + + // update secondary indexes + for _, mutIdx := range mutIdxs { + if len(ourRow) == 0 { + err = mutIdx.InsertEntry(ctx, cnfArt.Key, theirRow) + } else if len(theirRow) == 0 { + err = mutIdx.DeleteEntry(ctx, cnfArt.Key, ourRow) + } else { + err = mutIdx.UpdateEntry(ctx, cnfArt.Key, ourRow, theirRow) + } + if err != nil { + return nil, err + } + } + } + + // Update table + newMap, err := mutMap.Map(ctx) + if err != nil { + return nil, err + } + newIdx := durable.IndexFromProllyMap(newMap) + newTbl, err := tbl.UpdateRows(ctx, newIdx) + if err != nil { + return nil, err + } + + // Apply index set changes + for _, mutIdx := range mutIdxs { + m, err := mutIdx.Map(ctx) + if err != nil { + return nil, err + } + idxSet, err = idxSet.PutIndex(ctx, mutIdx.Name, durable.IndexFromProllyMap(m)) + if err != nil { + return nil, err + } + } + newTbl, err = newTbl.SetIndexSet(ctx, idxSet) + if err != nil { + return nil, err + } + + return newTbl, nil +} + +func resolvePkConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema, conflicts types.Map) (*doltdb.Table, error) { + // Create table editor + tmpDir, err := dEnv.TempTableFilesDir() + if err != nil { + return nil, err + } + opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} + tblEditor, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) + if err != nil { + return nil, err + } + + err = conflicts.Iter(ctx, func(key, val types.Value) (stop bool, err error) { + k := key.(types.Tuple) + cnf, err := conflict.ConflictFromTuple(val.(types.Tuple)) + if err != nil { + return true, err + } + + // row was removed + if types.IsNull(cnf.MergeValue) { + baseRow, err := row.FromNoms(sch, k, cnf.Base.(types.Tuple)) + if err != nil { + return true, err + } + err = tblEditor.DeleteRow(ctx, baseRow) + if err != nil { + return true, err + } + return false, nil + } + + newRow, err := row.FromNoms(sch, k, cnf.MergeValue.(types.Tuple)) + if err != nil { + return true, err + } + + if isValid, err := row.IsValid(newRow, sch); err != nil { + return true, err + } else if !isValid { + return true, table.NewBadRow(newRow, "error resolving conflicts", fmt.Sprintf("row with primary key %v in table %s does not match constraints or types of the table's schema.", key, tblName)) + } + + // row was added + if types.IsNull(cnf.Value) { + err = tblEditor.InsertRow(ctx, newRow, nil) + if err != nil { + return true, err + } + return false, nil + } + + // row was modified + oldRow, err := row.FromNoms(sch, k, cnf.Value.(types.Tuple)) + if err != nil { + return true, err + } + err = tblEditor.UpdateRow(ctx, oldRow, newRow, nil) + if err != nil { + return true, err + } + return false, nil + }) + if err != nil { + return nil, err + } + return tblEditor.Table(ctx) +} + +func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, conflicts types.Map) (*doltdb.Table, error) { + rowData, err := tbl.GetNomsRowData(ctx) + if err != nil { + return nil, err + } + + mapEditor := rowData.Edit() + err = conflicts.Iter(ctx, func(key, value types.Value) (stop bool, err error) { + cnf, err := conflict.ConflictFromTuple(value.(types.Tuple)) + if err != nil { + return true, err + } + + if types.IsNull(cnf.MergeValue) { + mapEditor.Remove(key) + } else { + mapEditor.Set(key, cnf.MergeValue) + } + + return false, nil + }) + if err != nil { + return nil, err + } + + rowData, err = mapEditor.Map(ctx) + if err != nil { + return nil, err + } + + return tbl.UpdateNomsRows(ctx, rowData) +} + +func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { + // Get conflicts + _, confIdx, err := tbl.GetConflicts(ctx) + if err != nil { + return nil, err + } + conflicts := durable.NomsMapFromConflictIndex(confIdx) + + if schema.IsKeyless(sch) { + return resolveKeylessConflicts(ctx, tbl, conflicts) + } + + return resolvePkConflicts(ctx, dEnv, tbl, tblName, sch, conflicts) +} + +func validateConstraintViolations(ctx *sql.Context, before, after *doltdb.RootValue, table string) error { + tables, err := after.GetTableNames(ctx) + if err != nil { + return err + } + + // todo: this is an expensive way to compute this + _, violators, err := merge.AddForeignKeyViolations(ctx, after, before, set.NewStrSet(tables), hash.Of(nil)) + if err != nil { + return err + } + if violators.Size() > 0 { + return fmt.Errorf("resolving conflicts for table %s created foreign key violations", table) + } + + return nil +} + +func clearTableAndUpdateRoot(ctx *sql.Context, root *doltdb.RootValue, tbl *doltdb.Table, tblName string) (*doltdb.RootValue, error) { + newTbl, err := tbl.ClearConflicts(ctx) + if err != nil { + return nil, err + } + newRoot, err := root.PutTable(ctx, tblName, newTbl) + if err != nil { + return nil, err + } + return newRoot, nil +} + +func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { + for _, tblName := range tblNames { + tbl, ok, err := root.GetTable(ctx, tblName) + if err != nil { + return err + } + if !ok { + return doltdb.ErrTableNotFound + } + + if has, err := tbl.HasConflicts(ctx); err != nil { + return err + } else if !has { + return nil + } + + sch, err := tbl.GetSchema(ctx) + if err != nil { + return err + } + _, ourSch, theirSch, err := tbl.GetConflictSchemas(ctx, tblName) + if err != nil { + return err + } + + if ours && !schema.ColCollsAreEqual(sch.GetAllCols(), ourSch.GetAllCols()) { + return ErrConfSchIncompatible + } else if !ours && !schema.ColCollsAreEqual(sch.GetAllCols(), theirSch.GetAllCols()) { + return ErrConfSchIncompatible + } + + if !ours { + if tbl.Format() == types.Format_DOLT { + tbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch) + } else { + tbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch) + } + if err != nil { + return err + } + } + + newRoot, err := clearTableAndUpdateRoot(ctx, root, tbl, tblName) + if err != nil { + return err + } + + err = validateConstraintViolations(ctx, root, newRoot, tblName) + if err != nil { + return err + } + + root = newRoot + } + return dSess.SetRoot(ctx, dbName, root) +} + +func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { + dbName := ctx.GetCurrentDatabase() + fmt.Printf("database name: %s", dbName) + + apr, err := cli.CreateConflictsResolveArgParser().Parse(args) + if err != nil { + return 1, err + } + + dSess := dsess.DSessFromSess(ctx.Session) + roots, ok := dSess.GetRoots(ctx, dbName) + if !ok { + return 1, fmt.Errorf("Could not load database %s", dbName) + } + + ours := apr.Contains(cli.OursFlag) + theirs := apr.Contains(cli.TheirsFlag) + if ours && theirs { + return 1, fmt.Errorf("specify only either --ours or --theirs") + } else if !ours && !theirs { + return 1, fmt.Errorf("--ours or --theirs must be supplied") + } + + if apr.NArg() == 0 { + return 1, fmt.Errorf("specify at least one table to resolve conflicts") + } + + // get all tables in conflict + root := roots.Working + tbls := apr.Args + if len(tbls) == 1 && tbls[0] == "." { + if allTables, err := root.TablesInConflict(ctx); err != nil { + return 1, err + } else { + tbls = allTables + } + } + + dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "") + err = ResolveConflicts(ctx, dEnv, dSess, root, dbName, ours, tbls) + if err != nil { + return 1, err + } + + return 0, nil +} From cbab3946c1492f68be2f15960fe66e5e7363733b Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 15:34:17 -0700 Subject: [PATCH 51/58] removing use of dEnv --- .../dprocedures/dolt_conflicts_resolve.go | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go index 308ab09493..47c8d84292 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go @@ -24,14 +24,12 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/conflict" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" - "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/merge" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" "github.com/dolthub/dolt/go/libraries/doltcore/table" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" - "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/libraries/utils/set" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" @@ -198,13 +196,8 @@ func resolveProllyConflicts(ctx *sql.Context, tbl *doltdb.Table, tblName string, return newTbl, nil } -func resolvePkConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema, conflicts types.Map) (*doltdb.Table, error) { +func resolvePkConflicts(ctx *sql.Context, opts editor.Options, tbl *doltdb.Table, tblName string, sch schema.Schema, conflicts types.Map) (*doltdb.Table, error) { // Create table editor - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: tmpDir} tblEditor, err := editor.NewTableEditor(ctx, tbl, sch, tblName, opts) if err != nil { return nil, err @@ -300,7 +293,7 @@ func resolveKeylessConflicts(ctx *sql.Context, tbl *doltdb.Table, conflicts type return tbl.UpdateNomsRows(ctx, rowData) } -func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { +func resolveNomsConflicts(ctx *sql.Context, opts editor.Options, tbl *doltdb.Table, tblName string, sch schema.Schema) (*doltdb.Table, error) { // Get conflicts _, confIdx, err := tbl.GetConflicts(ctx) if err != nil { @@ -312,7 +305,7 @@ func resolveNomsConflicts(ctx *sql.Context, dEnv *env.DoltEnv, tbl *doltdb.Table return resolveKeylessConflicts(ctx, tbl, conflicts) } - return resolvePkConflicts(ctx, dEnv, tbl, tblName, sch, conflicts) + return resolvePkConflicts(ctx, opts, tbl, tblName, sch, conflicts) } func validateConstraintViolations(ctx *sql.Context, before, after *doltdb.RootValue, table string) error { @@ -345,7 +338,7 @@ func clearTableAndUpdateRoot(ctx *sql.Context, root *doltdb.RootValue, tbl *dolt return newRoot, nil } -func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { +func ResolveConflicts(ctx *sql.Context, dSess *dsess.DoltSession, root *doltdb.RootValue, dbName string, ours bool, tblNames []string) error { for _, tblName := range tblNames { tbl, ok, err := root.GetTable(ctx, tblName) if err != nil { @@ -380,7 +373,12 @@ func ResolveConflicts(ctx *sql.Context, dEnv *env.DoltEnv, dSess *dsess.DoltSess if tbl.Format() == types.Format_DOLT { tbl, err = resolveProllyConflicts(ctx, tbl, tblName, sch) } else { - tbl, err = resolveNomsConflicts(ctx, dEnv, tbl, tblName, sch) + state, _, err := dSess.LookupDbState(ctx, dbName) + if err != nil { + return err + } + opts := state.WriteSession.GetOptions() + tbl, err = resolveNomsConflicts(ctx, opts, tbl, tblName, sch) } if err != nil { return err @@ -440,8 +438,7 @@ func DoDoltConflictsResolve(ctx *sql.Context, args []string) (int, error) { } } - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, filesys.LocalFS, doltdb.LocalDirDoltDB, "") - err = ResolveConflicts(ctx, dEnv, dSess, root, dbName, ours, tbls) + err = ResolveConflicts(ctx, dSess, root, dbName, ours, tbls) if err != nil { return 1, err } From f21b1c9c2c53fa757a05e8ef8c25a242b5db7ceb Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Wed, 28 Sep 2022 22:35:59 +0000 Subject: [PATCH 52/58] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- .../doltcore/sqle/dprocedures/dolt_conflicts_resolve.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go index 47c8d84292..5d8811c1d5 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_conflicts_resolve.go @@ -17,9 +17,10 @@ package dprocedures import ( "errors" "fmt" - "github.com/dolthub/go-mysql-server/sql" "io" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/conflict" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" From 9cc1d99b1acfa361b14c40ad767da182ee159a41 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 28 Sep 2022 15:54:07 -0700 Subject: [PATCH 53/58] bad test --- integration-tests/bats/foreign-keys.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/foreign-keys.bats b/integration-tests/bats/foreign-keys.bats index ce8d05f328..170d91926e 100644 --- a/integration-tests/bats/foreign-keys.bats +++ b/integration-tests/bats/foreign-keys.bats @@ -1323,7 +1323,7 @@ SQL run dolt sql < Date: Thu, 29 Sep 2022 06:24:32 +0000 Subject: [PATCH 54/58] [ga-bump-dep] Bump dependency in Dolt by Hydrocharged --- go/go.mod | 4 ++-- go/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go/go.mod b/go/go.mod index b79c9ada82..b9a71e5f1e 100644 --- a/go/go.mod +++ b/go/go.mod @@ -17,7 +17,7 @@ require ( github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20220927165657-8eb73ed0ff24 + github.com/dolthub/vitess v0.0.0-20220929061157-c71cf6a7768e github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible - github.com/dolthub/go-mysql-server v0.12.1-0.20220928174432-cd8b451c9bf4 + github.com/dolthub/go-mysql-server v0.12.1-0.20220929062247-323a847921de github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 3f02e10fb4..5a667aa77d 100644 --- a/go/go.sum +++ b/go/go.sum @@ -177,8 +177,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20220928174432-cd8b451c9bf4 h1:27KKhBenaRJy++ZiVeEKcXQv0yJawAc+oPozDGqEMuc= -github.com/dolthub/go-mysql-server v0.12.1-0.20220928174432-cd8b451c9bf4/go.mod h1:8zHF9V5MPmb3dWB2kGjeltnHQc15QdynK9GnwSutreA= +github.com/dolthub/go-mysql-server v0.12.1-0.20220929062247-323a847921de h1:YkKR9AOt/Mta3suApA5bEwwTF/GbdYLva3zVbP0lxi0= +github.com/dolthub/go-mysql-server v0.12.1-0.20220929062247-323a847921de/go.mod h1:Ndof+jmKE/AISRWgeyx+RUvNlAtMOPSUzTM/iCOfx70= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= @@ -187,8 +187,8 @@ github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66 h1:WRPDbpJWEnPxP github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66/go.mod h1:N5ZIbMGuDUpTpOFQ7HcsN6WSIpTGQjHP+Mz27AfmAgk= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= -github.com/dolthub/vitess v0.0.0-20220927165657-8eb73ed0ff24 h1:2ARLp21egNaEvWIN4/tfprcyAlWsRj4bdey+Hv7HMEM= -github.com/dolthub/vitess v0.0.0-20220927165657-8eb73ed0ff24/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs= +github.com/dolthub/vitess v0.0.0-20220929061157-c71cf6a7768e h1:vC5OgmUm1Dd8vQP1YqgRvYMbHvrLNdQkd3S7udbS3BQ= +github.com/dolthub/vitess v0.0.0-20220929061157-c71cf6a7768e/go.mod h1:oVFIBdqMFEkt4Xz2fzFJBNtzKhDEjwdCF0dzde39iKs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= From 41fc88476f30917f5a623cc555ec438e1fba40d1 Mon Sep 17 00:00:00 2001 From: Hydrocharged Date: Thu, 29 Sep 2022 12:57:41 +0000 Subject: [PATCH 55/58] [ga-bump-release] Update Dolt version to 0.41.7 and release v0.41.7 --- go/cmd/dolt/dolt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/dolt/dolt.go b/go/cmd/dolt/dolt.go index 1b48220cf1..752074cc6f 100644 --- a/go/cmd/dolt/dolt.go +++ b/go/cmd/dolt/dolt.go @@ -57,7 +57,7 @@ import ( ) const ( - Version = "0.41.6" + Version = "0.41.7" ) var dumpDocsCommand = &commands.DumpDocsCmd{} From 19cdb64b72886c2f974ccc790a1004d13011b541 Mon Sep 17 00:00:00 2001 From: Tim Sehn Date: Thu, 29 Sep 2022 09:08:34 -0700 Subject: [PATCH 56/58] Reference new socket interface functionality in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 025336c09d..dfdabb8c79 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ MySQL comes with a MySQL server called `mysqld` and a MySQL client called `mysql mysql Ver 8.0.29 for macos12.2 on x86_64 (Homebrew) ``` -Now, to connect the `mysql` client to Dolt, you have to force the MySQL client through the TCP interface by passing in a host and port. The default is the socket interface which Dolt supports, but is not on by default. The MySQL client also requires you specify a user, in this case `root`. +Now, to connect the `mysql` client to Dolt, you are going to force the MySQL client through the TCP interface by passing in a host and port. The default is the socket interface which Dolt supports, but is only available on `localhost`. So, it's better to show off the TCP insterface. The MySQL client also requires you specify a user, in this case `root`. ```bash % mysql --host 127.0.0.1 --port 3306 -uroot From 18aa514c469a8d82f33cccbd5c36a04187b0a14f Mon Sep 17 00:00:00 2001 From: Tim Sehn Date: Thu, 29 Sep 2022 09:16:26 -0700 Subject: [PATCH 57/58] Update README.md Co-authored-by: Jason Fulghum --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dfdabb8c79..19390d7b10 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ MySQL comes with a MySQL server called `mysqld` and a MySQL client called `mysql mysql Ver 8.0.29 for macos12.2 on x86_64 (Homebrew) ``` -Now, to connect the `mysql` client to Dolt, you are going to force the MySQL client through the TCP interface by passing in a host and port. The default is the socket interface which Dolt supports, but is only available on `localhost`. So, it's better to show off the TCP insterface. The MySQL client also requires you specify a user, in this case `root`. +Now, to connect the `mysql` client to Dolt, you are going to force the MySQL client through the TCP interface by passing in a host and port. The default is the socket interface which Dolt supports, but is only available on `localhost`. So, it's better to show off the TCP interface. The MySQL client also requires you specify a user, in this case `root`. ```bash % mysql --host 127.0.0.1 --port 3306 -uroot From f48ffcf4cdd3a29dabc38b1f4e458a9490d99a92 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Thu, 29 Sep 2022 09:50:52 -0700 Subject: [PATCH 58/58] go: sqle: cluster: PR feedback. --- .../doltcore/sqle/cluster/commithook.go | 90 +++++++++++-------- .../doltcore/sqle/cluster/controller.go | 2 + .../doltcore/sqle/cluster/interceptors.go | 25 +++++- 3 files changed, 75 insertions(+), 42 deletions(-) diff --git a/go/libraries/doltcore/sqle/cluster/commithook.go b/go/libraries/doltcore/sqle/cluster/commithook.go index c10f9616c5..93e484734d 100644 --- a/go/libraries/doltcore/sqle/cluster/commithook.go +++ b/go/libraries/doltcore/sqle/cluster/commithook.go @@ -19,6 +19,7 @@ import ( "errors" "io" "sync" + "sync/atomic" "time" "github.com/dolthub/go-mysql-server/sql" @@ -33,7 +34,7 @@ var _ doltdb.CommitHook = (*commithook)(nil) type commithook struct { rootLgr *logrus.Entry - lgr *logrus.Entry + lgr atomic.Value // *logrus.Entry remotename string dbname string lout io.Writer @@ -60,12 +61,15 @@ type commithook struct { tempDir string } -var errDestDBRootHashMoved error = errors.New("sqle: cluster: standby replication: destination database root hash moved during our write, while it is assumed we are the only writer.") +var errDestDBRootHashMoved error = errors.New("cluster/commithook: standby replication: destination database root hash moved during our write, while it is assumed we are the only writer.") + +const logFieldThread = "thread" +const logFieldRole = "role" func newCommitHook(lgr *logrus.Logger, remotename, dbname string, role Role, destDBF func(context.Context) (*doltdb.DoltDB, error), srcDB *doltdb.DoltDB, tempDir string) *commithook { var ret commithook - ret.rootLgr = lgr.WithField("thread", "Standby Replication - "+dbname+" to "+remotename) - ret.lgr = ret.rootLgr.WithField("role", string(role)) + ret.rootLgr = lgr.WithField(logFieldThread, "Standby Replication - "+dbname+" to "+remotename) + ret.lgr.Store(ret.rootLgr.WithField(logFieldRole, string(role))) ret.remotename = remotename ret.dbname = dbname ret.role = role @@ -80,15 +84,29 @@ func (h *commithook) Run(bt *sql.BackgroundThreads) error { return bt.Add("Standby Replication - "+h.dbname+" to "+h.remotename, h.run) } +func (h *commithook) run(ctx context.Context) { + // The hook comes up attempting to replicate the current head. + h.logger().Tracef("cluster/commithook: background thread: running.") + h.wg.Add(2) + go h.replicate(ctx) + go h.tick(ctx) + <-ctx.Done() + h.logger().Tracef("cluster/commithook: background thread: requested shutdown, signaling replication thread.") + h.cond.Signal() + h.wg.Wait() + h.logger().Tracef("cluster/commithook: background thread: completed.") +} + func (h *commithook) replicate(ctx context.Context) { defer h.wg.Done() - defer h.lgr.Tracef("cluster/commithook: background thread: replicate: shutdown.") + defer h.logger().Tracef("cluster/commithook: background thread: replicate: shutdown.") h.mu.Lock() defer h.mu.Unlock() for { + lgr := h.logger() // Shutdown for context canceled. if ctx.Err() != nil { - h.lgr.Tracef("cluster/commithook replicate thread exiting; saw ctx.Err(): %v", ctx.Err()) + lgr.Tracef("cluster/commithook replicate thread exiting; saw ctx.Err(): %v", ctx.Err()) if h.shouldReplicate() { // attempt a last true-up of our standby as we shutdown // TODO: context.WithDeadline based on config / convention? @@ -96,8 +114,8 @@ func (h *commithook) replicate(ctx context.Context) { } return } - if h.role == RolePrimary && h.nextHead == (hash.Hash{}) { - h.lgr.Tracef("cluster/commithook: fetching current head.") + if h.primaryNeedsInit() { + lgr.Tracef("cluster/commithook: fetching current head.") // When the replicate thread comes up, it attempts to replicate the current head. datasDB := doltdb.HackDatasDatabaseFromDoltDB(h.srcDB) cs := datas.ChunkStoreFromDatabase(datasDB) @@ -105,19 +123,19 @@ func (h *commithook) replicate(ctx context.Context) { h.nextHead, err = cs.Root(ctx) if err != nil { // TODO: if err != nil, something is really wrong; should shutdown or backoff. - h.lgr.Warningf("standby replication thread failed to load database root: %v", err) + lgr.Warningf("standby replication thread failed to load database root: %v", err) h.nextHead = hash.Hash{} } // We do not know when this head was written, but we - // starting trying to replicate it now. + // are starting to try to replicate it now. h.nextHeadIncomingTime = time.Now() } else if h.shouldReplicate() { h.attemptReplicate(ctx) } else { - h.lgr.Tracef("cluster/commithook: background thread: waiting for signal.") + lgr.Tracef("cluster/commithook: background thread: waiting for signal.") h.cond.Wait() - h.lgr.Tracef("cluster/commithook: background thread: woken up.") + lgr.Tracef("cluster/commithook: background thread: woken up.") } } } @@ -133,23 +151,29 @@ func (h *commithook) shouldReplicate() bool { return (h.nextPushAttempt == (time.Time{}) || time.Now().After(h.nextPushAttempt)) } +// called with h.mu locked. +func (h *commithook) primaryNeedsInit() bool { + return h.role == RolePrimary && h.nextHead == (hash.Hash{}) +} + // Called by the replicate thread to push the nextHead to the destDB and set // its root to the new value. // // preconditions: h.mu is locked and shouldReplicate() returned true. // when this function returns, h.mu is locked. func (h *commithook) attemptReplicate(ctx context.Context) { + lgr := h.logger() toPush := h.nextHead incomingTime := h.nextHeadIncomingTime destDB := h.destDB h.mu.Unlock() if destDB == nil { - h.lgr.Tracef("cluster/commithook: attempting to fetch destDB.") + lgr.Tracef("cluster/commithook: attempting to fetch destDB.") var err error destDB, err = h.destDBF(ctx) if err != nil { - h.lgr.Warnf("cluster/commithook: could not replicate to standby: error fetching destDB: %v.", err) + lgr.Warnf("cluster/commithook: could not replicate to standby: error fetching destDB: %v.", err) h.mu.Lock() // TODO: We could add some backoff here. if toPush == h.nextHead { @@ -157,16 +181,16 @@ func (h *commithook) attemptReplicate(ctx context.Context) { } return } - h.lgr.Tracef("cluster/commithook: fetched destDB") + lgr.Tracef("cluster/commithook: fetched destDB") h.mu.Lock() h.destDB = destDB h.mu.Unlock() } - h.lgr.Tracef("cluster/commithook: pushing chunks for root hash %v to destDB", toPush.String()) + lgr.Tracef("cluster/commithook: pushing chunks for root hash %v to destDB", toPush.String()) err := destDB.PullChunks(ctx, h.tempDir, h.srcDB, toPush, nil, nil) if err == nil { - h.lgr.Tracef("cluster/commithook: successfully pushed chunks, setting root") + lgr.Tracef("cluster/commithook: successfully pushed chunks, setting root") datasDB := doltdb.HackDatasDatabaseFromDoltDB(destDB) cs := datas.ChunkStoreFromDatabase(datasDB) var curRootHash hash.Hash @@ -181,12 +205,12 @@ func (h *commithook) attemptReplicate(ctx context.Context) { h.mu.Lock() if err == nil { - h.lgr.Tracef("cluster/commithook: successfully Commited chunks on destDB") + lgr.Tracef("cluster/commithook: successfully Commited chunks on destDB") h.lastPushedHead = toPush h.lastPushedSuccess = incomingTime h.nextPushAttempt = time.Time{} } else { - h.lgr.Warnf("cluster/commithook: failed to commit chunks on destDB: %v", err) + lgr.Warnf("cluster/commithook: failed to commit chunks on destDB: %v", err) // add some delay if a new head didn't come in while we were pushing. if toPush == h.nextHead { // TODO: We could add some backoff here. @@ -215,16 +239,18 @@ func (h *commithook) replicationLag() time.Duration { return time.Now().Sub(h.lastPushedSuccess) } +func (h *commithook) logger() *logrus.Entry { + return h.lgr.Load().(*logrus.Entry) +} + // TODO: Would be more efficient to only tick when we have outstanding work... func (h *commithook) tick(ctx context.Context) { defer h.wg.Done() - defer h.lgr.Trace("tick thread returning") ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): - time.Sleep(1 * time.Second) return case <-ticker.C: h.cond.Signal() @@ -232,19 +258,6 @@ func (h *commithook) tick(ctx context.Context) { } } -func (h *commithook) run(ctx context.Context) { - // The hook comes up attempting to replicate the current head. - h.lgr.Tracef("cluster/commithook: background thread: running.") - h.wg.Add(2) - go h.replicate(ctx) - go h.tick(ctx) - <-ctx.Done() - h.lgr.Tracef("cluster/commithook: background thread: requested shutdown, signaling replication thread.") - h.cond.Signal() - h.wg.Wait() - h.lgr.Tracef("cluster/commithook: background thread: completed.") -} - func (h *commithook) setRole(role Role) { h.mu.Lock() defer h.mu.Unlock() @@ -255,24 +268,25 @@ func (h *commithook) setRole(role Role) { h.lastPushedSuccess = time.Time{} h.nextPushAttempt = time.Time{} h.role = role - h.lgr = h.rootLgr.WithField("role", string(role)) + h.lgr.Store(h.rootLgr.WithField(logFieldRole, string(role))) h.cond.Signal() } // Execute on this commithook updates the target root hash we're attempting to // replicate and wakes the replication thread. func (h *commithook) Execute(ctx context.Context, ds datas.Dataset, db datas.Database) error { - h.lgr.Warnf("cluster/commithook: Execute called post commit") + lgr := h.logger() + lgr.Warnf("cluster/commithook: Execute called post commit") cs := datas.ChunkStoreFromDatabase(db) root, err := cs.Root(ctx) if err != nil { - h.lgr.Warnf("cluster/commithook: Execute: error retrieving local database root: %v", err) + lgr.Warnf("cluster/commithook: Execute: error retrieving local database root: %v", err) return err } h.mu.Lock() defer h.mu.Unlock() if root != h.nextHead { - h.lgr.Tracef("signaling replication thread to push new head: %v", root.String()) + lgr.Tracef("signaling replication thread to push new head: %v", root.String()) h.nextHeadIncomingTime = time.Now() h.nextHead = root h.nextPushAttempt = time.Time{} diff --git a/go/libraries/doltcore/sqle/cluster/controller.go b/go/libraries/doltcore/sqle/cluster/controller.go index 744a854f22..c6a7b6fad0 100644 --- a/go/libraries/doltcore/sqle/cluster/controller.go +++ b/go/libraries/doltcore/sqle/cluster/controller.go @@ -82,7 +82,9 @@ func NewController(lgr *logrus.Logger, cfg Config, pCfg config.ReadWriteConfig) commithooks: make([]*commithook, 0), lgr: lgr, } + ret.sinterceptor.lgr = lgr.WithFields(logrus.Fields{}) ret.sinterceptor.setRole(role, epoch) + ret.cinterceptor.lgr = lgr.WithFields(logrus.Fields{}) ret.cinterceptor.setRole(role, epoch) return ret, nil } diff --git a/go/libraries/doltcore/sqle/cluster/interceptors.go b/go/libraries/doltcore/sqle/cluster/interceptors.go index e972ad21d5..a0b40f0093 100644 --- a/go/libraries/doltcore/sqle/cluster/interceptors.go +++ b/go/libraries/doltcore/sqle/cluster/interceptors.go @@ -19,6 +19,8 @@ import ( "strconv" "sync" + "github.com/sirupsen/logrus" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -43,6 +45,7 @@ const clusterRoleEpochHeader = "x-dolt-cluster-role-epoch" // epoch than this server, this incterceptor coordinates with the Controller to // immediately transition to standby and to stop replicating to the standby. type clientinterceptor struct { + lgr *logrus.Entry role Role epoch int mu sync.Mutex @@ -93,10 +96,14 @@ func (ci *clientinterceptor) handleResponseHeaders(header metadata.MD, role Role epochs := header.Get(clusterRoleEpochHeader) roles := header.Get(clusterRoleHeader) if len(epochs) > 0 && len(roles) > 0 && roles[0] == string(RolePrimary) { - if retepoch, err := strconv.Atoi(epochs[0]); err == nil { - if retepoch > epoch { + if respepoch, err := strconv.Atoi(epochs[0]); err == nil { + if respepoch == epoch { + ci.lgr.Errorf("cluster: this server and the server replicating to it are both primary at the same epoch. force transitioning to standby.") + // TODO: Signal to controller that we are forced to become a standby at epoch |respepoch|... + } else if respepoch > epoch { // The server we replicate to thinks it is the primary at a higher epoch than us... - // TODO: Signal to controller that we are forced to become a standby at epoch |retepoch|... + ci.lgr.Warnf("cluster: this server is primary at epoch %d. the server replicating to it is primary at epoch %d. force transitioning to standby.", epoch, respepoch) + // TODO: Signal to controller that we are forced to become a standby at epoch |respepoch|... } } } @@ -124,6 +131,7 @@ func (ci *clientinterceptor) Options() []grpc.DialOption { // than our current epoch, this interceptor coordinates with the Controller to // immediately transition to standby and allow replication requests through. type serverinterceptor struct { + lgr *logrus.Entry role Role epoch int mu sync.Mutex @@ -172,8 +180,17 @@ func (si *serverinterceptor) handleRequestHeaders(header metadata.MD, role Role, roles := header.Get(clusterRoleHeader) if len(epochs) > 0 && len(roles) > 0 && roles[0] == string(RolePrimary) && role == RolePrimary { if reqepoch, err := strconv.Atoi(epochs[0]); err == nil { - if reqepoch > epoch { + if reqepoch == epoch { + // Misconfiguration in the cluster means this + // server and its standby are marked as Primary + // at the same epoch. We will become standby + // and our peer will become standby. An + // operator will need to get involved. + si.lgr.Errorf("cluster: this server and its standby replica are both primary at the same epoch. force transitioning to standby.") + // TODO: Signal to controller that we are forced to become a standby at epoch |reqepoch| + } else if reqepoch > epoch { // The client replicating to us thinks it is the primary at a higher epoch than us. + si.lgr.Warnf("cluster: this server is primary at epoch %d. the server replicating to it is primary at epoch %d. force transitioning to standby.", epoch, reqepoch) // TODO: Signal to controller that we are forced to become a standby at epoch |reqepoch| } }