mirror of
https://github.com/dolthub/dolt.git
synced 2026-05-01 03:29:12 -05:00
Add noms merge (#2768)
Add optional merging functionality to noms commit. noms merge <database> <left-dataset-name> <right-dataset-name> <output-dataset-name> The command above will look in the given Database for the two named Datasets and, if possible, merge their HeadValue()s and commit the result back to <output-dataset-name>. Fixes #2535
This commit is contained in:
cmasone-attic
@@ -21,6 +21,7 @@ var commands = []*util.Command{
|
||||
nomsDiff,
|
||||
nomsDs,
|
||||
nomsLog,
|
||||
nomsMerge,
|
||||
nomsMigrate,
|
||||
nomsServe,
|
||||
nomsShow,
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
// Copyright 2016 Attic Labs, Inc. All rights reserved.
|
||||
// Licensed under the Apache License, version 2.0:
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/attic-labs/noms/cmd/util"
|
||||
"github.com/attic-labs/noms/go/config"
|
||||
"github.com/attic-labs/noms/go/d"
|
||||
"github.com/attic-labs/noms/go/datas"
|
||||
"github.com/attic-labs/noms/go/merge"
|
||||
"github.com/attic-labs/noms/go/types"
|
||||
"github.com/attic-labs/noms/go/util/status"
|
||||
"github.com/attic-labs/noms/go/util/verbose"
|
||||
flag "github.com/juju/gnuflag"
|
||||
)
|
||||
|
||||
var (
|
||||
resolver string
|
||||
|
||||
nomsMerge = &util.Command{
|
||||
Run: runMerge,
|
||||
UsageLine: "merge [options] <database> <left-dataset-name> <right-dataset-name> <output-dataset-name>",
|
||||
Short: "Merges and commits the head values of two named datasets",
|
||||
Long: "See Spelling Objects at https://github.com/attic-labs/noms/blob/master/doc/spelling.md for details on the database argument.\nYu must provide a working database and the names of two Datasets you want to merge. The values at the heads of these Datasets will be merged, put into a new Commit object, and set as the Head of the third provided Dataset name.",
|
||||
Flags: setupMergeFlags,
|
||||
Nargs: 1, // if absolute-path not present we read it from stdin
|
||||
}
|
||||
datasetRe = regexp.MustCompile("^" + datas.DatasetRe.String() + "$")
|
||||
)
|
||||
|
||||
func setupMergeFlags() *flag.FlagSet {
|
||||
commitFlagSet := flag.NewFlagSet("merge", flag.ExitOnError)
|
||||
commitFlagSet.StringVar(&resolver, "policy", "n", "conflict resolution policy for merging. Defaults to 'n', which means no resolution strategy will be applied. Supported values are 'l' (left), 'r' (right) and 'p' (prompt). 'prompt' will bring up a simple command-line prompt allowing you to resolve conflicts by choosing between 'l' or 'r' on a case-by-case basis.")
|
||||
verbose.RegisterVerboseFlags(commitFlagSet)
|
||||
return commitFlagSet
|
||||
}
|
||||
|
||||
func checkIfTrue(b bool, format string, args ...interface{}) {
|
||||
if b {
|
||||
d.CheckErrorNoUsage(fmt.Errorf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
func runMerge(args []string) int {
|
||||
cfg := config.NewResolver()
|
||||
|
||||
if len(args) != 4 {
|
||||
d.CheckErrorNoUsage(fmt.Errorf("Incorrect number of arguments"))
|
||||
}
|
||||
db, err := cfg.GetDatabase(args[0])
|
||||
d.CheckError(err)
|
||||
defer db.Close()
|
||||
|
||||
leftDS, rightDS, outDS := resolveDatasets(db, args[1], args[2], args[3])
|
||||
left, right, ancestor := getMergeCandidates(db, leftDS, rightDS)
|
||||
policy := decidePolicy(resolver)
|
||||
pc := newMergeProgressChan()
|
||||
merged, err := policy(left, right, ancestor, db, pc)
|
||||
d.CheckErrorNoUsage(err)
|
||||
close(pc)
|
||||
|
||||
_, err = db.SetHead(outDS, db.WriteValue(datas.NewCommit(merged, types.NewSet(leftDS.HeadRef(), rightDS.HeadRef()), types.EmptyStruct)))
|
||||
d.PanicIfError(err)
|
||||
if !verbose.Quiet() {
|
||||
status.Printf("Done")
|
||||
status.Done()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func resolveDatasets(db datas.Database, leftName, rightName, outName string) (leftDS, rightDS, outDS datas.Dataset) {
|
||||
makeDS := func(dsName string) datas.Dataset {
|
||||
if !datasetRe.MatchString(dsName) {
|
||||
d.CheckErrorNoUsage(fmt.Errorf("Invalid dataset %s, must match %s", dsName, datas.DatasetRe.String()))
|
||||
}
|
||||
return db.GetDataset(dsName)
|
||||
}
|
||||
leftDS = makeDS(leftName)
|
||||
rightDS = makeDS(rightName)
|
||||
outDS = makeDS(outName)
|
||||
return
|
||||
}
|
||||
|
||||
func getMergeCandidates(db datas.Database, leftDS, rightDS datas.Dataset) (left, right, ancestor types.Value) {
|
||||
leftRef, ok := leftDS.MaybeHeadRef()
|
||||
checkIfTrue(!ok, "Dataset %s has no data", leftDS.ID())
|
||||
rightRef, ok := rightDS.MaybeHeadRef()
|
||||
checkIfTrue(!ok, "Dataset %s has no data", rightDS.ID())
|
||||
ancestorCommit, ok := getCommonAncestor(leftRef, rightRef, db)
|
||||
checkIfTrue(!ok, "Datasets %s and %s have no common ancestor", leftDS.ID(), rightDS.ID())
|
||||
|
||||
return leftDS.HeadValue(), rightDS.HeadValue(), ancestorCommit.Get(datas.ValueField)
|
||||
}
|
||||
|
||||
func getCommonAncestor(r1, r2 types.Ref, vr types.ValueReader) (a types.Struct, found bool) {
|
||||
aRef, found := datas.FindCommonAncestor(r1, r2, vr)
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
v := vr.ReadValue(aRef.TargetHash())
|
||||
if v == nil {
|
||||
panic(aRef.TargetHash().String() + " not found")
|
||||
}
|
||||
if !datas.IsCommitType(v.Type()) {
|
||||
panic("Not a commit: " + types.EncodedValueMaxLines(v, 10) + " ...")
|
||||
}
|
||||
return v.(types.Struct), true
|
||||
}
|
||||
|
||||
func newMergeProgressChan() chan struct{} {
|
||||
pc := make(chan struct{}, 128)
|
||||
go func() {
|
||||
count := 0
|
||||
for range pc {
|
||||
if !verbose.Quiet() {
|
||||
count++
|
||||
status.Printf("Applied %d changes...", count)
|
||||
}
|
||||
}
|
||||
}()
|
||||
return pc
|
||||
}
|
||||
|
||||
func decidePolicy(policy string) merge.Policy {
|
||||
var resolve merge.ResolveFunc
|
||||
switch policy {
|
||||
case "n", "N":
|
||||
resolve = merge.None
|
||||
case "l", "L":
|
||||
resolve = merge.Ours
|
||||
case "r", "R":
|
||||
resolve = merge.Theirs
|
||||
case "p", "P":
|
||||
resolve = func(aType, bType types.DiffChangeType, a, b types.Value, path types.Path) (change types.DiffChangeType, merged types.Value, ok bool) {
|
||||
return cliResolve(os.Stdin, os.Stdout, aType, bType, a, b, path)
|
||||
}
|
||||
default:
|
||||
d.CheckErrorNoUsage(fmt.Errorf("Unsupported merge policy: %s. Choices are n, l, r and a.", policy))
|
||||
}
|
||||
return merge.NewThreeWay(resolve)
|
||||
}
|
||||
|
||||
func cliResolve(in io.Reader, out io.Writer, aType, bType types.DiffChangeType, a, b types.Value, path types.Path) (change types.DiffChangeType, merged types.Value, ok bool) {
|
||||
stringer := func(v types.Value) (s string, success bool) {
|
||||
switch v := v.(type) {
|
||||
case types.Bool, types.Number, types.String:
|
||||
return fmt.Sprintf("%v", v), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
left, lOk := stringer(a)
|
||||
right, rOk := stringer(b)
|
||||
if !lOk || !rOk {
|
||||
return change, merged, false
|
||||
}
|
||||
|
||||
// TODO: Handle removes as well.
|
||||
fmt.Fprintf(out, "\nConflict at: %s\n", path.String())
|
||||
fmt.Fprintf(out, "Left: %s\nRight: %s\n\n", left, right)
|
||||
var choice rune
|
||||
for {
|
||||
fmt.Fprintln(out, "Enter 'l' to accept the left value, 'r' to accept the right value")
|
||||
_, err := fmt.Fscanf(in, "%c\n", &choice)
|
||||
d.PanicIfError(err)
|
||||
switch choice {
|
||||
case 'l', 'L':
|
||||
return aType, a, true
|
||||
case 'r', 'R':
|
||||
return bType, b, true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
// Copyright 2016 Attic Labs, Inc. All rights reserved.
|
||||
// Licensed under the Apache License, version 2.0:
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/attic-labs/noms/go/datas"
|
||||
"github.com/attic-labs/noms/go/spec"
|
||||
"github.com/attic-labs/noms/go/types"
|
||||
"github.com/attic-labs/noms/go/util/clienttest"
|
||||
"github.com/attic-labs/testify/assert"
|
||||
"github.com/attic-labs/testify/suite"
|
||||
)
|
||||
|
||||
type nomsMergeTestSuite struct {
|
||||
clienttest.ClientTestSuite
|
||||
}
|
||||
|
||||
func TestNomsMerge(t *testing.T) {
|
||||
suite.Run(t, &nomsMergeTestSuite{})
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) TearDownTest() {
|
||||
s.NoError(os.RemoveAll(s.LdbDir))
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) TestNomsMerge_Success() {
|
||||
left, right := "left", "right"
|
||||
p := s.setupMergeDataset(
|
||||
"parent",
|
||||
types.StructData{
|
||||
"num": types.Number(42),
|
||||
"str": types.String("foobar"),
|
||||
"lst": types.NewList(types.Number(1), types.String("foo")),
|
||||
"map": types.NewMap(types.Number(1), types.String("foo"),
|
||||
types.String("foo"), types.Number(1)),
|
||||
},
|
||||
types.NewSet())
|
||||
|
||||
l := s.setupMergeDataset(
|
||||
left,
|
||||
types.StructData{
|
||||
"num": types.Number(42),
|
||||
"str": types.String("foobaz"),
|
||||
"lst": types.NewList(types.Number(1), types.String("foo")),
|
||||
"map": types.NewMap(types.Number(1), types.String("foo"),
|
||||
types.String("foo"), types.Number(1)),
|
||||
},
|
||||
types.NewSet(p))
|
||||
|
||||
r := s.setupMergeDataset(
|
||||
right,
|
||||
types.StructData{
|
||||
"num": types.Number(42),
|
||||
"str": types.String("foobar"),
|
||||
"lst": types.NewList(types.Number(1), types.String("foo")),
|
||||
"map": types.NewMap(types.Number(1), types.String("foo"),
|
||||
types.String("foo"), types.Number(1), types.Number(2), types.String("bar")),
|
||||
},
|
||||
types.NewSet(p))
|
||||
|
||||
expected := types.NewStruct("", types.StructData{
|
||||
"num": types.Number(42),
|
||||
"str": types.String("foobaz"),
|
||||
"lst": types.NewList(types.Number(1), types.String("foo")),
|
||||
"map": types.NewMap(types.Number(1), types.String("foo"),
|
||||
types.String("foo"), types.Number(1), types.Number(2), types.String("bar")),
|
||||
})
|
||||
|
||||
output := "output"
|
||||
stdout, stderr, err := s.Run(main, []string{"merge", s.LdbDir, left, right, output})
|
||||
if err == nil {
|
||||
s.Equal("", stderr)
|
||||
s.validateDataset(output, expected, l, r)
|
||||
} else {
|
||||
s.Fail("Run failed", "err: %v\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) setupMergeDataset(name string, data types.StructData, p types.Set) types.Ref {
|
||||
db, ds, _ := spec.GetDataset(spec.CreateValueSpecString("ldb", s.LdbDir, name))
|
||||
defer db.Close()
|
||||
ds, err := db.Commit(ds, types.NewStruct("", data), datas.CommitOptions{Parents: p})
|
||||
s.NoError(err)
|
||||
return ds.HeadRef()
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) validateDataset(name string, expected types.Struct, parents ...types.Value) {
|
||||
db, ds, err := spec.GetDataset(spec.CreateValueSpecString("ldb", s.LdbDir, name))
|
||||
if s.NoError(err) {
|
||||
commit := ds.Head()
|
||||
s.True(commit.Get(datas.ParentsField).Equals(types.NewSet(parents...)))
|
||||
merged := ds.HeadValue()
|
||||
s.True(expected.Equals(merged), "%s != %s", types.EncodedValue(expected), types.EncodedValue(merged))
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) TestNomsMerge_Left() {
|
||||
left, right := "left", "right"
|
||||
p := s.setupMergeDataset("parent", types.StructData{"num": types.Number(42)}, types.NewSet())
|
||||
l := s.setupMergeDataset(left, types.StructData{"num": types.Number(43)}, types.NewSet(p))
|
||||
r := s.setupMergeDataset(right, types.StructData{"num": types.Number(44)}, types.NewSet(p))
|
||||
|
||||
expected := types.NewStruct("", types.StructData{"num": types.Number(43)})
|
||||
|
||||
output := "output"
|
||||
stdout, stderr, err := s.Run(main, []string{"merge", "--policy=l", s.LdbDir, left, right, output})
|
||||
if err == nil {
|
||||
s.Equal("", stderr)
|
||||
s.validateDataset(output, expected, l, r)
|
||||
} else {
|
||||
s.Fail("Run failed", "err: %v\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) TestNomsMerge_Right() {
|
||||
left, right := "left", "right"
|
||||
p := s.setupMergeDataset("parent", types.StructData{"num": types.Number(42)}, types.NewSet())
|
||||
l := s.setupMergeDataset(left, types.StructData{"num": types.Number(43)}, types.NewSet(p))
|
||||
r := s.setupMergeDataset(right, types.StructData{"num": types.Number(44)}, types.NewSet(p))
|
||||
|
||||
expected := types.NewStruct("", types.StructData{"num": types.Number(44)})
|
||||
|
||||
output := "output"
|
||||
stdout, stderr, err := s.Run(main, []string{"merge", "--policy=r", s.LdbDir, left, right, output})
|
||||
if err == nil {
|
||||
s.Equal("", stderr)
|
||||
s.validateDataset(output, expected, l, r)
|
||||
} else {
|
||||
s.Fail("Run failed", "err: %v\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) TestNomsMerge_Conflict() {
|
||||
left, right := "left", "right"
|
||||
p := s.setupMergeDataset("parent", types.StructData{"num": types.Number(42)}, types.NewSet())
|
||||
s.setupMergeDataset(left, types.StructData{"num": types.Number(43)}, types.NewSet(p))
|
||||
s.setupMergeDataset(right, types.StructData{"num": types.Number(44)}, types.NewSet(p))
|
||||
|
||||
s.Panics(func() { s.MustRun(main, []string{"merge", s.LdbDir, left, right, "output"}) })
|
||||
}
|
||||
|
||||
func (s *nomsMergeTestSuite) TestBadInput() {
|
||||
sp := spec.CreateDatabaseSpecString("ldb", s.LdbDir)
|
||||
l, r, o := "left", "right", "output"
|
||||
type c struct {
|
||||
args []string
|
||||
err string
|
||||
}
|
||||
cases := []c{
|
||||
{[]string{"foo"}, "error: Incorrect number of arguments\n"},
|
||||
{[]string{"foo", "bar"}, "error: Incorrect number of arguments\n"},
|
||||
{[]string{"foo", "bar", "baz"}, "error: Incorrect number of arguments\n"},
|
||||
{[]string{"foo", "bar", "baz", "quux", "five"}, "error: Incorrect number of arguments\n"},
|
||||
{[]string{sp, l + "!!", r, o}, "error: Invalid dataset " + l + "!!, must match [a-zA-Z0-9\\-_/]+\n"},
|
||||
{[]string{sp, l + "2", r, o}, "error: Dataset " + l + "2 has no data\n"},
|
||||
{[]string{sp, l, r + "2", o}, "error: Dataset " + r + "2 has no data\n"},
|
||||
{[]string{sp, l, r, "!invalid"}, "error: Invalid dataset !invalid, must match [a-zA-Z0-9\\-_/]+\n"},
|
||||
}
|
||||
|
||||
db, _ := spec.GetDatabase(sp)
|
||||
prep := func(dsName string) {
|
||||
ds := db.GetDataset(dsName)
|
||||
db.CommitValue(ds, types.NewMap(types.String("foo"), types.String("bar")))
|
||||
}
|
||||
prep(l)
|
||||
prep(r)
|
||||
db.Close()
|
||||
|
||||
for _, c := range cases {
|
||||
stdout, stderr, err := s.Run(main, append([]string{"merge"}, c.args...))
|
||||
s.Empty(stdout, "Expected empty stdout for case: %#v", c.args)
|
||||
if !s.NotNil(err, "Unexpected success for case: %#v\n", c.args) {
|
||||
continue
|
||||
}
|
||||
if mainErr, ok := err.(clienttest.ExitError); ok {
|
||||
s.Equal(1, mainErr.Code)
|
||||
s.Equal(c.err, stderr, "Unexpected output for case: %#v\n", c.args)
|
||||
} else {
|
||||
s.Fail("Run() recovered non-error panic", "err: %#v\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNomsMergeCliResolve(t *testing.T) {
|
||||
type c struct {
|
||||
input string
|
||||
aChange, bChange types.DiffChangeType
|
||||
aVal, bVal types.Value
|
||||
expectedChange types.DiffChangeType
|
||||
expected types.Value
|
||||
success bool
|
||||
}
|
||||
|
||||
cases := []c{
|
||||
{"l\n", types.DiffChangeAdded, types.DiffChangeAdded, types.String("foo"), types.String("bar"), types.DiffChangeAdded, types.String("foo"), true},
|
||||
{"r\n", types.DiffChangeAdded, types.DiffChangeAdded, types.String("foo"), types.String("bar"), types.DiffChangeAdded, types.String("bar"), true},
|
||||
{"l\n", types.DiffChangeAdded, types.DiffChangeAdded, types.Number(7), types.String("bar"), types.DiffChangeAdded, types.Number(7), true},
|
||||
{"r\n", types.DiffChangeModified, types.DiffChangeModified, types.Number(7), types.String("bar"), types.DiffChangeModified, types.String("bar"), true},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
input := bytes.NewBufferString(c.input)
|
||||
|
||||
changeType, newVal, ok := cliResolve(input, ioutil.Discard, c.aChange, c.bChange, c.aVal, c.bVal, types.Path{})
|
||||
if !c.success {
|
||||
assert.False(t, ok)
|
||||
} else if assert.True(t, ok) {
|
||||
assert.Equal(t, c.expectedChange, changeType)
|
||||
assert.True(t, c.expected.Equals(newVal))
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user