diff --git a/go/store/cmd/noms/noms_merge.go b/go/store/cmd/noms/noms_merge.go index b0ceb304a6..645a7e4555 100644 --- a/go/store/cmd/noms/noms_merge.go +++ b/go/store/cmd/noms/noms_merge.go @@ -117,13 +117,9 @@ func runMerge(ctx context.Context, args []string) int { p, err := types.NewList(ctx, db, leftHeadRef, rightHeadRef) d.PanicIfError(err) - cm, err := datas.NewCommit(ctx, merged, p, types.EmptyStruct(db.Format())) - d.PanicIfError(err) - - ref, err := db.WriteValue(ctx, cm) - d.PanicIfError(err) - - _, err = db.SetHead(ctx, outDS, ref) + _, err = db.Commit(ctx, outDS, merged, datas.CommitOptions{ + ParentsList: p, + }) d.PanicIfError(err) status.Printf("Done") diff --git a/go/store/cmd/noms/noms_root_test.go b/go/store/cmd/noms/noms_root_test.go index c22df8e4ed..3d73a1113c 100644 --- a/go/store/cmd/noms/noms_root_test.go +++ b/go/store/cmd/noms/noms_root_test.go @@ -55,7 +55,7 @@ func (s *nomsRootTestSuite) TestBasic() { ds, _ = ds.Database().CommitValue(context.Background(), ds, types.String("goodbye")) c2, _ := s.MustRun(main, []string{"root", dbSpecStr}) - s.Equal("avdab61n1s1d1emdee7kb7e49quisr5n\n", c2) + s.Equal("cac1ilk2nnbk5vmdctlg9r5abj0m1u6f\n", c2) // TODO: Would be good to test successful --update too, but requires changes to MustRun to allow // input because of prompt :(. diff --git a/go/store/cmd/noms/noms_show_test.go b/go/store/cmd/noms/noms_show_test.go index 904850a847..e141e39e49 100644 --- a/go/store/cmd/noms/noms_show_test.go +++ b/go/store/cmd/noms/noms_show_test.go @@ -46,9 +46,29 @@ type nomsShowTestSuite struct { const ( res1 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n meta: struct {},\n parents: set {},\n parents_list: [],\n value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj,\n}" res2 = "String - \"test string\"" - res3 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n meta: struct {},\n parents: set {\n #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n },\n parents_list: [\n #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n ],\n value: #82adk7hfcudg8fktittm672to66t6qeu,\n}" + res3 = `Commit{meta Struct,parents Set,parents_closure Ref,parents_list List,value Ref} - struct Commit { + meta: struct {}, + parents: set { + #4u3mpdq0o8at437p37i5u94fk2frr4qm, + }, + parents_closure: #pr2umfcqukd4ltrgkpvsjrig7afb9ghg, + parents_list: [ + #4u3mpdq0o8at437p37i5u94fk2frr4qm, + ], + value: #t43ks6746hf0fcefv5e9v1c02k2i0jr9, +}` res4 = "List> - [\n \"elem1\",\n 2,\n \"elem3\",\n]" - res5 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n meta: struct {},\n parents: set {\n #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n },\n parents_list: [\n #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n ],\n value: #5cgfu2vk4nc21m1vjkjjpd2kvcm2df7q,\n}" + res5 = `Commit{meta Struct,parents Set,parents_closure Ref,parents_list List,value Ref} - struct Commit { + meta: struct {}, + parents: set { + #idcre7pv1p74mfmidiguol1pu6rmt0bu, + }, + parents_closure: #7pl4tlkc531difn1f32vlaqdve5g04p0, + parents_list: [ + #idcre7pv1p74mfmidiguol1pu6rmt0bu, + ], + value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj, +}` ) func (s *nomsShowTestSuite) spec(str string) spec.Spec { diff --git a/go/store/datas/commit.go b/go/store/datas/commit.go index d020bb6af7..d9e7607089 100644 --- a/go/store/datas/commit.go +++ b/go/store/datas/commit.go @@ -40,21 +40,44 @@ const ( // `"parents"` is still written as a Set as well, so that commits // created with newer versions of still usable by older versions. ParentsListField = "parents_list" - ValueField = "value" - CommitMetaField = "meta" - CommitName = "Commit" + // Added in October, 2021. Stores a Ref)>. + // The key of the map is a Tuple, reffable to + // a Commit ref. The value of the map is a List of Refs pointing + // to the parents of the Commit which corresponds to the key. + // + // This structure is a materialized closure of a commit's parents. It + // is used for pull/fetch/push commit graph fan-out and for sub-O(n) + // FindCommonAncestor calculations. + ParentsClosureField = "parents_closure" + ValueField = "value" + CommitMetaField = "meta" + CommitName = "Commit" ) -var commitTemplate = types.MakeStructTemplate(CommitName, []string{CommitMetaField, ParentsField, ParentsListField, ValueField}) +var commitTemplateWithParentsClosure = types.MakeStructTemplate(CommitName, []string{ + CommitMetaField, + ParentsField, + ParentsClosureField, + ParentsListField, + ValueField, +}) + +var commitTemplateWithoutParentsClosure = types.MakeStructTemplate(CommitName, []string{ + CommitMetaField, + ParentsField, + ParentsListField, + ValueField, +}) var valueCommitType = nomdl.MustParseType(`Struct Commit { meta: Struct {}, parents: Set>>, + parents_closure?: Ref, // Ref>, parents_list?: List>>, value: Value, }`) -// NewCommit creates a new commit object. +// newCommit creates a new commit object. // // A commit has the following type: // @@ -63,16 +86,21 @@ var valueCommitType = nomdl.MustParseType(`Struct Commit { // meta: M, // parents: Set>>, // parentsList: List>>, +// parentsClosure: Ref, // Map>>, // value: T, // } // ``` // where M is a struct type and T is any type. -func NewCommit(ctx context.Context, value types.Value, parentsList types.List, meta types.Struct) (types.Struct, error) { +func newCommit(ctx context.Context, value types.Value, parentsList types.List, parentsClosure types.Ref, includeParentsClosure bool, meta types.Struct) (types.Struct, error) { parentsSet, err := parentsList.ToSet(ctx) if err != nil { return types.EmptyStruct(meta.Format()), err } - return commitTemplate.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, value}) + if includeParentsClosure { + return commitTemplateWithParentsClosure.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsClosure, parentsList, value}) + } else { + return commitTemplateWithoutParentsClosure.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, value}) + } } // FindCommonAncestor returns the most recent common ancestor of c1 and c2, if @@ -244,25 +272,50 @@ func findCommonRef(a, b types.RefSlice) (types.Ref, bool) { return types.Ref{}, false } -func makeCommitStructType(metaType, parentsType, parentsListType, valueType *types.Type) (*types.Type, error) { - return types.MakeStructType(CommitName, - types.StructField{ - Name: CommitMetaField, - Type: metaType, - }, - types.StructField{ - Name: ParentsField, - Type: parentsType, - }, - types.StructField{ - Name: ParentsListField, - Type: parentsListType, - }, - types.StructField{ - Name: ValueField, - Type: valueType, - }, - ) +func makeCommitStructType(metaType, parentsType, parentsListType, parentsClosureType, valueType *types.Type, includeParentsClosure bool) (*types.Type, error) { + if includeParentsClosure { + return types.MakeStructType(CommitName, + types.StructField{ + Name: CommitMetaField, + Type: metaType, + }, + types.StructField{ + Name: ParentsField, + Type: parentsType, + }, + types.StructField{ + Name: ParentsListField, + Type: parentsListType, + }, + types.StructField{ + Name: ParentsClosureField, + Type: parentsClosureType, + }, + types.StructField{ + Name: ValueField, + Type: valueType, + }, + ) + } else { + return types.MakeStructType(CommitName, + types.StructField{ + Name: CommitMetaField, + Type: metaType, + }, + types.StructField{ + Name: ParentsField, + Type: parentsType, + }, + types.StructField{ + Name: ParentsListField, + Type: parentsListType, + }, + types.StructField{ + Name: ValueField, + Type: valueType, + }, + ) + } } func getRefElementType(t *types.Type) *types.Type { diff --git a/go/store/datas/commit_test.go b/go/store/datas/commit_test.go index 8e6a5a4df4..4989e8bcdd 100644 --- a/go/store/datas/commit_test.go +++ b/go/store/datas/commit_test.go @@ -27,9 +27,11 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/dolthub/dolt/go/store/chunks" "github.com/dolthub/dolt/go/store/d" + "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/nomdl" "github.com/dolthub/dolt/go/store/types" ) @@ -91,6 +93,20 @@ func mustList(l types.List, err error) types.List { return l } +func mustMap(m types.Map, err error) types.Map { + d.PanicIfError(err) + return m +} + +func mustParentsClosure(t *testing.T, exists bool) func(types.Ref, bool, error) types.Ref { + return func(r types.Ref, got bool, err error) types.Ref { + t.Helper() + require.NoError(t, err) + require.Equal(t, exists, got) + return r + } +} + func mustType(t *types.Type, err error) *types.Type { d.PanicIfError(err) return t @@ -106,12 +122,17 @@ func mustValue(val types.Value, err error) types.Value { return val } +func mustTuple(val types.Tuple, err error) types.Tuple { + d.PanicIfError(err) + return val +} + func TestNewCommit(t *testing.T) { assert := assert.New(t) assertTypeEquals := func(e, a *types.Type) { t.Helper() - assert.True(a.Equals(e), "Actual: %s\nExpected %s", mustString(a.Describe(context.Background())), mustString(e.Describe(context.Background()))) + assert.True(a.Equals(e), "Actual: %s\nExpected %s\n%s", mustString(a.Describe(context.Background())), mustString(e.Describe(context.Background())), a.HumanReadableString()) } storage := &chunks.TestStorage{} @@ -119,7 +140,8 @@ func TestNewCommit(t *testing.T) { defer db.Close() parents := mustList(types.NewList(context.Background(), db)) - commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18)) + parentsClosure := mustParentsClosure(t, false)(getParentsClosure(context.Background(), db, parents)) + commit, err := newCommit(context.Background(), types.Float(1), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at, err := types.TypeOf(commit) assert.NoError(err) @@ -127,39 +149,54 @@ func TestNewCommit(t *testing.T) { types.EmptyStructType, mustType(types.MakeSetType(mustType(types.MakeUnionType()))), mustType(types.MakeListType(mustType(types.MakeUnionType()))), + mustType(types.MakeRefType(types.PrimitiveTypeMap[types.ValueKind])), types.PrimitiveTypeMap[types.FloatKind], + false, ) assert.NoError(err) assertTypeEquals(et, at) + _, err = db.WriteValue(context.Background(), commit) + assert.NoError(err) + // Committing another Float parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit, types.Format_7_18)))) - commit2, err := NewCommit(context.Background(), types.Float(2), parents, types.EmptyStruct(types.Format_7_18)) + parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents)) + commit2, err := newCommit(context.Background(), types.Float(2), parents, parentsClosure, true, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at2, err := types.TypeOf(commit2) assert.NoError(err) et2 := nomdl.MustParseType(`Struct Commit { meta: Struct {}, parents: Set>>, + parents_closure?: Ref, parents_list: List>>, value: Float, }`) assertTypeEquals(et2, at2) + _, err = db.WriteValue(context.Background(), commit2) + assert.NoError(err) + // Now commit a String parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18)))) - commit3, err := NewCommit(context.Background(), types.String("Hi"), parents, types.EmptyStruct(types.Format_7_18)) + parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents)) + commit3, err := newCommit(context.Background(), types.String("Hi"), parents, parentsClosure, true, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at3, err := types.TypeOf(commit3) assert.NoError(err) et3 := nomdl.MustParseType(`Struct Commit { meta: Struct {}, parents: Set>>, + parents_closure?: Ref, parents_list: List>>, value: Float | String, }`) assertTypeEquals(et3, at3) + _, err = db.WriteValue(context.Background(), commit3) + assert.NoError(err) + // Now commit a String with MetaInfo meta, err := types.NewStruct(types.Format_7_18, "Meta", types.StructData{"date": types.String("some date"), "number": types.Float(9)}) assert.NoError(err) @@ -169,7 +206,8 @@ func TestNewCommit(t *testing.T) { }`) assertTypeEquals(metaType, mustType(types.TypeOf(meta))) parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18)))) - commit4, err := NewCommit(context.Background(), types.String("Hi"), parents, meta) + parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents)) + commit4, err := newCommit(context.Background(), types.String("Hi"), parents, parentsClosure, true, meta) assert.NoError(err) at4, err := types.TypeOf(commit4) assert.NoError(err) @@ -179,19 +217,26 @@ func TestNewCommit(t *testing.T) { number: Float, }, parents: Set>>, + parents_closure?: Ref, parents_list: List>>, value: Float | String, }`) assertTypeEquals(et4, at4) + _, err = db.WriteValue(context.Background(), commit4) + assert.NoError(err) + // Merge-commit with different parent types parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18)), mustRef(types.NewRef(commit3, types.Format_7_18)))) - commit5, err := NewCommit( + parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents)) + commit5, err := newCommit( context.Background(), types.String("Hi"), parents, + parentsClosure, + true, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at5, err := types.TypeOf(commit5) @@ -199,6 +244,7 @@ func TestNewCommit(t *testing.T) { et5 := nomdl.MustParseType(`Struct Commit { meta: Struct {}, parents: Set>>, + parents_closure?: Ref, parents_list: List>>, value: Float | String, }`) @@ -294,18 +340,203 @@ func assertCommonAncestor(t *testing.T, expected, a, b types.Struct, ldb, rdb Da } } -func TestFindCommonAncestor(t *testing.T) { +// Add a commit and return it. +func addCommit(t *testing.T, db Database, datasetID string, val string, parents ...types.Struct) (types.Struct, types.Ref) { + ds, err := db.GetDataset(context.Background(), datasetID) + assert.NoError(t, err) + ds, err = db.Commit(context.Background(), ds, types.String(val), CommitOptions{ParentsList: mustList(toRefList(db, parents...))}) + assert.NoError(t, err) + return mustHead(ds), mustHeadRef(ds) +} + +func assertClosureMapValue(t *testing.T, vrw types.ValueReadWriter, v types.Value, h hash.Hash) bool { + cv, err := vrw.ReadValue(context.Background(), h) + if !assert.NoError(t, err) { + return false + } + s, ok := cv.(types.Struct) + if !assert.True(t, ok) { + return false + } + plv, ok, err := s.MaybeGet(ParentsListField) + if !assert.NoError(t, err) { + return false + } + if !assert.True(t, ok) { + return false + } + pl, ok := plv.(types.List) + if !assert.True(t, ok) { + return false + } + gl, ok := v.(types.List) + if !assert.True(t, ok) { + return false + } + if !assert.Equal(t, pl.Len(), gl.Len()) { + return false + } + for i := 0; i < int(pl.Len()); i++ { + piv, err := pl.Get(context.Background(), uint64(i)) + if !assert.NoError(t, err) { + return false + } + giv, err := gl.Get(context.Background(), uint64(i)) + if !assert.NoError(t, err) { + return false + } + pir, ok := piv.(types.Ref) + if !assert.True(t, ok) { + return false + } + gir, ok := giv.(types.Ref) + if !assert.True(t, ok) { + return false + } + if !assert.Equal(t, pir.TargetHash(), gir.TargetHash()) { + return false + } + } + return true +} + +func TestCommitParentsClosure(t *testing.T) { assert := assert.New(t) - // Add a commit and return it - addCommit := func(db Database, datasetID string, val string, parents ...types.Struct) types.Struct { - ds, err := db.GetDataset(context.Background(), datasetID) - assert.NoError(err) - ds, err = db.Commit(context.Background(), ds, types.String(val), CommitOptions{ParentsList: mustList(toRefList(db, parents...))}) - assert.NoError(err) - return mustHead(ds) + storage := &chunks.TestStorage{} + db := NewDatabase(storage.NewView()) + + type expected struct { + height int + hash hash.Hash } + assertCommitParentsClosure := func(s types.Struct, es []expected) { + v, ok, err := s.MaybeGet(ParentsClosureField) + if !assert.NoError(err) { + return + } + if len(es) == 0 { + assert.False(ok, "must not find parents_closure field when its length is 0") + return + } + if !assert.True(ok, "must find parents_closure field in commit.") { + return + } + r, ok := v.(types.Ref) + if !assert.True(ok, "parents_closure field must contain a ref value.") { + return + } + tv, err := r.TargetValue(context.Background(), db) + if !assert.NoError(err, "getting target value of parents_closure field must not error") { + return + } + m, ok := tv.(types.Map) + if !assert.True(ok, "parents_closure ref target value must contain a map value.") { + return + } + if !assert.Equal(len(es), int(m.Len()), "expected length %v and got %v", len(es), m.Len()) { + return + } + i := 0 + err = m.IterAll(context.Background(), func(k, v types.Value) error { + j := i + i++ + kt, ok := k.(types.Tuple) + if !assert.True(ok, "key type must be Tuple") { + return nil + } + if !assert.Equal(2, int(kt.Len()), "key must have length 2") { + return nil + } + hv, err := kt.Get(0) + if !assert.NoError(err) { + return nil + } + h, ok := hv.(types.Uint) + if !assert.True(ok, "key first field must be Uint") { + return nil + } + if !assert.Equal(es[j].height, int(uint64(h))) { + return nil + } + hv, err = kt.Get(1) + if !assert.NoError(err) { + return nil + } + b, ok := hv.(types.InlineBlob) + if !assert.True(ok, "key second field must be InlineBlob") { + return nil + } + var fh hash.Hash + copy(fh[:], []byte(b)) + if !assert.Equal(es[j].hash, fh, "hash for idx %d did not match", j) { + return nil + } + assertClosureMapValue(t, db, v, fh) + return nil + }) + assert.NoError(err) + } + + a, b, c, d := "ds-a", "ds-b", "ds-c", "ds-d" + a1, a1r := addCommit(t, db, a, "a1") + a2, a2r := addCommit(t, db, a, "a2", a1) + a3, a3r := addCommit(t, db, a, "a3", a2) + + b1, b1r := addCommit(t, db, b, "b1", a1) + b2, b2r := addCommit(t, db, b, "b2", b1) + b3, b3r := addCommit(t, db, b, "b3", b2) + + c1, c1r := addCommit(t, db, c, "c1", a3, b3) + + d1, _ := addCommit(t, db, d, "d1", c1, b3) + + assertCommitParentsClosure(a1, []expected{}) + assertCommitParentsClosure(a2, []expected{ + {1, a1r.TargetHash()}, + }) + assertCommitParentsClosure(a3, []expected{ + {1, a1r.TargetHash()}, + {2, a2r.TargetHash()}, + }) + + assertCommitParentsClosure(b1, []expected{ + {1, a1r.TargetHash()}, + }) + assertCommitParentsClosure(b2, []expected{ + {1, a1r.TargetHash()}, + {2, b1r.TargetHash()}, + }) + assertCommitParentsClosure(b3, []expected{ + {1, a1r.TargetHash()}, + {2, b1r.TargetHash()}, + {3, b2r.TargetHash()}, + }) + + assertCommitParentsClosure(c1, []expected{ + {1, a1r.TargetHash()}, + {2, a2r.TargetHash()}, + {2, b1r.TargetHash()}, + {3, a3r.TargetHash()}, + {3, b2r.TargetHash()}, + {4, b3r.TargetHash()}, + }) + + assertCommitParentsClosure(d1, []expected{ + {1, a1r.TargetHash()}, + {2, a2r.TargetHash()}, + {2, b1r.TargetHash()}, + {3, a3r.TargetHash()}, + {3, b2r.TargetHash()}, + {4, b3r.TargetHash()}, + {5, c1r.TargetHash()}, + }) +} + +func TestFindCommonAncestor(t *testing.T) { + assert := assert.New(t) + storage := &chunks.TestStorage{} db := NewDatabase(storage.NewView()) @@ -325,19 +556,19 @@ func TestFindCommonAncestor(t *testing.T) { // ds-d: d1<-d2 // a, b, c, d := "ds-a", "ds-b", "ds-c", "ds-d" - a1 := addCommit(db, a, "a1") - d1 := addCommit(db, d, "d1") - a2 := addCommit(db, a, "a2", a1) - c2 := addCommit(db, c, "c2", a1) - d2 := addCommit(db, d, "d2", d1) - a3 := addCommit(db, a, "a3", a2) - b3 := addCommit(db, b, "b3", a2) - c3 := addCommit(db, c, "c3", c2, d2) - a4 := addCommit(db, a, "a4", a3) - b4 := addCommit(db, b, "b4", b3) - a5 := addCommit(db, a, "a5", a4) - b5 := addCommit(db, b, "b5", b4, a3) - a6 := addCommit(db, a, "a6", a5, b5) + a1, _ := addCommit(t, db, a, "a1") + d1, _ := addCommit(t, db, d, "d1") + a2, _ := addCommit(t, db, a, "a2", a1) + c2, _ := addCommit(t, db, c, "c2", a1) + d2, _ := addCommit(t, db, d, "d2", d1) + a3, _ := addCommit(t, db, a, "a3", a2) + b3, _ := addCommit(t, db, b, "b3", a2) + c3, _ := addCommit(t, db, c, "c3", c2, d2) + a4, _ := addCommit(t, db, a, "a4", a3) + b4, _ := addCommit(t, db, b, "b4", b3) + a5, _ := addCommit(t, db, a, "a5", a4) + b5, _ := addCommit(t, db, b, "b5", b4, a3) + a6, _ := addCommit(t, db, a, "a6", a5, b5) assertCommonAncestor(t, a1, a1, a1, db, db) // All self assertCommonAncestor(t, a1, a1, a2, db, db) // One side self @@ -377,43 +608,43 @@ func TestFindCommonAncestor(t *testing.T) { // Rerun the tests when using two difference Databases for left and // right commits. Both databases have all the previous commits. a, b, c, d = "ds-a", "ds-b", "ds-c", "ds-d" - a1 = addCommit(db, a, "a1") - d1 = addCommit(db, d, "d1") - a2 = addCommit(db, a, "a2", a1) - c2 = addCommit(db, c, "c2", a1) - d2 = addCommit(db, d, "d2", d1) - a3 = addCommit(db, a, "a3", a2) - b3 = addCommit(db, b, "b3", a2) - c3 = addCommit(db, c, "c3", c2, d2) - a4 = addCommit(db, a, "a4", a3) - b4 = addCommit(db, b, "b4", b3) - a5 = addCommit(db, a, "a5", a4) - b5 = addCommit(db, b, "b5", b4, a3) - a6 = addCommit(db, a, "a6", a5, b5) + a1, _ = addCommit(t, db, a, "a1") + d1, _ = addCommit(t, db, d, "d1") + a2, _ = addCommit(t, db, a, "a2", a1) + c2, _ = addCommit(t, db, c, "c2", a1) + d2, _ = addCommit(t, db, d, "d2", d1) + a3, _ = addCommit(t, db, a, "a3", a2) + b3, _ = addCommit(t, db, b, "b3", a2) + c3, _ = addCommit(t, db, c, "c3", c2, d2) + a4, _ = addCommit(t, db, a, "a4", a3) + b4, _ = addCommit(t, db, b, "b4", b3) + a5, _ = addCommit(t, db, a, "a5", a4) + b5, _ = addCommit(t, db, b, "b5", b4, a3) + a6, _ = addCommit(t, db, a, "a6", a5, b5) - addCommit(rdb, a, "a1") - addCommit(rdb, d, "d1") - addCommit(rdb, a, "a2", a1) - addCommit(rdb, c, "c2", a1) - addCommit(rdb, d, "d2", d1) - addCommit(rdb, a, "a3", a2) - addCommit(rdb, b, "b3", a2) - addCommit(rdb, c, "c3", c2, d2) - addCommit(rdb, a, "a4", a3) - addCommit(rdb, b, "b4", b3) - addCommit(rdb, a, "a5", a4) - addCommit(rdb, b, "b5", b4, a3) - addCommit(rdb, a, "a6", a5, b5) + addCommit(t, rdb, a, "a1") + addCommit(t, rdb, d, "d1") + addCommit(t, rdb, a, "a2", a1) + addCommit(t, rdb, c, "c2", a1) + addCommit(t, rdb, d, "d2", d1) + addCommit(t, rdb, a, "a3", a2) + addCommit(t, rdb, b, "b3", a2) + addCommit(t, rdb, c, "c3", c2, d2) + addCommit(t, rdb, a, "a4", a3) + addCommit(t, rdb, b, "b4", b3) + addCommit(t, rdb, a, "a5", a4) + addCommit(t, rdb, b, "b5", b4, a3) + addCommit(t, rdb, a, "a6", a5, b5) // Additionally, |db| has a6<-a7<-a8<-a9. // |rdb| has a6<-ra7<-ra8<-ra9. - a7 := addCommit(db, a, "a7", a6) - a8 := addCommit(db, a, "a8", a7) - a9 := addCommit(db, a, "a9", a8) + a7, _ := addCommit(t, db, a, "a7", a6) + a8, _ := addCommit(t, db, a, "a8", a7) + a9, _ := addCommit(t, db, a, "a9", a8) - ra7 := addCommit(rdb, a, "ra7", a6) - ra8 := addCommit(rdb, a, "ra8", ra7) - ra9 := addCommit(rdb, a, "ra9", ra8) + ra7, _ := addCommit(t, rdb, a, "ra7", a6) + ra8, _ := addCommit(t, rdb, a, "ra8", ra7) + ra9, _ := addCommit(t, rdb, a, "ra9", ra8) assertCommonAncestor(t, a1, a1, a1, db, rdb) // All self assertCommonAncestor(t, a1, a1, a2, db, rdb) // One side self @@ -433,20 +664,26 @@ func TestNewCommitRegressionTest(t *testing.T) { defer db.Close() parents := mustList(types.NewList(context.Background(), db)) - c1, err := NewCommit(context.Background(), types.String("one"), parents, types.EmptyStruct(types.Format_7_18)) + parentsClosure := mustParentsClosure(t, false)(getParentsClosure(context.Background(), db, parents)) + c1, err := newCommit(context.Background(), types.String("one"), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18)) assert.NoError(t, err) - cx, err := NewCommit(context.Background(), types.Bool(true), parents, types.EmptyStruct(types.Format_7_18)) + cx, err := newCommit(context.Background(), types.Bool(true), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18)) + assert.NoError(t, err) + _, err = db.WriteValue(context.Background(), c1) + assert.NoError(t, err) + _, err = db.WriteValue(context.Background(), cx) assert.NoError(t, err) value := types.String("two") parents, err = types.NewList(context.Background(), db, mustRef(types.NewRef(c1, types.Format_7_18))) assert.NoError(t, err) + parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents)) meta, err := types.NewStruct(types.Format_7_18, "", types.StructData{ "basis": cx, }) assert.NoError(t, err) // Used to fail - _, err = NewCommit(context.Background(), value, parents, meta) + _, err = newCommit(context.Background(), value, parents, parentsClosure, true, meta) assert.NoError(t, err) } @@ -454,6 +691,7 @@ func TestPersistedCommitConsts(t *testing.T) { // changing constants that are persisted requires a migration strategy assert.Equal(t, "parents", ParentsField) assert.Equal(t, "parents_list", ParentsListField) + assert.Equal(t, "parents_closure", ParentsClosureField) assert.Equal(t, "value", ValueField) assert.Equal(t, "meta", CommitMetaField) assert.Equal(t, "Commit", CommitName) diff --git a/go/store/datas/database_common.go b/go/store/datas/database_common.go index 19286817d2..f58fb395a0 100644 --- a/go/store/datas/database_common.go +++ b/go/store/datas/database_common.go @@ -138,6 +138,142 @@ func (db *database) DatasetsInRoot(ctx context.Context, rootHash hash.Hash) (typ return val.(types.Map), nil } +func getParentsClosure(ctx context.Context, vrw types.ValueReadWriter, parentRefsL types.List) (types.Ref, bool, error) { + parentRefs := make([]types.Ref, int(parentRefsL.Len())) + parents := make([]types.Struct, len(parentRefs)) + if len(parents) == 0 { + return types.Ref{}, false, nil + } + err := parentRefsL.IterAll(ctx, func(v types.Value, i uint64) error { + r, ok := v.(types.Ref) + if !ok { + return errors.New("parentsRef element was not a Ref") + } + parentRefs[int(i)] = r + tv, err := r.TargetValue(ctx, vrw) + if err != nil { + return err + } + s, ok := tv.(types.Struct) + if !ok { + return errors.New("parentRef target value was not a Struct") + } + parents[int(i)] = s + return nil + }) + if err != nil { + return types.Ref{}, false, err + } + parentMaps := make([]types.Map, len(parents)) + parentParentLists := make([]types.List, len(parents)) + for i, p := range parents { + v, ok, err := p.MaybeGet(ParentsClosureField) + if err != nil { + return types.Ref{}, false, err + } + if !ok || types.IsNull(v) { + empty, err := types.NewMap(ctx, vrw) + if err != nil { + return types.Ref{}, false, err + } + parentMaps[i] = empty + } else { + r, ok := v.(types.Ref) + if !ok { + return types.Ref{}, false, errors.New("unexpected field value type for parents_closure in commit struct") + } + tv, err := r.TargetValue(ctx, vrw) + if err != nil { + return types.Ref{}, false, err + } + parentMaps[i], ok = tv.(types.Map) + if !ok { + return types.Ref{}, false, fmt.Errorf("unexpected target value type for parents_closure in commit struct: %v", tv) + } + } + v, ok, err = p.MaybeGet(ParentsListField) + if !ok || types.IsNull(v) { + empty, err := types.NewList(ctx, vrw) + if err != nil { + return types.Ref{}, false, err + } + parentParentLists[i] = empty + } else { + parentParentLists[i], ok = v.(types.List) + if !ok { + return types.Ref{}, false, errors.New("unexpected field value or type for parents_list in commit struct") + } + } + if parentMaps[i].Len() == 0 && parentParentLists[i].Len() != 0 { + // If one of the commits has an empty parents_closure, but non-empty parents, we will not record + // a parents_closure here. + return types.Ref{}, false, nil + } + } + // Convert parent lists to List> + for i, l := range parentParentLists { + newRefs := make([]types.Value, int(l.Len())) + err := l.IterAll(ctx, func(v types.Value, i uint64) error { + r, ok := v.(types.Ref) + if !ok { + return errors.New("unexpected entry type for parents_list in commit struct") + } + newRefs[int(i)], err = types.ToRefOfValue(r, vrw.Format()) + if err != nil { + return err + } + return nil + }) + if err != nil { + return types.Ref{}, false, err + } + parentParentLists[i], err = types.NewList(ctx, vrw, newRefs...) + if err != nil { + return types.Ref{}, false, err + } + } + editor := parentMaps[0].Edit() + for i, r := range parentRefs { + h := r.TargetHash() + key, err := types.NewTuple(vrw.Format(), types.Uint(r.Height()), types.InlineBlob(h[:])) + if err != nil { + editor.Close(ctx) + return types.Ref{}, false, err + } + editor.Set(key, parentParentLists[i]) + } + for i := 1; i < len(parentMaps); i++ { + changes := make(chan types.ValueChanged) + var derr error + go func() { + defer close(changes) + derr = parentMaps[1].Diff(ctx, parentMaps[0], changes) + }() + for c := range changes { + if c.ChangeType == types.DiffChangeAdded { + editor.Set(c.Key, c.NewValue) + } + } + if derr != nil { + editor.Close(ctx) + return types.Ref{}, false, derr + } + } + m, err := editor.Map(ctx) + if err != nil { + return types.Ref{}, false, err + } + r, err := vrw.WriteValue(ctx, m) + if err != nil { + return types.Ref{}, false, err + } + r, err = types.ToRefOfValue(r, vrw.Format()) + if err != nil { + return types.Ref{}, false, err + } + return r, true, nil +} + // Datasets returns the Map of Datasets in the current root. If you intend to edit the map and commit changes back, // then you should fetch the current root, then call DatasetsInRoot with that hash. Otherwise another writer could // change the root value between when you get the root hash and call this method. @@ -318,7 +454,12 @@ func (db *database) CommitDangling(ctx context.Context, v types.Value, opts Comm opts.Meta = types.EmptyStruct(db.Format()) } - commitStruct, err := NewCommit(ctx, v, opts.ParentsList, opts.Meta) + parentsClosure, includeParentsClosure, err := getParentsClosure(ctx, db, opts.ParentsList) + if err != nil { + return types.Struct{}, err + } + + commitStruct, err := newCommit(ctx, v, opts.ParentsList, parentsClosure, includeParentsClosure, opts.Meta) if err != nil { return types.Struct{}, err } @@ -473,7 +614,12 @@ func (db *database) doMerge( return types.Ref{}, err } - newCom, err := NewCommit(ctx, merged, parents, types.EmptyStruct(db.Format())) + parentsClosure, includeParentsClosure, err := getParentsClosure(ctx, db, parents) + if err != nil { + return types.Ref{}, err + } + + newCom, err := newCommit(ctx, merged, parents, parentsClosure, includeParentsClosure, types.EmptyStruct(db.Format())) if err != nil { return types.Ref{}, err } @@ -980,7 +1126,13 @@ func buildNewCommit(ctx context.Context, ds Dataset, v types.Value, opts CommitO if meta.IsZeroValue() { meta = types.EmptyStruct(ds.Database().Format()) } - return NewCommit(ctx, v, parents, meta) + + parentsClosure, includeParentsClosure, err := getParentsClosure(ctx, ds.Database(), parents) + if err != nil { + return types.EmptyStruct(ds.Database().Format()), err + } + + return newCommit(ctx, v, parents, parentsClosure, includeParentsClosure, meta) } func (db *database) doHeadUpdate(ctx context.Context, ds Dataset, updateFunc func(ds Dataset) error) (Dataset, error) { diff --git a/go/store/datas/puller_test.go b/go/store/datas/puller_test.go index ac74d0a79e..5daecbe1c4 100644 --- a/go/store/datas/puller_test.go +++ b/go/store/datas/puller_test.go @@ -32,14 +32,6 @@ import ( "github.com/dolthub/dolt/go/store/util/clienttest" ) -func mustTuple(tpl types.Tuple, err error) types.Tuple { - if err != nil { - panic(err) - } - - return tpl -} - func addTableValues(ctx context.Context, vrw types.ValueReadWriter, m types.Map, tableName string, alternatingKeyVals ...types.Value) (types.Map, error) { val, ok, err := m.MaybeGet(ctx, types.String(tableName)) diff --git a/go/store/datas/tag_test.go b/go/store/datas/tag_test.go index 049a90ae99..48f00fe5f0 100644 --- a/go/store/datas/tag_test.go +++ b/go/store/datas/tag_test.go @@ -38,7 +38,8 @@ func TestNewTag(t *testing.T) { defer db.Close() parents := mustList(types.NewList(context.Background(), db)) - commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18)) + parentsClosure := mustParentsClosure(t, false)(getParentsClosure(context.Background(), db, parents)) + commit, err := newCommit(context.Background(), types.Float(1), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18)) require.NoError(t, err) cmRef, err := types.NewRef(commit, types.Format_7_18) @@ -50,7 +51,9 @@ func TestNewTag(t *testing.T) { types.EmptyStructType, mustType(types.MakeSetType(mustType(types.MakeUnionType()))), mustType(types.MakeListType(mustType(types.MakeUnionType()))), + mustType(types.MakeRefType(types.PrimitiveTypeMap[types.ValueKind])), types.PrimitiveTypeMap[types.FloatKind], + false, ) require.NoError(t, err) et, err := makeTagStructType( diff --git a/go/store/marshal/decode_test.go b/go/store/marshal/decode_test.go index bba7373863..9b5a6411d7 100644 --- a/go/store/marshal/decode_test.go +++ b/go/store/marshal/decode_test.go @@ -343,7 +343,7 @@ func TestDecodeTypeMismatch(t *testing.T) { assertDecodeErrorMessage(t, types.Float(42), &b, "Cannot unmarshal from: Float to: bool details: ") var blob types.Blob - assertDecodeErrorMessage(t, mustValue(types.NewList(context.Background(), vs)), &blob, "Cannot unmarshal from: List<> to: types.Blob details: ") + assertDecodeErrorMessage(t, mustValue(types.NewList(context.Background(), vs)), &blob, "Cannot unmarshal from: List> to: types.Blob details: ") type S struct { X int diff --git a/go/store/merge/three_way_keyval_test.go b/go/store/merge/three_way_keyval_test.go index b80cb48538..5347dcde3a 100644 --- a/go/store/merge/three_way_keyval_test.go +++ b/go/store/merge/three_way_keyval_test.go @@ -238,7 +238,7 @@ func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NilConflict() { } func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_ImmediateConflict() { - s.tryThreeWayConflict(mustValue(types.NewSet(context.Background(), s.vs)), mustValue(s.create(mm2b)), mustValue(s.create(mm2)), "Cannot merge Set<> with "+s.typeStr) + s.tryThreeWayConflict(mustValue(types.NewSet(context.Background(), s.vs)), mustValue(s.create(mm2b)), mustValue(s.create(mm2)), "Cannot merge Set> with "+s.typeStr) s.tryThreeWayConflict(mustValue(s.create(mm2b)), mustValue(types.NewSet(context.Background(), s.vs)), mustValue(s.create(mm2)), "Cannot merge "+s.typeStr) } diff --git a/go/store/merge/three_way_set_test.go b/go/store/merge/three_way_set_test.go index e229fe3bcb..140215e76b 100644 --- a/go/store/merge/three_way_set_test.go +++ b/go/store/merge/three_way_set_test.go @@ -105,6 +105,6 @@ func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_Refs() { } func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_ImmediateConflict() { - s.tryThreeWayConflict(mustValue(types.NewMap(context.Background(), s.vs)), mustValue(s.create(ss1b)), mustValue(s.create(ss1)), "Cannot merge Map<> with "+s.typeStr) + s.tryThreeWayConflict(mustValue(types.NewMap(context.Background(), s.vs)), mustValue(s.create(ss1b)), mustValue(s.create(ss1)), "Cannot merge Map, Union<>> with "+s.typeStr) s.tryThreeWayConflict(mustValue(s.create(ss1b)), mustValue(types.NewMap(context.Background(), s.vs)), mustValue(s.create(ss1)), "Cannot merge "+s.typeStr) } diff --git a/go/store/nomdl/parser.go b/go/store/nomdl/parser.go index 42a7ea2d61..d2b868a8cb 100644 --- a/go/store/nomdl/parser.go +++ b/go/store/nomdl/parser.go @@ -212,6 +212,12 @@ func (p *Parser) parseSingleTypeWithToken(tok rune, tokenText string) (*types.Ty return types.PrimitiveTypeMap[types.TypeKind], nil case "Value": return types.PrimitiveTypeMap[types.ValueKind], nil + case "Tuple": + f := types.Format_Default + if p.vrw != nil { + f = p.vrw.Format() + } + return types.TypeOf(types.EmptyTuple(f)) case "Struct": return p.parseStructType() case "Map": @@ -352,6 +358,7 @@ func (p *Parser) parseMapType() (*types.Type, error) { // Set // Map // Struct +// Tuple // // Bool : // `true` diff --git a/go/store/types/encode_human_readable.go b/go/store/types/encode_human_readable.go index c92b848df3..467b3d98c2 100644 --- a/go/store/types/encode_human_readable.go +++ b/go/store/types/encode_human_readable.go @@ -255,7 +255,6 @@ func (w *hrsWriter) Write(ctx context.Context, v Value) error { return err } - w.outdent() w.write(")") case MapKind: @@ -445,11 +444,6 @@ func (w *hrsWriter) writeType(t *Type, seenStructs map[*Type]struct{}) { w.write(t.TargetKind().String()) w.write("<") for i, et := range t.Desc.(CompoundDesc).ElemTypes { - if et.TargetKind() == UnionKind && len(et.Desc.(CompoundDesc).ElemTypes) == 0 { - // If one of the element types is an empty union all the other element types must - // also be empty union types. - break - } if i != 0 { w.write(", ") } @@ -460,6 +454,9 @@ func (w *hrsWriter) writeType(t *Type, seenStructs map[*Type]struct{}) { } w.write(">") case UnionKind: + if len(t.Desc.(CompoundDesc).ElemTypes) == 0 { + w.write("Union<>") + } for i, et := range t.Desc.(CompoundDesc).ElemTypes { if i != 0 { w.write(" | ") diff --git a/go/store/types/encode_human_readable_test.go b/go/store/types/encode_human_readable_test.go index 800d82cf5c..f1dc0ac16d 100644 --- a/go/store/types/encode_human_readable_test.go +++ b/go/store/types/encode_human_readable_test.go @@ -259,11 +259,11 @@ func TestWriteHumanReadableType(t *testing.T) { assertWriteHRSEqual(t, "Map", mustType(MakeMapType(PrimitiveTypeMap[FloatKind], PrimitiveTypeMap[StringKind]))) assertWriteHRSEqual(t, "Float | String", mustType(MakeUnionType(PrimitiveTypeMap[FloatKind], PrimitiveTypeMap[StringKind]))) assertWriteHRSEqual(t, "Bool", mustType(MakeUnionType(PrimitiveTypeMap[BoolKind]))) - assertWriteHRSEqual(t, "", mustType(MakeUnionType())) + assertWriteHRSEqual(t, "Union<>", mustType(MakeUnionType())) assertWriteHRSEqual(t, "List", mustType(MakeListType(mustType(MakeUnionType(PrimitiveTypeMap[FloatKind], PrimitiveTypeMap[StringKind]))))) assertWriteHRSEqual(t, "List", mustType(MakeListType(mustType(MakeUnionType(PrimitiveTypeMap[IntKind], PrimitiveTypeMap[UintKind]))))) assertWriteHRSEqual(t, "List", mustType(MakeListType(mustType(MakeUnionType(PrimitiveTypeMap[IntKind], PrimitiveTypeMap[NullKind]))))) - assertWriteHRSEqual(t, "List<>", mustType(MakeListType(mustType(MakeUnionType())))) + assertWriteHRSEqual(t, "List>", mustType(MakeListType(mustType(MakeUnionType())))) } func TestRecursiveStruct(t *testing.T) { diff --git a/go/store/types/struct.go b/go/store/types/struct.go index 326673c7f2..951318d5c9 100644 --- a/go/store/types/struct.go +++ b/go/store/types/struct.go @@ -250,7 +250,7 @@ func readStructTypeOfValue(nbf *NomsBinFormat, dec *valueDecoder) (*Type, error) t, err := dec.readTypeOfValue(nbf) if err != nil { - return nil, err + return nil, fmt.Errorf("error decoding type of field %s: %w", fname, err) } typeFields[i] = StructField{