Merge pull request #2235 from dolthub/aaron/commit-parents-closure

go/store/datas: Materialize a parents closure map at commit time.

go/store/cmd/noms/noms_merge.go

@@ -117,13 +117,9 @@ func runMerge(ctx context.Context, args []string) int {
 	p, err := types.NewList(ctx, db, leftHeadRef, rightHeadRef)
 	d.PanicIfError(err)
 
-	cm, err := datas.NewCommit(ctx, merged, p, types.EmptyStruct(db.Format()))
-	d.PanicIfError(err)
-
-	ref, err := db.WriteValue(ctx, cm)
-	d.PanicIfError(err)
-
-	_, err = db.SetHead(ctx, outDS, ref)
+	_, err = db.Commit(ctx, outDS, merged, datas.CommitOptions{
+		ParentsList: p,
+	})
 	d.PanicIfError(err)
 
 	status.Printf("Done")

go/store/cmd/noms/noms_root_test.go

@@ -55,7 +55,7 @@ func (s *nomsRootTestSuite) TestBasic() {
 
 	ds, _ = ds.Database().CommitValue(context.Background(), ds, types.String("goodbye"))
 	c2, _ := s.MustRun(main, []string{"root", dbSpecStr})
-	s.Equal("avdab61n1s1d1emdee7kb7e49quisr5n\n", c2)
+	s.Equal("cac1ilk2nnbk5vmdctlg9r5abj0m1u6f\n", c2)
 
 	// TODO: Would be good to test successful --update too, but requires changes to MustRun to allow
 	// input because of prompt :(.

go/store/cmd/noms/noms_show_test.go

@@ -46,9 +46,29 @@ type nomsShowTestSuite struct {
 const (
 	res1 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {},\n  parents_list: [],\n  value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj,\n}"
 	res2 = "String - \"test string\""
-	res3 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {\n    #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n  },\n  parents_list: [\n    #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n  ],\n  value: #82adk7hfcudg8fktittm672to66t6qeu,\n}"
+	res3 = `Commit{meta Struct,parents Set,parents_closure Ref,parents_list List,value Ref} - struct Commit {
+  meta: struct {},
+  parents: set {
+    #4u3mpdq0o8at437p37i5u94fk2frr4qm,
+  },
+  parents_closure: #pr2umfcqukd4ltrgkpvsjrig7afb9ghg,
+  parents_list: [
+    #4u3mpdq0o8at437p37i5u94fk2frr4qm,
+  ],
+  value: #t43ks6746hf0fcefv5e9v1c02k2i0jr9,
+}`
 	res4 = "List<Union<Float,String>> - [\n  \"elem1\",\n  2,\n  \"elem3\",\n]"
-	res5 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {\n    #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n  },\n  parents_list: [\n    #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n  ],\n  value: #5cgfu2vk4nc21m1vjkjjpd2kvcm2df7q,\n}"
+	res5 = `Commit{meta Struct,parents Set,parents_closure Ref,parents_list List,value Ref} - struct Commit {
+  meta: struct {},
+  parents: set {
+    #idcre7pv1p74mfmidiguol1pu6rmt0bu,
+  },
+  parents_closure: #7pl4tlkc531difn1f32vlaqdve5g04p0,
+  parents_list: [
+    #idcre7pv1p74mfmidiguol1pu6rmt0bu,
+  ],
+  value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj,
+}`
 )
 
 func (s *nomsShowTestSuite) spec(str string) spec.Spec {

go/store/datas/commit.go

@@ -40,21 +40,44 @@ const (
 	// `"parents"` is still written as a Set as well, so that commits
 	// created with newer versions of still usable by older versions.
 	ParentsListField = "parents_list"
-	ValueField       = "value"
-	CommitMetaField  = "meta"
-	CommitName       = "Commit"
+	// Added in October, 2021. Stores a Ref<Value(Map<Tuple,List>)>.
+	// The key of the map is a Tuple<Height, InlineBlob(Hash)>, reffable to
+	// a Commit ref. The value of the map is a List of Ref<Value>s pointing
+	// to the parents of the Commit which corresponds to the key.
+	//
+	// This structure is a materialized closure of a commit's parents. It
+	// is used for pull/fetch/push commit graph fan-out and for sub-O(n)
+	// FindCommonAncestor calculations.
+	ParentsClosureField = "parents_closure"
+	ValueField          = "value"
+	CommitMetaField     = "meta"
+	CommitName          = "Commit"
 )
 
-var commitTemplate = types.MakeStructTemplate(CommitName, []string{CommitMetaField, ParentsField, ParentsListField, ValueField})
+var commitTemplateWithParentsClosure = types.MakeStructTemplate(CommitName, []string{
+	CommitMetaField,
+	ParentsField,
+	ParentsClosureField,
+	ParentsListField,
+	ValueField,
+})
+
+var commitTemplateWithoutParentsClosure = types.MakeStructTemplate(CommitName, []string{
+	CommitMetaField,
+	ParentsField,
+	ParentsListField,
+	ValueField,
+})
 
 var valueCommitType = nomdl.MustParseType(`Struct Commit {
         meta: Struct {},
         parents: Set<Ref<Cycle<Commit>>>,
+        parents_closure?: Ref<Value>, // Ref<Map<Value,Value>>,
         parents_list?: List<Ref<Cycle<Commit>>>,
         value: Value,
 }`)
 
-// NewCommit creates a new commit object.
+// newCommit creates a new commit object.
 //
 // A commit has the following type:
 //
@@ -63,16 +86,21 @@ var valueCommitType = nomdl.MustParseType(`Struct Commit {
 //   meta: M,
 //   parents: Set<Ref<Cycle<Commit>>>,
 //   parentsList: List<Ref<Cycle<Commit>>>,
+//   parentsClosure: Ref<Value>, // Map<Tuple,List<Ref<Value>>>,
 //   value: T,
 // }
 // ```
 // where M is a struct type and T is any type.
-func NewCommit(ctx context.Context, value types.Value, parentsList types.List, meta types.Struct) (types.Struct, error) {
+func newCommit(ctx context.Context, value types.Value, parentsList types.List, parentsClosure types.Ref, includeParentsClosure bool, meta types.Struct) (types.Struct, error) {
 	parentsSet, err := parentsList.ToSet(ctx)
 	if err != nil {
 		return types.EmptyStruct(meta.Format()), err
 	}
-	return commitTemplate.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, value})
+	if includeParentsClosure {
+		return commitTemplateWithParentsClosure.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsClosure, parentsList, value})
+	} else {
+		return commitTemplateWithoutParentsClosure.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, value})
+	}
 }
 
 // FindCommonAncestor returns the most recent common ancestor of c1 and c2, if
@@ -244,25 +272,50 @@ func findCommonRef(a, b types.RefSlice) (types.Ref, bool) {
 	return types.Ref{}, false
 }
 
-func makeCommitStructType(metaType, parentsType, parentsListType, valueType *types.Type) (*types.Type, error) {
-	return types.MakeStructType(CommitName,
-		types.StructField{
-			Name: CommitMetaField,
-			Type: metaType,
-		},
-		types.StructField{
-			Name: ParentsField,
-			Type: parentsType,
-		},
-		types.StructField{
-			Name: ParentsListField,
-			Type: parentsListType,
-		},
-		types.StructField{
-			Name: ValueField,
-			Type: valueType,
-		},
-	)
+func makeCommitStructType(metaType, parentsType, parentsListType, parentsClosureType, valueType *types.Type, includeParentsClosure bool) (*types.Type, error) {
+	if includeParentsClosure {
+		return types.MakeStructType(CommitName,
+			types.StructField{
+				Name: CommitMetaField,
+				Type: metaType,
+			},
+			types.StructField{
+				Name: ParentsField,
+				Type: parentsType,
+			},
+			types.StructField{
+				Name: ParentsListField,
+				Type: parentsListType,
+			},
+			types.StructField{
+				Name: ParentsClosureField,
+				Type: parentsClosureType,
+			},
+			types.StructField{
+				Name: ValueField,
+				Type: valueType,
+			},
+		)
+	} else {
+		return types.MakeStructType(CommitName,
+			types.StructField{
+				Name: CommitMetaField,
+				Type: metaType,
+			},
+			types.StructField{
+				Name: ParentsField,
+				Type: parentsType,
+			},
+			types.StructField{
+				Name: ParentsListField,
+				Type: parentsListType,
+			},
+			types.StructField{
+				Name: ValueField,
+				Type: valueType,
+			},
+		)
+	}
 }
 
 func getRefElementType(t *types.Type) *types.Type {

go/store/datas/commit_test.go

@@ -27,9 +27,11 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/d"
+	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/nomdl"
 	"github.com/dolthub/dolt/go/store/types"
 )
@@ -91,6 +93,20 @@ func mustList(l types.List, err error) types.List {
 	return l
 }
 
+func mustMap(m types.Map, err error) types.Map {
+	d.PanicIfError(err)
+	return m
+}
+
+func mustParentsClosure(t *testing.T, exists bool) func(types.Ref, bool, error) types.Ref {
+	return func(r types.Ref, got bool, err error) types.Ref {
+		t.Helper()
+		require.NoError(t, err)
+		require.Equal(t, exists, got)
+		return r
+	}
+}
+
 func mustType(t *types.Type, err error) *types.Type {
 	d.PanicIfError(err)
 	return t
@@ -106,12 +122,17 @@ func mustValue(val types.Value, err error) types.Value {
 	return val
 }
 
+func mustTuple(val types.Tuple, err error) types.Tuple {
+	d.PanicIfError(err)
+	return val
+}
+
 func TestNewCommit(t *testing.T) {
 	assert := assert.New(t)
 
 	assertTypeEquals := func(e, a *types.Type) {
 		t.Helper()
-		assert.True(a.Equals(e), "Actual: %s\nExpected %s", mustString(a.Describe(context.Background())), mustString(e.Describe(context.Background())))
+		assert.True(a.Equals(e), "Actual: %s\nExpected %s\n%s", mustString(a.Describe(context.Background())), mustString(e.Describe(context.Background())), a.HumanReadableString())
 	}
 
 	storage := &chunks.TestStorage{}
@@ -119,7 +140,8 @@ func TestNewCommit(t *testing.T) {
 	defer db.Close()
 
 	parents := mustList(types.NewList(context.Background(), db))
-	commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18))
+	parentsClosure := mustParentsClosure(t, false)(getParentsClosure(context.Background(), db, parents))
+	commit, err := newCommit(context.Background(), types.Float(1), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at, err := types.TypeOf(commit)
 	assert.NoError(err)
@@ -127,39 +149,54 @@ func TestNewCommit(t *testing.T) {
 		types.EmptyStructType,
 		mustType(types.MakeSetType(mustType(types.MakeUnionType()))),
 		mustType(types.MakeListType(mustType(types.MakeUnionType()))),
+		mustType(types.MakeRefType(types.PrimitiveTypeMap[types.ValueKind])),
 		types.PrimitiveTypeMap[types.FloatKind],
+		false,
 	)
 	assert.NoError(err)
 	assertTypeEquals(et, at)
 
+	_, err = db.WriteValue(context.Background(), commit)
+	assert.NoError(err)
+
 	// Committing another Float
 	parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit, types.Format_7_18))))
-	commit2, err := NewCommit(context.Background(), types.Float(2), parents, types.EmptyStruct(types.Format_7_18))
+	parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents))
+	commit2, err := newCommit(context.Background(), types.Float(2), parents, parentsClosure, true, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at2, err := types.TypeOf(commit2)
 	assert.NoError(err)
 	et2 := nomdl.MustParseType(`Struct Commit {
                 meta: Struct {},
                 parents: Set<Ref<Cycle<Commit>>>,
+                parents_closure?: Ref<Value>,
                 parents_list: List<Ref<Cycle<Commit>>>,
                 value: Float,
         }`)
 	assertTypeEquals(et2, at2)
 
+	_, err = db.WriteValue(context.Background(), commit2)
+	assert.NoError(err)
+
 	// Now commit a String
 	parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18))))
-	commit3, err := NewCommit(context.Background(), types.String("Hi"), parents, types.EmptyStruct(types.Format_7_18))
+	parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents))
+	commit3, err := newCommit(context.Background(), types.String("Hi"), parents, parentsClosure, true, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at3, err := types.TypeOf(commit3)
 	assert.NoError(err)
 	et3 := nomdl.MustParseType(`Struct Commit {
                 meta: Struct {},
                 parents: Set<Ref<Cycle<Commit>>>,
+                parents_closure?: Ref<Value>,
                 parents_list: List<Ref<Cycle<Commit>>>,
                 value: Float | String,
         }`)
 	assertTypeEquals(et3, at3)
 
+	_, err = db.WriteValue(context.Background(), commit3)
+	assert.NoError(err)
+
 	// Now commit a String with MetaInfo
 	meta, err := types.NewStruct(types.Format_7_18, "Meta", types.StructData{"date": types.String("some date"), "number": types.Float(9)})
 	assert.NoError(err)
@@ -169,7 +206,8 @@ func TestNewCommit(t *testing.T) {
 	}`)
 	assertTypeEquals(metaType, mustType(types.TypeOf(meta)))
 	parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18))))
-	commit4, err := NewCommit(context.Background(), types.String("Hi"), parents, meta)
+	parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents))
+	commit4, err := newCommit(context.Background(), types.String("Hi"), parents, parentsClosure, true, meta)
 	assert.NoError(err)
 	at4, err := types.TypeOf(commit4)
 	assert.NoError(err)
@@ -179,19 +217,26 @@ func TestNewCommit(t *testing.T) {
                         number: Float,
         	},
                 parents: Set<Ref<Cycle<Commit>>>,
+                parents_closure?: Ref<Value>,
                 parents_list: List<Ref<Cycle<Commit>>>,
                 value: Float | String,
         }`)
 	assertTypeEquals(et4, at4)
 
+	_, err = db.WriteValue(context.Background(), commit4)
+	assert.NoError(err)
+
 	// Merge-commit with different parent types
 	parents = mustList(types.NewList(context.Background(), db,
 		mustRef(types.NewRef(commit2, types.Format_7_18)),
 		mustRef(types.NewRef(commit3, types.Format_7_18))))
-	commit5, err := NewCommit(
+	parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents))
+	commit5, err := newCommit(
 		context.Background(),
 		types.String("Hi"),
 		parents,
+		parentsClosure,
+		true,
 		types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at5, err := types.TypeOf(commit5)
@@ -199,6 +244,7 @@ func TestNewCommit(t *testing.T) {
 	et5 := nomdl.MustParseType(`Struct Commit {
                 meta: Struct {},
                 parents: Set<Ref<Cycle<Commit>>>,
+                parents_closure?: Ref<Value>,
                 parents_list: List<Ref<Cycle<Commit>>>,
                 value: Float | String,
         }`)
@@ -294,18 +340,203 @@ func assertCommonAncestor(t *testing.T, expected, a, b types.Struct, ldb, rdb Da
 	}
 }
 
-func TestFindCommonAncestor(t *testing.T) {
+// Add a commit and return it.
+func addCommit(t *testing.T, db Database, datasetID string, val string, parents ...types.Struct) (types.Struct, types.Ref) {
+	ds, err := db.GetDataset(context.Background(), datasetID)
+	assert.NoError(t, err)
+	ds, err = db.Commit(context.Background(), ds, types.String(val), CommitOptions{ParentsList: mustList(toRefList(db, parents...))})
+	assert.NoError(t, err)
+	return mustHead(ds), mustHeadRef(ds)
+}
+
+func assertClosureMapValue(t *testing.T, vrw types.ValueReadWriter, v types.Value, h hash.Hash) bool {
+	cv, err := vrw.ReadValue(context.Background(), h)
+	if !assert.NoError(t, err) {
+		return false
+	}
+	s, ok := cv.(types.Struct)
+	if !assert.True(t, ok) {
+		return false
+	}
+	plv, ok, err := s.MaybeGet(ParentsListField)
+	if !assert.NoError(t, err) {
+		return false
+	}
+	if !assert.True(t, ok) {
+		return false
+	}
+	pl, ok := plv.(types.List)
+	if !assert.True(t, ok) {
+		return false
+	}
+	gl, ok := v.(types.List)
+	if !assert.True(t, ok) {
+		return false
+	}
+	if !assert.Equal(t, pl.Len(), gl.Len()) {
+		return false
+	}
+	for i := 0; i < int(pl.Len()); i++ {
+		piv, err := pl.Get(context.Background(), uint64(i))
+		if !assert.NoError(t, err) {
+			return false
+		}
+		giv, err := gl.Get(context.Background(), uint64(i))
+		if !assert.NoError(t, err) {
+			return false
+		}
+		pir, ok := piv.(types.Ref)
+		if !assert.True(t, ok) {
+			return false
+		}
+		gir, ok := giv.(types.Ref)
+		if !assert.True(t, ok) {
+			return false
+		}
+		if !assert.Equal(t, pir.TargetHash(), gir.TargetHash()) {
+			return false
+		}
+	}
+	return true
+}
+
+func TestCommitParentsClosure(t *testing.T) {
 	assert := assert.New(t)
 
-	// Add a commit and return it
-	addCommit := func(db Database, datasetID string, val string, parents ...types.Struct) types.Struct {
-		ds, err := db.GetDataset(context.Background(), datasetID)
-		assert.NoError(err)
-		ds, err = db.Commit(context.Background(), ds, types.String(val), CommitOptions{ParentsList: mustList(toRefList(db, parents...))})
-		assert.NoError(err)
-		return mustHead(ds)
+	storage := &chunks.TestStorage{}
+	db := NewDatabase(storage.NewView())
+
+	type expected struct {
+		height int
+		hash   hash.Hash
 	}
 
+	assertCommitParentsClosure := func(s types.Struct, es []expected) {
+		v, ok, err := s.MaybeGet(ParentsClosureField)
+		if !assert.NoError(err) {
+			return
+		}
+		if len(es) == 0 {
+			assert.False(ok, "must not find parents_closure field when its length is 0")
+			return
+		}
+		if !assert.True(ok, "must find parents_closure field in commit.") {
+			return
+		}
+		r, ok := v.(types.Ref)
+		if !assert.True(ok, "parents_closure field must contain a ref value.") {
+			return
+		}
+		tv, err := r.TargetValue(context.Background(), db)
+		if !assert.NoError(err, "getting target value of parents_closure field must not error") {
+			return
+		}
+		m, ok := tv.(types.Map)
+		if !assert.True(ok, "parents_closure ref target value must contain a map value.") {
+			return
+		}
+		if !assert.Equal(len(es), int(m.Len()), "expected length %v and got %v", len(es), m.Len()) {
+			return
+		}
+		i := 0
+		err = m.IterAll(context.Background(), func(k, v types.Value) error {
+			j := i
+			i++
+			kt, ok := k.(types.Tuple)
+			if !assert.True(ok, "key type must be Tuple") {
+				return nil
+			}
+			if !assert.Equal(2, int(kt.Len()), "key must have length 2") {
+				return nil
+			}
+			hv, err := kt.Get(0)
+			if !assert.NoError(err) {
+				return nil
+			}
+			h, ok := hv.(types.Uint)
+			if !assert.True(ok, "key first field must be Uint") {
+				return nil
+			}
+			if !assert.Equal(es[j].height, int(uint64(h))) {
+				return nil
+			}
+			hv, err = kt.Get(1)
+			if !assert.NoError(err) {
+				return nil
+			}
+			b, ok := hv.(types.InlineBlob)
+			if !assert.True(ok, "key second field must be InlineBlob") {
+				return nil
+			}
+			var fh hash.Hash
+			copy(fh[:], []byte(b))
+			if !assert.Equal(es[j].hash, fh, "hash for idx %d did not match", j) {
+				return nil
+			}
+			assertClosureMapValue(t, db, v, fh)
+			return nil
+		})
+		assert.NoError(err)
+	}
+
+	a, b, c, d := "ds-a", "ds-b", "ds-c", "ds-d"
+	a1, a1r := addCommit(t, db, a, "a1")
+	a2, a2r := addCommit(t, db, a, "a2", a1)
+	a3, a3r := addCommit(t, db, a, "a3", a2)
+
+	b1, b1r := addCommit(t, db, b, "b1", a1)
+	b2, b2r := addCommit(t, db, b, "b2", b1)
+	b3, b3r := addCommit(t, db, b, "b3", b2)
+
+	c1, c1r := addCommit(t, db, c, "c1", a3, b3)
+
+	d1, _ := addCommit(t, db, d, "d1", c1, b3)
+
+	assertCommitParentsClosure(a1, []expected{})
+	assertCommitParentsClosure(a2, []expected{
+		{1, a1r.TargetHash()},
+	})
+	assertCommitParentsClosure(a3, []expected{
+		{1, a1r.TargetHash()},
+		{2, a2r.TargetHash()},
+	})
+
+	assertCommitParentsClosure(b1, []expected{
+		{1, a1r.TargetHash()},
+	})
+	assertCommitParentsClosure(b2, []expected{
+		{1, a1r.TargetHash()},
+		{2, b1r.TargetHash()},
+	})
+	assertCommitParentsClosure(b3, []expected{
+		{1, a1r.TargetHash()},
+		{2, b1r.TargetHash()},
+		{3, b2r.TargetHash()},
+	})
+
+	assertCommitParentsClosure(c1, []expected{
+		{1, a1r.TargetHash()},
+		{2, a2r.TargetHash()},
+		{2, b1r.TargetHash()},
+		{3, a3r.TargetHash()},
+		{3, b2r.TargetHash()},
+		{4, b3r.TargetHash()},
+	})
+
+	assertCommitParentsClosure(d1, []expected{
+		{1, a1r.TargetHash()},
+		{2, a2r.TargetHash()},
+		{2, b1r.TargetHash()},
+		{3, a3r.TargetHash()},
+		{3, b2r.TargetHash()},
+		{4, b3r.TargetHash()},
+		{5, c1r.TargetHash()},
+	})
+}
+
+func TestFindCommonAncestor(t *testing.T) {
+	assert := assert.New(t)
+
 	storage := &chunks.TestStorage{}
 	db := NewDatabase(storage.NewView())
 
@@ -325,19 +556,19 @@ func TestFindCommonAncestor(t *testing.T) {
 	// ds-d: d1<-d2
 	//
 	a, b, c, d := "ds-a", "ds-b", "ds-c", "ds-d"
-	a1 := addCommit(db, a, "a1")
-	d1 := addCommit(db, d, "d1")
-	a2 := addCommit(db, a, "a2", a1)
-	c2 := addCommit(db, c, "c2", a1)
-	d2 := addCommit(db, d, "d2", d1)
-	a3 := addCommit(db, a, "a3", a2)
-	b3 := addCommit(db, b, "b3", a2)
-	c3 := addCommit(db, c, "c3", c2, d2)
-	a4 := addCommit(db, a, "a4", a3)
-	b4 := addCommit(db, b, "b4", b3)
-	a5 := addCommit(db, a, "a5", a4)
-	b5 := addCommit(db, b, "b5", b4, a3)
-	a6 := addCommit(db, a, "a6", a5, b5)
+	a1, _ := addCommit(t, db, a, "a1")
+	d1, _ := addCommit(t, db, d, "d1")
+	a2, _ := addCommit(t, db, a, "a2", a1)
+	c2, _ := addCommit(t, db, c, "c2", a1)
+	d2, _ := addCommit(t, db, d, "d2", d1)
+	a3, _ := addCommit(t, db, a, "a3", a2)
+	b3, _ := addCommit(t, db, b, "b3", a2)
+	c3, _ := addCommit(t, db, c, "c3", c2, d2)
+	a4, _ := addCommit(t, db, a, "a4", a3)
+	b4, _ := addCommit(t, db, b, "b4", b3)
+	a5, _ := addCommit(t, db, a, "a5", a4)
+	b5, _ := addCommit(t, db, b, "b5", b4, a3)
+	a6, _ := addCommit(t, db, a, "a6", a5, b5)
 
 	assertCommonAncestor(t, a1, a1, a1, db, db) // All self
 	assertCommonAncestor(t, a1, a1, a2, db, db) // One side self
@@ -377,43 +608,43 @@ func TestFindCommonAncestor(t *testing.T) {
 	// Rerun the tests when using two difference Databases for left and
 	// right commits. Both databases have all the previous commits.
 	a, b, c, d = "ds-a", "ds-b", "ds-c", "ds-d"
-	a1 = addCommit(db, a, "a1")
-	d1 = addCommit(db, d, "d1")
-	a2 = addCommit(db, a, "a2", a1)
-	c2 = addCommit(db, c, "c2", a1)
-	d2 = addCommit(db, d, "d2", d1)
-	a3 = addCommit(db, a, "a3", a2)
-	b3 = addCommit(db, b, "b3", a2)
-	c3 = addCommit(db, c, "c3", c2, d2)
-	a4 = addCommit(db, a, "a4", a3)
-	b4 = addCommit(db, b, "b4", b3)
-	a5 = addCommit(db, a, "a5", a4)
-	b5 = addCommit(db, b, "b5", b4, a3)
-	a6 = addCommit(db, a, "a6", a5, b5)
+	a1, _ = addCommit(t, db, a, "a1")
+	d1, _ = addCommit(t, db, d, "d1")
+	a2, _ = addCommit(t, db, a, "a2", a1)
+	c2, _ = addCommit(t, db, c, "c2", a1)
+	d2, _ = addCommit(t, db, d, "d2", d1)
+	a3, _ = addCommit(t, db, a, "a3", a2)
+	b3, _ = addCommit(t, db, b, "b3", a2)
+	c3, _ = addCommit(t, db, c, "c3", c2, d2)
+	a4, _ = addCommit(t, db, a, "a4", a3)
+	b4, _ = addCommit(t, db, b, "b4", b3)
+	a5, _ = addCommit(t, db, a, "a5", a4)
+	b5, _ = addCommit(t, db, b, "b5", b4, a3)
+	a6, _ = addCommit(t, db, a, "a6", a5, b5)
 
-	addCommit(rdb, a, "a1")
-	addCommit(rdb, d, "d1")
-	addCommit(rdb, a, "a2", a1)
-	addCommit(rdb, c, "c2", a1)
-	addCommit(rdb, d, "d2", d1)
-	addCommit(rdb, a, "a3", a2)
-	addCommit(rdb, b, "b3", a2)
-	addCommit(rdb, c, "c3", c2, d2)
-	addCommit(rdb, a, "a4", a3)
-	addCommit(rdb, b, "b4", b3)
-	addCommit(rdb, a, "a5", a4)
-	addCommit(rdb, b, "b5", b4, a3)
-	addCommit(rdb, a, "a6", a5, b5)
+	addCommit(t, rdb, a, "a1")
+	addCommit(t, rdb, d, "d1")
+	addCommit(t, rdb, a, "a2", a1)
+	addCommit(t, rdb, c, "c2", a1)
+	addCommit(t, rdb, d, "d2", d1)
+	addCommit(t, rdb, a, "a3", a2)
+	addCommit(t, rdb, b, "b3", a2)
+	addCommit(t, rdb, c, "c3", c2, d2)
+	addCommit(t, rdb, a, "a4", a3)
+	addCommit(t, rdb, b, "b4", b3)
+	addCommit(t, rdb, a, "a5", a4)
+	addCommit(t, rdb, b, "b5", b4, a3)
+	addCommit(t, rdb, a, "a6", a5, b5)
 
 	// Additionally, |db| has a6<-a7<-a8<-a9.
 	// |rdb| has a6<-ra7<-ra8<-ra9.
-	a7 := addCommit(db, a, "a7", a6)
-	a8 := addCommit(db, a, "a8", a7)
-	a9 := addCommit(db, a, "a9", a8)
+	a7, _ := addCommit(t, db, a, "a7", a6)
+	a8, _ := addCommit(t, db, a, "a8", a7)
+	a9, _ := addCommit(t, db, a, "a9", a8)
 
-	ra7 := addCommit(rdb, a, "ra7", a6)
-	ra8 := addCommit(rdb, a, "ra8", ra7)
-	ra9 := addCommit(rdb, a, "ra9", ra8)
+	ra7, _ := addCommit(t, rdb, a, "ra7", a6)
+	ra8, _ := addCommit(t, rdb, a, "ra8", ra7)
+	ra9, _ := addCommit(t, rdb, a, "ra9", ra8)
 
 	assertCommonAncestor(t, a1, a1, a1, db, rdb) // All self
 	assertCommonAncestor(t, a1, a1, a2, db, rdb) // One side self
@@ -433,20 +664,26 @@ func TestNewCommitRegressionTest(t *testing.T) {
 	defer db.Close()
 
 	parents := mustList(types.NewList(context.Background(), db))
-	c1, err := NewCommit(context.Background(), types.String("one"), parents, types.EmptyStruct(types.Format_7_18))
+	parentsClosure := mustParentsClosure(t, false)(getParentsClosure(context.Background(), db, parents))
+	c1, err := newCommit(context.Background(), types.String("one"), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(t, err)
-	cx, err := NewCommit(context.Background(), types.Bool(true), parents, types.EmptyStruct(types.Format_7_18))
+	cx, err := newCommit(context.Background(), types.Bool(true), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18))
+	assert.NoError(t, err)
+	_, err = db.WriteValue(context.Background(), c1)
+	assert.NoError(t, err)
+	_, err = db.WriteValue(context.Background(), cx)
 	assert.NoError(t, err)
 	value := types.String("two")
 	parents, err = types.NewList(context.Background(), db, mustRef(types.NewRef(c1, types.Format_7_18)))
 	assert.NoError(t, err)
+	parentsClosure = mustParentsClosure(t, true)(getParentsClosure(context.Background(), db, parents))
 	meta, err := types.NewStruct(types.Format_7_18, "", types.StructData{
 		"basis": cx,
 	})
 	assert.NoError(t, err)
 
 	// Used to fail
-	_, err = NewCommit(context.Background(), value, parents, meta)
+	_, err = newCommit(context.Background(), value, parents, parentsClosure, true, meta)
 	assert.NoError(t, err)
 }
 
@@ -454,6 +691,7 @@ func TestPersistedCommitConsts(t *testing.T) {
 	// changing constants that are persisted requires a migration strategy
 	assert.Equal(t, "parents", ParentsField)
 	assert.Equal(t, "parents_list", ParentsListField)
+	assert.Equal(t, "parents_closure", ParentsClosureField)
 	assert.Equal(t, "value", ValueField)
 	assert.Equal(t, "meta", CommitMetaField)
 	assert.Equal(t, "Commit", CommitName)

go/store/datas/database_common.go

@@ -138,6 +138,142 @@ func (db *database) DatasetsInRoot(ctx context.Context, rootHash hash.Hash) (typ
 	return val.(types.Map), nil
 }
 
+func getParentsClosure(ctx context.Context, vrw types.ValueReadWriter, parentRefsL types.List) (types.Ref, bool, error) {
+	parentRefs := make([]types.Ref, int(parentRefsL.Len()))
+	parents := make([]types.Struct, len(parentRefs))
+	if len(parents) == 0 {
+		return types.Ref{}, false, nil
+	}
+	err := parentRefsL.IterAll(ctx, func(v types.Value, i uint64) error {
+		r, ok := v.(types.Ref)
+		if !ok {
+			return errors.New("parentsRef element was not a Ref")
+		}
+		parentRefs[int(i)] = r
+		tv, err := r.TargetValue(ctx, vrw)
+		if err != nil {
+			return err
+		}
+		s, ok := tv.(types.Struct)
+		if !ok {
+			return errors.New("parentRef target value was not a Struct")
+		}
+		parents[int(i)] = s
+		return nil
+	})
+	if err != nil {
+		return types.Ref{}, false, err
+	}
+	parentMaps := make([]types.Map, len(parents))
+	parentParentLists := make([]types.List, len(parents))
+	for i, p := range parents {
+		v, ok, err := p.MaybeGet(ParentsClosureField)
+		if err != nil {
+			return types.Ref{}, false, err
+		}
+		if !ok || types.IsNull(v) {
+			empty, err := types.NewMap(ctx, vrw)
+			if err != nil {
+				return types.Ref{}, false, err
+			}
+			parentMaps[i] = empty
+		} else {
+			r, ok := v.(types.Ref)
+			if !ok {
+				return types.Ref{}, false, errors.New("unexpected field value type for parents_closure in commit struct")
+			}
+			tv, err := r.TargetValue(ctx, vrw)
+			if err != nil {
+				return types.Ref{}, false, err
+			}
+			parentMaps[i], ok = tv.(types.Map)
+			if !ok {
+				return types.Ref{}, false, fmt.Errorf("unexpected target value type for parents_closure in commit struct: %v", tv)
+			}
+		}
+		v, ok, err = p.MaybeGet(ParentsListField)
+		if !ok || types.IsNull(v) {
+			empty, err := types.NewList(ctx, vrw)
+			if err != nil {
+				return types.Ref{}, false, err
+			}
+			parentParentLists[i] = empty
+		} else {
+			parentParentLists[i], ok = v.(types.List)
+			if !ok {
+				return types.Ref{}, false, errors.New("unexpected field value or type for parents_list in commit struct")
+			}
+		}
+		if parentMaps[i].Len() == 0 && parentParentLists[i].Len() != 0 {
+			// If one of the commits has an empty parents_closure, but non-empty parents, we will not record
+			// a parents_closure here.
+			return types.Ref{}, false, nil
+		}
+	}
+	// Convert parent lists to List<Ref<Value>>
+	for i, l := range parentParentLists {
+		newRefs := make([]types.Value, int(l.Len()))
+		err := l.IterAll(ctx, func(v types.Value, i uint64) error {
+			r, ok := v.(types.Ref)
+			if !ok {
+				return errors.New("unexpected entry type for parents_list in commit struct")
+			}
+			newRefs[int(i)], err = types.ToRefOfValue(r, vrw.Format())
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+		if err != nil {
+			return types.Ref{}, false, err
+		}
+		parentParentLists[i], err = types.NewList(ctx, vrw, newRefs...)
+		if err != nil {
+			return types.Ref{}, false, err
+		}
+	}
+	editor := parentMaps[0].Edit()
+	for i, r := range parentRefs {
+		h := r.TargetHash()
+		key, err := types.NewTuple(vrw.Format(), types.Uint(r.Height()), types.InlineBlob(h[:]))
+		if err != nil {
+			editor.Close(ctx)
+			return types.Ref{}, false, err
+		}
+		editor.Set(key, parentParentLists[i])
+	}
+	for i := 1; i < len(parentMaps); i++ {
+		changes := make(chan types.ValueChanged)
+		var derr error
+		go func() {
+			defer close(changes)
+			derr = parentMaps[1].Diff(ctx, parentMaps[0], changes)
+		}()
+		for c := range changes {
+			if c.ChangeType == types.DiffChangeAdded {
+				editor.Set(c.Key, c.NewValue)
+			}
+		}
+		if derr != nil {
+			editor.Close(ctx)
+			return types.Ref{}, false, derr
+		}
+	}
+	m, err := editor.Map(ctx)
+	if err != nil {
+		return types.Ref{}, false, err
+	}
+	r, err := vrw.WriteValue(ctx, m)
+	if err != nil {
+		return types.Ref{}, false, err
+	}
+	r, err = types.ToRefOfValue(r, vrw.Format())
+	if err != nil {
+		return types.Ref{}, false, err
+	}
+	return r, true, nil
+}
+
 // Datasets returns the Map of Datasets in the current root. If you intend to edit the map and commit changes back,
 // then you should fetch the current root, then call DatasetsInRoot with that hash. Otherwise another writer could
 // change the root value between when you get the root hash and call this method.
@@ -318,7 +454,12 @@ func (db *database) CommitDangling(ctx context.Context, v types.Value, opts Comm
 		opts.Meta = types.EmptyStruct(db.Format())
 	}
 
-	commitStruct, err := NewCommit(ctx, v, opts.ParentsList, opts.Meta)
+	parentsClosure, includeParentsClosure, err := getParentsClosure(ctx, db, opts.ParentsList)
+	if err != nil {
+		return types.Struct{}, err
+	}
+
+	commitStruct, err := newCommit(ctx, v, opts.ParentsList, parentsClosure, includeParentsClosure, opts.Meta)
 	if err != nil {
 		return types.Struct{}, err
 	}
@@ -473,7 +614,12 @@ func (db *database) doMerge(
 		return types.Ref{}, err
 	}
 
-	newCom, err := NewCommit(ctx, merged, parents, types.EmptyStruct(db.Format()))
+	parentsClosure, includeParentsClosure, err := getParentsClosure(ctx, db, parents)
+	if err != nil {
+		return types.Ref{}, err
+	}
+
+	newCom, err := newCommit(ctx, merged, parents, parentsClosure, includeParentsClosure, types.EmptyStruct(db.Format()))
 	if err != nil {
 		return types.Ref{}, err
 	}
@@ -980,7 +1126,13 @@ func buildNewCommit(ctx context.Context, ds Dataset, v types.Value, opts CommitO
 	if meta.IsZeroValue() {
 		meta = types.EmptyStruct(ds.Database().Format())
 	}
-	return NewCommit(ctx, v, parents, meta)
+
+	parentsClosure, includeParentsClosure, err := getParentsClosure(ctx, ds.Database(), parents)
+	if err != nil {
+		return types.EmptyStruct(ds.Database().Format()), err
+	}
+
+	return newCommit(ctx, v, parents, parentsClosure, includeParentsClosure, meta)
 }
 
 func (db *database) doHeadUpdate(ctx context.Context, ds Dataset, updateFunc func(ds Dataset) error) (Dataset, error) {

go/store/datas/puller_test.go

@@ -32,14 +32,6 @@ import (
 	"github.com/dolthub/dolt/go/store/util/clienttest"
 )
 
-func mustTuple(tpl types.Tuple, err error) types.Tuple {
-	if err != nil {
-		panic(err)
-	}
-
-	return tpl
-}
-
 func addTableValues(ctx context.Context, vrw types.ValueReadWriter, m types.Map, tableName string, alternatingKeyVals ...types.Value) (types.Map, error) {
 	val, ok, err := m.MaybeGet(ctx, types.String(tableName))
 

go/store/datas/tag_test.go

@@ -38,7 +38,8 @@ func TestNewTag(t *testing.T) {
 	defer db.Close()
 
 	parents := mustList(types.NewList(context.Background(), db))
-	commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18))
+	parentsClosure := mustParentsClosure(t, false)(getParentsClosure(context.Background(), db, parents))
+	commit, err := newCommit(context.Background(), types.Float(1), parents, parentsClosure, false, types.EmptyStruct(types.Format_7_18))
 	require.NoError(t, err)
 
 	cmRef, err := types.NewRef(commit, types.Format_7_18)
@@ -50,7 +51,9 @@ func TestNewTag(t *testing.T) {
 		types.EmptyStructType,
 		mustType(types.MakeSetType(mustType(types.MakeUnionType()))),
 		mustType(types.MakeListType(mustType(types.MakeUnionType()))),
+		mustType(types.MakeRefType(types.PrimitiveTypeMap[types.ValueKind])),
 		types.PrimitiveTypeMap[types.FloatKind],
+		false,
 	)
 	require.NoError(t, err)
 	et, err := makeTagStructType(

go/store/marshal/decode_test.go

@@ -343,7 +343,7 @@ func TestDecodeTypeMismatch(t *testing.T) {
 	assertDecodeErrorMessage(t, types.Float(42), &b, "Cannot unmarshal from: Float to: bool details: ")
 
 	var blob types.Blob
-	assertDecodeErrorMessage(t, mustValue(types.NewList(context.Background(), vs)), &blob, "Cannot unmarshal from: List<> to: types.Blob details: ")
+	assertDecodeErrorMessage(t, mustValue(types.NewList(context.Background(), vs)), &blob, "Cannot unmarshal from: List<Union<>> to: types.Blob details: ")
 
 	type S struct {
 		X int

go/store/merge/three_way_keyval_test.go

@@ -238,7 +238,7 @@ func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NilConflict() {
 }
 
 func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_ImmediateConflict() {
-	s.tryThreeWayConflict(mustValue(types.NewSet(context.Background(), s.vs)), mustValue(s.create(mm2b)), mustValue(s.create(mm2)), "Cannot merge Set<> with "+s.typeStr)
+	s.tryThreeWayConflict(mustValue(types.NewSet(context.Background(), s.vs)), mustValue(s.create(mm2b)), mustValue(s.create(mm2)), "Cannot merge Set<Union<>> with "+s.typeStr)
 	s.tryThreeWayConflict(mustValue(s.create(mm2b)), mustValue(types.NewSet(context.Background(), s.vs)), mustValue(s.create(mm2)), "Cannot merge "+s.typeStr)
 }
 

go/store/merge/three_way_set_test.go

@@ -105,6 +105,6 @@ func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_Refs() {
 }
 
 func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_ImmediateConflict() {
-	s.tryThreeWayConflict(mustValue(types.NewMap(context.Background(), s.vs)), mustValue(s.create(ss1b)), mustValue(s.create(ss1)), "Cannot merge Map<> with "+s.typeStr)
+	s.tryThreeWayConflict(mustValue(types.NewMap(context.Background(), s.vs)), mustValue(s.create(ss1b)), mustValue(s.create(ss1)), "Cannot merge Map<Union<>, Union<>> with "+s.typeStr)
 	s.tryThreeWayConflict(mustValue(s.create(ss1b)), mustValue(types.NewMap(context.Background(), s.vs)), mustValue(s.create(ss1)), "Cannot merge "+s.typeStr)
 }

go/store/nomdl/parser.go

@@ -212,6 +212,12 @@ func (p *Parser) parseSingleTypeWithToken(tok rune, tokenText string) (*types.Ty
 		return types.PrimitiveTypeMap[types.TypeKind], nil
 	case "Value":
 		return types.PrimitiveTypeMap[types.ValueKind], nil
+	case "Tuple":
+		f := types.Format_Default
+		if p.vrw != nil {
+			f = p.vrw.Format()
+		}
+		return types.TypeOf(types.EmptyTuple(f))
 	case "Struct":
 		return p.parseStructType()
 	case "Map":
@@ -352,6 +358,7 @@ func (p *Parser) parseMapType() (*types.Type, error) {
 //   Set
 //   Map
 //   Struct
+//   Tuple
 //
 // Bool :
 //   `true`

go/store/types/encode_human_readable.go

@@ -255,7 +255,6 @@ func (w *hrsWriter) Write(ctx context.Context, v Value) error {
 			return err
 		}
 
-		w.outdent()
 		w.write(")")
 
 	case MapKind:
@@ -445,11 +444,6 @@ func (w *hrsWriter) writeType(t *Type, seenStructs map[*Type]struct{}) {
 		w.write(t.TargetKind().String())
 		w.write("<")
 		for i, et := range t.Desc.(CompoundDesc).ElemTypes {
-			if et.TargetKind() == UnionKind && len(et.Desc.(CompoundDesc).ElemTypes) == 0 {
-				// If one of the element types is an empty union all the other element types must
-				// also be empty union types.
-				break
-			}
 			if i != 0 {
 				w.write(", ")
 			}
@@ -460,6 +454,9 @@ func (w *hrsWriter) writeType(t *Type, seenStructs map[*Type]struct{}) {
 		}
 		w.write(">")
 	case UnionKind:
+		if len(t.Desc.(CompoundDesc).ElemTypes) == 0 {
+			w.write("Union<>")
+		}
 		for i, et := range t.Desc.(CompoundDesc).ElemTypes {
 			if i != 0 {
 				w.write(" | ")

go/store/types/encode_human_readable_test.go

@@ -259,11 +259,11 @@ func TestWriteHumanReadableType(t *testing.T) {
 	assertWriteHRSEqual(t, "Map<Float, String>", mustType(MakeMapType(PrimitiveTypeMap[FloatKind], PrimitiveTypeMap[StringKind])))
 	assertWriteHRSEqual(t, "Float | String", mustType(MakeUnionType(PrimitiveTypeMap[FloatKind], PrimitiveTypeMap[StringKind])))
 	assertWriteHRSEqual(t, "Bool", mustType(MakeUnionType(PrimitiveTypeMap[BoolKind])))
-	assertWriteHRSEqual(t, "", mustType(MakeUnionType()))
+	assertWriteHRSEqual(t, "Union<>", mustType(MakeUnionType()))
 	assertWriteHRSEqual(t, "List<Float | String>", mustType(MakeListType(mustType(MakeUnionType(PrimitiveTypeMap[FloatKind], PrimitiveTypeMap[StringKind])))))
 	assertWriteHRSEqual(t, "List<Int | Uint>", mustType(MakeListType(mustType(MakeUnionType(PrimitiveTypeMap[IntKind], PrimitiveTypeMap[UintKind])))))
 	assertWriteHRSEqual(t, "List<Int | Null>", mustType(MakeListType(mustType(MakeUnionType(PrimitiveTypeMap[IntKind], PrimitiveTypeMap[NullKind])))))
-	assertWriteHRSEqual(t, "List<>", mustType(MakeListType(mustType(MakeUnionType()))))
+	assertWriteHRSEqual(t, "List<Union<>>", mustType(MakeListType(mustType(MakeUnionType()))))
 }
 
 func TestRecursiveStruct(t *testing.T) {

go/store/types/struct.go

@@ -250,7 +250,7 @@ func readStructTypeOfValue(nbf *NomsBinFormat, dec *valueDecoder) (*Type, error)
 		t, err := dec.readTypeOfValue(nbf)
 
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error decoding type of field %s: %w", fname, err)
 		}
 
 		typeFields[i] = StructField{