store/datas: NewCommit -> newCommit, start threading parentsSkipList parameter through.

store/datas: Add ability to store parents_skip_list field to Commit Struct.

go/libraries/doltcore/sqle/sqlselect_test.go

@@ -733,7 +733,7 @@ var BasicSelectTests = []SelectTest{
 		Query: "select * from dolt_log",
 		ExpectedRows: []sql.Row{
 			{
-				"so275enkvulb96mkckbun1kjo9seg7c9",
+				"oqd9k78tffc3hji0vbo88v6ffr6cp6td",
 				"billy bob",
 				"bigbillieb@fake.horse",
 				time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC).In(LoadedLocalLocation()),
@@ -763,7 +763,7 @@ var BasicSelectTests = []SelectTest{
 		ExpectedRows: []sql.Row{
 			{
 				env.DefaultInitBranch,
-				"so275enkvulb96mkckbun1kjo9seg7c9",
+				"oqd9k78tffc3hji0vbo88v6ffr6cp6td",
 				"billy bob", "bigbillieb@fake.horse",
 				time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC).In(LoadedLocalLocation()),
 				"Initialize data repository",

go/store/cmd/noms/noms_ds_test.go

@@ -92,7 +92,7 @@ func (s *nomsDsTestSuite) TestNomsDs() {
 
 	// delete one dataset, print message at delete
 	rtnVal, _ = s.MustRun(main, []string{"ds", "-d", datasetName})
-	s.Equal("Deleted "+datasetName+" (was #7jrps2q0ubq0phnha37gd6051m8uqq6b)\n", rtnVal)
+	s.Equal("Deleted "+datasetName+" (was #3qqamcm2ur9htnmd692v4m4ck3ktgb64)\n", rtnVal)
 
 	// print datasets, just one left
 	rtnVal, _ = s.MustRun(main, []string{"ds", dbSpec})
@@ -100,7 +100,7 @@ func (s *nomsDsTestSuite) TestNomsDs() {
 
 	// delete the second dataset
 	rtnVal, _ = s.MustRun(main, []string{"ds", "-d", dataset2Name})
-	s.Equal("Deleted "+dataset2Name+" (was #hiqpb7kk36vo80uc36virlnrddru2o7h)\n", rtnVal)
+	s.Equal("Deleted "+dataset2Name+" (was #3b9t0n2432d4c3a8lhq8e5qsr13kd75a)\n", rtnVal)
 
 	// print datasets, none left
 	rtnVal, _ = s.MustRun(main, []string{"ds", dbSpec})

go/store/cmd/noms/noms_merge.go

@@ -117,13 +117,9 @@ func runMerge(ctx context.Context, args []string) int {
 	p, err := types.NewList(ctx, db, leftHeadRef, rightHeadRef)
 	d.PanicIfError(err)
 
-	cm, err := datas.NewCommit(ctx, merged, p, types.EmptyStruct(db.Format()))
-	d.PanicIfError(err)
-
-	ref, err := db.WriteValue(ctx, cm)
-	d.PanicIfError(err)
-
-	_, err = db.SetHead(ctx, outDS, ref)
+	_, err = db.Commit(ctx, outDS, merged, datas.CommitOptions{
+		ParentsList: p,
+	})
 	d.PanicIfError(err)
 
 	status.Printf("Done")

go/store/cmd/noms/noms_root_test.go

@@ -51,11 +51,11 @@ func (s *nomsRootTestSuite) TestBasic() {
 	dbSpecStr := spec.CreateDatabaseSpecString("nbs", s.DBDir)
 	ds, _ = ds.Database().CommitValue(context.Background(), ds, types.String("hello!"))
 	c1, _ := s.MustRun(main, []string{"root", dbSpecStr})
-	s.Equal("maojl4udo9a7mtk2rnhuc08r0u7hc0fn\n", c1)
+	s.Equal("k0ishq8q2ejoc4jla005tlrmbkse321j\n", c1)
 
 	ds, _ = ds.Database().CommitValue(context.Background(), ds, types.String("goodbye"))
 	c2, _ := s.MustRun(main, []string{"root", dbSpecStr})
-	s.Equal("avdab61n1s1d1emdee7kb7e49quisr5n\n", c2)
+	s.Equal("o7phbcbit2moq2v0p9bd2uufijihc8nr\n", c2)
 
 	// TODO: Would be good to test successful --update too, but requires changes to MustRun to allow
 	// input because of prompt :(.

go/store/cmd/noms/noms_show_test.go

@@ -44,11 +44,11 @@ type nomsShowTestSuite struct {
 }
 
 const (
-	res1 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {},\n  parents_list: [],\n  value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj,\n}"
+	res1 = "Commit{meta Struct,parents Set,parents_list List,parents_skip_list Union,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {},\n  parents_list: [],\n  parents_skip_list: (),\n  value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj,\n}"
 	res2 = "String - \"test string\""
-	res3 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {\n    #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n  },\n  parents_list: [\n    #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n  ],\n  value: #82adk7hfcudg8fktittm672to66t6qeu,\n}"
+	res3 = "Commit{meta Struct,parents Set,parents_list List,parents_skip_list Union,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {\n    #nbd25rh0hqop4alrckbvakgm7snnfpcu,\n  },\n  parents_list: [\n    #nbd25rh0hqop4alrckbvakgm7snnfpcu,\n  ],\n  parents_skip_list: (),\n  value: #82adk7hfcudg8fktittm672to66t6qeu,\n}"
 	res4 = "List<Union<Float,String>> - [\n  \"elem1\",\n  2,\n  \"elem3\",\n]"
-	res5 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {\n    #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n  },\n  parents_list: [\n    #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n  ],\n  value: #5cgfu2vk4nc21m1vjkjjpd2kvcm2df7q,\n}"
+	res5 = "Commit{meta Struct,parents Set,parents_list List,parents_skip_list Union,value Ref} - struct Commit {\n  meta: struct {},\n  parents: set {\n    #kqf9vsd3qd52decpt8min7fb7kpiepah,\n  },\n  parents_list: [\n    #kqf9vsd3qd52decpt8min7fb7kpiepah,\n  ],\n  parents_skip_list: (),\n  value: #5cgfu2vk4nc21m1vjkjjpd2kvcm2df7q,\n}"
 )
 
 func (s *nomsShowTestSuite) spec(str string) spec.Spec {

go/store/datas/commit.go

@@ -40,21 +40,32 @@ const (
 	// `"parents"` is still written as a Set as well, so that commits
 	// created with newer versions of still usable by older versions.
 	ParentsListField = "parents_list"
-	ValueField       = "value"
-	CommitMetaField  = "meta"
-	CommitName       = "Commit"
+	// Added in October, 2021. Stored as a Tuple of Tuples, where
+	// each entry in the inner tuples is a level of the skip list
+	// pointer for a given level to the given parent index.
+	ParentsSkipListField = "parents_skip_list"
+	ValueField           = "value"
+	CommitMetaField      = "meta"
+	CommitName           = "Commit"
 )
 
-var commitTemplate = types.MakeStructTemplate(CommitName, []string{CommitMetaField, ParentsField, ParentsListField, ValueField})
+var commitTemplate = types.MakeStructTemplate(CommitName, []string{
+	CommitMetaField,
+	ParentsField,
+	ParentsListField,
+	ParentsSkipListField,
+	ValueField,
+})
 
 var valueCommitType = nomdl.MustParseType(`Struct Commit {
         meta: Struct {},
         parents: Set<Ref<Cycle<Commit>>>,
         parents_list?: List<Ref<Cycle<Commit>>>,
+        parents_skip_list?: Value,
         value: Value,
 }`)
 
-// NewCommit creates a new commit object.
+// newCommit creates a new commit object.
 //
 // A commit has the following type:
 //
@@ -63,16 +74,17 @@ var valueCommitType = nomdl.MustParseType(`Struct Commit {
 //   meta: M,
 //   parents: Set<Ref<Cycle<Commit>>>,
 //   parentsList: List<Ref<Cycle<Commit>>>,
+//   parentsSkipList: Value,
 //   value: T,
 // }
 // ```
 // where M is a struct type and T is any type.
-func NewCommit(ctx context.Context, value types.Value, parentsList types.List, meta types.Struct) (types.Struct, error) {
+func newCommit(ctx context.Context, value types.Value, parentsList types.List, parentsSkipList types.Tuple, meta types.Struct) (types.Struct, error) {
 	parentsSet, err := parentsList.ToSet(ctx)
 	if err != nil {
 		return types.EmptyStruct(meta.Format()), err
 	}
-	return commitTemplate.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, value})
+	return commitTemplate.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, parentsSkipList, value})
 }
 
 // FindCommonAncestor returns the most recent common ancestor of c1 and c2, if
@@ -244,7 +256,7 @@ func findCommonRef(a, b types.RefSlice) (types.Ref, bool) {
 	return types.Ref{}, false
 }
 
-func makeCommitStructType(metaType, parentsType, parentsListType, valueType *types.Type) (*types.Type, error) {
+func makeCommitStructType(metaType, parentsType, parentsListType, parentsSkipListType, valueType *types.Type) (*types.Type, error) {
 	return types.MakeStructType(CommitName,
 		types.StructField{
 			Name: CommitMetaField,
@@ -258,6 +270,10 @@ func makeCommitStructType(metaType, parentsType, parentsListType, valueType *typ
 			Name: ParentsListField,
 			Type: parentsListType,
 		},
+		types.StructField{
+			Name: ParentsSkipListField,
+			Type: parentsSkipListType,
+		},
 		types.StructField{
 			Name: ValueField,
 			Type: valueType,

go/store/datas/commit_test.go

@@ -106,6 +106,11 @@ func mustValue(val types.Value, err error) types.Value {
 	return val
 }
 
+func mustTuple(val types.Tuple, err error) types.Tuple {
+	d.PanicIfError(err)
+	return val
+}
+
 func TestNewCommit(t *testing.T) {
 	assert := assert.New(t)
 
@@ -119,7 +124,8 @@ func TestNewCommit(t *testing.T) {
 	defer db.Close()
 
 	parents := mustList(types.NewList(context.Background(), db))
-	commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18))
+	parentsSkipList := mustTuple(getParentsSkipList(context.Background(), db, parents))
+	commit, err := newCommit(context.Background(), types.Float(1), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at, err := types.TypeOf(commit)
 	assert.NoError(err)
@@ -127,6 +133,7 @@ func TestNewCommit(t *testing.T) {
 		types.EmptyStructType,
 		mustType(types.MakeSetType(mustType(types.MakeUnionType()))),
 		mustType(types.MakeListType(mustType(types.MakeUnionType()))),
+		mustType(types.TypeOf(types.EmptyTuple(types.Format_7_18))),
 		types.PrimitiveTypeMap[types.FloatKind],
 	)
 	assert.NoError(err)
@@ -134,7 +141,8 @@ func TestNewCommit(t *testing.T) {
 
 	// Committing another Float
 	parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit, types.Format_7_18))))
-	commit2, err := NewCommit(context.Background(), types.Float(2), parents, types.EmptyStruct(types.Format_7_18))
+	parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents))
+	commit2, err := newCommit(context.Background(), types.Float(2), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at2, err := types.TypeOf(commit2)
 	assert.NoError(err)
@@ -142,13 +150,15 @@ func TestNewCommit(t *testing.T) {
                 meta: Struct {},
                 parents: Set<Ref<Cycle<Commit>>>,
                 parents_list: List<Ref<Cycle<Commit>>>,
+                parents_skip_list: Tuple,
                 value: Float,
         }`)
 	assertTypeEquals(et2, at2)
 
 	// Now commit a String
 	parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18))))
-	commit3, err := NewCommit(context.Background(), types.String("Hi"), parents, types.EmptyStruct(types.Format_7_18))
+	parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents))
+	commit3, err := newCommit(context.Background(), types.String("Hi"), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at3, err := types.TypeOf(commit3)
 	assert.NoError(err)
@@ -156,6 +166,7 @@ func TestNewCommit(t *testing.T) {
                 meta: Struct {},
                 parents: Set<Ref<Cycle<Commit>>>,
                 parents_list: List<Ref<Cycle<Commit>>>,
+                parents_skip_list: Tuple,
                 value: Float | String,
         }`)
 	assertTypeEquals(et3, at3)
@@ -169,7 +180,8 @@ func TestNewCommit(t *testing.T) {
 	}`)
 	assertTypeEquals(metaType, mustType(types.TypeOf(meta)))
 	parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18))))
-	commit4, err := NewCommit(context.Background(), types.String("Hi"), parents, meta)
+	parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents))
+	commit4, err := newCommit(context.Background(), types.String("Hi"), parents, parentsSkipList, meta)
 	assert.NoError(err)
 	at4, err := types.TypeOf(commit4)
 	assert.NoError(err)
@@ -180,6 +192,7 @@ func TestNewCommit(t *testing.T) {
         	},
                 parents: Set<Ref<Cycle<Commit>>>,
                 parents_list: List<Ref<Cycle<Commit>>>,
+                parents_skip_list: Tuple,
                 value: Float | String,
         }`)
 	assertTypeEquals(et4, at4)
@@ -188,10 +201,12 @@ func TestNewCommit(t *testing.T) {
 	parents = mustList(types.NewList(context.Background(), db,
 		mustRef(types.NewRef(commit2, types.Format_7_18)),
 		mustRef(types.NewRef(commit3, types.Format_7_18))))
-	commit5, err := NewCommit(
+	parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents))
+	commit5, err := newCommit(
 		context.Background(),
 		types.String("Hi"),
 		parents,
+		parentsSkipList,
 		types.EmptyStruct(types.Format_7_18))
 	assert.NoError(err)
 	at5, err := types.TypeOf(commit5)
@@ -200,6 +215,7 @@ func TestNewCommit(t *testing.T) {
                 meta: Struct {},
                 parents: Set<Ref<Cycle<Commit>>>,
                 parents_list: List<Ref<Cycle<Commit>>>,
+                parents_skip_list: Tuple,
                 value: Float | String,
         }`)
 	assertTypeEquals(et5, at5)
@@ -433,20 +449,22 @@ func TestNewCommitRegressionTest(t *testing.T) {
 	defer db.Close()
 
 	parents := mustList(types.NewList(context.Background(), db))
-	c1, err := NewCommit(context.Background(), types.String("one"), parents, types.EmptyStruct(types.Format_7_18))
+	parentsSkipList := mustTuple(getParentsSkipList(context.Background(), db, parents))
+	c1, err := newCommit(context.Background(), types.String("one"), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(t, err)
-	cx, err := NewCommit(context.Background(), types.Bool(true), parents, types.EmptyStruct(types.Format_7_18))
+	cx, err := newCommit(context.Background(), types.Bool(true), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18))
 	assert.NoError(t, err)
 	value := types.String("two")
 	parents, err = types.NewList(context.Background(), db, mustRef(types.NewRef(c1, types.Format_7_18)))
 	assert.NoError(t, err)
+	parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents))
 	meta, err := types.NewStruct(types.Format_7_18, "", types.StructData{
 		"basis": cx,
 	})
 	assert.NoError(t, err)
 
 	// Used to fail
-	_, err = NewCommit(context.Background(), value, parents, meta)
+	_, err = newCommit(context.Background(), value, parents, parentsSkipList, meta)
 	assert.NoError(t, err)
 }
 

go/store/datas/database_common.go

@@ -138,6 +138,11 @@ func (db *database) DatasetsInRoot(ctx context.Context, rootHash hash.Hash) (typ
 	return val.(types.Map), nil
 }
 
+func getParentsSkipList(ctx context.Context, vrw types.ValueReadWriter, parents types.List) (types.Tuple, error) {
+	// TODO
+	return types.EmptyTuple(vrw.Format()), nil
+}
+
 // Datasets returns the Map of Datasets in the current root. If you intend to edit the map and commit changes back,
 // then you should fetch the current root, then call DatasetsInRoot with that hash. Otherwise another writer could
 // change the root value between when you get the root hash and call this method.
@@ -318,7 +323,12 @@ func (db *database) CommitDangling(ctx context.Context, v types.Value, opts Comm
 		opts.Meta = types.EmptyStruct(db.Format())
 	}
 
-	commitStruct, err := NewCommit(ctx, v, opts.ParentsList, opts.Meta)
+	parentsSkipList, err := getParentsSkipList(ctx, db, opts.ParentsList)
+	if err != nil {
+		return types.Struct{}, err
+	}
+
+	commitStruct, err := newCommit(ctx, v, opts.ParentsList, parentsSkipList, opts.Meta)
 	if err != nil {
 		return types.Struct{}, err
 	}
@@ -473,7 +483,12 @@ func (db *database) doMerge(
 		return types.Ref{}, err
 	}
 
-	newCom, err := NewCommit(ctx, merged, parents, types.EmptyStruct(db.Format()))
+	parentsSkipList, err := getParentsSkipList(ctx, db, parents)
+	if err != nil {
+		return types.Ref{}, err
+	}
+
+	newCom, err := newCommit(ctx, merged, parents, parentsSkipList, types.EmptyStruct(db.Format()))
 	if err != nil {
 		return types.Ref{}, err
 	}
@@ -980,7 +995,13 @@ func buildNewCommit(ctx context.Context, ds Dataset, v types.Value, opts CommitO
 	if meta.IsZeroValue() {
 		meta = types.EmptyStruct(ds.Database().Format())
 	}
-	return NewCommit(ctx, v, parents, meta)
+
+	parentsSkipList, err := getParentsSkipList(ctx, ds.Database(), parents)
+	if err != nil {
+		return types.EmptyStruct(ds.Database().Format()), err
+	}
+
+	return newCommit(ctx, v, parents, parentsSkipList, meta)
 }
 
 func (db *database) doHeadUpdate(ctx context.Context, ds Dataset, updateFunc func(ds Dataset) error) (Dataset, error) {

go/store/datas/puller_test.go

@@ -32,14 +32,6 @@ import (
 	"github.com/dolthub/dolt/go/store/util/clienttest"
 )
 
-func mustTuple(tpl types.Tuple, err error) types.Tuple {
-	if err != nil {
-		panic(err)
-	}
-
-	return tpl
-}
-
 func addTableValues(ctx context.Context, vrw types.ValueReadWriter, m types.Map, tableName string, alternatingKeyVals ...types.Value) (types.Map, error) {
 	val, ok, err := m.MaybeGet(ctx, types.String(tableName))
 

go/store/datas/tag_test.go

@@ -38,7 +38,8 @@ func TestNewTag(t *testing.T) {
 	defer db.Close()
 
 	parents := mustList(types.NewList(context.Background(), db))
-	commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18))
+	parentsSkipList := mustTuple(getParentsSkipList(context.Background(), db, parents))
+	commit, err := newCommit(context.Background(), types.Float(1), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18))
 	require.NoError(t, err)
 
 	cmRef, err := types.NewRef(commit, types.Format_7_18)
@@ -50,6 +51,7 @@ func TestNewTag(t *testing.T) {
 		types.EmptyStructType,
 		mustType(types.MakeSetType(mustType(types.MakeUnionType()))),
 		mustType(types.MakeListType(mustType(types.MakeUnionType()))),
+		mustType(types.TypeOf(types.EmptyTuple(types.Format_7_18))),
 		types.PrimitiveTypeMap[types.FloatKind],
 	)
 	require.NoError(t, err)

go/store/nomdl/parser.go

@@ -212,6 +212,12 @@ func (p *Parser) parseSingleTypeWithToken(tok rune, tokenText string) (*types.Ty
 		return types.PrimitiveTypeMap[types.TypeKind], nil
 	case "Value":
 		return types.PrimitiveTypeMap[types.ValueKind], nil
+	case "Tuple":
+		f := types.Format_Default
+		if p.vrw != nil {
+			f = p.vrw.Format()
+		}
+		return types.TypeOf(types.EmptyTuple(f))
 	case "Struct":
 		return p.parseStructType()
 	case "Map":
@@ -352,6 +358,7 @@ func (p *Parser) parseMapType() (*types.Type, error) {
 //   Set
 //   Map
 //   Struct
+//   Tuple
 //
 // Bool :
 //   `true`

go/store/types/encode_human_readable.go

@@ -255,7 +255,6 @@ func (w *hrsWriter) Write(ctx context.Context, v Value) error {
 			return err
 		}
 
-		w.outdent()
 		w.write(")")
 
 	case MapKind: