From a2f660ff42c26dbd639e4b9099a04abcbccd7c54 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 28 Sep 2021 17:06:27 -0700 Subject: [PATCH] store/datas: NewCommit -> newCommit, start threading parentsSkipList parameter through. store/datas: Add ability to store parents_skip_list field to Commit Struct. --- go/libraries/doltcore/sqle/sqlselect_test.go | 4 +-- go/store/cmd/noms/noms_ds_test.go | 4 +-- go/store/cmd/noms/noms_merge.go | 10 ++---- go/store/cmd/noms/noms_root_test.go | 4 +-- go/store/cmd/noms/noms_show_test.go | 6 ++-- go/store/datas/commit.go | 32 +++++++++++++----- go/store/datas/commit_test.go | 34 +++++++++++++++----- go/store/datas/database_common.go | 27 ++++++++++++++-- go/store/datas/puller_test.go | 8 ----- go/store/datas/tag_test.go | 4 ++- go/store/nomdl/parser.go | 7 ++++ go/store/types/encode_human_readable.go | 1 - 12 files changed, 96 insertions(+), 45 deletions(-) diff --git a/go/libraries/doltcore/sqle/sqlselect_test.go b/go/libraries/doltcore/sqle/sqlselect_test.go index 22e6fa4d47..ad7be8c412 100644 --- a/go/libraries/doltcore/sqle/sqlselect_test.go +++ b/go/libraries/doltcore/sqle/sqlselect_test.go @@ -733,7 +733,7 @@ var BasicSelectTests = []SelectTest{ Query: "select * from dolt_log", ExpectedRows: []sql.Row{ { - "so275enkvulb96mkckbun1kjo9seg7c9", + "oqd9k78tffc3hji0vbo88v6ffr6cp6td", "billy bob", "bigbillieb@fake.horse", time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC).In(LoadedLocalLocation()), @@ -763,7 +763,7 @@ var BasicSelectTests = []SelectTest{ ExpectedRows: []sql.Row{ { env.DefaultInitBranch, - "so275enkvulb96mkckbun1kjo9seg7c9", + "oqd9k78tffc3hji0vbo88v6ffr6cp6td", "billy bob", "bigbillieb@fake.horse", time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC).In(LoadedLocalLocation()), "Initialize data repository", diff --git a/go/store/cmd/noms/noms_ds_test.go b/go/store/cmd/noms/noms_ds_test.go index 9dc3ce29a2..448dea1c79 100644 --- a/go/store/cmd/noms/noms_ds_test.go +++ b/go/store/cmd/noms/noms_ds_test.go @@ -92,7 +92,7 @@ func (s *nomsDsTestSuite) TestNomsDs() { // delete one dataset, print message at delete rtnVal, _ = s.MustRun(main, []string{"ds", "-d", datasetName}) - s.Equal("Deleted "+datasetName+" (was #7jrps2q0ubq0phnha37gd6051m8uqq6b)\n", rtnVal) + s.Equal("Deleted "+datasetName+" (was #3qqamcm2ur9htnmd692v4m4ck3ktgb64)\n", rtnVal) // print datasets, just one left rtnVal, _ = s.MustRun(main, []string{"ds", dbSpec}) @@ -100,7 +100,7 @@ func (s *nomsDsTestSuite) TestNomsDs() { // delete the second dataset rtnVal, _ = s.MustRun(main, []string{"ds", "-d", dataset2Name}) - s.Equal("Deleted "+dataset2Name+" (was #hiqpb7kk36vo80uc36virlnrddru2o7h)\n", rtnVal) + s.Equal("Deleted "+dataset2Name+" (was #3b9t0n2432d4c3a8lhq8e5qsr13kd75a)\n", rtnVal) // print datasets, none left rtnVal, _ = s.MustRun(main, []string{"ds", dbSpec}) diff --git a/go/store/cmd/noms/noms_merge.go b/go/store/cmd/noms/noms_merge.go index b0ceb304a6..645a7e4555 100644 --- a/go/store/cmd/noms/noms_merge.go +++ b/go/store/cmd/noms/noms_merge.go @@ -117,13 +117,9 @@ func runMerge(ctx context.Context, args []string) int { p, err := types.NewList(ctx, db, leftHeadRef, rightHeadRef) d.PanicIfError(err) - cm, err := datas.NewCommit(ctx, merged, p, types.EmptyStruct(db.Format())) - d.PanicIfError(err) - - ref, err := db.WriteValue(ctx, cm) - d.PanicIfError(err) - - _, err = db.SetHead(ctx, outDS, ref) + _, err = db.Commit(ctx, outDS, merged, datas.CommitOptions{ + ParentsList: p, + }) d.PanicIfError(err) status.Printf("Done") diff --git a/go/store/cmd/noms/noms_root_test.go b/go/store/cmd/noms/noms_root_test.go index c22df8e4ed..44a752cf48 100644 --- a/go/store/cmd/noms/noms_root_test.go +++ b/go/store/cmd/noms/noms_root_test.go @@ -51,11 +51,11 @@ func (s *nomsRootTestSuite) TestBasic() { dbSpecStr := spec.CreateDatabaseSpecString("nbs", s.DBDir) ds, _ = ds.Database().CommitValue(context.Background(), ds, types.String("hello!")) c1, _ := s.MustRun(main, []string{"root", dbSpecStr}) - s.Equal("maojl4udo9a7mtk2rnhuc08r0u7hc0fn\n", c1) + s.Equal("k0ishq8q2ejoc4jla005tlrmbkse321j\n", c1) ds, _ = ds.Database().CommitValue(context.Background(), ds, types.String("goodbye")) c2, _ := s.MustRun(main, []string{"root", dbSpecStr}) - s.Equal("avdab61n1s1d1emdee7kb7e49quisr5n\n", c2) + s.Equal("o7phbcbit2moq2v0p9bd2uufijihc8nr\n", c2) // TODO: Would be good to test successful --update too, but requires changes to MustRun to allow // input because of prompt :(. diff --git a/go/store/cmd/noms/noms_show_test.go b/go/store/cmd/noms/noms_show_test.go index 904850a847..ea9284dc69 100644 --- a/go/store/cmd/noms/noms_show_test.go +++ b/go/store/cmd/noms/noms_show_test.go @@ -44,11 +44,11 @@ type nomsShowTestSuite struct { } const ( - res1 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n meta: struct {},\n parents: set {},\n parents_list: [],\n value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj,\n}" + res1 = "Commit{meta Struct,parents Set,parents_list List,parents_skip_list Union,value Ref} - struct Commit {\n meta: struct {},\n parents: set {},\n parents_list: [],\n parents_skip_list: (),\n value: #nl181uu1ioc2j6t7mt9paidjlhlcjtgj,\n}" res2 = "String - \"test string\"" - res3 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n meta: struct {},\n parents: set {\n #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n },\n parents_list: [\n #4g7ggl6999v5mlucl4a507n7k3kvckiq,\n ],\n value: #82adk7hfcudg8fktittm672to66t6qeu,\n}" + res3 = "Commit{meta Struct,parents Set,parents_list List,parents_skip_list Union,value Ref} - struct Commit {\n meta: struct {},\n parents: set {\n #nbd25rh0hqop4alrckbvakgm7snnfpcu,\n },\n parents_list: [\n #nbd25rh0hqop4alrckbvakgm7snnfpcu,\n ],\n parents_skip_list: (),\n value: #82adk7hfcudg8fktittm672to66t6qeu,\n}" res4 = "List> - [\n \"elem1\",\n 2,\n \"elem3\",\n]" - res5 = "Commit{meta Struct,parents Set,parents_list List,value Ref} - struct Commit {\n meta: struct {},\n parents: set {\n #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n },\n parents_list: [\n #3tmg89vabs2k6hotdock1kuo13j4lmqv,\n ],\n value: #5cgfu2vk4nc21m1vjkjjpd2kvcm2df7q,\n}" + res5 = "Commit{meta Struct,parents Set,parents_list List,parents_skip_list Union,value Ref} - struct Commit {\n meta: struct {},\n parents: set {\n #kqf9vsd3qd52decpt8min7fb7kpiepah,\n },\n parents_list: [\n #kqf9vsd3qd52decpt8min7fb7kpiepah,\n ],\n parents_skip_list: (),\n value: #5cgfu2vk4nc21m1vjkjjpd2kvcm2df7q,\n}" ) func (s *nomsShowTestSuite) spec(str string) spec.Spec { diff --git a/go/store/datas/commit.go b/go/store/datas/commit.go index d020bb6af7..e6a8e305b7 100644 --- a/go/store/datas/commit.go +++ b/go/store/datas/commit.go @@ -40,21 +40,32 @@ const ( // `"parents"` is still written as a Set as well, so that commits // created with newer versions of still usable by older versions. ParentsListField = "parents_list" - ValueField = "value" - CommitMetaField = "meta" - CommitName = "Commit" + // Added in October, 2021. Stored as a Tuple of Tuples, where + // each entry in the inner tuples is a level of the skip list + // pointer for a given level to the given parent index. + ParentsSkipListField = "parents_skip_list" + ValueField = "value" + CommitMetaField = "meta" + CommitName = "Commit" ) -var commitTemplate = types.MakeStructTemplate(CommitName, []string{CommitMetaField, ParentsField, ParentsListField, ValueField}) +var commitTemplate = types.MakeStructTemplate(CommitName, []string{ + CommitMetaField, + ParentsField, + ParentsListField, + ParentsSkipListField, + ValueField, +}) var valueCommitType = nomdl.MustParseType(`Struct Commit { meta: Struct {}, parents: Set>>, parents_list?: List>>, + parents_skip_list?: Value, value: Value, }`) -// NewCommit creates a new commit object. +// newCommit creates a new commit object. // // A commit has the following type: // @@ -63,16 +74,17 @@ var valueCommitType = nomdl.MustParseType(`Struct Commit { // meta: M, // parents: Set>>, // parentsList: List>>, +// parentsSkipList: Value, // value: T, // } // ``` // where M is a struct type and T is any type. -func NewCommit(ctx context.Context, value types.Value, parentsList types.List, meta types.Struct) (types.Struct, error) { +func newCommit(ctx context.Context, value types.Value, parentsList types.List, parentsSkipList types.Tuple, meta types.Struct) (types.Struct, error) { parentsSet, err := parentsList.ToSet(ctx) if err != nil { return types.EmptyStruct(meta.Format()), err } - return commitTemplate.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, value}) + return commitTemplate.NewStruct(meta.Format(), []types.Value{meta, parentsSet, parentsList, parentsSkipList, value}) } // FindCommonAncestor returns the most recent common ancestor of c1 and c2, if @@ -244,7 +256,7 @@ func findCommonRef(a, b types.RefSlice) (types.Ref, bool) { return types.Ref{}, false } -func makeCommitStructType(metaType, parentsType, parentsListType, valueType *types.Type) (*types.Type, error) { +func makeCommitStructType(metaType, parentsType, parentsListType, parentsSkipListType, valueType *types.Type) (*types.Type, error) { return types.MakeStructType(CommitName, types.StructField{ Name: CommitMetaField, @@ -258,6 +270,10 @@ func makeCommitStructType(metaType, parentsType, parentsListType, valueType *typ Name: ParentsListField, Type: parentsListType, }, + types.StructField{ + Name: ParentsSkipListField, + Type: parentsSkipListType, + }, types.StructField{ Name: ValueField, Type: valueType, diff --git a/go/store/datas/commit_test.go b/go/store/datas/commit_test.go index 8e6a5a4df4..66c641bb0e 100644 --- a/go/store/datas/commit_test.go +++ b/go/store/datas/commit_test.go @@ -106,6 +106,11 @@ func mustValue(val types.Value, err error) types.Value { return val } +func mustTuple(val types.Tuple, err error) types.Tuple { + d.PanicIfError(err) + return val +} + func TestNewCommit(t *testing.T) { assert := assert.New(t) @@ -119,7 +124,8 @@ func TestNewCommit(t *testing.T) { defer db.Close() parents := mustList(types.NewList(context.Background(), db)) - commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18)) + parentsSkipList := mustTuple(getParentsSkipList(context.Background(), db, parents)) + commit, err := newCommit(context.Background(), types.Float(1), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at, err := types.TypeOf(commit) assert.NoError(err) @@ -127,6 +133,7 @@ func TestNewCommit(t *testing.T) { types.EmptyStructType, mustType(types.MakeSetType(mustType(types.MakeUnionType()))), mustType(types.MakeListType(mustType(types.MakeUnionType()))), + mustType(types.TypeOf(types.EmptyTuple(types.Format_7_18))), types.PrimitiveTypeMap[types.FloatKind], ) assert.NoError(err) @@ -134,7 +141,8 @@ func TestNewCommit(t *testing.T) { // Committing another Float parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit, types.Format_7_18)))) - commit2, err := NewCommit(context.Background(), types.Float(2), parents, types.EmptyStruct(types.Format_7_18)) + parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents)) + commit2, err := newCommit(context.Background(), types.Float(2), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at2, err := types.TypeOf(commit2) assert.NoError(err) @@ -142,13 +150,15 @@ func TestNewCommit(t *testing.T) { meta: Struct {}, parents: Set>>, parents_list: List>>, + parents_skip_list: Tuple, value: Float, }`) assertTypeEquals(et2, at2) // Now commit a String parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18)))) - commit3, err := NewCommit(context.Background(), types.String("Hi"), parents, types.EmptyStruct(types.Format_7_18)) + parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents)) + commit3, err := newCommit(context.Background(), types.String("Hi"), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at3, err := types.TypeOf(commit3) assert.NoError(err) @@ -156,6 +166,7 @@ func TestNewCommit(t *testing.T) { meta: Struct {}, parents: Set>>, parents_list: List>>, + parents_skip_list: Tuple, value: Float | String, }`) assertTypeEquals(et3, at3) @@ -169,7 +180,8 @@ func TestNewCommit(t *testing.T) { }`) assertTypeEquals(metaType, mustType(types.TypeOf(meta))) parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18)))) - commit4, err := NewCommit(context.Background(), types.String("Hi"), parents, meta) + parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents)) + commit4, err := newCommit(context.Background(), types.String("Hi"), parents, parentsSkipList, meta) assert.NoError(err) at4, err := types.TypeOf(commit4) assert.NoError(err) @@ -180,6 +192,7 @@ func TestNewCommit(t *testing.T) { }, parents: Set>>, parents_list: List>>, + parents_skip_list: Tuple, value: Float | String, }`) assertTypeEquals(et4, at4) @@ -188,10 +201,12 @@ func TestNewCommit(t *testing.T) { parents = mustList(types.NewList(context.Background(), db, mustRef(types.NewRef(commit2, types.Format_7_18)), mustRef(types.NewRef(commit3, types.Format_7_18)))) - commit5, err := NewCommit( + parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents)) + commit5, err := newCommit( context.Background(), types.String("Hi"), parents, + parentsSkipList, types.EmptyStruct(types.Format_7_18)) assert.NoError(err) at5, err := types.TypeOf(commit5) @@ -200,6 +215,7 @@ func TestNewCommit(t *testing.T) { meta: Struct {}, parents: Set>>, parents_list: List>>, + parents_skip_list: Tuple, value: Float | String, }`) assertTypeEquals(et5, at5) @@ -433,20 +449,22 @@ func TestNewCommitRegressionTest(t *testing.T) { defer db.Close() parents := mustList(types.NewList(context.Background(), db)) - c1, err := NewCommit(context.Background(), types.String("one"), parents, types.EmptyStruct(types.Format_7_18)) + parentsSkipList := mustTuple(getParentsSkipList(context.Background(), db, parents)) + c1, err := newCommit(context.Background(), types.String("one"), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18)) assert.NoError(t, err) - cx, err := NewCommit(context.Background(), types.Bool(true), parents, types.EmptyStruct(types.Format_7_18)) + cx, err := newCommit(context.Background(), types.Bool(true), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18)) assert.NoError(t, err) value := types.String("two") parents, err = types.NewList(context.Background(), db, mustRef(types.NewRef(c1, types.Format_7_18))) assert.NoError(t, err) + parentsSkipList = mustTuple(getParentsSkipList(context.Background(), db, parents)) meta, err := types.NewStruct(types.Format_7_18, "", types.StructData{ "basis": cx, }) assert.NoError(t, err) // Used to fail - _, err = NewCommit(context.Background(), value, parents, meta) + _, err = newCommit(context.Background(), value, parents, parentsSkipList, meta) assert.NoError(t, err) } diff --git a/go/store/datas/database_common.go b/go/store/datas/database_common.go index 19286817d2..5ca76b33f0 100644 --- a/go/store/datas/database_common.go +++ b/go/store/datas/database_common.go @@ -138,6 +138,11 @@ func (db *database) DatasetsInRoot(ctx context.Context, rootHash hash.Hash) (typ return val.(types.Map), nil } +func getParentsSkipList(ctx context.Context, vrw types.ValueReadWriter, parents types.List) (types.Tuple, error) { + // TODO + return types.EmptyTuple(vrw.Format()), nil +} + // Datasets returns the Map of Datasets in the current root. If you intend to edit the map and commit changes back, // then you should fetch the current root, then call DatasetsInRoot with that hash. Otherwise another writer could // change the root value between when you get the root hash and call this method. @@ -318,7 +323,12 @@ func (db *database) CommitDangling(ctx context.Context, v types.Value, opts Comm opts.Meta = types.EmptyStruct(db.Format()) } - commitStruct, err := NewCommit(ctx, v, opts.ParentsList, opts.Meta) + parentsSkipList, err := getParentsSkipList(ctx, db, opts.ParentsList) + if err != nil { + return types.Struct{}, err + } + + commitStruct, err := newCommit(ctx, v, opts.ParentsList, parentsSkipList, opts.Meta) if err != nil { return types.Struct{}, err } @@ -473,7 +483,12 @@ func (db *database) doMerge( return types.Ref{}, err } - newCom, err := NewCommit(ctx, merged, parents, types.EmptyStruct(db.Format())) + parentsSkipList, err := getParentsSkipList(ctx, db, parents) + if err != nil { + return types.Ref{}, err + } + + newCom, err := newCommit(ctx, merged, parents, parentsSkipList, types.EmptyStruct(db.Format())) if err != nil { return types.Ref{}, err } @@ -980,7 +995,13 @@ func buildNewCommit(ctx context.Context, ds Dataset, v types.Value, opts CommitO if meta.IsZeroValue() { meta = types.EmptyStruct(ds.Database().Format()) } - return NewCommit(ctx, v, parents, meta) + + parentsSkipList, err := getParentsSkipList(ctx, ds.Database(), parents) + if err != nil { + return types.EmptyStruct(ds.Database().Format()), err + } + + return newCommit(ctx, v, parents, parentsSkipList, meta) } func (db *database) doHeadUpdate(ctx context.Context, ds Dataset, updateFunc func(ds Dataset) error) (Dataset, error) { diff --git a/go/store/datas/puller_test.go b/go/store/datas/puller_test.go index ac74d0a79e..5daecbe1c4 100644 --- a/go/store/datas/puller_test.go +++ b/go/store/datas/puller_test.go @@ -32,14 +32,6 @@ import ( "github.com/dolthub/dolt/go/store/util/clienttest" ) -func mustTuple(tpl types.Tuple, err error) types.Tuple { - if err != nil { - panic(err) - } - - return tpl -} - func addTableValues(ctx context.Context, vrw types.ValueReadWriter, m types.Map, tableName string, alternatingKeyVals ...types.Value) (types.Map, error) { val, ok, err := m.MaybeGet(ctx, types.String(tableName)) diff --git a/go/store/datas/tag_test.go b/go/store/datas/tag_test.go index 049a90ae99..ea6c46849c 100644 --- a/go/store/datas/tag_test.go +++ b/go/store/datas/tag_test.go @@ -38,7 +38,8 @@ func TestNewTag(t *testing.T) { defer db.Close() parents := mustList(types.NewList(context.Background(), db)) - commit, err := NewCommit(context.Background(), types.Float(1), parents, types.EmptyStruct(types.Format_7_18)) + parentsSkipList := mustTuple(getParentsSkipList(context.Background(), db, parents)) + commit, err := newCommit(context.Background(), types.Float(1), parents, parentsSkipList, types.EmptyStruct(types.Format_7_18)) require.NoError(t, err) cmRef, err := types.NewRef(commit, types.Format_7_18) @@ -50,6 +51,7 @@ func TestNewTag(t *testing.T) { types.EmptyStructType, mustType(types.MakeSetType(mustType(types.MakeUnionType()))), mustType(types.MakeListType(mustType(types.MakeUnionType()))), + mustType(types.TypeOf(types.EmptyTuple(types.Format_7_18))), types.PrimitiveTypeMap[types.FloatKind], ) require.NoError(t, err) diff --git a/go/store/nomdl/parser.go b/go/store/nomdl/parser.go index 42a7ea2d61..d2b868a8cb 100644 --- a/go/store/nomdl/parser.go +++ b/go/store/nomdl/parser.go @@ -212,6 +212,12 @@ func (p *Parser) parseSingleTypeWithToken(tok rune, tokenText string) (*types.Ty return types.PrimitiveTypeMap[types.TypeKind], nil case "Value": return types.PrimitiveTypeMap[types.ValueKind], nil + case "Tuple": + f := types.Format_Default + if p.vrw != nil { + f = p.vrw.Format() + } + return types.TypeOf(types.EmptyTuple(f)) case "Struct": return p.parseStructType() case "Map": @@ -352,6 +358,7 @@ func (p *Parser) parseMapType() (*types.Type, error) { // Set // Map // Struct +// Tuple // // Bool : // `true` diff --git a/go/store/types/encode_human_readable.go b/go/store/types/encode_human_readable.go index c92b848df3..f94fe375ee 100644 --- a/go/store/types/encode_human_readable.go +++ b/go/store/types/encode_human_readable.go @@ -255,7 +255,6 @@ func (w *hrsWriter) Write(ctx context.Context, v Value) error { return err } - w.outdent() w.write(")") case MapKind: