mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-06 19:35:18 -05:00
Code Issues Packages Projects Releases 160 Wiki Activity

Remove in mem graphs (#3635)

Browse Source 
This patch removes the ability to keep alive uncommitted prolly-tree sequences.
This commit is contained in:
Rafael Weinstein
2017-08-29 13:12:10 -07:00
committed by GitHub
parent 61f3d87dcf
commit c3f98d1631
120 changed files with 2615 additions and 1933 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/noms/noms_blob_get_test.go
+4 -3
View File
@@ -26,13 +26,14 @@ type nbeSuite struct {
}
func (s *nbeSuite) TestNomsBlobGet() {
blobBytes := []byte("hello")
blob := types.NewBlob(bytes.NewBuffer(blobBytes))
sp, err := spec.ForDatabase(s.TempDir)
s.NoError(err)
defer sp.Close()
db := sp.GetDatabase()
blobBytes := []byte("hello")
blob := types.NewBlob(db, bytes.NewBuffer(blobBytes))
ref := db.WriteValue(blob)
_, err = db.CommitValue(db.GetDataset("datasetID"), ref)
s.NoError(err)
cmd/noms/noms_blob_put.go
+1 -1
View File
@@ -52,7 +52,7 @@ func nomsBlobPut(filePath string, dsPath string, concurrency int) int {
}
defer db.Close()
blob := types.NewStreamingBlob(db, readers...)
blob := types.NewBlob(db, readers...)
_, err = db.CommitValue(ds, blob)
if err != nil {
cmd/noms/noms_diff_test.go
+2 -2
View File
@@ -62,11 +62,11 @@ func (s *nomsDiffTestSuite) TestNomsDiffSummarize() {
out, _ = s.MustRun(main, []string{"diff", "--summarize", r1 + ".value", r2 + ".value"})
s.NotContains(out, "Comparing commit values")
ds, err = db.CommitValue(ds, types.NewList(types.Number(1), types.Number(2), types.Number(3), types.Number(4)))
ds, err = db.CommitValue(ds, types.NewList(db, types.Number(1), types.Number(2), types.Number(3), types.Number(4)))
s.NoError(err)
r3 := spec.CreateHashSpecString("nbs", s.DBDir, ds.HeadRef().TargetHash()) + ".value"
ds, err = db.CommitValue(ds, types.NewList(types.Number(1), types.Number(222), types.Number(4)))
ds, err = db.CommitValue(ds, types.NewList(db, types.Number(1), types.Number(222), types.Number(4)))
s.NoError(err)
r4 := spec.CreateHashSpecString("nbs", s.DBDir, ds.HeadRef().TargetHash()) + ".value"
cmd/noms/noms_log_test.go
+23 -22
View File
@@ -81,13 +81,13 @@ func addCommitWithValue(ds datas.Dataset, v types.Value) (datas.Dataset, error)
return ds.Database().CommitValue(ds, v)
}
func addBranchedDataset(newDs, parentDs datas.Dataset, v string) (datas.Dataset, error) {
p := types.NewSet(parentDs.HeadRef())
func addBranchedDataset(vrw types.ValueReadWriter, newDs, parentDs datas.Dataset, v string) (datas.Dataset, error) {
p := types.NewSet(vrw, parentDs.HeadRef())
return newDs.Database().Commit(newDs, types.String(v), datas.CommitOptions{Parents: p})
}
func mergeDatasets(ds1, ds2 datas.Dataset, v string) (datas.Dataset, error) {
p := types.NewSet(ds1.HeadRef(), ds2.HeadRef())
func mergeDatasets(vrw types.ValueReadWriter, ds1, ds2 datas.Dataset, v string) (datas.Dataset, error) {
p := types.NewSet(vrw, ds1.HeadRef(), ds2.HeadRef())
return ds1.Database().Commit(ds1, types.String(v), datas.CommitOptions{Parents: p})
}
@@ -169,7 +169,7 @@ func (s *nomsLogTestSuite) TestNomsGraph1() {
s.NoError(err)
b2 := db.GetDataset("b2")
b2, err = addBranchedDataset(b2, b1, "3.1")
b2, err = addBranchedDataset(db, b2, b1, "3.1")
s.NoError(err)
b1, err = addCommit(b1, "3.2")
@@ -178,19 +178,19 @@ func (s *nomsLogTestSuite) TestNomsGraph1() {
s.NoError(err)
b3 := db.GetDataset("b3")
b3, err = addBranchedDataset(b3, b2, "3.1.3")
b3, err = addBranchedDataset(db, b3, b2, "3.1.3")
s.NoError(err)
b3, err = addCommit(b3, "3.1.5")
s.NoError(err)
b3, err = addCommit(b3, "3.1.7")
s.NoError(err)
b2, err = mergeDatasets(b2, b3, "3.5")
b2, err = mergeDatasets(db, b2, b3, "3.5")
s.NoError(err)
b2, err = addCommit(b2, "3.7")
s.NoError(err)
b1, err = mergeDatasets(b1, b2, "4")
b1, err = mergeDatasets(db, b1, b2, "4")
s.NoError(err)
b1, err = addCommit(b1, "5")
@@ -225,10 +225,10 @@ func (s *nomsLogTestSuite) TestNomsGraph2() {
bc, err = addCommit(bc, "100")
s.NoError(err)
ba, err = mergeDatasets(ba, bb, "11")
ba, err = mergeDatasets(db, ba, bb, "11")
s.NoError(err)
_, err = mergeDatasets(ba, bc, "101")
_, err = mergeDatasets(db, ba, bc, "101")
s.NoError(err)
res, _ := s.MustRun(main, []string{"log", "--graph", "--show-value", spec.CreateValueSpecString("nbs", s.DBDir, "ba")})
@@ -253,24 +253,24 @@ func (s *nomsLogTestSuite) TestNomsGraph3() {
s.NoError(err)
x := db.GetDataset("x")
x, err = addBranchedDataset(x, w, "20-x")
x, err = addBranchedDataset(db, x, w, "20-x")
s.NoError(err)
y := db.GetDataset("y")
y, err = addBranchedDataset(y, w, "200-y")
y, err = addBranchedDataset(db, y, w, "200-y")
s.NoError(err)
z := db.GetDataset("z")
z, err = addBranchedDataset(z, w, "2000-z")
z, err = addBranchedDataset(db, z, w, "2000-z")
s.NoError(err)
w, err = mergeDatasets(w, x, "22-wx")
w, err = mergeDatasets(db, w, x, "22-wx")
s.NoError(err)
w, err = mergeDatasets(w, y, "222-wy")
w, err = mergeDatasets(db, w, y, "222-wy")
s.NoError(err)
_, err = mergeDatasets(w, z, "2222-wz")
_, err = mergeDatasets(db, w, z, "2222-wz")
s.NoError(err)
res, _ := s.MustRun(main, []string{"log", "--graph", "--show-value", spec.CreateValueSpecString("nbs", s.DBDir, "w")})
@@ -280,19 +280,20 @@ func (s *nomsLogTestSuite) TestNomsGraph3() {
}
func (s *nomsLogTestSuite) TestTruncation() {
sp, err := spec.ForDatabase(spec.CreateDatabaseSpecString("nbs", s.DBDir))
s.NoError(err)
defer sp.Close()
db := sp.GetDatabase()
toNomsList := func(l []string) types.List {
nv := []types.Value{}
for _, v := range l {
nv = append(nv, types.String(v))
}
return types.NewList(nv...)
return types.NewList(db, nv...)
}
sp, err := spec.ForDatabase(spec.CreateDatabaseSpecString("nbs", s.DBDir))
s.NoError(err)
defer sp.Close()
t := sp.GetDatabase().GetDataset("truncate")
t := db.GetDataset("truncate")
t, err = addCommit(t, "the first line")
s.NoError(err)
cmd/noms/noms_merge.go
+1 -1
View File
@@ -66,7 +66,7 @@ func runMerge(args []string) int {
d.CheckErrorNoUsage(err)
close(pc)
_, err = db.SetHead(outDS, db.WriteValue(datas.NewCommit(merged, types.NewSet(leftDS.HeadRef(), rightDS.HeadRef()), types.EmptyStruct)))
_, err = db.SetHead(outDS, db.WriteValue(datas.NewCommit(merged, types.NewSet(db, leftDS.HeadRef(), rightDS.HeadRef()), types.EmptyStruct)))
d.PanicIfError(err)
if !verbose.Quiet() {
status.Printf("Done")
cmd/noms/noms_merge_test.go
+58 -28
View File
@@ -32,44 +32,51 @@ func (s *nomsMergeTestSuite) TearDownTest() {
func (s *nomsMergeTestSuite) TestNomsMerge_Success() {
left, right := "left", "right"
parentSpec := s.spec("parent")
defer parentSpec.Close()
leftSpec := s.spec(left)
defer leftSpec.Close()
rightSpec := s.spec(right)
defer rightSpec.Close()
p := s.setupMergeDataset(
"parent",
parentSpec,
types.StructData{
"num": types.Number(42),
"str": types.String("foobar"),
"lst": types.NewList(types.Number(1), types.String("foo")),
"map": types.NewMap(types.Number(1), types.String("foo"),
"lst": types.NewList(parentSpec.GetDatabase(), types.Number(1), types.String("foo")),
"map": types.NewMap(parentSpec.GetDatabase(), types.Number(1), types.String("foo"),
types.String("foo"), types.Number(1)),
},
types.NewSet())
types.NewSet(parentSpec.GetDatabase()))
l := s.setupMergeDataset(
left,
leftSpec,
types.StructData{
"num": types.Number(42),
"str": types.String("foobaz"),
"lst": types.NewList(types.Number(1), types.String("foo")),
"map": types.NewMap(types.Number(1), types.String("foo"),
"lst": types.NewList(leftSpec.GetDatabase(), types.Number(1), types.String("foo")),
"map": types.NewMap(leftSpec.GetDatabase(), types.Number(1), types.String("foo"),
types.String("foo"), types.Number(1)),
},
types.NewSet(p))
types.NewSet(leftSpec.GetDatabase(), p))
r := s.setupMergeDataset(
right,
rightSpec,
types.StructData{
"num": types.Number(42),
"str": types.String("foobar"),
"lst": types.NewList(types.Number(1), types.String("foo")),
"map": types.NewMap(types.Number(1), types.String("foo"),
"lst": types.NewList(rightSpec.GetDatabase(), types.Number(1), types.String("foo")),
"map": types.NewMap(rightSpec.GetDatabase(), types.Number(1), types.String("foo"),
types.String("foo"), types.Number(1), types.Number(2), types.String("bar")),
},
types.NewSet(p))
types.NewSet(rightSpec.GetDatabase(), p))
expected := types.NewStruct("", types.StructData{
"num": types.Number(42),
"str": types.String("foobaz"),
"lst": types.NewList(types.Number(1), types.String("foo")),
"map": types.NewMap(types.Number(1), types.String("foo"),
"lst": types.NewList(parentSpec.GetDatabase(), types.Number(1), types.String("foo")),
"map": types.NewMap(parentSpec.GetDatabase(), types.Number(1), types.String("foo"),
types.String("foo"), types.Number(1), types.Number(2), types.String("bar")),
})
@@ -83,23 +90,26 @@ func (s *nomsMergeTestSuite) TestNomsMerge_Success() {
}
}
func (s *nomsMergeTestSuite) setupMergeDataset(name string, data types.StructData, p types.Set) types.Ref {
func (s *nomsMergeTestSuite) spec(name string) spec.Spec {
sp, err := spec.ForDataset(spec.CreateValueSpecString("nbs", s.DBDir, name))
s.NoError(err)
defer sp.Close()
return sp
}
func (s *nomsMergeTestSuite) setupMergeDataset(sp spec.Spec, data types.StructData, p types.Set) types.Ref {
ds := sp.GetDataset()
ds, err = sp.GetDatabase().Commit(ds, types.NewStruct("", data), datas.CommitOptions{Parents: p})
ds, err := sp.GetDatabase().Commit(ds, types.NewStruct("", data), datas.CommitOptions{Parents: p})
s.NoError(err)
return ds.HeadRef()
}
func (s *nomsMergeTestSuite) validateDataset(name string, expected types.Struct, parents ...types.Value) {
sp, err := spec.ForDataset(spec.CreateValueSpecString("nbs", s.DBDir, name))
db := sp.GetDatabase()
if s.NoError(err) {
defer sp.Close()
commit := sp.GetDataset().Head()
s.True(commit.Get(datas.ParentsField).Equals(types.NewSet(parents...)))
s.True(commit.Get(datas.ParentsField).Equals(types.NewSet(db, parents...)))
merged := sp.GetDataset().HeadValue()
s.True(expected.Equals(merged), "%s != %s", types.EncodedValue(expected), types.EncodedValue(merged))
}
@@ -107,9 +117,16 @@ func (s *nomsMergeTestSuite) validateDataset(name string, expected types.Struct,
func (s *nomsMergeTestSuite) TestNomsMerge_Left() {
left, right := "left", "right"
p := s.setupMergeDataset("parent", types.StructData{"num": types.Number(42)}, types.NewSet())
l := s.setupMergeDataset(left, types.StructData{"num": types.Number(43)}, types.NewSet(p))
r := s.setupMergeDataset(right, types.StructData{"num": types.Number(44)}, types.NewSet(p))
parentSpec := s.spec("parent")
defer parentSpec.Close()
leftSpec := s.spec(left)
defer leftSpec.Close()
rightSpec := s.spec(right)
defer rightSpec.Close()
p := s.setupMergeDataset(parentSpec, types.StructData{"num": types.Number(42)}, types.NewSet(parentSpec.GetDatabase()))
l := s.setupMergeDataset(leftSpec, types.StructData{"num": types.Number(43)}, types.NewSet(leftSpec.GetDatabase(), p))
r := s.setupMergeDataset(rightSpec, types.StructData{"num": types.Number(44)}, types.NewSet(rightSpec.GetDatabase(), p))
expected := types.NewStruct("", types.StructData{"num": types.Number(43)})
@@ -125,9 +142,16 @@ func (s *nomsMergeTestSuite) TestNomsMerge_Left() {
func (s *nomsMergeTestSuite) TestNomsMerge_Right() {
left, right := "left", "right"
p := s.setupMergeDataset("parent", types.StructData{"num": types.Number(42)}, types.NewSet())
l := s.setupMergeDataset(left, types.StructData{"num": types.Number(43)}, types.NewSet(p))
r := s.setupMergeDataset(right, types.StructData{"num": types.Number(44)}, types.NewSet(p))
parentSpec := s.spec("parent")
defer parentSpec.Close()
leftSpec := s.spec(left)
defer leftSpec.Close()
rightSpec := s.spec(right)
defer rightSpec.Close()
p := s.setupMergeDataset(parentSpec, types.StructData{"num": types.Number(42)}, types.NewSet(parentSpec.GetDatabase()))
l := s.setupMergeDataset(leftSpec, types.StructData{"num": types.Number(43)}, types.NewSet(leftSpec.GetDatabase(), p))
r := s.setupMergeDataset(rightSpec, types.StructData{"num": types.Number(44)}, types.NewSet(rightSpec.GetDatabase(), p))
expected := types.NewStruct("", types.StructData{"num": types.Number(44)})
@@ -143,9 +167,15 @@ func (s *nomsMergeTestSuite) TestNomsMerge_Right() {
func (s *nomsMergeTestSuite) TestNomsMerge_Conflict() {
left, right := "left", "right"
p := s.setupMergeDataset("parent", types.StructData{"num": types.Number(42)}, types.NewSet())
s.setupMergeDataset(left, types.StructData{"num": types.Number(43)}, types.NewSet(p))
s.setupMergeDataset(right, types.StructData{"num": types.Number(44)}, types.NewSet(p))
parentSpec := s.spec("parent")
defer parentSpec.Close()
leftSpec := s.spec(left)
defer leftSpec.Close()
rightSpec := s.spec(right)
defer rightSpec.Close()
p := s.setupMergeDataset(parentSpec, types.StructData{"num": types.Number(42)}, types.NewSet(parentSpec.GetDatabase()))
s.setupMergeDataset(leftSpec, types.StructData{"num": types.Number(43)}, types.NewSet(leftSpec.GetDatabase(), p))
s.setupMergeDataset(rightSpec, types.StructData{"num": types.Number(44)}, types.NewSet(rightSpec.GetDatabase(), p))
s.Panics(func() { s.MustRun(main, []string{"merge", s.DBDir, left, right, "output"}) })
}
@@ -171,7 +201,7 @@ func (s *nomsMergeTestSuite) TestBadInput() {
prep := func(dsName string) {
ds := db.GetDataset(dsName)
db.CommitValue(ds, types.NewMap(types.String("foo"), types.String("bar")))
db.CommitValue(ds, types.NewMap(db, types.String("foo"), types.String("bar")))
}
prep(l)
prep(r)
cmd/noms/noms_show_test.go
+10 -4
View File
@@ -32,14 +32,18 @@ const (
res5 = "Commit {\n meta: {},\n parents: {\n 3tmg89vabs2k6hotdock1kuo13j4lmqv,\n },\n value: 5cgfu2vk4nc21m1vjkjjpd2kvcm2df7q,\n}\n"
)
func (s *nomsShowTestSuite) writeTestData(str string, value types.Value) types.Ref {
func (s *nomsShowTestSuite) spec(str string) spec.Spec {
sp, err := spec.ForDataset(str)
s.NoError(err)
return sp
}
func (s *nomsShowTestSuite) writeTestData(str string, value types.Value) types.Ref {
sp := s.spec(str)
defer sp.Close()
db := sp.GetDatabase()
r1 := db.WriteValue(value)
_, err = db.CommitValue(sp.GetDataset(), r1)
_, err := db.CommitValue(sp.GetDataset(), r1)
s.NoError(err)
return r1
@@ -58,7 +62,9 @@ func (s *nomsShowTestSuite) TestNomsShow() {
res, _ = s.MustRun(main, []string{"show", str1})
s.Equal(res2, res)
list := types.NewList(types.String("elem1"), types.Number(2), types.String("elem3"))
sp := s.spec(str)
defer sp.Close()
list := types.NewList(sp.GetDatabase(), types.String("elem1"), types.Number(2), types.String("elem3"))
r = s.writeTestData(str, list)
res, _ = s.MustRun(main, []string{"show", str})
test.EqualsIgnoreHashes(s.T(), res3, res)
@@ -112,7 +118,7 @@ func (s *nomsShowTestSuite) TestNomsShowRaw() {
for i := 0; i < len(items); i++ {
items[i] = types.Number(i)
}
l := types.NewList(items...)
l := types.NewList(db, items...)
numChildChunks := 0
l.WalkRefs(func(r types.Ref) {
numChildChunks++
cmd/noms/splore/noms_splore_test.go
+23 -22
View File
@@ -67,14 +67,15 @@ func TestNomsSplore(t *testing.T) {
sp, err := spec.ForDataset(fmt.Sprintf("nbs:%s::ds", dir))
d.PanicIfError(err)
defer sp.Close()
db := sp.GetDatabase()
strct := types.NewStruct("StructName", types.StructData{
"blob": types.NewBlob(),
"blob": types.NewBlob(db),
"bool": types.Bool(true),
"list": types.NewList(types.Number(1), types.Number(2)),
"map": types.NewMap(types.String("a"), types.String("b"), types.String("c"), types.String("d")),
"list": types.NewList(db, types.Number(1), types.Number(2)),
"map": types.NewMap(db, types.String("a"), types.String("b"), types.String("c"), types.String("d")),
"number": types.Number(42),
"ref": sp.GetDatabase().WriteValue(types.Bool(true)),
"set": types.NewSet(types.Number(3), types.Number(4)),
"ref": db.WriteValue(types.Bool(true)),
"set": types.NewSet(db, types.Number(3), types.Number(4)),
"string": types.String("hello world"),
"typeCompound": types.MakeMapType(types.StringType, types.MakeListType(types.BoolType)),
"typePrimitive": types.NumberType,
@@ -587,28 +588,28 @@ func TestNomsSploreGetMetaChildren(t *testing.T) {
// A bunch of lists with just numbers or ref<number>s in them. None of these
// should be detected as meta sequences:
l1 := types.NewList()
l1 := types.NewList(db)
assert.Nil(getMetaChildren(l1))
l2 := types.NewList(types.Number(1))
l2 := types.NewList(db, types.Number(1))
assert.Nil(getMetaChildren(l2))
l3 := types.NewList(types.Number(1), types.Number(2))
l3 := types.NewList(db, types.Number(1), types.Number(2))
assert.Nil(getMetaChildren(l3))
l4 := types.NewList(db.WriteValue(types.Number(1)))
l4 := types.NewList(db, db.WriteValue(types.Number(1)))
assert.Nil(getMetaChildren(l4))
l5 := types.NewList(db.WriteValue(types.Number(1)), types.Number(2))
l5 := types.NewList(db, db.WriteValue(types.Number(1)), types.Number(2))
assert.Nil(getMetaChildren(l5))
l6 := types.NewList(db.WriteValue(types.Number(1)), db.WriteValue(types.Number(2)))
l6 := types.NewList(db, db.WriteValue(types.Number(1)), db.WriteValue(types.Number(2)))
assert.Nil(getMetaChildren(l6))
l7 := types.NewList(l1)
l7 := types.NewList(db, l1)
assert.Nil(getMetaChildren(l7))
l8 := types.NewList(l4)
l8 := types.NewList(db, l4)
assert.Nil(getMetaChildren(l8))
// List with more or equal ref<list> than elements. This can't possibly be a meta
@@ -617,21 +618,21 @@ func TestNomsSploreGetMetaChildren(t *testing.T) {
l1Ref := db.WriteValue(l1)
l2Ref := db.WriteValue(l2)
l3Ref := db.WriteValue(l3)
listRefList := types.NewList(l1Ref, l2Ref, l3Ref)
listRefList := types.NewList(db, l1Ref, l2Ref, l3Ref)
l9 := types.NewList(listRefList)
l9 := types.NewList(db, listRefList)
assert.Nil(getMetaChildren(l9))
l10 := types.NewList(types.Number(1), listRefList)
l10 := types.NewList(db, types.Number(1), listRefList)
assert.Nil(getMetaChildren(l10))
l11 := listRefList
assert.Nil(getMetaChildren(l11))
l12 := types.NewList(types.Number(1), types.Number(2), listRefList)
l12 := types.NewList(db, types.Number(1), types.Number(2), listRefList)
assert.Nil(getMetaChildren(l12))
l13 := types.NewList(types.Number(1), db.WriteValue(types.Number(2)), listRefList)
l13 := types.NewList(db, types.Number(1), db.WriteValue(types.Number(2)), listRefList)
assert.Nil(getMetaChildren(l13))
// List with fewer ref<list> as children. For now this is the closet
@@ -646,15 +647,15 @@ func TestNomsSploreGetMetaChildren(t *testing.T) {
nodeChild{Value: nodeInfo{HasChildren: true, ID: l3Hash, Name: "List" + l3Hash}},
}
l14 := types.NewList(types.Number(1), types.Number(2), types.Number(3), listRefList)
l14 := types.NewList(db, types.Number(1), types.Number(2), types.Number(3), listRefList)
assert.Equal(expectNodeChildren, getMetaChildren(l14))
l15 := types.NewList(types.Number(1), types.Number(2), db.WriteValue(types.Number(3)), listRefList)
l15 := types.NewList(db, types.Number(1), types.Number(2), db.WriteValue(types.Number(3)), listRefList)
assert.Equal(expectNodeChildren, getMetaChildren(l15))
l16 := types.NewList(types.Number(1), types.Number(2), types.Number(3), types.Number(4), listRefList)
l16 := types.NewList(db, types.Number(1), types.Number(2), types.Number(3), types.Number(4), listRefList)
assert.Equal(expectNodeChildren, getMetaChildren(l16))
l17 := types.NewList(types.Number(1), types.Number(2), db.WriteValue(types.Number(3)), db.WriteValue(types.Number(4)), listRefList)
l17 := types.NewList(db, types.Number(1), types.Number(2), db.WriteValue(types.Number(3)), db.WriteValue(types.Number(4)), listRefList)
assert.Equal(expectNodeChildren, getMetaChildren(l17))
}
go/datas/commit_test.go
+27 -14
View File
@@ -22,7 +22,11 @@ func TestNewCommit(t *testing.T) {
assert.True(a.Equals(e), "Actual: %s\nExpected %s", a.Describe(), e.Describe())
}
commit := NewCommit(types.Number(1), types.NewSet(), types.EmptyStruct)
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
commit := NewCommit(types.Number(1), types.NewSet(db), types.EmptyStruct)
at := types.TypeOf(commit)
et := makeCommitStructType(
types.EmptyStructType,
@@ -32,7 +36,7 @@ func TestNewCommit(t *testing.T) {
assertTypeEquals(et, at)
// Committing another Number
commit2 := NewCommit(types.Number(2), types.NewSet(types.NewRef(commit)), types.EmptyStruct)
commit2 := NewCommit(types.Number(2), types.NewSet(db, types.NewRef(commit)), types.EmptyStruct)
at2 := types.TypeOf(commit2)
et2 := nomdl.MustParseType(`struct Commit {
meta: struct {},
@@ -42,7 +46,7 @@ func TestNewCommit(t *testing.T) {
assertTypeEquals(et2, at2)
// Now commit a String
commit3 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), types.EmptyStruct)
commit3 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2)), types.EmptyStruct)
at3 := types.TypeOf(commit3)
et3 := nomdl.MustParseType(`struct Commit {
meta: struct {},
@@ -58,7 +62,7 @@ func TestNewCommit(t *testing.T) {
number: Number,
}`)
assertTypeEquals(metaType, types.TypeOf(meta))
commit4 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2)), meta)
commit4 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2)), meta)
at4 := types.TypeOf(commit4)
et4 := nomdl.MustParseType(`struct Commit {
meta: struct {} | struct Meta {
@@ -71,7 +75,7 @@ func TestNewCommit(t *testing.T) {
assertTypeEquals(et4, at4)
// Merge-commit with different parent types
commit5 := NewCommit(types.String("Hi"), types.NewSet(types.NewRef(commit2), types.NewRef(commit3)), types.EmptyStruct)
commit5 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2), types.NewRef(commit3)), types.EmptyStruct)
at5 := types.TypeOf(commit5)
et5 := nomdl.MustParseType(`struct Commit {
meta: struct {},
@@ -83,9 +87,14 @@ func TestNewCommit(t *testing.T) {
func TestCommitWithoutMetaField(t *testing.T) {
assert := assert.New(t)
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
metaCommit := types.NewStruct("Commit", types.StructData{
"value": types.Number(9),
"parents": types.NewSet(),
"parents": types.NewSet(db),
"meta": types.EmptyStruct,
})
assert.True(IsCommit(metaCommit))
@@ -93,19 +102,19 @@ func TestCommitWithoutMetaField(t *testing.T) {
noMetaCommit := types.NewStruct("Commit", types.StructData{
"value": types.Number(9),
"parents": types.NewSet(),
"parents": types.NewSet(db),
})
assert.False(IsCommit(noMetaCommit))
assert.False(IsCommitType(types.TypeOf(noMetaCommit)))
}
// Convert list of Struct's to Set<Ref>
func toRefSet(commits ...types.Struct) types.Set {
set := types.NewSet().Edit()
func toRefSet(vrw types.ValueReadWriter, commits ...types.Struct) types.Set {
set := types.NewSet(vrw).Edit()
for _, p := range commits {
set.Insert(types.NewRef(p))
}
return set.Set(nil)
return set.Set()
}
// Convert Set<Ref<Struct>> to a string of Struct.Get("value")'s
@@ -127,7 +136,7 @@ func TestFindCommonAncestor(t *testing.T) {
addCommit := func(datasetID string, val string, parents ...types.Struct) types.Struct {
ds := db.GetDataset(datasetID)
var err error
ds, err = db.Commit(ds, types.String(val), CommitOptions{Parents: toRefSet(parents...)})
ds, err = db.Commit(ds, types.String(val), CommitOptions{Parents: toRefSet(db, parents...)})
assert.NoError(err)
return ds.Head()
}
@@ -196,10 +205,14 @@ func TestFindCommonAncestor(t *testing.T) {
}
func TestNewCommitRegressionTest(t *testing.T) {
c1 := NewCommit(types.String("one"), types.NewSet(), types.EmptyStruct)
cx := NewCommit(types.Bool(true), types.NewSet(), types.EmptyStruct)
storage := &chunks.TestStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
c1 := NewCommit(types.String("one"), types.NewSet(db), types.EmptyStruct)
cx := NewCommit(types.Bool(true), types.NewSet(db), types.EmptyStruct)
value := types.String("two")
parents := types.NewSet(types.NewRef(c1))
parents := types.NewSet(db, types.NewRef(c1))
meta := types.NewStruct("", types.StructData{
"basis": cx,
})
go/datas/database_common.go
+7 -7
View File
@@ -54,7 +54,7 @@ func (db *database) Stats() interface{} {
func (db *database) Datasets() types.Map {
rootHash := db.rt.Root()
if rootHash.IsEmpty() {
return types.NewMap()
return types.NewMap(db)
}
return db.ReadValue(rootHash).(types.Map)
@@ -93,7 +93,7 @@ func (db *database) doSetHead(ds Dataset, newHeadRef types.Ref) error {
currentRootHash, currentDatasets := db.rt.Root(), db.Datasets()
commitRef := db.WriteValue(commit) // will be orphaned if the tryCommitChunks() below fails
currentDatasets = currentDatasets.Edit().Set(types.String(ds.ID()), types.ToRefOfValue(commitRef)).Map(nil)
currentDatasets = currentDatasets.Edit().Set(types.String(ds.ID()), types.ToRefOfValue(commitRef)).Map()
return db.tryCommitChunks(currentDatasets, currentRootHash)
}
@@ -161,11 +161,11 @@ func (db *database) doCommit(datasetID string, commit types.Struct, mergePolicy
if err != nil {
return err
}
commitRef = db.WriteValue(NewCommit(merged, types.NewSet(commitRef, currentHeadRef), types.EmptyStruct))
commitRef = db.WriteValue(NewCommit(merged, types.NewSet(db, commitRef, currentHeadRef), types.EmptyStruct))
}
}
}
currentDatasets = currentDatasets.Edit().Set(types.String(datasetID), types.ToRefOfValue(commitRef)).Map(nil)
currentDatasets = currentDatasets.Edit().Set(types.String(datasetID), types.ToRefOfValue(commitRef)).Map()
err = db.tryCommitChunks(currentDatasets, currentRootHash)
}
return err
@@ -188,7 +188,7 @@ func (db *database) doDelete(datasetIDstr string) error {
var err error
for {
currentDatasets = currentDatasets.Edit().Remove(datasetID).Map(nil)
currentDatasets = currentDatasets.Edit().Remove(datasetID).Map()
err = db.tryCommitChunks(currentDatasets, currentRootHash)
if err != ErrOptimisticLockFailed {
break
@@ -227,9 +227,9 @@ func (db *database) validateRefAsCommit(r types.Ref) types.Struct {
func buildNewCommit(ds Dataset, v types.Value, opts CommitOptions) types.Struct {
parents := opts.Parents
if (parents == types.Set{}) {
parents = types.NewSet()
parents = types.NewSet(ds.Database())
if headRef, ok := ds.MaybeHeadRef(); ok {
parents = parents.Edit().Insert(headRef).Set(nil)
parents = parents.Edit().Insert(headRef).Set()
}
}
go/datas/database_test.go
+28 -28
View File
@@ -81,17 +81,17 @@ func (suite *DatabaseSuite) TestCompletenessCheck() {
datasetID := "ds1"
ds1 := suite.db.GetDataset(datasetID)
se := types.NewSet().Edit()
se := types.NewSet(suite.db).Edit()
for i := 0; i < 100; i++ {
se.Insert(suite.db.WriteValue(types.Number(100)))
}
s := se.Set(nil)
s := se.Set()
ds1, err := suite.db.CommitValue(ds1, s)
suite.NoError(err)
s = ds1.HeadValue().(types.Set)
s = s.Edit().Insert(types.NewRef(types.Number(1000))).Set(nil) // danging ref
s = s.Edit().Insert(types.NewRef(types.Number(1000))).Set() // danging ref
suite.Panics(func() {
ds1, err = suite.db.CommitValue(ds1, s)
})
@@ -192,7 +192,7 @@ func (suite *DatabaseSuite) TestDatabaseCommit() {
// \----|c|
// Should be disallowed.
c := types.String("c")
ds, err = suite.db.Commit(ds, c, newOpts(aCommitRef))
ds, err = suite.db.Commit(ds, c, newOpts(suite.db, aCommitRef))
suite.Error(err)
suite.True(ds.HeadValue().Equals(b))
@@ -205,7 +205,7 @@ func (suite *DatabaseSuite) TestDatabaseCommit() {
// Attempt to recommit |b| with |a| as parent.
// Should be disallowed.
ds, err = suite.db.Commit(ds, b, newOpts(aCommitRef))
ds, err = suite.db.Commit(ds, b, newOpts(suite.db, aCommitRef))
suite.Error(err)
suite.True(ds.HeadValue().Equals(d))
@@ -239,8 +239,8 @@ func (suite *DatabaseSuite) TestDatasetsMapType() {
suite.NotPanics(func() { assertMapOfStringToRefOfCommit(suite.db.Datasets(), datasets, suite.db) })
}
func newOpts(parents ...types.Value) CommitOptions {
return CommitOptions{Parents: types.NewSet(parents...)}
func newOpts(vrw types.ValueReadWriter, parents ...types.Value) CommitOptions {
return CommitOptions{Parents: types.NewSet(vrw, parents...)}
}
func (suite *DatabaseSuite) TestDatabaseDuplicateCommit() {
@@ -262,41 +262,41 @@ func (suite *DatabaseSuite) TestDatabaseCommitMerge() {
ds1, ds2 := suite.db.GetDataset(datasetID1), suite.db.GetDataset(datasetID2)
var err error
v := types.NewMap(types.String("Hello"), types.Number(42))
v := types.NewMap(suite.db, types.String("Hello"), types.Number(42))
ds1, err = suite.db.CommitValue(ds1, v)
ds1First := ds1
suite.NoError(err)
ds1, err = suite.db.CommitValue(ds1, v.Edit().Set(types.String("Friends"), types.Bool(true)).Map(suite.db))
ds1, err = suite.db.CommitValue(ds1, v.Edit().Set(types.String("Friends"), types.Bool(true)).Map())
suite.NoError(err)
ds2, err = suite.db.CommitValue(ds2, types.String("Goodbye"))
suite.NoError(err)
// No common ancestor
_, err = suite.db.Commit(ds1, types.Number(47), newOpts(ds2.HeadRef()))
_, err = suite.db.Commit(ds1, types.Number(47), newOpts(suite.db, ds2.HeadRef()))
suite.IsType(ErrMergeNeeded, err, "%s", err)
// Unmergeable
_, err = suite.db.Commit(ds1, types.Number(47), newOptsWithMerge(merge.None, ds1First.HeadRef()))
_, err = suite.db.Commit(ds1, types.Number(47), newOptsWithMerge(suite.db, merge.None, ds1First.HeadRef()))
suite.IsType(&merge.ErrMergeConflict{}, err, "%s", err)
// Merge policies
newV := v.Edit().Set(types.String("Friends"), types.Bool(false)).Map(suite.db)
_, err = suite.db.Commit(ds1, newV, newOptsWithMerge(merge.None, ds1First.HeadRef()))
newV := v.Edit().Set(types.String("Friends"), types.Bool(false)).Map()
_, err = suite.db.Commit(ds1, newV, newOptsWithMerge(suite.db, merge.None, ds1First.HeadRef()))
suite.IsType(&merge.ErrMergeConflict{}, err, "%s", err)
theirs, err := suite.db.Commit(ds1, newV, newOptsWithMerge(merge.Theirs, ds1First.HeadRef()))
theirs, err := suite.db.Commit(ds1, newV, newOptsWithMerge(suite.db, merge.Theirs, ds1First.HeadRef()))
suite.NoError(err)
suite.True(types.Bool(true).Equals(theirs.HeadValue().(types.Map).Get(types.String("Friends"))))
newV = v.Edit().Set(types.String("Friends"), types.Number(47)).Map(suite.db)
ours, err := suite.db.Commit(ds1First, newV, newOptsWithMerge(merge.Ours, ds1First.HeadRef()))
newV = v.Edit().Set(types.String("Friends"), types.Number(47)).Map()
ours, err := suite.db.Commit(ds1First, newV, newOptsWithMerge(suite.db, merge.Ours, ds1First.HeadRef()))
suite.NoError(err)
suite.True(types.Number(47).Equals(ours.HeadValue().(types.Map).Get(types.String("Friends"))))
}
func newOptsWithMerge(policy merge.ResolveFunc, parents ...types.Value) CommitOptions {
return CommitOptions{Parents: types.NewSet(parents...), Policy: merge.NewThreeWay(policy)}
func newOptsWithMerge(vrw types.ValueReadWriter, policy merge.ResolveFunc, parents ...types.Value) CommitOptions {
return CommitOptions{Parents: types.NewSet(vrw, parents...), Policy: merge.NewThreeWay(policy)}
}
func (suite *DatabaseSuite) TestDatabaseDelete() {
@@ -516,33 +516,33 @@ func (suite *DatabaseSuite) TestDatabaseHeightOfCollections() {
// Set<String>
v1 := types.String("hello")
v2 := types.String("world")
s1 := types.NewSet(v1, v2)
s1 := types.NewSet(suite.db, v1, v2)
suite.Equal(uint64(1), suite.db.WriteValue(s1).Height())
// Set<Ref<String>>
s2 := types.NewSet(suite.db.WriteValue(v1), suite.db.WriteValue(v2))
s2 := types.NewSet(suite.db, suite.db.WriteValue(v1), suite.db.WriteValue(v2))
suite.Equal(uint64(2), suite.db.WriteValue(s2).Height())
// List<Set<String>>
v3 := types.String("foo")
v4 := types.String("bar")
s3 := types.NewSet(v3, v4)
l1 := types.NewList(s1, s3)
s3 := types.NewSet(suite.db, v3, v4)
l1 := types.NewList(suite.db, s1, s3)
suite.Equal(uint64(1), suite.db.WriteValue(l1).Height())
// List<Ref<Set<String>>
l2 := types.NewList(suite.db.WriteValue(s1), suite.db.WriteValue(s3))
l2 := types.NewList(suite.db, suite.db.WriteValue(s1), suite.db.WriteValue(s3))
suite.Equal(uint64(2), suite.db.WriteValue(l2).Height())
// List<Ref<Set<Ref<String>>>
s4 := types.NewSet(suite.db.WriteValue(v3), suite.db.WriteValue(v4))
l3 := types.NewList(suite.db.WriteValue(s4))
s4 := types.NewSet(suite.db, suite.db.WriteValue(v3), suite.db.WriteValue(v4))
l3 := types.NewList(suite.db, suite.db.WriteValue(s4))
suite.Equal(uint64(3), suite.db.WriteValue(l3).Height())
// List<Set<String> | RefValue<Set<String>>>
l4 := types.NewList(s1, suite.db.WriteValue(s3))
l4 := types.NewList(suite.db, s1, suite.db.WriteValue(s3))
suite.Equal(uint64(2), suite.db.WriteValue(l4).Height())
l5 := types.NewList(suite.db.WriteValue(s1), s3)
l5 := types.NewList(suite.db, suite.db.WriteValue(s1), s3)
suite.Equal(uint64(2), suite.db.WriteValue(l5).Height())
// Familiar with the "New Jersey Turnpike" drink? Here's the noms version of that...
@@ -553,7 +553,7 @@ func (suite *DatabaseSuite) TestDatabaseHeightOfCollections() {
}
andMore = append(andMore, setOfStringType, setOfRefOfStringType)
suite.db.WriteValue(types.NewList(andMore...))
suite.db.WriteValue(types.NewList(suite.db, andMore...))
}
func (suite *DatabaseSuite) TestMetaOption() {
go/datas/dataset_test.go
+2 -2
View File
@@ -31,7 +31,7 @@ func TestExplicitBranchUsingDatasets(t *testing.T) {
// ds1: |a|
// \ds2
ds2 := store.GetDataset(id2)
ds2, err = store.Commit(ds2, ds1.HeadValue(), CommitOptions{Parents: types.NewSet(ds1.HeadRef())})
ds2, err = store.Commit(ds2, ds1.HeadValue(), CommitOptions{Parents: types.NewSet(store, ds1.HeadRef())})
assert.NoError(err)
assert.True(ds2.Head().Get(ValueField).Equals(a))
@@ -50,7 +50,7 @@ func TestExplicitBranchUsingDatasets(t *testing.T) {
// ds1: |a| <- |b| <--|d|
// \ds2 <- |c| <--/
mergeParents := types.NewSet(types.NewRef(ds1.Head()), types.NewRef(ds2.Head()))
mergeParents := types.NewSet(store, types.NewRef(ds1.Head()), types.NewRef(ds2.Head()))
d := types.String("d")
ds2, err = store.Commit(ds2, d, CommitOptions{Parents: mergeParents})
assert.NoError(err)
go/datas/http_chunk_store_test.go
+21 -11
View File
@@ -151,7 +151,7 @@ func (suite *HTTPChunkStoreSuite) TestPutChunk() {
suite.True(suite.http.Has(c.Hash()))
suite.True(suite.http.Commit(hash.Hash{}, hash.Hash{}))
suite.Equal(1, suite.serverCS.Writes)
suite.Equal(2, suite.serverCS.Writes)
}
func (suite *HTTPChunkStoreSuite) TestPutChunksInOrder() {
@@ -159,20 +159,24 @@ func (suite *HTTPChunkStoreSuite) TestPutChunksInOrder() {
types.String("abc"),
types.String("def"),
}
le := types.NewList().Edit()
vs := types.NewValueStore(suite.serverCS)
defer vs.Close()
le := types.NewList(vs).Edit()
for _, val := range vals {
suite.http.Put(types.EncodeValue(val))
le.Append(types.NewRef(val))
}
suite.http.Put(types.EncodeValue(le.List(nil)))
suite.http.Put(types.EncodeValue(le.List()))
suite.True(suite.http.Commit(hash.Hash{}, hash.Hash{}))
suite.Equal(3, suite.serverCS.Writes)
suite.Equal(4, suite.serverCS.Writes)
}
func (suite *HTTPChunkStoreSuite) TestRebase() {
suite.Equal(hash.Hash{}, suite.http.Root())
c := types.EncodeValue(types.NewMap())
db := NewDatabase(suite.serverCS)
defer db.Close()
c := types.EncodeValue(types.NewMap(db))
suite.serverCS.Put(c)
suite.True(suite.serverCS.Commit(c.Hash(), hash.Hash{})) // change happens behind our backs
suite.Equal(hash.Hash{}, suite.http.Root()) // shouldn't be visible yet
@@ -182,7 +186,9 @@ func (suite *HTTPChunkStoreSuite) TestRebase() {
}
func (suite *HTTPChunkStoreSuite) TestRoot() {
c := types.EncodeValue(types.NewMap())
db := NewDatabase(suite.serverCS)
defer db.Close()
c := types.EncodeValue(types.NewMap(db))
suite.serverCS.Put(c)
suite.True(suite.http.Commit(c.Hash(), hash.Hash{}))
suite.Equal(c.Hash(), suite.serverCS.Root())
@@ -190,14 +196,17 @@ func (suite *HTTPChunkStoreSuite) TestRoot() {
func (suite *HTTPChunkStoreSuite) TestVersionMismatch() {
store := newBadVersionHTTPChunkStoreForTest(suite.serverCS)
defer store.Close()
c := types.EncodeValue(types.NewMap())
vs := types.NewValueStore(store)
defer vs.Close()
c := types.EncodeValue(types.NewMap(vs))
suite.serverCS.Put(c)
suite.Panics(func() { store.Commit(c.Hash(), hash.Hash{}) })
}
func (suite *HTTPChunkStoreSuite) TestCommit() {
c := types.EncodeValue(types.NewMap())
db := NewDatabase(suite.serverCS)
defer db.Close()
c := types.EncodeValue(types.NewMap(db))
suite.serverCS.Put(c)
suite.True(suite.http.Commit(c.Hash(), hash.Hash{}))
suite.Equal(c.Hash(), suite.serverCS.Root())
@@ -211,8 +220,9 @@ func (suite *HTTPChunkStoreSuite) TestEmptyHashCommit() {
func (suite *HTTPChunkStoreSuite) TestCommitWithParams() {
u := fmt.Sprintf("http://localhost:9000?access_token=%s&other=19", testAuthToken)
store := newAuthenticatingHTTPChunkStoreForTest(suite.Assert(), suite.serverCS, u)
defer store.Close()
c := types.EncodeValue(types.NewMap())
vs := types.NewValueStore(store)
defer vs.Close()
c := types.EncodeValue(types.NewMap(vs))
suite.serverCS.Put(c)
suite.True(store.Commit(c.Hash(), hash.Hash{}))
suite.Equal(c.Hash(), suite.serverCS.Root())
go/datas/pull_test.go
+21 -21
View File
@@ -149,7 +149,7 @@ func (suite *PullSuite) TestPullEverything() {
expectedReads := suite.sinkCS.Reads
l := buildListOfHeight(2, suite.source)
sourceRef := suite.commitToSource(l, types.NewSet())
sourceRef := suite.commitToSource(l, types.NewSet(suite.source))
pt := startProgressTracker()
Pull(suite.source, suite.sink, sourceRef, pt.Ch)
@@ -182,15 +182,15 @@ func (suite *PullSuite) TestPullEverything() {
// \ -1-> L0
func (suite *PullSuite) TestPullMultiGeneration() {
sinkL := buildListOfHeight(2, suite.sink)
suite.commitToSink(sinkL, types.NewSet())
suite.commitToSink(sinkL, types.NewSet(suite.sink))
expectedReads := suite.sinkCS.Reads
srcL := buildListOfHeight(2, suite.source)
sourceRef := suite.commitToSource(srcL, types.NewSet())
sourceRef := suite.commitToSource(srcL, types.NewSet(suite.source))
srcL = buildListOfHeight(4, suite.source)
sourceRef = suite.commitToSource(srcL, types.NewSet(sourceRef))
sourceRef = suite.commitToSource(srcL, types.NewSet(suite.source, sourceRef))
srcL = buildListOfHeight(5, suite.source)
sourceRef = suite.commitToSource(srcL, types.NewSet(sourceRef))
sourceRef = suite.commitToSource(srcL, types.NewSet(suite.source, sourceRef))
pt := startProgressTracker()
@@ -228,14 +228,14 @@ func (suite *PullSuite) TestPullMultiGeneration() {
// \ -1-> L0
func (suite *PullSuite) TestPullDivergentHistory() {
sinkL := buildListOfHeight(3, suite.sink)
sinkRef := suite.commitToSink(sinkL, types.NewSet())
sinkRef := suite.commitToSink(sinkL, types.NewSet(suite.sink))
srcL := buildListOfHeight(3, suite.source)
sourceRef := suite.commitToSource(srcL, types.NewSet())
sourceRef := suite.commitToSource(srcL, types.NewSet(suite.source))
sinkL = sinkL.Edit().Append(types.String("oy!")).List(nil)
sinkRef = suite.commitToSink(sinkL, types.NewSet(sinkRef))
srcL = srcL.Edit().Set(1, buildListOfHeight(5, suite.source)).List(nil)
sourceRef = suite.commitToSource(srcL, types.NewSet(sourceRef))
sinkL = sinkL.Edit().Append(types.String("oy!")).List()
sinkRef = suite.commitToSink(sinkL, types.NewSet(suite.sink, sinkRef))
srcL = srcL.Edit().Set(1, buildListOfHeight(5, suite.source)).List()
sourceRef = suite.commitToSource(srcL, types.NewSet(suite.source, sourceRef))
preReads := suite.sinkCS.Reads
pt := startProgressTracker()
@@ -268,17 +268,17 @@ func (suite *PullSuite) TestPullDivergentHistory() {
// \ -1-> L0
func (suite *PullSuite) TestPullUpdates() {
sinkL := buildListOfHeight(4, suite.sink)
suite.commitToSink(sinkL, types.NewSet())
suite.commitToSink(sinkL, types.NewSet(suite.sink))
expectedReads := suite.sinkCS.Reads
srcL := buildListOfHeight(4, suite.source)
sourceRef := suite.commitToSource(srcL, types.NewSet())
sourceRef := suite.commitToSource(srcL, types.NewSet(suite.source))
L3 := srcL.Get(1).(types.Ref).TargetValue(suite.source).(types.List)
L2 := L3.Get(1).(types.Ref).TargetValue(suite.source).(types.List)
L2 = L2.Edit().Append(suite.source.WriteValue(types.String("oy!"))).List(nil)
L3 = L3.Edit().Set(1, suite.source.WriteValue(L2)).List(nil)
srcL = srcL.Edit().Set(1, suite.source.WriteValue(L3)).List(nil)
sourceRef = suite.commitToSource(srcL, types.NewSet(sourceRef))
L2 = L2.Edit().Append(suite.source.WriteValue(types.String("oy!"))).List()
L3 = L3.Edit().Set(1, suite.source.WriteValue(L2)).List()
srcL = srcL.Edit().Set(1, suite.source.WriteValue(L3)).List()
sourceRef = suite.commitToSource(srcL, types.NewSet(suite.source, sourceRef))
pt := startProgressTracker()
@@ -306,15 +306,15 @@ func (suite *PullSuite) commitToSink(v types.Value, p types.Set) types.Ref {
return ds.HeadRef()
}
func buildListOfHeight(height int, vw types.ValueWriter) types.List {
func buildListOfHeight(height int, vrw types.ValueReadWriter) types.List {
unique := 0
l := types.NewList(types.Number(unique), types.Number(unique+1))
l := types.NewList(vrw, types.Number(unique), types.Number(unique+1))
unique += 2
for i := 0; i < height; i++ {
r1, r2 := vw.WriteValue(types.Number(unique)), vw.WriteValue(l)
r1, r2 := vrw.WriteValue(types.Number(unique)), vrw.WriteValue(l)
unique++
l = types.NewList(r1, r2)
l = types.NewList(vrw, r1, r2)
}
return l
}
go/datas/remote_database_handlers.go
+8 -8
View File
@@ -398,7 +398,7 @@ func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, cs c
// round trip to the client and just retry inline.
for to, from := proposed, last; !vs.Commit(to, from); {
// If committing failed, we go read out the map of Datasets at the root of the store, which is a Map[string]Ref<Commit>
rootMap := types.NewMap()
rootMap := types.NewMap(vs)
root := vs.Root()
if v := vs.ReadValue(root); v != nil {
rootMap = v.(types.Map)
@@ -426,24 +426,24 @@ func handleRootPost(w http.ResponseWriter, req *http.Request, ps URLParams, cs c
fmt.Fprintf(w, "%v", vs.Root().String())
}
func validateLast(last hash.Hash, vr types.ValueReader) types.Map {
func validateLast(last hash.Hash, vrw types.ValueReadWriter) types.Map {
if last.IsEmpty() {
return types.NewMap()
return types.NewMap(vrw)
}
lastVal := vr.ReadValue(last)
lastVal := vrw.ReadValue(last)
if lastVal == nil {
d.Panic("Can't Commit from a non-present Chunk")
}
return lastVal.(types.Map)
}
func validateProposed(proposed, last hash.Hash, vr types.ValueReader) types.Map {
func validateProposed(proposed, last hash.Hash, vrw types.ValueReadWriter) types.Map {
// Only allowed to skip this check if both last and proposed are empty, because that represents the special case of someone flushing chunks into an empty store.
if last.IsEmpty() && proposed.IsEmpty() {
return types.NewMap()
return types.NewMap(vrw)
}
// Ensure that proposed new Root is present in vr, is a Map and, if it has anything in it, that it's <String, <Ref<Commit>>
proposedVal := vr.ReadValue(proposed)
proposedVal := vrw.ReadValue(proposed)
if proposedVal == nil {
d.Panic("Can't set Root to a non-present Chunk")
}
@@ -550,7 +550,7 @@ func mergeDatasetMaps(a, b, parent types.Map, vrw types.ValueReadWriter) (types.
merged = apply(merged, aChange, aValue)
aChange, bChange = types.ValueChanged{}, types.ValueChanged{}
}
return merged.Map(vrw), nil
return merged.Map(), nil
}
func handleGraphQL(w http.ResponseWriter, req *http.Request, ps URLParams, cs chunks.ChunkStore) {
go/datas/remote_database_handlers_test.go
+19 -15
View File
@@ -29,6 +29,7 @@ func TestHandleWriteValue(t *testing.T) {
db := NewDatabase(storage.NewView())
l := types.NewList(
db,
db.WriteValue(types.Bool(true)),
db.WriteValue(types.Bool(false)),
)
@@ -36,9 +37,9 @@ func TestHandleWriteValue(t *testing.T) {
_, err := db.CommitValue(db.GetDataset("datasetID"), r)
assert.NoError(err)
newItem := types.NewEmptyBlob()
newItem := types.NewEmptyBlob(db)
itemChunk := types.EncodeValue(newItem)
l2 := l.Edit().Insert(1, types.NewRef(newItem)).List(nil)
l2 := l.Edit().Insert(1, types.NewRef(newItem)).List()
listChunk := types.EncodeValue(l2)
body := &bytes.Buffer{}
@@ -73,8 +74,10 @@ func TestHandleWriteValuePanic(t *testing.T) {
func TestHandleWriteValueDupChunks(t *testing.T) {
assert := assert.New(t)
storage := &chunks.MemoryStorage{}
db := NewDatabase(storage.NewView())
defer db.Close()
newItem := types.NewEmptyBlob()
newItem := types.NewEmptyBlob(db)
itemChunk := types.EncodeValue(newItem)
body := &bytes.Buffer{}
@@ -210,7 +213,7 @@ func TestHandleGetBlob(t *testing.T) {
)
assert.Equal(http.StatusBadRequest, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
b := types.NewStreamingBlob(db, bytes.NewBuffer([]byte(blobContents)))
b := types.NewBlob(db, bytes.NewBuffer([]byte(blobContents)))
// Test non-present hash
w = httptest.NewRecorder()
@@ -242,7 +245,7 @@ func TestHandleGetBlob(t *testing.T) {
// Test non-blob
r2 := db.WriteValue(types.Number(1))
ds, err = db.CommitValue(ds, r2)
_, err = db.CommitValue(ds, r2)
assert.NoError(err)
w = httptest.NewRecorder()
@@ -346,14 +349,14 @@ func TestHandlePostRoot(t *testing.T) {
HandleRootPost(w, newRequest("POST", "", url, nil, nil), params{}, storage.NewView())
validate(http.StatusOK, hash.Hash{}, w)
commit := buildTestCommit(types.String("head"))
commit := buildTestCommit(vs, types.String("head"))
commitRef := vs.WriteValue(commit)
firstHead := types.NewMap(types.String("dataset1"), types.ToRefOfValue(commitRef))
firstHead := types.NewMap(vs, types.String("dataset1"), types.ToRefOfValue(commitRef))
firstHeadRef := vs.WriteValue(firstHead)
vs.Commit(vs.Root(), vs.Root())
commit = buildTestCommit(types.String("second"), commitRef)
newHead := types.NewMap(types.String("dataset1"), types.ToRefOfValue(vs.WriteValue(commit)))
commit = buildTestCommit(vs, types.String("second"), commitRef)
newHead := types.NewMap(vs, types.String("dataset1"), types.ToRefOfValue(vs.WriteValue(commit)))
newHeadRef := vs.WriteValue(newHead)
vs.Commit(vs.Root(), vs.Root())
@@ -379,16 +382,18 @@ func buildPostRootURL(current, last hash.Hash) string {
return u.String()
}
func buildTestCommit(v types.Value, parents ...types.Value) types.Struct {
return NewCommit(v, types.NewSet(parents...), types.NewStruct("Meta", types.StructData{}))
func buildTestCommit(vrw types.ValueReadWriter, v types.Value, parents ...types.Value) types.Struct {
return NewCommit(v, types.NewSet(vrw, parents...), types.NewStruct("Meta", types.StructData{}))
}
func TestRejectPostRoot(t *testing.T) {
assert := assert.New(t)
storage := &chunks.MemoryStorage{}
cs := storage.NewView()
vs := types.NewValueStore(cs)
defer vs.Close()
newHead := types.NewMap(types.String("dataset1"), types.String("Not a Head"))
newHead := types.NewMap(vs, types.String("dataset1"), types.String("Not a Head"))
chunk := types.EncodeValue(newHead)
cs.Put(chunk)
persistChunks(cs)
@@ -400,9 +405,8 @@ func TestRejectPostRoot(t *testing.T) {
assert.Equal(http.StatusBadRequest, w.Code, "Handler error:\n%s", string(w.Body.Bytes()))
// Put in a legit commit
vs := types.NewValueStore(cs)
commit := buildTestCommit(types.String("commit"))
head := types.NewMap(types.String("dataset1"), types.ToRefOfValue(vs.WriteValue(commit)))
commit := buildTestCommit(vs, types.String("commit"))
head := types.NewMap(vs, types.String("dataset1"), types.ToRefOfValue(vs.WriteValue(commit)))
headRef := vs.WriteValue(head)
assert.True(vs.Commit(headRef.TargetHash(), vs.Root()))
go/diff/apply_patch.go
+17 -17
View File
@@ -124,36 +124,36 @@ func (stack *patchStack) updateNode(top *stackElem, parent types.Value) types.Va
switch top.changeType {
case types.DiffChangeAdded:
if realIdx > el.Len() {
nv = el.Edit().Append(top.newValue).List(nil)
nv = el.Edit().Append(top.newValue).List()
} else {
nv = el.Edit().Insert(realIdx, top.newValue).List(nil)
nv = el.Edit().Insert(realIdx, top.newValue).List()
}
case types.DiffChangeRemoved:
nv = el.Edit().RemoveAt(realIdx).List(nil)
nv = el.Edit().RemoveAt(realIdx).List()
case types.DiffChangeModified:
nv = el.Edit().Set(realIdx, top.newValue).List(nil)
nv = el.Edit().Set(realIdx, top.newValue).List()
}
return nv
case types.Map:
switch top.changeType {
case types.DiffChangeAdded:
return el.Edit().Set(part.Index, top.newValue).Map(nil)
return el.Edit().Set(part.Index, top.newValue).Map()
case types.DiffChangeRemoved:
return el.Edit().Remove(part.Index).Map(nil)
return el.Edit().Remove(part.Index).Map()
case types.DiffChangeModified:
if part.IntoKey {
newPart := types.IndexPath{Index: part.Index}
ov := newPart.Resolve(parent, nil)
return el.Edit().Remove(part.Index).Set(top.newValue, ov).Map(nil)
return el.Edit().Remove(part.Index).Set(top.newValue, ov).Map()
}
return el.Edit().Set(part.Index, top.newValue).Map(nil)
return el.Edit().Set(part.Index, top.newValue).Map()
}
case types.Set:
if top.oldValue != nil {
el = el.Edit().Remove(top.oldValue).Set(nil)
el = el.Edit().Remove(top.oldValue).Set()
}
if top.newValue != nil {
el = el.Edit().Insert(top.newValue).Set(nil)
el = el.Edit().Insert(top.newValue).Set()
}
return el
}
@@ -162,11 +162,11 @@ func (stack *patchStack) updateNode(top *stackElem, parent types.Value) types.Va
case types.Set:
switch top.changeType {
case types.DiffChangeAdded:
return el.Edit().Insert(top.newValue).Set(nil)
return el.Edit().Insert(top.newValue).Set()
case types.DiffChangeRemoved:
return el.Edit().Remove(top.oldValue).Set(nil)
return el.Edit().Remove(top.oldValue).Set()
case types.DiffChangeModified:
return el.Edit().Remove(top.oldValue).Insert(top.newValue).Set(nil)
return el.Edit().Remove(top.oldValue).Insert(top.newValue).Set()
}
case types.Map:
keyPart := types.HashIndexPath{Hash: part.Hash, IntoKey: true}
@@ -174,15 +174,15 @@ func (stack *patchStack) updateNode(top *stackElem, parent types.Value) types.Va
switch top.changeType {
case types.DiffChangeAdded:
k := top.newKeyValue
return el.Edit().Set(k, top.newValue).Map(nil)
return el.Edit().Set(k, top.newValue).Map()
case types.DiffChangeRemoved:
return el.Edit().Remove(k).Map(nil)
return el.Edit().Remove(k).Map()
case types.DiffChangeModified:
if part.IntoKey {
v := el.Get(k)
return el.Edit().Remove(k).Set(top.newValue, v).Map(nil)
return el.Edit().Remove(k).Set(top.newValue, v).Map()
}
return el.Edit().Set(k, top.newValue).Map(nil)
return el.Edit().Set(k, top.newValue).Map()
}
}
}
go/diff/apply_patch_test.go
+44 -28
View File
@@ -64,7 +64,7 @@ func vfk(keys ...string) []types.Value {
return values
}
func testValues() map[string]types.Value {
func testValues(vrw types.ValueReadWriter) map[string]types.Value {
if vm == nil {
vm = map[string]types.Value{
"k1": types.String("k1"),
@@ -99,21 +99,26 @@ func testValues() map[string]types.Value {
"ms2": mustMarshal(map[testKey]int{{1, 1}: 1, {4, 4}: 4, {5, 5}: 5}),
}
vm["mh1"] = types.NewMap(vfk("k1", "struct1", "k2", "l1")...)
vm["mh2"] = types.NewMap(vfk("k1", "n1", "k2", "l2", "k3", "l3")...)
vm["set1"] = types.NewSet()
vm["set2"] = types.NewSet(vfk("s1", "s2")...)
vm["set3"] = types.NewSet(vfk("s1", "s2", "s3")...)
vm["set1"] = types.NewSet(vfk("s2")...)
vm["seth1"] = types.NewSet(vfk("struct1", "struct2", "struct3")...)
vm["seth2"] = types.NewSet(vfk("struct2", "struct3")...)
vm["setj3"] = types.NewSet(vfk("struct1")...)
vm["mk1"] = types.NewMap(vfk("struct1", "s1", "struct2", "s2")...)
vm["mk2"] = types.NewMap(vfk("struct1", "s3", "struct4", "s4")...)
vm["mh1"] = types.NewMap(vrw, vfk("k1", "struct1", "k2", "l1")...)
vm["mh2"] = types.NewMap(vrw, vfk("k1", "n1", "k2", "l2", "k3", "l3")...)
vm["set1"] = types.NewSet(vrw)
vm["set2"] = types.NewSet(vrw, vfk("s1", "s2")...)
vm["set3"] = types.NewSet(vrw, vfk("s1", "s2", "s3")...)
vm["set1"] = types.NewSet(vrw, vfk("s2")...)
vm["seth1"] = types.NewSet(vrw, vfk("struct1", "struct2", "struct3")...)
vm["seth2"] = types.NewSet(vrw, vfk("struct2", "struct3")...)
vm["setj3"] = types.NewSet(vrw, vfk("struct1")...)
vm["mk1"] = types.NewMap(vrw, vfk("struct1", "s1", "struct2", "s2")...)
vm["mk2"] = types.NewMap(vrw, vfk("struct1", "s3", "struct4", "s4")...)
}
return vm
}
func newTestValueStore() *types.ValueStore {
st := &chunks.TestStorage{}
return types.NewValueStore(st.NewView())
}
func getPatch(g1, g2 types.Value) Patch {
dChan := make(chan Difference)
sChan := make(chan struct{})
@@ -138,9 +143,12 @@ func checkApplyPatch(assert *assert.Assertions, g1, expectedG2 types.Value, k1,
func TestPatches(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
cnt := 0
for k1, g1 := range testValues() {
for k2, expectedG2 := range testValues() {
for k1, g1 := range testValues(vs) {
for k2, expectedG2 := range testValues(vs) {
if k1 != k2 {
cnt++
checkApplyPatch(assert, g1, expectedG2, k1, k2)
@@ -152,21 +160,23 @@ func TestPatches(t *testing.T) {
func TestNestedLists(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
ol1 := mustMarshal([]string{"one", "two", "three", "four"})
nl1 := mustMarshal([]string{"two", "three"})
ol2 := mustMarshal([]int{2, 3})
nl2 := mustMarshal([]int{1, 2, 3, 4})
nl3 := mustMarshal([]bool{true, false, true})
g1 := types.NewList(ol1, ol2)
g2 := types.NewList(nl1, nl2, nl3)
g1 := types.NewList(vs, ol1, ol2)
g2 := types.NewList(vs, nl1, nl2, nl3)
checkApplyPatch(assert, g1, g2, "g1", "g2")
}
func TestUpdateNode(t *testing.T) {
assert := assert.New(t)
storage := &chunks.MemoryStorage{}
vs := types.NewValueStore(storage.NewView())
vs := newTestValueStore()
defer vs.Close()
doTest := func(pp types.PathPart, parent, ov, nv, exp types.Value, f testFunc) {
@@ -187,13 +197,13 @@ func TestUpdateNode(t *testing.T) {
return parent.(types.Struct).Get("f2")
})
l1 := types.NewList(types.String("one"), oldVal, types.String("three"))
l1 := types.NewList(vs, types.String("one"), oldVal, types.String("three"))
pp = types.IndexPath{Index: types.Number(1)}
doTest(pp, l1, oldVal, newVal, newVal, func(parent types.Value) types.Value {
return parent.(types.List).Get(1)
})
m1 := types.NewMap(types.String("k1"), types.Number(1), types.String("k2"), oldVal)
m1 := types.NewMap(vs, types.String("k1"), types.Number(1), types.String("k2"), oldVal)
pp = types.IndexPath{Index: types.String("k2")}
doTest(pp, m1, oldVal, newVal, newVal, func(parent types.Value) types.Value {
return parent.(types.Map).Get(types.String("k2"))
@@ -201,23 +211,23 @@ func TestUpdateNode(t *testing.T) {
k1 := types.NewStruct("Sizes", types.StructData{"height": types.Number(200), "width": types.Number(300)})
vs.WriteValue(k1)
m1 = types.NewMap(k1, oldVal)
m1 = types.NewMap(vs, k1, oldVal)
pp = types.HashIndexPath{Hash: k1.Hash()}
doTest(pp, m1, oldVal, newVal, newVal, func(parent types.Value) types.Value {
return parent.(types.Map).Get(k1)
})
set1 := types.NewSet(oldVal, k1)
set1 := types.NewSet(vs, oldVal, k1)
pp = types.IndexPath{Index: oldVal}
exp := types.NewSet(newVal, k1)
exp := types.NewSet(vs, newVal, k1)
doTest(pp, set1, oldVal, newVal, exp, func(parent types.Value) types.Value {
return parent
})
k2 := types.NewStruct("Sizes", types.StructData{"height": types.Number(300), "width": types.Number(500)})
set1 = types.NewSet(oldVal, k1)
set1 = types.NewSet(vs, oldVal, k1)
pp = types.HashIndexPath{Hash: k1.Hash()}
exp = types.NewSet(oldVal, k2)
exp = types.NewSet(vs, oldVal, k2)
doTest(pp, set1, k1, k2, exp, func(parent types.Value) types.Value {
return parent
})
@@ -337,8 +347,11 @@ func TestUpdateStruct(t *testing.T) {
func TestUpdateSet(t *testing.T) {
a := assert.New(t)
a1 := types.NewSet(types.Number(1), types.String("two"), mustMarshal([]string{"one", "two", "three"}))
a2 := types.NewSet(types.Number(3), types.String("three"), mustMarshal([]string{"one", "two", "three", "four"}))
vs := newTestValueStore()
defer vs.Close()
a1 := types.NewSet(vs, types.Number(1), types.String("two"), mustMarshal([]string{"one", "two", "three"}))
a2 := types.NewSet(vs, types.Number(3), types.String("three"), mustMarshal([]string{"one", "two", "three", "four"}))
checkApplyDiffs(a, a1, a2, true)
checkApplyDiffs(a, a1, a2, false)
@@ -347,7 +360,10 @@ func TestUpdateSet(t *testing.T) {
}
func mustMarshal(v interface{}) types.Value {
v1, err := marshal.Marshal(v)
vs := newTestValueStore()
defer vs.Close()
v1, err := marshal.Marshal(vs, v)
d.Chk.NoError(err)
return v1
}
go/diff/diff_test.go
+20 -7
View File
@@ -50,18 +50,24 @@ func valsToTypesValues(kv ...interface{}) []types.Value {
}
func createMap(kv ...interface{}) types.Map {
vs := newTestValueStore()
defer vs.Close()
keyValues := valsToTypesValues(kv...)
return types.NewMap(keyValues...)
return types.NewMap(vs, keyValues...)
}
func createSet(kv ...interface{}) types.Set {
vs := newTestValueStore()
defer vs.Close()
keyValues := valsToTypesValues(kv...)
return types.NewSet(keyValues...)
return types.NewSet(vs, keyValues...)
}
func createList(kv ...interface{}) types.List {
vs := newTestValueStore()
defer vs.Close()
keyValues := valsToTypesValues(kv...)
return types.NewList(keyValues...)
return types.NewList(vs, keyValues...)
}
func createStruct(name string, kv ...interface{}) types.Struct {
@@ -307,6 +313,10 @@ func TestNomsDiffPrintStruct(t *testing.T) {
func TestNomsDiffPrintMapWithStructKeys(t *testing.T) {
a := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
k1 := createStruct("TestKey", "name", "n1", "label", "l1")
expected1 := `(root) {
@@ -321,8 +331,8 @@ func TestNomsDiffPrintMapWithStructKeys(t *testing.T) {
}
`
m1 := types.NewMap(k1, types.Bool(true))
m2 := types.NewMap(k1, types.Bool(false))
m1 := types.NewMap(vs, k1, types.Bool(true))
m2 := types.NewMap(vs, k1, types.Bool(false))
tf := func(leftRight bool) {
buf := &bytes.Buffer{}
PrintDiff(buf, m1, m2, leftRight)
@@ -407,10 +417,13 @@ func TestNomsDiffPrintList(t *testing.T) {
func TestNomsDiffPrintBlob(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
expected := "- Blob (2.0 kB)\n+ Blob (11 B)\n"
expectedPaths1 := []string{``}
b1 := types.NewBlob(strings.NewReader(strings.Repeat("x", 2*1024)))
b2 := types.NewBlob(strings.NewReader("Hello World"))
b1 := types.NewBlob(vs, strings.NewReader(strings.Repeat("x", 2*1024)))
b2 := types.NewBlob(vs, strings.NewReader("Hello World"))
tf := func(leftRight bool) {
buf := &bytes.Buffer{}
go/marshal/decode_test.go
+164 -67
View File
@@ -14,6 +14,7 @@ import (
"strings"
"testing"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/d"
"github.com/attic-labs/noms/go/types"
"github.com/attic-labs/testify/assert"
@@ -21,6 +22,9 @@ import (
func TestDecode(tt *testing.T) {
assert := assert.New(tt)
st := &chunks.TestStorage{}
vs := types.NewValueStore(st.NewView())
defer vs.Close()
t := func(v types.Value, ptr interface{}, expected interface{}) {
p := reflect.ValueOf(ptr)
@@ -110,19 +114,19 @@ func TestDecode(tt *testing.T) {
}
var list types.List
list2 := types.NewList(types.Number(42))
list2 := types.NewList(vs, types.Number(42))
t(list2, &list, list2)
var m types.Map
map2 := types.NewMap(types.Number(42), types.String("Hi"))
map2 := types.NewMap(vs, types.Number(42), types.String("Hi"))
t(map2, &m, map2)
var set types.Set
set2 := types.NewSet(types.String("Bye"))
set2 := types.NewSet(vs, types.String("Bye"))
t(set2, &set, set2)
var blob types.Blob
blob2 := types.NewBlob(bytes.NewBufferString("hello"))
blob2 := types.NewBlob(vs, bytes.NewBufferString("hello"))
t(blob2, &blob, blob2)
type TestStruct struct {
@@ -169,10 +173,10 @@ func TestDecode(tt *testing.T) {
"b": types.Bool(false),
"c": types.String("bye"),
}),
"def": types.NewList(types.Number(42)),
"def": types.NewList(vs, types.Number(42)),
}), &t2, T2{
TestStruct{false, 1, "bye"},
types.NewList(types.Number(42)),
types.NewList(vs, types.Number(42)),
})
// extra fields
@@ -235,12 +239,20 @@ func TestDecodeNil(t *testing.T) {
assert.Equal(t, "Cannot unmarshal into Go nil value", err.Error())
}
func newTestValueStore() *types.ValueStore {
st := &chunks.TestStorage{}
return types.NewValueStore(st.NewView())
}
func TestDecodeTypeMismatch(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
var b bool
assertDecodeErrorMessage(t, types.Number(42), &b, "Cannot unmarshal Number into Go value of type bool")
var blob types.Blob
assertDecodeErrorMessage(t, types.NewList(), &blob, "Cannot unmarshal List<> into Go value of type types.Blob")
assertDecodeErrorMessage(t, types.NewList(vs), &blob, "Cannot unmarshal List<> into Go value of type types.Blob")
type S struct {
X int
@@ -478,12 +490,15 @@ func TestDecodeInvalidNamedFields(t *testing.T) {
}
func TestDecodeInvalidNomsType(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
type S struct {
A types.List
}
var s S
assertDecodeErrorMessage(t, types.NewStruct("S", types.StructData{
"a": types.NewMap(types.String("A"), types.Number(1)),
"a": types.NewMap(vs, types.String("A"), types.Number(1)),
}), &s, "Cannot unmarshal Map<String, Number> into Go value of type types.List")
}
@@ -535,49 +550,61 @@ func ExampleUnmarshal() {
func TestDecodeSlice(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var s []string
err := Unmarshal(types.NewList(types.String("a"), types.String("b"), types.String("c")), &s)
err := Unmarshal(types.NewList(vs, types.String("a"), types.String("b"), types.String("c")), &s)
assert.NoError(err)
assert.Equal([]string{"a", "b", "c"}, s)
err = Unmarshal(types.NewSet(types.String("a"), types.String("b"), types.String("c")), &s)
err = Unmarshal(types.NewSet(vs, types.String("a"), types.String("b"), types.String("c")), &s)
assert.NoError(err)
assert.Equal([]string{"a", "b", "c"}, s)
}
func TestDecodeSliceEmpty(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var s []string
err := Unmarshal(types.NewList(), &s)
err := Unmarshal(types.NewList(vs), &s)
assert.NoError(err)
assert.Equal([]string(nil), s)
err = Unmarshal(types.NewSet(), &s)
err = Unmarshal(types.NewSet(vs), &s)
assert.NoError(err)
assert.Equal([]string(nil), s)
s2 := []string{}
err = Unmarshal(types.NewList(), &s2)
err = Unmarshal(types.NewList(vs), &s2)
assert.NoError(err)
assert.Equal([]string{}, s2)
err = Unmarshal(types.NewSet(), &s2)
err = Unmarshal(types.NewSet(vs), &s2)
assert.NoError(err)
assert.Equal([]string{}, s2)
}
func TestDecodeSliceReuse(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
s := []string{"A", "B", "C", "D"}
s2 := s[1:3]
err := Unmarshal(types.NewList(types.String("a"), types.String("b")), &s)
err := Unmarshal(types.NewList(vs, types.String("a"), types.String("b")), &s)
assert.NoError(err)
assert.Equal([]string{"a", "b"}, s)
assert.Equal([]string{"b", "C"}, s2)
err = Unmarshal(types.NewSet(types.String("a"), types.String("b")), &s)
err = Unmarshal(types.NewSet(vs, types.String("a"), types.String("b")), &s)
assert.NoError(err)
assert.Equal([]string{"a", "b"}, s)
assert.Equal([]string{"b", "C"}, s2)
@@ -585,26 +612,34 @@ func TestDecodeSliceReuse(t *testing.T) {
func TestDecodeArray(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
s := [3]string{"", "", ""}
err := Unmarshal(types.NewList(types.String("a"), types.String("b"), types.String("c")), &s)
err := Unmarshal(types.NewList(vs, types.String("a"), types.String("b"), types.String("c")), &s)
assert.NoError(err)
assert.Equal([3]string{"a", "b", "c"}, s)
err = Unmarshal(types.NewSet(types.String("a"), types.String("b"), types.String("c")), &s)
err = Unmarshal(types.NewSet(vs, types.String("a"), types.String("b"), types.String("c")), &s)
assert.NoError(err)
assert.Equal([3]string{"a", "b", "c"}, s)
}
func TestDecodeArrayEmpty(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var s [0]string
err := Unmarshal(types.NewList(), &s)
err := Unmarshal(types.NewList(vs), &s)
assert.NoError(err)
assert.Equal([0]string{}, s)
err = Unmarshal(types.NewSet(), &s)
err = Unmarshal(types.NewSet(vs), &s)
assert.NoError(err)
assert.Equal([0]string{}, s)
}
@@ -612,18 +647,21 @@ func TestDecodeArrayEmpty(t *testing.T) {
func TestDecodeStructWithSlice(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
List []int
}
var s S
err := Unmarshal(types.NewStruct("S", types.StructData{
"list": types.NewList(types.Number(1), types.Number(2), types.Number(3)),
"list": types.NewList(vs, types.Number(1), types.Number(2), types.Number(3)),
}), &s)
assert.NoError(err)
assert.Equal(S{[]int{1, 2, 3}}, s)
err = Unmarshal(types.NewStruct("S", types.StructData{
"list": types.NewSet(types.Number(1), types.Number(2), types.Number(3)),
"list": types.NewSet(vs, types.Number(1), types.Number(2), types.Number(3)),
}), &s)
assert.NoError(err)
assert.Equal(S{[]int{1, 2, 3}}, s)
@@ -632,45 +670,66 @@ func TestDecodeStructWithSlice(t *testing.T) {
func TestDecodeStructWithArrayOfNomsValue(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
List [1]types.Set
}
var s S
err := Unmarshal(types.NewStruct("S", types.StructData{
"list": types.NewList(types.NewSet(types.Bool(true))),
"list": types.NewList(vs, types.NewSet(vs, types.Bool(true))),
}), &s)
assert.NoError(err)
assert.Equal(S{[1]types.Set{types.NewSet(types.Bool(true))}}, s)
assert.Equal(S{[1]types.Set{types.NewSet(vs, types.Bool(true))}}, s)
}
func TestDecodeWrongArrayLength(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
var l [2]string
assertDecodeErrorMessage(t, types.NewList(types.String("hi")), &l, "Cannot unmarshal List<String> into Go value of type [2]string, length does not match")
assertDecodeErrorMessage(t, types.NewList(vs, types.String("hi")), &l, "Cannot unmarshal List<String> into Go value of type [2]string, length does not match")
}
func TestDecodeWrongArrayType(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
var l [1]string
assertDecodeErrorMessage(t, types.NewList(types.Number(1)), &l, "Cannot unmarshal Number into Go value of type string")
assertDecodeErrorMessage(t, types.NewList(vs, types.Number(1)), &l, "Cannot unmarshal Number into Go value of type string")
}
func TestDecodeWrongSliceType(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
var l []string
assertDecodeErrorMessage(t, types.NewList(types.Number(1)), &l, "Cannot unmarshal Number into Go value of type string")
assertDecodeErrorMessage(t, types.NewList(vs, types.Number(1)), &l, "Cannot unmarshal Number into Go value of type string")
}
func TestDecodeSliceWrongNomsType(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
var l []string
assertDecodeErrorMessage(t, types.NewMap(types.String("a"), types.Number(1)), &l, "Cannot unmarshal Map<String, Number> into Go value of type []string")
assertDecodeErrorMessage(t, types.NewMap(vs, types.String("a"), types.Number(1)), &l, "Cannot unmarshal Map<String, Number> into Go value of type []string")
}
func TestDecodeArrayWrongNomsType(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
var l [1]string
assertDecodeErrorMessage(t, types.NewMap(types.String("a"), types.Number(1)), &l, "Cannot unmarshal Map<String, Number> into Go value of type [1]string")
assertDecodeErrorMessage(t, types.NewMap(vs, types.String("a"), types.Number(1)), &l, "Cannot unmarshal Map<String, Number> into Go value of type [1]string")
}
func TestDecodeRecursive(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type Node struct {
Value int
Children []Node
@@ -678,12 +737,13 @@ func TestDecodeRecursive(t *testing.T) {
v := types.NewStruct("Node", types.StructData{
"children": types.NewList(
vs,
types.NewStruct("Node", types.StructData{
"children": types.NewList(),
"children": types.NewList(vs),
"value": types.Number(2),
}),
types.NewStruct("Node", types.StructData{
"children": types.NewList(),
"children": types.NewList(vs),
"value": types.Number(3),
}),
),
@@ -704,9 +764,14 @@ func TestDecodeRecursive(t *testing.T) {
func TestDecodeMap(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var m map[string]int
testMap := types.NewMap(
vs,
types.String("a"), types.Number(1),
types.String("b"), types.Number(2),
types.String("c"), types.Number(3))
@@ -717,6 +782,7 @@ func TestDecodeMap(t *testing.T) {
m = map[string]int{"b": 2, "c": 333}
err = Unmarshal(types.NewMap(
vs,
types.String("a"), types.Number(1),
types.String("c"), types.Number(3)), &m)
assert.NoError(err)
@@ -728,6 +794,7 @@ func TestDecodeMap(t *testing.T) {
var m2 map[S]bool
err = Unmarshal(types.NewMap(
vs,
types.NewStruct("S", types.StructData{"n": types.String("Yes")}), types.Bool(true),
types.NewStruct("S", types.StructData{"n": types.String("No")}), types.Bool(false)), &m2)
assert.NoError(err)
@@ -737,25 +804,34 @@ func TestDecodeMap(t *testing.T) {
func TestDecodeMapEmpty(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var m map[string]int
err := Unmarshal(types.NewMap(), &m)
err := Unmarshal(types.NewMap(vs), &m)
assert.NoError(err)
assert.Equal(map[string]int(nil), m)
m2 := map[string]int{}
err = Unmarshal(types.NewMap(), &m2)
err = Unmarshal(types.NewMap(vs), &m2)
assert.NoError(err)
assert.Equal(map[string]int{}, m2)
}
func TestDecodeMapWrongNomsType(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
var m map[string]int
assertDecodeErrorMessage(t, types.NewList(types.String("a"), types.Number(1)), &m, "Cannot unmarshal List<Number | String> into Go value of type map[string]int")
assertDecodeErrorMessage(t, types.NewList(vs, types.String("a"), types.Number(1)), &m, "Cannot unmarshal List<Number | String> into Go value of type map[string]int")
}
func TestDecodeOntoInterface(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var i interface{}
err := Unmarshal(types.Number(1), &i)
assert.NoError(err)
@@ -769,19 +845,19 @@ func TestDecodeOntoInterface(t *testing.T) {
assert.NoError(err)
assert.Equal(true, i)
err = Unmarshal(types.NewList(types.String("abc")), &i)
err = Unmarshal(types.NewList(vs, types.String("abc")), &i)
assert.NoError(err)
assert.Equal([]string{"abc"}, i)
err = Unmarshal(types.NewMap(types.String("abc"), types.Number(1)), &i)
err = Unmarshal(types.NewMap(vs, types.String("abc"), types.Number(1)), &i)
assert.NoError(err)
assert.Equal(map[string]float64{"abc": float64(1)}, i)
err = Unmarshal(types.NewList(types.String("a"), types.Bool(true), types.Number(42)), &i)
err = Unmarshal(types.NewList(vs, types.String("a"), types.Bool(true), types.Number(42)), &i)
assert.NoError(err)
assert.Equal([]interface{}{"a", true, float64(42)}, i)
err = Unmarshal(types.NewMap(types.String("a"), types.Bool(true), types.Number(42), types.NewList()), &i)
err = Unmarshal(types.NewMap(vs, types.String("a"), types.Bool(true), types.Number(42), types.NewList(vs)), &i)
assert.NoError(err)
assert.Equal(map[interface{}]interface{}{"a": true, float64(42): []interface{}(nil)}, i)
}
@@ -803,6 +879,9 @@ func TestDecodeOntoInterfaceStruct(t *testing.T) {
func TestDecodeSet(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type T struct {
A map[int]struct{} `noms:",set"`
B map[int]struct{}
@@ -814,13 +893,13 @@ func TestDecodeSet(t *testing.T) {
}
ns := types.NewStruct("T", types.StructData{
"a": types.NewSet(types.Number(0), types.Number(1), types.Number(2)),
"b": types.NewMap(types.Number(3), types.EmptyStruct, types.Number(4), types.EmptyStruct, types.Number(5), types.EmptyStruct),
"c": types.NewSet(types.String("0"), types.String("1"), types.String("2")),
"d": types.NewMap(types.String("3"), types.EmptyStruct, types.String("4"), types.EmptyStruct, types.String("5"), types.EmptyStruct),
"e": types.NewSet(types.Number(6), types.Number(7), types.Number(8)),
"f": types.NewSet(types.Number(9), types.Number(10), types.Number(11)),
"g": types.NewList(types.Number(12), types.Number(13), types.Number(14)),
"a": types.NewSet(vs, types.Number(0), types.Number(1), types.Number(2)),
"b": types.NewMap(vs, types.Number(3), types.EmptyStruct, types.Number(4), types.EmptyStruct, types.Number(5), types.EmptyStruct),
"c": types.NewSet(vs, types.String("0"), types.String("1"), types.String("2")),
"d": types.NewMap(vs, types.String("3"), types.EmptyStruct, types.String("4"), types.EmptyStruct, types.String("5"), types.EmptyStruct),
"e": types.NewSet(vs, types.Number(6), types.Number(7), types.Number(8)),
"f": types.NewSet(vs, types.Number(9), types.Number(10), types.Number(11)),
"g": types.NewList(vs, types.Number(12), types.Number(13), types.Number(14)),
})
gs := T{}
@@ -836,13 +915,13 @@ func TestDecodeSet(t *testing.T) {
}, gs)
ns2 := types.NewStruct("T", types.StructData{
"a": types.NewSet(),
"b": types.NewMap(),
"c": types.NewSet(),
"d": types.NewMap(),
"e": types.NewSet(),
"f": types.NewSet(),
"g": types.NewList(),
"a": types.NewSet(vs),
"b": types.NewMap(vs),
"c": types.NewSet(vs),
"d": types.NewMap(vs),
"e": types.NewSet(vs),
"f": types.NewSet(vs),
"g": types.NewList(vs),
})
gs2 := T{
@@ -857,6 +936,9 @@ func TestDecodeSet(t *testing.T) {
func TestDecodeOpt(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
tc := []struct {
in types.Value
opt Opt
@@ -865,28 +947,28 @@ func TestDecodeOpt(t *testing.T) {
wantError string
}{
{
types.NewSet(types.String("a"), types.String("b")),
types.NewSet(vs, types.String("a"), types.String("b")),
Opt{},
&[]string{},
&[]string{"a", "b"},
"",
},
{
types.NewSet(types.String("a"), types.String("b")),
types.NewSet(vs, types.String("a"), types.String("b")),
Opt{Set: true},
&[]string{},
&[]string{"a", "b"},
"",
},
{
types.NewSet(types.String("a"), types.String("b")),
types.NewSet(vs, types.String("a"), types.String("b")),
Opt{Set: true},
&map[string]struct{}{},
&map[string]struct{}{"a": struct{}{}, "b": struct{}{}},
"",
},
{
types.NewSet(types.String("a"), types.String("b")),
types.NewSet(vs, types.String("a"), types.String("b")),
Opt{},
&map[string]struct{}{},
&map[string]struct{}{},
@@ -908,13 +990,16 @@ func TestDecodeOpt(t *testing.T) {
func TestDecodeNamedSet(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type T struct {
A map[int]struct{} `noms:"foo,set"`
}
ns := types.NewStruct("T", types.StructData{
"a": types.NewSet(types.Number(0)),
"foo": types.NewSet(types.Number(1)),
"a": types.NewSet(vs, types.Number(0)),
"foo": types.NewSet(vs, types.Number(1)),
})
gs := T{}
@@ -927,12 +1012,15 @@ func TestDecodeNamedSet(t *testing.T) {
func TestDecodeSetWrongMapType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type T1 struct {
A map[int]int `noms:",set"`
}
err := Unmarshal(types.NewStruct("T1", types.StructData{
"a": types.NewSet(types.Number(0)),
"a": types.NewSet(vs, types.Number(0)),
}), &T1{})
assert.Error(err)
assert.Equal("Cannot unmarshal Set<Number> into Go value of type map[int]int", err.Error())
@@ -942,7 +1030,7 @@ func TestDecodeSetWrongMapType(t *testing.T) {
}
err = Unmarshal(types.NewStruct("T2", types.StructData{
"a": types.NewSet(types.Number(0)),
"a": types.NewSet(vs, types.Number(0)),
}), &T2{})
assert.Error(err)
assert.Equal(`Cannot unmarshal Set<Number> into Go value of type map[int]struct {}, field missing "set" tag`, err.Error())
@@ -952,7 +1040,7 @@ func TestDecodeSetWrongMapType(t *testing.T) {
}
err = Unmarshal(types.NewStruct("T3", types.StructData{
"a": types.NewMap(types.Number(0), types.EmptyStruct),
"a": types.NewMap(vs, types.Number(0), types.EmptyStruct),
}), &T3{})
assert.Error(err)
assert.Equal(`Cannot unmarshal Map<Number, struct {}> into Go value of type map[int]struct {}, field has "set" tag`, err.Error())
@@ -1083,7 +1171,10 @@ func (u *primitiveMapType) UnmarshalNoms(v types.Value) error {
func TestUnmarshalerPrimitiveMapType(t *testing.T) {
assert := assert.New(t)
v := types.NewSet(types.String("a,foo"), types.String("b,bar"))
vs := newTestValueStore()
defer vs.Close()
v := types.NewSet(vs, types.String("a,foo"), types.String("b,bar"))
u := primitiveMapType{}
assert.NoError(Unmarshal(v, &u))
assert.Equal(primitiveMapType(map[string]string{
@@ -1143,14 +1234,17 @@ func TestUnmarshalerWrappedMarshalerType(t *testing.T) {
func TestUnmarshalerComplexStructType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
s := "foo|bar"
r := regexp.MustCompile(s)
v := types.NewStruct("TestComplexStructType", types.StructData{
"p": types.Number(43),
"ps": types.NewList(types.Number(2), types.Number(3)),
"pm": types.NewMap(types.String("x"), types.Number(101), types.String("y"), types.Number(102)),
"ps": types.NewList(vs, types.Number(2), types.Number(3)),
"pm": types.NewMap(vs, types.String("x"), types.Number(101), types.String("y"), types.Number(102)),
"pslice": types.String("a,b,c"),
"pmap": types.NewSet(types.String("c,123"), types.String("d,456")),
"pmap": types.NewSet(vs, types.String("c,123"), types.String("d,456")),
"pstruct": types.Number(5),
"b": types.String(s),
})
@@ -1216,12 +1310,15 @@ func TestUnmarshalNomsNotPointerDoesNotShareState(t *testing.T) {
func TestUnmarshalMustUnmarshal(t *testing.T) {
a := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type TestStruct struct{ F1 int }
v := MustMarshal(types.Number(1))
v := MustMarshal(vs, types.Number(1))
var out TestStruct
a.Panics(func() { MustUnmarshal(v, &out) })
v = MustMarshal(TestStruct{2})
v = MustMarshal(vs, TestStruct{2})
a.NotPanics(func() { MustUnmarshal(v, &out) })
}
go/marshal/encode.go
+38 -38
View File
@@ -91,12 +91,12 @@ import (
//
// Go pointers, complex, function are not supported. Attempting to encode such a
// value causes Marshal to return an UnsupportedTypeError.
func Marshal(v interface{}) (types.Value, error) {
return MarshalOpt(v, Opt{})
func Marshal(vrw types.ValueReadWriter, v interface{}) (types.Value, error) {
return MarshalOpt(vrw, v, Opt{})
}
// MarshalOpt is like Marshal but provides additional options.
func MarshalOpt(v interface{}, opt Opt) (nomsValue types.Value, err error) {
func MarshalOpt(vrw types.ValueReadWriter, v interface{}, opt Opt) (nomsValue types.Value, err error) {
defer func() {
if r := recover(); r != nil {
switch r := r.(type) {
@@ -109,23 +109,23 @@ func MarshalOpt(v interface{}, opt Opt) (nomsValue types.Value, err error) {
}
}
}()
nomsValue = MustMarshalOpt(v, opt)
nomsValue = MustMarshalOpt(vrw, v, opt)
return
}
// MustMarshal marshals a Go value to a Noms value using the same rules as
// Marshal(). Panics on failure.
func MustMarshal(v interface{}) types.Value {
return MustMarshalOpt(v, Opt{})
func MustMarshal(vrw types.ValueReadWriter, v interface{}) types.Value {
return MustMarshalOpt(vrw, v, Opt{})
}
// MustMarshalOpt is like MustMarshal, but with additional options.
func MustMarshalOpt(v interface{}, opt Opt) types.Value {
func MustMarshalOpt(vrw types.ValueReadWriter, v interface{}, opt Opt) types.Value {
rv := reflect.ValueOf(v)
nt := nomsTags{
set: opt.Set,
}
encoder := typeEncoder(rv.Type(), map[string]reflect.Type{}, nt)
encoder := typeEncoder(vrw, rv.Type(), map[string]reflect.Type{}, nt)
return encoder(rv)
}
@@ -134,7 +134,7 @@ type Marshaler interface {
// MarshalNoms returns the Noms Value encoding of a type, or an error.
// nil is not a valid return val - if both val and err are nil, Marshal will
// panic.
MarshalNoms() (val types.Value, err error)
MarshalNoms(vrw types.ValueReadWriter) (val types.Value, err error)
}
// StructNameMarshaler is an interface that can be implemented to define the
@@ -223,9 +223,9 @@ func nomsValueEncoder(v reflect.Value) types.Value {
return v.Interface().(types.Value)
}
func marshalerEncoder(t reflect.Type) encoderFunc {
func marshalerEncoder(vrw types.ValueReadWriter, t reflect.Type) encoderFunc {
return func(v reflect.Value) types.Value {
val, err := v.Interface().(Marshaler).MarshalNoms()
val, err := v.Interface().(Marshaler).MarshalNoms(vrw)
if err != nil {
panic(&marshalNomsError{err})
}
@@ -236,9 +236,9 @@ func marshalerEncoder(t reflect.Type) encoderFunc {
}
}
func typeEncoder(t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTags) encoderFunc {
func typeEncoder(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTags) encoderFunc {
if t.Implements(marshalerInterface) {
return marshalerEncoder(t)
return marshalerEncoder(vrw, t)
}
switch t.Kind() {
@@ -253,22 +253,22 @@ func typeEncoder(t reflect.Type, seenStructs map[string]reflect.Type, tags nomsT
case reflect.String:
return stringEncoder
case reflect.Struct:
return structEncoder(t, seenStructs)
return structEncoder(vrw, t, seenStructs)
case reflect.Slice, reflect.Array:
if shouldEncodeAsSet(t, tags) {
return setFromListEncoder(t, seenStructs)
return setFromListEncoder(vrw, t, seenStructs)
}
return listEncoder(t, seenStructs)
return listEncoder(vrw, t, seenStructs)
case reflect.Map:
if shouldEncodeAsSet(t, tags) {
return setEncoder(t, seenStructs)
return setEncoder(vrw, t, seenStructs)
}
return mapEncoder(t, seenStructs)
return mapEncoder(vrw, t, seenStructs)
case reflect.Interface:
return func(v reflect.Value) types.Value {
// Get the dynamic type.
v2 := reflect.ValueOf(v.Interface())
return typeEncoder(v2.Type(), seenStructs, tags)(v2)
return typeEncoder(vrw, v2.Type(), seenStructs, tags)(v2)
}
case reflect.Ptr:
// Allow implementations of types.Value (like *types.Type)
@@ -289,7 +289,7 @@ func getStructName(t reflect.Type) string {
return strings.Title(t.Name())
}
func structEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
func structEncoder(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
if t.Implements(nomsValueInterface) {
return nomsValueEncoder
}
@@ -302,7 +302,7 @@ func structEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderF
structName := getStructName(t)
seenStructs[t.Name()] = t
fields, knownShape, originalFieldIndex := typeFields(t, seenStructs, false, false)
fields, knownShape, originalFieldIndex := typeFields(vrw, t, seenStructs, false, false)
if knownShape {
fieldNames := make([]string, len(fields))
for i, f := range fields {
@@ -461,7 +461,7 @@ func validateField(f reflect.StructField, t reflect.Type) {
}
}
func typeFields(t reflect.Type, seenStructs map[string]reflect.Type, computeType, embedded bool) (fields fieldSlice, knownShape bool, originalFieldIndex []int) {
func typeFields(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type, computeType, embedded bool) (fields fieldSlice, knownShape bool, originalFieldIndex []int) {
knownShape = true
for i := 0; i < t.NumField(); i++ {
index := make([]int, 1)
@@ -478,7 +478,7 @@ func typeFields(t reflect.Type, seenStructs map[string]reflect.Type, computeType
}
if f.Anonymous && f.PkgPath == "" && !tags.hasName {
embeddedFields, embeddedKnownShape, embeddedOriginalFieldIndex := typeFields(f.Type, seenStructs, computeType, true)
embeddedFields, embeddedKnownShape, embeddedOriginalFieldIndex := typeFields(vrw, f.Type, seenStructs, computeType, true)
if embeddedOriginalFieldIndex != nil {
originalFieldIndex = append(index, embeddedOriginalFieldIndex...)
}
@@ -495,7 +495,7 @@ func typeFields(t reflect.Type, seenStructs map[string]reflect.Type, computeType
var nt *types.Type
validateField(f, t)
if computeType {
nt = encodeType(f.Type, seenStructs, tags)
nt = encodeType(vrw, f.Type, seenStructs, tags)
if nt == nil {
knownShape = false
}
@@ -507,7 +507,7 @@ func typeFields(t reflect.Type, seenStructs map[string]reflect.Type, computeType
fields = append(fields, field{
name: tags.name,
encoder: typeEncoder(f.Type, seenStructs, tags),
encoder: typeEncoder(vrw, f.Type, seenStructs, tags),
index: index,
nomsType: nt,
omitEmpty: tags.omitEmpty,
@@ -522,7 +522,7 @@ func typeFields(t reflect.Type, seenStructs map[string]reflect.Type, computeType
return
}
func listEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
func listEncoder(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
e := encoderCache.get(t)
if e != nil {
return e
@@ -540,16 +540,16 @@ func listEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFun
for i := 0; i < v.Len(); i++ {
values[i] = elemEncoder(v.Index(i))
}
return types.NewList(values...)
return types.NewList(vrw, values...)
}
encoderCache.set(t, e)
elemEncoder = typeEncoder(t.Elem(), seenStructs, nomsTags{})
elemEncoder = typeEncoder(vrw, t.Elem(), seenStructs, nomsTags{})
return e
}
// Encode set from array or slice
func setFromListEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
func setFromListEncoder(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
e := setEncoderCache.get(t)
if e != nil {
return e
@@ -567,15 +567,15 @@ func setFromListEncoder(t reflect.Type, seenStructs map[string]reflect.Type) enc
for i := 0; i < v.Len(); i++ {
values[i] = elemEncoder(v.Index(i))
}
return types.NewSet(values...)
return types.NewSet(vrw, values...)
}
setEncoderCache.set(t, e)
elemEncoder = typeEncoder(t.Elem(), seenStructs, nomsTags{})
elemEncoder = typeEncoder(vrw, t.Elem(), seenStructs, nomsTags{})
return e
}
func setEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
func setEncoder(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
e := setEncoderCache.get(t)
if e != nil {
return e
@@ -593,15 +593,15 @@ func setEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc
for i, k := range v.MapKeys() {
values[i] = encoder(k)
}
return types.NewSet(values...)
return types.NewSet(vrw, values...)
}
setEncoderCache.set(t, e)
encoder = typeEncoder(t.Key(), seenStructs, nomsTags{})
encoder = typeEncoder(vrw, t.Key(), seenStructs, nomsTags{})
return e
}
func mapEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
func mapEncoder(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc {
e := encoderCache.get(t)
if e != nil {
return e
@@ -622,12 +622,12 @@ func mapEncoder(t reflect.Type, seenStructs map[string]reflect.Type) encoderFunc
kvs[2*i] = keyEncoder(k)
kvs[2*i+1] = valueEncoder(v.MapIndex(k))
}
return types.NewMap(kvs...)
return types.NewMap(vrw, kvs...)
}
encoderCache.set(t, e)
keyEncoder = typeEncoder(t.Key(), seenStructs, nomsTags{})
valueEncoder = typeEncoder(t.Elem(), seenStructs, nomsTags{})
keyEncoder = typeEncoder(vrw, t.Key(), seenStructs, nomsTags{})
valueEncoder = typeEncoder(vrw, t.Elem(), seenStructs, nomsTags{})
return e
}
go/marshal/encode_test.go
+208 -97
View File
@@ -18,13 +18,16 @@ import (
)
func TestEncode(tt *testing.T) {
vs := newTestValueStore()
defer vs.Close()
t := func(exp types.Value, v interface{}) {
actual, err := Marshal(v)
actual, err := Marshal(vs, v)
assert.NoError(tt, err)
assert.True(tt, exp.Equals(actual))
// Encode again for fallthrough
actual2, err := Marshal(actual)
actual2, err := Marshal(vs, actual)
assert.NoError(tt, err)
assert.True(tt, exp.Equals(actual2))
}
@@ -93,10 +96,10 @@ func TestEncode(tt *testing.T) {
t(types.String(s), s)
}
t(types.NewList(types.Number(42)), types.NewList(types.Number(42)))
t(types.NewMap(types.Number(42), types.String("hi")), types.NewMap(types.Number(42), types.String("hi")))
t(types.NewSet(types.String("bye")), types.NewSet(types.String("bye")))
t(types.NewBlob(bytes.NewBufferString("hello")), types.NewBlob(bytes.NewBufferString("hello")))
t(types.NewList(vs, types.Number(42)), types.NewList(vs, types.Number(42)))
t(types.NewMap(vs, types.Number(42), types.String("hi")), types.NewMap(vs, types.Number(42), types.String("hi")))
t(types.NewSet(vs, types.String("bye")), types.NewSet(vs, types.String("bye")))
t(types.NewBlob(vs, bytes.NewBufferString("hello")), types.NewBlob(vs, bytes.NewBufferString("hello")))
type TestStruct struct {
Str string
@@ -127,14 +130,14 @@ func TestEncode(tt *testing.T) {
C float64
}
t(types.NewStruct("TestNestedStruct", types.StructData{
"a": types.NewList(types.String("hi")),
"a": types.NewList(vs, types.String("hi")),
"b": types.NewStruct("TestStruct", types.StructData{
"str": types.String("bye"),
"num": types.Number(5678),
}),
"c": types.Number(1234),
}), TestNestedStruct{
A: types.NewList(types.String("hi")),
A: types.NewList(vs, types.String("hi")),
B: TestStruct{
Str: "bye",
Num: 5678,
@@ -153,7 +156,10 @@ func TestEncode(tt *testing.T) {
}
func assertEncodeErrorMessage(t *testing.T, v interface{}, expectedMessage string) {
_, err := Marshal(v)
vs := newTestValueStore()
defer vs.Close()
_, err := Marshal(vs, v)
assert.Error(t, err)
assert.Equal(t, expectedMessage, err.Error())
}
@@ -167,6 +173,9 @@ func TestInvalidTypes(t *testing.T) {
func TestEncodeEmbeddedStructSkip(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type EmbeddedStruct struct {
X int
}
@@ -175,7 +184,7 @@ func TestEncodeEmbeddedStructSkip(t *testing.T) {
Y int
}
s := TestStruct{EmbeddedStruct{1}, 2}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("TestStruct", types.StructData{
"y": types.Number(2),
@@ -185,6 +194,9 @@ func TestEncodeEmbeddedStructSkip(t *testing.T) {
func TestEncodeEmbeddedStructWithName(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type EmbeddedStruct struct {
X int
}
@@ -193,7 +205,7 @@ func TestEncodeEmbeddedStructWithName(t *testing.T) {
Y int
}
s := TestStruct{EmbeddedStruct{1}, 2}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("TestStruct", types.StructData{
"em": types.NewStruct("EmbeddedStruct", types.StructData{
@@ -206,6 +218,9 @@ func TestEncodeEmbeddedStructWithName(t *testing.T) {
func TestEncodeEmbeddedStruct(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type EmbeddedStruct struct {
X int
}
@@ -213,7 +228,7 @@ func TestEncodeEmbeddedStruct(t *testing.T) {
EmbeddedStruct
}
s := TestStruct{EmbeddedStruct{1}}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("TestStruct", types.StructData{
"x": types.Number(1),
@@ -225,7 +240,7 @@ func TestEncodeEmbeddedStruct(t *testing.T) {
B int
}
s2 := TestOuter{0, TestStruct{EmbeddedStruct{1}}, 2}
v2, err := Marshal(s2)
v2, err := Marshal(vs, s2)
assert.NoError(err)
assert.True(types.NewStruct("TestOuter", types.StructData{
"a": types.Number(0),
@@ -237,6 +252,9 @@ func TestEncodeEmbeddedStruct(t *testing.T) {
func TestEncodeEmbeddedStructOriginal(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type EmbeddedStruct struct {
X int
O types.Struct `noms:",original"`
@@ -251,7 +269,7 @@ func TestEncodeEmbeddedStructOriginal(t *testing.T) {
B: true,
},
}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("TestStruct", types.StructData{
"b": types.Bool(true),
@@ -269,12 +287,15 @@ func TestEncodeNonExportedField(t *testing.T) {
func TestEncodeTaggingSkip(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Abc int `noms:"-"`
Def bool
}
s := S{42, true}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("S", types.StructData{
"def": types.Bool(true),
@@ -284,13 +305,16 @@ func TestEncodeTaggingSkip(t *testing.T) {
func TestEncodeNamedFields(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Aaa int `noms:"a"`
Bbb bool `noms:"B"`
Ccc string
}
s := S{42, true, "Hi"}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("S", types.StructData{
"a": types.Number(42),
@@ -309,6 +333,9 @@ func TestEncodeInvalidNamedFields(t *testing.T) {
func TestEncodeOmitEmpty(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
String string `noms:",omitempty"`
Bool bool `noms:",omitempty"`
@@ -341,7 +368,7 @@ func TestEncodeOmitEmpty(t *testing.T) {
Float32: 1,
Float64: 1,
}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("S", types.StructData{
"string": types.String("s"),
@@ -376,7 +403,7 @@ func TestEncodeOmitEmpty(t *testing.T) {
Float32: 0,
Float64: 0,
}
v2, err := Marshal(s2)
v2, err := Marshal(vs, s2)
assert.NoError(err)
assert.True(types.NewStruct("S", types.StructData{}).Equals(v2))
@@ -389,18 +416,18 @@ func TestEncodeOmitEmpty(t *testing.T) {
Slice: []int{0},
Map: map[int]int{0: 0},
}
v3, err := Marshal(s3)
v3, err := Marshal(vs, s3)
assert.NoError(err)
assert.True(types.NewStruct("S2", types.StructData{
"slice": types.NewList(types.Number(0)),
"map": types.NewMap(types.Number(0), types.Number(0)),
"slice": types.NewList(vs, types.Number(0)),
"map": types.NewMap(vs, types.Number(0), types.Number(0)),
}).Equals(v3))
s4 := S2{
Slice: []int{},
Map: map[int]int{},
}
v4, err := Marshal(s4)
v4, err := Marshal(vs, s4)
assert.NoError(err)
assert.True(types.NewStruct("S2", types.StructData{}).Equals(v4))
@@ -408,7 +435,7 @@ func TestEncodeOmitEmpty(t *testing.T) {
Slice: nil,
Map: nil,
}
v5, err := Marshal(s5)
v5, err := Marshal(vs, s5)
assert.NoError(err)
assert.True(types.NewStruct("S2", types.StructData{}).Equals(v5))
@@ -417,13 +444,13 @@ func TestEncodeOmitEmpty(t *testing.T) {
Value types.Value `noms:",omitempty"`
}
s6 := S3{
List: types.NewList(),
List: types.NewList(vs),
Value: types.Number(0),
}
v6, err := Marshal(s6)
v6, err := Marshal(vs, s6)
assert.NoError(err)
assert.True(types.NewStruct("S3", types.StructData{
"list": types.NewList(),
"list": types.NewList(vs),
"value": types.Number(0),
}).Equals(v6))
@@ -431,7 +458,7 @@ func TestEncodeOmitEmpty(t *testing.T) {
List: types.List{},
Value: nil,
}
v7, err := Marshal(s7)
v7, err := Marshal(vs, s7)
assert.NoError(err)
assert.True(types.NewStruct("S3", types.StructData{}).Equals(v7))
@@ -442,7 +469,7 @@ func TestEncodeOmitEmpty(t *testing.T) {
s8 := S4{
X: 1,
}
v8, err := Marshal(s8)
v8, err := Marshal(vs, s8)
assert.NoError(err)
assert.True(types.NewStruct("S4", types.StructData{
"y": types.Number(1),
@@ -451,17 +478,20 @@ func TestEncodeOmitEmpty(t *testing.T) {
s9 := S4{
X: 0,
}
v9, err := Marshal(s9)
v9, err := Marshal(vs, s9)
assert.NoError(err)
assert.True(types.NewStruct("S4", types.StructData{}).Equals(v9))
}
func ExampleMarshal() {
vs := newTestValueStore()
defer vs.Close()
type Person struct {
Given string
Male bool
}
arya, err := Marshal(Person{"Arya", false})
arya, err := Marshal(vs, Person{"Arya", false})
if err != nil {
fmt.Println(err)
return
@@ -473,49 +503,66 @@ func ExampleMarshal() {
func TestEncodeSlice(t *testing.T) {
assert := assert.New(t)
v, err := Marshal([]string{"a", "b", "c"})
vs := newTestValueStore()
defer vs.Close()
v, err := Marshal(vs, []string{"a", "b", "c"})
assert.NoError(err)
assert.True(types.NewList(types.String("a"), types.String("b"), types.String("c")).Equals(v))
assert.True(types.NewList(vs, types.String("a"), types.String("b"), types.String("c")).Equals(v))
}
func TestEncodeArray(t *testing.T) {
assert := assert.New(t)
v, err := Marshal([3]int{1, 2, 3})
vs := newTestValueStore()
defer vs.Close()
v, err := Marshal(vs, [3]int{1, 2, 3})
assert.NoError(err)
assert.True(types.NewList(types.Number(1), types.Number(2), types.Number(3)).Equals(v))
assert.True(types.NewList(vs, types.Number(1), types.Number(2), types.Number(3)).Equals(v))
}
func TestEncodeStructWithSlice(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
List []int
}
v, err := Marshal(S{[]int{1, 2, 3}})
v, err := Marshal(vs, S{[]int{1, 2, 3}})
assert.NoError(err)
assert.True(types.NewStruct("S", types.StructData{
"list": types.NewList(types.Number(1), types.Number(2), types.Number(3)),
"list": types.NewList(vs, types.Number(1), types.Number(2), types.Number(3)),
}).Equals(v))
}
func TestEncodeStructWithArrayOfNomsValue(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
List [1]types.Set
}
v, err := Marshal(S{[1]types.Set{types.NewSet(types.Bool(true))}})
v, err := Marshal(vs, S{[1]types.Set{types.NewSet(vs, types.Bool(true))}})
assert.NoError(err)
assert.True(types.NewStruct("S", types.StructData{
"list": types.NewList(types.NewSet(types.Bool(true))),
"list": types.NewList(vs, types.NewSet(vs, types.Bool(true))),
}).Equals(v))
}
func TestEncodeNomsTypePtr(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
testMarshal := func(g interface{}, expected types.Value) {
v, err := Marshal(g)
v, err := Marshal(vs, g)
assert.NoError(err)
assert.Equal(expected, v)
}
@@ -542,11 +589,14 @@ func TestEncodeNomsTypePtr(t *testing.T) {
func TestEncodeRecursive(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type Node struct {
Value int
Children []Node
}
v, err := Marshal(Node{
v, err := Marshal(vs, Node{
1, []Node{
{2, []Node{}},
{3, []Node(nil)},
@@ -568,12 +618,13 @@ func TestEncodeRecursive(t *testing.T) {
assert.True(types.NewStruct("Node", types.StructData{
"children": types.NewList(
vs,
types.NewStruct("Node", types.StructData{
"children": types.NewList(),
"children": types.NewList(vs),
"value": types.Number(2),
}),
types.NewStruct("Node", types.StructData{
"children": types.NewList(),
"children": types.NewList(vs),
"value": types.Number(3),
}),
),
@@ -584,9 +635,13 @@ func TestEncodeRecursive(t *testing.T) {
func TestEncodeMap(t *testing.T) {
assert := assert.New(t)
v, err := Marshal(map[string]int{"a": 1, "b": 2, "c": 3})
vs := newTestValueStore()
defer vs.Close()
v, err := Marshal(vs, map[string]int{"a": 1, "b": 2, "c": 3})
assert.NoError(err)
assert.True(types.NewMap(
vs,
types.String("a"), types.Number(1),
types.String("b"), types.Number(2),
types.String("c"), types.Number(3)).Equals(v))
@@ -594,34 +649,39 @@ func TestEncodeMap(t *testing.T) {
type S struct {
N string
}
v, err = Marshal(map[S]bool{S{"Yes"}: true, S{"No"}: false})
v, err = Marshal(vs, map[S]bool{S{"Yes"}: true, S{"No"}: false})
assert.NoError(err)
assert.True(types.NewMap(
vs,
types.NewStruct("S", types.StructData{"n": types.String("Yes")}), types.Bool(true),
types.NewStruct("S", types.StructData{"n": types.String("No")}), types.Bool(false)).Equals(v))
v, err = Marshal(map[string]int(nil))
v, err = Marshal(vs, map[string]int(nil))
assert.NoError(err)
assert.True(types.NewMap().Equals(v))
assert.True(types.NewMap(vs).Equals(v))
v, err = Marshal(map[string]int{})
v, err = Marshal(vs, map[string]int{})
assert.NoError(err)
assert.True(types.NewMap().Equals(v))
assert.True(types.NewMap(vs).Equals(v))
}
func TestEncodeInterface(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var i interface{}
i = []string{"a", "b"}
v, err := Marshal(i)
v, err := Marshal(vs, i)
assert.NoError(err)
assert.True(types.NewList(types.String("a"), types.String("b")).Equals(v))
assert.True(types.NewList(vs, types.String("a"), types.String("b")).Equals(v))
i = map[interface{}]interface{}{"a": true, struct{ Name string }{"b"}: 42}
v, err = Marshal(i)
v, err = Marshal(vs, i)
assert.NoError(err)
assert.True(types.NewMap(
vs,
types.String("a"), types.Bool(true),
types.NewStruct("", types.StructData{"name": types.String("b")}), types.Number(42),
).Equals(v))
@@ -630,7 +690,10 @@ func TestEncodeInterface(t *testing.T) {
func TestEncodeSet(t *testing.T) {
assert := assert.New(t)
v, err := Marshal(struct {
vs := newTestValueStore()
defer vs.Close()
v, err := Marshal(vs, struct {
A map[int]struct{} `noms:",set"`
B map[int]struct{}
C map[int]string `noms:",set"`
@@ -699,6 +762,9 @@ func TestEncodeSet(t *testing.T) {
func TestEncodeOpt(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
tc := []struct {
in interface{}
opt Opt
@@ -707,28 +773,28 @@ func TestEncodeOpt(t *testing.T) {
{
[]string{"a", "b"},
Opt{},
types.NewList(types.String("a"), types.String("b")),
types.NewList(vs, types.String("a"), types.String("b")),
},
{
[]string{"a", "b"},
Opt{Set: true},
types.NewSet(types.String("a"), types.String("b")),
types.NewSet(vs, types.String("a"), types.String("b")),
},
{
map[string]struct{}{"a": struct{}{}, "b": struct{}{}},
Opt{},
types.NewMap(types.String("a"), types.NewStruct("", nil), types.String("b"), types.NewStruct("", nil)),
types.NewMap(vs, types.String("a"), types.NewStruct("", nil), types.String("b"), types.NewStruct("", nil)),
},
{
map[string]struct{}{"a": struct{}{}, "b": struct{}{}},
Opt{Set: true},
types.NewSet(types.String("a"), types.String("b")),
types.NewSet(vs, types.String("a"), types.String("b")),
},
}
for _, t := range tc {
r, err := MarshalOpt(t.in, t.opt)
assert.Equal(t.wantValue, r)
r, err := MarshalOpt(vs, t.in, t.opt)
assert.True(t.wantValue.Equals(r))
assert.Nil(err)
}
}
@@ -736,7 +802,10 @@ func TestEncodeOpt(t *testing.T) {
func TestEncodeSetWithTags(t *testing.T) {
assert := assert.New(t)
v, err := Marshal(struct {
vs := newTestValueStore()
defer vs.Close()
v, err := Marshal(vs, struct {
A map[int]struct{} `noms:"foo,set"`
B map[int]struct{} `noms:",omitempty,set"`
C map[int]struct{} `noms:"bar,omitempty,set"`
@@ -757,15 +826,18 @@ func TestEncodeSetWithTags(t *testing.T) {
foo, ok := s.Get("foo").(types.Set)
assert.True(ok)
assert.True(types.NewSet(types.Number(0), types.Number(1)).Equals(foo))
assert.True(types.NewSet(vs, types.Number(0), types.Number(1)).Equals(foo))
bar, ok := s.Get("bar").(types.Set)
assert.True(ok)
assert.True(types.NewSet(types.Number(2), types.Number(3)).Equals(bar))
assert.True(types.NewSet(vs, types.Number(2), types.Number(3)).Equals(bar))
}
func TestInvalidTag(t *testing.T) {
_, err := Marshal(struct {
vs := newTestValueStore()
defer vs.Close()
_, err := Marshal(vs, struct {
F string `noms:",omitEmpty"`
}{"F"})
assert.Error(t, err)
@@ -775,12 +847,15 @@ func TestInvalidTag(t *testing.T) {
func TestEncodeCanSkipUnexportedField(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Abc int
notExported bool `noms:"-"`
}
s := S{42, true}
v, err := Marshal(s)
v, err := Marshal(vs, s)
assert.NoError(err)
assert.True(types.NewStruct("S", types.StructData{
"abc": types.Number(42),
@@ -790,6 +865,9 @@ func TestEncodeCanSkipUnexportedField(t *testing.T) {
func TestEncodeOriginal(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Foo int `noms:",omitempty"`
Bar types.Struct `noms:",original"`
@@ -806,21 +884,21 @@ func TestEncodeOriginal(t *testing.T) {
err = Unmarshal(orig, &s)
assert.NoError(err)
s.Foo = 43
assert.True(MustMarshal(s).Equals(orig.Set("foo", types.Number(43))))
assert.True(MustMarshal(vs, s).Equals(orig.Set("foo", types.Number(43))))
// New field extends old struct
orig = types.NewStruct("S", types.StructData{})
err = Unmarshal(orig, &s)
assert.NoError(err)
s.Foo = 43
assert.True(MustMarshal(s).Equals(orig.Set("foo", types.Number(43))))
assert.True(MustMarshal(vs, s).Equals(orig.Set("foo", types.Number(43))))
// Old struct name always used
orig = types.NewStruct("Q", types.StructData{})
err = Unmarshal(orig, &s)
assert.NoError(err)
s.Foo = 43
assert.True(MustMarshal(s).Equals(orig.Set("foo", types.Number(43))))
assert.True(MustMarshal(vs, s).Equals(orig.Set("foo", types.Number(43))))
// Field type of base are preserved
orig = types.NewStruct("S", types.StructData{
@@ -829,7 +907,7 @@ func TestEncodeOriginal(t *testing.T) {
err = Unmarshal(orig, &s)
assert.NoError(err)
s.Foo = 43
out := MustMarshal(s)
out := MustMarshal(vs, s)
assert.True(out.Equals(orig.Set("foo", types.Number(43))))
st2 := types.MakeStructTypeFromFields("S", types.FieldMap{
@@ -841,13 +919,16 @@ func TestEncodeOriginal(t *testing.T) {
s = S{
Foo: 42,
}
assert.True(MustMarshal(s).Equals(
assert.True(MustMarshal(vs, s).Equals(
types.NewStruct("S", types.StructData{"foo": types.Number(float64(42))})))
}
func TestNomsTypes(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Blob types.Blob
Bool types.Bool
@@ -856,15 +937,15 @@ func TestNomsTypes(t *testing.T) {
Type *types.Type
}
s := S{
Blob: types.NewBlob(),
Blob: types.NewBlob(vs),
Bool: types.Bool(true),
Number: types.Number(42),
String: types.String("hi"),
Type: types.NumberType,
}
assert.True(MustMarshal(s).Equals(
assert.True(MustMarshal(vs, s).Equals(
types.NewStruct("S", types.StructData{
"blob": types.NewBlob(),
"blob": types.NewBlob(vs),
"bool": types.Bool(true),
"number": types.Number(42),
"string": types.String("hi"),
@@ -875,72 +956,84 @@ func TestNomsTypes(t *testing.T) {
type primitiveType int
func (t primitiveType) MarshalNoms() (types.Value, error) {
func (t primitiveType) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
return types.Number(int(t) + 1), nil
}
func TestMarshalerPrimitiveType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
u := primitiveType(42)
v := MustMarshal(u)
v := MustMarshal(vs, u)
assert.Equal(types.Number(43), v)
}
type primitiveSliceType []string
func (u primitiveSliceType) MarshalNoms() (types.Value, error) {
func (u primitiveSliceType) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
return types.String(strings.Join(u, ",")), nil
}
func TestMarshalerPrimitiveSliceType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
u := primitiveSliceType([]string{"a", "b", "c"})
v := MustMarshal(u)
v := MustMarshal(vs, u)
assert.Equal(types.String("a,b,c"), v)
}
type primitiveMapType map[string]string
func (u primitiveMapType) MarshalNoms() (types.Value, error) {
func (u primitiveMapType) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
var vals types.ValueSlice
for k, v := range u {
vals = append(vals, types.String(k+","+v))
}
return types.NewSet(vals...), nil
return types.NewSet(vrw, vals...), nil
}
func TestMarshalerPrimitiveMapType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
u := primitiveMapType(map[string]string{
"a": "foo",
"b": "bar",
})
v := MustMarshal(u)
assert.True(types.NewSet(types.String("a,foo"), types.String("b,bar")).Equals(v))
v := MustMarshal(vs, u)
assert.True(types.NewSet(vs, types.String("a,foo"), types.String("b,bar")).Equals(v))
}
type primitiveStructType struct {
x, y int
}
func (u primitiveStructType) MarshalNoms() (types.Value, error) {
func (u primitiveStructType) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
return types.Number(u.x + u.y), nil
}
func TestMarshalerPrimitiveStructType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
u := primitiveStructType{1, 2}
v := MustMarshal(u)
v := MustMarshal(vs, u)
assert.Equal(types.Number(3), v)
}
type builtinType regexp.Regexp
func (u builtinType) MarshalNoms() (types.Value, error) {
func (u builtinType) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
r := regexp.Regexp(u)
return types.String(r.String()), nil
}
@@ -948,24 +1041,30 @@ func (u builtinType) MarshalNoms() (types.Value, error) {
func TestMarshalerBuiltinType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
s := "[a-z]+$"
r := regexp.MustCompile(s)
u := builtinType(*r)
v := MustMarshal(u)
v := MustMarshal(vs, u)
assert.Equal(types.String(s), v)
}
type wrappedMarshalerType primitiveType
func (u wrappedMarshalerType) MarshalNoms() (types.Value, error) {
func (u wrappedMarshalerType) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
return types.Number(int(u) + 2), nil
}
func TestMarshalerWrapperMarshalerType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
u := wrappedMarshalerType(primitiveType(42))
v := MustMarshal(u)
v := MustMarshal(vs, u)
assert.Equal(types.Number(44), v)
}
@@ -982,6 +1081,9 @@ type TestComplexStructType struct {
func TestMarshalerComplexStructType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
s := "foo|bar"
r := regexp.MustCompile(s)
u := TestComplexStructType{
@@ -1000,14 +1102,14 @@ func TestMarshalerComplexStructType(t *testing.T) {
B: builtinType(*r),
}
v := MustMarshal(u)
v := MustMarshal(vs, u)
assert.True(types.NewStruct("TestComplexStructType", types.StructData{
"p": types.Number(43),
"ps": types.NewList(types.Number(2), types.Number(3)),
"pm": types.NewMap(types.String("x"), types.Number(101), types.String("y"), types.Number(102)),
"ps": types.NewList(vs, types.Number(2), types.Number(3)),
"pm": types.NewMap(vs, types.String("x"), types.Number(101), types.String("y"), types.Number(102)),
"pslice": types.String("a,b,c"),
"pmap": types.NewSet(types.String("c,123"), types.String("d,456")),
"pmap": types.NewSet(vs, types.String("c,123"), types.String("d,456")),
"pstruct": types.Number(30),
"b": types.String(s),
}).Equals(v))
@@ -1017,35 +1119,38 @@ type returnsMarshalerError struct {
err error
}
func (u returnsMarshalerError) MarshalNoms() (types.Value, error) {
func (u returnsMarshalerError) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
return nil, u.err
}
type returnsMarshalerNil struct{}
func (u returnsMarshalerNil) MarshalNoms() (types.Value, error) {
func (u returnsMarshalerNil) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
return nil, nil
}
type panicsMarshaler struct{}
func (u panicsMarshaler) MarshalNoms() (types.Value, error) {
func (u panicsMarshaler) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
panic("panic")
}
func TestMarshalerErrors(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
expErr := errors.New("expected error")
m1 := returnsMarshalerError{expErr}
_, actErr := Marshal(m1)
_, actErr := Marshal(vs, m1)
assert.Equal(expErr, actErr)
m2 := returnsMarshalerNil{}
assert.Panics(func() { Marshal(m2) })
assert.Panics(func() { Marshal(vs, m2) })
m3 := panicsMarshaler{}
assert.Panics(func() { Marshal(m3) })
assert.Panics(func() { Marshal(vs, m3) })
}
type TestStructWithNameImpl struct {
@@ -1058,10 +1163,13 @@ func (ts TestStructWithNameImpl) MarshalNomsStructName() string {
func TestMarshalStructName(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
ts := TestStructWithNameImpl{
X: 1,
}
v := MustMarshal(ts)
v := MustMarshal(vs, ts)
assert.True(types.NewStruct("A", types.StructData{
"x": types.Number(1),
}).Equals(v), types.EncodedValue(v))
@@ -1077,10 +1185,13 @@ func (ts TestStructWithNameImpl2) MarshalNomsStructName() string {
func TestMarshalStructName2(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
ts := TestStructWithNameImpl2{
X: 1,
}
v := MustMarshal(ts)
v := MustMarshal(vs, ts)
assert.True(types.NewStruct("", types.StructData{
"x": types.Number(1),
}).Equals(v), types.EncodedValue(v))
go/marshal/encode_type.go
+15 -15
View File
@@ -22,12 +22,12 @@ import (
//
// If a Go struct contains a noms tag with original the field is skipped since
// the Noms type depends on the original Noms value which is not available.
func MarshalType(v interface{}) (nt *types.Type, err error) {
return MarshalTypeOpt(v, Opt{})
func MarshalType(vrw types.ValueReadWriter, v interface{}) (nt *types.Type, err error) {
return MarshalTypeOpt(vrw, v, Opt{})
}
// MarshalTypeOpt is like MarshalType but with additional options.
func MarshalTypeOpt(v interface{}, opt Opt) (nt *types.Type, err error) {
func MarshalTypeOpt(vrw types.ValueReadWriter, v interface{}, opt Opt) (nt *types.Type, err error) {
defer func() {
if r := recover(); r != nil {
switch r := r.(type) {
@@ -40,23 +40,23 @@ func MarshalTypeOpt(v interface{}, opt Opt) (nt *types.Type, err error) {
}
}
}()
nt = MustMarshalTypeOpt(v, opt)
nt = MustMarshalTypeOpt(vrw, v, opt)
return
}
// MustMarshalType computes a Noms type from a Go type or panics if there is an
// error.
func MustMarshalType(v interface{}) (nt *types.Type) {
return MustMarshalTypeOpt(v, Opt{})
func MustMarshalType(vrw types.ValueReadWriter, v interface{}) (nt *types.Type) {
return MustMarshalTypeOpt(vrw, v, Opt{})
}
// MustMarshalTypeOpt is like MustMarshalType but provides additional options.
func MustMarshalTypeOpt(v interface{}, opt Opt) (nt *types.Type) {
func MustMarshalTypeOpt(vrw types.ValueReadWriter, v interface{}, opt Opt) (nt *types.Type) {
rv := reflect.ValueOf(v)
tags := nomsTags{
set: opt.Set,
}
nt = encodeType(rv.Type(), map[string]reflect.Type{}, tags)
nt = encodeType(vrw, rv.Type(), map[string]reflect.Type{}, tags)
if nt == nil {
panic(&UnsupportedTypeError{Type: rv.Type()})
@@ -77,7 +77,7 @@ type TypeMarshaler interface {
var typeOfTypesType = reflect.TypeOf((*types.Type)(nil))
var typeMarshalerInterface = reflect.TypeOf((*TypeMarshaler)(nil)).Elem()
func encodeType(t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTags) *types.Type {
func encodeType(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTags) *types.Type {
if t.Implements(typeMarshalerInterface) {
v := reflect.Zero(t)
typ, err := v.Interface().(TypeMarshaler).MarshalNomsType()
@@ -137,9 +137,9 @@ func encodeType(t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTa
case reflect.String:
return types.StringType
case reflect.Struct:
return structEncodeType(t, seenStructs)
return structEncodeType(vrw, t, seenStructs)
case reflect.Array, reflect.Slice:
elemType := encodeType(t.Elem(), seenStructs, nomsTags{})
elemType := encodeType(vrw, t.Elem(), seenStructs, nomsTags{})
if elemType == nil {
break
}
@@ -148,7 +148,7 @@ func encodeType(t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTa
}
return types.MakeListType(elemType)
case reflect.Map:
keyType := encodeType(t.Key(), seenStructs, nomsTags{})
keyType := encodeType(vrw, t.Key(), seenStructs, nomsTags{})
if keyType == nil {
break
}
@@ -157,7 +157,7 @@ func encodeType(t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTa
return types.MakeSetType(keyType)
}
valueType := encodeType(t.Elem(), seenStructs, nomsTags{})
valueType := encodeType(vrw, t.Elem(), seenStructs, nomsTags{})
if valueType != nil {
return types.MakeMapType(keyType, valueType)
}
@@ -172,7 +172,7 @@ func encodeType(t reflect.Type, seenStructs map[string]reflect.Type, tags nomsTa
// the type but we also need to look at the value. In these cases this returns
// nil and we have to wait until we have a value to be able to determine the
// type.
func structEncodeType(t reflect.Type, seenStructs map[string]reflect.Type) *types.Type {
func structEncodeType(vrw types.ValueReadWriter, t reflect.Type, seenStructs map[string]reflect.Type) *types.Type {
name := getStructName(t)
if name != "" {
if _, ok := seenStructs[name]; ok {
@@ -181,7 +181,7 @@ func structEncodeType(t reflect.Type, seenStructs map[string]reflect.Type) *type
seenStructs[name] = t
}
fields, knownShape, _ := typeFields(t, seenStructs, true, false)
fields, knownShape, _ := typeFields(vrw, t, seenStructs, true, false)
var structType *types.Type
if knownShape {
go/marshal/encode_type_test.go
+129 -34
View File
@@ -16,10 +16,13 @@ import (
)
func TestMarshalTypeType(tt *testing.T) {
vs := newTestValueStore()
defer vs.Close()
t := func(exp *types.Type, ptr interface{}) {
p := reflect.ValueOf(ptr)
assert.NotEqual(tt, reflect.Ptr, p.Type().Kind())
actual, err := MarshalType(p.Interface())
actual, err := MarshalType(vs, p.Interface())
assert.NoError(tt, err)
assert.NotNil(tt, actual, "%#v", p.Interface())
assert.True(tt, exp.Equals(actual))
@@ -105,7 +108,10 @@ func TestMarshalTypeType(tt *testing.T) {
//
func assertMarshalTypeErrorMessage(t *testing.T, v interface{}, expectedMessage string) {
_, err := MarshalType(v)
vs := newTestValueStore()
defer vs.Close()
_, err := MarshalType(vs, v)
assert.Error(t, err)
assert.Equal(t, expectedMessage, err.Error())
}
@@ -117,6 +123,9 @@ func TestMarshalTypeInvalidTypes(t *testing.T) {
func TestMarshalTypeEmbeddedStruct(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type EmbeddedStruct struct {
B bool
}
@@ -126,7 +135,7 @@ func TestMarshalTypeEmbeddedStruct(t *testing.T) {
}
var s TestStruct
typ := MustMarshalType(s)
typ := MustMarshalType(vs, s)
assert.True(types.MakeStructTypeFromFields("TestStruct", types.FieldMap{
"a": types.NumberType,
@@ -137,6 +146,9 @@ func TestMarshalTypeEmbeddedStruct(t *testing.T) {
func TestMarshalTypeEmbeddedStructSkip(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type EmbeddedStruct struct {
B bool
}
@@ -146,7 +158,7 @@ func TestMarshalTypeEmbeddedStructSkip(t *testing.T) {
}
var s TestStruct
typ := MustMarshalType(s)
typ := MustMarshalType(vs, s)
assert.True(types.MakeStructTypeFromFields("TestStruct", types.FieldMap{
"a": types.NumberType,
@@ -156,6 +168,9 @@ func TestMarshalTypeEmbeddedStructSkip(t *testing.T) {
func TestMarshalTypeEmbeddedStructNamed(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type EmbeddedStruct struct {
B bool
}
@@ -165,7 +180,7 @@ func TestMarshalTypeEmbeddedStructNamed(t *testing.T) {
}
var s TestStruct
typ := MustMarshalType(s)
typ := MustMarshalType(vs, s)
assert.True(types.MakeStructTypeFromFields("TestStruct", types.FieldMap{
"a": types.NumberType,
@@ -185,12 +200,15 @@ func TestMarshalTypeEncodeNonExportedField(t *testing.T) {
func TestMarshalTypeEncodeTaggingSkip(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Abc int `noms:"-"`
Def bool
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeStructTypeFromFields("S", types.FieldMap{
"def": types.BoolType,
@@ -200,13 +218,16 @@ func TestMarshalTypeEncodeTaggingSkip(t *testing.T) {
func TestMarshalTypeNamedFields(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Aaa int `noms:"a"`
Bbb bool `noms:"B"`
Ccc string
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeStructTypeFromFields("S", types.FieldMap{
"a": types.NumberType,
@@ -226,22 +247,28 @@ func TestMarshalTypeInvalidNamedFields(t *testing.T) {
func TestMarshalTypeOmitEmpty(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
String string `noms:",omitempty"`
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeStructType("S", types.StructField{"string", types.StringType, true}).Equals(typ))
}
func ExampleMarshalType() {
vs := newTestValueStore()
defer vs.Close()
type Person struct {
Given string
Female bool
}
var person Person
personNomsType, err := MarshalType(person)
personNomsType, err := MarshalType(vs, person)
if err != nil {
fmt.Println(err)
return
@@ -256,16 +283,24 @@ func ExampleMarshalType() {
func TestMarshalTypeSlice(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
s := []string{"a", "b", "c"}
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeListType(types.StringType).Equals(typ))
}
func TestMarshalTypeArray(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
a := [3]int{1, 2, 3}
typ, err := MarshalType(a)
typ, err := MarshalType(vs, a)
assert.NoError(err)
assert.True(types.MakeListType(types.NumberType).Equals(typ))
}
@@ -273,11 +308,14 @@ func TestMarshalTypeArray(t *testing.T) {
func TestMarshalTypeStructWithSlice(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
List []int
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeStructTypeFromFields("S", types.FieldMap{
"list": types.MakeListType(types.NumberType),
@@ -287,12 +325,15 @@ func TestMarshalTypeStructWithSlice(t *testing.T) {
func TestMarshalTypeRecursive(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type Node struct {
Value int
Children []Node
}
var n Node
typ, err := MarshalType(n)
typ, err := MarshalType(vs, n)
assert.NoError(err)
typ2 := types.MakeStructType("Node",
@@ -311,8 +352,11 @@ func TestMarshalTypeRecursive(t *testing.T) {
func TestMarshalTypeMap(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var m map[string]int
typ, err := MarshalType(m)
typ, err := MarshalType(vs, m)
assert.NoError(err)
assert.True(types.MakeMapType(types.StringType, types.NumberType).Equals(typ))
@@ -321,7 +365,7 @@ func TestMarshalTypeMap(t *testing.T) {
}
var m2 map[S]bool
typ, err = MarshalType(m2)
typ, err = MarshalType(vs, m2)
assert.NoError(err)
assert.True(types.MakeMapType(
types.MakeStructTypeFromFields("S", types.FieldMap{
@@ -333,6 +377,9 @@ func TestMarshalTypeMap(t *testing.T) {
func TestMarshalTypeSet(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
A map[int]struct{} `noms:",set"`
B map[int]struct{}
@@ -344,7 +391,7 @@ func TestMarshalTypeSet(t *testing.T) {
H string `noms:",set"`
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
emptyStructType := types.MakeStructTypeFromFields("", types.FieldMap{})
@@ -364,6 +411,9 @@ func TestMarshalTypeSet(t *testing.T) {
func TestEncodeTypeOpt(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
tc := []struct {
in interface{}
opt Opt
@@ -392,7 +442,7 @@ func TestEncodeTypeOpt(t *testing.T) {
}
for _, t := range tc {
r, err := MarshalTypeOpt(t.in, t.opt)
r, err := MarshalTypeOpt(vs, t.in, t.opt)
assert.True(t.wantType.Equals(r))
assert.Nil(err)
}
@@ -401,6 +451,9 @@ func TestEncodeTypeOpt(t *testing.T) {
func TestMarshalTypeSetWithTags(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
A map[int]struct{} `noms:"foo,set"`
B map[int]struct{} `noms:",omitempty,set"`
@@ -408,7 +461,7 @@ func TestMarshalTypeSetWithTags(t *testing.T) {
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeStructType("S",
types.StructField{"foo", types.MakeSetType(types.NumberType), false},
@@ -418,11 +471,14 @@ func TestMarshalTypeSetWithTags(t *testing.T) {
}
func TestMarshalTypeInvalidTag(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
type S struct {
F string `noms:",omitEmpty"`
}
var s S
_, err := MarshalType(s)
_, err := MarshalType(vs, s)
assert.Error(t, err)
assert.Equal(t, `Unrecognized tag: omitEmpty`, err.Error())
}
@@ -430,12 +486,15 @@ func TestMarshalTypeInvalidTag(t *testing.T) {
func TestMarshalTypeCanSkipUnexportedField(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Abc int
notExported bool `noms:"-"`
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeStructTypeFromFields("S", types.FieldMap{
"abc": types.NumberType,
@@ -445,13 +504,16 @@ func TestMarshalTypeCanSkipUnexportedField(t *testing.T) {
func TestMarshalTypeOriginal(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Foo int `noms:",omitempty"`
Bar types.Struct `noms:",original"`
}
var s S
typ, err := MarshalType(s)
typ, err := MarshalType(vs, s)
assert.NoError(err)
assert.True(types.MakeStructType("S",
types.StructField{"foo", types.NumberType, true},
@@ -461,6 +523,9 @@ func TestMarshalTypeOriginal(t *testing.T) {
func TestMarshalTypeNomsTypes(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type S struct {
Blob types.Blob
Bool types.Bool
@@ -469,7 +534,7 @@ func TestMarshalTypeNomsTypes(t *testing.T) {
Type *types.Type
}
var s S
assert.True(MustMarshalType(s).Equals(
assert.True(MustMarshalType(vs, s).Equals(
types.MakeStructTypeFromFields("S", types.FieldMap{
"blob": types.BlobType,
"bool": types.BoolType,
@@ -487,8 +552,11 @@ func (t primitiveType) MarshalNomsType() (*types.Type, error) {
func TestTypeMarshalerPrimitiveType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var u primitiveType
typ := MustMarshalType(u)
typ := MustMarshalType(vs, u)
assert.Equal(types.NumberType, typ)
}
@@ -499,8 +567,11 @@ func (u primitiveSliceType) MarshalNomsType() (*types.Type, error) {
func TestTypeMarshalerPrimitiveSliceType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var u primitiveSliceType
typ := MustMarshalType(u)
typ := MustMarshalType(vs, u)
assert.Equal(types.StringType, typ)
}
@@ -511,16 +582,22 @@ func (u primitiveMapType) MarshalNomsType() (*types.Type, error) {
func TestTypeMarshalerPrimitiveMapType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var u primitiveMapType
typ := MustMarshalType(u)
typ := MustMarshalType(vs, u)
assert.Equal(types.MakeSetType(types.StringType), typ)
}
func TestTypeMarshalerPrimitiveStructTypeNoMarshalNomsType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var u primitiveStructType
_, err := MarshalType(u)
_, err := MarshalType(vs, u)
assert.Error(err)
assert.Equal("Cannot marshal type which implements marshal.Marshaler, perhaps implement marshal.TypeMarshaler for marshal.primitiveStructType", err.Error())
}
@@ -532,8 +609,11 @@ func (u builtinType) MarshalNomsType() (*types.Type, error) {
func TestTypeMarshalerBuiltinType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var u builtinType
typ := MustMarshalType(u)
typ := MustMarshalType(vs, u)
assert.Equal(types.StringType, typ)
}
@@ -544,8 +624,11 @@ func (u wrappedMarshalerType) MarshalNomsType() (*types.Type, error) {
func TestTypeMarshalerWrapperMarshalerType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var u wrappedMarshalerType
typ := MustMarshalType(u)
typ := MustMarshalType(vs, u)
assert.Equal(types.NumberType, typ)
}
@@ -564,31 +647,40 @@ func (u panicsMarshaler) MarshalNomsType() (*types.Type, error) {
func TestTypeMarshalerErrors(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
expErr := errors.New("expected error")
var m1 returnsMarshalerError
_, actErr := MarshalType(m1)
_, actErr := MarshalType(vs, m1)
assert.Equal(expErr, actErr)
var m2 returnsMarshalerNil
assert.Panics(func() { MarshalType(m2) })
assert.Panics(func() { MarshalType(vs, m2) })
var m3 panicsMarshaler
assert.Panics(func() { MarshalType(m3) })
assert.Panics(func() { MarshalType(vs, m3) })
}
func TestMarshalTypeStructName(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var ts TestStructWithNameImpl
typ := MustMarshalType(ts)
typ := MustMarshalType(vs, ts)
assert.True(types.MakeStructType("A", types.StructField{"x", types.NumberType, false}).Equals(typ), typ.Describe())
}
func TestMarshalTypeStructName2(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
var ts TestStructWithNameImpl2
typ := MustMarshalType(ts)
typ := MustMarshalType(vs, ts)
assert.True(types.MakeStructType("", types.StructField{"x", types.NumberType, false}).Equals(typ), typ.Describe())
}
@@ -606,7 +698,10 @@ func (f OutFace) MarshalNomsStructName() string {
}
func TestMarshalTypeOutface(t *testing.T) {
typ := MustMarshalType(OutPhoto{})
vs := newTestValueStore()
defer vs.Close()
typ := MustMarshalType(vs, OutPhoto{})
expectedType := nomdl.MustParseType(`struct OutPhoto {
faces: Set<struct Face {
blob: Ref<Value>,
go/merge/three_way.go
+13 -13
View File
@@ -184,12 +184,12 @@ func (m *merger) threeWay(a, b, parent types.Value, path types.Path) (merged typ
switch a.Kind() {
case types.ListKind:
if aList, bList, pList, ok := listAssert(a, b, parent); ok {
if aList, bList, pList, ok := listAssert(m.vrw, a, b, parent); ok {
return threeWayListMerge(aList, bList, pList)
}
case types.MapKind:
if aMap, bMap, pMap, ok := mapAssert(a, b, parent); ok {
if aMap, bMap, pMap, ok := mapAssert(m.vrw, a, b, parent); ok {
return m.threeWayMapMerge(aMap, bMap, pMap, path)
}
@@ -203,7 +203,7 @@ func (m *merger) threeWay(a, b, parent types.Value, path types.Path) (merged typ
}
case types.SetKind:
if aSet, bSet, pSet, ok := setAssert(a, b, parent); ok {
if aSet, bSet, pSet, ok := setAssert(m.vrw, a, b, parent); ok {
return m.threeWaySetMerge(aSet, bSet, pSet, path)
}
@@ -225,9 +225,9 @@ func (m *merger) threeWayMapMerge(a, b, parent types.Map, path types.Path) (merg
defer updateProgress(m.progress)
switch change.ChangeType {
case types.DiffChangeAdded, types.DiffChangeModified:
return mapCandidate{target.getValue().(types.Map).Edit().Set(change.Key, newVal).Map(nil)}
return mapCandidate{target.getValue().(types.Map).Edit().Set(change.Key, newVal).Map()}
case types.DiffChangeRemoved:
return mapCandidate{target.getValue().(types.Map).Edit().Remove(change.Key).Map(nil)}
return mapCandidate{target.getValue().(types.Map).Edit().Remove(change.Key).Map()}
default:
panic("Not Reached")
}
@@ -240,9 +240,9 @@ func (m *merger) threeWaySetMerge(a, b, parent types.Set, path types.Path) (merg
defer updateProgress(m.progress)
switch change.ChangeType {
case types.DiffChangeAdded, types.DiffChangeModified:
return setCandidate{target.getValue().(types.Set).Edit().Insert(newVal).Set(nil)}
return setCandidate{target.getValue().(types.Set).Edit().Insert(newVal).Set()}
case types.DiffChangeRemoved:
return setCandidate{target.getValue().(types.Set).Edit().Remove(newVal).Set(nil)}
return setCandidate{target.getValue().(types.Set).Edit().Remove(newVal).Set()}
default:
panic("Not Reached")
}
@@ -273,26 +273,26 @@ func (m *merger) threeWayStructMerge(a, b, parent types.Struct, path types.Path)
return m.threeWayOrderedSequenceMerge(structCandidate{a}, structCandidate{b}, structCandidate{parent}, apply, path)
}
func listAssert(a, b, parent types.Value) (aList, bList, pList types.List, ok bool) {
func listAssert(vrw types.ValueReadWriter, a, b, parent types.Value) (aList, bList, pList types.List, ok bool) {
var aOk, bOk, pOk bool
aList, aOk = a.(types.List)
bList, bOk = b.(types.List)
if parent != nil {
pList, pOk = parent.(types.List)
} else {
pList, pOk = types.NewList(), true
pList, pOk = types.NewList(vrw), true
}
return aList, bList, pList, aOk && bOk && pOk
}
func mapAssert(a, b, parent types.Value) (aMap, bMap, pMap types.Map, ok bool) {
func mapAssert(vrw types.ValueReadWriter, a, b, parent types.Value) (aMap, bMap, pMap types.Map, ok bool) {
var aOk, bOk, pOk bool
aMap, aOk = a.(types.Map)
bMap, bOk = b.(types.Map)
if parent != nil {
pMap, pOk = parent.(types.Map)
} else {
pMap, pOk = types.NewMap(), true
pMap, pOk = types.NewMap(vrw), true
}
return aMap, bMap, pMap, aOk && bOk && pOk
}
@@ -318,14 +318,14 @@ func refAssert(a, b, parent types.Value, vrw types.ValueReadWriter) (aValue, bVa
return aValue, bValue, pValue, aOk && bOk && pOk
}
func setAssert(a, b, parent types.Value) (aSet, bSet, pSet types.Set, ok bool) {
func setAssert(vrw types.ValueReadWriter, a, b, parent types.Value) (aSet, bSet, pSet types.Set, ok bool) {
var aOk, bOk, pOk bool
aSet, aOk = a.(types.Set)
bSet, bOk = b.(types.Set)
if parent != nil {
pSet, pOk = parent.(types.Set)
} else {
pSet, pOk = types.NewSet(), true
pSet, pOk = types.NewSet(vrw), true
}
return aSet, bSet, pSet, aOk && bOk && pOk
}
go/merge/three_way_keyval_test.go
+5 -5
View File
@@ -75,7 +75,7 @@ func (s *ThreeWayMapMergeSuite) SetupSuite() {
s.create = func(seq seq) (val types.Value) {
if seq != nil {
keyValues := valsToTypesValues(s.create, seq.items()...)
val = types.NewMap(keyValues...)
val = types.NewMap(s.vs, keyValues...)
}
return
}
@@ -209,8 +209,8 @@ func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NilConflict() {
}
func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_ImmediateConflict() {
s.tryThreeWayConflict(types.NewSet(), s.create(mm2b), s.create(mm2), "Cannot merge Set<> with "+s.typeStr)
s.tryThreeWayConflict(s.create(mm2b), types.NewSet(), s.create(mm2), "Cannot merge "+s.typeStr)
s.tryThreeWayConflict(types.NewSet(s.vs), s.create(mm2b), s.create(mm2), "Cannot merge Set<> with "+s.typeStr)
s.tryThreeWayConflict(s.create(mm2b), types.NewSet(s.vs), s.create(mm2), "Cannot merge "+s.typeStr)
}
func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RefConflict() {
@@ -226,8 +226,8 @@ func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_RefConflict() {
}
func (s *ThreeWayKeyValMergeSuite) TestThreeWayMerge_NestedConflict() {
a := mm2a.set("k2", types.NewSet())
s.tryThreeWayConflict(s.create(a), s.create(mm2b), s.create(mm2), types.EncodedValue(types.NewSet()))
a := mm2a.set("k2", types.NewSet(s.vs))
s.tryThreeWayConflict(s.create(a), s.create(mm2b), s.create(mm2), types.EncodedValue(types.NewSet(s.vs)))
s.tryThreeWayConflict(s.create(a), s.create(mm2b), s.create(mm2), types.EncodedValue(s.create(aa1b)))
}
go/merge/three_way_list.go
+1 -1
View File
@@ -120,7 +120,7 @@ func apply(source, target types.List, offset uint64, s types.Splice) types.List
}
toAdd[i] = v
}
return target.Edit().Splice(s.SpAt+offset, s.SpRemoved, toAdd...).List(nil)
return target.Edit().Splice(s.SpAt+offset, s.SpRemoved, toAdd...).List()
}
func describeSplice(s types.Splice) string {
go/merge/three_way_list_test.go
+1 -1
View File
@@ -23,7 +23,7 @@ func (s *ThreeWayListMergeSuite) SetupSuite() {
s.create = func(i seq) (val types.Value) {
if i != nil {
items := valsToTypesValues(s.create, i.items()...)
val = types.NewList(items...)
val = types.NewList(s.vs, items...)
}
return
}
go/merge/three_way_set_test.go
+3 -3
View File
@@ -29,7 +29,7 @@ func (s *ThreeWaySetMergeSuite) SetupSuite() {
s.create = func(i seq) (val types.Value) {
if i != nil {
keyValues := valsToTypesValues(s.create, i.items()...)
val = types.NewSet(keyValues...)
val = types.NewSet(s.vs, keyValues...)
}
return
}
@@ -80,6 +80,6 @@ func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_Refs() {
}
func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_ImmediateConflict() {
s.tryThreeWayConflict(types.NewMap(), s.create(ss1b), s.create(ss1), "Cannot merge Map<> with "+s.typeStr)
s.tryThreeWayConflict(s.create(ss1b), types.NewMap(), s.create(ss1), "Cannot merge "+s.typeStr)
s.tryThreeWayConflict(types.NewMap(s.vs), s.create(ss1b), s.create(ss1), "Cannot merge Map<> with "+s.typeStr)
s.tryThreeWayConflict(s.create(ss1b), types.NewMap(s.vs), s.create(ss1), "Cannot merge "+s.typeStr)
}
go/ngql/query.go
+7 -7
View File
@@ -31,7 +31,7 @@ const (
throughKey = "through"
valueKey = "value"
valuesKey = "values"
vrKey = "vr"
vrwKey = "vrw"
)
// NewRootQueryObject creates a "root" query object that can be used to
@@ -61,22 +61,22 @@ func (tc *TypeConverter) NewRootQueryObject(rootValue types.Value) *graphql.Obje
// NewContext creates a new context.Context with the extra data added to it
// that is required by ngql.
func NewContext(vr types.ValueReader) context.Context {
return context.WithValue(context.Background(), vrKey, vr)
func NewContext(vrw types.ValueReader) context.Context {
return context.WithValue(context.Background(), vrwKey, vrw)
}
// Query takes |rootValue|, builds a GraphQL scheme from rootValue.Type() and
// executes |query| against it, encoding the result to |w|.
func Query(rootValue types.Value, query string, vr types.ValueReader, w io.Writer) {
func Query(rootValue types.Value, query string, vrw types.ValueReadWriter, w io.Writer) {
schemaConfig := graphql.SchemaConfig{}
tc := NewTypeConverter()
queryWithSchemaConfig(rootValue, query, schemaConfig, vr, tc, w)
queryWithSchemaConfig(rootValue, query, schemaConfig, vrw, tc, w)
}
func queryWithSchemaConfig(rootValue types.Value, query string, schemaConfig graphql.SchemaConfig, vr types.ValueReader, tc *TypeConverter, w io.Writer) {
func queryWithSchemaConfig(rootValue types.Value, query string, schemaConfig graphql.SchemaConfig, vrw types.ValueReadWriter, tc *TypeConverter, w io.Writer) {
schemaConfig.Query = tc.NewRootQueryObject(rootValue)
schema, _ := graphql.NewSchema(schemaConfig)
ctx := NewContext(vr)
ctx := NewContext(vrw)
r := graphql.Do(graphql.Params{
Schema: schema,
go/ngql/query_test.go
+76 -70
View File
@@ -95,27 +95,27 @@ func (suite *QueryGraphQLSuite) TestEmbeddedStruct() {
func (suite *QueryGraphQLSuite) TestListBasic() {
for _, valuesKey := range []string{"elements", "values"} {
list := types.NewList()
list := types.NewList(suite.vs)
suite.assertQueryResult(list, "{root{size}}", `{"data":{"root":{"size":0}}}`)
suite.assertQueryResult(list, "{root{"+valuesKey+"}}", `{"data":{"root":{}}}`)
list = types.NewList(types.String("foo"), types.String("bar"), types.String("baz"))
list = types.NewList(suite.vs, types.String("foo"), types.String("bar"), types.String("baz"))
suite.assertQueryResult(list, "{root{"+valuesKey+"}}", `{"data":{"root":{"`+valuesKey+`":["foo","bar","baz"]}}}`)
suite.assertQueryResult(list, "{root{size}}", `{"data":{"root":{"size":3}}}`)
suite.assertQueryResult(list, "{root{"+valuesKey+"(at:1,count:2)}}", `{"data":{"root":{"`+valuesKey+`":["bar","baz"]}}}`)
list = types.NewList(types.Bool(true), types.Bool(false), types.Bool(false))
list = types.NewList(suite.vs, types.Bool(true), types.Bool(false), types.Bool(false))
suite.assertQueryResult(list, "{root{"+valuesKey+"}}", `{"data":{"root":{"`+valuesKey+`":[true,false,false]}}}`)
suite.assertQueryResult(list, "{root{"+valuesKey+"(at:1,count:2)}}", `{"data":{"root":{"`+valuesKey+`":[false,false]}}}`)
list = types.NewList(types.Number(1), types.Number(1.1), types.Number(-100))
list = types.NewList(suite.vs, types.Number(1), types.Number(1.1), types.Number(-100))
suite.assertQueryResult(list, "{root{"+valuesKey+"}}", `{"data":{"root":{"`+valuesKey+`":[1,1.1,-100]}}}`)
suite.assertQueryResult(list, "{root{"+valuesKey+"(at:1,count:2)}}", `{"data":{"root":{"`+valuesKey+`":[1.1,-100]}}}`)
list = types.NewList(types.String("a"), types.String("b"), types.String("c"))
list = types.NewList(suite.vs, types.String("a"), types.String("b"), types.String("c"))
suite.assertQueryResult(list, "{root{"+valuesKey+"(at:4)}}", `{"data":{"root":{"`+valuesKey+`":[]}}}`)
suite.assertQueryResult(list, "{root{"+valuesKey+"(count:0)}}", `{"data":{"root":{"`+valuesKey+`":[]}}}`)
suite.assertQueryResult(list, "{root{"+valuesKey+"(count:10)}}", `{"data":{"root":{"`+valuesKey+`":["a","b","c"]}}}`)
@@ -124,7 +124,7 @@ func (suite *QueryGraphQLSuite) TestListBasic() {
}
func (suite *QueryGraphQLSuite) TestListOfStruct() {
list := types.NewList(
list := types.NewList(suite.vs,
types.NewStruct("Foo", types.StructData{
"a": types.Number(28),
"b": types.String("foo"),
@@ -145,7 +145,7 @@ func (suite *QueryGraphQLSuite) TestListOfStruct() {
}
func (suite *QueryGraphQLSuite) TestListOfStructWithOptionalFields() {
list := types.NewList(
list := types.NewList(suite.vs,
types.NewStruct("Foo", types.StructData{
"a": types.Number(1),
}),
@@ -169,27 +169,27 @@ func (suite *QueryGraphQLSuite) TestListOfStructWithOptionalFields() {
func (suite *QueryGraphQLSuite) TestSetBasic() {
for _, valuesKey := range []string{"elements", "values"} {
set := types.NewSet()
set := types.NewSet(suite.vs)
suite.assertQueryResult(set, "{root{size}}", `{"data":{"root":{"size":0}}}`)
suite.assertQueryResult(set, "{root{"+valuesKey+"}}", `{"data":{"root":{}}}`)
set = types.NewSet(types.String("foo"), types.String("bar"), types.String("baz"))
set = types.NewSet(suite.vs, types.String("foo"), types.String("bar"), types.String("baz"))
suite.assertQueryResult(set, "{root{"+valuesKey+"}}", `{"data":{"root":{"`+valuesKey+`":["bar","baz","foo"]}}}`)
suite.assertQueryResult(set, "{root{size}}", `{"data":{"root":{"size":3}}}`)
suite.assertQueryResult(set, "{root{"+valuesKey+"(count:2)}}", `{"data":{"root":{"`+valuesKey+`":["bar","baz"]}}}`)
set = types.NewSet(types.Bool(true), types.Bool(false))
set = types.NewSet(suite.vs, types.Bool(true), types.Bool(false))
suite.assertQueryResult(set, "{root{"+valuesKey+"}}", `{"data":{"root":{"`+valuesKey+`":[false,true]}}}`)
suite.assertQueryResult(set, "{root{"+valuesKey+"(count:1)}}", `{"data":{"root":{"`+valuesKey+`":[false]}}}`)
set = types.NewSet(types.Number(1), types.Number(1.1), types.Number(-100))
set = types.NewSet(suite.vs, types.Number(1), types.Number(1.1), types.Number(-100))
suite.assertQueryResult(set, "{root{"+valuesKey+"}}", `{"data":{"root":{"`+valuesKey+`":[-100,1,1.1]}}}`)
suite.assertQueryResult(set, "{root{"+valuesKey+"(count:2)}}", `{"data":{"root":{"`+valuesKey+`":[-100,1]}}}`)
set = types.NewSet(types.String("a"), types.String("b"), types.String("c"), types.String("d"))
set = types.NewSet(suite.vs, types.String("a"), types.String("b"), types.String("c"), types.String("d"))
suite.assertQueryResult(set, "{root{"+valuesKey+"(count:0)}}", `{"data":{"root":{"`+valuesKey+`":[]}}}`)
suite.assertQueryResult(set, "{root{"+valuesKey+"(count:2)}}", `{"data":{"root":{"`+valuesKey+`":["a","b"]}}}`)
@@ -204,7 +204,7 @@ func (suite *QueryGraphQLSuite) TestSetBasic() {
}
func (suite *QueryGraphQLSuite) TestSetOfStruct() {
set := types.NewSet(
set := types.NewSet(suite.vs,
types.NewStruct("Foo", types.StructData{
"a": types.Number(28),
"b": types.String("foo"),
@@ -227,11 +227,11 @@ func (suite *QueryGraphQLSuite) TestSetOfStruct() {
func (suite *QueryGraphQLSuite) TestMapBasic() {
for _, entriesKey := range []string{"elements", "entries"} {
m := types.NewMap()
m := types.NewMap(suite.vs)
suite.assertQueryResult(m, "{root{size}}", `{"data":{"root":{"size":0}}}`)
suite.assertQueryResult(m, "{root{"+entriesKey+"}}", `{"data":{"root":{}}}`)
m = types.NewMap(
m = types.NewMap(suite.vs,
types.String("a"), types.Number(1),
types.String("b"), types.Number(2),
types.String("c"), types.Number(3),
@@ -244,7 +244,7 @@ func (suite *QueryGraphQLSuite) TestMapBasic() {
}
func (suite *QueryGraphQLSuite) TestMapOfStruct() {
m := types.NewMap(
m := types.NewMap(suite.vs,
types.String("foo"), types.NewStruct("Foo", types.StructData{
"a": types.Number(28),
"b": types.String("foo"),
@@ -279,14 +279,14 @@ func (suite *QueryGraphQLSuite) TestRef() {
suite.assertQueryResult(r, "{root{targetValue{a}}}", `{"data":{"root":{"targetValue":{"a":28}}}}`)
suite.assertQueryResult(r, "{root{targetValue{a b}}}", `{"data":{"root":{"targetValue":{"a":28,"b":"foo"}}}}`)
r = suite.vs.WriteValue(types.NewList(types.String("foo"), types.String("bar"), types.String("baz")))
r = suite.vs.WriteValue(types.NewList(suite.vs, types.String("foo"), types.String("bar"), types.String("baz")))
suite.assertQueryResult(r, "{root{targetValue{values}}}", `{"data":{"root":{"targetValue":{"values":["foo","bar","baz"]}}}}`)
suite.assertQueryResult(r, "{root{targetValue{values(at:1,count:2)}}}", `{"data":{"root":{"targetValue":{"values":["bar","baz"]}}}}`)
}
func (suite *QueryGraphQLSuite) TestListOfUnionOfStructs() {
list := types.NewList(
list := types.NewList(suite.vs,
types.NewStruct("Foo", types.StructData{
"a": types.Number(28),
"b": types.String("baz"),
@@ -308,7 +308,7 @@ func (suite *QueryGraphQLSuite) TestListOfUnionOfStructs() {
}
func (suite *QueryGraphQLSuite) TestListOfUnionOfStructsConflictingFieldTypes() {
list := types.NewList(
list := types.NewList(suite.vs,
types.NewStruct("Foo", types.StructData{
"a": types.Number(28),
}),
@@ -329,7 +329,7 @@ func (suite *QueryGraphQLSuite) TestListOfUnionOfStructsConflictingFieldTypes()
}
func (suite *QueryGraphQLSuite) TestListOfUnionOfScalars() {
list := types.NewList(
list := types.NewList(suite.vs,
types.Number(28),
types.String("bar"),
types.Bool(true),
@@ -349,10 +349,10 @@ func (suite *QueryGraphQLSuite) TestCyclicStructs() {
s1 := types.NewStruct("A", types.StructData{
"a": types.String("aaa"),
"b": types.NewSet(
"b": types.NewSet(suite.vs,
types.NewStruct("A", types.StructData{
"a": types.String("bbb"),
"b": types.NewSet(),
"b": types.NewSet(suite.vs),
})),
})
@@ -432,14 +432,14 @@ func (suite *QueryGraphQLSuite) TestCyclicStructsWithUnion() {
}
func (suite *QueryGraphQLSuite) TestNestedCollection() {
list := types.NewList(
types.NewSet(
types.NewMap(types.Number(10), types.String("foo")),
types.NewMap(types.Number(20), types.String("bar")),
list := types.NewList(suite.vs,
types.NewSet(suite.vs,
types.NewMap(suite.vs, types.Number(10), types.String("foo")),
types.NewMap(suite.vs, types.Number(20), types.String("bar")),
),
types.NewSet(
types.NewMap(types.Number(30), types.String("baz")),
types.NewMap(types.Number(40), types.String("bat")),
types.NewSet(suite.vs,
types.NewMap(suite.vs, types.Number(30), types.String("baz")),
types.NewMap(suite.vs, types.Number(40), types.String("bat")),
),
)
@@ -450,7 +450,7 @@ func (suite *QueryGraphQLSuite) TestNestedCollection() {
}
func (suite *QueryGraphQLSuite) TestLoFi() {
b := types.NewBlob(bytes.NewBufferString("I am a blob"))
b := types.NewBlob(suite.vs, bytes.NewBufferString("I am a blob"))
suite.assertQueryResult(b, "{root}", `{"data":{"root":"0123456789abcdefghijklmnopqrstuv"}}`)
@@ -468,7 +468,7 @@ func (suite *QueryGraphQLSuite) TestError() {
func (suite *QueryGraphQLSuite) TestMapArgs() {
for _, entriesKey := range []string{"elements", "entries"} {
m := types.NewMap(
m := types.NewMap(suite.vs,
types.String("a"), types.Number(1),
types.String("c"), types.Number(2),
types.String("e"), types.Number(3),
@@ -567,7 +567,7 @@ func (suite *QueryGraphQLSuite) TestMapArgs() {
func (suite *QueryGraphQLSuite) TestMapKeysArg() {
for _, entriesKey := range []string{"elements", "entries"} {
m := types.NewMap(
m := types.NewMap(suite.vs,
types.String("a"), types.Number(1),
types.String("c"), types.Number(2),
types.String("e"), types.Number(3),
@@ -578,7 +578,7 @@ func (suite *QueryGraphQLSuite) TestMapKeysArg() {
suite.assertQueryResult(m, `{root{`+entriesKey+`(keys:[]){value}}}`,
`{"data":{"root":{"`+entriesKey+`":[]}}}`)
m = types.NewMap(
m = types.NewMap(suite.vs,
types.Number(1), types.String("a"),
types.Number(2), types.String("c"),
types.Number(3), types.String("e"),
@@ -601,7 +601,7 @@ func (suite *QueryGraphQLSuite) TestMapKeysArg() {
func (suite *QueryGraphQLSuite) TestSetArgs() {
for _, valuesKey := range []string{"elements", "values"} {
s := types.NewSet(
s := types.NewSet(suite.vs,
types.String("a"),
types.String("c"),
types.String("e"),
@@ -702,7 +702,7 @@ func (suite *QueryGraphQLSuite) TestSetArgs() {
}
func (suite *QueryGraphQLSuite) TestMapValues() {
m := types.NewMap(
m := types.NewMap(suite.vs,
types.String("a"), types.Number(1),
types.String("c"), types.Number(2),
types.String("e"), types.Number(3),
@@ -780,7 +780,7 @@ func (suite *QueryGraphQLSuite) TestMapValues() {
}
func (suite *QueryGraphQLSuite) TestMapKeys() {
m := types.NewMap(
m := types.NewMap(suite.vs,
types.String("a"), types.Number(1),
types.String("c"), types.Number(2),
types.String("e"), types.Number(3),
@@ -859,7 +859,7 @@ func (suite *QueryGraphQLSuite) TestMapKeys() {
func (suite *QueryGraphQLSuite) TestMapNullable() {
// When selecting the result based on keys the values may be null.
m := types.NewMap(
m := types.NewMap(suite.vs,
types.String("a"), types.Number(1),
types.String("c"), types.Number(2),
)
@@ -1020,11 +1020,11 @@ func (suite *QueryGraphQLSuite) TestMutationCollectionArgs() {
}
func (suite *QueryGraphQLSuite) TestMapWithComplexKeys() {
m := types.NewMap(
types.NewList(types.String("a")), types.Number(1),
types.NewList(types.String("c")), types.Number(2),
types.NewList(types.String("e")), types.Number(3),
types.NewList(types.String("g")), types.Number(4),
m := types.NewMap(suite.vs,
types.NewList(suite.vs, types.String("a")), types.Number(1),
types.NewList(suite.vs, types.String("c")), types.Number(2),
types.NewList(suite.vs, types.String("e")), types.Number(3),
types.NewList(suite.vs, types.String("g")), types.Number(4),
)
suite.assertQueryResult(m, `{root{values(key: ["e"])}}`, `{"data":{"root":{"values":[3]}}}`)
@@ -1050,7 +1050,7 @@ func (suite *QueryGraphQLSuite) TestMapWithComplexKeys() {
}
}}`)
m2 := types.NewMap(
m2 := types.NewMap(suite.vs,
types.NewStruct("", types.StructData{
"n": types.String("a"),
}), types.Number(1),
@@ -1075,11 +1075,11 @@ func (suite *QueryGraphQLSuite) TestMapWithComplexKeys() {
}
func (suite *QueryGraphQLSuite) TestSetWithComplexKeys() {
s := types.NewSet(
types.NewList(types.String("a")),
types.NewList(types.String("c")),
types.NewList(types.String("e")),
types.NewList(types.String("g")),
s := types.NewSet(suite.vs,
types.NewList(suite.vs, types.String("a")),
types.NewList(suite.vs, types.String("c")),
types.NewList(suite.vs, types.String("e")),
types.NewList(suite.vs, types.String("g")),
)
suite.assertQueryResult(s, `{root{values(key: ["e"]) { values }}}`,
@@ -1090,7 +1090,7 @@ func (suite *QueryGraphQLSuite) TestSetWithComplexKeys() {
suite.assertQueryResult(s, `{root{values(key: ["g"], through: ["c"]) { values }}}`,
`{"data":{"root":{"values":[{"values":["g"]},{"values":["a"]},{"values":["c"]}]}}}`)
s2 := types.NewSet(
s2 := types.NewSet(suite.vs,
types.NewStruct("", types.StructData{
"n": types.String("a"),
}),
@@ -1130,28 +1130,28 @@ func (suite *QueryGraphQLSuite) TestInputToNomsValue() {
test(types.String("hi"), "hi")
test(types.String(""), "")
test(types.NewList(types.Number(42)), []interface{}{float64(42)})
test(types.NewList(types.Number(1), types.Number(2)), []interface{}{float64(1), float64(2)})
test(types.NewList(suite.vs, types.Number(42)), []interface{}{float64(42)})
test(types.NewList(suite.vs, types.Number(1), types.Number(2)), []interface{}{float64(1), float64(2)})
test(types.NewSet(types.Number(42)), []interface{}{float64(42)})
test(types.NewSet(types.Number(1), types.Number(2)), []interface{}{float64(1), float64(2)})
test(types.NewSet(suite.vs, types.Number(42)), []interface{}{float64(42)})
test(types.NewSet(suite.vs, types.Number(1), types.Number(2)), []interface{}{float64(1), float64(2)})
test(types.NewMap(
test(types.NewMap(suite.vs,
types.String("a"), types.Number(1),
types.String("b"), types.Number(2),
), []interface{}{
map[string]interface{}{"key": "a", "value": 1},
map[string]interface{}{"key": "b", "value": 2},
})
test(types.NewMap(
types.NewList(types.String("a")), types.Number(1),
types.NewList(types.String("b")), types.Number(2),
test(types.NewMap(suite.vs,
types.NewList(suite.vs, types.String("a")), types.Number(1),
types.NewList(suite.vs, types.String("b")), types.Number(2),
), []interface{}{
map[string]interface{}{"key": []interface{}{"a"}, "value": 1},
map[string]interface{}{"key": []interface{}{"b"}, "value": 2},
})
test(types.NewMap(
test(types.NewMap(suite.vs,
types.NewStruct("S", types.StructData{"a": types.Number(1)}), types.Number(11),
types.NewStruct("S", types.StructData{"a": types.Number(2)}), types.Number(22),
), []interface{}{
@@ -1159,7 +1159,7 @@ func (suite *QueryGraphQLSuite) TestInputToNomsValue() {
map[string]interface{}{"key": map[string]interface{}{"a": float64(2)}, "value": 22},
})
test(types.NewSet(
test(types.NewSet(suite.vs,
types.NewStruct("S", types.StructData{"a": types.Number(1)}),
types.NewStruct("S", types.StructData{"a": types.Number(2)}),
), []interface{}{
@@ -1239,13 +1239,13 @@ func (suite *QueryGraphQLSuite) TestVariables() {
suite.JSONEq(expected, string(b))
}
v := types.NewList(types.Number(0), types.Number(1), types.Number(2), types.Number(3))
v := types.NewList(suite.vs, types.Number(0), types.Number(1), types.Number(2), types.Number(3))
test(v, `{"data":{"root":{"values":[0,1,2,3]}}}`, `query Test($c: Int) { root { values(count: $c) } }`, nil)
test(v, `{"data":{"root":{"values":[0,1]}}}`, `query Test($c: Int) { root { values(count: $c) } }`, map[string]interface{}{
"c": 2,
})
m := types.NewMap(
m := types.NewMap(suite.vs,
types.String("a"), types.Number(0),
types.String("b"), types.Number(1),
types.String("c"), types.Number(2),
@@ -1264,7 +1264,7 @@ func (suite *QueryGraphQLSuite) TestVariables() {
"ks": []string{"a", "c"},
})
m2 := types.NewMap(
m2 := types.NewMap(suite.vs,
types.NewStruct("S", types.StructData{"n": types.String("a")}), types.Number(0),
types.NewStruct("S", types.StructData{"n": types.String("b")}), types.Number(1),
types.NewStruct("S", types.StructData{"n": types.String("c")}), types.Number(2),
@@ -1326,9 +1326,9 @@ func (suite *QueryGraphQLSuite) TestVariables() {
},
)
m3 := types.NewMap(
types.NewMap(types.Number(0), types.String("zero")), types.Bool(false),
types.NewMap(types.Number(1), types.String("one")), types.Bool(true),
m3 := types.NewMap(suite.vs,
types.NewMap(suite.vs, types.Number(0), types.String("zero")), types.Bool(false),
types.NewMap(suite.vs, types.Number(1), types.String("one")), types.Bool(true),
)
keyNomsType := types.TypeOf(m3).Desc.(types.CompoundDesc).ElemTypes[0]
tc := NewTypeConverter()
@@ -1408,7 +1408,7 @@ func (suite *QueryGraphQLSuite) TestNameFunc() {
"b": types.Number(2),
})
list := types.NewList(aVal, bVal)
list := types.NewList(suite.vs, aVal, bVal)
tc := NewTypeConverter()
tc.NameFunc = func(nomsType *types.Type, isInputType bool) string {
@@ -1445,7 +1445,7 @@ func (suite *QueryGraphQLSuite) TestNameFunc() {
}`
test(tc, list, expected, query, nil)
set := types.NewSet(aVal,
set := types.NewSet(suite.vs, aVal,
types.NewStruct("A", types.StructData{
"a": types.Number(2),
}),
@@ -1488,7 +1488,7 @@ func (suite *QueryGraphQLSuite) TestNameFunc() {
func TestGetListElementsWithSet(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
v := types.NewSet(types.Number(0), types.Number(1), types.Number(2))
v := types.NewSet(vs, types.Number(0), types.Number(1), types.Number(2))
r := getListElements(vs, v, map[string]interface{}{})
assert.Equal([]interface{}{float64(0), float64(1), float64(2)}, r)
@@ -1506,6 +1506,9 @@ func TestGetListElementsWithSet(t *testing.T) {
func TestNoErrorOnNonCyclicTypeRefsInputType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type User struct {
ID string `noms:"id"`
}
@@ -1515,7 +1518,7 @@ func TestNoErrorOnNonCyclicTypeRefsInputType(t *testing.T) {
}
var a Account
typ := marshal.MustMarshalType(a)
typ := marshal.MustMarshalType(vs, a)
tc := NewTypeConverter()
_, err := tc.NomsTypeToGraphQLInputType(typ)
assert.NoError(err)
@@ -1524,12 +1527,15 @@ func TestNoErrorOnNonCyclicTypeRefsInputType(t *testing.T) {
func TestErrorOnCyclicTypeRefsInputType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
type Node struct {
Children map[string]Node
}
var n Node
typ := marshal.MustMarshalType(n)
typ := marshal.MustMarshalType(vs, n)
tc := NewTypeConverter()
_, err := tc.NomsTypeToGraphQLInputType(typ)
assert.Error(err)
go/ngql/types.go
+29 -29
View File
@@ -96,7 +96,7 @@ type getFieldFn func(v interface{}, fieldName string, ctx context.Context) types
// When a field name is resolved, it may take key:value arguments. A
// getSubvaluesFn handles returning one or more *noms* values whose presence is
// indicated by the provided arguments.
type getSubvaluesFn func(vr types.ValueReader, v types.Value, args map[string]interface{}) interface{}
type getSubvaluesFn func(vrw types.ValueReadWriter, v types.Value, args map[string]interface{}) interface{}
// GraphQL requires all memberTypes in a Union to be Structs, so when a noms
// union contains a scalar, we represent it in that context as a "boxed" value.
@@ -404,7 +404,7 @@ var listArgs = graphql.FieldConfigArgument{
countKey: &graphql.ArgumentConfig{Type: graphql.Int},
}
func getListElements(vr types.ValueReader, v types.Value, args map[string]interface{}) interface{} {
func getListElements(vrw types.ValueReadWriter, v types.Value, args map[string]interface{}) interface{} {
l := v.(types.Collection)
idx := 0
count := int(l.Len())
@@ -450,10 +450,10 @@ func getListElements(vr types.ValueReader, v types.Value, args map[string]interf
return values
}
func getSetElements(vr types.ValueReader, v types.Value, args map[string]interface{}) interface{} {
func getSetElements(vrw types.ValueReadWriter, v types.Value, args map[string]interface{}) interface{} {
s := v.(types.Set)
iter, nomsKey, nomsThrough, count, singleExactMatch := getCollectionArgs(vr, s, args, iteratorFactory{
iter, nomsKey, nomsThrough, count, singleExactMatch := getCollectionArgs(vrw, s, args, iteratorFactory{
IteratorFrom: func(from types.Value) interface{} {
return s.IteratorFrom(from)
},
@@ -497,7 +497,7 @@ func getSetElements(vr types.ValueReader, v types.Value, args map[string]interfa
return values
}
func getCollectionArgs(vr types.ValueReader, col types.Collection, args map[string]interface{}, factory iteratorFactory) (iter interface{}, nomsKey, nomsThrough types.Value, count uint64, singleExactMatch bool) {
func getCollectionArgs(vrw types.ValueReadWriter, col types.Collection, args map[string]interface{}, factory iteratorFactory) (iter interface{}, nomsKey, nomsThrough types.Value, count uint64, singleExactMatch bool) {
typ := types.TypeOf(col)
length := col.Len()
nomsKeyType := typ.Desc.(types.CompoundDesc).ElemTypes[0]
@@ -507,7 +507,7 @@ func getCollectionArgs(vr types.ValueReader, col types.Collection, args map[stri
nomsKeys := make(types.ValueSlice, len(slice))
for i, v := range slice {
var nomsValue types.Value
nomsValue = InputToNomsValue(vr, v, nomsKeyType)
nomsValue = InputToNomsValue(vrw, v, nomsKeyType)
nomsKeys[i] = nomsValue
}
count = uint64(len(slice))
@@ -518,12 +518,12 @@ func getCollectionArgs(vr types.ValueReader, col types.Collection, args map[stri
return
}
nomsThrough = getThroughArg(vr, nomsKeyType, args)
nomsThrough = getThroughArg(vrw, nomsKeyType, args)
count, singleExactMatch = getCountArg(length, args)
if key, ok := args[keyKey]; ok {
nomsKey = InputToNomsValue(vr, key, nomsKeyType)
nomsKey = InputToNomsValue(vrw, key, nomsKeyType)
iter = factory.IteratorFrom(nomsKey)
} else if at, ok := args[atKey]; ok {
idx := at.(int)
@@ -546,10 +546,10 @@ func getCollectionArgs(vr types.ValueReader, col types.Collection, args map[stri
type mapAppender func(slice []interface{}, k, v types.Value) []interface{}
func getMapElements(vr types.ValueReader, v types.Value, args map[string]interface{}, app mapAppender) (interface{}, error) {
func getMapElements(vrw types.ValueReadWriter, v types.Value, args map[string]interface{}, app mapAppender) (interface{}, error) {
m := v.(types.Map)
iter, nomsKey, nomsThrough, count, singleExactMatch := getCollectionArgs(vr, m, args, iteratorFactory{
iter, nomsKey, nomsThrough, count, singleExactMatch := getCollectionArgs(vrw, m, args, iteratorFactory{
IteratorFrom: func(from types.Value) interface{} {
return m.IteratorFrom(from)
},
@@ -612,9 +612,9 @@ func getCountArg(count uint64, args map[string]interface{}) (c uint64, singleExa
return count, false
}
func getThroughArg(vr types.ValueReader, nomsKeyType *types.Type, args map[string]interface{}) types.Value {
func getThroughArg(vrw types.ValueReadWriter, nomsKeyType *types.Type, args map[string]interface{}) types.Value {
if through, ok := args[throughKey]; ok {
return InputToNomsValue(vr, through, nomsKeyType)
return InputToNomsValue(vrw, through, nomsKeyType)
}
return nil
}
@@ -819,8 +819,8 @@ func (tc *TypeConverter) listAndSetToGraphQLObject(nomsType *types.Type) *graphq
Args: args,
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
c := p.Source.(types.Collection)
vr := p.Context.Value(vrKey).(types.ValueReader)
return getSubvalues(vr, c, p.Args), nil
vrw := p.Context.Value(vrwKey).(types.ValueReadWriter)
return getSubvalues(vrw, c, p.Args), nil
},
}
fields[valuesKey] = valuesField
@@ -863,8 +863,8 @@ func (tc *TypeConverter) mapToGraphQLObject(nomsType *types.Type) *graphql.Objec
Args: args,
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
c := p.Source.(types.Collection)
vr := p.Context.Value(vrKey).(types.ValueReader)
return getMapElements(vr, c, p.Args, mapAppendEntry)
vrw := p.Context.Value(vrwKey).(types.ValueReadWriter)
return getMapElements(vrw, c, p.Args, mapAppendEntry)
},
}
fields[entriesKey] = entriesField
@@ -875,8 +875,8 @@ func (tc *TypeConverter) mapToGraphQLObject(nomsType *types.Type) *graphql.Objec
Args: args,
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
c := p.Source.(types.Collection)
vr := p.Context.Value(vrKey).(types.ValueReader)
return getMapElements(vr, c, p.Args, mapAppendKey)
vrw := p.Context.Value(vrwKey).(types.ValueReadWriter)
return getMapElements(vrw, c, p.Args, mapAppendKey)
},
}
fields[valuesKey] = &graphql.Field{
@@ -884,8 +884,8 @@ func (tc *TypeConverter) mapToGraphQLObject(nomsType *types.Type) *graphql.Objec
Args: args,
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
c := p.Source.(types.Collection)
vr := p.Context.Value(vrKey).(types.ValueReader)
return getMapElements(vr, c, p.Args, mapAppendValue)
vrw := p.Context.Value(vrwKey).(types.ValueReadWriter)
return getMapElements(vrw, c, p.Args, mapAppendValue)
},
}
}
@@ -933,7 +933,7 @@ func (tc *TypeConverter) refToGraphQLObject(nomsType *types.Type) *graphql.Objec
Type: targetType,
Resolve: func(p graphql.ResolveParams) (interface{}, error) {
r := p.Source.(types.Ref)
return MaybeGetScalar(r.TargetValue(p.Context.Value(vrKey).(types.ValueReader))), nil
return MaybeGetScalar(r.TargetValue(p.Context.Value(vrwKey).(types.ValueReader))), nil
},
},
}
@@ -959,7 +959,7 @@ func MaybeGetScalar(v types.Value) interface{} {
// InputToNomsValue converts a GraphQL input value (as used in arguments and
// variables) to a Noms value.
func InputToNomsValue(vr types.ValueReader, arg interface{}, nomsType *types.Type) types.Value {
func InputToNomsValue(vrw types.ValueReadWriter, arg interface{}, nomsType *types.Type) types.Value {
switch nomsType.TargetKind() {
case types.BoolKind:
return types.Bool(arg.(bool))
@@ -975,12 +975,12 @@ func InputToNomsValue(vr types.ValueReader, arg interface{}, nomsType *types.Typ
sl := arg.([]interface{})
vs := make(types.ValueSlice, len(sl))
for i, v := range sl {
vs[i] = InputToNomsValue(vr, v, elemType)
vs[i] = InputToNomsValue(vrw, v, elemType)
}
if nomsType.TargetKind() == types.ListKind {
return types.NewList(vs...)
return types.NewList(vrw, vs...)
}
return types.NewSet(vs...)
return types.NewSet(vrw, vs...)
case types.MapKind:
// Maps are passed as [{key: K, value: V}, ...]
keyType := nomsType.Desc.(types.CompoundDesc).ElemTypes[0]
@@ -989,17 +989,17 @@ func InputToNomsValue(vr types.ValueReader, arg interface{}, nomsType *types.Typ
kvs := make(types.ValueSlice, 2*len(sl))
for i, v := range sl {
v := v.(map[string]interface{})
kvs[2*i] = InputToNomsValue(vr, v["key"], keyType)
kvs[2*i+1] = InputToNomsValue(vr, v["value"], valType)
kvs[2*i] = InputToNomsValue(vrw, v["key"], keyType)
kvs[2*i+1] = InputToNomsValue(vrw, v["value"], valType)
}
return types.NewMap(kvs...)
return types.NewMap(vrw, kvs...)
case types.StructKind:
desc := nomsType.Desc.(types.StructDesc)
data := make(types.StructData, desc.Len())
m := arg.(map[string]interface{})
desc.IterFields(func(name string, t *types.Type, optional bool) {
if m[name] != nil || !optional {
data[name] = InputToNomsValue(vr, m[name], t)
data[name] = InputToNomsValue(vrw, m[name], t)
}
})
return types.NewStruct(desc.Name, data)
go/perf/codec-perf-rig/main.go
+20 -20
View File
@@ -57,7 +57,7 @@ func main() {
db := datas.NewDatabase(storage.NewView())
ds := db.GetDataset("test")
t1 := time.Now()
col := buildFns[i](buildCount, valueFn)
col := buildFns[i](db, buildCount, valueFn)
ds, err := db.CommitValue(ds, col)
d.Chk.NoError(err)
buildDuration := time.Since(t1)
@@ -73,7 +73,7 @@ func main() {
db = datas.NewDatabase(storage.NewView())
ds = db.GetDataset("test")
t1 = time.Now()
col = buildIncrFns[i](insertCount, valueFn)
col = buildIncrFns[i](db, insertCount, valueFn)
ds, err = db.CommitValue(ds, col)
d.Chk.NoError(err)
incrDuration := time.Since(t1)
@@ -95,7 +95,7 @@ func main() {
blobBytes := makeBlobBytes(*blobSize)
t1 := time.Now()
blob := types.NewBlob(bytes.NewReader(blobBytes))
blob := types.NewBlob(db, bytes.NewReader(blobBytes))
db.CommitValue(ds, blob)
buildDuration := time.Since(t1)
@@ -116,7 +116,7 @@ func rate(d time.Duration, size uint64) string {
}
type createValueFn func(i uint64) types.Value
type buildCollectionFn func(count uint64, createFn createValueFn) types.Collection
type buildCollectionFn func(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection
type readCollectionFn func(value types.Collection)
func makeBlobBytes(byteLength uint64) []byte {
@@ -162,22 +162,22 @@ func createStruct(i uint64) types.Value {
})
}
func buildList(count uint64, createFn createValueFn) types.Collection {
func buildList(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
values := make([]types.Value, count)
for i := uint64(0); i < count; i++ {
values[i] = createFn(i)
}
return types.NewList(values...)
return types.NewList(vrw, values...)
}
func buildListIncrementally(count uint64, createFn createValueFn) types.Collection {
l := types.NewList().Edit()
func buildListIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
l := types.NewList(vrw).Edit()
for i := uint64(0); i < count; i++ {
l.Insert(i, createFn(i))
l.Append(createFn(i))
}
return l.List(nil)
return l.List()
}
func readList(c types.Collection) {
@@ -185,22 +185,22 @@ func readList(c types.Collection) {
})
}
func buildSet(count uint64, createFn createValueFn) types.Collection {
func buildSet(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
values := make([]types.Value, count)
for i := uint64(0); i < count; i++ {
values[i] = createFn(i)
}
return types.NewSet(values...)
return types.NewSet(vrw, values...)
}
func buildSetIncrementally(count uint64, createFn createValueFn) types.Collection {
s := types.NewSet().Edit()
func buildSetIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
s := types.NewSet(vrw).Edit()
for i := uint64(0); i < count; i++ {
s.Insert(createFn(i))
}
return s.Set(nil)
return s.Set()
}
func readSet(c types.Collection) {
@@ -208,23 +208,23 @@ func readSet(c types.Collection) {
})
}
func buildMap(count uint64, createFn createValueFn) types.Collection {
func buildMap(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
values := make([]types.Value, count*2)
for i := uint64(0); i < count*2; i++ {
values[i] = createFn(i)
}
return types.NewMap(values...)
return types.NewMap(vrw, values...)
}
func buildMapIncrementally(count uint64, createFn createValueFn) types.Collection {
me := types.NewMap().Edit()
func buildMapIncrementally(vrw types.ValueReadWriter, count uint64, createFn createValueFn) types.Collection {
me := types.NewMap(vrw).Edit()
for i := uint64(0); i < count*2; i += 2 {
me.Set(createFn(i), createFn(i+1))
}
return me.Map(nil)
return me.Map()
}
func readMap(c types.Collection) {
go/perf/suite/suite.go
+9 -8
View File
@@ -237,6 +237,8 @@ func Run(datasetID string, t *testing.T, suiteT perfSuiteT) {
}
defer func() {
db := sp.GetDatabase()
reps := make([]types.Value, *perfRepeatFlag)
for i, rep := range testReps {
timesSlice := types.ValueSlice{}
@@ -247,20 +249,18 @@ func Run(datasetID string, t *testing.T, suiteT perfSuiteT) {
"total": types.Number(info.total.Nanoseconds()),
}))
}
reps[i] = types.NewMap(timesSlice...)
reps[i] = types.NewMap(db, timesSlice...)
}
record := types.NewStruct("", map[string]types.Value{
"environment": suite.getEnvironment(),
"environment": suite.getEnvironment(db),
"nomsRevision": types.String(suite.getGitHead(path.Join(suite.AtticLabs, "noms"))),
"testdataRevision": types.String(suite.getGitHead(suite.Testdata)),
"reps": types.NewList(reps...),
"reps": types.NewList(db, reps...),
})
db := sp.GetDatabase()
ds := db.GetDataset(*perfPrefixFlag + datasetID)
var err error
ds, err = db.CommitValue(ds, record)
_, err := db.CommitValue(ds, record)
assert.NoError(err)
}()
@@ -274,6 +274,7 @@ func Run(datasetID string, t *testing.T, suiteT perfSuiteT) {
serverHost, stopServerFn := suite.StartRemoteDatabase()
suite.DatabaseSpec = serverHost
suite.Database = datas.NewDatabase(datas.NewHTTPChunkStore(serverHost, ""))
defer suite.Database.Close()
if t, ok := suiteT.(SetupRepSuite); ok {
t.SetupRep()
@@ -418,7 +419,7 @@ func callSafe(name string, fun reflect.Value, args ...interface{}) error {
})
}
func (suite *PerfSuite) getEnvironment() types.Value {
func (suite *PerfSuite) getEnvironment(vrw types.ValueReadWriter) types.Value {
assert := suite.NewAssert()
env := environment{
@@ -450,7 +451,7 @@ func (suite *PerfSuite) getEnvironment() types.Value {
assert.NoError(err)
env.Host = *hostInfo
envStruct, err := marshal.Marshal(env)
envStruct, err := marshal.Marshal(vrw, env)
assert.NoError(err)
return envStruct
}
go/perf/suite/suite_test.go
+1
View File
@@ -279,6 +279,7 @@ func TestPrefixFlag(t *testing.T) {
}
func TestRunFlag(t *testing.T) {
t.Skip("Fails on Jenkins most of the times")
assert := assert.New(t)
type expect struct {
go/spec/absolute_path_test.go
+6 -6
View File
@@ -33,12 +33,12 @@ func TestAbsolutePathToAndFromString(t *testing.T) {
func TestAbsolutePaths(t *testing.T) {
assert := assert.New(t)
storage := &chunks.MemoryStorage{}
db := datas.NewDatabase(storage.NewView())
s0, s1 := types.String("foo"), types.String("bar")
list := types.NewList(s0, s1)
emptySet := types.NewSet()
list := types.NewList(db, s0, s1)
emptySet := types.NewSet(db)
db := datas.NewDatabase(storage.NewView())
db.WriteValue(s0)
db.WriteValue(s1)
db.WriteValue(list)
@@ -84,13 +84,13 @@ func TestAbsolutePaths(t *testing.T) {
func TestReadAbsolutePaths(t *testing.T) {
assert := assert.New(t)
storage := &chunks.MemoryStorage{}
db := datas.NewDatabase(storage.NewView())
s0, s1 := types.String("foo"), types.String("bar")
list := types.NewList(s0, s1)
list := types.NewList(db, s0, s1)
db := datas.NewDatabase(storage.NewView())
ds := db.GetDataset("ds")
ds, err := db.CommitValue(ds, list)
_, err := db.CommitValue(ds, list)
assert.NoError(err)
vals, err := ReadAbsolutePaths(db, "ds.value[0]", "ds.value[1]")
go/spec/spec_test.go
+1 -1
View File
@@ -91,7 +91,7 @@ func TestMemDatasetPathSpec(t *testing.T) {
db := spec.GetDatabase()
ds := db.GetDataset("test")
ds, err = db.CommitValue(ds, types.NewList(types.Number(42)))
ds, err = db.CommitValue(ds, types.NewList(db, types.Number(42)))
assert.NoError(err)
assert.Equal(types.Number(42), spec.GetValue())
go/types/blob.go
+17 -32
View File
@@ -25,8 +25,8 @@ func newBlob(seq sequence) Blob {
return Blob{seq, &hash.Hash{}}
}
func NewEmptyBlob() Blob {
return Blob{newBlobLeafSequence(nil, []byte{}), &hash.Hash{}}
func NewEmptyBlob(vrw ValueReadWriter) Blob {
return Blob{newBlobLeafSequence(vrw, []byte{}), &hash.Hash{}}
}
func (b Blob) Edit() *BlobEditor {
@@ -135,14 +135,14 @@ func (b Blob) CopyReadAhead(w io.Writer, chunkSize uint64, concurrency int) (n i
// to visit the rightmost prolly tree chunks of this Blob, and the leftmost
// prolly tree chunks of other, so it's efficient.
func (b Blob) Concat(other Blob) Blob {
seq := concat(b.seq, other.seq, func(cur *sequenceCursor, vr ValueReader) *sequenceChunker {
return b.newChunker(cur, vr)
seq := concat(b.seq, other.seq, func(cur *sequenceCursor, vrw ValueReadWriter) *sequenceChunker {
return b.newChunker(cur, vrw)
})
return newBlob(seq)
}
func (b Blob) newChunker(cur *sequenceCursor, vr ValueReader) *sequenceChunker {
return newSequenceChunker(cur, 0, vr, nil, makeBlobLeafChunkFn(vr), newIndexedMetaSequenceChunkFn(BlobKind, vr), hashValueByte)
func (b Blob) newChunker(cur *sequenceCursor, vrw ValueReadWriter) *sequenceChunker {
return newSequenceChunker(cur, 0, vrw, makeBlobLeafChunkFn(vrw), newIndexedMetaSequenceChunkFn(BlobKind, vrw), hashValueByte)
}
// Collection interface
@@ -163,7 +163,7 @@ func (b Blob) hashPointer() *hash.Hash {
}
// Value interface
func (b Blob) Value(vrw ValueReadWriter) Value {
func (b Blob) Value() Value {
return b
}
@@ -231,7 +231,7 @@ func (cbr *BlobReader) Seek(offset int64, whence int) (int64, error) {
return abs, nil
}
func makeBlobLeafChunkFn(vr ValueReader) makeChunkFn {
func makeBlobLeafChunkFn(vrw ValueReadWriter) makeChunkFn {
return func(level uint64, items []sequenceItem) (Collection, orderedKey, uint64) {
d.PanicIfFalse(level == 0)
buff := make([]byte, len(items))
@@ -240,34 +240,26 @@ func makeBlobLeafChunkFn(vr ValueReader) makeChunkFn {
buff[i] = v.(byte)
}
return chunkBlobLeaf(vr, buff)
return chunkBlobLeaf(vrw, buff)
}
}
func chunkBlobLeaf(vr ValueReader, buff []byte) (Collection, orderedKey, uint64) {
blob := newBlob(newBlobLeafSequence(vr, buff))
func chunkBlobLeaf(vrw ValueReadWriter, buff []byte) (Collection, orderedKey, uint64) {
blob := newBlob(newBlobLeafSequence(vrw, buff))
return blob, orderedKeyFromInt(len(buff)), uint64(len(buff))
}
// NewBlob creates a Blob by reading from every Reader in rs and concatenating
// the result. NewBlob uses one goroutine per Reader. Chunks are kept in memory
// as they're created - to reduce memory pressure and write to disk instead,
// use NewStreamingBlob with a non-nil reader.
func NewBlob(rs ...io.Reader) Blob {
return readBlobsP(nil, rs...)
}
// NewStreamingBlob creates a Blob by reading from every Reader in rs and
// concatenating the result. NewStreamingBlob uses one goroutine per Reader.
// NewBlob creates a Blob by reading from every Reader in rs and
// concatenating the result. NewBlob uses one goroutine per Reader.
// If vrw is not nil, chunks are written to vrw instead of kept in memory.
func NewStreamingBlob(vrw ValueReadWriter, rs ...io.Reader) Blob {
func NewBlob(vrw ValueReadWriter, rs ...io.Reader) Blob {
return readBlobsP(vrw, rs...)
}
func readBlobsP(vrw ValueReadWriter, rs ...io.Reader) Blob {
switch len(rs) {
case 0:
return NewEmptyBlob()
return NewEmptyBlob(vrw)
case 1:
return readBlob(rs[0], vrw)
}
@@ -295,7 +287,7 @@ func readBlobsP(vrw ValueReadWriter, rs ...io.Reader) Blob {
}
func readBlob(r io.Reader, vrw ValueReadWriter) Blob {
sc := newEmptySequenceChunker(vrw, vrw, makeBlobLeafChunkFn(vrw), newIndexedMetaSequenceChunkFn(BlobKind, vrw), func(item sequenceItem, rv *rollingValueHasher) {
sc := newEmptySequenceChunker(vrw, makeBlobLeafChunkFn(vrw), newIndexedMetaSequenceChunkFn(BlobKind, vrw), func(item sequenceItem, rv *rollingValueHasher) {
rv.HashByte(item.(byte))
})
@@ -328,14 +320,7 @@ func readBlob(r io.Reader, vrw ValueReadWriter) Blob {
go func(ch chan metaTuple, cp []byte) {
col, key, numLeaves := chunkBlobLeaf(vrw, cp)
var ref Ref
if vrw != nil {
ref = vrw.WriteValue(col)
col = nil
} else {
ref = NewRef(col)
}
ch <- newMetaTuple(ref, key, numLeaves, col)
ch <- newMetaTuple(vrw.WriteValue(col), key, numLeaves)
}(ch, cp)
offset = 0
go/types/blob_editor.go
+5 -9
View File
@@ -28,21 +28,17 @@ func (be *BlobEditor) Kind() NomsKind {
return BlobKind
}
func (be *BlobEditor) Value(vrw ValueReadWriter) Value {
return be.Blob(vrw)
func (be *BlobEditor) Value() Value {
return be.Blob()
}
func (be *BlobEditor) Blob(vrw ValueReadWriter) Blob {
func (be *BlobEditor) Blob() Blob {
if be.edits == nil {
return be.b // no edits
}
seq := be.b.sequence()
vr := seq.valueReader()
if vrw != nil {
vr = vrw
}
vrw := seq.valueReadWriter()
curs := make([]chan *sequenceCursor, 0)
for edit := be.edits; edit != nil; edit = edit.next {
@@ -63,7 +59,7 @@ func (be *BlobEditor) Blob(vrw ValueReadWriter) Blob {
idx++
if ch == nil {
ch = newSequenceChunker(cur, 0, vr, vrw, makeBlobLeafChunkFn(vr), newIndexedMetaSequenceChunkFn(BlobKind, vr), hashValueByte)
ch = newSequenceChunker(cur, 0, vrw, makeBlobLeafChunkFn(vrw), newIndexedMetaSequenceChunkFn(BlobKind, vrw), hashValueByte)
} else {
ch.advanceTo(cur)
}
go/types/blob_editor_test.go
+3 -3
View File
@@ -35,7 +35,7 @@ func TestBlobReadWriteFuzzer(t *testing.T) {
cr := newCountingReader()
for i := 0; i < rounds; i++ {
b := NewBlob()
b := NewBlob(vs)
f, _ := ioutil.TempFile("", "buff")
be := b.Edit()
@@ -68,13 +68,13 @@ func TestBlobReadWriteFuzzer(t *testing.T) {
}
if j%flushEvery == 0 {
// Flush
b = be.Blob(vs)
b = be.Blob()
be = b.Edit()
}
}
f.Sync()
b = be.Blob(vs)
b = be.Blob()
f.Seek(0, 0)
info, err := f.Stat()
go/types/blob_leaf_sequence.go
+5 -2
View File
@@ -4,13 +4,16 @@
package types
import "github.com/attic-labs/noms/go/d"
type blobLeafSequence struct {
leafSequence
data []byte
}
func newBlobLeafSequence(vr ValueReader, data []byte) sequence {
return blobLeafSequence{leafSequence{vr, len(data), BlobKind}, data}
func newBlobLeafSequence(vrw ValueReadWriter, data []byte) sequence {
d.PanicIfTrue(vrw == nil)
return blobLeafSequence{leafSequence{vrw, len(data), BlobKind}, data}
}
// sequence interface
go/types/blob_test.go
+24 -19
View File
@@ -60,9 +60,11 @@ type blobTestSuite struct {
}
func newBlobTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int, expectAppendChunkDiff int) *blobTestSuite {
vrw := newTestValueStore()
length := 1 << size
buff := randomBuff(size)
blob := NewBlob(bytes.NewReader(buff))
blob := NewBlob(vrw, bytes.NewReader(buff))
return &blobTestSuite{
collectionTestSuite: collectionTestSuite{
col: blob,
@@ -81,13 +83,13 @@ func newBlobTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff in
dup := make([]byte, length+1)
dup[0] = 0
copy(dup[1:], buff)
return NewBlob(bytes.NewReader(dup))
return NewBlob(vrw, bytes.NewReader(dup))
},
appendOne: func() Collection {
dup := make([]byte, length+1)
copy(dup, buff)
dup[len(dup)-1] = 0
return NewBlob(bytes.NewReader(dup))
return NewBlob(vrw, bytes.NewReader(dup))
},
},
buff: buff,
@@ -193,9 +195,10 @@ func TestBlobFromReaderThatReturnsDataAndError(t *testing.T) {
// See issue #264.
// This tests the case of building a Blob from a reader who returns both data and an error for the final Read() call.
assert := assert.New(t)
vrw := newTestValueStore()
tr := &testReader{buf: &bytes.Buffer{}}
b := NewBlob(tr)
b := NewBlob(vrw, tr)
actual := &bytes.Buffer{}
io.Copy(actual, b.Reader())
@@ -206,26 +209,27 @@ func TestBlobFromReaderThatReturnsDataAndError(t *testing.T) {
func TestBlobSplice(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
blob := NewEmptyBlob()
blob := NewEmptyBlob(vrw)
buf := new(bytes.Buffer)
blob = blob.Edit().Splice(0, 0, []byte("I'll do anything")).Blob(nil)
blob = blob.Edit().Splice(0, 0, []byte("I'll do anything")).Blob()
buf.Reset()
buf.ReadFrom(blob.Reader())
assert.Equal(buf.String(), "I'll do anything")
blob = blob.Edit().Splice(16, 0, []byte(" for arv")).Blob(nil)
blob = blob.Edit().Splice(16, 0, []byte(" for arv")).Blob()
buf.Reset()
buf.ReadFrom(blob.Reader())
assert.Equal(buf.String(), "I'll do anything for arv")
blob = blob.Edit().Splice(0, 0, []byte("Yes, ")).Blob(nil)
blob = blob.Edit().Splice(0, 0, []byte("Yes, ")).Blob()
buf.Reset()
buf.ReadFrom(blob.Reader())
assert.Equal(buf.String(), "Yes, I'll do anything for arv")
blob = blob.Edit().Splice(5, 20, []byte("it's hard to satisfy")).Blob(nil)
blob = blob.Edit().Splice(5, 20, []byte("it's hard to satisfy")).Blob()
buf.Reset()
buf.ReadFrom(blob.Reader())
assert.Equal(buf.String(), "Yes, it's hard to satisfy arv")
@@ -244,22 +248,22 @@ func TestBlobConcat(t *testing.T) {
split := func(b Blob, at int64) (Blob, Blob) {
read1, read2 := b.Reader(), b.Reader()
b1 := NewBlob(&io.LimitedReader{read1, at})
b1 := NewBlob(vs, &io.LimitedReader{read1, at})
read2.Seek(at, 0)
b2 := NewBlob(read2)
b2 := NewBlob(vs, read2)
return reload(b1), reload(b2)
}
// Random 1MB Blob.
// Note that List.Concat is exhaustively tested, don't worry here.
r := rand.New(rand.NewSource(0))
b := NewBlob(&io.LimitedReader{r, 1e6})
b := NewBlob(vs, &io.LimitedReader{r, 1e6})
b = reload(b)
b1 := NewEmptyBlob().Concat(b)
b1 := NewEmptyBlob(vs).Concat(b)
assert.True(b.Equals(b1))
b2 := b.Concat(NewEmptyBlob())
b2 := b.Concat(NewEmptyBlob(vs))
assert.True(b.Equals(b2))
b3, b4 := split(b, 10)
@@ -274,6 +278,7 @@ func TestBlobConcat(t *testing.T) {
func TestBlobNewParallel(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
readAll := func(b Blob) []byte {
data, err := ioutil.ReadAll(b.Reader())
@@ -281,13 +286,13 @@ func TestBlobNewParallel(t *testing.T) {
return data
}
b := NewBlob()
b := NewBlob(vrw)
assert.True(b.Len() == 0)
b = NewBlob(strings.NewReader("abc"))
b = NewBlob(vrw, strings.NewReader("abc"))
assert.Equal("abc", string(readAll(b)))
b = NewBlob(strings.NewReader("abc"), strings.NewReader("def"))
b = NewBlob(vrw, strings.NewReader("abc"), strings.NewReader("def"))
assert.Equal("abcdef", string(readAll(b)))
p, size := 100, 1024
@@ -301,7 +306,7 @@ func TestBlobNewParallel(t *testing.T) {
readers[i] = bytes.NewBuffer(data[i*size : (i+1)*size])
}
b = NewBlob(readers...)
b = NewBlob(vrw, readers...)
assert.Equal(data, readAll(b))
}
@@ -318,7 +323,7 @@ func TestStreamingParallelBlob(t *testing.T) {
}
vs := newTestValueStore()
blob := NewStreamingBlob(vs, readers...)
blob := NewBlob(vs, readers...)
outBuff := &bytes.Buffer{}
blob.Copy(outBuff)
assert.True(bytes.Compare(buff, outBuff.Bytes()) == 0)
go/types/bool.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
type Bool bool
// Value interface
func (v Bool) Value(vrw ValueReadWriter) Value {
func (v Bool) Value() Value {
return v
}
go/types/codec.go
+6 -6
View File
@@ -27,26 +27,26 @@ func EncodeValue(v Value) chunks.Chunk {
return c
}
func DecodeFromBytes(data []byte, vr ValueReader) Value {
func DecodeFromBytes(data []byte, vrw ValueReadWriter) Value {
br := &binaryNomsReader{data, 0}
dec := newValueDecoder(br, vr)
dec := newValueDecoder(br, vrw)
v := dec.readValue()
d.PanicIfFalse(br.pos() == uint32(len(data)))
return v
}
func decodeFromBytesWithValidation(data []byte, vr ValueReader) Value {
func decodeFromBytesWithValidation(data []byte, vrw ValueReadWriter) Value {
br := &binaryNomsReader{data, 0}
dec := newValueDecoderWithValidation(br, vr)
dec := newValueDecoderWithValidation(br, vrw)
v := dec.readValue()
d.PanicIfFalse(br.pos() == uint32(len(data)))
return v
}
// DecodeValue decodes a value from a chunk source. It is an error to provide an empty chunk.
func DecodeValue(c chunks.Chunk, vr ValueReader) Value {
func DecodeValue(c chunks.Chunk, vrw ValueReadWriter) Value {
d.PanicIfTrue(c.IsEmpty())
v := DecodeFromBytes(c.Data(), vr)
v := DecodeFromBytes(c.Data(), vrw)
if cacher, ok := v.(hashCacher); ok {
assignHash(cacher, c.Hash())
}
go/types/collection_test.go
+1 -6
View File
@@ -46,12 +46,7 @@ func (suite *collectionTestSuite) TestChunkCountAndType() {
}
func (suite *collectionTestSuite) TestRoundTripAndValidate() {
vs := newTestValueStore()
r := vs.WriteValue(suite.col)
v2 := vs.ReadValue(r.TargetHash()).(Collection)
suite.True(v2.Equals(suite.col))
suite.True(suite.col.Equals(v2))
suite.True(suite.validate(v2))
suite.True(suite.validate(suite.col))
}
func (suite *collectionTestSuite) TestPrependChunkDiff() {
go/types/compare_test.go
+9 -8
View File
@@ -16,6 +16,7 @@ var prefix = []byte{0x01, 0x02, 0x03, 0x04}
func TestCompareTotalOrdering(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
// values in increasing order. Some of these are compared by ref so changing the serialization might change the ordering.
values := []Value{
@@ -24,7 +25,7 @@ func TestCompareTotalOrdering(t *testing.T) {
String("a"), String("b"), String("c"),
// The order of these are done by the hash.
NewSet(Number(0), Number(1), Number(2), Number(3)),
NewSet(vrw, Number(0), Number(1), Number(2), Number(3)),
BoolType,
// Value - values cannot be value
@@ -63,11 +64,11 @@ func TestCompareDifferentPrimitiveTypes(t *testing.T) {
nums := ValueSlice{Number(1), Number(2), Number(3)}
words := ValueSlice{String("k1"), String("v1")}
blob := NewBlob(bytes.NewBuffer([]byte{1, 2, 3}))
nList := NewList(nums...)
nMap := NewMap(words...)
blob := NewBlob(vrw, bytes.NewBuffer([]byte{1, 2, 3}))
nList := NewList(vrw, nums...)
nMap := NewMap(vrw, words...)
nRef := NewRef(blob)
nSet := NewSet(nums...)
nSet := NewSet(vrw, nums...)
nStruct := NewStruct("teststruct", map[string]Value{"f1": Number(1)})
vals := ValueSlice{Bool(true), Number(19), String("hellow"), blob, nList, nMap, nRef, nSet, nStruct}
@@ -77,7 +78,7 @@ func TestCompareDifferentPrimitiveTypes(t *testing.T) {
for j, v2 := range vals {
iBytes := [1024]byte{}
jBytes := [1024]byte{}
res := compareEncodedKey(encodeGraphKey(iBytes[:0], v1, vrw), encodeGraphKey(jBytes[:0], v2, vrw))
res := compareEncodedKey(encodeGraphKey(iBytes[:0], v1), encodeGraphKey(jBytes[:0], v2))
assert.Equal(compareInts(i, j), res)
}
}
@@ -123,8 +124,8 @@ func TestCompareEncodedKeys(t *testing.T) {
bs1 := [initialBufferSize]byte{}
bs2 := [initialBufferSize]byte{}
e1, _ := encodeKeys(bs1[:0], 0x01020304, MapKind, k1, vrw)
e2, _ := encodeKeys(bs2[:0], 0x01020304, MapKind, k2, vrw)
e1, _ := encodeKeys(bs1[:0], 0x01020304, MapKind, k1)
e2, _ := encodeKeys(bs2[:0], 0x01020304, MapKind, k2)
assert.Equal(-1, comp.Compare(e1, e2))
}
go/types/encode_human_readable_test.go
+41 -27
View File
@@ -69,23 +69,25 @@ func TestWriteHumanReadableRef(t *testing.T) {
}
func TestWriteHumanReadableCollections(t *testing.T) {
l := NewList(Number(0), Number(1), Number(2), Number(3))
vrw := newTestValueStore()
l := NewList(vrw, Number(0), Number(1), Number(2), Number(3))
assertWriteHRSEqual(t, "[ // 4 items\n 0,\n 1,\n 2,\n 3,\n]", l)
assertWriteTaggedHRSEqual(t, "List<Number>([ // 4 items\n 0,\n 1,\n 2,\n 3,\n])", l)
s := NewSet(Number(0), Number(1), Number(2), Number(3))
s := NewSet(vrw, Number(0), Number(1), Number(2), Number(3))
assertWriteHRSEqual(t, "{ // 4 items\n 0,\n 1,\n 2,\n 3,\n}", s)
assertWriteTaggedHRSEqual(t, "Set<Number>({ // 4 items\n 0,\n 1,\n 2,\n 3,\n})", s)
m := NewMap(Number(0), Bool(false), Number(1), Bool(true))
m := NewMap(vrw, Number(0), Bool(false), Number(1), Bool(true))
assertWriteHRSEqual(t, "{\n 0: false,\n 1: true,\n}", m)
assertWriteTaggedHRSEqual(t, "Map<Number, Bool>({\n 0: false,\n 1: true,\n})", m)
l2 := NewList()
l2 := NewList(vrw)
assertWriteHRSEqual(t, "[]", l2)
assertWriteTaggedHRSEqual(t, "List<>([])", l2)
l3 := NewList(Number(0))
l3 := NewList(vrw, Number(0))
assertWriteHRSEqual(t, "[\n 0,\n]", l3)
assertWriteTaggedHRSEqual(t, "List<Number>([\n 0,\n])", l3)
@@ -93,18 +95,20 @@ func TestWriteHumanReadableCollections(t *testing.T) {
for i := range nums {
nums[i] = Number(0)
}
l4 := NewList(nums...)
l4 := NewList(vrw, nums...)
assertWriteTaggedHRSEqual(t, "List<Number>([ // 2,000 items\n"+strings.Repeat(" 0,\n", 2000)+"])", l4)
}
func TestWriteHumanReadableNested(t *testing.T) {
l := NewList(Number(0), Number(1))
l2 := NewList(Number(2), Number(3))
vrw := newTestValueStore()
s := NewSet(String("a"), String("b"))
s2 := NewSet(String("c"), String("d"))
l := NewList(vrw, Number(0), Number(1))
l2 := NewList(vrw, Number(2), Number(3))
m := NewMap(s, l, s2, l2)
s := NewSet(vrw, String("a"), String("b"))
s2 := NewSet(vrw, String("c"), String("d"))
m := NewMap(vrw, s, l, s2, l2)
assertWriteHRSEqual(t, `{
{
"c",
@@ -149,6 +153,8 @@ func TestWriteHumanReadableStruct(t *testing.T) {
}
func TestWriteHumanReadableListOfStruct(t *testing.T) {
vrw := newTestValueStore()
str1 := NewStruct("S3", StructData{
"x": Number(1),
})
@@ -158,7 +164,7 @@ func TestWriteHumanReadableListOfStruct(t *testing.T) {
str3 := NewStruct("S3", StructData{
"x": Number(3),
})
l := NewList(str1, str2, str3)
l := NewList(vrw, str1, str2, str3)
assertWriteHRSEqual(t, `[
S3 {
x: 1,
@@ -186,25 +192,27 @@ func TestWriteHumanReadableListOfStruct(t *testing.T) {
}
func TestWriteHumanReadableBlob(t *testing.T) {
assertWriteHRSEqual(t, "", NewEmptyBlob())
assertWriteTaggedHRSEqual(t, "Blob()", NewEmptyBlob())
vrw := newTestValueStore()
b1 := NewBlob(bytes.NewBuffer([]byte{0x01}))
assertWriteHRSEqual(t, "", NewEmptyBlob(vrw))
assertWriteTaggedHRSEqual(t, "Blob()", NewEmptyBlob(vrw))
b1 := NewBlob(vrw, bytes.NewBuffer([]byte{0x01}))
assertWriteHRSEqual(t, "01", b1)
assertWriteTaggedHRSEqual(t, "Blob(01)", b1)
b2 := NewBlob(bytes.NewBuffer([]byte{0x01, 0x02}))
b2 := NewBlob(vrw, bytes.NewBuffer([]byte{0x01, 0x02}))
assertWriteHRSEqual(t, "01 02", b2)
assertWriteTaggedHRSEqual(t, "Blob(01 02)", b2)
b3 := NewBlob(bytes.NewBuffer([]byte{
b3 := NewBlob(vrw, bytes.NewBuffer([]byte{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
}))
assertWriteHRSEqual(t, "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f", b3)
assertWriteTaggedHRSEqual(t, "Blob(00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f)", b3)
b4 := NewBlob(bytes.NewBuffer([]byte{
b4 := NewBlob(vrw, bytes.NewBuffer([]byte{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10,
@@ -217,25 +225,27 @@ func TestWriteHumanReadableBlob(t *testing.T) {
bs[i] = byte(i)
}
b5 := NewBlob(bytes.NewBuffer(bs))
b5 := NewBlob(vrw, bytes.NewBuffer(bs))
assertWriteHRSEqual(t, "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f // 256 B\n10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f\n20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f\n30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f\n40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f\n50 51 52 53 54 55 56 57 58 59 5a 5b 5c 5d 5e 5f\n60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f\n70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f\n80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f\n90 91 92 93 94 95 96 97 98 99 9a 9b 9c 9d 9e 9f\na0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af\nb0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb bc bd be bf\nc0 c1 c2 c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf\nd0 d1 d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df\ne0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef\nf0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff", b5)
assertWriteTaggedHRSEqual(t, "Blob(00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f // 256 B\n10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f\n20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f\n30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f\n40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f\n50 51 52 53 54 55 56 57 58 59 5a 5b 5c 5d 5e 5f\n60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f\n70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f\n80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f\n90 91 92 93 94 95 96 97 98 99 9a 9b 9c 9d 9e 9f\na0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af\nb0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb bc bd be bf\nc0 c1 c2 c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf\nd0 d1 d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df\ne0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef\nf0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff)", b5)
b6 := NewBlob(bytes.NewBuffer(make([]byte, 16*100)))
b6 := NewBlob(vrw, bytes.NewBuffer(make([]byte, 16*100)))
row := strings.Repeat("00 ", 15) + "00"
s := strings.Repeat(row+"\n", 98) + row
assertWriteTaggedHRSEqual(t, "Blob("+row+" // 1.6 kB\n"+s+")", b6)
}
func TestWriteHumanReadableListOfBlob(t *testing.T) {
b1 := NewBlob(bytes.NewBuffer([]byte{0x01}))
b2 := NewBlob(bytes.NewBuffer([]byte{0x02}))
b3 := NewBlob(bytes.NewBuffer([]byte{
vrw := newTestValueStore()
b1 := NewBlob(vrw, bytes.NewBuffer([]byte{0x01}))
b2 := NewBlob(vrw, bytes.NewBuffer([]byte{0x02}))
b3 := NewBlob(vrw, bytes.NewBuffer([]byte{
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10,
}))
l := NewList(b1, NewEmptyBlob(), b2, b3)
l := NewList(vrw, b1, NewEmptyBlob(vrw), b2, b3)
assertWriteHRSEqual(t, "[ // 4 items\n 01,\n ,\n 02,\n 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f // 17 B\n 10,\n]", l)
assertWriteTaggedHRSEqual(t, "List<Blob>([ // 4 items\n 01,\n ,\n 02,\n 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f // 17 B\n 10,\n])", l)
}
@@ -391,23 +401,27 @@ func TestWriteHumanReadableWriterError(t *testing.T) {
}
func TestEmptyCollections(t *testing.T) {
vrw := newTestValueStore()
a := MakeStructType("Nothing")
assertWriteTaggedHRSEqual(t, "Type(struct Nothing {})", a)
b := NewStruct("Rien", StructData{})
assertWriteTaggedHRSEqual(t, "struct Rien {}({})", b)
c := MakeMapType(BlobType, NumberType)
assertWriteTaggedHRSEqual(t, "Type(Map<Blob, Number>)", c)
d := NewMap()
d := NewMap(vrw)
assertWriteTaggedHRSEqual(t, "Map<>({})", d)
e := MakeSetType(StringType)
assertWriteTaggedHRSEqual(t, "Type(Set<String>)", e)
f := NewSet()
f := NewSet(vrw)
assertWriteTaggedHRSEqual(t, "Set<>({})", f)
}
func TestEncodedValueMaxLines(t *testing.T) {
assert := assert.New(t)
l1 := NewList(generateNumbersAsValues(11)...)
vrw := newTestValueStore()
l1 := NewList(vrw, generateNumbersAsValues(11)...)
expected := strings.Join(strings.SplitAfterN(EncodedValue(l1), "\n", 6)[:5], "")
assert.Equal(expected, EncodedValueMaxLines(l1, 5))
go/types/encoding_test.go
+85 -46
View File
@@ -114,8 +114,9 @@ func assertEncoding(t *testing.T, expect []interface{}, v Value) {
}
func TestRoundTrips(t *testing.T) {
vs := newTestValueStore()
assertRoundTrips := func(v Value) {
vs := newTestValueStore()
out := DecodeValue(EncodeValue(v), vs)
assert.True(t, v.Equals(out))
}
@@ -163,13 +164,13 @@ func TestRoundTrips(t *testing.T) {
assertRoundTrips(NewStruct("", StructData{"a": Bool(true), "b": String("foo"), "c": Number(2.3)}))
listLeaf := newList(newListLeafSequence(nil, Number(4), Number(5), Number(6), Number(7)))
listLeaf := newList(newListLeafSequence(vs, Number(4), Number(5), Number(6), Number(7)))
assertRoundTrips(listLeaf)
assertRoundTrips(newList(newListMetaSequence(1, []metaTuple{
newMetaTuple(NewRef(listLeaf), orderedKeyFromInt(10), 10, nil),
newMetaTuple(NewRef(listLeaf), orderedKeyFromInt(20), 20, nil),
}, nil)))
newMetaTuple(NewRef(listLeaf), orderedKeyFromInt(10), 10),
newMetaTuple(NewRef(listLeaf), orderedKeyFromInt(20), 20),
}, vs)))
}
func TestNonFiniteNumbers(tt *testing.T) {
@@ -231,24 +232,30 @@ func TestWritePrimitives(t *testing.T) {
}
func TestWriteSimpleBlob(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(BlobKind), uint64(0), []byte{0x00, 0x01},
},
NewBlob(bytes.NewBuffer([]byte{0x00, 0x01})),
NewBlob(vrw, bytes.NewBuffer([]byte{0x00, 0x01})),
)
}
func TestWriteList(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(ListKind), uint64(0), uint64(4) /* len */, uint8(NumberKind), Number(0), uint8(NumberKind), Number(1), uint8(NumberKind), Number(2), uint8(NumberKind), Number(3),
},
NewList(Number(0), Number(1), Number(2), Number(3)),
NewList(vrw, Number(0), Number(1), Number(2), Number(3)),
)
}
func TestWriteListOfList(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(ListKind), uint64(0),
@@ -256,49 +263,57 @@ func TestWriteListOfList(t *testing.T) {
uint8(ListKind), uint64(0), uint64(1) /* len */, uint8(NumberKind), Number(0),
uint8(ListKind), uint64(0), uint64(3) /* len */, uint8(NumberKind), Number(1), uint8(NumberKind), Number(2), uint8(NumberKind), Number(3),
},
NewList(NewList(Number(0)), NewList(Number(1), Number(2), Number(3))),
NewList(vrw, NewList(vrw, Number(0)), NewList(vrw, Number(1), Number(2), Number(3))),
)
}
func TestWriteSet(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(SetKind), uint64(0), uint64(4), /* len */
uint8(NumberKind), Number(0), uint8(NumberKind), Number(1), uint8(NumberKind), Number(2), uint8(NumberKind), Number(3),
},
NewSet(Number(3), Number(1), Number(2), Number(0)),
NewSet(vrw, Number(3), Number(1), Number(2), Number(0)),
)
}
func TestWriteSetOfSet(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(SetKind), uint64(0), uint64(2), // len
uint8(SetKind), uint64(0), uint64(3) /* len */, uint8(NumberKind), Number(1), uint8(NumberKind), Number(2), uint8(NumberKind), Number(3),
uint8(SetKind), uint64(0), uint64(1) /* len */, uint8(NumberKind), Number(0),
},
NewSet(NewSet(Number(0)), NewSet(Number(1), Number(2), Number(3))),
NewSet(vrw, NewSet(vrw, Number(0)), NewSet(vrw, Number(1), Number(2), Number(3))),
)
}
func TestWriteMap(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(MapKind), uint64(0), uint64(2), /* len */
uint8(StringKind), "a", uint8(BoolKind), false, uint8(StringKind), "b", uint8(BoolKind), true,
},
NewMap(String("a"), Bool(false), String("b"), Bool(true)),
NewMap(vrw, String("a"), Bool(false), String("b"), Bool(true)),
)
}
func TestWriteMapOfMap(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(MapKind), uint64(0), uint64(1), // len
uint8(MapKind), uint64(0), uint64(1) /* len */, uint8(StringKind), "a", uint8(NumberKind), Number(0),
uint8(SetKind), uint64(0), uint64(1) /* len */, uint8(BoolKind), true,
},
NewMap(NewMap(String("a"), Number(0)), NewSet(Bool(true))),
NewMap(vrw, NewMap(vrw, String("a"), Number(0)), NewSet(vrw, Bool(true))),
)
}
@@ -316,9 +331,9 @@ func TestWriteCompoundBlob(t *testing.T) {
uint8(RefKind), r3.String(), uint8(BlobKind), uint64(33), uint8(NumberKind), Number(60), uint64(60),
},
newBlob(newBlobMetaSequence(1, []metaTuple{
newMetaTuple(constructRef(r1, BlobType, 11), orderedKeyFromInt(20), 20, nil),
newMetaTuple(constructRef(r2, BlobType, 22), orderedKeyFromInt(40), 40, nil),
newMetaTuple(constructRef(r3, BlobType, 33), orderedKeyFromInt(60), 60, nil),
newMetaTuple(constructRef(r1, BlobType, 11), orderedKeyFromInt(20), 20),
newMetaTuple(constructRef(r2, BlobType, 22), orderedKeyFromInt(40), 40),
newMetaTuple(constructRef(r3, BlobType, 33), orderedKeyFromInt(60), 60),
}, newTestValueStore())),
)
}
@@ -355,13 +370,15 @@ func TestWriteStructTooMuchData(t *testing.T) {
}
func TestWriteStructWithList(t *testing.T) {
vrw := newTestValueStore()
// struct S {l: List<String>}({l: ["a", "b"]})
assertEncoding(t,
[]interface{}{
uint8(StructKind), "S", uint64(1), /* len */
"l", uint8(ListKind), uint64(0), uint64(2) /* len */, uint8(StringKind), "a", uint8(StringKind), "b",
},
NewStruct("S", StructData{"l": NewList(String("a"), String("b"))}),
NewStruct("S", StructData{"l": NewList(vrw, String("a"), String("b"))}),
)
// struct S {l: List<>}({l: []})
@@ -370,7 +387,7 @@ func TestWriteStructWithList(t *testing.T) {
uint8(StructKind), "S", uint64(1), /* len */
"l", uint8(ListKind), uint64(0), uint64(0), /* len */
},
NewStruct("S", StructData{"l": NewList()}),
NewStruct("S", StructData{"l": NewList(vrw)}),
)
}
@@ -393,18 +410,22 @@ func TestWriteStructWithStruct(t *testing.T) {
}
func TestWriteStructWithBlob(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(StructKind), "S", uint64(1), /* len */
"b", uint8(BlobKind), uint64(0), []byte{0x00, 0x01},
},
NewStruct("S", StructData{"b": NewBlob(bytes.NewBuffer([]byte{0x00, 0x01}))}),
NewStruct("S", StructData{"b": NewBlob(vrw, bytes.NewBuffer([]byte{0x00, 0x01}))}),
)
}
func TestWriteCompoundList(t *testing.T) {
list1 := newList(newListLeafSequence(nil, Number(0)))
list2 := newList(newListLeafSequence(nil, Number(1), Number(2), Number(3)))
vrw := newTestValueStore()
list1 := newList(newListLeafSequence(vrw, Number(0)))
list2 := newList(newListLeafSequence(vrw, Number(1), Number(2), Number(3)))
assertEncoding(t,
[]interface{}{
uint8(ListKind), uint64(1), uint64(2), // len,
@@ -412,15 +433,17 @@ func TestWriteCompoundList(t *testing.T) {
uint8(RefKind), list2.Hash().String(), uint8(ListKind), uint8(NumberKind), uint64(1), uint8(NumberKind), Number(3), uint64(3),
},
newList(newListMetaSequence(1, []metaTuple{
newMetaTuple(NewRef(list1), orderedKeyFromInt(1), 1, list1),
newMetaTuple(NewRef(list2), orderedKeyFromInt(3), 3, list2),
newMetaTuple(NewRef(list1), orderedKeyFromInt(1), 1),
newMetaTuple(NewRef(list2), orderedKeyFromInt(3), 3),
}, nil)),
)
}
func TestWriteCompoundSet(t *testing.T) {
set1 := newSet(newSetLeafSequence(nil, Number(0), Number(1)))
set2 := newSet(newSetLeafSequence(nil, Number(2), Number(3), Number(4)))
vrw := newTestValueStore()
set1 := newSet(newSetLeafSequence(vrw, Number(0), Number(1)))
set2 := newSet(newSetLeafSequence(vrw, Number(2), Number(3), Number(4)))
assertEncoding(t,
[]interface{}{
@@ -429,16 +452,18 @@ func TestWriteCompoundSet(t *testing.T) {
uint8(RefKind), set2.Hash().String(), uint8(SetKind), uint8(NumberKind), uint64(1), uint8(NumberKind), Number(4), uint64(3),
},
newSet(newSetMetaSequence(1, []metaTuple{
newMetaTuple(NewRef(set1), orderedKeyFromInt(1), 2, set1),
newMetaTuple(NewRef(set2), orderedKeyFromInt(4), 3, set2),
}, nil)),
newMetaTuple(NewRef(set1), orderedKeyFromInt(1), 2),
newMetaTuple(NewRef(set2), orderedKeyFromInt(4), 3),
}, vrw)),
)
}
func TestWriteCompoundSetOfBlobs(t *testing.T) {
vrw := newTestValueStore()
// Blobs are interesting because unlike the numbers used in TestWriteCompondSet, refs are sorted by their hashes, not their value.
newBlobOfInt := func(i int) Blob {
return NewBlob(strings.NewReader(strconv.Itoa(i)))
return NewBlob(vrw, strings.NewReader(strconv.Itoa(i)))
}
blob0 := newBlobOfInt(0)
@@ -447,8 +472,8 @@ func TestWriteCompoundSetOfBlobs(t *testing.T) {
blob3 := newBlobOfInt(3)
blob4 := newBlobOfInt(4)
set1 := newSet(newSetLeafSequence(nil, blob0, blob1))
set2 := newSet(newSetLeafSequence(nil, blob2, blob3, blob4))
set1 := newSet(newSetLeafSequence(vrw, blob0, blob1))
set2 := newSet(newSetLeafSequence(vrw, blob2, blob3, blob4))
assertEncoding(t,
[]interface{}{
@@ -458,20 +483,22 @@ func TestWriteCompoundSetOfBlobs(t *testing.T) {
uint8(RefKind), set2.Hash().String(), uint8(SetKind), uint8(BlobKind), uint64(1), uint8(RefKind), blob4.Hash().String(), uint8(BoolKind), uint64(0), uint64(3),
},
newSet(newSetMetaSequence(1, []metaTuple{
newMetaTuple(NewRef(set1), newOrderedKey(blob1), 2, set1),
newMetaTuple(NewRef(set2), newOrderedKey(blob4), 3, set2),
}, nil)),
newMetaTuple(NewRef(set1), newOrderedKey(blob1), 2),
newMetaTuple(NewRef(set2), newOrderedKey(blob4), 3),
}, vrw)),
)
}
func TestWriteListOfUnion(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
// Note that the order of members in a union is determined based on a hash computation; the particular ordering of Number, Bool, String was determined empirically. This must not change unless deliberately and explicitly revving the persistent format.
[]interface{}{
uint8(ListKind), uint64(0),
uint64(4) /* len */, uint8(StringKind), "0", uint8(NumberKind), Number(1), uint8(StringKind), "2", uint8(BoolKind), true,
},
NewList(
NewList(vrw,
String("0"),
Number(1),
String("2"),
@@ -481,16 +508,20 @@ func TestWriteListOfUnion(t *testing.T) {
}
func TestWriteListOfStruct(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(ListKind), uint64(0), uint64(1), /* len */
uint8(StructKind), "S", uint64(1) /* len */, "x", uint8(NumberKind), Number(42),
},
NewList(NewStruct("S", StructData{"x": Number(42)})),
NewList(vrw, NewStruct("S", StructData{"x": Number(42)})),
)
}
func TestWriteListOfUnionWithType(t *testing.T) {
vrw := newTestValueStore()
structType := MakeStructType("S", StructField{"x", NumberType, false})
assertEncoding(t,
@@ -501,7 +532,7 @@ func TestWriteListOfUnionWithType(t *testing.T) {
uint8(TypeKind), uint8(TypeKind),
uint8(TypeKind), uint8(StructKind), "S", uint64(1) /* len */, "x", uint8(NumberKind), false,
},
NewList(
NewList(vrw,
Bool(true),
NumberType,
TypeType,
@@ -522,16 +553,20 @@ func TestWriteRef(t *testing.T) {
}
func TestWriteListOfTypes(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(ListKind), uint64(0), uint64(2), /* len */
uint8(TypeKind), uint8(BoolKind), uint8(TypeKind), uint8(StringKind),
},
NewList(BoolType, StringType),
NewList(vrw, BoolType, StringType),
)
}
func nomsTestWriteRecursiveStruct(t *testing.T) {
vrw := newTestValueStore()
// struct A6 {
// cs: List<A6>
// v: Number
@@ -543,37 +578,41 @@ func nomsTestWriteRecursiveStruct(t *testing.T) {
uint8(NumberKind), Number(42),
},
// {v: 42, cs: [{v: 555, cs: []}]}
NewStruct("A6", StructData{"cs": NewList(), "v": Number(42)}),
NewStruct("A6", StructData{"cs": NewList(vrw), "v": Number(42)}),
)
}
func TestWriteUnionList(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(ListKind), uint64(0), uint64(3), /* len */
uint8(NumberKind), Number(23), uint8(StringKind), "hi", uint8(NumberKind), Number(42),
},
NewList(Number(23), String("hi"), Number(42)),
NewList(vrw, Number(23), String("hi"), Number(42)),
)
}
func TestWriteEmptyUnionList(t *testing.T) {
vrw := newTestValueStore()
assertEncoding(t,
[]interface{}{
uint8(ListKind), uint64(0), uint64(0), /* len */
},
NewList(),
NewList(vrw),
)
}
type bogusType int
func (bg bogusType) Value(vrw ValueReadWriter) Value { return bg }
func (bg bogusType) Equals(other Value) bool { return false }
func (bg bogusType) Less(other Value) bool { return false }
func (bg bogusType) Hash() hash.Hash { return hash.Hash{} }
func (bg bogusType) WalkValues(cb ValueCallback) {}
func (bg bogusType) WalkRefs(cb RefCallback) {}
func (bg bogusType) Value() Value { return bg }
func (bg bogusType) Equals(other Value) bool { return false }
func (bg bogusType) Less(other Value) bool { return false }
func (bg bogusType) Hash() hash.Hash { return hash.Hash{} }
func (bg bogusType) WalkValues(cb ValueCallback) {}
func (bg bogusType) WalkRefs(cb RefCallback) {}
func (bg bogusType) Kind() NomsKind {
return CycleKind
}
go/types/equals_test.go
+15 -14
View File
@@ -13,6 +13,7 @@ import (
func TestValueEquals(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
values := []func() Value{
func() Value { return Bool(false) },
@@ -24,29 +25,29 @@ func TestValueEquals(t *testing.T) {
func() Value { return String("hi") },
func() Value { return String("bye") },
func() Value {
return NewBlob(&bytes.Buffer{})
return NewBlob(vrw, &bytes.Buffer{})
},
func() Value {
return NewBlob(bytes.NewBufferString("hi"))
return NewBlob(vrw, bytes.NewBufferString("hi"))
},
func() Value {
return NewBlob(bytes.NewBufferString("bye"))
return NewBlob(vrw, bytes.NewBufferString("bye"))
},
func() Value {
b1 := NewBlob(bytes.NewBufferString("hi"))
b2 := NewBlob(bytes.NewBufferString("bye"))
b1 := NewBlob(vrw, bytes.NewBufferString("hi"))
b2 := NewBlob(vrw, bytes.NewBufferString("bye"))
return newBlob(newBlobMetaSequence(1, []metaTuple{
newMetaTuple(NewRef(b1), orderedKeyFromInt(2), 2, b1),
newMetaTuple(NewRef(b2), orderedKeyFromInt(5), 5, b2),
newMetaTuple(NewRef(b1), orderedKeyFromInt(2), 2),
newMetaTuple(NewRef(b2), orderedKeyFromInt(5), 5),
}, nil))
},
func() Value { return NewList() },
func() Value { return NewList(String("foo")) },
func() Value { return NewList(String("bar")) },
func() Value { return NewMap() },
func() Value { return NewMap(String("a"), String("a")) },
func() Value { return NewSet() },
func() Value { return NewSet(String("hi")) },
func() Value { return NewList(vrw) },
func() Value { return NewList(vrw, String("foo")) },
func() Value { return NewList(vrw, String("bar")) },
func() Value { return NewMap(vrw) },
func() Value { return NewMap(vrw, String("a"), String("a")) },
func() Value { return NewSet(vrw) },
func() Value { return NewSet(vrw, String("hi")) },
func() Value { return BoolType },
func() Value { return StringType },
go/types/get_hash_test.go
+12 -12
View File
@@ -33,32 +33,32 @@ func TestEnsureHash(t *testing.T) {
getHashOverride = nil
}()
bl := newBlob(newBlobLeafSequence(nil, []byte("hi")))
cb := newBlob(newBlobMetaSequence(1, []metaTuple{{Ref{}, newOrderedKey(Number(2)), 2, bl}}, vs))
bl := newBlob(newBlobLeafSequence(vs, []byte("hi")))
cb := newBlob(newBlobMetaSequence(1, []metaTuple{{vs.WriteValue(bl), newOrderedKey(Number(2)), 2}}, vs))
ll := newList(newListLeafSequence(nil, String("foo")))
cl := newList(newMetaSequence(ListKind, 1, []metaTuple{{Ref{}, newOrderedKey(Number(1)), 1, ll}}, vs))
ll := newList(newListLeafSequence(vs, String("foo")))
cl := newList(newMetaSequence(ListKind, 1, []metaTuple{{vs.WriteValue(ll), newOrderedKey(Number(1)), 1}}, vs))
newStringOrderedKey := func(s string) orderedKey {
return newOrderedKey(String(s))
}
ml := newMap(newMapLeafSequence(nil, mapEntry{String("foo"), String("bar")}))
cm := newMap(newMetaSequence(MapKind, 1, []metaTuple{{Ref{}, newStringOrderedKey("foo"), 1, ml}}, vs))
ml := newMap(newMapLeafSequence(vs, mapEntry{String("foo"), String("bar")}))
cm := newMap(newMetaSequence(MapKind, 1, []metaTuple{{vs.WriteValue(ml), newStringOrderedKey("foo"), 1}}, vs))
sl := newSet(newSetLeafSequence(nil, String("foo")))
cps := newSet(newMetaSequence(SetKind, 1, []metaTuple{{Ref{}, newStringOrderedKey("foo"), 1, sl}}, vs))
sl := newSet(newSetLeafSequence(vs, String("foo")))
cps := newSet(newMetaSequence(SetKind, 1, []metaTuple{{vs.WriteValue(sl), newStringOrderedKey("foo"), 1}}, vs))
count = byte(1)
values := []Value{
newBlob(newBlobLeafSequence(nil, []byte{})),
newBlob(newBlobLeafSequence(vs, []byte{})),
cb,
newList(newListLeafSequence(nil, String("bar"))),
newList(newListLeafSequence(vs, String("bar"))),
cl,
cm,
newMap(newMapLeafSequence(nil)),
newMap(newMapLeafSequence(vs)),
cps,
newSet(newSetLeafSequence(nil)),
newSet(newSetLeafSequence(vs)),
}
for i := 0; i < 2; i++ {
for j, v := range values {
go/types/graph_builder.go
+3 -3
View File
@@ -199,11 +199,11 @@ func (b *GraphBuilder) pushNewKeyOnStack(key Value, kind NomsKind) {
var ch *sequenceChunker
switch kind {
case MapKind:
ch = newEmptyMapSequenceChunker(b.vrw, b.vrw)
ch = newEmptyMapSequenceChunker(b.vrw)
case SetKind:
ch = newEmptySetSequenceChunker(b.vrw, b.vrw)
ch = newEmptySetSequenceChunker(b.vrw)
case ListKind:
ch = newEmptyListSequenceChunker(b.vrw, b.vrw)
ch = newEmptyListSequenceChunker(b.vrw)
default:
panic("bad 'kind' value in GraphBuilder, newElem()")
}
go/types/graph_builder_test.go
+8 -8
View File
@@ -77,7 +77,7 @@ func TestGraphBuilderEncodeDecodeAsKey(t *testing.T) {
} else {
expectedRes = append(expectedRes, nil)
}
bs = encodeGraphKey(bs, k, vrw)
bs = encodeGraphKey(bs, k)
}
res := ValueSlice{}
for pos := 0; pos < numKeys; pos++ {
@@ -107,7 +107,7 @@ func TestGraphBuilderEncodeDecodeAsValue(t *testing.T) {
bs := byteBuf[:0]
numKeys := len(keys)
for _, k := range keys {
bs = encodeGraphValue(bs, k, vrw)
bs = encodeGraphValue(bs, k)
}
res := ValueSlice{}
for pos := 0; pos < numKeys; pos++ {
@@ -157,7 +157,7 @@ func TestGraphBuilderMapSetGraphOp(t *testing.T) {
// |avgSize| parameters. The graph will contain nested maps with a
// depth == |levels|, each map will contain |avgSize| elements of different
// types.
func createTestMap(levels, avgSize int, valGen func() Value) Map {
func createTestMap(vrw ValueReadWriter, levels, avgSize int, valGen func() Value) Map {
sampleSize := func() int {
size := (int(rand.Int31()) % avgSize) + (avgSize / 2)
if size < 2 {
@@ -177,11 +177,11 @@ func createTestMap(levels, avgSize int, valGen func() Value) Map {
if numElems%2 != 0 {
numElems -= 1
}
return NewMap(elems[:numElems]...)
return NewMap(vrw, elems[:numElems]...)
case 1:
return NewSet(elems...)
return NewSet(vrw, elems...)
case 2:
return NewList(elems...)
return NewList(vrw, elems...)
}
panic("unreachable")
}
@@ -203,7 +203,7 @@ func createTestMap(levels, avgSize int, valGen func() Value) Map {
}
}
}
return NewMap(kvs...)
return NewMap(vrw, kvs...)
}
return genChildren(0)
@@ -247,7 +247,7 @@ func TestGraphBuilderNestedMapSet(t *testing.T) {
vs := newTestValueStore()
defer vs.Close()
expected := createTestMap(3, 4, valGen)
expected := createTestMap(vs, 3, 4, valGen)
b := NewGraphBuilder(vs, MapKind)
ops := []testGraphOp{}
go/types/incremental_test.go
+11 -11
View File
@@ -12,18 +12,18 @@ import (
"github.com/attic-labs/testify/assert"
)
var (
testVals = []Value{
func getTestVals(vrw ValueReadWriter) []Value {
return []Value{
Bool(true),
Number(1),
String("hi"),
NewBlob(bytes.NewReader([]byte("hi"))),
NewBlob(vrw, bytes.NewReader([]byte("hi"))),
// compoundBlob
NewSet(String("hi")),
NewList(String("hi")),
NewMap(String("hi"), String("hi")),
NewSet(vrw, String("hi")),
NewList(vrw, String("hi")),
NewMap(vrw, String("hi"), String("hi")),
}
)
}
func isEncodedOutOfLine(v Value) int {
switch v.(type) {
@@ -39,7 +39,7 @@ func TestIncrementalLoadList(t *testing.T) {
cs := ts.NewView()
vs := NewValueStore(cs)
expected := NewList(testVals...)
expected := NewList(vs, getTestVals(vs)...)
hash := vs.WriteValue(expected).TargetHash()
vs.Commit(vs.Root(), vs.Root())
@@ -69,7 +69,7 @@ func SkipTestIncrementalLoadSet(t *testing.T) {
cs := ts.NewView()
vs := NewValueStore(cs)
expected := NewSet(testVals...)
expected := NewSet(vs, getTestVals(vs)...)
ref := vs.WriteValue(expected).TargetHash()
actualVar := vs.ReadValue(ref)
@@ -90,7 +90,7 @@ func SkipTestIncrementalLoadMap(t *testing.T) {
cs := ts.NewView()
vs := NewValueStore(cs)
expected := NewMap(testVals...)
expected := NewMap(vs, getTestVals(vs)...)
ref := vs.WriteValue(expected).TargetHash()
actualVar := vs.ReadValue(ref)
@@ -115,7 +115,7 @@ func SkipTestIncrementalAddRef(t *testing.T) {
expectedItem := Number(42)
ref := vs.WriteValue(expectedItem)
expected := NewList(ref)
expected := NewList(vs, ref)
ref = vs.WriteValue(expected)
actualVar := vs.ReadValue(ref.TargetHash())
go/types/indexed_sequences.go
+8 -16
View File
@@ -9,12 +9,12 @@ import (
"github.com/attic-labs/noms/go/hash"
)
func newListMetaSequence(level uint64, tuples []metaTuple, vr ValueReader) metaSequence {
return newMetaSequence(ListKind, level, tuples, vr)
func newListMetaSequence(level uint64, tuples []metaTuple, vrw ValueReadWriter) metaSequence {
return newMetaSequence(ListKind, level, tuples, vrw)
}
func newBlobMetaSequence(level uint64, tuples []metaTuple, vr ValueReader) metaSequence {
return newMetaSequence(BlobKind, level, tuples, vr)
func newBlobMetaSequence(level uint64, tuples []metaTuple, vrw ValueReadWriter) metaSequence {
return newMetaSequence(BlobKind, level, tuples, vrw)
}
// advanceCursorToOffset advances the cursor as close as possible to idx
@@ -51,7 +51,7 @@ func advanceCursorToOffset(cur *sequenceCursor, idx uint64) uint64 {
// If |sink| is not nil, chunks will be eagerly written as they're created. Otherwise they are
// written when the root is written.
func newIndexedMetaSequenceChunkFn(kind NomsKind, source ValueReader) makeChunkFn {
func newIndexedMetaSequenceChunkFn(kind NomsKind, source ValueReadWriter) makeChunkFn {
return func(level uint64, items []sequenceItem) (Collection, orderedKey, uint64) {
tuples := make([]metaTuple, len(items))
numLeaves := uint64(0)
@@ -84,7 +84,8 @@ func orderedKeyFromSum(msd []metaTuple) orderedKey {
// loads the set of leaf nodes which contain the items [startIdx -> endIdx).
// Returns the set of nodes and the offset within the first sequence which corresponds to |startIdx|.
func loadLeafNodes(cols []Collection, startIdx, endIdx uint64) ([]Collection, uint64) {
vr := cols[0].sequence().valueReader()
vrw := cols[0].sequence().valueReadWriter()
d.PanicIfTrue(vrw == nil)
if cols[0].sequence().isLeaf() {
for _, c := range cols {
@@ -121,9 +122,6 @@ func loadLeafNodes(cols []Collection, startIdx, endIdx uint64) ([]Collection, ui
hs := hash.HashSet{}
for _, mt := range childTuples {
if mt.child != nil {
continue
}
hs.Insert(mt.ref.TargetHash())
}
@@ -132,8 +130,7 @@ func loadLeafNodes(cols []Collection, startIdx, endIdx uint64) ([]Collection, ui
if len(hs) > 0 {
valueChan := make(chan Value, len(hs))
go func() {
d.PanicIfTrue(vr == nil)
vr.ReadManyValues(hs, valueChan)
vrw.ReadManyValues(hs, valueChan)
close(valueChan)
}()
for value := range valueChan {
@@ -143,11 +140,6 @@ func loadLeafNodes(cols []Collection, startIdx, endIdx uint64) ([]Collection, ui
childCols := make([]Collection, len(childTuples))
for i, mt := range childTuples {
if mt.child != nil {
childCols[i] = mt.child.(Collection)
continue
}
childCols[i] = fetched[mt.ref.TargetHash()]
}
go/types/leaf_sequence.go
+3 -3
View File
@@ -5,7 +5,7 @@
package types
type leafSequence struct {
vr ValueReader
vrw ValueReadWriter
length int
kind NomsKind
}
@@ -18,8 +18,8 @@ func (seq leafSequence) numLeaves() uint64 {
return uint64(seq.length)
}
func (seq leafSequence) valueReader() ValueReader {
return seq.vr
func (seq leafSequence) valueReadWriter() ValueReadWriter {
return seq.vrw
}
func (seq leafSequence) getChildSequence(idx int) sequence {
go/types/list.go
+12 -12
View File
@@ -29,8 +29,8 @@ func newList(seq sequence) List {
// NewList creates a new List where the type is computed from the elements in the list, populated
// with values, chunking if and when needed.
func NewList(values ...Value) List {
ch := newEmptyListSequenceChunker(nil, nil)
func NewList(vrw ValueReadWriter, values ...Value) List {
ch := newEmptyListSequenceChunker(vrw)
for _, v := range values {
ch.Append(v)
}
@@ -44,7 +44,7 @@ func NewStreamingList(vrw ValueReadWriter, values <-chan Value) <-chan List {
out := make(chan List, 1)
go func() {
defer close(out)
ch := newEmptyListSequenceChunker(vrw, vrw)
ch := newEmptyListSequenceChunker(vrw)
for v := range values {
ch.Append(v)
}
@@ -78,7 +78,7 @@ func (l List) hashPointer() *hash.Hash {
}
// Value interface
func (l List) Value(vrw ValueReadWriter) Value {
func (l List) Value() Value {
return l
}
@@ -147,8 +147,8 @@ func (l List) Map(mf MapFunc) []interface{} {
// to visit the rightmost prolly tree chunks of this List, and the leftmost
// prolly tree chunks of other, so it's efficient.
func (l List) Concat(other List) List {
seq := concat(l.seq, other.seq, func(cur *sequenceCursor, vr ValueReader) *sequenceChunker {
return l.newChunker(cur, vr)
seq := concat(l.seq, other.seq, func(cur *sequenceCursor, vrw ValueReadWriter) *sequenceChunker {
return l.newChunker(cur, vrw)
})
return newList(seq)
}
@@ -225,13 +225,13 @@ func (l List) DiffWithLimit(last List, changes chan<- Splice, closeChan <-chan s
indexedSequenceDiff(last.seq, 0, l.seq, 0, changes, closeChan, maxSpliceMatrixSize)
}
func (l List) newChunker(cur *sequenceCursor, vr ValueReader) *sequenceChunker {
return newSequenceChunker(cur, 0, vr, nil, makeListLeafChunkFn(vr), newIndexedMetaSequenceChunkFn(ListKind, vr), hashValueBytes)
func (l List) newChunker(cur *sequenceCursor, vrw ValueReadWriter) *sequenceChunker {
return newSequenceChunker(cur, 0, vrw, makeListLeafChunkFn(vrw), newIndexedMetaSequenceChunkFn(ListKind, vrw), hashValueBytes)
}
// If |sink| is not nil, chunks will be eagerly written as they're created. Otherwise they are
// written when the root is written.
func makeListLeafChunkFn(vr ValueReader) makeChunkFn {
func makeListLeafChunkFn(vrw ValueReadWriter) makeChunkFn {
return func(level uint64, items []sequenceItem) (Collection, orderedKey, uint64) {
d.PanicIfFalse(level == 0)
values := make([]Value, len(items))
@@ -240,11 +240,11 @@ func makeListLeafChunkFn(vr ValueReader) makeChunkFn {
values[i] = v.(Value)
}
list := newList(newListLeafSequence(vr, values...))
list := newList(newListLeafSequence(vrw, values...))
return list, orderedKeyFromInt(len(values)), uint64(len(values))
}
}
func newEmptyListSequenceChunker(vr ValueReader, vw ValueWriter) *sequenceChunker {
return newEmptySequenceChunker(vr, vw, makeListLeafChunkFn(vr), newIndexedMetaSequenceChunkFn(ListKind, vr), hashValueBytes)
func newEmptyListSequenceChunker(vrw ValueReadWriter) *sequenceChunker {
return newEmptySequenceChunker(vrw, makeListLeafChunkFn(vrw), newIndexedMetaSequenceChunkFn(ListKind, vrw), hashValueBytes)
}
go/types/list_editor.go
+6 -10
View File
@@ -23,21 +23,17 @@ func (le *ListEditor) Kind() NomsKind {
return ListKind
}
func (le *ListEditor) Value(vrw ValueReadWriter) Value {
return le.List(vrw)
func (le *ListEditor) Value() Value {
return le.List()
}
func (le *ListEditor) List(vrw ValueReadWriter) List {
func (le *ListEditor) List() List {
if le.edits == nil {
return le.l // no edits
}
seq := le.l.sequence()
vr := seq.valueReader()
if vrw != nil {
vr = vrw
}
vrw := seq.valueReadWriter()
cursChan := make(chan chan *sequenceCursor)
spliceChan := make(chan chan listEdit)
@@ -68,7 +64,7 @@ func (le *ListEditor) List(vrw ValueReadWriter) List {
idx := i
wg.Add(1)
go func() {
edit.inserted[idx] = v.Value(vrw)
edit.inserted[idx] = v.Value()
wg.Done()
}()
}
@@ -93,7 +89,7 @@ func (le *ListEditor) List(vrw ValueReadWriter) List {
sp := <-<-spliceChan
if ch == nil {
ch = newSequenceChunker(cur, 0, vr, vrw, makeListLeafChunkFn(vr), newIndexedMetaSequenceChunkFn(ListKind, vr), hashValueBytes)
ch = newSequenceChunker(cur, 0, vrw, makeListLeafChunkFn(vrw), newIndexedMetaSequenceChunkFn(ListKind, vrw), hashValueBytes)
} else {
ch.advanceTo(cur)
}
go/types/list_editor_test.go
+59 -51
View File
@@ -11,16 +11,16 @@ import (
"github.com/attic-labs/testify/assert"
)
func listOfInts(vals ...int) List {
func listOfInts(vrw ValueReadWriter, vals ...int) List {
vs := ValueSlice{}
for _, v := range vals {
vs = append(vs, Number(v))
}
return NewList(vs...)
return NewList(vrw, vs...)
}
func testEditor(vals ...int) *ListEditor {
return NewListEditor(listOfInts(vals...))
func testEditor(vrw ValueReadWriter, vals ...int) *ListEditor {
return NewListEditor(listOfInts(vrw, vals...))
}
func edit(le *ListEditor, idx, remove int, insert ...int) {
@@ -31,7 +31,7 @@ func edit(le *ListEditor, idx, remove int, insert ...int) {
le.Splice(uint64(idx), uint64(remove), vals...)
}
func assertState(t *testing.T, le *ListEditor, expectItems []int, expectEditCount int) {
func assertState(t *testing.T, vrw ValueReadWriter, le *ListEditor, expectItems []int, expectEditCount int) {
assert.Equal(t, uint64(len(expectItems)), le.Len())
for i, v := range expectItems {
@@ -45,124 +45,130 @@ func assertState(t *testing.T, le *ListEditor, expectItems []int, expectEditCoun
assert.Equal(t, expectEditCount, actualEditCount)
assert.True(t, listOfInts(expectItems...).Equals(le.List(nil)))
assert.True(t, listOfInts(vrw, expectItems...).Equals(le.List()))
}
func TestListEditorBasic(t *testing.T) {
vrw := newTestValueStore()
t.Run("remove a few", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 2, 2)
assertState(t, le, []int{0, 1, 4, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 4, 5}, 1)
})
t.Run("insert a few", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 2, 0, 9, 8, 7)
assertState(t, le, []int{0, 1, 9, 8, 7, 2, 3, 4, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 9, 8, 7, 2, 3, 4, 5}, 1)
})
t.Run("remove 2, insert 3", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 2, 2, 9, 8, 7)
assertState(t, le, []int{0, 1, 9, 8, 7, 4, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 9, 8, 7, 4, 5}, 1)
})
t.Run("insert 2 twice", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 2, 0, 9, 10)
assertState(t, le, []int{0, 1, 9, 10, 2, 3, 4, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 9, 10, 2, 3, 4, 5}, 1)
edit(le, 7, 0, 8, 9)
assertState(t, le, []int{0, 1, 9, 10, 2, 3, 4, 8, 9, 5}, 2)
assertState(t, vrw, le, []int{0, 1, 9, 10, 2, 3, 4, 8, 9, 5}, 2)
})
t.Run("remove 2 twice", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5, 6, 7)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5, 6, 7)
edit(le, 5, 2)
assertState(t, le, []int{0, 1, 2, 3, 4, 7}, 1)
assertState(t, vrw, le, []int{0, 1, 2, 3, 4, 7}, 1)
edit(le, 1, 2)
assertState(t, le, []int{0, 3, 4, 7}, 2)
assertState(t, vrw, le, []int{0, 3, 4, 7}, 2)
})
}
func TestCollapseSplices(t *testing.T) {
vrw := newTestValueStore()
t.Run("left adjacent", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5, 6, 7)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5, 6, 7)
edit(le, 4, 3)
assertState(t, le, []int{0, 1, 2, 3, 7}, 1)
assertState(t, vrw, le, []int{0, 1, 2, 3, 7}, 1)
edit(le, 1, 3)
assertState(t, le, []int{0, 7}, 1)
assertState(t, vrw, le, []int{0, 7}, 1)
})
t.Run("left adjacent 2", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5, 6, 7)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5, 6, 7)
edit(le, 4, 3, 0, 0)
assertState(t, le, []int{0, 1, 2, 3, 0, 0, 7}, 1)
assertState(t, vrw, le, []int{0, 1, 2, 3, 0, 0, 7}, 1)
edit(le, 1, 3, 5, 5)
assertState(t, le, []int{0, 5, 5, 0, 0, 7}, 1)
assertState(t, vrw, le, []int{0, 5, 5, 0, 0, 7}, 1)
})
t.Run("left consume", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5, 6, 7)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5, 6, 7)
edit(le, 2, 4)
assertState(t, le, []int{0, 1, 6, 7}, 1)
assertState(t, vrw, le, []int{0, 1, 6, 7}, 1)
edit(le, 1, 2)
assertState(t, le, []int{0, 7}, 1)
assertState(t, vrw, le, []int{0, 7}, 1)
})
t.Run("left overlap ", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 3, 2, 7, 8, 9)
assertState(t, le, []int{0, 1, 2, 7, 8, 9, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 2, 7, 8, 9, 5}, 1)
edit(le, 0, 4)
assertState(t, le, []int{8, 9, 5}, 1)
assertState(t, vrw, le, []int{8, 9, 5}, 1)
})
t.Run("undo 1", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 2, 3)
assertState(t, le, []int{0, 1, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 5}, 1)
edit(le, 2, 0, 2, 3, 4)
assertState(t, le, []int{0, 1, 2, 3, 4, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 2, 3, 4, 5}, 1)
})
t.Run("undo 2", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 2, 0, 9, 8, 7)
assertState(t, le, []int{0, 1, 9, 8, 7, 2, 3, 4, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 9, 8, 7, 2, 3, 4, 5}, 1)
edit(le, 2, 3)
assertState(t, le, []int{0, 1, 2, 3, 4, 5}, 0)
assertState(t, vrw, le, []int{0, 1, 2, 3, 4, 5}, 0)
})
t.Run("splice middile of splice", func(t *testing.T) {
le := testEditor(0, 1)
le := testEditor(vrw, 0, 1)
edit(le, 1, 0, 9, 8, 7, 6)
assertState(t, le, []int{0, 9, 8, 7, 6, 1}, 1)
assertState(t, vrw, le, []int{0, 9, 8, 7, 6, 1}, 1)
edit(le, 2, 2)
assertState(t, le, []int{0, 9, 6, 1}, 1)
assertState(t, vrw, le, []int{0, 9, 6, 1}, 1)
})
}
func TestFuzzFails(t *testing.T) {
vrw := newTestValueStore()
t.Run("Case 1", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24)
edit(le, 23, 0, 0, 3, 2)
assertState(t, le, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 0, 3, 2, 23, 24}, 1)
assertState(t, vrw, le, []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 0, 3, 2, 23, 24}, 1)
edit(le, 5, 15, 1, 2, 9, 8)
assertState(t, le, []int{0, 1, 2, 3, 4, 1, 2, 9, 8, 20, 21, 22, 0, 3, 2, 23, 24}, 2)
assertState(t, vrw, le, []int{0, 1, 2, 3, 4, 1, 2, 9, 8, 20, 21, 22, 0, 3, 2, 23, 24}, 2)
edit(le, 4, 7, 7)
assertState(t, le, []int{0, 1, 2, 3, 7, 22, 0, 3, 2, 23, 24}, 2)
assertState(t, vrw, le, []int{0, 1, 2, 3, 7, 22, 0, 3, 2, 23, 24}, 2)
})
t.Run("Case 2", func(t *testing.T) {
le := testEditor(0, 1, 2, 3, 4, 5)
le := testEditor(vrw, 0, 1, 2, 3, 4, 5)
edit(le, 5, 0, 1, 7, 5, 3, 13, 17)
assertState(t, le, []int{0, 1, 2, 3, 4, 1, 7, 5, 3, 13, 17, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 2, 3, 4, 1, 7, 5, 3, 13, 17, 5}, 1)
edit(le, 2, 2, 16, 5, 12, 5, 15, 0, 15, 15, 7)
assertState(t, le, []int{0, 1, 16, 5, 12, 5, 15, 0, 15, 15, 7, 4, 1, 7, 5, 3, 13, 17, 5}, 2)
assertState(t, vrw, le, []int{0, 1, 16, 5, 12, 5, 15, 0, 15, 15, 7, 4, 1, 7, 5, 3, 13, 17, 5}, 2)
edit(le, 8, 5, 4, 13)
assertState(t, le, []int{0, 1, 16, 5, 12, 5, 15, 0, 4, 13, 7, 5, 3, 13, 17, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 16, 5, 12, 5, 15, 0, 4, 13, 7, 5, 3, 13, 17, 5}, 1)
edit(le, 6, 2, 8, 2, 6, 3, 14, 6)
assertState(t, le, []int{0, 1, 16, 5, 12, 5, 8, 2, 6, 3, 14, 6, 4, 13, 7, 5, 3, 13, 17, 5}, 1)
assertState(t, vrw, le, []int{0, 1, 16, 5, 12, 5, 8, 2, 6, 3, 14, 6, 4, 13, 7, 5, 3, 13, 17, 5}, 1)
})
}
@@ -182,6 +188,8 @@ func TestListSpliceFuzzer(t *testing.T) {
maxInsertCount := uint64(50)
maxInt := uint64(100)
vrw := newTestValueStore()
r := rand.New(rand.NewSource(0))
nextRandInt := func(from, to uint64) uint64 {
@@ -201,15 +209,15 @@ func TestListSpliceFuzzer(t *testing.T) {
for i := 0; i < rounds; i++ {
tl := newTestList(startCount)
le := tl.toList().Edit()
le := tl.toList(vrw).Edit()
for j := 0; j < splices; j++ {
idx, removed, insert := nextRandomSplice(len(tl))
tl = tl.Splice(int(idx), int(removed), insert...)
le.Splice(idx, removed, AsValuables(insert)...)
}
expect := tl.toList()
actual := le.List(nil)
expect := tl.toList(vrw)
actual := le.List()
assert.True(t, expect.Equals(actual))
}
}
go/types/list_iterator_test.go
+2 -1
View File
@@ -12,9 +12,10 @@ import (
func TestListIterator(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
numbers := append(generateNumbersAsValues(10), Number(20), Number(25))
l := NewList(numbers...)
l := NewList(vrw, numbers...)
i := l.Iterator()
vs := iterToSlice(i)
assert.True(vs.Equals(numbers), "Expected: %v != actual: %v", numbers, vs)
go/types/list_leaf_sequence.go
+5 -2
View File
@@ -4,13 +4,16 @@
package types
import "github.com/attic-labs/noms/go/d"
type listLeafSequence struct {
leafSequence
values []Value
}
func newListLeafSequence(vr ValueReader, v ...Value) sequence {
return listLeafSequence{leafSequence{vr, len(v), ListKind}, v}
func newListLeafSequence(vrw ValueReadWriter, v ...Value) sequence {
d.PanicIfTrue(vrw == nil)
return listLeafSequence{leafSequence{vrw, len(v), ListKind}, v}
}
// sequence interface
go/types/list_test.go
+199 -135
View File
@@ -55,8 +55,8 @@ func (tl testList) Diff(last testList) []Splice {
func(i uint64, j uint64) bool { return last[i] == tl[j] })
}
func (tl testList) toList() List {
return NewList(tl...)
func (tl testList) toList(vrw ValueReadWriter) List {
return NewList(vrw, tl...)
}
func newTestList(length int) testList {
@@ -71,8 +71,8 @@ func newTestListFromList(list List) testList {
return tl
}
func validateList(t *testing.T, l List, values ValueSlice) {
assert.True(t, l.Equals(NewList(values...)))
func validateList(t *testing.T, vrw ValueReadWriter, l List, values ValueSlice) {
assert.True(t, l.Equals(NewList(vrw, values...)))
out := ValueSlice{}
l.IterAll(func(v Value, idx uint64) {
out = append(out, v)
@@ -86,10 +86,12 @@ type listTestSuite struct {
}
func newListTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int, expectAppendChunkDiff int) *listTestSuite {
vrw := newTestValueStore()
length := 1 << size
elems := newTestList(length)
tr := MakeListType(NumberType)
list := NewList(elems...)
list := NewList(vrw, elems...)
return &listTestSuite{
collectionTestSuite: collectionTestSuite{
col: list,
@@ -110,13 +112,13 @@ func newListTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff in
dup := make([]Value, length+1)
dup[0] = Number(0)
copy(dup[1:], elems)
return NewList(dup...)
return NewList(vrw, dup...)
},
appendOne: func() Collection {
dup := make([]Value, length+1)
copy(dup, elems)
dup[len(dup)-1] = Number(0)
return NewList(dup...)
return NewList(vrw, dup...)
},
},
elems: elems,
@@ -169,49 +171,53 @@ func TestListSuite8K(t *testing.T) {
func TestListInsert(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
tl := newTestList(1024)
list := tl.toList()
list := tl.toList(vrw)
for i := 0; i < len(tl); i += 16 {
tl = tl.Insert(i, Number(i))
list = list.Edit().Insert(uint64(i), Number(i)).List(nil)
list = list.Edit().Insert(uint64(i), Number(i)).List()
}
assert.True(tl.toList().Equals(list))
assert.True(tl.toList(vrw).Equals(list))
}
func TestListRemove(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
tl := newTestList(1024)
list := tl.toList()
list := tl.toList(vrw)
for i := len(tl) - 16; i >= 0; i -= 16 {
tl = tl.Remove(i, i+4)
list = list.Edit().Remove(uint64(i), uint64(i+4)).List(nil)
list = list.Edit().Remove(uint64(i), uint64(i+4)).List()
}
assert.True(tl.toList().Equals(list))
assert.True(tl.toList(vrw).Equals(list))
}
func TestListRemoveAt(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
l0 := NewList()
l0 = l0.Edit().Append(Bool(false), Bool(true)).List(nil)
l1 := l0.Edit().RemoveAt(1).List(nil)
assert.True(NewList(Bool(false)).Equals(l1))
l1 = l1.Edit().RemoveAt(0).List(nil)
assert.True(NewList().Equals(l1))
l0 := NewList(vrw)
l0 = l0.Edit().Append(Bool(false), Bool(true)).List()
l1 := l0.Edit().RemoveAt(1).List()
assert.True(NewList(vrw, Bool(false)).Equals(l1))
l1 = l1.Edit().RemoveAt(0).List()
assert.True(NewList(vrw).Equals(l1))
assert.Panics(func() {
l1.Edit().RemoveAt(0).List(nil)
l1.Edit().RemoveAt(0).List()
})
}
@@ -267,7 +273,7 @@ func TestStreamingListCreation(t *testing.T) {
vs := newTestValueStore()
simpleList := getTestList()
cl := NewList(simpleList...)
cl := NewList(vs, simpleList...)
valueChan := make(chan Value)
listChan := NewStreamingList(vs, valueChan)
for _, v := range simpleList {
@@ -286,13 +292,15 @@ func TestListAppend(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
if testing.Short() {
t.Skip("Skipping test in short mode.")
}
assert := assert.New(t)
newList := func(items testList) List {
return NewList(items...)
return NewList(vrw, items...)
}
listToSimple := func(cl List) (simple testList) {
@@ -303,11 +311,11 @@ func TestListAppend(t *testing.T) {
}
cl := newList(getTestList())
cl2 := cl.Edit().Append(Number(42)).List(nil)
cl3 := cl2.Edit().Append(Number(43)).List(nil)
cl4 := cl3.Edit().Append(getTestList().AsValuables()...).List(nil)
cl5 := cl4.Edit().Append(Number(44), Number(45)).List(nil)
cl6 := cl5.Edit().Append(getTestList().AsValuables()...).List(nil)
cl2 := cl.Edit().Append(Number(42)).List()
cl3 := cl2.Edit().Append(Number(43)).List()
cl4 := cl3.Edit().Append(getTestList().AsValuables()...).List()
cl5 := cl4.Edit().Append(Number(44), Number(45)).List()
cl6 := cl5.Edit().Append(getTestList().AsValuables()...).List()
expected := getTestList()
assert.Equal(expected, listToSimple(cl))
@@ -348,12 +356,14 @@ func TestListValidateInsertAscending(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
values := generateNumbersAsValues(1000)
s := NewList()
s := NewList(vrw)
for i, v := range values {
s = s.Edit().Insert(uint64(i), v).List(nil)
validateList(t, s, values[0:i+1])
s = s.Edit().Insert(uint64(i), v).List()
validateList(t, vrw, s, values[0:i+1])
}
}
@@ -365,14 +375,16 @@ func TestListValidateInsertAtZero(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
values := generateNumbersAsValues(1000)
s := NewList()
s := NewList(vrw)
count := len(values)
for count > 0 {
count--
v := values[count]
s = s.Edit().Insert(uint64(0), v).List(nil)
validateList(t, s, values[count:])
s = s.Edit().Insert(uint64(0), v).List()
validateList(t, vrw, s, values[count:])
}
}
@@ -382,14 +394,16 @@ func TestListInsertNothing(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
cl := getTestList().toList()
vrw := newTestValueStore()
assert.True(cl.Equals(cl.Edit().Insert(0).List(nil)))
cl := getTestList().toList(vrw)
assert.True(cl.Equals(cl.Edit().Insert(0).List()))
for i := uint64(1); i < getTestListLen(); i *= 2 {
assert.True(cl.Equals(cl.Edit().Insert(i).List(nil)))
assert.True(cl.Equals(cl.Edit().Insert(i).List()))
}
assert.True(cl.Equals(cl.Edit().Insert(cl.Len() - 1).List(nil)))
assert.True(cl.Equals(cl.Edit().Insert(cl.Len()).List(nil)))
assert.True(cl.Equals(cl.Edit().Insert(cl.Len() - 1).List()))
assert.True(cl.Equals(cl.Edit().Insert(cl.Len()).List()))
}
func TestListInsertStart(t *testing.T) {
@@ -401,42 +415,44 @@ func TestListInsertStart(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
cl := getTestList().toList()
cl2 := cl.Edit().Insert(0, Number(42)).List(nil)
cl3 := cl2.Edit().Insert(0, Number(43)).List(nil)
cl4 := cl3.Edit().Insert(0, getTestList().AsValuables()...).List(nil)
cl5 := cl4.Edit().Insert(0, Number(44), Number(45)).List(nil)
cl6 := cl5.Edit().Insert(0, getTestList().AsValuables()...).List(nil)
vrw := newTestValueStore()
cl := getTestList().toList(vrw)
cl2 := cl.Edit().Insert(0, Number(42)).List()
cl3 := cl2.Edit().Insert(0, Number(43)).List()
cl4 := cl3.Edit().Insert(0, getTestList().AsValuables()...).List()
cl5 := cl4.Edit().Insert(0, Number(44), Number(45)).List()
cl6 := cl5.Edit().Insert(0, getTestList().AsValuables()...).List()
expected := getTestList()
assert.Equal(expected, testListFromNomsList(cl))
assert.Equal(getTestListLen(), cl.Len())
assert.True(expected.toList().Equals(cl))
assert.True(expected.toList(vrw).Equals(cl))
expected = expected.Insert(0, Number(42))
assert.Equal(expected, testListFromNomsList(cl2))
assert.Equal(getTestListLen()+1, cl2.Len())
assert.True(expected.toList().Equals(cl2))
assert.True(expected.toList(vrw).Equals(cl2))
expected = expected.Insert(0, Number(43))
assert.Equal(expected, testListFromNomsList(cl3))
assert.Equal(getTestListLen()+2, cl3.Len())
assert.True(expected.toList().Equals(cl3))
assert.True(expected.toList(vrw).Equals(cl3))
expected = expected.Insert(0, getTestList()...)
assert.Equal(expected, testListFromNomsList(cl4))
assert.Equal(2*getTestListLen()+2, cl4.Len())
assert.True(expected.toList().Equals(cl4))
assert.True(expected.toList(vrw).Equals(cl4))
expected = expected.Insert(0, Number(44), Number(45))
assert.Equal(expected, testListFromNomsList(cl5))
assert.Equal(2*getTestListLen()+4, cl5.Len())
assert.True(expected.toList().Equals(cl5))
assert.True(expected.toList(vrw).Equals(cl5))
expected = expected.Insert(0, getTestList()...)
assert.Equal(expected, testListFromNomsList(cl6))
assert.Equal(3*getTestListLen()+4, cl6.Len())
assert.True(expected.toList().Equals(cl6))
assert.True(expected.toList(vrw).Equals(cl6))
}
func TestListInsertMiddle(t *testing.T) {
@@ -448,48 +464,50 @@ func TestListInsertMiddle(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
cl := getTestList().toList()
cl2 := cl.Edit().Insert(100, Number(42)).List(nil)
cl3 := cl2.Edit().Insert(200, Number(43)).List(nil)
cl4 := cl3.Edit().Insert(300, getTestList().AsValuables()...).List(nil)
cl5 := cl4.Edit().Insert(400, Number(44), Number(45)).List(nil)
cl6 := cl5.Edit().Insert(500, getTestList().AsValuables()...).List(nil)
cl7 := cl6.Edit().Insert(600, Number(100)).List(nil)
vrw := newTestValueStore()
cl := getTestList().toList(vrw)
cl2 := cl.Edit().Insert(100, Number(42)).List()
cl3 := cl2.Edit().Insert(200, Number(43)).List()
cl4 := cl3.Edit().Insert(300, getTestList().AsValuables()...).List()
cl5 := cl4.Edit().Insert(400, Number(44), Number(45)).List()
cl6 := cl5.Edit().Insert(500, getTestList().AsValuables()...).List()
cl7 := cl6.Edit().Insert(600, Number(100)).List()
expected := getTestList()
assert.Equal(expected, testListFromNomsList(cl))
assert.Equal(getTestListLen(), cl.Len())
assert.True(expected.toList().Equals(cl))
assert.True(expected.toList(vrw).Equals(cl))
expected = expected.Insert(100, Number(42))
assert.Equal(expected, testListFromNomsList(cl2))
assert.Equal(getTestListLen()+1, cl2.Len())
assert.True(expected.toList().Equals(cl2))
assert.True(expected.toList(vrw).Equals(cl2))
expected = expected.Insert(200, Number(43))
assert.Equal(expected, testListFromNomsList(cl3))
assert.Equal(getTestListLen()+2, cl3.Len())
assert.True(expected.toList().Equals(cl3))
assert.True(expected.toList(vrw).Equals(cl3))
expected = expected.Insert(300, getTestList()...)
assert.Equal(expected, testListFromNomsList(cl4))
assert.Equal(2*getTestListLen()+2, cl4.Len())
assert.True(expected.toList().Equals(cl4))
assert.True(expected.toList(vrw).Equals(cl4))
expected = expected.Insert(400, Number(44), Number(45))
assert.Equal(expected, testListFromNomsList(cl5))
assert.Equal(2*getTestListLen()+4, cl5.Len())
assert.True(expected.toList().Equals(cl5))
assert.True(expected.toList(vrw).Equals(cl5))
expected = expected.Insert(500, getTestList()...)
assert.Equal(expected, testListFromNomsList(cl6))
assert.Equal(3*getTestListLen()+4, cl6.Len())
assert.True(expected.toList().Equals(cl6))
assert.True(expected.toList(vrw).Equals(cl6))
expected = expected.Insert(600, Number(100))
assert.Equal(expected, testListFromNomsList(cl7))
assert.Equal(3*getTestListLen()+5, cl7.Len())
assert.True(expected.toList().Equals(cl7))
assert.True(expected.toList(vrw).Equals(cl7))
}
func TestListInsertRanges(t *testing.T) {
@@ -501,14 +519,16 @@ func TestListInsertRanges(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
testList := getTestList()
whole := testList.toList()
whole := testList.toList(vrw)
// Compare list equality. Increment by 256 (16^2) because each iteration requires building a new list, which is slow.
for incr, i := 256, 0; i < len(testList)-incr; i += incr {
for window := 1; window <= incr; window *= 16 {
testListPart := testList.Remove(i, i+window)
actual := testListPart.toList().Edit().Insert(uint64(i), testList[i:i+window].AsValuables()...).List(nil)
actual := testListPart.toList(vrw).Edit().Insert(uint64(i), testList[i:i+window].AsValuables()...).List()
assert.Equal(whole.Len(), actual.Len())
assert.True(whole.Equals(actual))
}
@@ -516,7 +536,7 @@ func TestListInsertRanges(t *testing.T) {
// Compare list length, which doesn't require building a new list every iteration, so the increment can be smaller.
for incr, i := 10, 0; i < len(testList); i += incr {
assert.Equal(len(testList)+incr, int(whole.Edit().Insert(uint64(i), testList[0:incr].AsValuables()...).List(nil).Len()))
assert.Equal(len(testList)+incr, int(whole.Edit().Insert(uint64(i), testList[0:incr].AsValuables()...).List().Len()))
}
}
@@ -526,14 +546,16 @@ func TestListRemoveNothing(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
cl := getTestList().toList()
vrw := newTestValueStore()
assert.True(cl.Equals(cl.Edit().Remove(0, 0).List(nil)))
cl := getTestList().toList(vrw)
assert.True(cl.Equals(cl.Edit().Remove(0, 0).List()))
for i := uint64(1); i < getTestListLen(); i *= 2 {
assert.True(cl.Equals(cl.Edit().Remove(i, i).List(nil)))
assert.True(cl.Equals(cl.Edit().Remove(i, i).List()))
}
assert.True(cl.Equals(cl.Edit().Remove(cl.Len()-1, cl.Len()-1).List(nil)))
assert.True(cl.Equals(cl.Edit().Remove(cl.Len(), cl.Len()).List(nil)))
assert.True(cl.Equals(cl.Edit().Remove(cl.Len()-1, cl.Len()-1).List()))
assert.True(cl.Equals(cl.Edit().Remove(cl.Len(), cl.Len()).List()))
}
func TestListRemoveEverything(t *testing.T) {
@@ -542,9 +564,11 @@ func TestListRemoveEverything(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
cl := getTestList().toList().Edit().Remove(0, getTestListLen()).List(nil)
vrw := newTestValueStore()
assert.True(NewList().Equals(cl))
cl := getTestList().toList(vrw).Edit().Remove(0, getTestListLen()).List()
assert.True(NewList(vrw).Equals(cl))
assert.Equal(0, int(cl.Len()))
}
@@ -557,24 +581,26 @@ func TestListRemoveAtMiddle(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
cl := getTestList().toList()
cl2 := cl.Edit().RemoveAt(100).List(nil)
cl3 := cl2.Edit().RemoveAt(200).List(nil)
vrw := newTestValueStore()
cl := getTestList().toList(vrw)
cl2 := cl.Edit().RemoveAt(100).List()
cl3 := cl2.Edit().RemoveAt(200).List()
expected := getTestList()
assert.Equal(expected, testListFromNomsList(cl))
assert.Equal(getTestListLen(), cl.Len())
assert.True(expected.toList().Equals(cl))
assert.True(expected.toList(vrw).Equals(cl))
expected = expected.RemoveAt(100)
assert.Equal(expected, testListFromNomsList(cl2))
assert.Equal(getTestListLen()-1, cl2.Len())
assert.True(expected.toList().Equals(cl2))
assert.True(expected.toList(vrw).Equals(cl2))
expected = expected.RemoveAt(200)
assert.Equal(expected, testListFromNomsList(cl3))
assert.Equal(getTestListLen()-2, cl3.Len())
assert.True(expected.toList().Equals(cl3))
assert.True(expected.toList(vrw).Equals(cl3))
}
func TestListRemoveRanges(t *testing.T) {
@@ -586,15 +612,17 @@ func TestListRemoveRanges(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
testList := getTestList()
whole := testList.toList()
whole := testList.toList(vrw)
// Compare list equality. Increment by 256 (16^2) because each iteration requires building a new list, which is slow.
for incr, i := 256, 0; i < len(testList)-incr; i += incr {
for window := 1; window <= incr; window *= 16 {
testListPart := testList.Remove(i, i+window)
expected := testListPart.toList()
actual := whole.Edit().Remove(uint64(i), uint64(i+window)).List(nil)
expected := testListPart.toList(vrw)
actual := whole.Edit().Remove(uint64(i), uint64(i+window)).List()
assert.Equal(expected.Len(), actual.Len())
assert.True(expected.Equals(actual))
}
@@ -602,7 +630,7 @@ func TestListRemoveRanges(t *testing.T) {
// Compare list length, which doesn't require building a new list every iteration, so the increment can be smaller.
for incr, i := 10, 0; i < len(testList)-incr; i += incr {
assert.Equal(len(testList)-incr, int(whole.Edit().Remove(uint64(i), uint64(i+incr)).List(nil).Len()))
assert.Equal(len(testList)-incr, int(whole.Edit().Remove(uint64(i), uint64(i+incr)).List().Len()))
}
}
@@ -615,12 +643,14 @@ func TestListRemoveAtEnd(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
tl := getTestListWithLen(testListSize / 10)
cl := tl.toList()
cl := tl.toList(vrw)
for i := len(tl) - 1; i >= 0; i-- {
cl = cl.Edit().Remove(uint64(i), uint64(i+1)).List(nil)
expect := tl[0:i].toList()
cl = cl.Edit().Remove(uint64(i), uint64(i+1)).List()
expect := tl[0:i].toList(vrw)
assert.True(expect.Equals(cl))
}
}
@@ -634,15 +664,17 @@ func TestListSet(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
testList := getTestList()
cl := testList.toList()
cl := testList.toList(vrw)
testIdx := func(idx int, testEquality bool) {
newVal := Number(-1) // Test values are never < 0
cl2 := cl.Edit().Set(uint64(idx), newVal).List(nil)
cl2 := cl.Edit().Set(uint64(idx), newVal).List()
assert.False(cl.Equals(cl2))
if testEquality {
assert.True(testList.Set(idx, newVal).toList().Equals(cl2))
assert.True(testList.Set(idx, newVal).toList(vrw).Equals(cl2))
}
}
@@ -659,17 +691,20 @@ func TestListSet(t *testing.T) {
}
func TestListFirstNNumbers(t *testing.T) {
vrw := newTestValueStore()
nums := generateNumbersAsValues(testListSize)
NewList(nums...)
NewList(vrw, nums...)
}
func TestListRefOfStructFirstNNumbers(t *testing.T) {
if testing.Short() {
t.Skip("Skipping test in short mode.")
}
vrw := newTestValueStore()
nums := generateNumbersAsRefOfStructs(testListSize)
NewList(nums...)
NewList(vrw, nums...)
}
func TestListModifyAfterRead(t *testing.T) {
@@ -680,15 +715,15 @@ func TestListModifyAfterRead(t *testing.T) {
vs := newTestValueStore()
list := getTestList().toList()
list := getTestList().toList(vs)
// Drop chunk values.
list = vs.ReadValue(vs.WriteValue(list).TargetHash()).(List)
// Modify/query. Once upon a time this would crash.
llen := list.Len()
z := list.Get(0)
list = list.Edit().RemoveAt(0).List(nil)
list = list.Edit().RemoveAt(0).List()
assert.Equal(llen-1, list.Len())
list = list.Edit().Append(z).List(nil)
list = list.Edit().Append(z).List()
assert.Equal(llen, list.Len())
}
@@ -720,10 +755,12 @@ func TestListDiffIdentical(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums := generateNumbersAsValues(5)
l1 := NewList(nums...)
l2 := NewList(nums...)
l1 := NewList(vrw, nums...)
l2 := NewList(vrw, nums...)
diff1 := accumulateDiffSplices(l1, l2)
diff2 := accumulateDiffSplices(l2, l1)
@@ -736,10 +773,12 @@ func TestListDiffVersusEmpty(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := generateNumbersAsValues(5)
l1 := NewList(nums1...)
l2 := NewList()
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw)
diff1 := accumulateDiffSplices(l1, l2)
diff2 := accumulateDiffSplices(l2, l1)
@@ -758,11 +797,13 @@ func TestListDiffReverse(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := generateNumbersAsValues(5000)
nums2 := reverseValues(nums1)
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff1 := accumulateDiffSplices(l1, l2)
diff2 := accumulateDiffSplices(l2, l1)
@@ -782,11 +823,13 @@ func TestListDiffReverseWithLargerLimit(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := generateNumbersAsValues(5000)
nums2 := reverseValues(nums1)
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff1 := accumulateDiffSplicesWithLimit(l1, l2, 27e6)
diff2 := accumulateDiffSplicesWithLimit(l2, l1, 27e6)
@@ -808,14 +851,16 @@ func TestListDiffRemove5x100(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := generateNumbersAsValues(5000)
nums2 := generateNumbersAsValues(5000)
for count := 5; count > 0; count-- {
nums2 = spliceValues(nums2, (count-1)*1000, 100)
}
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff1 := accumulateDiffSplices(l1, l2)
diff2 := accumulateDiffSplices(l2, l1)
@@ -839,14 +884,16 @@ func TestListDiffAdd5x5(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := generateNumbersAsValues(5000)
nums2 := generateNumbersAsValues(5000)
for count := 5; count > 0; count-- {
nums2 = spliceValues(nums2, (count-1)*1000, 0, Number(0), Number(1), Number(2), Number(3), Number(4))
}
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff1 := accumulateDiffSplices(l1, l2)
diff2 := accumulateDiffSplices(l2, l1)
@@ -870,6 +917,8 @@ func TestListDiffReplaceReverse5x100(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := generateNumbersAsValues(5000)
nums2 := generateNumbersAsValues(5000)
@@ -877,8 +926,8 @@ func TestListDiffReplaceReverse5x100(t *testing.T) {
out := reverseValues(nums2[(count-1)*1000 : (count-1)*1000+100])
nums2 = spliceValues(nums2, (count-1)*1000, 100, out...)
}
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff := accumulateDiffSplices(l2, l1)
diffExpected := []Splice{
@@ -900,11 +949,13 @@ func TestListDiffString1(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := []Value{String("one"), String("two"), String("three")}
nums2 := []Value{String("one"), String("two"), String("three")}
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff := accumulateDiffSplices(l2, l1)
assert.Equal(0, len(diff))
@@ -914,11 +965,13 @@ func TestListDiffString2(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := []Value{String("one"), String("two"), String("three")}
nums2 := []Value{String("one"), String("two"), String("three"), String("four")}
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff := accumulateDiffSplices(l2, l1)
diffExpected := []Splice{
@@ -931,11 +984,13 @@ func TestListDiffString3(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
nums1 := []Value{String("one"), String("two"), String("three")}
nums2 := []Value{String("one"), String("two"), String("four")}
l1 := NewList(nums1...)
l2 := NewList(nums2...)
l1 := NewList(vrw, nums1...)
l2 := NewList(vrw, nums2...)
diff := accumulateDiffSplices(l2, l1)
diffExpected := []Splice{
@@ -956,7 +1011,7 @@ func TestListDiffLargeWithSameMiddle(t *testing.T) {
cs1 := storage.NewView()
vs1 := NewValueStore(cs1)
nums1 := generateNumbersAsValues(4000)
l1 := NewList(nums1...)
l1 := NewList(vs1, nums1...)
hash1 := vs1.WriteValue(l1).TargetHash()
vs1.Commit(vs1.Root(), vs1.Root())
@@ -965,7 +1020,7 @@ func TestListDiffLargeWithSameMiddle(t *testing.T) {
cs2 := storage.NewView()
vs2 := NewValueStore(cs2)
nums2 := generateNumbersAsValuesFromToBy(5, 3550, 1)
l2 := NewList(nums2...)
l2 := NewList(vs2, nums2...)
hash2 := vs2.WriteValue(l2).TargetHash()
vs2.Commit(vs1.Root(), vs1.Root())
refList2 := vs2.ReadValue(hash2).(List)
@@ -991,18 +1046,20 @@ func TestListDiffLargeWithSameMiddle(t *testing.T) {
func TestListDiffAllValuesInSequenceRemoved(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
newSequenceMetaTuple := func(vs ...Value) metaTuple {
seq := newListLeafSequence(nil, vs...)
seq := newListLeafSequence(vrw, vs...)
list := newList(seq)
return newMetaTuple(NewRef(list), orderedKeyFromInt(len(vs)), uint64(len(vs)), list)
return newMetaTuple(vrw.WriteValue(list), orderedKeyFromInt(len(vs)), uint64(len(vs)))
}
m1 := newSequenceMetaTuple(Number(1), Number(2), Number(3))
m2 := newSequenceMetaTuple(Number(4), Number(5), Number(6), Number(7), Number(8))
m3 := newSequenceMetaTuple(Number(9), Number(10), Number(11), Number(12), Number(13), Number(14), Number(15))
l1 := newList(newListMetaSequence(1, []metaTuple{m1, m3}, nil)) // [1, 2, 3][9, 10, 11, 12, 13, 14, 15]
l2 := newList(newListMetaSequence(1, []metaTuple{m1, m2, m3}, nil)) // [1, 2, 3][4, 5, 6, 7, 8][9, 10, 11, 12, 13, 14, 15]
l1 := newList(newListMetaSequence(1, []metaTuple{m1, m3}, vrw)) // [1, 2, 3][9, 10, 11, 12, 13, 14, 15]
l2 := newList(newListMetaSequence(1, []metaTuple{m1, m2, m3}, vrw)) // [1, 2, 3][4, 5, 6, 7, 8][9, 10, 11, 12, 13, 14, 15]
diff := accumulateDiffSplices(l2, l1)
@@ -1018,21 +1075,22 @@ func TestListTypeAfterMutations(t *testing.T) {
defer normalProductionChunks()
assert := assert.New(t)
vrw := newTestValueStore()
test := func(n int, c interface{}) {
values := generateNumbersAsValues(n)
l := NewList(values...)
l := NewList(vrw, values...)
assert.Equal(l.Len(), uint64(n))
assert.IsType(c, l.sequence())
assert.True(TypeOf(l).Equals(MakeListType(NumberType)))
l = l.Edit().Append(String("a")).List(nil)
l = l.Edit().Append(String("a")).List()
assert.Equal(l.Len(), uint64(n+1))
assert.IsType(c, l.sequence())
assert.True(TypeOf(l).Equals(MakeListType(MakeUnionType(NumberType, StringType))))
l = l.Edit().Splice(l.Len()-1, 1).List(nil)
l = l.Edit().Splice(l.Len()-1, 1).List()
assert.Equal(l.Len(), uint64(n))
assert.IsType(c, l.sequence())
assert.True(TypeOf(l).Equals(MakeListType(NumberType)))
@@ -1054,12 +1112,12 @@ func TestListRemoveLastWhenNotLoaded(t *testing.T) {
}
tl := newTestList(1024)
nl := tl.toList()
nl := tl.toList(vs)
for len(tl) > 0 {
tl = tl[:len(tl)-1]
nl = reload(nl.Edit().RemoveAt(uint64(len(tl))).List(nil))
assert.True(tl.toList().Equals(nl))
nl = reload(nl.Edit().RemoveAt(uint64(len(tl))).List())
assert.True(tl.toList(vs).Equals(nl))
}
}
@@ -1086,11 +1144,11 @@ func TestListConcat(t *testing.T) {
listSlice[i] = Number(r.Intn(size))
}
list := listSlice.toList()
list := listSlice.toList(vs)
for i := from; i < to; i += by {
fst := reload(vs, listSlice[:i].toList())
snd := reload(vs, listSlice[i:].toList())
fst := reload(vs, listSlice[:i].toList(vs))
snd := reload(vs, listSlice[i:].toList(vs))
actual := fst.Concat(snd)
assert.True(list.Equals(actual),
"fail at %d/%d (with expected length %d, actual %d)", i, size, list.Len(), actual.Len())
@@ -1113,6 +1171,8 @@ func TestListConcat(t *testing.T) {
func TestListConcatDifferentTypes(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
fst := generateNumbersAsValuesFromToBy(0, testListSize/2, 1)
snd := generateNumbersAsStructsFromToBy(testListSize/2, testListSize, 1)
@@ -1120,13 +1180,15 @@ func TestListConcatDifferentTypes(t *testing.T) {
whole = append(whole, fst...)
whole = append(whole, snd...)
concat := NewList(fst...).Concat(NewList(snd...))
assert.True(NewList(whole...).Equals(concat))
concat := NewList(vrw, fst...).Concat(NewList(vrw, snd...))
assert.True(NewList(vrw, whole...).Equals(concat))
}
func TestListWithStructShouldHaveOptionalFields(t *testing.T) {
assert := assert.New(t)
list := NewList(
vrw := newTestValueStore()
list := NewList(vrw,
NewStruct("Foo", StructData{
"a": Number(1),
}),
@@ -1144,10 +1206,12 @@ func TestListWithStructShouldHaveOptionalFields(t *testing.T) {
}
func TestListWithNil(t *testing.T) {
vrw := newTestValueStore()
assert.Panics(t, func() {
NewList(nil)
NewList(vrw, nil)
})
assert.Panics(t, func() {
NewList(Number(42), nil)
NewList(vrw, Number(42), nil)
})
}
go/types/map.go
+9 -8
View File
@@ -27,9 +27,9 @@ func mapHashValueBytes(item sequenceItem, rv *rollingValueHasher) {
hashValueBytes(entry.value, rv)
}
func NewMap(kv ...Value) Map {
func NewMap(vrw ValueReadWriter, kv ...Value) Map {
entries := buildMapData(kv)
ch := newEmptyMapSequenceChunker(nil, nil)
ch := newEmptyMapSequenceChunker(vrw)
for _, entry := range entries {
ch.Append(entry)
@@ -46,6 +46,7 @@ func NewMap(kv ...Value) Map {
// closed by the caller, a finished Map will be sent to the output channel. See
// graph_builder.go for building collections with values that are not in order.
func NewStreamingMap(vrw ValueReadWriter, kvs <-chan Value) <-chan Map {
d.PanicIfTrue(vrw == nil)
return newStreamingMap(vrw, kvs, func(vrw ValueReadWriter, kvs <-chan Value, outChan chan<- Map) {
go readMapInput(vrw, kvs, outChan)
})
@@ -61,7 +62,7 @@ func newStreamingMap(vrw ValueReadWriter, kvs <-chan Value, readFunc streamingMa
func readMapInput(vrw ValueReadWriter, kvs <-chan Value, outChan chan<- Map) {
defer close(outChan)
ch := newEmptyMapSequenceChunker(vrw, vrw)
ch := newEmptyMapSequenceChunker(vrw)
var lastK Value
nextIsKey := true
var k Value
@@ -127,7 +128,7 @@ func (m Map) hashPointer() *hash.Hash {
}
// Value interface
func (m Map) Value(vrw ValueReadWriter) Value {
func (m Map) Value() Value {
return m
}
@@ -317,7 +318,7 @@ func buildMapData(values []Value) mapEntrySlice {
// If |vw| is not nil, chunks will be eagerly written as they're created. Otherwise they are
// written when the root is written.
func makeMapLeafChunkFn(vr ValueReader) makeChunkFn {
func makeMapLeafChunkFn(vrw ValueReadWriter) makeChunkFn {
return func(level uint64, items []sequenceItem) (Collection, orderedKey, uint64) {
d.PanicIfFalse(level == 0)
mapData := make([]mapEntry, len(items), len(items))
@@ -330,7 +331,7 @@ func makeMapLeafChunkFn(vr ValueReader) makeChunkFn {
mapData[i] = entry
}
m := newMap(newMapLeafSequence(vr, mapData...))
m := newMap(newMapLeafSequence(vrw, mapData...))
var key orderedKey
if len(mapData) > 0 {
key = newOrderedKey(mapData[len(mapData)-1].key)
@@ -339,6 +340,6 @@ func makeMapLeafChunkFn(vr ValueReader) makeChunkFn {
}
}
func newEmptyMapSequenceChunker(vr ValueReader, vw ValueWriter) *sequenceChunker {
return newEmptySequenceChunker(vr, vw, makeMapLeafChunkFn(vr), newOrderedMetaSequenceChunkFn(MapKind, vr), mapHashValueBytes)
func newEmptyMapSequenceChunker(vrw ValueReadWriter) *sequenceChunker {
return newEmptySequenceChunker(vrw, makeMapLeafChunkFn(vrw), newOrderedMetaSequenceChunkFn(MapKind, vrw), mapHashValueBytes)
}
go/types/map_editor.go
+6 -9
View File
@@ -30,19 +30,16 @@ func (me *MapEditor) Kind() NomsKind {
return MapKind
}
func (me *MapEditor) Value(vrw ValueReadWriter) Value {
return me.Map(vrw)
func (me *MapEditor) Value() Value {
return me.Map()
}
func (me *MapEditor) Map(vrw ValueReadWriter) Map {
func (me *MapEditor) Map() Map {
if len(me.edits) == 0 {
return me.m // no edits
}
vr := me.m.sequence().valueReader()
if vrw != nil {
vr = vrw
}
vrw := me.m.sequence().valueReadWriter()
me.normalize()
@@ -79,7 +76,7 @@ func (me *MapEditor) Map(vrw ValueReadWriter) Map {
}
go func() {
sv := edit.value.Value(vrw)
sv := edit.value.Value()
if e, ok := sv.(Emptyable); ok {
if e.Empty() {
sv = nil
@@ -116,7 +113,7 @@ func (me *MapEditor) Map(vrw ValueReadWriter) Map {
}
if ch == nil {
ch = newSequenceChunker(cur, 0, vr, vrw, makeMapLeafChunkFn(vr), newOrderedMetaSequenceChunkFn(MapKind, vr), mapHashValueBytes)
ch = newSequenceChunker(cur, 0, vrw, makeMapLeafChunkFn(vrw), newOrderedMetaSequenceChunkFn(MapKind, vrw), mapHashValueBytes)
} else {
ch.advanceTo(cur)
}
go/types/map_iterator_test.go
+4 -2
View File
@@ -13,12 +13,14 @@ import (
func TestMapIterator(t *testing.T) {
assert := assert.New(t)
me := NewMap().Edit()
vrw := newTestValueStore()
me := NewMap(vrw).Edit()
for i := 0; i < 5; i++ {
me.Set(String(string(byte(65+i))), Number(i))
}
m := me.Map(nil)
m := me.Map()
test := func(it MapIterator, start int, msg string) {
for i := start; i < 5; i++ {
k, v := it.Next()
go/types/map_leaf_sequence.go
+5 -2
View File
@@ -4,6 +4,8 @@
package types
import "github.com/attic-labs/noms/go/d"
type mapLeafSequence struct {
leafSequence
data []mapEntry // sorted by entry.key.Hash()
@@ -33,8 +35,9 @@ func (mes mapEntrySlice) Equals(other mapEntrySlice) bool {
return true
}
func newMapLeafSequence(vr ValueReader, data ...mapEntry) orderedSequence {
return mapLeafSequence{leafSequence{vr, len(data), MapKind}, data}
func newMapLeafSequence(vrw ValueReadWriter, data ...mapEntry) orderedSequence {
d.PanicIfTrue(vrw == nil)
return mapLeafSequence{leafSequence{vrw, len(data), MapKind}, data}
}
// sequence interface
go/types/map_test.go
+236 -156
View File
@@ -87,12 +87,12 @@ func (tm testMap) Diff(last testMap) (added []Value, removed []Value, modified [
return
}
func (tm testMap) toMap() Map {
func (tm testMap) toMap(vrw ValueReadWriter) Map {
keyvals := []Value{}
for _, entry := range tm.entries {
keyvals = append(keyvals, entry.key, entry.value)
}
return NewMap(keyvals...)
return NewMap(vrw, keyvals...)
}
func toValuable(vs ValueSlice) []Valuable {
@@ -158,9 +158,9 @@ func newRandomTestMap(length int, gen genValueFn) testMap {
return testMap{entries, gen(mask + 1)}
}
func validateMap(t *testing.T, m Map, entries mapEntrySlice) {
func validateMap(t *testing.T, vrw ValueReadWriter, m Map, entries mapEntrySlice) {
tm := testMap{entries: entries}
assert.True(t, m.Equals(tm.toMap()))
assert.True(t, m.Equals(tm.toMap(vrw)))
out := mapEntrySlice{}
m.IterAll(func(k Value, v Value) {
@@ -176,11 +176,13 @@ type mapTestSuite struct {
}
func newMapTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int, expectAppendChunkDiff int, gen genValueFn) *mapTestSuite {
vrw := newTestValueStore()
length := 1 << size
keyType := TypeOf(gen(0))
elems := newSortedTestMap(length, gen)
tr := MakeMapType(keyType, NumberType)
tmap := NewMap(elems.FlattenAll()...)
tmap := NewMap(vrw, elems.FlattenAll()...)
return &mapTestSuite{
collectionTestSuite: collectionTestSuite{
col: tmap,
@@ -219,7 +221,7 @@ func newMapTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int
for _, entry := range dup {
flat = append(flat, entry.key, entry.value)
}
return NewMap(flat...)
return NewMap(vrw, flat...)
},
appendOne: func() Collection {
dup := make([]mapEntry, length+1)
@@ -229,7 +231,7 @@ func newMapTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int
for _, entry := range dup {
flat = append(flat, entry.key, entry.value)
}
return NewMap(flat...)
return NewMap(vrw, flat...)
},
},
elems: elems,
@@ -331,9 +333,9 @@ func getTestRefToNativeOrderMap(scale int, vw ValueWriter) testMap {
})
}
func getTestRefToValueOrderMap(scale int, vw ValueWriter) testMap {
func getTestRefToValueOrderMap(scale int, vrw ValueReadWriter) testMap {
return newRandomTestMap(64*scale, func(i int) Value {
return vw.WriteValue(NewSet(Number(i)))
return vrw.WriteValue(NewSet(vrw, Number(i)))
})
}
@@ -378,11 +380,13 @@ func TestMapDiff(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
testMap1 := newRandomTestMap(64*2, newNumber)
testMap2 := newRandomTestMap(64*2, newNumber)
testMapAdded, testMapRemoved, testMapModified := testMap1.Diff(testMap2)
map1 := testMap1.toMap()
map2 := testMap2.toMap()
map1 := testMap1.toMap(vrw)
map2 := testMap2.toMap(vrw)
mapDiffAdded, mapDiffRemoved, mapDiffModified := accumulateMapDiffChanges(map1, map2)
assert.Equal(t, testMapAdded, mapDiffAdded, "testMap.diff != map.diff")
@@ -405,43 +409,50 @@ func TestMapMutationReadWriteCount(t *testing.T) {
})
}
m := newRandomTestMap(20000, newLargeStruct).toMap()
every := 500
ts := &chunks.TestStorage{}
cs := ts.NewView()
vs := newValueStoreWithCacheAndPending(cs, 0, 0)
me := NewMap(vs).Edit()
for i := 0; i < 4000; i++ {
me.Set(Number(i), newLargeStruct(i))
}
m := me.Map()
r := vs.WriteValue(m)
vs.Commit(vs.Root(), vs.Root())
m = r.TargetValue(vs).(Map)
cs.Writes = 0
cs.Reads = 0
every := 100
i := 0
me := vs.ReadValue(r.TargetHash()).(Map).Edit()
m.IterAll(func(k, v Value) {
me = m.Edit()
for i := 0; i < 4000; i++ {
if i%every == 0 {
s := v.(Struct)
k := Number(i)
s := me.Get(Number(i)).(Struct)
s = s.Set("Number", Number(float64(s.Get("Number").(Number))+1))
me.Set(k, s)
}
i++
})
}
cs.Writes = 0
cs.Reads = 0
m = me.Map()
me.Map(vs)
vs.Commit(vs.Root(), vs.Root())
assert.Equal(t, uint64(3), NewRef(m).Height())
assert.Equal(t, 82, cs.Reads)
assert.Equal(t, 49, cs.Writes)
assert.Equal(t, uint64(2), NewRef(m).Height())
assert.Equal(t, 40, cs.Reads)
assert.Equal(t, 17, cs.Writes)
}
func TestMapInfiniteChunkBug(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
keyLen := chunkWindow + 1
buff := &bytes.Buffer{}
@@ -451,26 +462,30 @@ func TestMapInfiniteChunkBug(t *testing.T) {
prefix := buff.String()
me := NewMap().Edit()
me := NewMap(vrw).Edit()
for i := 0; i < 10000; i++ {
me.Set(String(prefix+fmt.Sprintf("%d", i)), Number(i))
}
me.Map(nil)
me.Map()
}
func TestNewMap(t *testing.T) {
assert := assert.New(t)
m := NewMap()
vrw := newTestValueStore()
m := NewMap(vrw)
assert.Equal(uint64(0), m.Len())
m = NewMap(String("foo1"), String("bar1"), String("foo2"), String("bar2"))
m = NewMap(vrw, String("foo1"), String("bar1"), String("foo2"), String("bar2"))
assert.Equal(uint64(2), m.Len())
assert.True(String("bar1").Equals(m.Get(String("foo1"))))
assert.True(String("bar2").Equals(m.Get(String("foo2"))))
}
func TestMapUniqueKeysString(t *testing.T) {
vrw := newTestValueStore()
assert := assert.New(t)
l := []Value{
String("hello"), String("world"),
@@ -478,13 +493,15 @@ func TestMapUniqueKeysString(t *testing.T) {
String("bar"), String("foo"),
String("hello"), String("foo"),
}
m := NewMap(l...)
m := NewMap(vrw, l...)
assert.Equal(uint64(3), m.Len())
assert.True(String("foo").Equals(m.Get(String("hello"))))
}
func TestMapUniqueKeysNumber(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
l := []Value{
Number(4), Number(1),
Number(0), Number(2),
@@ -492,7 +509,7 @@ func TestMapUniqueKeysNumber(t *testing.T) {
Number(3), Number(4),
Number(1), Number(5),
}
m := NewMap(l...)
m := NewMap(vrw, l...)
assert.Equal(uint64(4), m.Len())
assert.True(Number(5).Equals(m.Get(Number(1))))
}
@@ -509,7 +526,7 @@ func TestMapHas(t *testing.T) {
vs := newTestValueStore()
doTest := func(tm testMap) {
m := tm.toMap()
m := tm.toMap(vs)
m2 := vs.ReadValue(vs.WriteValue(m).TargetHash()).(Map)
for _, entry := range tm.entries {
k, v := entry.key, entry.value
@@ -529,11 +546,12 @@ func TestMapHas(t *testing.T) {
func TestMapHasRemove(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
me := NewMap().Edit()
me := NewMap(vrw).Edit()
bothHave := func(k Value) bool {
meHas := me.Has(k)
mHas := me.Map(nil).Has(k)
mHas := me.Map().Has(k)
assert.Equal(meHas, mHas)
return meHas
}
@@ -576,7 +594,7 @@ func TestMapHasRemove(t *testing.T) {
assert.False(bothHave(String("b")))
assert.False(bothHave(String("c")))
m := me.Map(nil)
m := me.Map()
assert.True(m.Len() == 0)
}
@@ -588,13 +606,15 @@ func TestMapRemove(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
doTest := func(incr int, tm testMap) {
whole := tm.toMap()
whole := tm.toMap(vrw)
run := func(i int) {
expected := tm.Remove(i, i+1).toMap()
actual := whole.Edit().Remove(tm.entries[i].key).Map(nil)
expected := tm.Remove(i, i+1).toMap(vrw)
actual := whole.Edit().Remove(tm.entries[i].key).Map()
assert.Equal(expected.Len(), actual.Len())
assert.True(expected.Equals(actual))
diffMapTest(assert, expected, actual, 0, 0, 0)
@@ -617,9 +637,11 @@ func TestMapRemoveNonexistentKey(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
tm := getTestNativeOrderMap(2)
original := tm.toMap()
actual := original.Edit().Remove(Number(-1)).Map(nil) // rand.Int63 returns non-negative numbers.
original := tm.toMap(vrw)
actual := original.Edit().Remove(Number(-1)).Map() // rand.Int63 returns non-negative numbers.
assert.Equal(original.Len(), actual.Len())
assert.True(original.Equals(actual))
@@ -631,12 +653,14 @@ func TestMapFirst(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
m1 := NewMap()
vrw := newTestValueStore()
m1 := NewMap(vrw)
k, v := m1.First()
assert.Nil(k)
assert.Nil(v)
m1 = m1.Edit().Set(String("foo"), String("bar")).Set(String("hot"), String("dog")).Map(nil)
m1 = m1.Edit().Set(String("foo"), String("bar")).Set(String("hot"), String("dog")).Map()
ak, av := m1.First()
var ek, ev Value
@@ -658,8 +682,10 @@ func TestMapFirst2(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
doTest := func(tm testMap) {
m := tm.toMap()
m := tm.toMap(vrw)
sort.Stable(tm.entries)
actualKey, actualValue := m.First()
assert.True(tm.entries[0].key.Equals(actualKey))
@@ -678,12 +704,14 @@ func TestMapLast(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
m1 := NewMap()
vrw := newTestValueStore()
m1 := NewMap(vrw)
k, v := m1.First()
assert.Nil(k)
assert.Nil(v)
m1 = m1.Edit().Set(String("foo"), String("bar")).Set(String("hot"), String("dog")).Map(nil)
m1 = m1.Edit().Set(String("foo"), String("bar")).Set(String("hot"), String("dog")).Map()
ak, av := m1.Last()
var ek, ev Value
@@ -705,8 +733,10 @@ func TestMapLast2(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
doTest := func(tm testMap) {
m := tm.toMap()
m := tm.toMap(vrw)
sort.Stable(tm.entries)
actualKey, actualValue := m.Last()
assert.True(tm.entries[len(tm.entries)-1].key.Equals(actualKey))
@@ -725,10 +755,12 @@ func TestMapSetGet(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
me := NewMap().Edit()
vrw := newTestValueStore()
me := NewMap(vrw).Edit()
bothAre := func(k Value) Value {
meV := me.Get(k)
mV := me.Map(nil).Get(k)
mV := me.Map().Get(k)
assert.True((meV == nil && mV == nil) || meV.(Value).Equals(mV))
return mV
}
@@ -772,23 +804,25 @@ func TestMapSetGet(t *testing.T) {
assert.Nil(bothAre(String("z")))
assert.Nil(bothAre(String("never-inserted")))
m := me.Map(nil)
m := me.Map()
assert.True(m.Len() == 0)
}
func validateMapInsertion(t *testing.T, tm testMap) {
allMe := NewMap().Edit()
incrMe := NewMap().Edit()
vrw := newTestValueStore()
allMe := NewMap(vrw).Edit()
incrMe := NewMap(vrw).Edit()
for i, entry := range tm.entries {
allMe.Set(entry.key, entry.value)
incrMe.Set(entry.key, entry.value)
m1 := allMe.Map(nil)
m2 := incrMe.Map(nil)
m1 := allMe.Map()
m2 := incrMe.Map()
validateMap(t, m1, tm.entries[0:i+1])
validateMap(t, m2, tm.entries[0:i+1])
validateMap(t, vrw, m1, tm.entries[0:i+1])
validateMap(t, vrw, m2, tm.entries[0:i+1])
incrMe = m2.Edit()
}
@@ -813,12 +847,14 @@ func TestMapSet(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
doTest := func(incr, offset int, tm testMap) {
expected := tm.toMap()
expected := tm.toMap(vrw)
run := func(from, to int) {
actual := tm.Remove(from, to).toMap().Edit().SetM(toValuable(tm.Flatten(from, to))...).Map(nil)
actual := tm.Remove(from, to).toMap(vrw).Edit().SetM(toValuable(tm.Flatten(from, to))...).Map()
assert.Equal(expected.Len(), actual.Len())
assert.True(expected.Equals(actual))
diffMapTest(assert, expected, actual, 0, 0, 0)
@@ -838,14 +874,16 @@ func TestMapSet(t *testing.T) {
func TestMapSetM(t *testing.T) {
assert := assert.New(t)
m1 := NewMap()
m2 := m1.Edit().SetM().Map(nil)
vrw := newTestValueStore()
m1 := NewMap(vrw)
m2 := m1.Edit().SetM().Map()
assert.True(m1.Equals(m2))
m3 := m2.Edit().SetM(String("foo"), String("bar"), String("hot"), String("dog")).Map(nil)
m3 := m2.Edit().SetM(String("foo"), String("bar"), String("hot"), String("dog")).Map()
assert.Equal(uint64(2), m3.Len())
assert.True(String("bar").Equals(m3.Get(String("foo"))))
assert.True(String("dog").Equals(m3.Get(String("hot"))))
m4 := m3.Edit().SetM(String("mon"), String("key")).Map(nil)
m4 := m3.Edit().SetM(String("mon"), String("key")).Map()
assert.Equal(uint64(2), m3.Len())
assert.Equal(uint64(3), m4.Len())
}
@@ -858,20 +896,22 @@ func TestMapSetExistingKeyToNewValue(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
tm := getTestNativeOrderMap(2)
original := tm.toMap()
original := tm.toMap(vrw)
expectedWorking := tm
actual := original
for i, entry := range tm.entries {
newValue := Number(int64(entry.value.(Number)) + 1)
expectedWorking = expectedWorking.SetValue(i, newValue)
actual = actual.Edit().Set(entry.key, newValue).Map(nil)
actual = actual.Edit().Set(entry.key, newValue).Map()
}
expected := expectedWorking.toMap()
expected := expectedWorking.toMap(vrw)
assert.Equal(expected.Len(), actual.Len())
assert.True(expected.Equals(actual))
assert.False(original.Equals(actual))
@@ -881,7 +921,9 @@ func TestMapSetExistingKeyToNewValue(t *testing.T) {
// BUG 98
func TestMapDuplicateSet(t *testing.T) {
assert := assert.New(t)
m1 := NewMap(Bool(true), Bool(true), Number(42), Number(42), Number(42), Number(42))
vrw := newTestValueStore()
m1 := NewMap(vrw, Bool(true), Bool(true), Number(42), Number(42), Number(42), Number(42))
assert.Equal(uint64(2), m1.Len())
}
@@ -893,10 +935,12 @@ func TestMapMaybeGet(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
doTest := func(tm testMap) {
m := tm.toMap()
m := tm.toMap(vrw)
for _, entry := range tm.entries {
v, ok := m.MaybeGet(entry.key)
if assert.True(ok, "%v should have been in the map!", entry.key) {
@@ -915,7 +959,9 @@ func TestMapMaybeGet(t *testing.T) {
func TestMapIter(t *testing.T) {
assert := assert.New(t)
m := NewMap()
vrw := newTestValueStore()
m := NewMap(vrw)
type entry struct {
key Value
@@ -942,7 +988,7 @@ func TestMapIter(t *testing.T) {
m.Iter(cb)
assert.Equal(0, len(results))
m = m.Edit().Set(String("a"), Number(0)).Set(String("b"), Number(1)).Map(nil)
m = m.Edit().Set(String("a"), Number(0)).Set(String("b"), Number(1)).Map()
m.Iter(cb)
assert.Equal(2, len(results))
assert.True(got(String("a"), Number(0)))
@@ -962,8 +1008,10 @@ func TestMapIter2(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
doTest := func(tm testMap) {
m := tm.toMap()
m := tm.toMap(vrw)
sort.Sort(tm.entries)
idx := uint64(0)
endAt := uint64(64)
@@ -990,13 +1038,15 @@ func TestMapIter2(t *testing.T) {
func TestMapAny(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
p := func(k, v Value) bool {
return k.Equals(String("foo")) && v.Equals(String("bar"))
}
assert.False(NewMap().Any(p))
assert.False(NewMap(String("foo"), String("baz")).Any(p))
assert.True(NewMap(String("foo"), String("bar")).Any(p))
assert.False(NewMap(vrw).Any(p))
assert.False(NewMap(vrw, String("foo"), String("baz")).Any(p))
assert.True(NewMap(vrw, String("foo"), String("bar")).Any(p))
}
func TestMapIterAll(t *testing.T) {
@@ -1007,10 +1057,12 @@ func TestMapIterAll(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vrw := newTestValueStore()
assert := assert.New(t)
doTest := func(tm testMap) {
m := tm.toMap()
m := tm.toMap(vrw)
sort.Sort(tm.entries)
idx := uint64(0)
@@ -1030,9 +1082,11 @@ func TestMapIterAll(t *testing.T) {
func TestMapEquals(t *testing.T) {
assert := assert.New(t)
m1 := NewMap()
vrw := newTestValueStore()
m1 := NewMap(vrw)
m2 := m1
m3 := NewMap()
m3 := NewMap(vrw)
assert.True(m1.Equals(m2))
assert.True(m2.Equals(m1))
@@ -1045,8 +1099,8 @@ func TestMapEquals(t *testing.T) {
diffMapTest(assert, m3, m1, 0, 0, 0)
diffMapTest(assert, m3, m2, 0, 0, 0)
m1 = NewMap(String("foo"), Number(0.0), String("bar"), NewList())
m2 = m2.Edit().Set(String("foo"), Number(0.0)).Set(String("bar"), NewList()).Map(nil)
m1 = NewMap(vrw, String("foo"), Number(0.0), String("bar"), NewList(vrw))
m2 = m2.Edit().Set(String("foo"), Number(0.0)).Set(String("bar"), NewList(vrw)).Map()
assert.True(m1.Equals(m2))
assert.True(m2.Equals(m1))
assert.False(m2.Equals(m3))
@@ -1062,8 +1116,10 @@ func TestMapEquals(t *testing.T) {
func TestMapNotStringKeys(t *testing.T) {
assert := assert.New(t)
b1 := NewBlob(bytes.NewBufferString("blob1"))
b2 := NewBlob(bytes.NewBufferString("blob2"))
vrw := newTestValueStore()
b1 := NewBlob(vrw, bytes.NewBufferString("blob1"))
b2 := NewBlob(vrw, bytes.NewBufferString("blob2"))
l := []Value{
Bool(true), String("true"),
Bool(false), String("false"),
@@ -1071,14 +1127,14 @@ func TestMapNotStringKeys(t *testing.T) {
Number(0), String("Number: 0"),
b1, String("blob1"),
b2, String("blob2"),
NewList(), String("empty list"),
NewList(NewList()), String("list of list"),
NewMap(), String("empty map"),
NewMap(NewMap(), NewMap()), String("map of map/map"),
NewSet(), String("empty set"),
NewSet(NewSet()), String("map of set/set"),
NewList(vrw), String("empty list"),
NewList(vrw, NewList(vrw)), String("list of list"),
NewMap(vrw), String("empty map"),
NewMap(vrw, NewMap(vrw), NewMap(vrw)), String("map of map/map"),
NewSet(vrw), String("empty set"),
NewSet(vrw, NewSet(vrw)), String("map of set/set"),
}
m1 := NewMap(l...)
m1 := NewMap(vrw, l...)
assert.Equal(uint64(12), m1.Len())
for i := 0; i < len(l); i += 2 {
assert.True(m1.Get(l[i]).Equals(l[i+1]))
@@ -1086,8 +1142,8 @@ func TestMapNotStringKeys(t *testing.T) {
assert.Nil(m1.Get(Number(42)))
}
func testMapOrder(assert *assert.Assertions, keyType, valueType *Type, tuples []Value, expectOrdering []Value) {
m := NewMap(tuples...)
func testMapOrder(assert *assert.Assertions, vrw ValueReadWriter, keyType, valueType *Type, tuples []Value, expectOrdering []Value) {
m := NewMap(vrw, tuples...)
i := 0
m.IterAll(func(key, value Value) {
assert.Equal(expectOrdering[i].Hash().String(), key.Hash().String())
@@ -1101,7 +1157,9 @@ func TestMapOrdering(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
testMapOrder(assert,
vrw := newTestValueStore()
testMapOrder(assert, vrw,
StringType, StringType,
[]Value{
String("a"), String("unused"),
@@ -1121,7 +1179,7 @@ func TestMapOrdering(t *testing.T) {
},
)
testMapOrder(assert,
testMapOrder(assert, vrw,
NumberType, StringType,
[]Value{
Number(0), String("unused"),
@@ -1141,7 +1199,7 @@ func TestMapOrdering(t *testing.T) {
},
)
testMapOrder(assert,
testMapOrder(assert, vrw,
NumberType, StringType,
[]Value{
Number(0), String("unused"),
@@ -1161,7 +1219,7 @@ func TestMapOrdering(t *testing.T) {
},
)
testMapOrder(assert,
testMapOrder(assert, vrw,
NumberType, StringType,
[]Value{
Number(0.0001), String("unused"),
@@ -1181,7 +1239,7 @@ func TestMapOrdering(t *testing.T) {
},
)
testMapOrder(assert,
testMapOrder(assert, vrw,
ValueType, StringType,
[]Value{
String("a"), String("unused"),
@@ -1201,7 +1259,7 @@ func TestMapOrdering(t *testing.T) {
},
)
testMapOrder(assert,
testMapOrder(assert, vrw,
BoolType, StringType,
[]Value{
Bool(true), String("unused"),
@@ -1217,53 +1275,59 @@ func TestMapOrdering(t *testing.T) {
func TestMapEmpty(t *testing.T) {
assert := assert.New(t)
me := NewMap().Edit()
vrw := newTestValueStore()
me := NewMap(vrw).Edit()
empty := func() bool {
return me.Map(nil).Empty()
return me.Map().Empty()
}
assert.True(empty())
me.Set(Bool(false), String("hi"))
assert.False(empty())
me.Set(NewList(), NewMap())
me.Set(NewList(vrw), NewMap(vrw))
assert.False(empty())
}
func TestMapType(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
emptyMapType := MakeMapType(MakeUnionType(), MakeUnionType())
m := NewMap()
m := NewMap(vrw)
assert.True(TypeOf(m).Equals(emptyMapType))
m2 := m.Edit().Remove(String("B")).Map(nil)
m2 := m.Edit().Remove(String("B")).Map()
assert.True(emptyMapType.Equals(TypeOf(m2)))
tr := MakeMapType(StringType, NumberType)
m2 = m.Edit().Set(String("A"), Number(1)).Map(nil)
m2 = m.Edit().Set(String("A"), Number(1)).Map()
assert.True(tr.Equals(TypeOf(m2)))
m2 = m.Edit().Set(String("B"), Number(2)).Set(String("C"), Number(2)).Map(nil)
m2 = m.Edit().Set(String("B"), Number(2)).Set(String("C"), Number(2)).Map()
assert.True(tr.Equals(TypeOf(m2)))
m3 := m2.Edit().Set(String("A"), Bool(true)).Map(nil)
m3 := m2.Edit().Set(String("A"), Bool(true)).Map()
assert.True(MakeMapType(StringType, MakeUnionType(BoolType, NumberType)).Equals(TypeOf(m3)), TypeOf(m3).Describe())
m4 := m3.Edit().Set(Bool(true), Number(1)).Map(nil)
m4 := m3.Edit().Set(Bool(true), Number(1)).Map()
assert.True(MakeMapType(MakeUnionType(BoolType, StringType), MakeUnionType(BoolType, NumberType)).Equals(TypeOf(m4)))
}
func TestMapChunks(t *testing.T) {
assert := assert.New(t)
l1 := NewMap(Number(0), Number(1))
vrw := newTestValueStore()
l1 := NewMap(vrw, Number(0), Number(1))
c1 := getChunks(l1)
assert.Len(c1, 0)
l2 := NewMap(NewRef(Number(0)), Number(1))
l2 := NewMap(vrw, NewRef(Number(0)), Number(1))
c2 := getChunks(l2)
assert.Len(c2, 1)
l3 := NewMap(Number(0), NewRef(Number(1)))
l3 := NewMap(vrw, Number(0), NewRef(Number(1)))
c3 := getChunks(l3)
assert.Len(c3, 1)
}
@@ -1274,12 +1338,14 @@ func TestMapFirstNNumbers(t *testing.T) {
}
assert := assert.New(t)
vrw := newTestValueStore()
kvs := []Value{}
for i := 0; i < testMapSize; i++ {
kvs = append(kvs, Number(i), Number(i+1))
}
m := NewMap(kvs...)
m := NewMap(vrw, kvs...)
assert.Equal(deriveCollectionHeight(m), getRefHeightOfCollection(m))
}
@@ -1299,7 +1365,7 @@ func TestMapRefOfStructFirstNNumbers(t *testing.T) {
kvs = append(kvs, k, v)
}
m := NewMap(kvs...)
m := NewMap(vs, kvs...)
// height + 1 because the leaves are Ref values (with height 1).
assert.Equal(deriveCollectionHeight(m)+1, getRefHeightOfCollection(m))
}
@@ -1311,24 +1377,26 @@ func TestMapModifyAfterRead(t *testing.T) {
defer normalProductionChunks()
vs := newTestValueStore()
m := getTestNativeOrderMap(2).toMap()
m := getTestNativeOrderMap(2).toMap(vs)
// Drop chunk values.
m = vs.ReadValue(vs.WriteValue(m).TargetHash()).(Map)
// Modify/query. Once upon a time this would crash.
fst, fstval := m.First()
m = m.Edit().Remove(fst).Map(nil)
m = m.Edit().Remove(fst).Map()
assert.False(m.Has(fst))
fst2, _ := m.First()
assert.True(m.Has(fst2))
m = m.Edit().Set(fst, fstval).Map(nil)
m = m.Edit().Set(fst, fstval).Map()
assert.True(m.Has(fst))
}
func TestMapTypeAfterMutations(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
test := func(n int, c interface{}) {
values := make([]Value, 2*n)
for i := 0; i < n; i++ {
@@ -1336,17 +1404,17 @@ func TestMapTypeAfterMutations(t *testing.T) {
values[2*i+1] = Number(i)
}
m := NewMap(values...)
m := NewMap(vrw, values...)
assert.Equal(m.Len(), uint64(n))
assert.IsType(c, m.sequence())
assert.True(TypeOf(m).Equals(MakeMapType(NumberType, NumberType)))
m = m.Edit().Set(String("a"), String("a")).Map(nil)
m = m.Edit().Set(String("a"), String("a")).Map()
assert.Equal(m.Len(), uint64(n+1))
assert.IsType(c, m.sequence())
assert.True(TypeOf(m).Equals(MakeMapType(MakeUnionType(NumberType, StringType), MakeUnionType(NumberType, StringType))))
m = m.Edit().Remove(String("a")).Map(nil)
m = m.Edit().Remove(String("a")).Map()
assert.Equal(m.Len(), uint64(n))
assert.IsType(c, m.sequence())
assert.True(TypeOf(m).Equals(MakeMapType(NumberType, NumberType)))
@@ -1359,33 +1427,35 @@ func TestMapTypeAfterMutations(t *testing.T) {
func TestCompoundMapWithValuesOfEveryType(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
v := Number(42)
kvs := []Value{
// Values
Bool(true), v,
Number(0), v,
String("hello"), v,
NewBlob(bytes.NewBufferString("buf")), v,
NewSet(Bool(true)), v,
NewList(Bool(true)), v,
NewMap(Bool(true), Number(0)), v,
NewBlob(vrw, bytes.NewBufferString("buf")), v,
NewSet(vrw, Bool(true)), v,
NewList(vrw, Bool(true)), v,
NewMap(vrw, Bool(true), Number(0)), v,
NewStruct("", StructData{"field": Bool(true)}), v,
// Refs of values
NewRef(Bool(true)), v,
NewRef(Number(0)), v,
NewRef(String("hello")), v,
NewRef(NewBlob(bytes.NewBufferString("buf"))), v,
NewRef(NewSet(Bool(true))), v,
NewRef(NewList(Bool(true))), v,
NewRef(NewMap(Bool(true), Number(0))), v,
NewRef(NewBlob(vrw, bytes.NewBufferString("buf"))), v,
NewRef(NewSet(vrw, Bool(true))), v,
NewRef(NewList(vrw, Bool(true))), v,
NewRef(NewMap(vrw, Bool(true), Number(0))), v,
NewRef(NewStruct("", StructData{"field": Bool(true)})), v,
}
m := NewMap(kvs...)
m := NewMap(vrw, kvs...)
for i := 1; m.sequence().isLeaf(); i++ {
k := Number(i)
kvs = append(kvs, k, v)
m = m.Edit().Set(k, v).Map(nil)
m = m.Edit().Set(k, v).Map()
}
assert.Equal(len(kvs)/2, int(m.Len()))
@@ -1405,7 +1475,7 @@ func TestCompoundMapWithValuesOfEveryType(t *testing.T) {
for len(kvs) > 0 {
k := kvs[0]
kvs = kvs[2:]
m = m.Edit().Remove(k).Map(nil)
m = m.Edit().Remove(k).Map()
assert.False(m.Has(k))
assert.Equal(len(kvs)/2, int(m.Len()))
}
@@ -1423,21 +1493,23 @@ func TestMapRemoveLastWhenNotLoaded(t *testing.T) {
}
tm := getTestNativeOrderMap(4)
nm := tm.toMap()
nm := tm.toMap(vs)
for len(tm.entries) > 0 {
entr := tm.entries
last := entr[len(entr)-1]
entr = entr[:len(entr)-1]
tm.entries = entr
nm = reload(nm.Edit().Remove(last.key).Map(nil))
assert.True(tm.toMap().Equals(nm))
nm = reload(nm.Edit().Remove(last.key).Map())
assert.True(tm.toMap(vs).Equals(nm))
}
}
func TestMapIterFrom(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
test := func(m Map, start, end Value) ValueSlice {
res := ValueSlice{}
m.IterFrom(start, func(k, v Value) bool {
@@ -1451,7 +1523,7 @@ func TestMapIterFrom(t *testing.T) {
}
kvs := generateNumbersAsValuesFromToBy(-50, 50, 1)
m1 := NewMap(kvs...)
m1 := NewMap(vrw, kvs...)
assert.True(kvs.Equals(test(m1, nil, Number(1000))))
assert.True(kvs.Equals(test(m1, Number(-1000), Number(1000))))
assert.True(kvs.Equals(test(m1, Number(-50), Number(1000))))
@@ -1466,8 +1538,10 @@ func TestMapIterFrom(t *testing.T) {
func TestMapAt(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
values := []Value{Bool(false), Number(42), String("a"), String("b"), String("c"), String("d")}
m := NewMap(values...)
m := NewMap(vrw, values...)
for i := 0; i < len(values); i += 2 {
k, v := m.At(uint64(i / 2))
@@ -1482,7 +1556,9 @@ func TestMapAt(t *testing.T) {
func TestMapWithStructShouldHaveOptionalFields(t *testing.T) {
assert := assert.New(t)
list := NewMap(
vrw := newTestValueStore()
list := NewMap(vrw,
String("one"),
NewStruct("Foo", StructData{
"a": Number(1),
@@ -1502,7 +1578,7 @@ func TestMapWithStructShouldHaveOptionalFields(t *testing.T) {
).Equals(TypeOf(list)))
// transpose
list = NewMap(
list = NewMap(vrw,
NewStruct("Foo", StructData{
"a": Number(1),
}),
@@ -1525,63 +1601,67 @@ func TestMapWithStructShouldHaveOptionalFields(t *testing.T) {
}
func TestMapWithNil(t *testing.T) {
vrw := newTestValueStore()
assert.Panics(t, func() {
NewMap(nil, Number(42))
})
assert.Panics(t, func() {
NewSet(Number(42), nil)
NewSet(vrw, Number(42), nil)
})
assert.Panics(t, func() {
NewMap(String("a"), String("b"), nil, Number(42))
NewMap(vrw, String("a"), String("b"), nil, Number(42))
})
assert.Panics(t, func() {
NewSet(String("a"), String("b"), Number(42), nil)
NewSet(vrw, String("a"), String("b"), Number(42), nil)
})
}
func TestNestedEditing(t *testing.T) {
me0 := NewMap().Edit()
vrw := newTestValueStore()
me0 := NewMap(vrw).Edit()
// m.a.a
me1a := NewMap().Edit()
me1a := NewMap(vrw).Edit()
me0.Set(String("a"), me1a)
se2a := NewSet().Edit()
se2a := NewSet(vrw).Edit()
me1a.Set(String("a"), se2a)
se2a.Insert(String("a"))
// m.b.b
me1b := NewMap().Edit()
me1b := NewMap(vrw).Edit()
me0.Set(String("b"), me1b)
se2b := NewSet().Edit()
se2b := NewSet(vrw).Edit()
me1b.Set(String("b"), se2b)
se2b.Insert(String("b"))
mOut := me0.Map(nil)
assert.True(t, mOut.Equals(NewMap(
String("a"), NewMap(
String("a"), NewSet(String("a")),
mOut := me0.Map()
assert.True(t, mOut.Equals(NewMap(vrw,
String("a"), NewMap(vrw,
String("a"), NewSet(vrw, String("a")),
),
String("b"), NewMap(
String("b"), NewSet(String("b")),
String("b"), NewMap(vrw,
String("b"), NewSet(vrw, String("b")),
),
)))
se2a.Remove(String("a")).Insert(String("aa"))
se2b.Remove(String("b")).Insert(String("bb"))
mOut = me0.Map(nil)
assert.True(t, mOut.Equals(NewMap(
String("a"), NewMap(
String("a"), NewSet(String("aa")),
mOut = me0.Map()
assert.True(t, mOut.Equals(NewMap(vrw,
String("a"), NewMap(vrw,
String("a"), NewSet(vrw, String("aa")),
),
String("b"), NewMap(
String("b"), NewSet(String("bb")),
String("b"), NewMap(vrw,
String("b"), NewSet(vrw, String("bb")),
),
)))
se2a.Remove(String("aa"))
se2b.Remove(String("bb"))
mOut = me0.Map(nil)
assert.True(t, mOut.Equals(NewMap())) // remove empty
mOut = me0.Map()
assert.True(t, mOut.Equals(NewMap(vrw))) // remove empty
}
go/types/meta_sequence.go
+15 -70
View File
@@ -17,9 +17,9 @@ const (
var emptyKey = orderedKey{}
func newMetaTuple(ref Ref, key orderedKey, numLeaves uint64, child Collection) metaTuple {
func newMetaTuple(ref Ref, key orderedKey, numLeaves uint64) metaTuple {
d.PanicIfFalse(Ref{} != ref)
return metaTuple{ref, key, numLeaves, child}
return metaTuple{ref, key, numLeaves}
}
// metaTuple is a node in a Prolly Tree, consisting of data in the node (either tree leaves or other metaSequences), and a Value annotation for exploring the tree (e.g. the largest item if this an ordered sequence).
@@ -27,13 +27,9 @@ type metaTuple struct {
ref Ref
key orderedKey
numLeaves uint64
child Collection // may be nil
}
func (mt metaTuple) getChildSequence(vr ValueReader) sequence {
if mt.child != nil {
return mt.child.sequence()
}
return mt.ref.TargetValue(vr).(Collection).sequence()
}
@@ -82,12 +78,12 @@ type metaSequence struct {
kind NomsKind
level uint64
tuples []metaTuple
vr ValueReader
vrw ValueReadWriter
}
func newMetaSequence(kind NomsKind, level uint64, tuples []metaTuple, vr ValueReader) metaSequence {
func newMetaSequence(kind NomsKind, level uint64, tuples []metaTuple, vrw ValueReadWriter) metaSequence {
d.PanicIfFalse(level > 0)
return metaSequence{kind, level, tuples, vr}
return metaSequence{kind, level, tuples, vrw}
}
func (ms metaSequence) data() []metaTuple {
@@ -122,8 +118,8 @@ func (ms metaSequence) seqLen() int {
return len(ms.tuples)
}
func (ms metaSequence) valueReader() ValueReader {
return ms.vr
func (ms metaSequence) valueReadWriter() ValueReadWriter {
return ms.vrw
}
func (ms metaSequence) WalkRefs(cb RefCallback) {
@@ -160,7 +156,7 @@ func (ms metaSequence) isLeaf() bool {
// metaSequence interface
func (ms metaSequence) getChildSequence(idx int) sequence {
mt := ms.tuples[idx]
return mt.getChildSequence(ms.vr)
return mt.getChildSequence(ms.vrw)
}
// Returns the sequences pointed to by all items[i], s.t. start <= i < end, and returns the
@@ -200,18 +196,18 @@ func (ms metaSequence) getCompositeChildSequence(start uint64, length uint64) se
}
if childIsMeta {
return newMetaSequence(ms.kind, ms.level-1, metaItems, ms.vr)
return newMetaSequence(ms.kind, ms.level-1, metaItems, ms.vrw)
}
if isIndexedSequence {
return newListLeafSequence(ms.vr, valueItems...)
return newListLeafSequence(ms.vrw, valueItems...)
}
if MapKind == ms.Kind() {
return newMapLeafSequence(ms.vr, mapItems...)
return newMapLeafSequence(ms.vrw, mapItems...)
}
return newSetLeafSequence(ms.vr, valueItems...)
return newSetLeafSequence(ms.vrw, valueItems...)
}
// fetches child sequences from start (inclusive) to end (exclusive) and respects uncommitted child
@@ -225,11 +221,7 @@ func (ms metaSequence) getChildren(start, end uint64) (seqs []sequence) {
for i := start; i < end; i++ {
mt := ms.tuples[i]
if mt.child != nil {
seqs[i-start] = mt.child.sequence()
} else {
hs[mt.ref.TargetHash()] = struct{}{}
}
hs[mt.ref.TargetHash()] = struct{}{}
}
if len(hs) == 0 {
@@ -239,7 +231,7 @@ func (ms metaSequence) getChildren(start, end uint64) (seqs []sequence) {
// Fetch committed child sequences in a single batch
valueChan := make(chan Value, len(hs))
go func() {
ms.vr.ReadManyValues(hs, valueChan)
ms.vrw.ReadManyValues(hs, valueChan)
close(valueChan)
}()
children := make(map[hash.Hash]sequence, len(hs))
@@ -249,10 +241,6 @@ func (ms metaSequence) getChildren(start, end uint64) (seqs []sequence) {
for i := start; i < end; i++ {
mt := ms.tuples[i]
if mt.child != nil {
continue
}
childSeq := children[mt.ref.TargetHash()]
d.Chk.NotNil(childSeq)
seqs[i-start] = childSeq
@@ -261,17 +249,6 @@ func (ms metaSequence) getChildren(start, end uint64) (seqs []sequence) {
return
}
func readMetaTupleValue(item sequenceItem, vr ValueReader) Value {
mt := item.(metaTuple)
if mt.child != nil {
return mt.child
}
r := mt.ref.TargetHash()
d.PanicIfTrue(r.IsEmpty())
return vr.ReadValue(r)
}
func metaHashValueBytes(item sequenceItem, rv *rollingValueHasher) {
mt := item.(metaTuple)
v := mt.key.v
@@ -301,7 +278,7 @@ func (es emptySequence) numLeaves() uint64 {
return 0
}
func (es emptySequence) valueReader() ValueReader {
func (es emptySequence) valueReadWriter() ValueReadWriter {
return nil
}
@@ -350,35 +327,3 @@ func (es emptySequence) treeLevel() uint64 {
func (es emptySequence) isLeaf() bool {
return es.level == 0
}
// Invokes |cb| on all "uncommitted" ptree nodes reachable from |v| in
// "bottom-up" order. It will only follow refs in the case of an in-memory
// reference to an uncommitted child. Note that |v| maybe not necessarily be
// a collection, but a collection with uncommitted children maybe be reachable
// from it (e.g. a Struct).
func iterateUncommittedChildren(v Value, cb func(sv Value)) {
if _, ok := v.(*Type); ok {
// Types can be in-memory cyclic, but they cannot reference uncommitted
// nodes. Just avoid traversing into them.
return
}
if c, ok := v.(Collection); ok {
if ms, ok := c.sequence().(metaSequence); ok {
// Only traverse uncommitted children of meta sequences
for _, mt := range ms.tuples {
if mt.child != nil {
iterateUncommittedChildren(mt.child, cb)
cb(mt.child)
}
}
return
}
}
// Traverse subvalues in search of nested collections with uncommitted
// ptree nodes.
v.WalkValues(func(v Value) {
iterateUncommittedChildren(v, cb)
})
}
go/types/number.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
type Number float64
// Value interface
func (v Number) Value(vrw ValueReadWriter) Value {
func (v Number) Value() Value {
return v
}
go/types/opcache.go
+19 -29
View File
@@ -119,7 +119,7 @@ type ldbOpCache struct {
type ldbOpCacheIterator struct {
iter ldbIterator.Iterator
vr ValueReader
vrw ValueReadWriter
}
func newLdbOpCacheStore(vrw ValueReadWriter) *ldbOpCacheStore {
@@ -158,13 +158,13 @@ func (opc *ldbOpCache) insertLdbOp(allKeys ValueSlice, opKind NomsKind, val Valu
ldbKeyBytes := [initialBufferSize]byte{}
ldbValBytes := [initialBufferSize]byte{}
ldbKey, valuesToEncode := encodeKeys(ldbKeyBytes[:0], opc.colId, opKind, allKeys, opc.vrw)
ldbKey, valuesToEncode := encodeKeys(ldbKeyBytes[:0], opc.colId, opKind, allKeys)
// val may be nil when dealing with sets, since the val is the key.
if val != nil {
valuesToEncode = append(valuesToEncode, val)
}
ldbVal := encodeValues(ldbValBytes[:0], valuesToEncode, opc.vrw)
ldbVal := encodeValues(ldbValBytes[:0], valuesToEncode)
err := opc.ldb.Put(ldbKey, ldbVal, nil)
d.Chk.NoError(err)
@@ -200,7 +200,7 @@ func (i *ldbOpCacheIterator) GraphOp() (ValueSlice, NomsKind, sequenceItem) {
graphKeys := ValueSlice{}
for pos := uint8(0); pos < numKeys; pos++ {
var gk Value
ldbKey, gk = decodeValue(ldbKey, false, i.vr)
ldbKey, gk = decodeValue(ldbKey, false, i.vrw)
graphKeys = append(graphKeys, gk)
}
@@ -213,7 +213,7 @@ func (i *ldbOpCacheIterator) GraphOp() (ValueSlice, NomsKind, sequenceItem) {
values := ValueSlice{}
for pos := uint8(0); pos < numEncodedValues; pos++ {
var gk Value
ldbVal, gk = decodeValue(ldbVal, true, i.vr)
ldbVal, gk = decodeValue(ldbVal, true, i.vrw)
values = append(values, gk)
}
@@ -248,7 +248,7 @@ func (i *ldbOpCacheIterator) GraphOp() (ValueSlice, NomsKind, sequenceItem) {
func (opc *ldbOpCache) NewIterator() opCacheIterator {
prefix := [4]byte{}
binary.BigEndian.PutUint32(prefix[:], opc.colId)
return &ldbOpCacheIterator{iter: opc.ldb.NewIterator(util.BytesPrefix(prefix[:]), nil), vr: opc.vrw}
return &ldbOpCacheIterator{iter: opc.ldb.NewIterator(util.BytesPrefix(prefix[:]), nil), vrw: opc.vrw}
}
func (i *ldbOpCacheIterator) Next() bool {
@@ -260,7 +260,7 @@ func (i *ldbOpCacheIterator) Release() {
}
// encodeKeys() serializes a list of keys to the byte slice |bs|.
func encodeKeys(bs []byte, colId uint32, opKind NomsKind, keys []Value, vrw ValueReadWriter) ([]byte, []Value) {
func encodeKeys(bs []byte, colId uint32, opKind NomsKind, keys []Value) ([]byte, []Value) {
// All ldb keys start with a 4-byte collection id that serves as a namespace
// that keeps them separate from other collections.
idHolder := [4]byte{}
@@ -276,7 +276,7 @@ func encodeKeys(bs []byte, colId uint32, opKind NomsKind, keys []Value, vrw Valu
valuesToEncode := ValueSlice{}
for _, gk := range keys {
bs = encodeGraphKey(bs, gk, vrw)
bs = encodeGraphKey(bs, gk)
if !isKindOrderedByValue(gk.Kind()) {
valuesToEncode = append(valuesToEncode, gk)
}
@@ -284,24 +284,24 @@ func encodeKeys(bs []byte, colId uint32, opKind NomsKind, keys []Value, vrw Valu
return bs, valuesToEncode
}
func encodeValues(bs []byte, valuesToEncode []Value, vrw ValueReadWriter) []byte {
func encodeValues(bs []byte, valuesToEncode []Value) []byte {
// Encode allValues into the ldbVal byte slice.
bs = append(bs, uint8(len(valuesToEncode)))
for _, k := range valuesToEncode {
bs = encodeGraphValue(bs, k, vrw)
bs = encodeGraphValue(bs, k)
}
return bs
}
func encodeGraphKey(bs []byte, v Value, vrw ValueReadWriter) []byte {
return encodeForGraph(bs, v, false, vrw)
func encodeGraphKey(bs []byte, v Value) []byte {
return encodeForGraph(bs, v, false)
}
func encodeGraphValue(bs []byte, v Value, vrw ValueReadWriter) []byte {
return encodeForGraph(bs, v, true, vrw)
func encodeGraphValue(bs []byte, v Value) []byte {
return encodeForGraph(bs, v, true)
}
func encodeForGraph(bs []byte, v Value, asValue bool, vrw ValueReadWriter) []byte {
func encodeForGraph(bs []byte, v Value, asValue bool) []byte {
// Note: encToSlice() and append() will both grow the backing store of |bs|
// as necessary. Always call them when writing to |bs|.
if asValue || isKindOrderedByValue(v.Kind()) {
@@ -309,7 +309,7 @@ func encodeForGraph(bs []byte, v Value, asValue bool, vrw ValueReadWriter) []byt
// noms-kind(1-byte), serialization-len(4-bytes), serialization(n-bytes)
buf := [initialBufferSize]byte{}
uint32buf := [4]byte{}
encodedVal := encToSlice(v, buf[:], vrw)
encodedVal := encToSlice(v, buf[:])
binary.BigEndian.PutUint32(uint32buf[:], uint32(len(encodedVal)))
bs = append(bs, uint8(v.Kind()))
bs = append(bs, uint32buf[:]...)
@@ -323,29 +323,19 @@ func encodeForGraph(bs []byte, v Value, asValue bool, vrw ValueReadWriter) []byt
return bs
}
func decodeValue(bs []byte, asValue bool, vr ValueReader) ([]byte, Value) {
func decodeValue(bs []byte, asValue bool, vrw ValueReadWriter) ([]byte, Value) {
kind := NomsKind(bs[0])
var v Value
if asValue || isKindOrderedByValue(kind) {
encodedLen := binary.BigEndian.Uint32(bs[1:5])
v = DecodeFromBytes(bs[5:5+encodedLen], vr)
v = DecodeFromBytes(bs[5:5+encodedLen], vrw)
return bs[5+encodedLen:], v
}
return bs[1+hash.ByteLen:], nil
}
// Note that, if 'v' are prolly trees, any in-memory child chunks will be written to vw at this time.
func encToSlice(v Value, initBuf []byte, vw ValueWriter) []byte {
// TODO: This is really unfortunate because WriteValue, in the case of
// ValueStore will internally also iterateUncommittedChildren. The problem
// here is a lack of clarity in the contract of ValueWriter. It's not explicit
// that WriteValue is responsible for writing uncommitted child nodes of
// ptrees, nor is there sufficient public API for any implementor of it to do
// so.
iterateUncommittedChildren(v, func(sv Value) {
vw.WriteValue(v)
})
func encToSlice(v Value, initBuf []byte) []byte {
// TODO: Are there enough calls to this that it's worth re-using a nomsWriter and valueEncoder?
w := &binaryNomsWriter{initBuf, 0}
enc := newValueEncoder(w)
go/types/opcache_test.go
+17 -14
View File
@@ -31,16 +31,17 @@ func (suite *OpCacheSuite) TearDownTest() {
}
func (suite *OpCacheSuite) TestMapSet() {
opCacheStore := newLdbOpCacheStore(suite.vs)
vs := suite.vs
opCacheStore := newLdbOpCacheStore(vs)
oc := opCacheStore.opCache()
defer opCacheStore.destroy()
entries := mapEntrySlice{
{NewList(Number(8), Number(0)), String("ahoy")},
{String("A key"), NewBlob(bytes.NewBufferString("A value"))},
{NewList(vs, Number(8), Number(0)), String("ahoy")},
{String("A key"), NewBlob(vs, bytes.NewBufferString("A value"))},
{Number(1), Bool(true)},
{Bool(false), Number(1)},
{NewBlob(bytes.NewBuffer([]byte{0xff, 0, 0})), NewMap()},
{NewBlob(vs, bytes.NewBuffer([]byte{0xff, 0, 0})), NewMap(vs)},
{Bool(true), Number(42)},
{NewStruct("thing1", StructData{"a": Number(7)}), Number(42)},
{String("struct"), NewStruct("thing2", nil)},
@@ -64,19 +65,20 @@ func (suite *OpCacheSuite) TestMapSet() {
}
func (suite *OpCacheSuite) TestSetInsert() {
opCacheStore := newLdbOpCacheStore(suite.vs)
vs := suite.vs
opCacheStore := newLdbOpCacheStore(vs)
oc := opCacheStore.opCache()
defer opCacheStore.destroy()
entries := ValueSlice{
NewList(Number(8), Number(0)),
NewList(vs, Number(8), Number(0)),
String("ahoy"),
NewBlob(bytes.NewBufferString("A value")),
NewBlob(vs, bytes.NewBufferString("A value")),
Number(1),
Bool(true),
Bool(false),
NewBlob(bytes.NewBuffer([]byte{0xff, 0, 0})),
NewMap(),
NewBlob(vs, bytes.NewBuffer([]byte{0xff, 0, 0})),
NewMap(vs),
Number(42),
NewStruct("thing1", StructData{"a": Number(7)}),
String("struct"),
@@ -101,19 +103,20 @@ func (suite *OpCacheSuite) TestSetInsert() {
}
func (suite *OpCacheSuite) TestListAppend() {
opCacheStore := newLdbOpCacheStore(suite.vs)
vs := suite.vs
opCacheStore := newLdbOpCacheStore(vs)
oc := opCacheStore.opCache()
defer opCacheStore.destroy()
entries := ValueSlice{
NewList(Number(8), Number(0)),
NewList(vs, Number(8), Number(0)),
String("ahoy"),
NewBlob(bytes.NewBufferString("A value")),
NewBlob(vs, bytes.NewBufferString("A value")),
Number(1),
Bool(true),
Bool(false),
NewBlob(bytes.NewBuffer([]byte{0xff, 0, 0})),
NewMap(),
NewBlob(vs, bytes.NewBuffer([]byte{0xff, 0, 0})),
NewMap(vs),
Number(42),
NewStruct("thing1", StructData{"a": Number(7)}),
String("struct"),
go/types/ordered_sequences.go
+7 -7
View File
@@ -15,12 +15,12 @@ type orderedSequence interface {
getKey(idx int) orderedKey
}
func newSetMetaSequence(level uint64, tuples []metaTuple, vr ValueReader) metaSequence {
return newMetaSequence(SetKind, level, tuples, vr)
func newSetMetaSequence(level uint64, tuples []metaTuple, vrw ValueReadWriter) metaSequence {
return newMetaSequence(SetKind, level, tuples, vrw)
}
func newMapMetaSequence(level uint64, tuples []metaTuple, vr ValueReader) metaSequence {
return newMetaSequence(MapKind, level, tuples, vr)
func newMapMetaSequence(level uint64, tuples []metaTuple, vrw ValueReadWriter) metaSequence {
return newMetaSequence(MapKind, level, tuples, vrw)
}
func newCursorAtValue(seq orderedSequence, val Value, forInsertion bool, last bool, readAhead bool) *sequenceCursor {
@@ -90,7 +90,7 @@ func getMapValue(cur *sequenceCursor) Value {
// If |vw| is not nil, chunks will be eagerly written as they're created. Otherwise they are
// written when the root is written.
func newOrderedMetaSequenceChunkFn(kind NomsKind, vr ValueReader) makeChunkFn {
func newOrderedMetaSequenceChunkFn(kind NomsKind, vrw ValueReadWriter) makeChunkFn {
return func(level uint64, items []sequenceItem) (Collection, orderedKey, uint64) {
tuples := make([]metaTuple, len(items))
numLeaves := uint64(0)
@@ -106,10 +106,10 @@ func newOrderedMetaSequenceChunkFn(kind NomsKind, vr ValueReader) makeChunkFn {
var col Collection
if kind == SetKind {
col = newSet(newSetMetaSequence(level, tuples, vr))
col = newSet(newSetMetaSequence(level, tuples, vrw))
} else {
d.PanicIfFalse(MapKind == kind)
col = newMap(newMapMetaSequence(level, tuples, vr))
col = newMap(newMapMetaSequence(level, tuples, vrw))
}
return col, tuples[len(tuples)-1].key, numLeaves
go/types/ordered_sequences_diff_test.go
+14 -9
View File
@@ -59,6 +59,8 @@ func accumulateOrderedSequenceDiffChanges(o1, o2 orderedSequence, df diffFn) (ad
}
func (suite *diffTestSuite) TestDiff() {
vs := newTestValueStore()
type valFn func(int, int, int) ValueSlice
type colFn func([]Value) Collection
@@ -92,11 +94,10 @@ func (suite *diffTestSuite) TestDiff() {
runTestDf(name, vf, cf, orderedSequenceDiffBest)
}
newSetAsCol := func(vs []Value) Collection { return NewSet(vs...) }
newMapAsCol := func(vs []Value) Collection { return NewMap(vs...) }
newSetAsCol := func(vals []Value) Collection { return NewSet(vs, vals...) }
newMapAsCol := func(vals []Value) Collection { return NewMap(vs, vals...) }
rw := func(col Collection) Collection {
vs := newTestValueStore()
h := vs.WriteValue(col).TargetHash()
vs.Commit(vs.Root(), vs.Root())
return vs.ReadValue(h).(Collection)
@@ -156,10 +157,12 @@ func TestOrderedSequencesDisjoint(t *testing.T) {
}
func TestOrderedSequencesDiffCloseWithoutReading(t *testing.T) {
vs := newTestValueStore()
runTest := func(df diffFn) {
s1 := NewSet().seq
s1 := NewSet(vs).seq
// A single item should be enough, but generate lots anyway.
s2 := NewSet(generateNumbersAsValuesFromToBy(0, 1000, 1)...).seq
s2 := NewSet(vs, generateNumbersAsValuesFromToBy(0, 1000, 1)...).seq
changeChan := make(chan ValueChanged)
closeChan := make(chan struct{})
@@ -182,17 +185,19 @@ func TestOrderedSequencesDiffCloseWithoutReading(t *testing.T) {
func TestOrderedSequenceDiffWithMetaNodeGap(t *testing.T) {
assert := assert.New(t)
vrw := newTestValueStore()
newSetSequenceMt := func(v ...Value) metaTuple {
seq := newSetLeafSequence(nil, v...)
seq := newSetLeafSequence(vrw, v...)
set := newSet(seq)
return newMetaTuple(NewRef(set), newOrderedKey(v[len(v)-1]), uint64(len(v)), set)
return newMetaTuple(vrw.WriteValue(set), newOrderedKey(v[len(v)-1]), uint64(len(v)))
}
m1 := newSetSequenceMt(Number(1), Number(2))
m2 := newSetSequenceMt(Number(3), Number(4))
m3 := newSetSequenceMt(Number(5), Number(6))
s1 := newSetMetaSequence(1, []metaTuple{m1, m3}, nil)
s2 := newSetMetaSequence(1, []metaTuple{m1, m2, m3}, nil)
s1 := newSetMetaSequence(1, []metaTuple{m1, m3}, vrw)
s2 := newSetMetaSequence(1, []metaTuple{m1, m2, m3}, vrw)
runTest := func(df diffFn) {
changes := make(chan ValueChanged)
go/types/path_test.go
+30 -17
View File
@@ -89,6 +89,7 @@ func TestPathStructType(t *testing.T) {
func TestPathIndex(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
var v Value
resolvesTo := func(expVal, expKey Value, str string) {
@@ -96,7 +97,7 @@ func TestPathIndex(t *testing.T) {
assertResolvesTo(assert, expKey, v, str+"@key")
}
v = NewList(Number(1), Number(3), String("foo"), Bool(false))
v = NewList(vs, Number(1), Number(3), String("foo"), Bool(false))
resolvesTo(Number(1), Number(0), "[0]")
resolvesTo(Number(3), Number(1), "[1]")
@@ -109,7 +110,7 @@ func TestPathIndex(t *testing.T) {
resolvesTo(String("foo"), Number(2), "[-2]")
resolvesTo(Bool(false), Number(3), "[-1]")
v = NewMap(
v = NewMap(vs,
Bool(false), Number(23),
Number(1), String("foo"),
Number(2.3), Number(4.5),
@@ -168,20 +169,22 @@ func TestPathIndexType(t *testing.T) {
func TestPathHashIndex(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
b := Bool(true)
br := NewRef(b)
i := Number(0)
str := String("foo")
l := NewList(b, i, str)
l := NewList(vs, b, i, str)
lr := NewRef(l)
m := NewMap(
m := NewMap(vs,
b, br,
br, i,
i, str,
l, lr,
lr, b,
)
s := NewSet(b, br, i, str, l, lr)
s := NewSet(vs, b, br, i, str, l, lr)
resolvesTo := func(col, key, expVal, expKey Value) {
assertResolvesTo(assert, expVal, col, hashIdx(key))
@@ -212,31 +215,35 @@ func TestPathHashIndexOfSingletonCollection(t *testing.T) {
// This test is to make sure we don't accidentally return |b| if it's the only element.
assert := assert.New(t)
vs := newTestValueStore()
resolvesToNil := func(col, val Value) {
assertResolvesTo(assert, nil, col, hashIdx(val))
}
b := Bool(true)
resolvesToNil(NewMap(b, b), b)
resolvesToNil(NewSet(b), b)
resolvesToNil(NewMap(vs, b, b), b)
resolvesToNil(NewSet(vs, b), b)
}
func TestPathMulti(t *testing.T) {
assert := assert.New(t)
m1 := NewMap(
vs := newTestValueStore()
m1 := NewMap(vs,
String("a"), String("foo"),
String("b"), String("bar"),
String("c"), String("car"),
)
m2 := NewMap(
m2 := NewMap(vs,
Bool(false), String("earth"),
String("d"), String("dar"),
m1, String("fire"),
)
l := NewList(m1, m2)
l := NewList(vs, m1, m2)
s := NewStruct("", StructData{
"foo": l,
@@ -422,12 +429,14 @@ func TestPathEquals(t *testing.T) {
func TestPathCanBePathIndex(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
assert.True(ValueCanBePathIndex(Bool(true)))
assert.True(ValueCanBePathIndex(Number(5)))
assert.True(ValueCanBePathIndex(String("yes")))
assert.False(ValueCanBePathIndex(NewRef(String("yes"))))
assert.False(ValueCanBePathIndex(NewBlob(bytes.NewReader([]byte("yes")))))
assert.False(ValueCanBePathIndex(NewBlob(vs, bytes.NewReader([]byte("yes")))))
}
func TestCopyPath(t *testing.T) {
@@ -468,12 +477,14 @@ func TestMustParsePath(t *testing.T) {
func TestPathType(t *testing.T) {
assert := assert.New(t)
m := NewMap(
vs := newTestValueStore()
m := NewMap(vs,
String("string"), String("foo"),
String("bool"), Bool(false),
String("number"), Number(42),
String("List<number|string>"), NewList(Number(42), String("foo")),
String("Map<Bool, Bool>"), NewMap(Bool(true), Bool(false)))
String("List<number|string>"), NewList(vs, Number(42), String("foo")),
String("Map<Bool, Bool>"), NewMap(vs, Bool(true), Bool(false)))
m.IterAll(func(k, cv Value) {
ks := k.(String)
@@ -511,13 +522,15 @@ func TestPathTarget(t *testing.T) {
func TestPathAtAnnotation(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
var v Value
resolvesTo := func(expVal, expKey Value, str string) {
assertResolvesTo(assert, expVal, v, str)
assertResolvesTo(assert, expKey, v, str+"@key")
}
v = NewList(Number(1), Number(3), String("foo"), Bool(false))
v = NewList(vs, Number(1), Number(3), String("foo"), Bool(false))
resolvesTo(Number(1), nil, "@at(0)")
resolvesTo(Number(3), nil, "@at(1)")
@@ -530,7 +543,7 @@ func TestPathAtAnnotation(t *testing.T) {
resolvesTo(String("foo"), nil, "@at(-2)")
resolvesTo(Bool(false), nil, "@at(-1)")
v = NewSet(
v = NewSet(vs,
Bool(false),
Number(1),
Number(2.3),
@@ -548,7 +561,7 @@ func TestPathAtAnnotation(t *testing.T) {
resolvesTo(Number(2.3), Number(2.3), "@at(-2)")
resolvesTo(String("two"), String("two"), `@at(-1)`)
v = NewMap(
v = NewMap(vs,
Bool(false), Number(23),
Number(1), String("foo"),
Number(2.3), Number(4.5),
go/types/perf/perf_test.go
+2 -2
View File
@@ -86,7 +86,7 @@ func (s *perfSuite) Test05Concat10mValues2kTimes() {
l1Len, l2Len := l1.Len(), l2.Len()
l1Last, l2Last := last(l1), last(l2)
l3 := types.NewList()
l3 := types.NewList(s.Database)
for i := uint64(0); i < 1e3; i++ { // 1k iterations * 2 concat ops = 2k times
// Include some basic sanity checks.
l3 = l3.Concat(l1)
@@ -151,7 +151,7 @@ func (s *perfSuite) testBuild500megBlob(p int) {
}
})
b := types.NewBlob(readers...)
b := types.NewBlob(s.Database, readers...)
assert.Equal(uint64(size), b.Len())
}
go/types/ref.go
+1 -1
View File
@@ -55,7 +55,7 @@ func (r Ref) TargetType() *Type {
}
// Value interface
func (r Ref) Value(vrw ValueReadWriter) Value {
func (r Ref) Value() Value {
return r
}
go/types/ref_test.go
+15 -7
View File
@@ -13,9 +13,11 @@ import (
func TestRefInList(t *testing.T) {
assert := assert.New(t)
l := NewList()
vs := newTestValueStore()
l := NewList(vs)
r := NewRef(l)
l = l.Edit().Append(r).List(nil)
l = l.Edit().Append(r).List()
r2 := l.Get(0)
assert.True(r.Equals(r2))
}
@@ -23,9 +25,11 @@ func TestRefInList(t *testing.T) {
func TestRefInSet(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
r := NewRef(s)
s = s.Edit().Insert(r).Set(nil)
s = s.Edit().Insert(r).Set()
r2 := s.First()
assert.True(r.Equals(r2))
}
@@ -33,9 +37,11 @@ func TestRefInSet(t *testing.T) {
func TestRefInMap(t *testing.T) {
assert := assert.New(t)
m := NewMap()
vs := newTestValueStore()
m := NewMap(vs)
r := NewRef(m)
m = m.Edit().Set(Number(0), r).Set(r, Number(1)).Map(nil)
m = m.Edit().Set(Number(0), r).Set(r, Number(1)).Map()
r2 := m.Get(Number(0))
assert.True(r.Equals(r2))
@@ -46,7 +52,9 @@ func TestRefInMap(t *testing.T) {
func TestRefChunks(t *testing.T) {
assert := assert.New(t)
l := NewList()
vs := newTestValueStore()
l := NewList(vs)
r := NewRef(l)
assert.Len(getChunks(r), 1)
assert.Equal(r, getChunks(r)[0])
go/types/sequence.go
+1 -1
View File
@@ -12,7 +12,7 @@ type sequence interface {
getItem(idx int) sequenceItem
seqLen() int
numLeaves() uint64
valueReader() ValueReader
valueReadWriter() ValueReadWriter
WalkRefs(cb RefCallback)
typeOf() *Type
Kind() NomsKind
go/types/sequence_chunker.go
+9 -17
View File
@@ -11,8 +11,7 @@ type hashValueBytesFn func(item sequenceItem, rv *rollingValueHasher)
type sequenceChunker struct {
cur *sequenceCursor
level uint64
vr ValueReader
vw ValueWriter
vrw ValueReadWriter
parent *sequenceChunker
current []sequenceItem
makeChunk, parentMakeChunk makeChunkFn
@@ -25,22 +24,22 @@ type sequenceChunker struct {
// makeChunkFn takes a sequence of items to chunk, and returns the result of chunking those items, a tuple of a reference to that chunk which can itself be chunked + its underlying value.
type makeChunkFn func(level uint64, values []sequenceItem) (Collection, orderedKey, uint64)
func newEmptySequenceChunker(vr ValueReader, vw ValueWriter, makeChunk, parentMakeChunk makeChunkFn, hashValueBytes hashValueBytesFn) *sequenceChunker {
return newSequenceChunker(nil, uint64(0), vr, vw, makeChunk, parentMakeChunk, hashValueBytes)
func newEmptySequenceChunker(vrw ValueReadWriter, makeChunk, parentMakeChunk makeChunkFn, hashValueBytes hashValueBytesFn) *sequenceChunker {
return newSequenceChunker(nil, uint64(0), vrw, makeChunk, parentMakeChunk, hashValueBytes)
}
func newSequenceChunker(cur *sequenceCursor, level uint64, vr ValueReader, vw ValueWriter, makeChunk, parentMakeChunk makeChunkFn, hashValueBytes hashValueBytesFn) *sequenceChunker {
func newSequenceChunker(cur *sequenceCursor, level uint64, vrw ValueReadWriter, makeChunk, parentMakeChunk makeChunkFn, hashValueBytes hashValueBytesFn) *sequenceChunker {
d.PanicIfFalse(makeChunk != nil)
d.PanicIfFalse(parentMakeChunk != nil)
d.PanicIfFalse(hashValueBytes != nil)
d.PanicIfTrue(vrw == nil)
// |cur| will be nil if this is a new sequence, implying this is a new tree, or the tree has grown in height relative to its original chunked form.
sc := &sequenceChunker{
cur,
level,
vr,
vw,
vrw,
nil,
[]sequenceItem{},
makeChunk, parentMakeChunk,
@@ -164,7 +163,7 @@ func (sc *sequenceChunker) createParent() {
// Clone the parent cursor because otherwise calling cur.advance() will affect our parent - and vice versa - in surprising ways. Instead, Skip moves forward our parent's cursor if we advance across a boundary.
parent = sc.cur.parent
}
sc.parent = newSequenceChunker(parent, sc.level+1, sc.vr, sc.vw, sc.parentMakeChunk, sc.parentMakeChunk, metaHashValueBytes)
sc.parent = newSequenceChunker(parent, sc.level+1, sc.vrw, sc.parentMakeChunk, sc.parentMakeChunk, metaHashValueBytes)
sc.parent.isLeaf = false
}
@@ -172,14 +171,7 @@ func (sc *sequenceChunker) createSequence() (sequence, metaTuple) {
// If the sequence chunker has a ValueWriter, eagerly write sequences.
col, key, numLeaves := sc.makeChunk(sc.level, sc.current)
seq := col.sequence()
var ref Ref
if sc.vw != nil {
ref = sc.vw.WriteValue(col)
col = nil
} else {
ref = NewRef(col)
}
mt := newMetaTuple(ref, key, numLeaves, col)
mt := newMetaTuple(sc.vrw.WriteValue(col), key, numLeaves)
sc.current = []sequenceItem{}
return seq, mt
@@ -242,7 +234,7 @@ func (sc *sequenceChunker) Done() sequence {
mt := sc.current[0].(metaTuple)
for {
child := mt.getChildSequence(sc.vr)
child := mt.getChildSequence(sc.vrw)
if _, ok := child.(metaSequence); !ok || child.seqLen() > 1 {
return child
}
go/types/sequence_concat.go
+4 -4
View File
@@ -6,7 +6,7 @@ package types
import "github.com/attic-labs/noms/go/d"
type newSequenceChunkerFn func(cur *sequenceCursor, vr ValueReader) *sequenceChunker
type newSequenceChunkerFn func(cur *sequenceCursor, vrw ValueReadWriter) *sequenceChunker
func concat(fst, snd sequence, newSequenceChunker newSequenceChunkerFn) sequence {
if fst.numLeaves() == 0 {
@@ -19,11 +19,11 @@ func concat(fst, snd sequence, newSequenceChunker newSequenceChunkerFn) sequence
// concat works by tricking the sequenceChunker into resuming chunking at a
// cursor to the end of fst, then finalizing chunking to the start of snd - by
// swapping fst cursors for snd cursors in the middle of chunking.
vr := fst.valueReader()
if vr != snd.valueReader() {
vrw := fst.valueReadWriter()
if vrw != snd.valueReadWriter() {
d.Panic("cannot concat sequences from different databases")
}
chunker := newSequenceChunker(newCursorAtIndex(fst, fst.numLeaves(), false), vr)
chunker := newSequenceChunker(newCursorAtIndex(fst, fst.numLeaves(), false), vrw)
for cur, ch := newCursorAtIndex(snd, 0, false), chunker; ch != nil; ch = ch.parent {
// Note that if snd is shallower than fst, then higher chunkers will have
go/types/sequence_cursor.go
+1 -1
View File
@@ -80,7 +80,7 @@ func newSequenceCursor(parent *sequenceCursor, seq sequence, idx int, readAhead
d.PanicIfFalse(idx >= 0)
}
readAhead = readAhead && !seq.isLeaf() && seq.valueReader() != nil
readAhead = readAhead && !seq.isLeaf() && seq.valueReadWriter() != nil
return &sequenceCursor{parent, seq, idx, readAhead, nil}
}
go/types/sequence_cursor_test.go
+1 -1
View File
@@ -51,7 +51,7 @@ func (ts testSequence) getCompareFn(other sequence) compareFn {
}
}
func (ts testSequence) valueReader() ValueReader {
func (ts testSequence) valueReadWriter() ValueReadWriter {
panic("not reached")
}
go/types/set.go
+9 -8
View File
@@ -21,9 +21,9 @@ func newSet(seq orderedSequence) Set {
return Set{seq, &hash.Hash{}}
}
func NewSet(v ...Value) Set {
func NewSet(vrw ValueReadWriter, v ...Value) Set {
data := buildSetData(v)
ch := newEmptySetSequenceChunker(nil, nil)
ch := newEmptySetSequenceChunker(vrw)
for _, v := range data {
ch.Append(v)
@@ -47,6 +47,7 @@ func NewStreamingSet(vrw ValueReadWriter, vChan <-chan Value) <-chan Set {
type streamingSetReadFunc func(vrw ValueReadWriter, vChan <-chan Value, outChan chan<- Set)
func newStreamingSet(vrw ValueReadWriter, vChan <-chan Value, readFunc streamingSetReadFunc) <-chan Set {
d.PanicIfTrue(vrw == nil)
outChan := make(chan Set, 1)
readFunc(vrw, vChan, outChan)
return outChan
@@ -54,7 +55,7 @@ func newStreamingSet(vrw ValueReadWriter, vChan <-chan Value, readFunc streaming
func readSetInput(vrw ValueReadWriter, vChan <-chan Value, outChan chan<- Set) {
defer close(outChan)
ch := newEmptySetSequenceChunker(vrw, vrw)
ch := newEmptySetSequenceChunker(vrw)
var lastV Value
for v := range vChan {
d.PanicIfTrue(v == nil)
@@ -114,7 +115,7 @@ func (s Set) hashPointer() *hash.Hash {
}
// Value interface
func (s Set) Value(vrw ValueReadWriter) Value {
func (s Set) Value() Value {
return s
}
@@ -240,7 +241,7 @@ func buildSetData(values ValueSlice) ValueSlice {
return append(uniqueSorted, last)
}
func makeSetLeafChunkFn(vr ValueReader) makeChunkFn {
func makeSetLeafChunkFn(vrw ValueReadWriter) makeChunkFn {
return func(level uint64, items []sequenceItem) (Collection, orderedKey, uint64) {
d.PanicIfFalse(level == 0)
setData := make([]Value, len(items), len(items))
@@ -253,7 +254,7 @@ func makeSetLeafChunkFn(vr ValueReader) makeChunkFn {
setData[i] = v
}
set := newSet(newSetLeafSequence(vr, setData...))
set := newSet(newSetLeafSequence(vrw, setData...))
var key orderedKey
if len(setData) > 0 {
key = newOrderedKey(setData[len(setData)-1])
@@ -263,6 +264,6 @@ func makeSetLeafChunkFn(vr ValueReader) makeChunkFn {
}
}
func newEmptySetSequenceChunker(vr ValueReader, vw ValueWriter) *sequenceChunker {
return newEmptySequenceChunker(vr, vw, makeSetLeafChunkFn(vr), newOrderedMetaSequenceChunkFn(SetKind, vr), hashValueBytes)
func newEmptySetSequenceChunker(vrw ValueReadWriter) *sequenceChunker {
return newEmptySequenceChunker(vrw, makeSetLeafChunkFn(vrw), newOrderedMetaSequenceChunkFn(SetKind, vrw), hashValueBytes)
}
go/types/set_editor.go
+5 -8
View File
@@ -30,19 +30,16 @@ func (se *SetEditor) Kind() NomsKind {
return SetKind
}
func (se *SetEditor) Value(vrw ValueReadWriter) Value {
return se.Set(vrw)
func (se *SetEditor) Value() Value {
return se.Set()
}
func (se *SetEditor) Set(vrw ValueReadWriter) Set {
func (se *SetEditor) Set() Set {
if len(se.edits) == 0 {
return se.s // no edits
}
vr := se.s.sequence().valueReader()
if vrw != nil {
vr = vrw
}
vrw := se.s.sequence().valueReadWriter()
se.normalize()
@@ -93,7 +90,7 @@ func (se *SetEditor) Set(vrw ValueReadWriter) Set {
}
if ch == nil {
ch = newSequenceChunker(cur, 0, vr, vrw, makeSetLeafChunkFn(vr), newOrderedMetaSequenceChunkFn(SetKind, vr), hashValueBytes)
ch = newSequenceChunker(cur, 0, vrw, makeSetLeafChunkFn(vrw), newOrderedMetaSequenceChunkFn(SetKind, vrw), hashValueBytes)
} else {
ch.advanceTo(cur)
}
go/types/set_iterator_test.go
+89 -72
View File
@@ -14,11 +14,13 @@ import (
func TestSetIterator(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
numbers := append(generateNumbersAsValues(10), Number(20), Number(25))
s := NewSet(numbers...)
s := NewSet(vs, numbers...)
i := s.Iterator()
vs := iterToSlice(i)
assert.True(vs.Equals(numbers), "Expected: %v != actual: %v", numbers, vs)
vals := iterToSlice(i)
assert.True(vals.Equals(numbers), "Expected: %v != actual: %v", numbers, vs)
i = s.Iterator()
assert.Panics(func() { i.SkipTo(nil) })
@@ -39,90 +41,96 @@ func TestSetIterator(t *testing.T) {
assert.Equal(Number(3), i.SkipTo(Number(3)))
assert.Equal(Number(4), i.Next())
empty := NewSet()
empty := NewSet(vs)
assert.Nil(empty.Iterator().Next())
assert.Nil(empty.Iterator().SkipTo(Number(-30)))
single := NewSet(Number(42)).Iterator()
single := NewSet(vs, Number(42)).Iterator()
assert.Equal(Number(42), single.SkipTo(Number(42)))
assert.Equal(nil, single.SkipTo(Number(42)))
single = NewSet(Number(42)).Iterator()
single = NewSet(vs, Number(42)).Iterator()
assert.Equal(Number(42), single.SkipTo(Number(42)))
assert.Equal(nil, single.Next())
single = NewSet(Number(42)).Iterator()
single = NewSet(vs, Number(42)).Iterator()
assert.Equal(Number(42), single.SkipTo(Number(21)))
}
func TestSetIteratorAt(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
numbers := append(generateNumbersAsValues(5), Number(10))
s := NewSet(numbers...)
s := NewSet(vs, numbers...)
i := s.IteratorAt(0)
vs := iterToSlice(i)
assert.True(vs.Equals(numbers), "Expected: %v != actual: %v", numbers, vs)
vals := iterToSlice(i)
assert.True(vals.Equals(numbers), "Expected: %v != actual: %v", numbers, vs)
i = s.IteratorAt(2)
vs = iterToSlice(i)
assert.True(vs.Equals(numbers[2:]), "Expected: %v != actual: %v", numbers[2:], vs)
vals = iterToSlice(i)
assert.True(vals.Equals(numbers[2:]), "Expected: %v != actual: %v", numbers[2:], vs)
i = s.IteratorAt(10)
vs = iterToSlice(i)
assert.True(vs.Equals(nil), "Expected: %v != actual: %v", nil, vs)
vals = iterToSlice(i)
assert.True(vals.Equals(nil), "Expected: %v != actual: %v", nil, vs)
}
func TestSetIteratorFrom(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
numbers := append(generateNumbersAsValues(5), Number(10), Number(20))
s := NewSet(numbers...)
s := NewSet(vs, numbers...)
i := s.IteratorFrom(Number(0))
vs := iterToSlice(i)
assert.True(vs.Equals(numbers), "Expected: %v != actual: %v", numbers, vs)
vals := iterToSlice(i)
assert.True(vals.Equals(numbers), "Expected: %v != actual: %v", numbers, vs)
i = s.IteratorFrom(Number(2))
vs = iterToSlice(i)
assert.True(vs.Equals(numbers[2:]), "Expected: %v != actual: %v", numbers[2:], vs)
vals = iterToSlice(i)
assert.True(vals.Equals(numbers[2:]), "Expected: %v != actual: %v", numbers[2:], vs)
i = s.IteratorFrom(Number(10))
vs = iterToSlice(i)
assert.True(vs.Equals(ValueSlice{Number(10), Number(20)}), "Expected: %v != actual: %v", nil, vs)
vals = iterToSlice(i)
assert.True(vals.Equals(ValueSlice{Number(10), Number(20)}), "Expected: %v != actual: %v", nil, vs)
i = s.IteratorFrom(Number(20))
vs = iterToSlice(i)
assert.True(vs.Equals(ValueSlice{Number(20)}), "Expected: %v != actual: %v", nil, vs)
vals = iterToSlice(i)
assert.True(vals.Equals(ValueSlice{Number(20)}), "Expected: %v != actual: %v", nil, vs)
i = s.IteratorFrom(Number(100))
vs = iterToSlice(i)
assert.True(vs.Equals(nil), "Expected: %v != actual: %v", nil, vs)
vals = iterToSlice(i)
assert.True(vals.Equals(nil), "Expected: %v != actual: %v", nil, vs)
// Not present. Starts at next larger.
i = s.IteratorFrom(Number(15))
vs = iterToSlice(i)
assert.True(vs.Equals(ValueSlice{Number(20)}), "Expected: %v != actual: %v", nil, vs)
vals = iterToSlice(i)
assert.True(vals.Equals(ValueSlice{Number(20)}), "Expected: %v != actual: %v", nil, vs)
}
func TestUnionIterator(t *testing.T) {
assert := assert.New(t)
set1 := NewSet(generateNumbersAsValuesFromToBy(0, 10, 1)...)
set2 := NewSet(generateNumbersAsValuesFromToBy(5, 15, 1)...)
set3 := NewSet(generateNumbersAsValuesFromToBy(10, 20, 1)...)
set4 := NewSet(generateNumbersAsValuesFromToBy(15, 25, 1)...)
vs := newTestValueStore()
set1 := NewSet(vs, generateNumbersAsValuesFromToBy(0, 10, 1)...)
set2 := NewSet(vs, generateNumbersAsValuesFromToBy(5, 15, 1)...)
set3 := NewSet(vs, generateNumbersAsValuesFromToBy(10, 20, 1)...)
set4 := NewSet(vs, generateNumbersAsValuesFromToBy(15, 25, 1)...)
ui1 := NewUnionIterator(set1.Iterator(), set2.Iterator())
vs := iterToSlice(ui1)
vals := iterToSlice(ui1)
expectedRes := generateNumbersAsValues(15)
assert.True(vs.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
assert.True(vals.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
ui1 = NewUnionIterator(set1.Iterator(), set4.Iterator())
ui2 := NewUnionIterator(set3.Iterator(), set2.Iterator())
ui3 := NewUnionIterator(ui1, ui2)
vs = iterToSlice(ui3)
vals = iterToSlice(ui3)
expectedRes = generateNumbersAsValues(25)
assert.True(vs.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
assert.True(vals.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
ui1 = NewUnionIterator(set1.Iterator(), set4.Iterator())
ui2 = NewUnionIterator(set3.Iterator(), set2.Iterator())
@@ -140,34 +148,36 @@ func TestUnionIterator(t *testing.T) {
assert.Equal(Number(24), ui3.SkipTo(Number(24)))
assert.Nil(ui3.SkipTo(Number(25)))
singleElemSet := NewSet(Number(4))
emptySet := NewSet()
singleElemSet := NewSet(vs, Number(4))
emptySet := NewSet(vs)
ui10 := NewUnionIterator(singleElemSet.Iterator(), singleElemSet.Iterator())
ui20 := NewUnionIterator(emptySet.Iterator(), emptySet.Iterator())
ui30 := NewUnionIterator(ui10, ui20)
vs = iterToSlice(ui30)
vals = iterToSlice(ui30)
expectedRes = ValueSlice{Number(4)}
assert.True(vs.Equals(expectedRes), "%v != %v\n", expectedRes, vs)
assert.True(vals.Equals(expectedRes), "%v != %v\n", expectedRes, vs)
}
func TestIntersectionIterator(t *testing.T) {
assert := assert.New(t)
byTwos := NewSet(generateNumbersAsValuesFromToBy(0, 200, 2)...)
byThrees := NewSet(generateNumbersAsValuesFromToBy(0, 200, 3)...)
byFives := NewSet(generateNumbersAsValuesFromToBy(0, 200, 5)...)
vs := newTestValueStore()
byTwos := NewSet(vs, generateNumbersAsValuesFromToBy(0, 200, 2)...)
byThrees := NewSet(vs, generateNumbersAsValuesFromToBy(0, 200, 3)...)
byFives := NewSet(vs, generateNumbersAsValuesFromToBy(0, 200, 5)...)
i1 := NewIntersectionIterator(byTwos.Iterator(), byThrees.Iterator())
vs := iterToSlice(i1)
vals := iterToSlice(i1)
expectedRes := generateNumbersAsValuesFromToBy(0, 200, 6)
assert.True(vs.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
assert.True(vals.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
it1 := NewIntersectionIterator(byTwos.Iterator(), byThrees.Iterator())
it2 := NewIntersectionIterator(it1, byFives.Iterator())
vs = iterToSlice(it2)
vals = iterToSlice(it2)
expectedRes = generateNumbersAsValuesFromToBy(0, 200, 30)
assert.True(vs.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
assert.True(vals.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
it1 = NewIntersectionIterator(byThrees.Iterator(), byFives.Iterator())
it2 = NewIntersectionIterator(it1, byTwos.Iterator())
@@ -184,24 +194,26 @@ func TestIntersectionIterator(t *testing.T) {
func TestCombinationIterator(t *testing.T) {
assert := assert.New(t)
byTwos := NewSet(generateNumbersAsValuesFromToBy(0, 70, 2)...)
byThrees := NewSet(generateNumbersAsValuesFromToBy(0, 70, 3)...)
byFives := NewSet(generateNumbersAsValuesFromToBy(0, 70, 5)...)
bySevens := NewSet(generateNumbersAsValuesFromToBy(0, 70, 7)...)
vs := newTestValueStore()
byTwos := NewSet(vs, generateNumbersAsValuesFromToBy(0, 70, 2)...)
byThrees := NewSet(vs, generateNumbersAsValuesFromToBy(0, 70, 3)...)
byFives := NewSet(vs, generateNumbersAsValuesFromToBy(0, 70, 5)...)
bySevens := NewSet(vs, generateNumbersAsValuesFromToBy(0, 70, 7)...)
it1 := NewIntersectionIterator(byTwos.Iterator(), bySevens.Iterator())
it2 := NewIntersectionIterator(byFives.Iterator(), byThrees.Iterator())
ut1 := NewUnionIterator(it1, it2)
vs := iterToSlice(ut1)
vals := iterToSlice(ut1)
expectedRes := intsToValueSlice(0, 14, 15, 28, 30, 42, 45, 56, 60)
assert.True(vs.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
assert.True(vals.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
ut1 = NewUnionIterator(byTwos.Iterator(), bySevens.Iterator())
it2 = NewIntersectionIterator(byFives.Iterator(), byThrees.Iterator())
ut2 := NewIntersectionIterator(ut1, it2)
vs = iterToSlice(ut2)
vals = iterToSlice(ut2)
expectedRes = intsToValueSlice(0, 30, 60)
assert.True(vs.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
assert.True(vals.Equals(expectedRes), "Expected: %v != actual: %v", expectedRes, vs)
}
type UnionTestIterator struct {
@@ -230,6 +242,8 @@ func NewUnionTestIterator(i1, i2 SetIterator, cntr *int) SetIterator {
func TestUnionComplexity(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
numSets := 256
numElemsPerSet := 1000
logNumSets := int(math.Ceil(math.Log2(float64(numSets))))
@@ -237,17 +251,17 @@ func TestUnionComplexity(t *testing.T) {
expectedMax := logNumSets*totalElems + numSets
callCount1 := 0
iter := iterize(createSetsWithDistinctNumbers(numSets, numElemsPerSet), NewUnionTestIterator, &callCount1)
vs := iterToSlice(iter)
iter := iterize(createSetsWithDistinctNumbers(vs, numSets, numElemsPerSet), NewUnionTestIterator, &callCount1)
vals := iterToSlice(iter)
expected := generateNumbersAsValueSlice(numSets * numElemsPerSet)
assert.True(expected.Equals(vs), "expected: %v != actual: %v", expected, vs)
assert.True(expected.Equals(vals), "expected: %v != actual: %v", expected, vals)
assert.True(expectedMax > callCount1, "callCount: %d exceeds expectedMax: %d", callCount1, expectedMax)
callCount2 := 0
iter = iterize(createSetsWithSameNumbers(numSets, numElemsPerSet), NewUnionTestIterator, &callCount2)
vs = iterToSlice(iter)
iter = iterize(createSetsWithSameNumbers(vs, numSets, numElemsPerSet), NewUnionTestIterator, &callCount2)
vals = iterToSlice(iter)
expected = generateNumbersAsValueSlice(numElemsPerSet)
assert.True(expected.Equals(vs), "expected: %v != actual: %v", expected, vs)
assert.True(expected.Equals(vals), "expected: %v != actual: %v", expected, vals)
assert.True(expectedMax > callCount2, "callCount: %d exceeds expectedMax: %d", callCount2, expectedMax)
}
@@ -277,6 +291,8 @@ func NewIntersectionTestIterator(i1, i2 SetIterator, cntr *int) SetIterator {
func TestIntersectComplexity(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
numSets := 256
numElemsPerSet := 1000
logNumSets := int(math.Ceil(math.Log2(float64(numSets))))
@@ -284,41 +300,42 @@ func TestIntersectComplexity(t *testing.T) {
expectedMax := logNumSets*totalElems + numSets
callCount1 := 0
iter := iterize(createSetsWithDistinctNumbers(numSets, numElemsPerSet), NewIntersectionTestIterator, &callCount1)
vs := iterToSlice(iter)
iter := iterize(createSetsWithDistinctNumbers(vs, numSets, numElemsPerSet), NewIntersectionTestIterator, &callCount1)
vals := iterToSlice(iter)
expected := ValueSlice{}
assert.True(expected.Equals(vs), "expected: %v != actual: %v", expected, vs)
assert.True(expected.Equals(vals), "expected: %v != actual: %v", expected, vals)
assert.True(expectedMax > callCount1, "callCount: %d exceeds expectedMax: %d", callCount1, expectedMax)
callCount2 := 0
iter = iterize(createSetsWithSameNumbers(numSets, numElemsPerSet), NewIntersectionTestIterator, &callCount2)
vs = iterToSlice(iter)
iter = iterize(createSetsWithSameNumbers(vs, numSets, numElemsPerSet), NewIntersectionTestIterator, &callCount2)
vals = iterToSlice(iter)
expected = generateNumbersAsValueSlice(numElemsPerSet)
assert.True(expected.Equals(vs), "expected: %v != actual: %v", expected, vs)
assert.True(expected.Equals(vals), "expected: %v != actual: %v", expected, vals)
assert.True(expectedMax > callCount2, "callCount: %d exceeds expectedMax: %d", callCount2, expectedMax)
}
func createSetsWithDistinctNumbers(numSets, numElemsPerSet int) []SetIterator {
func createSetsWithDistinctNumbers(vrw ValueReadWriter, numSets, numElemsPerSet int) []SetIterator {
iterSlice := []SetIterator{}
for i := 0; i < numSets; i++ {
vs := ValueSlice{}
vals := ValueSlice{}
for j := 0; j < numElemsPerSet; j++ {
vs = append(vs, Number(i+(numSets*j)))
vals = append(vals, Number(i+(numSets*j)))
}
s := NewSet(vs...)
s := NewSet(vrw, vals...)
iterSlice = append(iterSlice, s.Iterator())
}
return iterSlice
}
func createSetsWithSameNumbers(numSets, numElemsPerSet int) []SetIterator {
func createSetsWithSameNumbers(vrw ValueReadWriter, numSets, numElemsPerSet int) []SetIterator {
vs := ValueSlice{}
for j := 0; j < numElemsPerSet; j++ {
vs = append(vs, Number(j))
}
iterSlice := []SetIterator{}
for i := 0; i < numSets; i++ {
iterSlice = append(iterSlice, NewSet(vs...).Iterator())
iterSlice = append(iterSlice, NewSet(vrw, vs...).Iterator())
}
return iterSlice
}
go/types/set_leaf_sequence.go
+5 -2
View File
@@ -4,13 +4,16 @@
package types
import "github.com/attic-labs/noms/go/d"
type setLeafSequence struct {
leafSequence
data []Value // sorted by Hash()
}
func newSetLeafSequence(vr ValueReader, v ...Value) orderedSequence {
return setLeafSequence{leafSequence{vr, len(v), SetKind}, v}
func newSetLeafSequence(vrw ValueReadWriter, v ...Value) orderedSequence {
d.PanicIfTrue(vrw == nil)
return setLeafSequence{leafSequence{vrw, len(v), SetKind}, v}
}
// sequence interface
go/types/set_test.go
+172 -117
View File
@@ -69,8 +69,8 @@ func (ts testSet) Diff(last testSet) (added []Value, removed []Value) {
return
}
func (ts testSet) toSet() Set {
return NewSet(ts...)
func (ts testSet) toSet(vrw ValueReadWriter) Set {
return NewSet(vrw, ts...)
}
func newSortedTestSet(length int, gen genValueFn) (values testSet) {
@@ -104,8 +104,8 @@ func newRandomTestSet(length int, gen genValueFn) testSet {
return values
}
func validateSet(t *testing.T, s Set, values ValueSlice) {
assert.True(t, s.Equals(NewSet(values...)))
func validateSet(t *testing.T, vrw ValueReadWriter, s Set, values ValueSlice) {
assert.True(t, s.Equals(NewSet(vrw, values...)))
out := ValueSlice{}
s.IterAll(func(v Value) {
out = append(out, v)
@@ -119,11 +119,13 @@ type setTestSuite struct {
}
func newSetTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int, expectAppendChunkDiff int, gen genValueFn) *setTestSuite {
vs := newTestValueStore()
length := 1 << size
elemType := TypeOf(gen(0))
elems := newSortedTestSet(length, gen)
tr := MakeSetType(elemType)
set := NewSet(elems...)
set := NewSet(vs, elems...)
return &setTestSuite{
collectionTestSuite: collectionTestSuite{
col: set,
@@ -149,13 +151,13 @@ func newSetTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int
dup := make([]Value, length+1)
dup[0] = Number(-1)
copy(dup[1:], elems)
return NewSet(dup...)
return NewSet(vs, dup...)
},
appendOne: func() Collection {
dup := make([]Value, length+1)
copy(dup, elems)
dup[len(dup)-1] = Number(length + 1)
return NewSet(dup...)
return NewSet(vs, dup...)
},
},
elems: elems,
@@ -253,9 +255,9 @@ func getTestRefToNativeOrderSet(scale int, vw ValueWriter) testSet {
})
}
func getTestRefToValueOrderSet(scale int, vw ValueWriter) testSet {
func getTestRefToValueOrderSet(scale int, vrw ValueReadWriter) testSet {
return newRandomTestSet(64*scale, func(v int) Value {
return vw.WriteValue(NewSet(Number(v)))
return vrw.WriteValue(NewSet(vrw, Number(v)))
})
}
@@ -293,37 +295,43 @@ func diffSetTest(assert *assert.Assertions, s1 Set, s2 Set, numAddsExpected int,
func TestNewSet(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
assert.True(MakeSetType(MakeUnionType()).Equals(TypeOf(s)))
assert.Equal(uint64(0), s.Len())
s = NewSet(Number(0))
s = NewSet(vs, Number(0))
assert.True(MakeSetType(NumberType).Equals(TypeOf(s)))
s = NewSet()
s = NewSet(vs)
assert.IsType(MakeSetType(NumberType), TypeOf(s))
s2 := s.Edit().Remove(Number(1)).Set(nil)
s2 := s.Edit().Remove(Number(1)).Set()
assert.IsType(TypeOf(s), TypeOf(s2))
}
func TestSetLen(t *testing.T) {
assert := assert.New(t)
s0 := NewSet()
vs := newTestValueStore()
s0 := NewSet(vs)
assert.Equal(uint64(0), s0.Len())
s1 := NewSet(Bool(true), Number(1), String("hi"))
s1 := NewSet(vs, Bool(true), Number(1), String("hi"))
assert.Equal(uint64(3), s1.Len())
diffSetTest(assert, s0, s1, 0, 3)
diffSetTest(assert, s1, s0, 3, 0)
s2 := s1.Edit().Insert(Bool(false)).Set(nil)
s2 := s1.Edit().Insert(Bool(false)).Set()
assert.Equal(uint64(4), s2.Len())
diffSetTest(assert, s0, s2, 0, 4)
diffSetTest(assert, s2, s0, 4, 0)
diffSetTest(assert, s1, s2, 0, 1)
diffSetTest(assert, s2, s1, 1, 0)
s3 := s2.Edit().Remove(Bool(true)).Set(nil)
s3 := s2.Edit().Remove(Bool(true)).Set()
assert.Equal(uint64(3), s3.Len())
diffSetTest(assert, s2, s3, 1, 0)
diffSetTest(assert, s3, s2, 0, 1)
@@ -331,28 +339,34 @@ func TestSetLen(t *testing.T) {
func TestSetEmpty(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
assert.True(s.Empty())
assert.Equal(uint64(0), s.Len())
}
func TestSetEmptyInsert(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
assert.True(s.Empty())
s = s.Edit().Insert(Bool(false)).Set(nil)
s = s.Edit().Insert(Bool(false)).Set()
assert.False(s.Empty())
assert.Equal(uint64(1), s.Len())
}
func TestSetEmptyInsertRemove(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
assert.True(s.Empty())
s = s.Edit().Insert(Bool(false)).Set(nil)
s = s.Edit().Insert(Bool(false)).Set()
assert.False(s.Empty())
assert.Equal(uint64(1), s.Len())
s = s.Edit().Remove(Bool(false)).Set(nil)
s = s.Edit().Remove(Bool(false)).Set()
assert.True(s.Empty())
assert.Equal(uint64(0), s.Len())
}
@@ -360,13 +374,17 @@ func TestSetEmptyInsertRemove(t *testing.T) {
// BUG 98
func TestSetDuplicateInsert(t *testing.T) {
assert := assert.New(t)
s1 := NewSet(Bool(true), Number(42), Number(42))
vs := newTestValueStore()
s1 := NewSet(vs, Bool(true), Number(42), Number(42))
assert.Equal(uint64(2), s1.Len())
}
func TestSetUniqueKeysString(t *testing.T) {
assert := assert.New(t)
s1 := NewSet(String("hello"), String("world"), String("hello"))
vs := newTestValueStore()
s1 := NewSet(vs, String("hello"), String("world"), String("hello"))
assert.Equal(uint64(2), s1.Len())
assert.True(s1.Has(String("hello")))
assert.True(s1.Has(String("world")))
@@ -375,7 +393,9 @@ func TestSetUniqueKeysString(t *testing.T) {
func TestSetUniqueKeysNumber(t *testing.T) {
assert := assert.New(t)
s1 := NewSet(Number(4), Number(1), Number(0), Number(0), Number(1), Number(3))
vs := newTestValueStore()
s1 := NewSet(vs, Number(4), Number(1), Number(0), Number(0), Number(1), Number(3))
assert.Equal(uint64(4), s1.Len())
assert.True(s1.Has(Number(4)))
assert.True(s1.Has(Number(1)))
@@ -386,7 +406,9 @@ func TestSetUniqueKeysNumber(t *testing.T) {
func TestSetHas(t *testing.T) {
assert := assert.New(t)
s1 := NewSet(Bool(true), Number(1), String("hi"))
vs := newTestValueStore()
s1 := NewSet(vs, Bool(true), Number(1), String("hi"))
assert.True(s1.Has(Bool(true)))
assert.False(s1.Has(Bool(false)))
assert.True(s1.Has(Number(1)))
@@ -394,7 +416,7 @@ func TestSetHas(t *testing.T) {
assert.True(s1.Has(String("hi")))
assert.False(s1.Has(String("ho")))
s2 := s1.Edit().Insert(Bool(false)).Set(nil)
s2 := s1.Edit().Insert(Bool(false)).Set()
assert.True(s2.Has(Bool(false)))
assert.True(s2.Has(Bool(true)))
@@ -411,10 +433,10 @@ func TestSetHas2(t *testing.T) {
defer normalProductionChunks()
assert := assert.New(t)
vs := newTestValueStore()
doTest := func(ts testSet) {
set := ts.toSet()
set := ts.toSet(vs)
set2 := vs.ReadValue(vs.WriteValue(set).TargetHash()).(Set)
for _, v := range ts {
assert.True(set.Has(v))
@@ -429,11 +451,11 @@ func TestSetHas2(t *testing.T) {
doTest(getTestRefToValueOrderSet(2, vs))
}
func validateSetInsertion(t *testing.T, values ValueSlice) {
s := NewSet()
func validateSetInsertion(t *testing.T, vrw ValueReadWriter, values ValueSlice) {
s := NewSet(vrw)
for i, v := range values {
s = s.Edit().Insert(v).Set(nil)
validateSet(t, s, values[0:i+1])
s = s.Edit().Insert(v).Set()
validateSet(t, vrw, s, values[0:i+1])
}
}
@@ -445,23 +467,27 @@ func TestSetValidateInsertAscending(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
validateSetInsertion(t, generateNumbersAsValues(300))
vs := newTestValueStore()
validateSetInsertion(t, vs, generateNumbersAsValues(300))
}
func TestSetInsert(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
v1 := Bool(false)
v2 := Bool(true)
v3 := Number(0)
assert.False(s.Has(v1))
s = s.Edit().Insert(v1).Set(nil)
s = s.Edit().Insert(v1).Set()
assert.True(s.Has(v1))
s = s.Edit().Insert(v2).Set(nil)
s = s.Edit().Insert(v2).Set()
assert.True(s.Has(v1))
assert.True(s.Has(v2))
s2 := s.Edit().Insert(v3).Set(nil)
s2 := s.Edit().Insert(v3).Set()
assert.True(s.Has(v1))
assert.True(s.Has(v2))
assert.False(s.Has(v3))
@@ -479,11 +505,12 @@ func TestSetInsert2(t *testing.T) {
defer normalProductionChunks()
assert := assert.New(t)
vs := newTestValueStore()
doTest := func(incr, offset int, ts testSet) {
expected := ts.toSet()
expected := ts.toSet(vs)
run := func(from, to int) {
actual := ts.Remove(from, to).toSet().Edit().Insert(ts[from:to]...).Set(nil)
actual := ts.Remove(from, to).toSet(vs).Edit().Insert(ts[from:to]...).Set()
assert.Equal(expected.Len(), actual.Len())
assert.True(expected.Equals(actual))
diffSetTest(assert, expected, actual, 0, 0)
@@ -507,9 +534,11 @@ func TestSetInsertExistingValue(t *testing.T) {
smallTestChunks()
defer normalProductionChunks()
vs := newTestValueStore()
ts := getTestNativeOrderSet(2)
original := ts.toSet()
actual := original.Edit().Insert(ts[0]).Set(nil)
original := ts.toSet(vs)
actual := original.Edit().Insert(ts[0]).Set()
assert.Equal(original.Len(), actual.Len())
assert.True(original.Equals(actual))
@@ -517,18 +546,20 @@ func TestSetInsertExistingValue(t *testing.T) {
func TestSetRemove(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
v1 := Bool(false)
v2 := Bool(true)
v3 := Number(0)
s := NewSet(v1, v2, v3)
s := NewSet(vs, v1, v2, v3)
assert.True(s.Has(v1))
assert.True(s.Has(v2))
assert.True(s.Has(v3))
s = s.Edit().Remove(v1).Set(nil)
s = s.Edit().Remove(v1).Set()
assert.False(s.Has(v1))
assert.True(s.Has(v2))
assert.True(s.Has(v3))
s2 := s.Edit().Remove(v2).Set(nil)
s2 := s.Edit().Remove(v2).Set()
assert.False(s.Has(v1))
assert.True(s.Has(v2))
assert.True(s.Has(v3))
@@ -546,12 +577,13 @@ func TestSetRemove2(t *testing.T) {
defer normalProductionChunks()
assert := assert.New(t)
vs := newTestValueStore()
doTest := func(incr, offset int, ts testSet) {
whole := ts.toSet()
whole := ts.toSet(vs)
run := func(from, to int) {
expected := ts.Remove(from, to).toSet()
actual := whole.Edit().Remove(ts[from:to]...).Set(nil)
expected := ts.Remove(from, to).toSet(vs)
actual := whole.Edit().Remove(ts[from:to]...).Set()
assert.Equal(expected.Len(), actual.Len())
assert.True(expected.Equals(actual))
diffSetTest(assert, expected, actual, 0, 0)
@@ -571,10 +603,11 @@ func TestSetRemove2(t *testing.T) {
func TestSetRemoveNonexistentValue(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
ts := getTestNativeOrderSet(2)
original := ts.toSet()
actual := original.Edit().Remove(Number(-1)).Set(nil) // rand.Int63 returns non-negative values.
original := ts.toSet(vs)
actual := original.Edit().Remove(Number(-1)).Set() // rand.Int63 returns non-negative values.
assert.Equal(original.Len(), actual.Len())
assert.True(original.Equals(actual))
@@ -582,27 +615,30 @@ func TestSetRemoveNonexistentValue(t *testing.T) {
func TestSetFirst(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
assert.Nil(s.First())
s = s.Edit().Insert(Number(1)).Set(nil)
s = s.Edit().Insert(Number(1)).Set()
assert.NotNil(s.First())
s = s.Edit().Insert(Number(2)).Set(nil)
s = s.Edit().Insert(Number(2)).Set()
assert.NotNil(s.First())
s2 := s.Edit().Remove(Number(1)).Set(nil)
s2 := s.Edit().Remove(Number(1)).Set()
assert.NotNil(s2.First())
s2 = s2.Edit().Remove(Number(2)).Set(nil)
s2 = s2.Edit().Remove(Number(2)).Set()
assert.Nil(s2.First())
}
func TestSetOfStruct(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
elems := []Value{}
for i := 0; i < 200; i++ {
elems = append(elems, NewStruct("S1", StructData{"o": Number(i)}))
}
s := NewSet(elems...)
s := NewSet(vs, elems...)
for i := 0; i < 200; i++ {
assert.True(s.Has(elems[i]))
}
@@ -610,17 +646,19 @@ func TestSetOfStruct(t *testing.T) {
func TestSetIter(t *testing.T) {
assert := assert.New(t)
s := NewSet(Number(0), Number(1), Number(2), Number(3), Number(4))
acc := NewSet()
vs := newTestValueStore()
s := NewSet(vs, Number(0), Number(1), Number(2), Number(3), Number(4))
acc := NewSet(vs)
s.Iter(func(v Value) bool {
_, ok := v.(Number)
assert.True(ok)
acc = acc.Edit().Insert(v).Set(nil)
acc = acc.Edit().Insert(v).Set()
return false
})
assert.True(s.Equals(acc))
acc = NewSet()
acc = NewSet(vs)
s.Iter(func(v Value) bool {
return true
})
@@ -629,12 +667,13 @@ func TestSetIter(t *testing.T) {
func TestSetIter2(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
smallTestChunks()
defer normalProductionChunks()
doTest := func(ts testSet) {
set := ts.toSet()
set := ts.toSet(vs)
sort.Sort(ValueSlice(ts))
idx := uint64(0)
endAt := uint64(64)
@@ -659,24 +698,27 @@ func TestSetIter2(t *testing.T) {
func TestSetIterAll(t *testing.T) {
assert := assert.New(t)
s := NewSet(Number(0), Number(1), Number(2), Number(3), Number(4))
acc := NewSet()
vs := newTestValueStore()
s := NewSet(vs, Number(0), Number(1), Number(2), Number(3), Number(4))
acc := NewSet(vs)
s.IterAll(func(v Value) {
_, ok := v.(Number)
assert.True(ok)
acc = acc.Edit().Insert(v).Set(nil)
acc = acc.Edit().Insert(v).Set()
})
assert.True(s.Equals(acc))
}
func TestSetIterAll2(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
smallTestChunks()
defer normalProductionChunks()
doTest := func(ts testSet) {
set := ts.toSet()
set := ts.toSet(vs)
sort.Sort(ValueSlice(ts))
idx := uint64(0)
@@ -692,8 +734,8 @@ func TestSetIterAll2(t *testing.T) {
doTest(getTestRefToValueOrderSet(2, newTestValueStore()))
}
func testSetOrder(assert *assert.Assertions, valueType *Type, value []Value, expectOrdering []Value) {
m := NewSet(value...)
func testSetOrder(assert *assert.Assertions, vrw ValueReadWriter, valueType *Type, value []Value, expectOrdering []Value) {
m := NewSet(vrw, value...)
i := 0
m.IterAll(func(value Value) {
assert.Equal(expectOrdering[i].Hash().String(), value.Hash().String())
@@ -703,8 +745,9 @@ func testSetOrder(assert *assert.Assertions, valueType *Type, value []Value, exp
func TestSetOrdering(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
testSetOrder(assert,
testSetOrder(assert, vs,
StringType,
[]Value{
String("a"),
@@ -724,7 +767,7 @@ func TestSetOrdering(t *testing.T) {
},
)
testSetOrder(assert,
testSetOrder(assert, vs,
NumberType,
[]Value{
Number(0),
@@ -744,7 +787,7 @@ func TestSetOrdering(t *testing.T) {
},
)
testSetOrder(assert,
testSetOrder(assert, vs,
NumberType,
[]Value{
Number(0),
@@ -764,7 +807,7 @@ func TestSetOrdering(t *testing.T) {
},
)
testSetOrder(assert,
testSetOrder(assert, vs,
NumberType,
[]Value{
Number(0.0001),
@@ -784,7 +827,7 @@ func TestSetOrdering(t *testing.T) {
},
)
testSetOrder(assert,
testSetOrder(assert, vs,
ValueType,
[]Value{
String("a"),
@@ -805,7 +848,7 @@ func TestSetOrdering(t *testing.T) {
},
)
testSetOrder(assert,
testSetOrder(assert, vs,
BoolType,
[]Value{
Bool(true),
@@ -822,32 +865,35 @@ func TestSetOrdering(t *testing.T) {
func TestSetType(t *testing.T) {
assert := assert.New(t)
s := NewSet()
vs := newTestValueStore()
s := NewSet(vs)
assert.True(TypeOf(s).Equals(MakeSetType(MakeUnionType())))
s = NewSet(Number(0))
s = NewSet(vs, Number(0))
assert.True(TypeOf(s).Equals(MakeSetType(NumberType)))
s2 := s.Edit().Remove(Number(1)).Set(nil)
s2 := s.Edit().Remove(Number(1)).Set()
assert.True(TypeOf(s2).Equals(MakeSetType(NumberType)))
s2 = s.Edit().Insert(Number(0), Number(1)).Set(nil)
s2 = s.Edit().Insert(Number(0), Number(1)).Set()
assert.True(TypeOf(s).Equals(TypeOf(s2)))
s3 := s.Edit().Insert(Bool(true)).Set(nil)
s3 := s.Edit().Insert(Bool(true)).Set()
assert.True(TypeOf(s3).Equals(MakeSetType(MakeUnionType(BoolType, NumberType))))
s4 := s.Edit().Insert(Number(3), Bool(true)).Set(nil)
s4 := s.Edit().Insert(Number(3), Bool(true)).Set()
assert.True(TypeOf(s4).Equals(MakeSetType(MakeUnionType(BoolType, NumberType))))
}
func TestSetChunks(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
l1 := NewSet(Number(0))
l1 := NewSet(vs, Number(0))
c1 := getChunks(l1)
assert.Len(c1, 0)
l2 := NewSet(NewRef(Number(0)))
l2 := NewSet(vs, NewRef(Number(0)))
c2 := getChunks(l2)
assert.Len(c2, 1)
}
@@ -860,7 +906,7 @@ func TestSetChunks2(t *testing.T) {
vs := newTestValueStore()
doTest := func(ts testSet) {
set := ts.toSet()
set := ts.toSet(vs)
set2chunks := getChunks(vs.ReadValue(vs.WriteValue(set).TargetHash()))
for i, r := range getChunks(set) {
assert.True(TypeOf(r).Equals(TypeOf(set2chunks[i])), "%s != %s", TypeOf(r).Describe(), TypeOf(set2chunks[i]).Describe())
@@ -875,9 +921,10 @@ func TestSetChunks2(t *testing.T) {
func TestSetFirstNNumbers(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
nums := generateNumbersAsValues(testSetSize)
s := NewSet(nums...)
s := NewSet(vs, nums...)
assert.Equal(deriveCollectionHeight(s), getRefHeightOfCollection(s))
}
@@ -886,9 +933,10 @@ func TestSetRefOfStructFirstNNumbers(t *testing.T) {
t.Skip("Skipping test in short mode.")
}
assert := assert.New(t)
vs := newTestValueStore()
nums := generateNumbersAsRefOfStructs(testSetSize)
s := NewSet(nums...)
s := NewSet(vs, nums...)
// height + 1 because the leaves are Ref values (with height 1).
assert.Equal(deriveCollectionHeight(s)+1, getRefHeightOfCollection(s))
}
@@ -899,20 +947,21 @@ func TestSetModifyAfterRead(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
set := getTestNativeOrderSet(2).toSet()
set := getTestNativeOrderSet(2).toSet(vs)
// Drop chunk values.
set = vs.ReadValue(vs.WriteValue(set).TargetHash()).(Set)
// Modify/query. Once upon a time this would crash.
fst := set.First()
set = set.Edit().Remove(fst).Set(nil)
set = set.Edit().Remove(fst).Set()
assert.False(set.Has(fst))
assert.True(set.Has(set.First()))
set = set.Edit().Insert(fst).Set(nil)
set = set.Edit().Insert(fst).Set()
assert.True(set.Has(fst))
}
func TestSetTypeAfterMutations(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
smallTestChunks()
defer normalProductionChunks()
@@ -920,17 +969,17 @@ func TestSetTypeAfterMutations(t *testing.T) {
test := func(n int, c interface{}) {
values := generateNumbersAsValues(n)
s := NewSet(values...)
s := NewSet(vs, values...)
assert.Equal(s.Len(), uint64(n))
assert.IsType(c, s.sequence())
assert.True(TypeOf(s).Equals(MakeSetType(NumberType)))
s = s.Edit().Insert(String("a")).Set(nil)
s = s.Edit().Insert(String("a")).Set()
assert.Equal(s.Len(), uint64(n+1))
assert.IsType(c, s.sequence())
assert.True(TypeOf(s).Equals(MakeSetType(MakeUnionType(NumberType, StringType))))
s = s.Edit().Remove(String("a")).Set(nil)
s = s.Edit().Remove(String("a")).Set()
assert.Equal(s.Len(), uint64(n))
assert.IsType(c, s.sequence())
assert.True(TypeOf(s).Equals(MakeSetType(NumberType)))
@@ -942,51 +991,52 @@ func TestSetTypeAfterMutations(t *testing.T) {
func TestChunkedSetWithValuesOfEveryType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
smallTestChunks()
defer normalProductionChunks()
vs := []Value{
vals := []Value{
// Values
Bool(true),
Number(0),
String("hello"),
NewBlob(bytes.NewBufferString("buf")),
NewSet(Bool(true)),
NewList(Bool(true)),
NewMap(Bool(true), Number(0)),
NewBlob(vs, bytes.NewBufferString("buf")),
NewSet(vs, Bool(true)),
NewList(vs, Bool(true)),
NewMap(vs, Bool(true), Number(0)),
NewStruct("", StructData{"field": Bool(true)}),
// Refs of values
NewRef(Bool(true)),
NewRef(Number(0)),
NewRef(String("hello")),
NewRef(NewBlob(bytes.NewBufferString("buf"))),
NewRef(NewSet(Bool(true))),
NewRef(NewList(Bool(true))),
NewRef(NewMap(Bool(true), Number(0))),
NewRef(NewBlob(vs, bytes.NewBufferString("buf"))),
NewRef(NewSet(vs, Bool(true))),
NewRef(NewList(vs, Bool(true))),
NewRef(NewMap(vs, Bool(true), Number(0))),
NewRef(NewStruct("", StructData{"field": Bool(true)})),
}
s := NewSet(vs...)
s := NewSet(vs, vals...)
for i := 1; s.sequence().isLeaf(); i++ {
v := Number(i)
vs = append(vs, v)
s = s.Edit().Insert(v).Set(nil)
vals = append(vals, v)
s = s.Edit().Insert(v).Set()
}
assert.Equal(len(vs), int(s.Len()))
assert.Equal(len(vals), int(s.Len()))
assert.True(bool(s.First().(Bool)))
for _, v := range vs {
for _, v := range vals {
assert.True(s.Has(v))
}
for len(vs) > 0 {
v := vs[0]
vs = vs[1:]
s = s.Edit().Remove(v).Set(nil)
for len(vals) > 0 {
v := vals[0]
vals = vals[1:]
s = s.Edit().Remove(v).Set()
assert.False(s.Has(v))
assert.Equal(len(vs), int(s.Len()))
assert.Equal(len(vals), int(s.Len()))
}
}
@@ -1002,21 +1052,22 @@ func TestSetRemoveLastWhenNotLoaded(t *testing.T) {
}
ts := getTestNativeOrderSet(8)
ns := ts.toSet()
ns := ts.toSet(vs)
for len(ts) > 0 {
last := ts[len(ts)-1]
ts = ts[:len(ts)-1]
ns = reload(ns.Edit().Remove(last).Set(nil))
assert.True(ts.toSet().Equals(ns))
ns = reload(ns.Edit().Remove(last).Set())
assert.True(ts.toSet(vs).Equals(ns))
}
}
func TestSetAt(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
values := []Value{Bool(false), Number(42), String("a"), String("b"), String("c")}
s := NewSet(values...)
s := NewSet(vs, values...)
for i, v := range values {
assert.Equal(v, s.At(uint64(i)))
@@ -1029,7 +1080,9 @@ func TestSetAt(t *testing.T) {
func TestSetWithStructShouldHaveOptionalFields(t *testing.T) {
assert := assert.New(t)
list := NewSet(
vs := newTestValueStore()
list := NewSet(vs,
NewStruct("Foo", StructData{
"a": Number(1),
}),
@@ -1047,10 +1100,12 @@ func TestSetWithStructShouldHaveOptionalFields(t *testing.T) {
}
func TestSetWithNil(t *testing.T) {
vs := newTestValueStore()
assert.Panics(t, func() {
NewSet(nil)
NewSet(vs, nil)
})
assert.Panics(t, func() {
NewSet(Number(42), nil)
NewSet(vs, Number(42), nil)
})
}
go/types/string.go
+1 -1
View File
@@ -10,7 +10,7 @@ import "github.com/attic-labs/noms/go/hash"
type String string
// Value interface
func (s String) Value(vrw ValueReadWriter) Value {
func (s String) Value() Value {
return s
}
go/types/struct.go
+1 -1
View File
@@ -110,7 +110,7 @@ func (s Struct) hashPointer() *hash.Hash {
}
// Value interface
func (s Struct) Value(vrw ValueReadWriter) Value {
func (s Struct) Value() Value {
return s
}
go/types/struct_test.go
+25 -23
View File
@@ -54,6 +54,7 @@ func TestGenericStructNew(t *testing.T) {
func TestGenericStructSet(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
s := NewStruct("S3", StructData{"b": Bool(true), "o": String("hi")})
s2 := s.Set("b", Bool(false))
@@ -77,8 +78,8 @@ func TestGenericStructSet(t *testing.T) {
).Equals(TypeOf(s5)))
// Subtype is not equal.
s6 := NewStruct("", StructData{"l": NewList(Number(0), Number(1), Bool(false), Bool(true))})
s7 := s6.Set("l", NewList(Number(2), Number(3)))
s6 := NewStruct("", StructData{"l": NewList(vs, Number(0), Number(1), Bool(false), Bool(true))})
s7 := s6.Set("l", NewList(vs, Number(2), Number(3)))
t7 := MakeStructTypeFromFields("", FieldMap{
"l": MakeListType(NumberType),
})
@@ -119,6 +120,7 @@ func assertValueChangeEqual(assert *assert.Assertions, c1, c2 ValueChanged) {
func TestStructDiff(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
assertDiff := func(expect []ValueChanged, s1, s2 Struct) {
changes := make(chan ValueChanged)
@@ -165,46 +167,46 @@ func TestStructDiff(t *testing.T) {
s1, NewStruct("NewType", StructData{"a": Bool(true), "c": Number(4), "d": Number(5)}))
s2 := NewStruct("", StructData{
"a": NewList(Number(0), Number(1)),
"b": NewMap(String("foo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(Number(0), Number(1), String("foo")),
"a": NewList(vs, Number(0), Number(1)),
"b": NewMap(vs, String("foo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(vs, Number(0), Number(1), String("foo")),
})
assertDiff([]ValueChanged{},
s2, NewStruct("", StructData{
"a": NewList(Number(0), Number(1)),
"b": NewMap(String("foo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(Number(0), Number(1), String("foo")),
"a": NewList(vs, Number(0), Number(1)),
"b": NewMap(vs, String("foo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(vs, Number(0), Number(1), String("foo")),
}))
assertDiff([]ValueChanged{
vc(DiffChangeModified, "a", NewList(Number(1), Number(1)), NewList(Number(0), Number(1))),
vc(DiffChangeModified, "b", NewMap(String("foo"), Bool(true), String("bar"), Bool(true)), NewMap(String("foo"), Bool(false), String("bar"), Bool(true))),
vc(DiffChangeModified, "a", NewList(vs, Number(1), Number(1)), NewList(vs, Number(0), Number(1))),
vc(DiffChangeModified, "b", NewMap(vs, String("foo"), Bool(true), String("bar"), Bool(true)), NewMap(vs, String("foo"), Bool(false), String("bar"), Bool(true))),
},
s2, NewStruct("", StructData{
"a": NewList(Number(1), Number(1)),
"b": NewMap(String("foo"), Bool(true), String("bar"), Bool(true)),
"c": NewSet(Number(0), Number(1), String("foo")),
"a": NewList(vs, Number(1), Number(1)),
"b": NewMap(vs, String("foo"), Bool(true), String("bar"), Bool(true)),
"c": NewSet(vs, Number(0), Number(1), String("foo")),
}))
assertDiff([]ValueChanged{
vc(DiffChangeModified, "a", NewList(Number(0)), NewList(Number(0), Number(1))),
vc(DiffChangeModified, "c", NewSet(Number(0), Number(2), String("foo")), NewSet(Number(0), Number(1), String("foo"))),
vc(DiffChangeModified, "a", NewList(vs, Number(0)), NewList(vs, Number(0), Number(1))),
vc(DiffChangeModified, "c", NewSet(vs, Number(0), Number(2), String("foo")), NewSet(vs, Number(0), Number(1), String("foo"))),
},
s2, NewStruct("", StructData{
"a": NewList(Number(0)),
"b": NewMap(String("foo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(Number(0), Number(2), String("foo")),
"a": NewList(vs, Number(0)),
"b": NewMap(vs, String("foo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(vs, Number(0), Number(2), String("foo")),
}))
assertDiff([]ValueChanged{
vc(DiffChangeModified, "b", NewMap(String("boo"), Bool(false), String("bar"), Bool(true)), NewMap(String("foo"), Bool(false), String("bar"), Bool(true))),
vc(DiffChangeModified, "c", NewSet(Number(0), Number(1), String("bar")), NewSet(Number(0), Number(1), String("foo"))),
vc(DiffChangeModified, "b", NewMap(vs, String("boo"), Bool(false), String("bar"), Bool(true)), NewMap(vs, String("foo"), Bool(false), String("bar"), Bool(true))),
vc(DiffChangeModified, "c", NewSet(vs, Number(0), Number(1), String("bar")), NewSet(vs, Number(0), Number(1), String("foo"))),
},
s2, NewStruct("", StructData{
"a": NewList(Number(0), Number(1)),
"b": NewMap(String("boo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(Number(0), Number(1), String("bar")),
"a": NewList(vs, Number(0), Number(1)),
"b": NewMap(vs, String("boo"), Bool(false), String("bar"), Bool(true)),
"c": NewSet(vs, Number(0), Number(1), String("bar")),
}))
}
go/types/subtype_test.go
+94 -70
View File
@@ -52,37 +52,47 @@ func TestAssertTypePrimitives(t *testing.T) {
}
func TestAssertTypeValue(t *testing.T) {
vs := newTestValueStore()
assertSubtype(ValueType, Bool(true))
assertSubtype(ValueType, Number(1))
assertSubtype(ValueType, String("abc"))
l := NewList(Number(0), Number(1), Number(2), Number(3))
l := NewList(vs, Number(0), Number(1), Number(2), Number(3))
assertSubtype(ValueType, l)
}
func TestAssertTypeBlob(t *testing.T) {
blob := NewBlob(bytes.NewBuffer([]byte{0x00, 0x01}))
vs := newTestValueStore()
blob := NewBlob(vs, bytes.NewBuffer([]byte{0x00, 0x01}))
assertAll(t, BlobType, blob)
}
func TestAssertTypeList(tt *testing.T) {
vs := newTestValueStore()
listOfNumberType := MakeListType(NumberType)
l := NewList(Number(0), Number(1), Number(2), Number(3))
l := NewList(vs, Number(0), Number(1), Number(2), Number(3))
assertSubtype(listOfNumberType, l)
assertAll(tt, listOfNumberType, l)
assertSubtype(MakeListType(ValueType), l)
}
func TestAssertTypeMap(tt *testing.T) {
vs := newTestValueStore()
mapOfNumberToStringType := MakeMapType(NumberType, StringType)
m := NewMap(Number(0), String("a"), Number(2), String("b"))
m := NewMap(vs, Number(0), String("a"), Number(2), String("b"))
assertSubtype(mapOfNumberToStringType, m)
assertAll(tt, mapOfNumberToStringType, m)
assertSubtype(MakeMapType(ValueType, ValueType), m)
}
func TestAssertTypeSet(tt *testing.T) {
vs := newTestValueStore()
setOfNumberType := MakeSetType(NumberType)
s := NewSet(Number(0), Number(1), Number(2), Number(3))
s := NewSet(vs, Number(0), Number(1), Number(2), Number(3))
assertSubtype(setOfNumberType, s)
assertAll(tt, setOfNumberType, s)
assertSubtype(MakeSetType(ValueType), s)
@@ -105,6 +115,8 @@ func TestAssertTypeStruct(tt *testing.T) {
}
func TestAssertTypeUnion(tt *testing.T) {
vs := newTestValueStore()
assertSubtype(MakeUnionType(NumberType), Number(42))
assertSubtype(MakeUnionType(NumberType, StringType), Number(42))
assertSubtype(MakeUnionType(NumberType, StringType), String("hi"))
@@ -113,17 +125,17 @@ func TestAssertTypeUnion(tt *testing.T) {
assertSubtype(MakeUnionType(NumberType, StringType, BoolType), Bool(true))
lt := MakeListType(MakeUnionType(NumberType, StringType))
assertSubtype(lt, NewList(Number(1), String("hi"), Number(2), String("bye")))
assertSubtype(lt, NewList(vs, Number(1), String("hi"), Number(2), String("bye")))
st := MakeSetType(StringType)
assertSubtype(MakeUnionType(st, NumberType), Number(42))
assertSubtype(MakeUnionType(st, NumberType), NewSet(String("a"), String("b")))
assertSubtype(MakeUnionType(st, NumberType), NewSet(vs, String("a"), String("b")))
assertInvalid(tt, MakeUnionType(), Number(42))
assertInvalid(tt, MakeUnionType(StringType), Number(42))
assertInvalid(tt, MakeUnionType(StringType, BoolType), Number(42))
assertInvalid(tt, MakeUnionType(st, StringType), Number(42))
assertInvalid(tt, MakeUnionType(st, NumberType), NewSet(Number(1), Number(2)))
assertInvalid(tt, MakeUnionType(st, NumberType), NewSet(vs, Number(1), Number(2)))
}
func TestAssertConcreteTypeIsUnion(tt *testing.T) {
@@ -156,32 +168,40 @@ func TestAssertConcreteTypeIsUnion(tt *testing.T) {
}
func TestAssertTypeEmptyListUnion(tt *testing.T) {
vs := newTestValueStore()
lt := MakeListType(MakeUnionType())
assertSubtype(lt, NewList())
assertSubtype(lt, NewList(vs))
}
func TestAssertTypeEmptyList(tt *testing.T) {
vs := newTestValueStore()
lt := MakeListType(NumberType)
assertSubtype(lt, NewList())
assertSubtype(lt, NewList(vs))
// List<> not a subtype of List<Number>
assertInvalid(tt, MakeListType(MakeUnionType()), NewList(Number(1)))
assertInvalid(tt, MakeListType(MakeUnionType()), NewList(vs, Number(1)))
}
func TestAssertTypeEmptySet(tt *testing.T) {
vs := newTestValueStore()
st := MakeSetType(NumberType)
assertSubtype(st, NewSet())
assertSubtype(st, NewSet(vs))
// Set<> not a subtype of Set<Number>
assertInvalid(tt, MakeSetType(MakeUnionType()), NewSet(Number(1)))
assertInvalid(tt, MakeSetType(MakeUnionType()), NewSet(vs, Number(1)))
}
func TestAssertTypeEmptyMap(tt *testing.T) {
vs := newTestValueStore()
mt := MakeMapType(NumberType, StringType)
assertSubtype(mt, NewMap())
assertSubtype(mt, NewMap(vs))
// Map<> not a subtype of Map<Number, Number>
assertInvalid(tt, MakeMapType(MakeUnionType(), MakeUnionType()), NewMap(Number(1), Number(2)))
assertInvalid(tt, MakeMapType(MakeUnionType(), MakeUnionType()), NewMap(vs, Number(1), Number(2)))
}
func TestAssertTypeStructSubtypeByName(tt *testing.T) {
@@ -222,9 +242,11 @@ func TestAssertTypeStructSubtypeExtraFields(tt *testing.T) {
}
func TestAssertTypeStructSubtype(tt *testing.T) {
vs := newTestValueStore()
c1 := NewStruct("Commit", StructData{
"value": Number(1),
"parents": NewSet(),
"parents": NewSet(vs),
})
t1 := MakeStructType("Commit",
StructField{"parents", MakeSetType(MakeUnionType()), false},
@@ -240,7 +262,7 @@ func TestAssertTypeStructSubtype(tt *testing.T) {
c2 := NewStruct("Commit", StructData{
"value": Number(2),
"parents": NewSet(NewRef(c1)),
"parents": NewSet(vs, NewRef(c1)),
})
assertSubtype(t11, c2)
}
@@ -498,6 +520,8 @@ func TestIsSubtypeDisallowExtraStructFields(tt *testing.T) {
func TestIsValueSubtypeOf(tt *testing.T) {
assert := assert.New(tt)
vs := newTestValueStore()
assertTrue := func(v Value, t *Type) {
assert.True(IsValueSubtypeOf(v, t))
}
@@ -513,12 +537,12 @@ func TestIsValueSubtypeOf(tt *testing.T) {
{Bool(true), BoolType},
{Number(42), NumberType},
{String("s"), StringType},
{NewEmptyBlob(), BlobType},
{NewEmptyBlob(vs), BlobType},
{BoolType, TypeType},
{NewList(Number(42)), MakeListType(NumberType)},
{NewSet(Number(42)), MakeSetType(NumberType)},
{NewList(vs, Number(42)), MakeListType(NumberType)},
{NewSet(vs, Number(42)), MakeSetType(NumberType)},
{NewRef(Number(42)), MakeRefType(NumberType)},
{NewMap(Number(42), String("a")), MakeMapType(NumberType, StringType)},
{NewMap(vs, Number(42), String("a")), MakeMapType(NumberType, StringType)},
{NewStruct("A", StructData{}), MakeStructType("A")},
// Not including CycleType or Union here
}
@@ -542,27 +566,27 @@ func TestIsValueSubtypeOf(tt *testing.T) {
assertFalse(String("abc"), MakeUnionType(BoolType, NumberType))
assertFalse(String("abc"), MakeUnionType())
assertTrue(NewList(), MakeListType(NumberType))
assertTrue(NewList(Number(0), Number(1), Number(2), Number(3)), MakeListType(NumberType))
assertFalse(NewList(Number(0), Number(1), Number(2), Number(3)), MakeListType(BoolType))
assertTrue(NewList(Number(0), Number(1), Number(2), Number(3)), MakeListType(MakeUnionType(NumberType, BoolType)))
assertTrue(NewList(Number(0), Bool(true)), MakeListType(MakeUnionType(NumberType, BoolType)))
assertFalse(NewList(Number(0)), MakeListType(MakeUnionType()))
assertTrue(NewList(), MakeListType(MakeUnionType()))
assertTrue(NewList(vs), MakeListType(NumberType))
assertTrue(NewList(vs, Number(0), Number(1), Number(2), Number(3)), MakeListType(NumberType))
assertFalse(NewList(vs, Number(0), Number(1), Number(2), Number(3)), MakeListType(BoolType))
assertTrue(NewList(vs, Number(0), Number(1), Number(2), Number(3)), MakeListType(MakeUnionType(NumberType, BoolType)))
assertTrue(NewList(vs, Number(0), Bool(true)), MakeListType(MakeUnionType(NumberType, BoolType)))
assertFalse(NewList(vs, Number(0)), MakeListType(MakeUnionType()))
assertTrue(NewList(vs), MakeListType(MakeUnionType()))
{
newChunkedList := func(vs ...Value) List {
newChunkedList := func(vals ...Value) List {
newSequenceMetaTuple := func(v Value) metaTuple {
seq := newListLeafSequence(nil, v)
seq := newListLeafSequence(vs, v)
list := newList(seq)
return newMetaTuple(NewRef(list), newOrderedKey(v), 1, list)
return newMetaTuple(vs.WriteValue(list), newOrderedKey(v), 1)
}
tuples := make([]metaTuple, len(vs))
for i, v := range vs {
tuples := make([]metaTuple, len(vals))
for i, v := range vals {
tuples[i] = newSequenceMetaTuple(v)
}
return newList(newListMetaSequence(1, tuples, nil))
return newList(newListMetaSequence(1, tuples, vs))
}
assertTrue(newChunkedList(Number(0), Number(1), Number(2), Number(3)), MakeListType(NumberType))
@@ -572,27 +596,27 @@ func TestIsValueSubtypeOf(tt *testing.T) {
assertFalse(newChunkedList(Number(0)), MakeListType(MakeUnionType()))
}
assertTrue(NewSet(), MakeSetType(NumberType))
assertTrue(NewSet(Number(0), Number(1), Number(2), Number(3)), MakeSetType(NumberType))
assertFalse(NewSet(Number(0), Number(1), Number(2), Number(3)), MakeSetType(BoolType))
assertTrue(NewSet(Number(0), Number(1), Number(2), Number(3)), MakeSetType(MakeUnionType(NumberType, BoolType)))
assertTrue(NewSet(Number(0), Bool(true)), MakeSetType(MakeUnionType(NumberType, BoolType)))
assertFalse(NewSet(Number(0)), MakeSetType(MakeUnionType()))
assertTrue(NewSet(), MakeSetType(MakeUnionType()))
assertTrue(NewSet(vs), MakeSetType(NumberType))
assertTrue(NewSet(vs, Number(0), Number(1), Number(2), Number(3)), MakeSetType(NumberType))
assertFalse(NewSet(vs, Number(0), Number(1), Number(2), Number(3)), MakeSetType(BoolType))
assertTrue(NewSet(vs, Number(0), Number(1), Number(2), Number(3)), MakeSetType(MakeUnionType(NumberType, BoolType)))
assertTrue(NewSet(vs, Number(0), Bool(true)), MakeSetType(MakeUnionType(NumberType, BoolType)))
assertFalse(NewSet(vs, Number(0)), MakeSetType(MakeUnionType()))
assertTrue(NewSet(vs), MakeSetType(MakeUnionType()))
{
newChunkedSet := func(vs ...Value) Set {
newChunkedSet := func(vals ...Value) Set {
newSequenceMetaTuple := func(v Value) metaTuple {
seq := newSetLeafSequence(nil, v)
seq := newSetLeafSequence(vs, v)
set := newSet(seq)
return newMetaTuple(NewRef(set), newOrderedKey(v), 1, set)
return newMetaTuple(vs.WriteValue(set), newOrderedKey(v), 1)
}
tuples := make([]metaTuple, len(vs))
for i, v := range vs {
tuples := make([]metaTuple, len(vals))
for i, v := range vals {
tuples[i] = newSequenceMetaTuple(v)
}
return newSet(newSetMetaSequence(1, tuples, nil))
return newSet(newSetMetaSequence(1, tuples, vs))
}
assertTrue(newChunkedSet(Number(0), Number(1), Number(2), Number(3)), MakeSetType(NumberType))
assertFalse(newChunkedSet(Number(0), Number(1), Number(2), Number(3)), MakeSetType(BoolType))
@@ -601,31 +625,31 @@ func TestIsValueSubtypeOf(tt *testing.T) {
assertFalse(newChunkedSet(Number(0)), MakeSetType(MakeUnionType()))
}
assertTrue(NewMap(), MakeMapType(NumberType, StringType))
assertTrue(NewMap(Number(0), String("a"), Number(1), String("b")), MakeMapType(NumberType, StringType))
assertFalse(NewMap(Number(0), String("a"), Number(1), String("b")), MakeMapType(BoolType, StringType))
assertFalse(NewMap(Number(0), String("a"), Number(1), String("b")), MakeMapType(NumberType, BoolType))
assertTrue(NewMap(Number(0), String("a"), Number(1), String("b")), MakeMapType(MakeUnionType(NumberType, BoolType), StringType))
assertTrue(NewMap(Number(0), String("a"), Number(1), String("b")), MakeMapType(NumberType, MakeUnionType(BoolType, StringType)))
assertTrue(NewMap(Number(0), String("a"), Bool(true), String("b")), MakeMapType(MakeUnionType(NumberType, BoolType), StringType))
assertTrue(NewMap(Number(0), String("a"), Number(1), Bool(true)), MakeMapType(NumberType, MakeUnionType(BoolType, StringType)))
assertFalse(NewMap(Number(0), String("a")), MakeMapType(MakeUnionType(), StringType))
assertFalse(NewMap(Number(0), String("a")), MakeMapType(NumberType, MakeUnionType()))
assertTrue(NewMap(), MakeMapType(MakeUnionType(), MakeUnionType()))
assertTrue(NewMap(vs), MakeMapType(NumberType, StringType))
assertTrue(NewMap(vs, Number(0), String("a"), Number(1), String("b")), MakeMapType(NumberType, StringType))
assertFalse(NewMap(vs, Number(0), String("a"), Number(1), String("b")), MakeMapType(BoolType, StringType))
assertFalse(NewMap(vs, Number(0), String("a"), Number(1), String("b")), MakeMapType(NumberType, BoolType))
assertTrue(NewMap(vs, Number(0), String("a"), Number(1), String("b")), MakeMapType(MakeUnionType(NumberType, BoolType), StringType))
assertTrue(NewMap(vs, Number(0), String("a"), Number(1), String("b")), MakeMapType(NumberType, MakeUnionType(BoolType, StringType)))
assertTrue(NewMap(vs, Number(0), String("a"), Bool(true), String("b")), MakeMapType(MakeUnionType(NumberType, BoolType), StringType))
assertTrue(NewMap(vs, Number(0), String("a"), Number(1), Bool(true)), MakeMapType(NumberType, MakeUnionType(BoolType, StringType)))
assertFalse(NewMap(vs, Number(0), String("a")), MakeMapType(MakeUnionType(), StringType))
assertFalse(NewMap(vs, Number(0), String("a")), MakeMapType(NumberType, MakeUnionType()))
assertTrue(NewMap(vs), MakeMapType(MakeUnionType(), MakeUnionType()))
{
newChunkedMap := func(vs ...Value) Map {
newChunkedMap := func(vals ...Value) Map {
newSequenceMetaTuple := func(e mapEntry) metaTuple {
seq := newMapLeafSequence(nil, e)
seq := newMapLeafSequence(vs, e)
m := newMap(seq)
return newMetaTuple(NewRef(m), newOrderedKey(e.key), 1, m)
return newMetaTuple(vs.WriteValue(m), newOrderedKey(e.key), 1)
}
tuples := make([]metaTuple, len(vs)/2)
for i := 0; i < len(vs); i += 2 {
tuples[i/2] = newSequenceMetaTuple(mapEntry{vs[i], vs[i+1]})
tuples := make([]metaTuple, len(vals)/2)
for i := 0; i < len(vals); i += 2 {
tuples[i/2] = newSequenceMetaTuple(mapEntry{vals[i], vals[i+1]})
}
return newMap(newMapMetaSequence(1, tuples, nil))
return newMap(newMapMetaSequence(1, tuples, vs))
}
assertTrue(newChunkedMap(Number(0), String("a"), Number(1), String("b")), MakeMapType(NumberType, StringType))
@@ -684,10 +708,10 @@ func TestIsValueSubtypeOf(tt *testing.T) {
assertTrue(
NewStruct("Node", StructData{
"value": Number(1),
"children": NewList(
"children": NewList(vs,
NewStruct("Node", StructData{
"value": Number(2),
"children": NewList(),
"children": NewList(vs),
}),
),
}),
@@ -700,10 +724,10 @@ func TestIsValueSubtypeOf(tt *testing.T) {
assertFalse( // inner Node has wrong type.
NewStruct("Node", StructData{
"value": Number(1),
"children": NewList(
"children": NewList(vs,
NewStruct("Node", StructData{
"value": Bool(true),
"children": NewList(),
"children": NewList(vs),
}),
),
}),
@@ -721,7 +745,7 @@ func TestIsValueSubtypeOf(tt *testing.T) {
}
rv := NewStruct("Node", StructData{
"value": value,
"children": NewList(childrenAsRefs...),
"children": NewList(vs, childrenAsRefs...),
})
return rv
}
go/types/type.go
+1 -1
View File
@@ -38,7 +38,7 @@ func (t *Type) TargetKind() NomsKind {
}
// Value interface
func (t *Type) Value(vrw ValueReadWriter) Value {
func (t *Type) Value() Value {
return t
}
go/types/value.go
+1 -1
View File
@@ -16,7 +16,7 @@ type Valuable interface {
// Kind is the NomsKind describing the kind of value this is.
Kind() NomsKind
Value(vrw ValueReadWriter) Value
Value() Value
}
// Emptyable is an interface for Values which may or may not be empty
go/types/value_decoder.go
+11 -11
View File
@@ -13,17 +13,17 @@ import (
type valueDecoder struct {
nomsReader
vr ValueReader
vrw ValueReadWriter
validating bool
}
// |tc| must be locked as long as the valueDecoder is being used
func newValueDecoder(nr nomsReader, vr ValueReader) *valueDecoder {
return &valueDecoder{nr, vr, false}
func newValueDecoder(nr nomsReader, vrw ValueReadWriter) *valueDecoder {
return &valueDecoder{nr, vrw, false}
}
func newValueDecoderWithValidation(nr nomsReader, vr ValueReader) *valueDecoder {
return &valueDecoder{nr, vr, true}
func newValueDecoderWithValidation(nr nomsReader, vrw ValueReadWriter) *valueDecoder {
return &valueDecoder{nr, vrw, true}
}
func (r *valueDecoder) readKind() NomsKind {
@@ -74,7 +74,7 @@ func (r *valueDecoder) readTypeInner(seenStructs map[string]*Type) *Type {
func (r *valueDecoder) readBlobLeafSequence() sequence {
b := r.readBytes()
return newBlobLeafSequence(r.vr, b)
return newBlobLeafSequence(r.vrw, b)
}
func (r *valueDecoder) readValueSequence() ValueSlice {
@@ -91,12 +91,12 @@ func (r *valueDecoder) readValueSequence() ValueSlice {
func (r *valueDecoder) readListLeafSequence() sequence {
data := r.readValueSequence()
return listLeafSequence{leafSequence{r.vr, len(data), ListKind}, data}
return listLeafSequence{leafSequence{r.vrw, len(data), ListKind}, data}
}
func (r *valueDecoder) readSetLeafSequence() orderedSequence {
data := r.readValueSequence()
return setLeafSequence{leafSequence{r.vr, len(data), SetKind}, data}
return setLeafSequence{leafSequence{r.vrw, len(data), SetKind}, data}
}
func (r *valueDecoder) readMapLeafSequence() orderedSequence {
@@ -108,7 +108,7 @@ func (r *valueDecoder) readMapLeafSequence() orderedSequence {
data = append(data, mapEntry{k, v})
}
return mapLeafSequence{leafSequence{r.vr, len(data), MapKind}, data}
return mapLeafSequence{leafSequence{r.vrw, len(data), MapKind}, data}
}
func (r *valueDecoder) readMetaSequence(k NomsKind, level uint64) metaSequence {
@@ -126,10 +126,10 @@ func (r *valueDecoder) readMetaSequence(k NomsKind, level uint64) metaSequence {
key = newOrderedKey(v)
}
numLeaves := r.readCount()
data = append(data, newMetaTuple(ref, key, numLeaves, nil))
data = append(data, newMetaTuple(ref, key, numLeaves))
}
return newMetaSequence(k, level, data, r.vr)
return newMetaSequence(k, level, data, r.vrw)
}
func (r *valueDecoder) readValue() Value {
go/types/value_store.go
-8
View File
@@ -199,14 +199,6 @@ func (lvs *ValueStore) ReadManyValues(hashes hash.HashSet, foundValues chan<- Va
// an appropriately-typed types.Ref. v is not guaranteed to be actually
// written until after Flush().
func (lvs *ValueStore) WriteValue(v Value) Ref {
iterateUncommittedChildren(v, func(sv Value) {
lvs.writeValueInternal(sv)
})
return lvs.writeValueInternal(v)
}
func (lvs *ValueStore) writeValueInternal(v Value) Ref {
lvs.versOnce.Do(lvs.expectVersion)
d.PanicIfFalse(v != nil)
go/types/value_store_test.go
+15 -16
View File
@@ -5,6 +5,7 @@
package types
import (
"fmt"
"testing"
"github.com/attic-labs/noms/go/chunks"
@@ -112,6 +113,7 @@ func (cbs *checkingChunkStore) expect(rs ...Ref) {
}
func (cbs *checkingChunkStore) Put(c chunks.Chunk) {
fmt.Println("Put", c.Hash().String())
if cbs.a.NotZero(len(cbs.expectedOrder), "Unexpected Put of %s", c.Hash()) {
cbs.a.Equal(cbs.expectedOrder[0], c.Hash())
cbs.expectedOrder = cbs.expectedOrder[1:]
@@ -142,21 +144,21 @@ func TestFlushOrder(t *testing.T) {
n := Number(42)
sr, nr := vs.WriteValue(s), vs.WriteValue(n)
ccs.expect(sr, nr)
ml := NewList(sr, nr)
ml := NewList(vs, sr, nr)
b := NewEmptyBlob()
b := NewEmptyBlob(vs)
br, mlr := vs.WriteValue(b), vs.WriteValue(ml)
ccs.expect(br, mlr)
ml1 := NewList(br, mlr)
ml1 := NewList(vs, br, mlr)
f := Bool(false)
fr := vs.WriteValue(f)
ccs.expect(fr)
ml2 := NewList(fr)
ml2 := NewList(vs, fr)
ml1r, ml2r := vs.WriteValue(ml1), vs.WriteValue(ml2)
ccs.expect(ml1r, ml2r)
l := NewList(ml1r, ml2r)
l := NewList(vs, ml1r, ml2r)
r := vs.WriteValue(l)
ccs.expect(r)
@@ -167,15 +169,12 @@ func TestFlushOverSize(t *testing.T) {
assert := assert.New(t)
storage := &chunks.TestStorage{}
ccs := &checkingChunkStore{storage.NewView(), assert, nil}
vs := newValueStoreWithCacheAndPending(ccs, 0, 10)
vs := newValueStoreWithCacheAndPending(ccs, 0, 30)
s := String("oy")
sr := vs.WriteValue(s)
l := NewList(sr)
ccs.expect(sr, NewRef(l))
vs.WriteValue(l)
vs.Commit(vs.Root(), vs.Root())
ccs.expect(sr)
NewList(vs, sr) // will write the root chunk
}
func TestTolerateTopDown(t *testing.T) {
@@ -193,11 +192,11 @@ func TestTolerateTopDown(t *testing.T) {
sr := vs.WriteValue(S)
ccs.expect(sr)
ML := NewList(sr)
ML := NewList(vs, sr)
mlr := vs.WriteValue(ML)
ccs.expect(mlr)
L := NewList(mlr)
L := NewList(vs, mlr)
lr := vs.WriteValue(L)
ccs.expect(lr)
@@ -225,7 +224,7 @@ func TestPanicOnBadVersion(t *testing.T) {
t.Run("Write", func(t *testing.T) {
cvs := NewValueStore(&badVersionStore{ChunkStore: storage.NewView()})
assert.Panics(t, func() {
cvs.WriteValue(NewEmptyBlob())
cvs.WriteValue(NewEmptyBlob(cvs))
cvs.Commit(cvs.Root(), cvs.Root())
})
})
@@ -236,7 +235,7 @@ func TestPanicIfDangling(t *testing.T) {
vs := newTestValueStore()
r := NewRef(Bool(true))
l := NewList(r)
l := NewList(vs, r)
vs.WriteValue(l)
assert.Panics(func() {
@@ -249,7 +248,7 @@ func TestSkipEnforceCompleteness(t *testing.T) {
vs.SetEnforceCompleteness(false)
r := NewRef(Bool(true))
l := NewList(r)
l := NewList(vs, r)
vs.WriteValue(l)
vs.Commit(vs.Root(), vs.Root())
go/types/walk.go
+1 -5
View File
@@ -46,11 +46,7 @@ func WalkValues(target Value, vr ValueReader, cb SkipValueCallback) {
if col, ok := v.(Collection); ok && !col.sequence().isLeaf() {
ms := col.sequence().(metaSequence)
for _, mt := range ms.tuples {
if mt.child != nil {
values = append(values, valueRec{mt.child, false})
} else {
refs[mt.ref.TargetHash()] = false
}
refs[mt.ref.TargetHash()] = false
}
continue
}
go/types/walk_test.go
+12 -12
View File
@@ -61,7 +61,7 @@ func (suite *WalkAllTestSuite) TestWalkValuesDuplicates() {
func (suite *WalkAllTestSuite) TestWalkAvoidBlobChunks() {
buff := randomBuff(16)
blob := NewBlob(bytes.NewReader(buff))
blob := NewBlob(suite.vs, bytes.NewReader(buff))
r := suite.vs.WriteValue(blob)
suite.True(r.Height() > 1)
outBlob := suite.vs.ReadValue(r.TargetHash()).(Blob)
@@ -90,7 +90,7 @@ func (suite *WalkAllTestSuite) TestWalkMultilevelList() {
for i := 0; i < count; i++ {
nums[i] = Number(i)
}
l := NewList(nums...)
l := NewList(suite.vs, nums...)
suite.True(NewRef(l).Height() > 1)
suite.assertCallbackCount(l, count+1)
@@ -160,7 +160,7 @@ func (suite *WalkTestSuite) skipWorker(composite Value) (reached []Value) {
// Skipping a sub-tree must allow other items in the list to be processed.
func (suite *WalkTestSuite) TestSkipListElement() {
wholeList := NewList(suite.mustSkip, suite.shouldSee, suite.shouldSee)
wholeList := NewList(suite.vs, suite.mustSkip, suite.shouldSee, suite.shouldSee)
reached := suite.skipWorker(wholeList)
for _, v := range []Value{wholeList, suite.mustSkip, suite.shouldSee, suite.shouldSeeItem} {
suite.Contains(reached, v, "Doesn't contain %+v", v)
@@ -169,7 +169,7 @@ func (suite *WalkTestSuite) TestSkipListElement() {
}
func (suite *WalkTestSuite) TestSkipSetElement() {
wholeSet := NewSet(suite.mustSkip, suite.shouldSee).Edit().Insert(suite.shouldSee).Set(nil)
wholeSet := NewSet(suite.vs, suite.mustSkip, suite.shouldSee).Edit().Insert(suite.shouldSee).Set()
reached := suite.skipWorker(wholeSet)
for _, v := range []Value{wholeSet, suite.mustSkip, suite.shouldSee, suite.shouldSeeItem} {
suite.Contains(reached, v, "Doesn't contain %+v", v)
@@ -179,8 +179,8 @@ func (suite *WalkTestSuite) TestSkipSetElement() {
func (suite *WalkTestSuite) TestSkipMapValue() {
shouldAlsoSeeItem := String("Also good")
shouldAlsoSee := NewSet(shouldAlsoSeeItem)
wholeMap := NewMap(suite.shouldSee, suite.mustSkip, shouldAlsoSee, suite.shouldSee)
shouldAlsoSee := NewSet(suite.vs, shouldAlsoSeeItem)
wholeMap := NewMap(suite.vs, suite.shouldSee, suite.mustSkip, shouldAlsoSee, suite.shouldSee)
reached := suite.skipWorker(wholeMap)
for _, v := range []Value{wholeMap, suite.shouldSee, suite.shouldSeeItem, suite.mustSkip, shouldAlsoSee, shouldAlsoSeeItem} {
suite.Contains(reached, v, "Doesn't contain %+v", v)
@@ -189,7 +189,7 @@ func (suite *WalkTestSuite) TestSkipMapValue() {
}
func (suite *WalkTestSuite) TestSkipMapKey() {
wholeMap := NewMap(suite.mustSkip, suite.shouldSee, suite.shouldSee, suite.shouldSee)
wholeMap := NewMap(suite.vs, suite.mustSkip, suite.shouldSee, suite.shouldSee, suite.shouldSee)
reached := suite.skipWorker(wholeMap)
for _, v := range []Value{wholeMap, suite.mustSkip, suite.shouldSee, suite.shouldSeeItem} {
suite.Contains(reached, v, "Doesn't contain %+v", v)
@@ -198,17 +198,17 @@ func (suite *WalkTestSuite) TestSkipMapKey() {
}
func (suite *WalkAllTestSuite) NewList(vs ...Value) Ref {
v := NewList(vs...)
v := NewList(suite.vs, vs...)
return suite.vs.WriteValue(v)
}
func (suite *WalkAllTestSuite) NewMap(vs ...Value) Ref {
v := NewMap(vs...)
v := NewMap(suite.vs, vs...)
return suite.vs.WriteValue(v)
}
func (suite *WalkAllTestSuite) NewSet(vs ...Value) Ref {
v := NewSet(vs...)
v := NewSet(suite.vs, vs...)
return suite.vs.WriteValue(v)
}
@@ -246,7 +246,7 @@ func (suite *WalkTestSuite) SetupTest() {
suite.ts = storage.NewView()
suite.vs = NewValueStore(suite.ts)
suite.shouldSeeItem = String("zzz")
suite.shouldSee = NewList(suite.shouldSeeItem)
suite.shouldSee = NewList(suite.vs, suite.shouldSeeItem)
suite.deadValue = Number(0xDEADBEEF)
suite.mustSkip = NewList(suite.deadValue)
suite.mustSkip = NewList(suite.vs, suite.deadValue)
}
go/util/datetime/date_time.go
+1 -1
View File
@@ -39,7 +39,7 @@ func Now() DateTime {
// MarshalNoms makes DateTime implement marshal.Marshaler and it makes
// DateTime marshal into a Noms struct with type DateTimeType.
func (dt DateTime) MarshalNoms() (types.Value, error) {
func (dt DateTime) MarshalNoms(vrw types.ValueReadWriter) (types.Value, error) {
return dateTimeTemplate.NewStruct([]types.Value{types.Number(float64(dt.Unix()) + float64(dt.Nanosecond())*1e-9)}), nil
}
go/util/datetime/date_time_test.go
+23 -5
View File
@@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/marshal"
"github.com/attic-labs/noms/go/types"
"github.com/attic-labs/testify/assert"
@@ -16,10 +17,13 @@ import (
func TestBasics(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
// Since we are using float64 in noms we cannot represent all possible times.
dt := DateTime{time.Unix(1234567, 1234567)}
nomsValue, err := marshal.Marshal(dt)
nomsValue, err := marshal.Marshal(vs, dt)
assert.NoError(err)
var dt2 DateTime
@@ -76,8 +80,11 @@ func TestUnmarshalInvalid(t *testing.T) {
func TestMarshal(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
test := func(dt DateTime, expected float64) {
v, err := marshal.Marshal(dt)
v, err := marshal.Marshal(vs, dt)
assert.NoError(err)
assert.True(types.NewStruct("DateTime", types.StructData{
@@ -97,21 +104,32 @@ func TestMarshal(t *testing.T) {
func TestMarshalType(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
dt := DateTime{time.Unix(0, 0)}
typ := marshal.MustMarshalType(dt)
typ := marshal.MustMarshalType(vs, dt)
assert.Equal(DateTimeType, typ)
v := marshal.MustMarshal(dt)
v := marshal.MustMarshal(vs, dt)
assert.Equal(typ, types.TypeOf(v))
}
func newTestValueStore() *types.ValueStore {
st := &chunks.TestStorage{}
return types.NewValueStore(st.NewView())
}
func TestZeroValues(t *testing.T) {
assert := assert.New(t)
vs := newTestValueStore()
defer vs.Close()
dt1 := DateTime{}
assert.True(dt1.IsZero())
nomsDate, _ := dt1.MarshalNoms()
nomsDate, _ := dt1.MarshalNoms(vs)
dt2 := DateTime{}
marshal.Unmarshal(nomsDate, &dt2)
go/util/jsontonoms/json_to_noms.go
+10 -10
View File
@@ -11,7 +11,7 @@ import (
"github.com/attic-labs/noms/go/types"
)
func nomsValueFromDecodedJSONBase(o interface{}, useStruct bool, namedStructs bool) types.Value {
func nomsValueFromDecodedJSONBase(vrw types.ValueReadWriter, o interface{}, useStruct bool, namedStructs bool) types.Value {
switch o := o.(type) {
case string:
return types.String(o)
@@ -24,12 +24,12 @@ func nomsValueFromDecodedJSONBase(o interface{}, useStruct bool, namedStructs bo
case []interface{}:
items := make([]types.Value, 0, len(o))
for _, v := range o {
nv := nomsValueFromDecodedJSONBase(v, useStruct, namedStructs)
nv := nomsValueFromDecodedJSONBase(vrw, v, useStruct, namedStructs)
if nv != nil {
items = append(items, nv)
}
}
return types.NewList(items...)
return types.NewList(vrw, items...)
case map[string]interface{}:
var v types.Value
if useStruct {
@@ -42,7 +42,7 @@ func nomsValueFromDecodedJSONBase(o interface{}, useStruct bool, namedStructs bo
continue
}
}
nv := nomsValueFromDecodedJSONBase(v, useStruct, namedStructs)
nv := nomsValueFromDecodedJSONBase(vrw, v, useStruct, namedStructs)
if nv != nil {
k := types.EscapeStructField(k)
fields[k] = nv
@@ -52,12 +52,12 @@ func nomsValueFromDecodedJSONBase(o interface{}, useStruct bool, namedStructs bo
} else {
kv := make([]types.Value, 0, len(o)*2)
for k, v := range o {
nv := nomsValueFromDecodedJSONBase(v, useStruct, namedStructs)
nv := nomsValueFromDecodedJSONBase(vrw, v, useStruct, namedStructs)
if nv != nil {
kv = append(kv, types.String(k), nv)
}
}
v = types.NewMap(kv...)
v = types.NewMap(vrw, kv...)
}
return v
@@ -81,14 +81,14 @@ func nomsValueFromDecodedJSONBase(o interface{}, useStruct bool, namedStructs bo
// Composites:
// - []interface{}
// - map[string]interface{}
func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
return nomsValueFromDecodedJSONBase(o, useStruct, false)
func NomsValueFromDecodedJSON(vrw types.ValueReadWriter, o interface{}, useStruct bool) types.Value {
return nomsValueFromDecodedJSONBase(vrw, o, useStruct, false)
}
// NomsValueUsingNamedStructsFromDecodedJSON performs the same function as
// NomsValueFromDecodedJson except that it always decodes JSON objects into
// structs. If the JSON object has a string field name '_name' it uses the
// value of that field as the name of the Noms struct.
func NomsValueUsingNamedStructsFromDecodedJSON(o interface{}) types.Value {
return nomsValueFromDecodedJSONBase(o, true, true)
func NomsValueUsingNamedStructsFromDecodedJSON(vrw types.ValueReadWriter, o interface{}) types.Value {
return nomsValueFromDecodedJSONBase(vrw, o, true, true)
}

Some files were not shown because too many files have changed in this diff Show More