mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-01-25 03:09:00 -06:00
Code Issues Packages Projects Releases 123 Wiki Activity

Added metaSequence + enc/dec & tests

Browse Source
This commit is contained in:
Rafael Weinstein
2015-11-10 15:29:44 -08:00
parent 9f4f656a24
commit f0ebce2d74
16 changed files with 299 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
types/blob.go
View File

@@ -6,7 +6,6 @@ import (
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/attic-labs/buzhash"
"github.com/attic-labs/noms/chunks"
"github.com/attic-labs/noms/ref"
)
const (
@@ -38,8 +37,7 @@ func NewMemoryBlob(r io.Reader) (Blob, error) {
func NewBlob(r io.Reader, cs chunks.ChunkStore) (Blob, error) {
length := uint64(0)
offsets := []uint64{}
chunks := []ref.Ref{}
tuples := metaSequenceData{}
var blob blobLeaf
for {
buf := bytes.Buffer{}
@@ -54,9 +52,9 @@ func NewBlob(r io.Reader, cs chunks.ChunkStore) (Blob, error) {
}
length += n
offsets = append(offsets, length)
blob = newBlobLeaf(buf.Bytes())
chunks = append(chunks, WriteValue(blob, cs))
ref := WriteValue(blob, cs)
tuples = append(tuples, metaTuple{ref, UInt64(length)})
if err == io.EOF {
break
@@ -67,13 +65,11 @@ func NewBlob(r io.Reader, cs chunks.ChunkStore) (Blob, error) {
return newBlobLeaf([]byte{}), nil
}
if len(chunks) == 1 {
if len(tuples) == 1 {
return blob, nil
}
co := compoundObject{offsets, chunks, &ref.Ref{}, cs}
co = splitCompoundObject(co, cs)
return compoundBlob{co}, nil
return splitCompoundBlob(newCompoundBlob(tuples, cs), cs), nil
}
// copyChunk copies from src to dst until a chunk boundary is found.

141
types/compound_blob.go
View File

@@ -1,22 +1,47 @@
package types
import (
"crypto/sha1"
"errors"
"io"
"sort"
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/attic-labs/buzhash"
"github.com/attic-labs/noms/chunks"
"github.com/attic-labs/noms/d"
"github.com/attic-labs/noms/ref"
)
const (
objectWindowSize = 8 * sha1.Size
objectPattern = uint32(1<<6 - 1) // Average size of 64 elements
)
// compoundBlob represents a list of Blobs.
// It implements the Blob interface.
type compoundBlob struct {
compoundObject
tuples metaSequenceData
ref *ref.Ref
cs chunks.ChunkSource
}
func newCompoundBlob(offsets []uint64, chunks []ref.Ref, cs chunks.ChunkSource) compoundBlob {
return compoundBlob{compoundObject{offsets, chunks, &ref.Ref{}, cs}}
var typeRefForCompoundBlob = MakeCompoundTypeRef(MetaSequenceKind, MakePrimitiveTypeRef(BlobKind))
func newCompoundBlob(tuples metaSequenceData, cs chunks.ChunkSource) compoundBlob {
return buildCompoundBlob(tuples, typeRefForCompoundBlob, cs).(compoundBlob)
}
func buildCompoundBlob(tuples metaSequenceData, t Type, cs chunks.ChunkSource) Value {
d.Chk.True(t.Equals(typeRefForCompoundBlob))
return compoundBlob{tuples, &ref.Ref{}, cs}
}
func getSequenceData(v Value) metaSequenceData {
return v.(compoundBlob).tuples
}
func init() {
registerMetaValue(BlobKind, buildCompoundBlob, getSequenceData)
}
// Reader implements the Blob interface
@@ -24,6 +49,37 @@ func (cb compoundBlob) Reader() io.ReadSeeker {
return &compoundBlobReader{cb: cb}
}
func (cb compoundBlob) Equals(other Value) bool {
return other != nil && typeRefForCompoundBlob.Equals(other.Type()) && cb.Ref() == other.Ref()
}
func (cb compoundBlob) Ref() ref.Ref {
return EnsureRef(cb.ref, cb)
}
func (cb compoundBlob) Type() Type {
return typeRefForCompoundBlob
}
func (cb compoundBlob) ChildValues() []Value {
res := make([]Value, len(cb.tuples))
for i, t := range cb.tuples {
res[i] = NewRefOfBlob(t.ref)
}
return res
}
func (cb compoundBlob) Chunks() (chunks []ref.Ref) {
for _, tuple := range cb.tuples {
chunks = append(chunks, tuple.ref)
}
return
}
func (cb compoundBlob) Len() uint64 {
return cb.tuples[len(cb.tuples)-1].uint64Value()
}
type compoundBlobReader struct {
cb compoundBlob
currentReader io.ReadSeeker
@@ -32,7 +88,7 @@ type compoundBlobReader struct {
}
func (cbr *compoundBlobReader) Read(p []byte) (n int, err error) {
for cbr.currentBlobIndex < len(cbr.cb.chunks) {
for cbr.currentBlobIndex < len(cbr.cb.tuples) {
if cbr.currentReader == nil {
if err = cbr.updateReader(); err != nil {
return
@@ -81,7 +137,7 @@ func (cbr *compoundBlobReader) Seek(offset int64, whence int) (int64, error) {
if cbr.currentReader != nil {
offset := abs
if cbr.currentBlobIndex > 0 {
offset -= int64(cbr.cb.offsets[cbr.currentBlobIndex-1])
offset -= int64(cbr.cb.tuples[cbr.currentBlobIndex-1].uint64Value())
}
if _, err := cbr.currentReader.Seek(offset, 0); err != nil {
return 0, err
@@ -92,14 +148,14 @@ func (cbr *compoundBlobReader) Seek(offset int64, whence int) (int64, error) {
}
func (cbr *compoundBlobReader) findBlobOffset(abs uint64) int {
return sort.Search(len(cbr.cb.offsets), func(i int) bool {
return cbr.cb.offsets[i] > abs
return sort.Search(len(cbr.cb.tuples), func(i int) bool {
return cbr.cb.tuples[i].uint64Value() > abs
})
}
func (cbr *compoundBlobReader) updateReader() error {
if cbr.currentBlobIndex < len(cbr.cb.chunks) {
v := ReadValue(cbr.cb.chunks[cbr.currentBlobIndex], cbr.cb.cs)
if cbr.currentBlobIndex < len(cbr.cb.tuples) {
v := ReadValue(cbr.cb.tuples[cbr.currentBlobIndex].ref, cbr.cb.cs)
cbr.currentReader = v.(Blob).Reader()
} else {
cbr.currentReader = nil
@@ -107,14 +163,67 @@ func (cbr *compoundBlobReader) updateReader() error {
return nil
}
func (cb compoundBlob) Ref() ref.Ref {
return EnsureRef(cb.ref, cb)
// splitCompoundBlob chunks a compound list/blob into smaller compound
// lists/blobs. If no split was made the same compoundBlob is returned.
func splitCompoundBlob(cb compoundBlob, cs chunks.ChunkSink) compoundBlob {
tuples := metaSequenceData{}
startIndex := uint64(0)
h := buzhash.NewBuzHash(objectWindowSize)
for i := 0; i < len(cb.tuples); i++ {
c := cb.tuples[i].ref
digest := c.Digest()
_, err := h.Write(digest[:])
d.Chk.NoError(err)
if h.Sum32()&objectPattern == objectPattern {
h = buzhash.NewBuzHash(objectWindowSize)
c := makeSubObject(cb, startIndex, uint64(i)+1, cs)
startIndex = uint64(i) + 1
tuples = append(tuples, metaTuple{c, cb.tuples[i].value})
}
}
// No split, use original.
if startIndex == 0 {
return cb
}
// Add remaining.
if startIndex != uint64(len(cb.tuples)) {
c := makeSubObject(cb, startIndex, uint64(len(cb.tuples)), cs)
tuples = append(tuples, metaTuple{c, cb.tuples[len(cb.tuples)-1].value})
}
// Single chunk, use original.
if len(tuples) == 1 {
return cb
}
// It is possible that the splitting the object produces the exact same
// compound object.
if len(tuples) == len(cb.tuples) {
return cb
}
// Split again.
return splitCompoundBlob(newCompoundBlob(tuples, cb.cs), cs)
}
func (cb compoundBlob) Equals(other Value) bool {
return other != nil && typeRefForBlob.Equals(other.Type()) && cb.Ref() == other.Ref()
}
func makeSubObject(cb compoundBlob, startIndex, endIndex uint64, cs chunks.ChunkSink) ref.Ref {
d.Chk.True(endIndex-startIndex > 0)
if endIndex-startIndex == 1 {
return cb.tuples[startIndex].ref
}
func (cb compoundBlob) Type() Type {
return typeRefForBlob
tuples := make([]metaTuple, endIndex-startIndex)
copy(tuples, cb.tuples[startIndex:endIndex])
startOffset := uint64(0)
if startIndex > 0 {
startOffset = cb.tuples[startIndex-1].uint64Value()
}
for i := startIndex; i < endIndex; i++ {
tuples[i-startIndex].value = UInt64(cb.tuples[i].uint64Value() - startOffset)
}
return WriteValue(newCompoundBlob(tuples, cb.cs), cs)
}

53
types/compound_blob_test.go
View File

@@ -11,21 +11,18 @@ import (
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert"
"github.com/attic-labs/noms/chunks"
"github.com/attic-labs/noms/ref"
)
func getTestCompoundBlob(datas ...string) compoundBlob {
blobs := make([]ref.Ref, len(datas))
offsets := make([]uint64, len(datas))
tuples := make([]metaTuple, len(datas))
length := uint64(0)
ms := chunks.NewMemoryStore()
for i, s := range datas {
b, _ := NewBlob(bytes.NewBufferString(s), ms)
blobs[i] = WriteValue(b, ms)
length += uint64(len(s))
offsets[i] = length
tuples[i] = metaTuple{WriteValue(b, ms), UInt64(length)}
}
return newCompoundBlob(offsets, blobs, ms)
return newCompoundBlob(tuples, ms)
}
type randReader struct {
@@ -113,7 +110,7 @@ func TestCompoundBlobChunks(t *testing.T) {
bl1 := newBlobLeaf([]byte("hello"))
bl2 := newBlobLeaf([]byte("world"))
cb = newCompoundBlob([]uint64{5, 10}, []ref.Ref{WriteValue(bl1, cs), WriteValue(bl2, cs)}, cs)
cb = newCompoundBlob([]metaTuple{{WriteValue(bl1, cs), UInt64(uint64(5))}, {WriteValue(bl2, cs), UInt64(uint64(10))}}, cs)
assert.Equal(2, len(cb.Chunks()))
}
@@ -139,15 +136,15 @@ func TestCompoundBlobSameChunksWithPrefix(t *testing.T) {
// chunks 31
assert.Equal(cb2.Len(), cb1.Len()+uint64(6))
assert.Equal(2, len(cb1.chunks))
assert.Equal(2, len(cb2.chunks))
assert.NotEqual(cb1.chunks[0], cb2.chunks[0])
assert.Equal(cb1.chunks[1], cb2.chunks[1])
assert.Equal(2, len(cb1.tuples))
assert.Equal(2, len(cb2.tuples))
assert.NotEqual(cb1.tuples[0].ref, cb2.tuples[0].ref)
assert.Equal(cb1.tuples[1].ref, cb2.tuples[1].ref)
chunks1 := ReadValue(cb1.chunks[0], cb1.cs).(compoundBlob).chunks
chunks2 := ReadValue(cb2.chunks[0], cb2.cs).(compoundBlob).chunks
assert.NotEqual(chunks1[0], chunks2[0])
assert.Equal(chunks1[1], chunks2[1])
tuples1 := ReadValue(cb1.tuples[0].ref, cb1.cs).(compoundBlob).tuples
tuples2 := ReadValue(cb2.tuples[0].ref, cb2.cs).(compoundBlob).tuples
assert.NotEqual(tuples1[0].ref, tuples2[0].ref)
assert.Equal(tuples1[1].ref, tuples2[1].ref)
}
func TestCompoundBlobSameChunksWithSuffix(t *testing.T) {
@@ -172,16 +169,16 @@ func TestCompoundBlobSameChunksWithSuffix(t *testing.T) {
// chunks 31 - only last chunk is different
assert.Equal(cb2.Len(), cb1.Len()+uint64(6))
assert.Equal(2, len(cb1.chunks))
assert.Equal(len(cb1.chunks), len(cb2.chunks))
assert.Equal(cb1.chunks[0], cb2.chunks[0])
assert.NotEqual(cb1.chunks[1], cb2.chunks[1])
assert.Equal(2, len(cb1.tuples))
assert.Equal(len(cb1.tuples), len(cb2.tuples))
assert.Equal(cb1.tuples[0].ref, cb2.tuples[0].ref)
assert.NotEqual(cb1.tuples[1].ref, cb2.tuples[1].ref)
chunks1 := ReadValue(cb1.chunks[1], cb1.cs).(compoundBlob).chunks
chunks2 := ReadValue(cb2.chunks[1], cb2.cs).(compoundBlob).chunks
assert.Equal(chunks1[0], chunks2[0])
assert.Equal(chunks1[len(chunks1)-2], chunks2[len(chunks2)-2])
assert.NotEqual(chunks1[len(chunks1)-1], chunks2[len(chunks2)-1])
tuples1 := ReadValue(cb1.tuples[1].ref, cb1.cs).(compoundBlob).tuples
tuples2 := ReadValue(cb2.tuples[1].ref, cb2.cs).(compoundBlob).tuples
assert.Equal(tuples1[0].ref, tuples2[0].ref)
assert.Equal(tuples1[len(tuples1)-2].ref, tuples2[len(tuples2)-2].ref)
assert.NotEqual(tuples1[len(tuples1)-1].ref, tuples2[len(tuples2)-1].ref)
}
func printBlob(b Blob, indent int) {
@@ -190,10 +187,10 @@ func printBlob(b Blob, indent int) {
case blobLeaf:
fmt.Printf("%sblobLeaf, len: %d\n", indentString, b.Len())
case compoundBlob:
fmt.Printf("%scompoundBlob, len: %d, chunks: %d\n", indentString, b.Len(), len(b.offsets))
fmt.Printf("%scompoundBlob, len: %d, chunks: %d\n", indentString, b.Len(), len(b.tuples))
indent++
for _, sb := range b.chunks {
printBlob(ReadValue(sb, b.cs).(Blob), indent)
for _, t := range b.tuples {
printBlob(ReadValue(t.ref, b.cs).(Blob), indent)
}
}
}
@@ -202,5 +199,5 @@ func TestCompoundBlobTypeRef(t *testing.T) {
assert := assert.New(t)
cb := getTestCompoundBlob("hello", "world")
assert.True(cb.Type().Equals(MakePrimitiveTypeRef(BlobKind)))
assert.True(cb.Type().Equals(MakeCompoundTypeRef(MetaSequenceKind, MakePrimitiveTypeRef(BlobKind))))
}

107
types/compound_object.go
View File

@@ -1,107 +0,0 @@
package types
import (
"crypto/sha1"
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/kch42/buzhash"
"github.com/attic-labs/noms/chunks"
"github.com/attic-labs/noms/d"
"github.com/attic-labs/noms/ref"
)
const (
objectWindowSize = 8 * sha1.Size
objectPattern = uint32(1<<6 - 1) // Average size of 64 elements
)
type compoundObject struct {
offsets []uint64
chunks []ref.Ref
ref *ref.Ref
cs chunks.ChunkSource
}
func (co compoundObject) Len() uint64 {
return co.offsets[len(co.offsets)-1]
}
func (co compoundObject) Chunks() []ref.Ref {
return co.chunks
}
func (co compoundObject) ChildValues() []Value {
res := make([]Value, len(co.chunks))
for i, r := range co.chunks {
res[i] = NewRefOfBlob(r)
}
return res
}
// splitCompoundObject chunks a compound list/blob into smaller compound
// lists/blobs. If no split was made the same compoundObject is returned.
func splitCompoundObject(co compoundObject, cs chunks.ChunkSink) compoundObject {
offsets := []uint64{}
chunks := []ref.Ref{}
startIndex := uint64(0)
h := buzhash.NewBuzHash(objectWindowSize)
for i := 0; i < len(co.offsets); i++ {
c := co.chunks[i]
digest := c.Digest()
_, err := h.Write(digest[:])
d.Chk.NoError(err)
if h.Sum32()&objectPattern == objectPattern {
h = buzhash.NewBuzHash(objectWindowSize)
c := makeSubObject(co, startIndex, uint64(i)+1, cs)
startIndex = uint64(i) + 1
offsets = append(offsets, co.offsets[i])
chunks = append(chunks, c)
}
}
// No split, use original.
if startIndex == 0 {
return co
}
// Add remaining.
if startIndex != uint64(len(co.offsets)) {
c := makeSubObject(co, startIndex, uint64(len(co.offsets)), cs)
offsets = append(offsets, co.offsets[len(co.offsets)-1])
chunks = append(chunks, c)
}
// Single chunk, use original.
if len(offsets) == 1 {
return co
}
// It is possible that the splitting the object produces the exact same
// compound object.
if len(offsets) == len(co.offsets) {
return co
}
// Split again.
return splitCompoundObject(compoundObject{offsets, chunks, &ref.Ref{}, co.cs}, cs)
}
func makeSubObject(co compoundObject, startIndex, endIndex uint64, cs chunks.ChunkSink) ref.Ref {
d.Chk.True(endIndex-startIndex > 0)
if endIndex-startIndex == 1 {
return co.chunks[startIndex]
}
chunks := make([]ref.Ref, endIndex-startIndex)
copy(chunks, co.chunks[startIndex:endIndex])
offsets := make([]uint64, endIndex-startIndex)
startOffset := uint64(0)
if startIndex > 0 {
startOffset = co.offsets[startIndex-1]
}
for i := startIndex; i < endIndex; i++ {
offsets[i-startIndex] = co.offsets[i] - startOffset
}
return WriteValue(compoundBlob{compoundObject{offsets, chunks, &ref.Ref{}, co.cs}}, cs)
}

34
types/decode_noms_value.go
View File

@@ -59,7 +59,7 @@ func (r *jsonArrayReader) readRef() ref.Ref {
func (r *jsonArrayReader) readTypeRefAsTag() Type {
kind := r.readKind()
switch kind {
case ListKind, SetKind, RefKind:
case ListKind, SetKind, RefKind, MetaSequenceKind:
elemType := r.readTypeRefAsTag()
return MakeCompoundTypeRef(kind, elemType)
case MapKind:
@@ -135,6 +135,33 @@ func (r *jsonArrayReader) readMap(t Type, pkg *Package) Value {
return valueFromTypeRef(newMapFromData(data, t), t)
}
func indexTypeForMetaSequence(t Type) Type {
desc := t.Desc.(CompoundDesc)
concreteType := desc.ElemTypes[0]
switch concreteType.Kind() {
case MapKind, SetKind:
return concreteType.Desc.(CompoundDesc).ElemTypes[0]
case BlobKind, ListKind:
return MakePrimitiveTypeRef(UInt64Kind)
}
panic("unreached")
}
func (r *jsonArrayReader) readMetaSequence(t Type, pkg *Package) Value {
data := metaSequenceData{}
indexType := indexTypeForMetaSequence(t)
for !r.atEnd() {
ref := r.readRef()
v := r.readValueWithoutTag(indexType, pkg)
data = append(data, metaTuple{ref, v})
}
t = fixupTypeRef(t, pkg)
return newMetaSequenceFromData(data, t, r.cs)
}
func (r *jsonArrayReader) readEnum(t Type, pkg *Package) Value {
t = fixupTypeRef(t, pkg)
return enumFromTypeRef(uint32(r.read().(float64)), t)
@@ -218,6 +245,9 @@ func (r *jsonArrayReader) readValueWithoutTag(t Type, pkg *Package) Value {
return r.readTypeRefKindToValue(t, pkg)
case UnresolvedKind:
return r.readUnresolvedKindToValue(t, pkg)
case MetaSequenceKind:
r2 := newJsonArrayReader(r.readArray(), r.cs)
return r2.readMetaSequence(t, pkg)
}
panic("not reachable")
}
@@ -264,7 +294,7 @@ func (r *jsonArrayReader) readTypeRefAsValue(pkg *Package) Type {
ids = append(ids, r2.readString())
}
return MakeEnumTypeRef(name, ids...)
case ListKind, MapKind, RefKind, SetKind:
case ListKind, MapKind, RefKind, SetKind, MetaSequenceKind:
r2 := newJsonArrayReader(r.readArray(), r.cs)
elemTypes := []Type{}
for !r2.atEnd() {

19
types/decode_noms_value_test.go
View File

@@ -188,6 +188,25 @@ func TestReadValueSetOfUInt16(t *testing.T) {
assert.True(s2.Equals(s))
}
func TestReadCompoundBlob(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewMemoryStore()
r1 := ref.Parse("sha1-0000000000000000000000000000000000000001")
r2 := ref.Parse("sha1-0000000000000000000000000000000000000002")
r3 := ref.Parse("sha1-0000000000000000000000000000000000000003")
a := parseJson(`[%d, %d, ["%s", 20, "%s", 40, "%s", 60]]`, MetaSequenceKind, BlobKind, r1, r2, r3)
r := newJsonArrayReader(a, cs)
m := r.readTopLevelValue()
_, ok := m.(compoundBlob)
assert.True(ok)
m2 := newCompoundBlob([]metaTuple{{r1, UInt64(20)}, {r2, UInt64(40)}, {r3, UInt64(60)}}, cs)
assert.True(m.Type().Equals(m2.Type()))
assert.Equal(m.Ref().String(), m2.Ref().String())
}
func TestReadStruct(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewMemoryStore()

15
types/encode_noms_value.go
View File

@@ -57,6 +57,9 @@ func (w *jsonArrayWriter) writeTypeRefAsTag(t Type) {
if pkg != nil {
writeChildValueInternal(*pkg, w.cs)
}
case MetaSequenceKind:
concreteType := t.Desc.(CompoundDesc).ElemTypes[0]
w.writeTypeRefAsTag(concreteType)
}
}
@@ -126,6 +129,16 @@ func (w *jsonArrayWriter) writeValue(v Value, tr Type, pkg *Package) {
case ValueKind:
w.writeTypeRefAsTag(v.Type())
w.writeValue(v, v.Type(), pkg)
case MetaSequenceKind:
w2 := newJsonArrayWriter(w.cs)
indexType := indexTypeForMetaSequence(tr)
tr = fixupTypeRef(tr, pkg)
m := internalValueFromTypeRef(v, tr)
for _, tuple := range getDataFromMetaSequence(m) {
w2.writeRef(tuple.ref)
w2.writeValue(tuple.value, indexType, pkg)
}
w.write(w2.toArray())
default:
d.Chk.Fail("Unknown NomsKind")
}
@@ -142,7 +155,7 @@ func (w *jsonArrayWriter) writeTypeRefAsValue(v Type) {
w2.write(id)
}
w.write(w2.toArray())
case ListKind, MapKind, RefKind, SetKind:
case ListKind, MapKind, RefKind, SetKind, MetaSequenceKind:
w2 := newJsonArrayWriter(w.cs)
for _, elemType := range v.Desc.(CompoundDesc).ElemTypes {
w2.writeTypeRefAsValue(elemType)

16
types/encode_noms_value_test.go
View File

@@ -131,6 +131,22 @@ func TestWriteMapOfMap(t *testing.T) {
assert.EqualValues([]interface{}{MapKind, MapKind, StringKind, Int64Kind, SetKind, BoolKind, []interface{}{[]interface{}{"a", int64(0)}, []interface{}{true}}}, w.toArray())
}
func TestWriteCompoundBlob(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewMemoryStore()
r1 := ref.Parse("sha1-0000000000000000000000000000000000000001")
r2 := ref.Parse("sha1-0000000000000000000000000000000000000002")
r3 := ref.Parse("sha1-0000000000000000000000000000000000000003")
v := newCompoundBlob([]metaTuple{{r1, UInt64(20)}, {r2, UInt64(40)}, {r3, UInt64(60)}}, cs)
w := newJsonArrayWriter(cs)
w.writeTopLevelValue(v)
// the order of the elements is based on the ref of the value.
assert.EqualValues([]interface{}{MetaSequenceKind, BlobKind, []interface{}{r1.String(), uint64(20), r2.String(), uint64(40), r3.String(), uint64(60)}}, w.toArray())
}
func TestWriteEmptyStruct(t *testing.T) {
assert := assert.New(t)
cs := chunks.NewMemoryStore()

3
types/equals_test.go
View File

@@ -6,7 +6,6 @@ import (
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert"
"github.com/attic-labs/noms/chunks"
"github.com/attic-labs/noms/ref"
)
func TestValueEquals(t *testing.T) {
@@ -64,7 +63,7 @@ func TestValueEquals(t *testing.T) {
ms := chunks.NewMemoryStore()
b1, _ := NewBlob(bytes.NewBufferString("hi"), ms)
b2, _ := NewBlob(bytes.NewBufferString("bye"), ms)
return newCompoundBlob([]uint64{2, 5}, []ref.Ref{WriteValue(b1, ms), WriteValue(b2, ms)}, ms)
return newCompoundBlob([]metaTuple{{WriteValue(b1, ms), UInt64(uint64(2))}, {WriteValue(b2, ms), UInt64(uint64(5))}}, ms)
},
func() Value { return NewList() },
func() Value { return NewList(NewString("foo")) },

2
types/get_ref_test.go
View File

@@ -43,7 +43,7 @@ func TestEnsureRef(t *testing.T) {
}()
bl := newBlobLeaf([]byte("hi"))
cb := newCompoundBlob([]uint64{2}, []ref.Ref{WriteValue(bl, cs)}, cs)
cb := newCompoundBlob([]metaTuple{{WriteValue(bl, cs), UInt64(uint64(2))}}, cs)
values := []Value{
newBlobLeaf([]byte{}),

55
types/meta_sequence.go Normal file
View File

@@ -0,0 +1,55 @@
package types
import (
"github.com/attic-labs/noms/chunks"
"github.com/attic-labs/noms/ref"
)
// metaSequence is a logical abstraction, but has no concrete "base" implementation. A Meta Sequence is a non-leaf (internal) node of a "probably" tree, which results from the chunking of an ordered or unordered sequence of values.
type metaTuple struct {
ref ref.Ref
value Value
}
func (mt metaTuple) uint64Value() uint64 {
return uint64(mt.value.(UInt64))
}
type metaSequenceData []metaTuple
type metaBuilderFunc func(tuples metaSequenceData, t Type, cs chunks.ChunkSource) Value
type metaReaderFunc func(v Value) metaSequenceData
type metaSequenceFuncs struct {
builder metaBuilderFunc
reader metaReaderFunc
}
var (
metaFuncMap map[NomsKind]metaSequenceFuncs = map[NomsKind]metaSequenceFuncs{}
)
func registerMetaValue(k NomsKind, bf metaBuilderFunc, rf metaReaderFunc) {
metaFuncMap[k] = metaSequenceFuncs{bf, rf}
}
func newMetaSequenceFromData(tuples metaSequenceData, t Type, cs chunks.ChunkSource) Value {
concreteType := t.Desc.(CompoundDesc).ElemTypes[0]
if s, ok := metaFuncMap[concreteType.Kind()]; ok {
return s.builder(tuples, t, cs)
}
panic("not reached")
}
func getDataFromMetaSequence(v Value) metaSequenceData {
concreteType := v.Type().Desc.(CompoundDesc).ElemTypes[0]
if s, ok := metaFuncMap[concreteType.Kind()]; ok {
return s.reader(v)
}
panic("not reached")
}

1
types/noms_kind.go
View File

@@ -28,6 +28,7 @@ const (
TypeRefKind
UnresolvedKind
PackageKind
MetaSequenceKind
)
// IsPrimitiveKind returns true if k represents a Noms primitive type, which excludes collections (List, Map, Set), Refs, Enums, Structs, Symbolic and Unresolved type references.

12
types/read_value.go
View File

@@ -26,17 +26,7 @@ func ReadValue(r ref.Ref, cs chunks.ChunkSource) Value {
d.Chk.NoError(err)
return newBlobLeaf(data)
case []interface{}:
tv := fromTypedEncodeable(v, cs)
if tv, ok := tv.(compoundBlobStruct); ok {
return convertToCompoundBlob(tv, cs)
}
return tv
return fromTypedEncodeable(v, cs)
}
panic("Unreachable")
}
func convertToCompoundBlob(cbs compoundBlobStruct, cs chunks.ChunkSource) compoundBlob {
offsets := cbs.Offsets().Def()
chunks := cbs.Blobs().Def()
return newCompoundBlob(offsets, chunks, cs)
}

2
types/type_desc.go
View File

@@ -140,6 +140,8 @@ func (c CompoundDesc) Describe() string {
return "Ref(" + descElems() + ")"
case SetKind:
return "Set(" + descElems() + ")"
case MetaSequenceKind:
return "Meta(" + descElems() + ")"
default:
panic(fmt.Errorf("Kind is not compound: %v", c.kind))
}

3
types/type_ref.go
View File

@@ -152,6 +152,7 @@ func MakePrimitiveTypeRefByString(p string) Type {
func MakeCompoundTypeRef(kind NomsKind, elemTypes ...Type) Type {
if len(elemTypes) == 1 {
d.Chk.NotEqual(MapKind, kind, "MapKind requires 2 element types.")
d.Chk.True(kind == RefKind || kind == ListKind || kind == SetKind || kind == MetaSequenceKind)
} else {
d.Chk.Equal(MapKind, kind)
d.Chk.Len(elemTypes, 2, "MapKind requires 2 element types.")
@@ -181,7 +182,7 @@ func buildType(n string, desc TypeDesc) Type {
return Type{name: name{name: n}, Desc: desc, ref: &ref.Ref{}}
}
switch desc.Kind() {
case ListKind, RefKind, SetKind, MapKind, EnumKind, StructKind, UnresolvedKind:
case ListKind, RefKind, SetKind, MapKind, EnumKind, StructKind, UnresolvedKind, MetaSequenceKind:
return Type{name: name{name: n}, Desc: desc, ref: &ref.Ref{}}
default:
d.Exp.Fail("Unrecognized Kind:", "%v", desc.Kind())

10
types/write_value.go
View File

@@ -38,22 +38,12 @@ func toEncodeable(v Value, cs chunks.ChunkSink) interface{} {
switch v := v.(type) {
case blobLeaf:
return v.Reader()
case compoundBlob:
tv := processCompoundBlob(v, cs)
return encNomsValue(tv, cs)
case Package:
processPackageChildren(v, cs)
}
return encNomsValue(v, cs)
}
func processCompoundBlob(cb compoundBlob, cs chunks.ChunkSink) compoundBlobStruct {
return compoundBlobStructDef{
Offsets: cb.offsets,
Blobs: cb.chunks,
}.New()
}
func processPackageChildren(p Package, cs chunks.ChunkSink) {
for _, r := range p.dependencies {
p := LookupPackage(r)