diff --git a/clients/music/mp3_importer/mp3_importer.go b/clients/music/mp3_importer/mp3_importer.go index 7846b3dbb2..82cc07ed95 100644 --- a/clients/music/mp3_importer/mp3_importer.go +++ b/clients/music/mp3_importer/mp3_importer.go @@ -31,7 +31,7 @@ func addMp3(ds *dataset.Dataset, filename string) { } defer mp3_file.Close() - mp3_data, err := types.NewBlob(bufio.NewReader(mp3_file)) + mp3_data, err := types.NewBlob(bufio.NewReader(mp3_file), ds.Store()) if err != nil { log.Fatalf("Failed to read mp3 data from %s: %s\n", filename, err) } diff --git a/marshal/marshal.go b/marshal/marshal.go index 7e948e03cd..279be27426 100644 --- a/marshal/marshal.go +++ b/marshal/marshal.go @@ -317,7 +317,7 @@ func isNilPtrOrNilInterface(v reflect.Value) bool { func readerEncoder(v reflect.Value) types.Value { d.Chk.True(v.Type().Implements(readerType)) - blob, err := types.NewBlob(v.Interface().(io.Reader)) + blob, err := types.NewMemoryBlob(v.Interface().(io.Reader)) d.Exp.NoError(err, "Failed to marshal reader into blob") return blob } @@ -398,10 +398,10 @@ func newMapEncoder(t reflect.Type) encoderFunc { func encodeByteSlice(v reflect.Value) types.Value { if v.IsNil() { - nom, _ := types.NewBlob(&bytes.Buffer{}) + nom, _ := types.NewMemoryBlob(&bytes.Buffer{}) return nom } - nom, err := types.NewBlob(bytes.NewReader(v.Bytes())) + nom, err := types.NewMemoryBlob(bytes.NewReader(v.Bytes())) if err != nil { panic(err) } diff --git a/marshal/marshal_test.go b/marshal/marshal_test.go index c66e6bf72b..423b1862c3 100644 --- a/marshal/marshal_test.go +++ b/marshal/marshal_test.go @@ -462,7 +462,7 @@ var pallValue = All{ // Used in creating canned marshaled values below. func makeNewBlob(b []byte) types.Blob { - blob, err := types.NewBlob(bytes.NewBuffer(b)) + blob, err := types.NewMemoryBlob(bytes.NewBuffer(b)) if err != nil { panic(err) // Sigh } diff --git a/marshal/unmarshal_test.go b/marshal/unmarshal_test.go index e08fc82c0b..ac4d92be0e 100644 --- a/marshal/unmarshal_test.go +++ b/marshal/unmarshal_test.go @@ -411,7 +411,7 @@ func byteSlice(b byte, times int) (out []byte) { } func blob(b ...byte) types.Blob { - blob, err := types.NewBlob(bytes.NewReader(b)) + blob, err := types.NewMemoryBlob(bytes.NewReader(b)) if err != nil { panic(err) } diff --git a/nomdl/codegen/test/struct_primitives_test.go b/nomdl/codegen/test/struct_primitives_test.go index c7931e10d2..59f8674b0c 100644 --- a/nomdl/codegen/test/struct_primitives_test.go +++ b/nomdl/codegen/test/struct_primitives_test.go @@ -104,7 +104,7 @@ func TestAccessors(t *testing.T) { assert.Equal("bye", st.String()) assert.True(st.Blob().Equals(types.NewEmptyBlob())) - b, err := types.NewBlob(strings.NewReader("hello")) + b, err := types.NewMemoryBlob(strings.NewReader("hello")) assert.NoError(err) st.SetBlob(b) assert.True(st.Blob().Equals(types.NewEmptyBlob())) diff --git a/types/blob.go b/types/blob.go index 6f58bb4621..5f7fe8b503 100644 --- a/types/blob.go +++ b/types/blob.go @@ -5,6 +5,7 @@ import ( "io" "github.com/attic-labs/noms/Godeps/_workspace/src/github.com/attic-labs/buzhash" + "github.com/attic-labs/noms/chunks" "github.com/attic-labs/noms/ref" ) @@ -31,7 +32,11 @@ func NewEmptyBlob() Blob { return newBlobLeaf([]byte{}) } -func NewBlob(r io.Reader) (Blob, error) { +func NewMemoryBlob(r io.Reader) (Blob, error) { + return NewBlob(r, chunks.NewMemoryStore()) +} + +func NewBlob(r io.Reader, cs chunks.ChunkStore) (Blob, error) { length := uint64(0) offsets := []uint64{} blobs := []Future{} @@ -51,7 +56,7 @@ func NewBlob(r io.Reader) (Blob, error) { length += n offsets = append(offsets, length) blob = newBlobLeaf(buf.Bytes()) - blobs = append(blobs, futureFromValue(blob)) + blobs = append(blobs, futureFromRef(WriteValue(blob, cs))) if err == io.EOF { break @@ -66,7 +71,7 @@ func NewBlob(r io.Reader) (Blob, error) { return blob, nil } - co := compoundObject{offsets, blobs, &ref.Ref{}, nil} + co := compoundObject{offsets, blobs, &ref.Ref{}, cs} co = splitCompoundObject(co, compoundObjectToBlobFuture) return compoundBlob{co}, nil } diff --git a/types/blob_test.go b/types/blob_test.go index 250143b585..4878cbdb8e 100644 --- a/types/blob_test.go +++ b/types/blob_test.go @@ -20,21 +20,21 @@ func AssertSymNe(assert *assert.Assertions, a, b Value) { func TestBlobLen(t *testing.T) { assert := assert.New(t) - b, err := NewBlob(&bytes.Buffer{}) + b, err := NewMemoryBlob(&bytes.Buffer{}) assert.NoError(err) assert.Equal(uint64(0), b.Len()) - b, err = NewBlob(bytes.NewBuffer([]byte{0x01})) + b, err = NewMemoryBlob(bytes.NewBuffer([]byte{0x01})) assert.NoError(err) assert.Equal(uint64(1), b.Len()) } func TestBlobEquals(t *testing.T) { assert := assert.New(t) - b1, _ := NewBlob(bytes.NewBuffer([]byte{0x01})) + b1, _ := NewMemoryBlob(bytes.NewBuffer([]byte{0x01})) b11 := b1 - b12, _ := NewBlob(bytes.NewBuffer([]byte{0x01})) - b2, _ := NewBlob(bytes.NewBuffer([]byte{0x02})) - b3, _ := NewBlob(bytes.NewBuffer([]byte{0x02, 0x03})) + b12, _ := NewMemoryBlob(bytes.NewBuffer([]byte{0x01})) + b2, _ := NewMemoryBlob(bytes.NewBuffer([]byte{0x02})) + b3, _ := NewMemoryBlob(bytes.NewBuffer([]byte{0x02, 0x03})) AssertSymEq(assert, b1, b11) AssertSymEq(assert, b1, b12) AssertSymNe(assert, b1, b2) @@ -72,7 +72,7 @@ func TestBlobFromReaderThatReturnsDataAndError(t *testing.T) { assert := assert.New(t) tr := &testReader{buf: &bytes.Buffer{}} - b, err := NewBlob(tr) + b, err := NewMemoryBlob(tr) assert.NoError(err) actual := &bytes.Buffer{} diff --git a/types/compound_blob_test.go b/types/compound_blob_test.go index 76bee3cb1b..c5110fa7c3 100644 --- a/types/compound_blob_test.go +++ b/types/compound_blob_test.go @@ -17,13 +17,14 @@ func getTestCompoundBlob(datas ...string) compoundBlob { blobs := make([]Future, len(datas)) offsets := make([]uint64, len(datas)) length := uint64(0) + ms := chunks.NewMemoryStore() for i, s := range datas { - b, _ := NewBlob(bytes.NewBufferString(s)) + b, _ := NewBlob(bytes.NewBufferString(s), ms) blobs[i] = futureFromValue(b) length += uint64(len(s)) offsets[i] = length } - return newCompoundBlob(offsets, blobs, nil) + return newCompoundBlob(offsets, blobs, ms) } type randReader struct { @@ -53,7 +54,7 @@ func getRandomBlob(t *testing.T) compoundBlob { t.Skip("Skipping test in short mode.") } r := getRandomReader() - b, err := NewBlob(r) + b, err := NewMemoryBlob(r) assert.NoError(t, err) return b.(compoundBlob) } @@ -266,7 +267,7 @@ func TestCompoundBlobSameChunksWithPrefix(t *testing.T) { buf := bytes.NewBufferString("prefix") r := io.MultiReader(buf, rr) - b, err := NewBlob(r) + b, err := NewMemoryBlob(r) assert.NoError(err) cb2 := b.(compoundBlob) @@ -299,7 +300,7 @@ func TestCompoundBlobSameChunksWithSuffix(t *testing.T) { buf := bytes.NewBufferString("suffix") r := io.MultiReader(rr, buf) - b, err := NewBlob(r) + b, err := NewMemoryBlob(r) assert.NoError(err) cb2 := b.(compoundBlob) diff --git a/types/decode_noms_value.go b/types/decode_noms_value.go index 270d0849a6..106845f9de 100644 --- a/types/decode_noms_value.go +++ b/types/decode_noms_value.go @@ -2,6 +2,7 @@ package types import ( "encoding/base64" + "io/ioutil" "strings" "github.com/attic-labs/noms/chunks" @@ -97,9 +98,9 @@ func (r *jsonArrayReader) readTypeRefAsTag() TypeRef { func (r *jsonArrayReader) readBlob(t TypeRef) Value { s := r.readString() decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(s)) - b, err := NewBlob(decoder) + b, err := ioutil.ReadAll(decoder) d.Exp.NoError(err) - return b + return newBlobLeaf(b) } func (r *jsonArrayReader) readList(t TypeRef, pkg *Package) Value { diff --git a/types/decode_noms_value_test.go b/types/decode_noms_value_test.go index 97f18b1ca4..d0bb3680a9 100644 --- a/types/decode_noms_value_test.go +++ b/types/decode_noms_value_test.go @@ -83,7 +83,7 @@ func TestReadPrimitives(t *testing.T) { test(NewString("hi"), `[%d, "hi"]`, StringKind) - blob, err := NewBlob(bytes.NewBuffer([]byte{0x00, 0x01})) + blob, err := NewMemoryBlob(bytes.NewBuffer([]byte{0x00, 0x01})) assert.NoError(err) test(blob, `[%d, "AAE="]`, BlobKind) } @@ -528,7 +528,7 @@ func TestReadStructWithBlob(t *testing.T) { v := r.readTopLevelValue().(Map) - blob, err := NewBlob(bytes.NewBuffer([]byte{0x00, 0x01})) + blob, err := NewMemoryBlob(bytes.NewBuffer([]byte{0x00, 0x01})) assert.NoError(err) assert.True(v.Get(NewString("b")).Equals(blob)) } diff --git a/types/encode_noms_value_test.go b/types/encode_noms_value_test.go index dc13eadaac..3915488a13 100644 --- a/types/encode_noms_value_test.go +++ b/types/encode_noms_value_test.go @@ -35,7 +35,7 @@ func TestWritePrimitives(t *testing.T) { f(StringKind, NewString("hi"), "hi") - blob, err := NewBlob(bytes.NewBuffer([]byte{0x00, 0x01})) + blob, err := NewMemoryBlob(bytes.NewBuffer([]byte{0x00, 0x01})) assert.NoError(err) f(BlobKind, blob, "AAE=") } @@ -295,7 +295,7 @@ func TestWriteStructWithBlob(t *testing.T) { }, Choices{})}, []ref.Ref{}) pkgRef := RegisterPackage(&pkg) tref := MakeTypeRef(pkgRef, 0) - b, _ := NewBlob(bytes.NewBuffer([]byte{0x00, 0x01})) + b, _ := NewMemoryBlob(bytes.NewBuffer([]byte{0x00, 0x01})) v := NewMap(NewString("b"), b) w := newJsonArrayWriter(cs) @@ -351,7 +351,7 @@ func TestWriteListOfValue(t *testing.T) { cs := chunks.NewMemoryStore() tref := MakeCompoundTypeRef(ListKind, MakePrimitiveTypeRef(ValueKind)) - blob, _ := NewBlob(bytes.NewBuffer([]byte{0x01})) + blob, _ := NewMemoryBlob(bytes.NewBuffer([]byte{0x01})) v := NewList( Bool(true), UInt8(1), diff --git a/types/equals_test.go b/types/equals_test.go index 156dd652e7..eb6ab6a2ae 100644 --- a/types/equals_test.go +++ b/types/equals_test.go @@ -44,20 +44,20 @@ func TestPrimitiveEquals(t *testing.T) { func() Value { return NewString("hi") }, func() Value { return NewString("bye") }, func() Value { - v, _ := NewBlob(&bytes.Buffer{}) + v, _ := NewMemoryBlob(&bytes.Buffer{}) return v }, func() Value { - v, _ := NewBlob(bytes.NewBufferString("hi")) + v, _ := NewMemoryBlob(bytes.NewBufferString("hi")) return v }, func() Value { - v, _ := NewBlob(bytes.NewBufferString("bye")) + v, _ := NewMemoryBlob(bytes.NewBufferString("bye")) return v }, func() Value { - b1, _ := NewBlob(bytes.NewBufferString("hi")) - b2, _ := NewBlob(bytes.NewBufferString("bye")) + b1, _ := NewMemoryBlob(bytes.NewBufferString("hi")) + b2, _ := NewMemoryBlob(bytes.NewBufferString("bye")) return newCompoundBlob([]uint64{2, 5}, []Future{futureFromValue(b1), futureFromValue(b2)}, nil) }, func() Value { return NewList() }, diff --git a/types/map_test.go b/types/map_test.go index 5b5e477130..eddefbf97e 100644 --- a/types/map_test.go +++ b/types/map_test.go @@ -166,8 +166,8 @@ func TestMapEquals(t *testing.T) { func TestMapNotStringKeys(t *testing.T) { assert := assert.New(t) - b1, _ := NewBlob(bytes.NewBufferString("blob1")) - b2, _ := NewBlob(bytes.NewBufferString("blob2")) + b1, _ := NewMemoryBlob(bytes.NewBufferString("blob1")) + b2, _ := NewMemoryBlob(bytes.NewBufferString("blob2")) l := []Value{ Bool(true), NewString("true"), Bool(false), NewString("false"), diff --git a/types/read_value_test.go b/types/read_value_test.go index 109cfaeb24..7c5c5ac78e 100644 --- a/types/read_value_test.go +++ b/types/read_value_test.go @@ -21,7 +21,7 @@ func TestReadValueBlobLeafDecode(t *testing.T) { blobLeafDecode := func(r io.Reader) Value { i := decode(r) - b, err := NewBlob(i.(io.Reader)) + b, err := NewMemoryBlob(i.(io.Reader)) assert.NoError(err) return b } diff --git a/types/string.go b/types/string.go index b884c5e60d..1341e51b6e 100644 --- a/types/string.go +++ b/types/string.go @@ -1,10 +1,6 @@ package types -import ( - "bytes" - - "github.com/attic-labs/noms/ref" -) +import "github.com/attic-labs/noms/ref" type String struct { s string @@ -15,10 +11,6 @@ func NewString(s string) String { return String{s, &ref.Ref{}} } -func (fs String) Blob() (Blob, error) { - return NewBlob(bytes.NewBufferString(fs.s)) -} - func (fs String) String() string { return fs.s } diff --git a/types/write_value_test.go b/types/write_value_test.go index bb00a01df2..1765f36c8c 100644 --- a/types/write_value_test.go +++ b/types/write_value_test.go @@ -27,7 +27,7 @@ func TestWriteValue(t *testing.T) { // Encoding details for each codec is tested elsewhere. // Here we just want to make sure codecs are selected correctly. - b, err := NewBlob(bytes.NewBuffer([]byte{0x00, 0x01, 0x02})) + b, err := NewMemoryBlob(bytes.NewBuffer([]byte{0x00, 0x01, 0x02})) assert.NoError(err) testEncode(string([]byte{'b', ' ', 0x00, 0x01, 0x02}), b) @@ -43,7 +43,7 @@ func TestWriteBlobLeaf(t *testing.T) { cs := chunks.NewMemoryStore() buf := bytes.NewBuffer([]byte{}) - b1, err := NewBlob(buf) + b1, err := NewMemoryBlob(buf) assert.NoError(err) bl1, ok := b1.(blobLeaf) assert.True(ok) @@ -52,7 +52,7 @@ func TestWriteBlobLeaf(t *testing.T) { assert.Equal("sha1-e1bc846440ec2fb557a5a271e785cd4c648883fa", r1.String()) buf = bytes.NewBufferString("Hello, World!") - b2, err := NewBlob(buf) + b2, err := NewMemoryBlob(buf) assert.NoError(err) bl2, ok := b2.(blobLeaf) assert.True(ok)