From 0622c8c860c9c8dbb41018c7e1452dec3f6189c7 Mon Sep 17 00:00:00 2001 From: Aaron Boodman Date: Sat, 31 Oct 2015 00:04:04 -0700 Subject: [PATCH] Remove Future --- types/append_chunk.go | 16 ++++++++ types/blob.go | 8 ++-- types/compound_blob.go | 10 ++--- types/compound_blob_struct.noms | 1 + types/compound_blob_test.go | 45 +++++++++++----------- types/compound_object.go | 45 ++++++++++------------ types/equals_test.go | 3 +- types/future.go | 66 --------------------------------- types/future_test.go | 25 ------------- types/get_ref_test.go | 2 +- types/list.go | 2 +- types/map.go | 4 +- types/map_test.go | 14 ------- types/read_value.go | 8 +--- types/set.go | 2 +- types/set_test.go | 13 ------- types/unresolved_future.go | 41 -------------------- types/write_value.go | 17 +-------- 18 files changed, 79 insertions(+), 243 deletions(-) create mode 100644 types/append_chunk.go delete mode 100644 types/future.go delete mode 100644 types/future_test.go delete mode 100644 types/unresolved_future.go diff --git a/types/append_chunk.go b/types/append_chunk.go new file mode 100644 index 0000000000..f617caeb4f --- /dev/null +++ b/types/append_chunk.go @@ -0,0 +1,16 @@ +package types + +import ( + "github.com/attic-labs/noms/ref" +) + +type targetRef interface { + TargetRef() ref.Ref +} + +func appendChunk(chunks []ref.Ref, v Value) []ref.Ref { + if v.TypeRef().Kind() == RefKind { + chunks = append(chunks, v.(targetRef).TargetRef()) + } + return chunks +} diff --git a/types/blob.go b/types/blob.go index a84b1ef727..4892316981 100644 --- a/types/blob.go +++ b/types/blob.go @@ -39,7 +39,7 @@ func NewMemoryBlob(r io.Reader) (Blob, error) { func NewBlob(r io.Reader, cs chunks.ChunkStore) (Blob, error) { length := uint64(0) offsets := []uint64{} - blobs := []Future{} + chunks := []ref.Ref{} var blob blobLeaf for { buf := bytes.Buffer{} @@ -56,7 +56,7 @@ func NewBlob(r io.Reader, cs chunks.ChunkStore) (Blob, error) { length += n offsets = append(offsets, length) blob = newBlobLeaf(buf.Bytes()) - blobs = append(blobs, futureFromRef(WriteValue(blob, cs))) + chunks = append(chunks, WriteValue(blob, cs)) if err == io.EOF { break @@ -67,11 +67,11 @@ func NewBlob(r io.Reader, cs chunks.ChunkStore) (Blob, error) { return newBlobLeaf([]byte{}), nil } - if len(blobs) == 1 { + if len(chunks) == 1 { return blob, nil } - co := compoundObject{offsets, blobs, &ref.Ref{}, cs} + co := compoundObject{offsets, chunks, &ref.Ref{}, cs} co = splitCompoundObject(co, cs) return compoundBlob{co}, nil } diff --git a/types/compound_blob.go b/types/compound_blob.go index 8bfce2b013..853340ee0a 100644 --- a/types/compound_blob.go +++ b/types/compound_blob.go @@ -15,8 +15,8 @@ type compoundBlob struct { compoundObject } -func newCompoundBlob(offsets []uint64, futures []Future, cs chunks.ChunkSource) compoundBlob { - return compoundBlob{compoundObject{offsets, futures, &ref.Ref{}, cs}} +func newCompoundBlob(offsets []uint64, chunks []ref.Ref, cs chunks.ChunkSource) compoundBlob { + return compoundBlob{compoundObject{offsets, chunks, &ref.Ref{}, cs}} } // Reader implements the Blob interface @@ -32,7 +32,7 @@ type compoundBlobReader struct { } func (cbr *compoundBlobReader) Read(p []byte) (n int, err error) { - for cbr.currentBlobIndex < len(cbr.cb.futures) { + for cbr.currentBlobIndex < len(cbr.cb.chunks) { if cbr.currentReader == nil { if err = cbr.updateReader(); err != nil { return @@ -98,8 +98,8 @@ func (cbr *compoundBlobReader) findBlobOffset(abs uint64) int { } func (cbr *compoundBlobReader) updateReader() error { - if cbr.currentBlobIndex < len(cbr.cb.futures) { - v := cbr.cb.futures[cbr.currentBlobIndex].Deref(cbr.cb.cs) + if cbr.currentBlobIndex < len(cbr.cb.chunks) { + v := ReadValue(cbr.cb.chunks[cbr.currentBlobIndex], cbr.cb.cs) cbr.currentReader = v.(Blob).Reader() } else { cbr.currentReader = nil diff --git a/types/compound_blob_struct.noms b/types/compound_blob_struct.noms index a8fc20d6ac..bafd309172 100644 --- a/types/compound_blob_struct.noms +++ b/types/compound_blob_struct.noms @@ -1,4 +1,5 @@ struct compoundBlobStruct { Offsets: List(UInt64) // The offsets of the end of the related blobs. + // TODO: Rename to Chunks. Blobs: List(Ref(Blob)) } diff --git a/types/compound_blob_test.go b/types/compound_blob_test.go index 081ea2194f..94005a32ef 100644 --- a/types/compound_blob_test.go +++ b/types/compound_blob_test.go @@ -11,16 +11,17 @@ import ( "github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert" "github.com/attic-labs/noms/chunks" + "github.com/attic-labs/noms/ref" ) func getTestCompoundBlob(datas ...string) compoundBlob { - blobs := make([]Future, len(datas)) + blobs := make([]ref.Ref, len(datas)) offsets := make([]uint64, len(datas)) length := uint64(0) ms := chunks.NewMemoryStore() for i, s := range datas { b, _ := NewBlob(bytes.NewBufferString(s), ms) - blobs[i] = futureFromRef(WriteValue(b, ms)) + blobs[i] = WriteValue(b, ms) length += uint64(len(s)) offsets[i] = length } @@ -112,7 +113,7 @@ func TestCompoundBlobChunks(t *testing.T) { bl1 := newBlobLeaf([]byte("hello")) bl2 := newBlobLeaf([]byte("world")) - cb = newCompoundBlob([]uint64{5, 10}, []Future{futureFromRef(WriteValue(bl1, cs)), futureFromRef(WriteValue(bl2, cs))}, cs) + cb = newCompoundBlob([]uint64{5, 10}, []ref.Ref{WriteValue(bl1, cs), WriteValue(bl2, cs)}, cs) assert.Equal(2, len(cb.Chunks())) } @@ -138,15 +139,15 @@ func TestCompoundBlobSameChunksWithPrefix(t *testing.T) { // chunks 31 assert.Equal(cb2.Len(), cb1.Len()+uint64(6)) - assert.Equal(2, len(cb1.futures)) - assert.Equal(2, len(cb2.futures)) - assert.NotEqual(cb1.futures[0].Ref(), cb2.futures[0].Ref()) - assert.Equal(cb1.futures[1].Ref(), cb2.futures[1].Ref()) + assert.Equal(2, len(cb1.chunks)) + assert.Equal(2, len(cb2.chunks)) + assert.NotEqual(cb1.chunks[0], cb2.chunks[0]) + assert.Equal(cb1.chunks[1], cb2.chunks[1]) - futures1 := cb1.futures[0].Deref(cb1.cs).(compoundBlob).futures - futures2 := cb2.futures[0].Deref(cb2.cs).(compoundBlob).futures - assert.NotEqual(futures1[0].Ref(), futures2[0].Ref()) - assert.Equal(futures1[1].Ref(), futures2[1].Ref()) + chunks1 := ReadValue(cb1.chunks[0], cb1.cs).(compoundBlob).chunks + chunks2 := ReadValue(cb2.chunks[0], cb2.cs).(compoundBlob).chunks + assert.NotEqual(chunks1[0], chunks2[0]) + assert.Equal(chunks1[1], chunks2[1]) } func TestCompoundBlobSameChunksWithSuffix(t *testing.T) { @@ -171,16 +172,16 @@ func TestCompoundBlobSameChunksWithSuffix(t *testing.T) { // chunks 31 - only last chunk is different assert.Equal(cb2.Len(), cb1.Len()+uint64(6)) - assert.Equal(2, len(cb1.futures)) - assert.Equal(len(cb1.futures), len(cb2.futures)) - assert.Equal(cb1.futures[0].Ref(), cb2.futures[0].Ref()) - assert.NotEqual(cb1.futures[1].Ref(), cb2.futures[1].Ref()) + assert.Equal(2, len(cb1.chunks)) + assert.Equal(len(cb1.chunks), len(cb2.chunks)) + assert.Equal(cb1.chunks[0], cb2.chunks[0]) + assert.NotEqual(cb1.chunks[1], cb2.chunks[1]) - futures1 := cb1.futures[1].Deref(cb1.cs).(compoundBlob).futures - futures2 := cb2.futures[1].Deref(cb2.cs).(compoundBlob).futures - assert.Equal(futures1[0].Ref(), futures2[0].Ref()) - assert.Equal(futures1[len(futures1)-2].Ref(), futures2[len(futures2)-2].Ref()) - assert.NotEqual(futures1[len(futures1)-1].Ref(), futures2[len(futures2)-1].Ref()) + chunks1 := ReadValue(cb1.chunks[1], cb1.cs).(compoundBlob).chunks + chunks2 := ReadValue(cb2.chunks[1], cb2.cs).(compoundBlob).chunks + assert.Equal(chunks1[0], chunks2[0]) + assert.Equal(chunks1[len(chunks1)-2], chunks2[len(chunks2)-2]) + assert.NotEqual(chunks1[len(chunks1)-1], chunks2[len(chunks2)-1]) } func printBlob(b Blob, indent int) { @@ -191,8 +192,8 @@ func printBlob(b Blob, indent int) { case compoundBlob: fmt.Printf("%scompoundBlob, len: %d, chunks: %d\n", indentString, b.Len(), len(b.offsets)) indent++ - for _, sb := range b.futures { - printBlob(sb.Deref(b.cs).(Blob), indent) + for _, sb := range b.chunks { + printBlob(ReadValue(sb, b.cs).(Blob), indent) } } } diff --git a/types/compound_object.go b/types/compound_object.go index 18cf8f8968..0b730e570a 100644 --- a/types/compound_object.go +++ b/types/compound_object.go @@ -16,7 +16,7 @@ const ( type compoundObject struct { offsets []uint64 - futures []Future + chunks []ref.Ref ref *ref.Ref cs chunks.ChunkSource } @@ -25,41 +25,34 @@ func (co compoundObject) Len() uint64 { return co.offsets[len(co.offsets)-1] } -func (co compoundObject) Chunks() (chunks []ref.Ref) { - for _, f := range co.futures { - chunks = appendChunks(chunks, f) - } - return +func (co compoundObject) Chunks() []ref.Ref { + return co.chunks } -type compoundObjectToFuture func(co compoundObject) Future - // splitCompoundObject chunks a compound list/blob into smaller compound // lists/blobs. If no split was made the same compoundObject is returned. func splitCompoundObject(co compoundObject, cs chunks.ChunkSink) compoundObject { offsets := []uint64{} - futures := []Future{} + chunks := []ref.Ref{} - toFuture := func(co compoundObject) Future { - cb := compoundBlob{co} - r := WriteValue(cb, cs) - return futureFromRef(r) + toRef := func(co compoundObject) ref.Ref { + return WriteValue(compoundBlob{co}, cs) } startIndex := uint64(0) h := buzhash.NewBuzHash(objectWindowSize) for i := 0; i < len(co.offsets); i++ { - future := co.futures[i] - digest := future.Ref().Digest() + c := co.chunks[i] + digest := c.Digest() _, err := h.Write(digest[:]) d.Chk.NoError(err) if h.Sum32()&objectPattern == objectPattern { h = buzhash.NewBuzHash(objectWindowSize) - future := makeSubObject(co, startIndex, uint64(i)+1, toFuture) + c := makeSubObject(co, startIndex, uint64(i)+1, toRef) startIndex = uint64(i) + 1 offsets = append(offsets, co.offsets[i]) - futures = append(futures, future) + chunks = append(chunks, c) } } @@ -70,9 +63,9 @@ func splitCompoundObject(co compoundObject, cs chunks.ChunkSink) compoundObject // Add remaining. if startIndex != uint64(len(co.offsets)) { - future := makeSubObject(co, startIndex, uint64(len(co.offsets)), toFuture) + c := makeSubObject(co, startIndex, uint64(len(co.offsets)), toRef) offsets = append(offsets, co.offsets[len(co.offsets)-1]) - futures = append(futures, future) + chunks = append(chunks, c) } // Single chunk, use original. @@ -87,17 +80,19 @@ func splitCompoundObject(co compoundObject, cs chunks.ChunkSink) compoundObject } // Split again. - return splitCompoundObject(compoundObject{offsets, futures, &ref.Ref{}, co.cs}, cs) + return splitCompoundObject(compoundObject{offsets, chunks, &ref.Ref{}, co.cs}, cs) } -func makeSubObject(co compoundObject, startIndex, endIndex uint64, toFuture compoundObjectToFuture) Future { +type compoundObjectToRef func(co compoundObject) ref.Ref + +func makeSubObject(co compoundObject, startIndex, endIndex uint64, toRef compoundObjectToRef) ref.Ref { d.Chk.True(endIndex-startIndex > 0) if endIndex-startIndex == 1 { - return co.futures[startIndex] + return co.chunks[startIndex] } - futures := make([]Future, endIndex-startIndex) - copy(futures, co.futures[startIndex:endIndex]) + chunks := make([]ref.Ref, endIndex-startIndex) + copy(chunks, co.chunks[startIndex:endIndex]) offsets := make([]uint64, endIndex-startIndex) startOffset := uint64(0) if startIndex > 0 { @@ -106,5 +101,5 @@ func makeSubObject(co compoundObject, startIndex, endIndex uint64, toFuture comp for i := startIndex; i < endIndex; i++ { offsets[i-startIndex] = co.offsets[i] - startOffset } - return toFuture(compoundObject{offsets, futures, &ref.Ref{}, co.cs}) + return toRef(compoundObject{offsets, chunks, &ref.Ref{}, co.cs}) } diff --git a/types/equals_test.go b/types/equals_test.go index d22ea41917..f21fad81e4 100644 --- a/types/equals_test.go +++ b/types/equals_test.go @@ -6,6 +6,7 @@ import ( "github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert" "github.com/attic-labs/noms/chunks" + "github.com/attic-labs/noms/ref" ) func TestValueEquals(t *testing.T) { @@ -63,7 +64,7 @@ func TestValueEquals(t *testing.T) { ms := chunks.NewMemoryStore() b1, _ := NewBlob(bytes.NewBufferString("hi"), ms) b2, _ := NewBlob(bytes.NewBufferString("bye"), ms) - return newCompoundBlob([]uint64{2, 5}, []Future{futureFromRef(WriteValue(b1, ms)), futureFromRef(WriteValue(b2, ms))}, ms) + return newCompoundBlob([]uint64{2, 5}, []ref.Ref{WriteValue(b1, ms), WriteValue(b2, ms)}, ms) }, func() Value { return NewList() }, func() Value { return NewList(NewString("foo")) }, diff --git a/types/future.go b/types/future.go deleted file mode 100644 index 069e77a9a8..0000000000 --- a/types/future.go +++ /dev/null @@ -1,66 +0,0 @@ -package types - -import ( - "github.com/attic-labs/noms/chunks" - "github.com/attic-labs/noms/d" - "github.com/attic-labs/noms/ref" -) - -// Future encapsulates a Value which may or may not be available yet. -type Future interface { - // Returns the Ref of the value without fetching it. - Ref() ref.Ref - - // Returns the Value if we already have it, nil otherwise. - Val() Value - - // Fetch the Future value if necessary, then return it. Multiple calls to deref only result in one fetch. - Deref(cs chunks.ChunkSource) Value - - // BUG 141. The lazy loading api is clearly not working. - Release() -} - -func futuresEqual(f1, f2 Future) bool { - // If we already have both values, then use their Equals() methods since for primitives it is faster than computing a reference. - if f1.Val() != nil && f2.Val() != nil { - return f1.Val().Equals(f2.Val()) - } else { - return f1.Ref() == f2.Ref() - } -} - -func futureEqualsValue(f Future, v Value) bool { - d.Chk.NotNil(v) - if f.Val() != nil { - return f.Val().Equals(v) - } else { - return f.Ref() == v.Ref() - } -} - -type targetRef interface { - TargetRef() ref.Ref -} - -func appendChunks(chunks []ref.Ref, f Future) []ref.Ref { - if uf, ok := f.(*unresolvedFuture); ok { - chunks = append(chunks, uf.Ref()) - } else if f != nil { - v := f.Val() - if v != nil { - if v.TypeRef().Kind() == RefKind { - chunks = append(chunks, v.(targetRef).TargetRef()) - } - } - } - - return chunks -} - -func appendValueToChunks(chunks []ref.Ref, v Value) []ref.Ref { - if v.TypeRef().Kind() == RefKind { - chunks = append(chunks, v.(targetRef).TargetRef()) - } - return chunks -} diff --git a/types/future_test.go b/types/future_test.go deleted file mode 100644 index 94daa791fe..0000000000 --- a/types/future_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package types - -import ( - "testing" - - "github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert" - "github.com/attic-labs/noms/chunks" -) - -func TestUnresolvedFuture(t *testing.T) { - assert := assert.New(t) - - cs := chunks.NewTestStore() - v := NewString("hello") - r := WriteValue(v, cs) - - f := futureFromRef(r) - v2 := f.Deref(cs) - assert.Equal(1, cs.Reads) - assert.True(v.Equals(v2)) - - v3 := f.Deref(cs) - assert.Equal(1, cs.Reads) - assert.True(v2.Equals(v3)) -} diff --git a/types/get_ref_test.go b/types/get_ref_test.go index 9b0403aec1..6dd49cb70b 100644 --- a/types/get_ref_test.go +++ b/types/get_ref_test.go @@ -43,7 +43,7 @@ func TestEnsureRef(t *testing.T) { }() bl := newBlobLeaf([]byte("hi")) - cb := newCompoundBlob([]uint64{2}, []Future{futureFromRef(WriteValue(bl, cs))}, cs) + cb := newCompoundBlob([]uint64{2}, []ref.Ref{WriteValue(bl, cs)}, cs) values := []Value{ newBlobLeaf([]byte{}), diff --git a/types/list.go b/types/list.go index 84a41c61e0..7595ee31c9 100644 --- a/types/list.go +++ b/types/list.go @@ -183,7 +183,7 @@ func (l List) Equals(other Value) bool { func (l List) Chunks() (chunks []ref.Ref) { for _, v := range l.values { - chunks = appendValueToChunks(chunks, v) + chunks = appendChunk(chunks, v) } return } diff --git a/types/map.go b/types/map.go index 98633322c5..f49c78f11b 100644 --- a/types/map.go +++ b/types/map.go @@ -124,8 +124,8 @@ func (m Map) Equals(other Value) bool { func (m Map) Chunks() (chunks []ref.Ref) { for _, entry := range m.data { - chunks = appendValueToChunks(chunks, entry.key) - chunks = appendValueToChunks(chunks, entry.value) + chunks = appendChunk(chunks, entry.key) + chunks = appendChunk(chunks, entry.value) } return } diff --git a/types/map_test.go b/types/map_test.go index 8e3a763277..403321d304 100644 --- a/types/map_test.go +++ b/types/map_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert" - "github.com/attic-labs/noms/chunks" ) func TestNewMap(t *testing.T) { @@ -202,19 +201,6 @@ func TestMapEmpty(t *testing.T) { assert.False(m.Empty()) } -func TestMapFutures(t *testing.T) { - assert := assert.New(t) - - cs := chunks.NewTestStore() - k := NewString("hello") - kRef := WriteValue(k, cs) - - m := NewMap(NewRef(kRef), Int64(0xbeefcafe)) - - assert.Len(m.Chunks(), 1) - assert.EqualValues(kRef, m.Chunks()[0]) -} - func TestMapTypeRef(t *testing.T) { assert := assert.New(t) m := NewMap() diff --git a/types/read_value.go b/types/read_value.go index cf33d9701b..8efb95bbe1 100644 --- a/types/read_value.go +++ b/types/read_value.go @@ -37,10 +37,6 @@ func ReadValue(r ref.Ref, cs chunks.ChunkSource) Value { func convertToCompoundBlob(cbs compoundBlobStruct, cs chunks.ChunkSource) compoundBlob { offsets := cbs.Offsets().Def() - refs := cbs.Blobs().Def() - futures := make([]Future, len(refs)) - for i, r := range refs { - futures[i] = futureFromRef(r) - } - return newCompoundBlob(offsets, futures, cs) + chunks := cbs.Blobs().Def() + return newCompoundBlob(offsets, chunks, cs) } diff --git a/types/set.go b/types/set.go index fde5215d94..7eb72d1c01 100644 --- a/types/set.go +++ b/types/set.go @@ -120,7 +120,7 @@ func (s Set) Equals(other Value) bool { func (s Set) Chunks() (chunks []ref.Ref) { for _, v := range s.data { - chunks = appendValueToChunks(chunks, v) + chunks = appendChunk(chunks, v) } return } diff --git a/types/set_test.go b/types/set_test.go index f9c045a6ba..7cef7b7c46 100644 --- a/types/set_test.go +++ b/types/set_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert" - "github.com/attic-labs/noms/chunks" ) func TestSetLen(t *testing.T) { @@ -125,18 +124,6 @@ func TestSetAny(t *testing.T) { assert.NotNil(s.Any()) } -func TestSetFutures(t *testing.T) { - assert := assert.New(t) - - cs := chunks.NewTestStore() - v := NewString("hello") - r := WriteValue(v, cs) - s := NewSet(NewRef(r), Int64(0xbeefcafe)) - - assert.Len(s.Chunks(), 1) - assert.EqualValues(r, s.Chunks()[0]) -} - func TestSetIter(t *testing.T) { assert := assert.New(t) diff --git a/types/unresolved_future.go b/types/unresolved_future.go deleted file mode 100644 index 7a393446bd..0000000000 --- a/types/unresolved_future.go +++ /dev/null @@ -1,41 +0,0 @@ -package types - -import ( - "sync" - - "github.com/attic-labs/noms/chunks" - "github.com/attic-labs/noms/ref" -) - -func futureFromRef(ref ref.Ref) Future { - return &unresolvedFuture{ref: ref, mu: &sync.Mutex{}} -} - -type unresolvedFuture struct { - val Value - ref ref.Ref - mu *sync.Mutex -} - -func (f *unresolvedFuture) Val() Value { - return f.val -} - -func (f *unresolvedFuture) Deref(cs chunks.ChunkSource) Value { - f.mu.Lock() - defer f.mu.Unlock() - if f.val != nil { - return f.val - } - - f.val = ReadValue(f.ref, cs) - return f.val -} - -func (f *unresolvedFuture) Ref() ref.Ref { - return f.ref -} - -func (f *unresolvedFuture) Release() { - f.val = nil -} diff --git a/types/write_value.go b/types/write_value.go index c96bdde3ab..9aa8773c3d 100644 --- a/types/write_value.go +++ b/types/write_value.go @@ -48,14 +48,9 @@ func toEncodeable(v Value, cs chunks.ChunkSink) interface{} { } func processCompoundBlob(cb compoundBlob, cs chunks.ChunkSink) compoundBlobStruct { - refs := make([]ref.Ref, len(cb.futures)) - for idx, f := range cb.futures { - processChild(f, cs) - refs[idx] = f.Ref() - } return compoundBlobStructDef{ Offsets: cb.offsets, - Blobs: refs, + Blobs: cb.chunks, }.New() } @@ -67,13 +62,3 @@ func processPackageChildren(p Package, cs chunks.ChunkSink) { } } } - -func processChild(f Future, cs chunks.ChunkSink) interface{} { - if v, ok := f.(*unresolvedFuture); ok { - return v.Ref() - } - - v := f.Val() - d.Exp.NotNil(v) - return writeChildValueInternal(v, cs) -}