mirror of
https://github.com/dolthub/dolt.git
synced 2026-02-04 18:49:00 -06:00
metaSequenceCursor
This commit is contained in:
Rafael Weinstein
1
types/blob_ref.noms
Normal file
1
types/blob_ref.noms
Normal file
@@ -0,0 +1 @@
|
||||
using Ref(Blob)
|
||||
65
types/blob_ref.noms.go
Normal file
65
types/blob_ref.noms.go
Normal file
@@ -0,0 +1,65 @@
|
||||
// This file was generated by nomdl/codegen.
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"github.com/attic-labs/noms/chunks"
|
||||
"github.com/attic-labs/noms/ref"
|
||||
)
|
||||
|
||||
// RefOfBlob
|
||||
|
||||
type RefOfBlob struct {
|
||||
target ref.Ref
|
||||
ref *ref.Ref
|
||||
}
|
||||
|
||||
func NewRefOfBlob(target ref.Ref) RefOfBlob {
|
||||
return RefOfBlob{target, &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (r RefOfBlob) TargetRef() ref.Ref {
|
||||
return r.target
|
||||
}
|
||||
|
||||
func (r RefOfBlob) Ref() ref.Ref {
|
||||
return EnsureRef(r.ref, r)
|
||||
}
|
||||
|
||||
func (r RefOfBlob) Equals(other Value) bool {
|
||||
return other != nil && __typeRefForRefOfBlob.Equals(other.Type()) && r.Ref() == other.Ref()
|
||||
}
|
||||
|
||||
func (r RefOfBlob) Chunks() (chunks []ref.Ref) {
|
||||
chunks = append(chunks, r.Type().Chunks()...)
|
||||
chunks = append(chunks, r.target)
|
||||
return
|
||||
}
|
||||
|
||||
func (r RefOfBlob) ChildValues() []Value {
|
||||
return nil
|
||||
}
|
||||
|
||||
// A Noms Value that describes RefOfBlob.
|
||||
var __typeRefForRefOfBlob Type
|
||||
|
||||
func (m RefOfBlob) Type() Type {
|
||||
return __typeRefForRefOfBlob
|
||||
}
|
||||
|
||||
func init() {
|
||||
__typeRefForRefOfBlob = MakeCompoundTypeRef(RefKind, MakePrimitiveTypeRef(BlobKind))
|
||||
RegisterRef(__typeRefForRefOfBlob, builderForRefOfBlob)
|
||||
}
|
||||
|
||||
func builderForRefOfBlob(r ref.Ref) Value {
|
||||
return NewRefOfBlob(r)
|
||||
}
|
||||
|
||||
func (r RefOfBlob) TargetValue(cs chunks.ChunkSource) Blob {
|
||||
return ReadValue(r.target, cs).(Blob)
|
||||
}
|
||||
|
||||
func (r RefOfBlob) SetTargetValue(val Blob, cs chunks.ChunkSink) RefOfBlob {
|
||||
return NewRefOfBlob(WriteValue(val, cs))
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"crypto/sha1"
|
||||
"errors"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/attic-labs/buzhash"
|
||||
"github.com/attic-labs/noms/chunks"
|
||||
@@ -44,9 +43,22 @@ func init() {
|
||||
registerMetaValue(BlobKind, buildCompoundBlob, getSequenceData)
|
||||
}
|
||||
|
||||
// Reader implements the Blob interface
|
||||
func (cb compoundBlob) Reader() io.ReadSeeker {
|
||||
return &compoundBlobReader{cb: cb}
|
||||
length := uint64(cb.lastTuple().value.(UInt64))
|
||||
return &compoundBlobReader{cursor: newMetaSequenceCursor(cb, cb.cs), length: length, cs: cb.cs}
|
||||
}
|
||||
|
||||
// MetaSequence
|
||||
func (cb compoundBlob) tupleAt(idx int) metaTuple {
|
||||
return cb.tuples[idx]
|
||||
}
|
||||
|
||||
func (cb compoundBlob) tupleCount() int {
|
||||
return len(cb.tuples)
|
||||
}
|
||||
|
||||
func (cb compoundBlob) lastTuple() metaTuple {
|
||||
return cb.tuples[cb.tupleCount()-1]
|
||||
}
|
||||
|
||||
func (cb compoundBlob) Equals(other Value) bool {
|
||||
@@ -81,86 +93,79 @@ func (cb compoundBlob) Len() uint64 {
|
||||
}
|
||||
|
||||
type compoundBlobReader struct {
|
||||
cb compoundBlob
|
||||
currentReader io.ReadSeeker
|
||||
currentBlobIndex int
|
||||
offset int64
|
||||
cursor *metaSequenceCursor
|
||||
currentReader io.ReadSeeker
|
||||
chunkStart, chunkOffset, length uint64
|
||||
cs chunks.ChunkSource
|
||||
}
|
||||
|
||||
func (cbr *compoundBlobReader) Read(p []byte) (n int, err error) {
|
||||
for cbr.currentBlobIndex < len(cbr.cb.tuples) {
|
||||
if cbr.currentReader == nil {
|
||||
if err = cbr.updateReader(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
n, err = cbr.currentReader.Read(p)
|
||||
if n > 0 || err != io.EOF {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
cbr.offset += int64(n)
|
||||
return
|
||||
}
|
||||
|
||||
cbr.currentBlobIndex++
|
||||
cbr.currentReader = nil
|
||||
if cbr.currentReader == nil {
|
||||
cbr.updateReader()
|
||||
}
|
||||
return 0, io.EOF
|
||||
|
||||
n, err = cbr.currentReader.Read(p)
|
||||
if n > 0 || err != io.EOF {
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
cbr.chunkOffset += uint64(n)
|
||||
return
|
||||
}
|
||||
|
||||
if !cbr.cursor.advance() {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
cbr.chunkStart = cbr.chunkStart + cbr.chunkOffset
|
||||
cbr.chunkOffset = 0
|
||||
cbr.currentReader = nil
|
||||
return cbr.Read(p)
|
||||
}
|
||||
|
||||
func (cbr *compoundBlobReader) Seek(offset int64, whence int) (int64, error) {
|
||||
var abs int64
|
||||
abs := int64(cbr.chunkStart) + int64(cbr.chunkOffset)
|
||||
|
||||
switch whence {
|
||||
case 0:
|
||||
abs = offset
|
||||
case 1:
|
||||
abs = int64(cbr.offset) + offset
|
||||
abs += offset
|
||||
case 2:
|
||||
abs = int64(cbr.cb.Len()) + offset
|
||||
abs = int64(cbr.length) + offset
|
||||
default:
|
||||
return 0, errors.New("Blob.Reader.Seek: invalid whence")
|
||||
}
|
||||
|
||||
if abs < 0 {
|
||||
return 0, errors.New("Blob.Reader.Seek: negative position")
|
||||
}
|
||||
|
||||
cbr.offset = abs
|
||||
currentBlobIndex := cbr.currentBlobIndex
|
||||
cbr.currentBlobIndex = cbr.findBlobOffset(uint64(abs))
|
||||
if currentBlobIndex != cbr.currentBlobIndex {
|
||||
if err := cbr.updateReader(); err != nil {
|
||||
return int64(0), err
|
||||
}
|
||||
}
|
||||
if cbr.currentReader != nil {
|
||||
offset := abs
|
||||
if cbr.currentBlobIndex > 0 {
|
||||
offset -= int64(cbr.cb.tuples[cbr.currentBlobIndex-1].uint64Value())
|
||||
}
|
||||
if _, err := cbr.currentReader.Seek(offset, 0); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
seekAbs := uint64(abs)
|
||||
|
||||
return abs, nil
|
||||
chunkStart := cbr.cursor.seek(func(v, parent Value) bool {
|
||||
d.Chk.NotNil(v)
|
||||
d.Chk.NotNil(parent)
|
||||
|
||||
return seekAbs < uint64(parent.(UInt64))+uint64(v.(UInt64))
|
||||
}, func(parent, prev, current Value) Value {
|
||||
pv := uint64(0)
|
||||
if prev != nil {
|
||||
pv = uint64(prev.(UInt64))
|
||||
}
|
||||
|
||||
return UInt64(uint64(parent.(UInt64)) + pv)
|
||||
}, UInt64(0))
|
||||
|
||||
cbr.chunkStart = uint64(chunkStart.(UInt64))
|
||||
cbr.chunkOffset = seekAbs - cbr.chunkStart
|
||||
cbr.currentReader = nil
|
||||
return int64(seekAbs), nil
|
||||
}
|
||||
|
||||
func (cbr *compoundBlobReader) findBlobOffset(abs uint64) int {
|
||||
return sort.Search(len(cbr.cb.tuples), func(i int) bool {
|
||||
return cbr.cb.tuples[i].uint64Value() > abs
|
||||
})
|
||||
}
|
||||
|
||||
func (cbr *compoundBlobReader) updateReader() error {
|
||||
if cbr.currentBlobIndex < len(cbr.cb.tuples) {
|
||||
v := ReadValue(cbr.cb.tuples[cbr.currentBlobIndex].ref, cbr.cb.cs)
|
||||
cbr.currentReader = v.(Blob).Reader()
|
||||
} else {
|
||||
cbr.currentReader = nil
|
||||
}
|
||||
return nil
|
||||
func (cbr *compoundBlobReader) updateReader() {
|
||||
cbr.currentReader = ReadValue(cbr.cursor.current().ref, cbr.cs).(blobLeaf).Reader()
|
||||
cbr.currentReader.Seek(int64(cbr.chunkOffset), 0)
|
||||
}
|
||||
|
||||
// splitCompoundBlob chunks a compound list/blob into smaller compound
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
struct compoundBlobStruct {
|
||||
Offsets: List(UInt64) // The offsets of the end of the related blobs.
|
||||
// TODO: Rename to Chunks.
|
||||
Blobs: List(Ref(Blob))
|
||||
}
|
||||
@@ -1,474 +0,0 @@
|
||||
// This file was generated by nomdl/codegen.
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"github.com/attic-labs/noms/chunks"
|
||||
"github.com/attic-labs/noms/ref"
|
||||
)
|
||||
|
||||
var __typesPackageInFile_compound_blob_struct_CachedRef ref.Ref
|
||||
|
||||
// This function builds up a Noms value that describes the type
|
||||
// package implemented by this file and registers it with the global
|
||||
// type package definition cache.
|
||||
func init() {
|
||||
p := NewPackage([]Type{
|
||||
MakeStructTypeRef("compoundBlobStruct",
|
||||
[]Field{
|
||||
Field{"Offsets", MakeCompoundTypeRef(ListKind, MakePrimitiveTypeRef(UInt64Kind)), false},
|
||||
Field{"Blobs", MakeCompoundTypeRef(ListKind, MakeCompoundTypeRef(RefKind, MakePrimitiveTypeRef(BlobKind))), false},
|
||||
},
|
||||
Choices{},
|
||||
),
|
||||
}, []ref.Ref{})
|
||||
__typesPackageInFile_compound_blob_struct_CachedRef = RegisterPackage(&p)
|
||||
}
|
||||
|
||||
// compoundBlobStruct
|
||||
|
||||
type compoundBlobStruct struct {
|
||||
_Offsets ListOfUInt64
|
||||
_Blobs ListOfRefOfBlob
|
||||
|
||||
ref *ref.Ref
|
||||
}
|
||||
|
||||
func NewcompoundBlobStruct() compoundBlobStruct {
|
||||
return compoundBlobStruct{
|
||||
_Offsets: NewListOfUInt64(),
|
||||
_Blobs: NewListOfRefOfBlob(),
|
||||
|
||||
ref: &ref.Ref{},
|
||||
}
|
||||
}
|
||||
|
||||
type compoundBlobStructDef struct {
|
||||
Offsets ListOfUInt64Def
|
||||
Blobs ListOfRefOfBlobDef
|
||||
}
|
||||
|
||||
func (def compoundBlobStructDef) New() compoundBlobStruct {
|
||||
return compoundBlobStruct{
|
||||
_Offsets: def.Offsets.New(),
|
||||
_Blobs: def.Blobs.New(),
|
||||
ref: &ref.Ref{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) Def() (d compoundBlobStructDef) {
|
||||
d.Offsets = s._Offsets.Def()
|
||||
d.Blobs = s._Blobs.Def()
|
||||
return
|
||||
}
|
||||
|
||||
var __typeRefForcompoundBlobStruct Type
|
||||
|
||||
func (m compoundBlobStruct) Type() Type {
|
||||
return __typeRefForcompoundBlobStruct
|
||||
}
|
||||
|
||||
func init() {
|
||||
__typeRefForcompoundBlobStruct = MakeTypeRef(__typesPackageInFile_compound_blob_struct_CachedRef, 0)
|
||||
RegisterStruct(__typeRefForcompoundBlobStruct, builderForcompoundBlobStruct, readerForcompoundBlobStruct)
|
||||
}
|
||||
|
||||
func builderForcompoundBlobStruct(values []Value) Value {
|
||||
i := 0
|
||||
s := compoundBlobStruct{ref: &ref.Ref{}}
|
||||
s._Offsets = values[i].(ListOfUInt64)
|
||||
i++
|
||||
s._Blobs = values[i].(ListOfRefOfBlob)
|
||||
i++
|
||||
return s
|
||||
}
|
||||
|
||||
func readerForcompoundBlobStruct(v Value) []Value {
|
||||
values := []Value{}
|
||||
s := v.(compoundBlobStruct)
|
||||
values = append(values, s._Offsets)
|
||||
values = append(values, s._Blobs)
|
||||
return values
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) Equals(other Value) bool {
|
||||
return other != nil && __typeRefForcompoundBlobStruct.Equals(other.Type()) && s.Ref() == other.Ref()
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) Ref() ref.Ref {
|
||||
return EnsureRef(s.ref, s)
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) Chunks() (chunks []ref.Ref) {
|
||||
chunks = append(chunks, __typeRefForcompoundBlobStruct.Chunks()...)
|
||||
chunks = append(chunks, s._Offsets.Chunks()...)
|
||||
chunks = append(chunks, s._Blobs.Chunks()...)
|
||||
return
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) ChildValues() (ret []Value) {
|
||||
ret = append(ret, s._Offsets)
|
||||
ret = append(ret, s._Blobs)
|
||||
return
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) Offsets() ListOfUInt64 {
|
||||
return s._Offsets
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) SetOffsets(val ListOfUInt64) compoundBlobStruct {
|
||||
s._Offsets = val
|
||||
s.ref = &ref.Ref{}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) Blobs() ListOfRefOfBlob {
|
||||
return s._Blobs
|
||||
}
|
||||
|
||||
func (s compoundBlobStruct) SetBlobs(val ListOfRefOfBlob) compoundBlobStruct {
|
||||
s._Blobs = val
|
||||
s.ref = &ref.Ref{}
|
||||
return s
|
||||
}
|
||||
|
||||
// ListOfUInt64
|
||||
|
||||
type ListOfUInt64 struct {
|
||||
l List
|
||||
ref *ref.Ref
|
||||
}
|
||||
|
||||
func NewListOfUInt64() ListOfUInt64 {
|
||||
return ListOfUInt64{NewList(), &ref.Ref{}}
|
||||
}
|
||||
|
||||
type ListOfUInt64Def []uint64
|
||||
|
||||
func (def ListOfUInt64Def) New() ListOfUInt64 {
|
||||
l := make([]Value, len(def))
|
||||
for i, d := range def {
|
||||
l[i] = UInt64(d)
|
||||
}
|
||||
return ListOfUInt64{NewList(l...), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Def() ListOfUInt64Def {
|
||||
d := make([]uint64, l.Len())
|
||||
for i := uint64(0); i < l.Len(); i++ {
|
||||
d[i] = uint64(l.l.Get(i).(UInt64))
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Equals(other Value) bool {
|
||||
return other != nil && __typeRefForListOfUInt64.Equals(other.Type()) && l.Ref() == other.Ref()
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Ref() ref.Ref {
|
||||
return EnsureRef(l.ref, l)
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Chunks() (chunks []ref.Ref) {
|
||||
chunks = append(chunks, l.Type().Chunks()...)
|
||||
chunks = append(chunks, l.l.Chunks()...)
|
||||
return
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) ChildValues() []Value {
|
||||
return append([]Value{}, l.l.ChildValues()...)
|
||||
}
|
||||
|
||||
// A Noms Value that describes ListOfUInt64.
|
||||
var __typeRefForListOfUInt64 Type
|
||||
|
||||
func (m ListOfUInt64) Type() Type {
|
||||
return __typeRefForListOfUInt64
|
||||
}
|
||||
|
||||
func init() {
|
||||
__typeRefForListOfUInt64 = MakeCompoundTypeRef(ListKind, MakePrimitiveTypeRef(UInt64Kind))
|
||||
RegisterValue(__typeRefForListOfUInt64, builderForListOfUInt64, readerForListOfUInt64)
|
||||
}
|
||||
|
||||
func builderForListOfUInt64(v Value) Value {
|
||||
return ListOfUInt64{v.(List), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func readerForListOfUInt64(v Value) Value {
|
||||
return v.(ListOfUInt64).l
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Len() uint64 {
|
||||
return l.l.Len()
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Empty() bool {
|
||||
return l.Len() == uint64(0)
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Get(i uint64) uint64 {
|
||||
return uint64(l.l.Get(i).(UInt64))
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Slice(idx uint64, end uint64) ListOfUInt64 {
|
||||
return ListOfUInt64{l.l.Slice(idx, end), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Set(i uint64, val uint64) ListOfUInt64 {
|
||||
return ListOfUInt64{l.l.Set(i, UInt64(val)), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Append(v ...uint64) ListOfUInt64 {
|
||||
return ListOfUInt64{l.l.Append(l.fromElemSlice(v)...), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Insert(idx uint64, v ...uint64) ListOfUInt64 {
|
||||
return ListOfUInt64{l.l.Insert(idx, l.fromElemSlice(v)...), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) Remove(idx uint64, end uint64) ListOfUInt64 {
|
||||
return ListOfUInt64{l.l.Remove(idx, end), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) RemoveAt(idx uint64) ListOfUInt64 {
|
||||
return ListOfUInt64{(l.l.RemoveAt(idx)), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) fromElemSlice(p []uint64) []Value {
|
||||
r := make([]Value, len(p))
|
||||
for i, v := range p {
|
||||
r[i] = UInt64(v)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
type ListOfUInt64IterCallback func(v uint64, i uint64) (stop bool)
|
||||
|
||||
func (l ListOfUInt64) Iter(cb ListOfUInt64IterCallback) {
|
||||
l.l.Iter(func(v Value, i uint64) bool {
|
||||
return cb(uint64(v.(UInt64)), i)
|
||||
})
|
||||
}
|
||||
|
||||
type ListOfUInt64IterAllCallback func(v uint64, i uint64)
|
||||
|
||||
func (l ListOfUInt64) IterAll(cb ListOfUInt64IterAllCallback) {
|
||||
l.l.IterAll(func(v Value, i uint64) {
|
||||
cb(uint64(v.(UInt64)), i)
|
||||
})
|
||||
}
|
||||
|
||||
func (l ListOfUInt64) IterAllP(concurrency int, cb ListOfUInt64IterAllCallback) {
|
||||
l.l.IterAllP(concurrency, func(v Value, i uint64) {
|
||||
cb(uint64(v.(UInt64)), i)
|
||||
})
|
||||
}
|
||||
|
||||
type ListOfUInt64FilterCallback func(v uint64, i uint64) (keep bool)
|
||||
|
||||
func (l ListOfUInt64) Filter(cb ListOfUInt64FilterCallback) ListOfUInt64 {
|
||||
out := l.l.Filter(func(v Value, i uint64) bool {
|
||||
return cb(uint64(v.(UInt64)), i)
|
||||
})
|
||||
return ListOfUInt64{out, &ref.Ref{}}
|
||||
}
|
||||
|
||||
// ListOfRefOfBlob
|
||||
|
||||
type ListOfRefOfBlob struct {
|
||||
l List
|
||||
ref *ref.Ref
|
||||
}
|
||||
|
||||
func NewListOfRefOfBlob() ListOfRefOfBlob {
|
||||
return ListOfRefOfBlob{NewList(), &ref.Ref{}}
|
||||
}
|
||||
|
||||
type ListOfRefOfBlobDef []ref.Ref
|
||||
|
||||
func (def ListOfRefOfBlobDef) New() ListOfRefOfBlob {
|
||||
l := make([]Value, len(def))
|
||||
for i, d := range def {
|
||||
l[i] = NewRefOfBlob(d)
|
||||
}
|
||||
return ListOfRefOfBlob{NewList(l...), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Def() ListOfRefOfBlobDef {
|
||||
d := make([]ref.Ref, l.Len())
|
||||
for i := uint64(0); i < l.Len(); i++ {
|
||||
d[i] = l.l.Get(i).(RefOfBlob).TargetRef()
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Equals(other Value) bool {
|
||||
return other != nil && __typeRefForListOfRefOfBlob.Equals(other.Type()) && l.Ref() == other.Ref()
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Ref() ref.Ref {
|
||||
return EnsureRef(l.ref, l)
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Chunks() (chunks []ref.Ref) {
|
||||
chunks = append(chunks, l.Type().Chunks()...)
|
||||
chunks = append(chunks, l.l.Chunks()...)
|
||||
return
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) ChildValues() []Value {
|
||||
return append([]Value{}, l.l.ChildValues()...)
|
||||
}
|
||||
|
||||
// A Noms Value that describes ListOfRefOfBlob.
|
||||
var __typeRefForListOfRefOfBlob Type
|
||||
|
||||
func (m ListOfRefOfBlob) Type() Type {
|
||||
return __typeRefForListOfRefOfBlob
|
||||
}
|
||||
|
||||
func init() {
|
||||
__typeRefForListOfRefOfBlob = MakeCompoundTypeRef(ListKind, MakeCompoundTypeRef(RefKind, MakePrimitiveTypeRef(BlobKind)))
|
||||
RegisterValue(__typeRefForListOfRefOfBlob, builderForListOfRefOfBlob, readerForListOfRefOfBlob)
|
||||
}
|
||||
|
||||
func builderForListOfRefOfBlob(v Value) Value {
|
||||
return ListOfRefOfBlob{v.(List), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func readerForListOfRefOfBlob(v Value) Value {
|
||||
return v.(ListOfRefOfBlob).l
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Len() uint64 {
|
||||
return l.l.Len()
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Empty() bool {
|
||||
return l.Len() == uint64(0)
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Get(i uint64) RefOfBlob {
|
||||
return l.l.Get(i).(RefOfBlob)
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Slice(idx uint64, end uint64) ListOfRefOfBlob {
|
||||
return ListOfRefOfBlob{l.l.Slice(idx, end), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Set(i uint64, val RefOfBlob) ListOfRefOfBlob {
|
||||
return ListOfRefOfBlob{l.l.Set(i, val), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Append(v ...RefOfBlob) ListOfRefOfBlob {
|
||||
return ListOfRefOfBlob{l.l.Append(l.fromElemSlice(v)...), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Insert(idx uint64, v ...RefOfBlob) ListOfRefOfBlob {
|
||||
return ListOfRefOfBlob{l.l.Insert(idx, l.fromElemSlice(v)...), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) Remove(idx uint64, end uint64) ListOfRefOfBlob {
|
||||
return ListOfRefOfBlob{l.l.Remove(idx, end), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) RemoveAt(idx uint64) ListOfRefOfBlob {
|
||||
return ListOfRefOfBlob{(l.l.RemoveAt(idx)), &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) fromElemSlice(p []RefOfBlob) []Value {
|
||||
r := make([]Value, len(p))
|
||||
for i, v := range p {
|
||||
r[i] = v
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
type ListOfRefOfBlobIterCallback func(v RefOfBlob, i uint64) (stop bool)
|
||||
|
||||
func (l ListOfRefOfBlob) Iter(cb ListOfRefOfBlobIterCallback) {
|
||||
l.l.Iter(func(v Value, i uint64) bool {
|
||||
return cb(v.(RefOfBlob), i)
|
||||
})
|
||||
}
|
||||
|
||||
type ListOfRefOfBlobIterAllCallback func(v RefOfBlob, i uint64)
|
||||
|
||||
func (l ListOfRefOfBlob) IterAll(cb ListOfRefOfBlobIterAllCallback) {
|
||||
l.l.IterAll(func(v Value, i uint64) {
|
||||
cb(v.(RefOfBlob), i)
|
||||
})
|
||||
}
|
||||
|
||||
func (l ListOfRefOfBlob) IterAllP(concurrency int, cb ListOfRefOfBlobIterAllCallback) {
|
||||
l.l.IterAllP(concurrency, func(v Value, i uint64) {
|
||||
cb(v.(RefOfBlob), i)
|
||||
})
|
||||
}
|
||||
|
||||
type ListOfRefOfBlobFilterCallback func(v RefOfBlob, i uint64) (keep bool)
|
||||
|
||||
func (l ListOfRefOfBlob) Filter(cb ListOfRefOfBlobFilterCallback) ListOfRefOfBlob {
|
||||
out := l.l.Filter(func(v Value, i uint64) bool {
|
||||
return cb(v.(RefOfBlob), i)
|
||||
})
|
||||
return ListOfRefOfBlob{out, &ref.Ref{}}
|
||||
}
|
||||
|
||||
// RefOfBlob
|
||||
|
||||
type RefOfBlob struct {
|
||||
target ref.Ref
|
||||
ref *ref.Ref
|
||||
}
|
||||
|
||||
func NewRefOfBlob(target ref.Ref) RefOfBlob {
|
||||
return RefOfBlob{target, &ref.Ref{}}
|
||||
}
|
||||
|
||||
func (r RefOfBlob) TargetRef() ref.Ref {
|
||||
return r.target
|
||||
}
|
||||
|
||||
func (r RefOfBlob) Ref() ref.Ref {
|
||||
return EnsureRef(r.ref, r)
|
||||
}
|
||||
|
||||
func (r RefOfBlob) Equals(other Value) bool {
|
||||
return other != nil && __typeRefForRefOfBlob.Equals(other.Type()) && r.Ref() == other.Ref()
|
||||
}
|
||||
|
||||
func (r RefOfBlob) Chunks() (chunks []ref.Ref) {
|
||||
chunks = append(chunks, r.Type().Chunks()...)
|
||||
chunks = append(chunks, r.target)
|
||||
return
|
||||
}
|
||||
|
||||
func (r RefOfBlob) ChildValues() []Value {
|
||||
return nil
|
||||
}
|
||||
|
||||
// A Noms Value that describes RefOfBlob.
|
||||
var __typeRefForRefOfBlob Type
|
||||
|
||||
func (m RefOfBlob) Type() Type {
|
||||
return __typeRefForRefOfBlob
|
||||
}
|
||||
|
||||
func init() {
|
||||
__typeRefForRefOfBlob = MakeCompoundTypeRef(RefKind, MakePrimitiveTypeRef(BlobKind))
|
||||
RegisterRef(__typeRefForRefOfBlob, builderForRefOfBlob)
|
||||
}
|
||||
|
||||
func builderForRefOfBlob(r ref.Ref) Value {
|
||||
return NewRefOfBlob(r)
|
||||
}
|
||||
|
||||
func (r RefOfBlob) TargetValue(cs chunks.ChunkSource) Blob {
|
||||
return ReadValue(r.target, cs).(Blob)
|
||||
}
|
||||
|
||||
func (r RefOfBlob) SetTargetValue(val Blob, cs chunks.ChunkSink) RefOfBlob {
|
||||
return NewRefOfBlob(WriteValue(val, cs))
|
||||
}
|
||||
@@ -25,26 +25,15 @@ func getTestCompoundBlob(datas ...string) compoundBlob {
|
||||
return newCompoundBlob(tuples, ms)
|
||||
}
|
||||
|
||||
type randReader struct {
|
||||
s rand.Source
|
||||
i int
|
||||
size int
|
||||
}
|
||||
|
||||
func (r *randReader) Read(p []byte) (n int, err error) {
|
||||
start := r.i
|
||||
for i := range p {
|
||||
if r.i == r.size {
|
||||
return r.i - start, io.EOF
|
||||
}
|
||||
p[i] = byte(r.s.Int63() & 0xff)
|
||||
r.i++
|
||||
func getRandomReader() io.ReadSeeker {
|
||||
length := int(5e5)
|
||||
s := rand.NewSource(42)
|
||||
buff := make([]byte, 5e5, 5e5)
|
||||
for i := 0; i < length; i++ {
|
||||
buff[i] = byte(s.Int63() & 0xff)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func getRandomReader() io.Reader {
|
||||
return &randReader{rand.NewSource(42), 0, 5e5}
|
||||
return bytes.NewReader(buff)
|
||||
}
|
||||
|
||||
func getRandomBlob(t *testing.T) compoundBlob {
|
||||
@@ -57,6 +46,28 @@ func getRandomBlob(t *testing.T) compoundBlob {
|
||||
return b.(compoundBlob)
|
||||
}
|
||||
|
||||
func testByteRange(assert *assert.Assertions, offset, length int64, expect io.ReadSeeker, actual io.ReadSeeker) {
|
||||
n, err := expect.Seek(offset, 0)
|
||||
assert.Equal(offset, n)
|
||||
assert.NoError(err)
|
||||
|
||||
b1 := &bytes.Buffer{}
|
||||
n, err = io.CopyN(b1, expect, length)
|
||||
assert.Equal(length, n)
|
||||
assert.NoError(err)
|
||||
|
||||
n, err = actual.Seek(offset, 0)
|
||||
assert.Equal(offset, n)
|
||||
assert.NoError(err)
|
||||
|
||||
b2 := &bytes.Buffer{}
|
||||
n, err = io.CopyN(b2, actual, length)
|
||||
assert.Equal(length, n)
|
||||
assert.NoError(err)
|
||||
|
||||
assert.Equal(b1.Bytes(), b2.Bytes())
|
||||
}
|
||||
|
||||
func TestCompoundBlobReader(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping test in short mode.")
|
||||
@@ -74,12 +85,18 @@ func TestCompoundBlobReader(t *testing.T) {
|
||||
r := getRandomReader()
|
||||
bs2, err := ioutil.ReadAll(r)
|
||||
assert.Equal(bs2, bs)
|
||||
testByteRange(assert, 200453, 100232, r, ab.Reader())
|
||||
testByteRange(assert, 100, 10, r, ab.Reader())
|
||||
testByteRange(assert, 2340, 2630, r, ab.Reader())
|
||||
testByteRange(assert, 432423, 50000, r, ab.Reader())
|
||||
testByteRange(assert, 1, 10, r, ab.Reader())
|
||||
|
||||
ref := WriteValue(cb, cb.cs.(chunks.ChunkStore))
|
||||
cb2 := ReadValue(ref, cb.cs)
|
||||
bs3, err := ioutil.ReadAll(cb2.(Blob).Reader())
|
||||
assert.NoError(err)
|
||||
assert.Equal("helloworld", string(bs3))
|
||||
|
||||
}
|
||||
|
||||
type testBlob struct {
|
||||
@@ -92,6 +109,70 @@ func (b testBlob) Reader() io.ReadSeeker {
|
||||
return b.blobLeaf.Reader()
|
||||
}
|
||||
|
||||
func TestCompoundBlobReaderSeek(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
cb := getTestCompoundBlob("hi", "bye")
|
||||
|
||||
r := cb.Reader()
|
||||
_, err := r.Seek(0, 4)
|
||||
assert.Error(err)
|
||||
|
||||
_, err = r.Seek(-1, 0)
|
||||
assert.Error(err)
|
||||
|
||||
p := []byte{0}
|
||||
|
||||
n, err := r.Seek(3, 0)
|
||||
assert.NoError(err)
|
||||
assert.Equal(int64(3), n)
|
||||
|
||||
n2, err := r.Read(p)
|
||||
assert.NoError(err)
|
||||
assert.Equal(1, n2)
|
||||
assert.Equal("y", string(p))
|
||||
|
||||
n, err = r.Seek(-1, 1)
|
||||
assert.NoError(err)
|
||||
// assert.Equal(int64(3), n)
|
||||
|
||||
n2, err = r.Read(p)
|
||||
assert.NoError(err)
|
||||
assert.Equal(1, n2)
|
||||
assert.Equal("y", string(p))
|
||||
|
||||
n, err = r.Seek(-5, 2)
|
||||
assert.NoError(err)
|
||||
assert.Equal(int64(0), n)
|
||||
|
||||
n2, err = r.Read(p)
|
||||
assert.NoError(err)
|
||||
assert.Equal(1, n2)
|
||||
assert.Equal("h", string(p))
|
||||
|
||||
n, err = r.Seek(100, 0)
|
||||
assert.NoError(err)
|
||||
assert.Equal(int64(100), n)
|
||||
|
||||
n2, err = r.Read(p)
|
||||
assert.Equal(io.EOF, err)
|
||||
assert.Equal(0, n2)
|
||||
|
||||
n, err = r.Seek(-99, 1)
|
||||
assert.NoError(err)
|
||||
assert.Equal(int64(1), n)
|
||||
|
||||
n2, err = r.Read(p)
|
||||
assert.NoError(err)
|
||||
assert.Equal(1, n2)
|
||||
assert.Equal("i", string(p))
|
||||
|
||||
n2, err = r.Read(p)
|
||||
assert.NoError(err)
|
||||
assert.Equal(1, n2)
|
||||
assert.Equal("b", string(p))
|
||||
}
|
||||
|
||||
func TestCompoundBlobLen(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
cb := getTestCompoundBlob("hello", "world")
|
||||
|
||||
@@ -7,6 +7,13 @@ import (
|
||||
|
||||
// metaSequence is a logical abstraction, but has no concrete "base" implementation. A Meta Sequence is a non-leaf (internal) node of a "probably" tree, which results from the chunking of an ordered or unordered sequence of values.
|
||||
|
||||
type metaSequence interface {
|
||||
tupleAt(idx int) metaTuple
|
||||
lastTuple() metaTuple
|
||||
tupleCount() int
|
||||
Ref() ref.Ref
|
||||
}
|
||||
|
||||
type metaTuple struct {
|
||||
ref ref.Ref
|
||||
value Value
|
||||
|
||||
93
types/meta_sequence_cursor.go
Normal file
93
types/meta_sequence_cursor.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/attic-labs/noms/chunks"
|
||||
"github.com/attic-labs/noms/d"
|
||||
)
|
||||
|
||||
// metaSequenceCursor allows traversal of a tree of metaSequence nodes.
|
||||
|
||||
type metaSequenceCursor struct {
|
||||
parent *metaSequenceCursor
|
||||
sequence metaSequence
|
||||
idx int
|
||||
cs chunks.ChunkSource
|
||||
}
|
||||
|
||||
func newMetaSequenceCursor(root metaSequence, cs chunks.ChunkSource) *metaSequenceCursor {
|
||||
cursor := &metaSequenceCursor{nil, root, 0, cs}
|
||||
parent := cursor
|
||||
child := ReadValue(cursor.current().ref, cs)
|
||||
if ms, ok := child.(metaSequence); ok {
|
||||
cursor = newMetaSequenceCursor(ms, cs)
|
||||
cursor.parent = parent
|
||||
}
|
||||
|
||||
return cursor
|
||||
}
|
||||
|
||||
func (ms *metaSequenceCursor) advance() bool {
|
||||
newIdx := ms.idx + 1
|
||||
|
||||
if newIdx < ms.sequence.tupleCount() {
|
||||
ms.idx = newIdx
|
||||
return true
|
||||
}
|
||||
|
||||
if ms.parent != nil && ms.parent.advance() {
|
||||
ms.readSequence()
|
||||
ms.idx = 0
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (ms *metaSequenceCursor) readSequence() {
|
||||
if ms.sequence.Ref() == ms.parent.current().ref {
|
||||
return
|
||||
}
|
||||
|
||||
ms.sequence = ReadValue(ms.parent.current().ref, ms.cs).(metaSequence)
|
||||
d.Chk.NotNil(ms.sequence)
|
||||
}
|
||||
|
||||
func (ms *metaSequenceCursor) current() metaTuple {
|
||||
d.Chk.NotNil(ms.sequence)
|
||||
d.Chk.True(ms.idx >= 0 && ms.idx < ms.sequence.tupleCount())
|
||||
return ms.sequence.tupleAt(ms.idx)
|
||||
}
|
||||
|
||||
type metaSequenceSeekFn func(v, parent Value) bool
|
||||
type seekParentValueFn func(parent, prev, curr Value) Value
|
||||
|
||||
// |seek| will never advance the cursor beyond the final tuple in the cursor, even if seekFn never returns true
|
||||
func (ms *metaSequenceCursor) seek(seekFn metaSequenceSeekFn, parentValueFn seekParentValueFn, parentValue Value) Value {
|
||||
d.Chk.NotNil(seekFn) // parentValueFn is allowed to be nil
|
||||
|
||||
if ms.parent != nil {
|
||||
parentValue = ms.parent.seek(seekFn, parentValueFn, parentValue)
|
||||
ms.readSequence()
|
||||
}
|
||||
|
||||
ms.idx = sort.Search(ms.sequence.tupleCount(), func(i int) bool {
|
||||
return seekFn(ms.sequence.tupleAt(i).value, parentValue)
|
||||
})
|
||||
|
||||
if ms.idx >= ms.sequence.tupleCount() {
|
||||
ms.idx = ms.sequence.tupleCount() - 1
|
||||
}
|
||||
|
||||
if parentValueFn == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var prev Value
|
||||
if ms.idx > 0 {
|
||||
prev = ms.sequence.tupleAt(ms.idx - 1).value
|
||||
}
|
||||
|
||||
return parentValueFn(parentValue, prev, ms.sequence.tupleAt(ms.idx).value)
|
||||
}
|
||||
@@ -2,4 +2,4 @@ package types
|
||||
|
||||
//go:generate go run gen/main.go
|
||||
|
||||
//go:generate go run ../nomdl/codegen/codegen.go -in=compound_blob_struct.noms
|
||||
//go:generate go run ../nomdl/codegen/codegen.go -in=blob_ref.noms
|
||||
|
||||
Reference in New Issue
Block a user