mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-01-30 10:45:18 -06:00
Code Issues Packages Projects Releases 124 Wiki Activity

IterAllP for typed Maps

Browse Source 
Fixes #564
This commit is contained in:
Chris Masone
2015-11-05 12:00:22 -08:00
parent 2fcd3ce3af
commit 4e79367d1a
8 changed files with 251 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
nomdl/codegen/map.tmpl
View File

@@ -116,6 +116,11 @@ func (m {{.Name}}) IterAll(cb {{.Name}}IterAllCallback) {
})
}
func (m {{.Name}}) IterAllP(concurrency int, cb {{.Name}}IterAllCallback) {
m.m.IterAllP(concurrency, func(k, v {{$typesPackage}}Value) {
cb({{valueToUser "k" .KeyType}}, {{valueToUser "v" .ValueType}})
})
}
type {{.Name}}FilterCallback func(k {{userType .KeyType}}, v {{userType .ValueType}}) (keep bool)

12
nomdl/codegen/test/gen/map.noms.go
View File

@@ -121,6 +121,12 @@ func (m MapOfBoolToString) IterAll(cb MapOfBoolToStringIterAllCallback) {
})
}
func (m MapOfBoolToString) IterAllP(concurrency int, cb MapOfBoolToStringIterAllCallback) {
m.m.IterAllP(concurrency, func(k, v types.Value) {
cb(bool(k.(types.Bool)), v.(types.String).String())
})
}
type MapOfBoolToStringFilterCallback func(k bool, v string) (keep bool)
func (m MapOfBoolToString) Filter(cb MapOfBoolToStringFilterCallback) MapOfBoolToString {
@@ -247,6 +253,12 @@ func (m MapOfStringToValue) IterAll(cb MapOfStringToValueIterAllCallback) {
})
}
func (m MapOfStringToValue) IterAllP(concurrency int, cb MapOfStringToValueIterAllCallback) {
m.m.IterAllP(concurrency, func(k, v types.Value) {
cb(k.(types.String).String(), v)
})
}
type MapOfStringToValueFilterCallback func(k string, v types.Value) (keep bool)
func (m MapOfStringToValue) Filter(cb MapOfStringToValueFilterCallback) MapOfStringToValue {

141
nomdl/codegen/test/gen/struct_with_dup_list.noms.go
View File

@@ -109,144 +109,3 @@ func (s StructWithDupList) SetL(val ListOfUInt8) StructWithDupList {
s.ref = &ref.Ref{}
return s
}
// ListOfUInt8
type ListOfUInt8 struct {
l types.List
ref *ref.Ref
}
func NewListOfUInt8() ListOfUInt8 {
return ListOfUInt8{types.NewList(), &ref.Ref{}}
}
type ListOfUInt8Def []uint8
func (def ListOfUInt8Def) New() ListOfUInt8 {
l := make([]types.Value, len(def))
for i, d := range def {
l[i] = types.UInt8(d)
}
return ListOfUInt8{types.NewList(l...), &ref.Ref{}}
}
func (l ListOfUInt8) Def() ListOfUInt8Def {
d := make([]uint8, l.Len())
for i := uint64(0); i < l.Len(); i++ {
d[i] = uint8(l.l.Get(i).(types.UInt8))
}
return d
}
func (l ListOfUInt8) Equals(other types.Value) bool {
return other != nil && __typeRefForListOfUInt8.Equals(other.TypeRef()) && l.Ref() == other.Ref()
}
func (l ListOfUInt8) Ref() ref.Ref {
return types.EnsureRef(l.ref, l)
}
func (l ListOfUInt8) Chunks() (chunks []ref.Ref) {
chunks = append(chunks, l.TypeRef().Chunks()...)
chunks = append(chunks, l.l.Chunks()...)
return
}
// A Noms Value that describes ListOfUInt8.
var __typeRefForListOfUInt8 types.TypeRef
func (m ListOfUInt8) TypeRef() types.TypeRef {
return __typeRefForListOfUInt8
}
func init() {
__typeRefForListOfUInt8 = types.MakeCompoundTypeRef(types.ListKind, types.MakePrimitiveTypeRef(types.UInt8Kind))
types.RegisterValue(__typeRefForListOfUInt8, builderForListOfUInt8, readerForListOfUInt8)
}
func builderForListOfUInt8(v types.Value) types.Value {
return ListOfUInt8{v.(types.List), &ref.Ref{}}
}
func readerForListOfUInt8(v types.Value) types.Value {
return v.(ListOfUInt8).l
}
func (l ListOfUInt8) Len() uint64 {
return l.l.Len()
}
func (l ListOfUInt8) Empty() bool {
return l.Len() == uint64(0)
}
func (l ListOfUInt8) Get(i uint64) uint8 {
return uint8(l.l.Get(i).(types.UInt8))
}
func (l ListOfUInt8) Slice(idx uint64, end uint64) ListOfUInt8 {
return ListOfUInt8{l.l.Slice(idx, end), &ref.Ref{}}
}
func (l ListOfUInt8) Set(i uint64, val uint8) ListOfUInt8 {
return ListOfUInt8{l.l.Set(i, types.UInt8(val)), &ref.Ref{}}
}
func (l ListOfUInt8) Append(v ...uint8) ListOfUInt8 {
return ListOfUInt8{l.l.Append(l.fromElemSlice(v)...), &ref.Ref{}}
}
func (l ListOfUInt8) Insert(idx uint64, v ...uint8) ListOfUInt8 {
return ListOfUInt8{l.l.Insert(idx, l.fromElemSlice(v)...), &ref.Ref{}}
}
func (l ListOfUInt8) Remove(idx uint64, end uint64) ListOfUInt8 {
return ListOfUInt8{l.l.Remove(idx, end), &ref.Ref{}}
}
func (l ListOfUInt8) RemoveAt(idx uint64) ListOfUInt8 {
return ListOfUInt8{(l.l.RemoveAt(idx)), &ref.Ref{}}
}
func (l ListOfUInt8) fromElemSlice(p []uint8) []types.Value {
r := make([]types.Value, len(p))
for i, v := range p {
r[i] = types.UInt8(v)
}
return r
}
type ListOfUInt8IterCallback func(v uint8, i uint64) (stop bool)
func (l ListOfUInt8) Iter(cb ListOfUInt8IterCallback) {
l.l.Iter(func(v types.Value, i uint64) bool {
return cb(uint8(v.(types.UInt8)), i)
})
}
type ListOfUInt8IterAllCallback func(v uint8, i uint64)
func (l ListOfUInt8) IterAll(cb ListOfUInt8IterAllCallback) {
l.l.IterAll(func(v types.Value, i uint64) {
cb(uint8(v.(types.UInt8)), i)
})
}
func (l ListOfUInt8) IterAllP(concurrency int, cb ListOfUInt8IterAllCallback) {
l.l.IterAllP(concurrency, func(v types.Value, i uint64) {
cb(uint8(v.(types.UInt8)), i)
})
}
type ListOfUInt8FilterCallback func(v uint8, i uint64) (keep bool)
func (l ListOfUInt8) Filter(cb ListOfUInt8FilterCallback) ListOfUInt8 {
nl := NewListOfUInt8()
l.IterAll(func(v uint8, i uint64) {
if cb(v, i) {
nl = nl.Append(v)
}
})
return nl
}

141
nomdl/codegen/test/gen/struct_with_list.noms.go
View File

@@ -163,3 +163,144 @@ func (s StructWithList) SetI(val int64) StructWithList {
s.ref = &ref.Ref{}
return s
}
// ListOfUInt8
type ListOfUInt8 struct {
l types.List
ref *ref.Ref
}
func NewListOfUInt8() ListOfUInt8 {
return ListOfUInt8{types.NewList(), &ref.Ref{}}
}
type ListOfUInt8Def []uint8
func (def ListOfUInt8Def) New() ListOfUInt8 {
l := make([]types.Value, len(def))
for i, d := range def {
l[i] = types.UInt8(d)
}
return ListOfUInt8{types.NewList(l...), &ref.Ref{}}
}
func (l ListOfUInt8) Def() ListOfUInt8Def {
d := make([]uint8, l.Len())
for i := uint64(0); i < l.Len(); i++ {
d[i] = uint8(l.l.Get(i).(types.UInt8))
}
return d
}
func (l ListOfUInt8) Equals(other types.Value) bool {
return other != nil && __typeRefForListOfUInt8.Equals(other.TypeRef()) && l.Ref() == other.Ref()
}
func (l ListOfUInt8) Ref() ref.Ref {
return types.EnsureRef(l.ref, l)
}
func (l ListOfUInt8) Chunks() (chunks []ref.Ref) {
chunks = append(chunks, l.TypeRef().Chunks()...)
chunks = append(chunks, l.l.Chunks()...)
return
}
// A Noms Value that describes ListOfUInt8.
var __typeRefForListOfUInt8 types.TypeRef
func (m ListOfUInt8) TypeRef() types.TypeRef {
return __typeRefForListOfUInt8
}
func init() {
__typeRefForListOfUInt8 = types.MakeCompoundTypeRef(types.ListKind, types.MakePrimitiveTypeRef(types.UInt8Kind))
types.RegisterValue(__typeRefForListOfUInt8, builderForListOfUInt8, readerForListOfUInt8)
}
func builderForListOfUInt8(v types.Value) types.Value {
return ListOfUInt8{v.(types.List), &ref.Ref{}}
}
func readerForListOfUInt8(v types.Value) types.Value {
return v.(ListOfUInt8).l
}
func (l ListOfUInt8) Len() uint64 {
return l.l.Len()
}
func (l ListOfUInt8) Empty() bool {
return l.Len() == uint64(0)
}
func (l ListOfUInt8) Get(i uint64) uint8 {
return uint8(l.l.Get(i).(types.UInt8))
}
func (l ListOfUInt8) Slice(idx uint64, end uint64) ListOfUInt8 {
return ListOfUInt8{l.l.Slice(idx, end), &ref.Ref{}}
}
func (l ListOfUInt8) Set(i uint64, val uint8) ListOfUInt8 {
return ListOfUInt8{l.l.Set(i, types.UInt8(val)), &ref.Ref{}}
}
func (l ListOfUInt8) Append(v ...uint8) ListOfUInt8 {
return ListOfUInt8{l.l.Append(l.fromElemSlice(v)...), &ref.Ref{}}
}
func (l ListOfUInt8) Insert(idx uint64, v ...uint8) ListOfUInt8 {
return ListOfUInt8{l.l.Insert(idx, l.fromElemSlice(v)...), &ref.Ref{}}
}
func (l ListOfUInt8) Remove(idx uint64, end uint64) ListOfUInt8 {
return ListOfUInt8{l.l.Remove(idx, end), &ref.Ref{}}
}
func (l ListOfUInt8) RemoveAt(idx uint64) ListOfUInt8 {
return ListOfUInt8{(l.l.RemoveAt(idx)), &ref.Ref{}}
}
func (l ListOfUInt8) fromElemSlice(p []uint8) []types.Value {
r := make([]types.Value, len(p))
for i, v := range p {
r[i] = types.UInt8(v)
}
return r
}
type ListOfUInt8IterCallback func(v uint8, i uint64) (stop bool)
func (l ListOfUInt8) Iter(cb ListOfUInt8IterCallback) {
l.l.Iter(func(v types.Value, i uint64) bool {
return cb(uint8(v.(types.UInt8)), i)
})
}
type ListOfUInt8IterAllCallback func(v uint8, i uint64)
func (l ListOfUInt8) IterAll(cb ListOfUInt8IterAllCallback) {
l.l.IterAll(func(v types.Value, i uint64) {
cb(uint8(v.(types.UInt8)), i)
})
}
func (l ListOfUInt8) IterAllP(concurrency int, cb ListOfUInt8IterAllCallback) {
l.l.IterAllP(concurrency, func(v types.Value, i uint64) {
cb(uint8(v.(types.UInt8)), i)
})
}
type ListOfUInt8FilterCallback func(v uint8, i uint64) (keep bool)
func (l ListOfUInt8) Filter(cb ListOfUInt8FilterCallback) ListOfUInt8 {
nl := NewListOfUInt8()
l.IterAll(func(v uint8, i uint64) {
if cb(v, i) {
nl = nl.Append(v)
}
})
return nl
}

10
types/list.go
View File

@@ -14,10 +14,6 @@ type List struct {
ref *ref.Ref
}
type listIterFunc func(v Value, index uint64) (stop bool)
type listIterAllFunc func(v Value, index uint64)
type MapFunc func(v Value, index uint64) interface{}
var listTypeRef = MakeCompoundTypeRef(ListKind, MakePrimitiveTypeRef(ValueKind))
func NewList(v ...Value) List {
@@ -44,6 +40,8 @@ func (l List) Get(idx uint64) Value {
return l.values[idx]
}
type listIterFunc func(v Value, index uint64) (stop bool)
func (l List) Iter(f listIterFunc) {
for i, v := range l.values {
if f(v, uint64(i)) {
@@ -52,6 +50,8 @@ func (l List) Iter(f listIterFunc) {
}
}
type listIterAllFunc func(v Value, index uint64)
func (l List) IterAll(f listIterAllFunc) {
for i, v := range l.values {
f(v, uint64(i))
@@ -86,6 +86,8 @@ func (l List) iterInternal(sem chan int, lf listIterAllFunc, offset uint64) {
wg.Wait()
}
type MapFunc func(v Value, index uint64) interface{}
func (l List) Map(mf MapFunc) []interface{} {
return l.MapP(1, mf)
}

25
types/map.go
View File

@@ -1,7 +1,9 @@
package types
import (
"runtime"
"sort"
"sync"
"github.com/attic-labs/noms/d"
"github.com/attic-labs/noms/ref"
@@ -101,7 +103,28 @@ func (m Map) IterAll(cb mapIterAllCallback) {
}
}
// TODO: Implement IterAllP() BUG 573
func (m Map) IterAllP(concurrency int, f mapIterAllCallback) {
if concurrency == 0 {
concurrency = runtime.NumCPU()
}
sem := make(chan int, concurrency)
wg := sync.WaitGroup{}
for idx := range m.data {
wg.Add(1)
sem <- 1
go func(idx int) {
defer wg.Done()
md := m.data[idx]
f(md.key, md.value)
<-sem
}(idx)
}
wg.Wait()
}
type mapFilterCallback func(key, value Value) (keep bool)

57
types/map_test.go
View File

@@ -2,6 +2,8 @@ package types
import (
"bytes"
"runtime"
"sync"
"testing"
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert"
@@ -134,6 +136,61 @@ func TestMapIter(t *testing.T) {
assert.True(got(NewString("a"), Int32(0)) || got(NewString("b"), Int32(1)))
}
func TestMapIterAllP(t *testing.T) {
assert := assert.New(t)
testIter := func(concurrency, mapLen int) {
values := make([]Value, 2*mapLen)
for i := 0; i < mapLen; i++ {
values[2*i] = UInt64(i)
values[2*i+1] = UInt64(i)
}
m := NewMap(values...)
cur := 0
mu := sync.Mutex{}
getCur := func() int {
mu.Lock()
defer mu.Unlock()
return cur
}
expectConcurreny := concurrency
if concurrency == 0 {
expectConcurreny = runtime.NumCPU()
}
visited := make([]bool, mapLen)
f := func(k, v Value) {
mu.Lock()
cur++
mu.Unlock()
for getCur() < expectConcurreny {
}
visited[v.(UInt64)] = true
}
if concurrency == 1 {
m.IterAll(f)
} else {
m.IterAllP(concurrency, f)
}
numVisited := 0
for _, visit := range visited {
if visit {
numVisited++
}
}
assert.Equal(mapLen, numVisited, "IterAllP was not called with every map key")
}
testIter(0, 100)
testIter(10, 1000)
testIter(1, 100000)
testIter(64, 100000)
}
func TestMapFilter(t *testing.T) {
assert := assert.New(t)

10
types/set.go
View File

@@ -16,10 +16,6 @@ type Set struct {
ref *ref.Ref
}
type setIterCallback func(v Value) bool
type setIterAllCallback func(v Value)
type setFilterCallback func(v Value) (keep bool)
var setTypeRef = MakeCompoundTypeRef(SetKind, MakePrimitiveTypeRef(ValueKind))
func NewSet(v ...Value) Set {
@@ -80,6 +76,8 @@ func (s Set) Subtract(others ...Set) Set {
return result
}
type setIterCallback func(v Value) bool
func (s Set) Iter(cb setIterCallback) {
for _, v := range s.data {
if cb(v) {
@@ -88,6 +86,8 @@ func (s Set) Iter(cb setIterCallback) {
}
}
type setIterAllCallback func(v Value)
func (s Set) IterAll(cb setIterAllCallback) {
for _, v := range s.data {
cb(v)
@@ -116,6 +116,8 @@ func (s Set) IterAllP(concurrency int, f setIterAllCallback) {
wg.Wait()
}
type setFilterCallback func(v Value) (keep bool)
func (s Set) Filter(cb setFilterCallback) Set {
data := setData{}
for _, v := range s.data {