Fix xml_importer and pitchmap/index to correctly use typed containers.

Fixes #554

clients/pitchmap/index/index.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"sync"
 
+	"github.com/attic-labs/noms/chunks"
 	"github.com/attic-labs/noms/clients/util"
 	"github.com/attic-labs/noms/d"
 	"github.com/attic-labs/noms/datas"
@@ -21,53 +22,55 @@ var (
 	outputID = flag.String("output-ds", "", "dataset to store data in.")
 )
 
-func processPitcher(m MapOfStringToString) (id, name string) {
-	id = m.Get("-id")
-	name = fmt.Sprintf("%s, %s", m.Get("-last_name"), m.Get("-first_name"))
+func processPitcher(m MapOfStringToValue) (id, name string) {
+	id = m.getAsString("-id")
+	name = fmt.Sprintf("%s, %s", m.getAsString("-last_name"), m.getAsString("-first_name"))
 	return
 }
 
-func checkPitch(v MapOfStringToString) bool {
+func checkPitch(v MapOfStringToValue) bool {
 	return v.Has("-px") && v.Has("-pz")
 }
 
-func getPitch(v MapOfStringToString) Pitch {
-	x, _ := strconv.ParseFloat(v.Get("-px"), 64)
-	z, _ := strconv.ParseFloat(v.Get("-pz"), 64)
-	return NewPitch().SetX(x).SetZ(z)
+func getPitch(v MapOfStringToValue) PitchDef {
+	x, _ := strconv.ParseFloat(v.getAsString("-px"), 64)
+	z, _ := strconv.ParseFloat(v.getAsString("-pz"), 64)
+	return PitchDef{X: x, Z: z}
 }
 
-func processPitches(v types.Value) (pitches []Pitch) {
+func processPitches(v types.Value) (pitches []PitchDef) {
 	switch v := v.(type) {
 	case types.List:
 		for i := uint64(0); i < v.Len(); i++ {
 			pitches = append(pitches, processPitches(v.Get(i))...)
 		}
-	case MapOfStringToString:
+	case MapOfStringToValue:
 		if checkPitch(v) {
 			pitches = append(pitches, getPitch(v))
 		}
 	case nil:
 		return // Yes, an at-bat can end with no pitches thrown.
 	default:
-		d.Chk.Fail("No pitch should be %+v, which is of type %s!\n", v, reflect.TypeOf(v).String())
+		d.Chk.Fail("Impossible pitch", "No pitch should be %+v, which is of type %s!\n", v, reflect.TypeOf(v).String())
 	}
 	return
 }
 
-func processInning(m MapOfStringToValue) map[string][]Pitch {
-	pitchCounts := map[string][]Pitch{}
+func (m MapOfStringToValue) getAsString(k string) string {
+	return m.Get(k).(types.String).String()
+}
+
+func processInning(m MapOfStringToValue) map[string][]PitchDef {
+	pitchCounts := map[string][]PitchDef{}
 
 	// This is brittle, figure out how to do it without being super verbose.
-	top := m.Get("top")
-	if _, ok := top.(types.Map); !ok {
+	top, ok := m.Get("top").(MapOfStringToValue)
+	if !ok {
 		// If "top" is anything other than a map, give up
 		return pitchCounts
 	}
 
-	halves := []MapOfStringToValue{
-		top.(MapOfStringToValue),
-	}
+	halves := []MapOfStringToValue{top}
 
 	if bot := m.Get("bottom"); bot != nil {
 		halves = append(halves, bot.(MapOfStringToValue))
@@ -82,71 +85,63 @@ func processInning(m MapOfStringToValue) map[string][]Pitch {
 	for _, half := range halves {
 		atbat := half.Get("atbat")
 		switch atbat := atbat.(type) {
-		case ListOfMapOfStringToValue:
+		case types.List:
 			for i := uint64(0); i < atbat.Len(); i++ {
-				ab := atbat.Get(i)
-				addPitch(ab)
+				addPitch(atbat.Get(i).(MapOfStringToValue))
 			}
 		case MapOfStringToValue:
 			// Apparently, if there's only one, it's encoded directly as a singleton. Yay, data!
 			addPitch(atbat)
 		default:
+			d.Chk.Fail("Impossible half", "No half should be %+v, which is of type %s!\n", atbat, reflect.TypeOf(atbat).String())
 		}
 	}
 	return pitchCounts
 }
 
-func getIndex(input types.List) MapOfStringToListOfPitch {
-	mu := sync.Mutex{}
-	pitchers := NewMapOfStringToString()
+func getIndex(input ListOfRefOfMapOfStringToValue, cs chunks.ChunkStore) MapOfStringToListOfPitch {
+	pitcherMu := sync.Mutex{}
+	inningMu := sync.Mutex{}
+	pitchers := map[string]string{}
+	innings := []map[string][]PitchDef{}
 
 	// Walk through the list in inputDataset and basically switch
 	// on the top-level key to know if it's an inning or a pitcher.
-	innings := input.MapP(512, func(item types.Value, i uint64) interface{} {
-		m := item.(MapOfStringToValue)
+	input.IterAllP(512, func(item RefOfMapOfStringToValue, i uint64) {
+		m := item.TargetValue(cs)
 
 		if key := "inning"; m.Has(key) {
-			return processInning(m.Get(key).(MapOfStringToValue))
+			inning := processInning(m.Get(key).(MapOfStringToValue))
+			inningMu.Lock()
+			innings = append(innings, inning)
+			inningMu.Unlock()
 		}
 
 		if key := "Player"; m.Has(key) {
-			id, name := processPitcher(m.Get(key).(MapOfStringToString))
+			id, name := processPitcher(m.Get(key).(MapOfStringToValue))
+
 			if id != "" && name != "" {
-				mu.Lock()
-				pitchers = pitchers.Set(id, name)
-				mu.Unlock()
+				pitcherMu.Lock()
+				pitchers[id] = name
+				pitcherMu.Unlock()
 			}
 		}
-
-		return nil
 	})
 
-	pitchCounts := NewMapOfStringToListOfPitch()
+	pitchCounts := MapOfStringToListOfPitchDef{}
 	for _, inning := range innings {
-		if inning == nil {
-			continue
-		}
-
-		for id, p := range inning.(map[string][]Pitch) {
-			pitches := NewListOfPitch()
-			if pitchCounts.Has(id) {
-				pitches = pitchCounts.Get(id)
-			}
-			pitchCounts = pitchCounts.Set(id, pitches.Append(p...))
+		for id, p := range inning {
+			pitchCounts[id] = append(pitchCounts[id], p...)
 		}
 	}
 
-	namedPitchCounts := NewMapOfStringToListOfPitch()
-	pitchCounts.Iter(func(id string, p ListOfPitch) (stop bool) {
-		if pitchers.Has(id) {
-			namedPitchCounts = namedPitchCounts.Set(pitchers.Get(id), p)
-		} else {
-			d.Chk.Fail("Unknown pitcher!", id)
+	namedPitchCounts := MapOfStringToListOfPitchDef{}
+	for id, p := range pitchCounts {
+		if name, ok := pitchers[id]; d.Chk.True(ok, "Unknown pitcher: %s", id) {
+			namedPitchCounts[name] = p
 		}
-		return
-	})
-
-	return namedPitchCounts
+	}
+	return namedPitchCounts.New()
 }
 
 func main() {
@@ -168,8 +163,8 @@ func main() {
 		inputDataset := dataset.NewDataset(ds, *inputID)
 		outputDataset := dataset.NewDataset(ds, *outputID)
 
-		input := inputDataset.Head().Value().(types.List)
-		output := getIndex(input)
+		input := inputDataset.Head().Value().(ListOfRefOfMapOfStringToValue)
+		output := getIndex(input, ds)
 
 		_, ok := outputDataset.Commit(output)
 		d.Exp.True(ok, "Could not commit due to conflicting edit")

clients/pitchmap/index/types.noms

@@ -3,7 +3,7 @@ struct Pitch {
   Z: Float64
 }
 
-using List(Map(String, Value))
 using Map(String, List(Pitch))
-using Map(String, String)
+using List(Ref(Map(String, Value)))
+using Ref(Map(String, Value))
 using Map(String, Value)

clients/pitchmap/index/types.noms.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"github.com/attic-labs/noms/chunks"
 	"github.com/attic-labs/noms/ref"
 	"github.com/attic-labs/noms/types"
 )
@@ -127,147 +128,6 @@ func (s Pitch) SetZ(val float64) Pitch {
 	return s
 }
 
-// ListOfMapOfStringToValue
-
-type ListOfMapOfStringToValue struct {
-	l   types.List
-	ref *ref.Ref
-}
-
-func NewListOfMapOfStringToValue() ListOfMapOfStringToValue {
-	return ListOfMapOfStringToValue{types.NewList(), &ref.Ref{}}
-}
-
-type ListOfMapOfStringToValueDef []MapOfStringToValueDef
-
-func (def ListOfMapOfStringToValueDef) New() ListOfMapOfStringToValue {
-	l := make([]types.Value, len(def))
-	for i, d := range def {
-		l[i] = d.New()
-	}
-	return ListOfMapOfStringToValue{types.NewList(l...), &ref.Ref{}}
-}
-
-func (l ListOfMapOfStringToValue) Def() ListOfMapOfStringToValueDef {
-	d := make([]MapOfStringToValueDef, l.Len())
-	for i := uint64(0); i < l.Len(); i++ {
-		d[i] = l.l.Get(i).(MapOfStringToValue).Def()
-	}
-	return d
-}
-
-func (l ListOfMapOfStringToValue) Equals(other types.Value) bool {
-	return other != nil && __typeRefForListOfMapOfStringToValue.Equals(other.TypeRef()) && l.Ref() == other.Ref()
-}
-
-func (l ListOfMapOfStringToValue) Ref() ref.Ref {
-	return types.EnsureRef(l.ref, l)
-}
-
-func (l ListOfMapOfStringToValue) Chunks() (chunks []ref.Ref) {
-	chunks = append(chunks, l.TypeRef().Chunks()...)
-	chunks = append(chunks, l.l.Chunks()...)
-	return
-}
-
-// A Noms Value that describes ListOfMapOfStringToValue.
-var __typeRefForListOfMapOfStringToValue types.TypeRef
-
-func (m ListOfMapOfStringToValue) TypeRef() types.TypeRef {
-	return __typeRefForListOfMapOfStringToValue
-}
-
-func init() {
-	__typeRefForListOfMapOfStringToValue = types.MakeCompoundTypeRef(types.ListKind, types.MakeCompoundTypeRef(types.MapKind, types.MakePrimitiveTypeRef(types.StringKind), types.MakePrimitiveTypeRef(types.ValueKind)))
-	types.RegisterValue(__typeRefForListOfMapOfStringToValue, builderForListOfMapOfStringToValue, readerForListOfMapOfStringToValue)
-}
-
-func builderForListOfMapOfStringToValue(v types.Value) types.Value {
-	return ListOfMapOfStringToValue{v.(types.List), &ref.Ref{}}
-}
-
-func readerForListOfMapOfStringToValue(v types.Value) types.Value {
-	return v.(ListOfMapOfStringToValue).l
-}
-
-func (l ListOfMapOfStringToValue) Len() uint64 {
-	return l.l.Len()
-}
-
-func (l ListOfMapOfStringToValue) Empty() bool {
-	return l.Len() == uint64(0)
-}
-
-func (l ListOfMapOfStringToValue) Get(i uint64) MapOfStringToValue {
-	return l.l.Get(i).(MapOfStringToValue)
-}
-
-func (l ListOfMapOfStringToValue) Slice(idx uint64, end uint64) ListOfMapOfStringToValue {
-	return ListOfMapOfStringToValue{l.l.Slice(idx, end), &ref.Ref{}}
-}
-
-func (l ListOfMapOfStringToValue) Set(i uint64, val MapOfStringToValue) ListOfMapOfStringToValue {
-	return ListOfMapOfStringToValue{l.l.Set(i, val), &ref.Ref{}}
-}
-
-func (l ListOfMapOfStringToValue) Append(v ...MapOfStringToValue) ListOfMapOfStringToValue {
-	return ListOfMapOfStringToValue{l.l.Append(l.fromElemSlice(v)...), &ref.Ref{}}
-}
-
-func (l ListOfMapOfStringToValue) Insert(idx uint64, v ...MapOfStringToValue) ListOfMapOfStringToValue {
-	return ListOfMapOfStringToValue{l.l.Insert(idx, l.fromElemSlice(v)...), &ref.Ref{}}
-}
-
-func (l ListOfMapOfStringToValue) Remove(idx uint64, end uint64) ListOfMapOfStringToValue {
-	return ListOfMapOfStringToValue{l.l.Remove(idx, end), &ref.Ref{}}
-}
-
-func (l ListOfMapOfStringToValue) RemoveAt(idx uint64) ListOfMapOfStringToValue {
-	return ListOfMapOfStringToValue{(l.l.RemoveAt(idx)), &ref.Ref{}}
-}
-
-func (l ListOfMapOfStringToValue) fromElemSlice(p []MapOfStringToValue) []types.Value {
-	r := make([]types.Value, len(p))
-	for i, v := range p {
-		r[i] = v
-	}
-	return r
-}
-
-type ListOfMapOfStringToValueIterCallback func(v MapOfStringToValue, i uint64) (stop bool)
-
-func (l ListOfMapOfStringToValue) Iter(cb ListOfMapOfStringToValueIterCallback) {
-	l.l.Iter(func(v types.Value, i uint64) bool {
-		return cb(v.(MapOfStringToValue), i)
-	})
-}
-
-type ListOfMapOfStringToValueIterAllCallback func(v MapOfStringToValue, i uint64)
-
-func (l ListOfMapOfStringToValue) IterAll(cb ListOfMapOfStringToValueIterAllCallback) {
-	l.l.IterAll(func(v types.Value, i uint64) {
-		cb(v.(MapOfStringToValue), i)
-	})
-}
-
-func (l ListOfMapOfStringToValue) IterAllP(concurrency int, cb ListOfMapOfStringToValueIterAllCallback) {
-	l.l.IterAllP(concurrency, func(v types.Value, i uint64) {
-		cb(v.(MapOfStringToValue), i)
-	})
-}
-
-type ListOfMapOfStringToValueFilterCallback func(v MapOfStringToValue, i uint64) (keep bool)
-
-func (l ListOfMapOfStringToValue) Filter(cb ListOfMapOfStringToValueFilterCallback) ListOfMapOfStringToValue {
-	nl := NewListOfMapOfStringToValue()
-	l.IterAll(func(v MapOfStringToValue, i uint64) {
-		if cb(v, i) {
-			nl = nl.Append(v)
-		}
-	})
-	return nl
-}
-
 // MapOfStringToListOfPitch
 
 type MapOfStringToListOfPitch struct {
@@ -394,130 +254,196 @@ func (m MapOfStringToListOfPitch) Filter(cb MapOfStringToListOfPitchFilterCallba
 	return nm
 }
 
-// MapOfStringToString
+// ListOfRefOfMapOfStringToValue
 
-type MapOfStringToString struct {
-	m   types.Map
+type ListOfRefOfMapOfStringToValue struct {
+	l   types.List
 	ref *ref.Ref
 }
 
-func NewMapOfStringToString() MapOfStringToString {
-	return MapOfStringToString{types.NewMap(), &ref.Ref{}}
+func NewListOfRefOfMapOfStringToValue() ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{types.NewList(), &ref.Ref{}}
 }
 
-type MapOfStringToStringDef map[string]string
+type ListOfRefOfMapOfStringToValueDef []ref.Ref
 
-func (def MapOfStringToStringDef) New() MapOfStringToString {
-	kv := make([]types.Value, 0, len(def)*2)
-	for k, v := range def {
-		kv = append(kv, types.NewString(k), types.NewString(v))
+func (def ListOfRefOfMapOfStringToValueDef) New() ListOfRefOfMapOfStringToValue {
+	l := make([]types.Value, len(def))
+	for i, d := range def {
+		l[i] = NewRefOfMapOfStringToValue(d)
 	}
-	return MapOfStringToString{types.NewMap(kv...), &ref.Ref{}}
+	return ListOfRefOfMapOfStringToValue{types.NewList(l...), &ref.Ref{}}
 }
 
-func (m MapOfStringToString) Def() MapOfStringToStringDef {
-	def := make(map[string]string)
-	m.m.Iter(func(k, v types.Value) bool {
-		def[k.(types.String).String()] = v.(types.String).String()
-		return false
-	})
-	return def
+func (l ListOfRefOfMapOfStringToValue) Def() ListOfRefOfMapOfStringToValueDef {
+	d := make([]ref.Ref, l.Len())
+	for i := uint64(0); i < l.Len(); i++ {
+		d[i] = l.l.Get(i).(RefOfMapOfStringToValue).TargetRef()
+	}
+	return d
 }
 
-func (m MapOfStringToString) Equals(other types.Value) bool {
-	return other != nil && __typeRefForMapOfStringToString.Equals(other.TypeRef()) && m.Ref() == other.Ref()
+func (l ListOfRefOfMapOfStringToValue) Equals(other types.Value) bool {
+	return other != nil && __typeRefForListOfRefOfMapOfStringToValue.Equals(other.TypeRef()) && l.Ref() == other.Ref()
 }
 
-func (m MapOfStringToString) Ref() ref.Ref {
-	return types.EnsureRef(m.ref, m)
+func (l ListOfRefOfMapOfStringToValue) Ref() ref.Ref {
+	return types.EnsureRef(l.ref, l)
 }
 
-func (m MapOfStringToString) Chunks() (chunks []ref.Ref) {
-	chunks = append(chunks, m.TypeRef().Chunks()...)
-	chunks = append(chunks, m.m.Chunks()...)
+func (l ListOfRefOfMapOfStringToValue) Chunks() (chunks []ref.Ref) {
+	chunks = append(chunks, l.TypeRef().Chunks()...)
+	chunks = append(chunks, l.l.Chunks()...)
 	return
 }
 
-// A Noms Value that describes MapOfStringToString.
-var __typeRefForMapOfStringToString types.TypeRef
+// A Noms Value that describes ListOfRefOfMapOfStringToValue.
+var __typeRefForListOfRefOfMapOfStringToValue types.TypeRef
 
-func (m MapOfStringToString) TypeRef() types.TypeRef {
-	return __typeRefForMapOfStringToString
+func (m ListOfRefOfMapOfStringToValue) TypeRef() types.TypeRef {
+	return __typeRefForListOfRefOfMapOfStringToValue
 }
 
 func init() {
-	__typeRefForMapOfStringToString = types.MakeCompoundTypeRef(types.MapKind, types.MakePrimitiveTypeRef(types.StringKind), types.MakePrimitiveTypeRef(types.StringKind))
-	types.RegisterValue(__typeRefForMapOfStringToString, builderForMapOfStringToString, readerForMapOfStringToString)
+	__typeRefForListOfRefOfMapOfStringToValue = types.MakeCompoundTypeRef(types.ListKind, types.MakeCompoundTypeRef(types.RefKind, types.MakeCompoundTypeRef(types.MapKind, types.MakePrimitiveTypeRef(types.StringKind), types.MakePrimitiveTypeRef(types.ValueKind))))
+	types.RegisterValue(__typeRefForListOfRefOfMapOfStringToValue, builderForListOfRefOfMapOfStringToValue, readerForListOfRefOfMapOfStringToValue)
 }
 
-func builderForMapOfStringToString(v types.Value) types.Value {
-	return MapOfStringToString{v.(types.Map), &ref.Ref{}}
+func builderForListOfRefOfMapOfStringToValue(v types.Value) types.Value {
+	return ListOfRefOfMapOfStringToValue{v.(types.List), &ref.Ref{}}
 }
 
-func readerForMapOfStringToString(v types.Value) types.Value {
-	return v.(MapOfStringToString).m
+func readerForListOfRefOfMapOfStringToValue(v types.Value) types.Value {
+	return v.(ListOfRefOfMapOfStringToValue).l
 }
 
-func (m MapOfStringToString) Empty() bool {
-	return m.m.Empty()
+func (l ListOfRefOfMapOfStringToValue) Len() uint64 {
+	return l.l.Len()
 }
 
-func (m MapOfStringToString) Len() uint64 {
-	return m.m.Len()
+func (l ListOfRefOfMapOfStringToValue) Empty() bool {
+	return l.Len() == uint64(0)
 }
 
-func (m MapOfStringToString) Has(p string) bool {
-	return m.m.Has(types.NewString(p))
+func (l ListOfRefOfMapOfStringToValue) Get(i uint64) RefOfMapOfStringToValue {
+	return l.l.Get(i).(RefOfMapOfStringToValue)
 }
 
-func (m MapOfStringToString) Get(p string) string {
-	return m.m.Get(types.NewString(p)).(types.String).String()
+func (l ListOfRefOfMapOfStringToValue) Slice(idx uint64, end uint64) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Slice(idx, end), &ref.Ref{}}
 }
 
-func (m MapOfStringToString) MaybeGet(p string) (string, bool) {
-	v, ok := m.m.MaybeGet(types.NewString(p))
-	if !ok {
-		return "", false
+func (l ListOfRefOfMapOfStringToValue) Set(i uint64, val RefOfMapOfStringToValue) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Set(i, val), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Append(v ...RefOfMapOfStringToValue) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Append(l.fromElemSlice(v)...), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Insert(idx uint64, v ...RefOfMapOfStringToValue) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Insert(idx, l.fromElemSlice(v)...), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Remove(idx uint64, end uint64) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Remove(idx, end), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) RemoveAt(idx uint64) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{(l.l.RemoveAt(idx)), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) fromElemSlice(p []RefOfMapOfStringToValue) []types.Value {
+	r := make([]types.Value, len(p))
+	for i, v := range p {
+		r[i] = v
 	}
-	return v.(types.String).String(), ok
+	return r
 }
 
-func (m MapOfStringToString) Set(k string, v string) MapOfStringToString {
-	return MapOfStringToString{m.m.Set(types.NewString(k), types.NewString(v)), &ref.Ref{}}
-}
+type ListOfRefOfMapOfStringToValueIterCallback func(v RefOfMapOfStringToValue, i uint64) (stop bool)
 
-// TODO: Implement SetM?
-
-func (m MapOfStringToString) Remove(p string) MapOfStringToString {
-	return MapOfStringToString{m.m.Remove(types.NewString(p)), &ref.Ref{}}
-}
-
-type MapOfStringToStringIterCallback func(k string, v string) (stop bool)
-
-func (m MapOfStringToString) Iter(cb MapOfStringToStringIterCallback) {
-	m.m.Iter(func(k, v types.Value) bool {
-		return cb(k.(types.String).String(), v.(types.String).String())
+func (l ListOfRefOfMapOfStringToValue) Iter(cb ListOfRefOfMapOfStringToValueIterCallback) {
+	l.l.Iter(func(v types.Value, i uint64) bool {
+		return cb(v.(RefOfMapOfStringToValue), i)
 	})
 }
 
-type MapOfStringToStringIterAllCallback func(k string, v string)
+type ListOfRefOfMapOfStringToValueIterAllCallback func(v RefOfMapOfStringToValue, i uint64)
 
-func (m MapOfStringToString) IterAll(cb MapOfStringToStringIterAllCallback) {
-	m.m.IterAll(func(k, v types.Value) {
-		cb(k.(types.String).String(), v.(types.String).String())
+func (l ListOfRefOfMapOfStringToValue) IterAll(cb ListOfRefOfMapOfStringToValueIterAllCallback) {
+	l.l.IterAll(func(v types.Value, i uint64) {
+		cb(v.(RefOfMapOfStringToValue), i)
 	})
 }
 
-type MapOfStringToStringFilterCallback func(k string, v string) (keep bool)
+func (l ListOfRefOfMapOfStringToValue) IterAllP(concurrency int, cb ListOfRefOfMapOfStringToValueIterAllCallback) {
+	l.l.IterAllP(concurrency, func(v types.Value, i uint64) {
+		cb(v.(RefOfMapOfStringToValue), i)
+	})
+}
 
-func (m MapOfStringToString) Filter(cb MapOfStringToStringFilterCallback) MapOfStringToString {
-	nm := NewMapOfStringToString()
-	m.IterAll(func(k string, v string) {
-		if cb(k, v) {
-			nm = nm.Set(k, v)
+type ListOfRefOfMapOfStringToValueFilterCallback func(v RefOfMapOfStringToValue, i uint64) (keep bool)
+
+func (l ListOfRefOfMapOfStringToValue) Filter(cb ListOfRefOfMapOfStringToValueFilterCallback) ListOfRefOfMapOfStringToValue {
+	nl := NewListOfRefOfMapOfStringToValue()
+	l.IterAll(func(v RefOfMapOfStringToValue, i uint64) {
+		if cb(v, i) {
+			nl = nl.Append(v)
 		}
 	})
-	return nm
+	return nl
+}
+
+// RefOfMapOfStringToValue
+
+type RefOfMapOfStringToValue struct {
+	target ref.Ref
+	ref    *ref.Ref
+}
+
+func NewRefOfMapOfStringToValue(target ref.Ref) RefOfMapOfStringToValue {
+	return RefOfMapOfStringToValue{target, &ref.Ref{}}
+}
+
+func (r RefOfMapOfStringToValue) TargetRef() ref.Ref {
+	return r.target
+}
+
+func (r RefOfMapOfStringToValue) Ref() ref.Ref {
+	return types.EnsureRef(r.ref, r)
+}
+
+func (r RefOfMapOfStringToValue) Equals(other types.Value) bool {
+	return other != nil && __typeRefForRefOfMapOfStringToValue.Equals(other.TypeRef()) && r.Ref() == other.Ref()
+}
+
+func (r RefOfMapOfStringToValue) Chunks() (chunks []ref.Ref) {
+	chunks = append(chunks, r.TypeRef().Chunks()...)
+	chunks = append(chunks, r.target)
+	return
+}
+
+// A Noms Value that describes RefOfMapOfStringToValue.
+var __typeRefForRefOfMapOfStringToValue types.TypeRef
+
+func (m RefOfMapOfStringToValue) TypeRef() types.TypeRef {
+	return __typeRefForRefOfMapOfStringToValue
+}
+
+func init() {
+	__typeRefForRefOfMapOfStringToValue = types.MakeCompoundTypeRef(types.RefKind, types.MakeCompoundTypeRef(types.MapKind, types.MakePrimitiveTypeRef(types.StringKind), types.MakePrimitiveTypeRef(types.ValueKind)))
+	types.RegisterFromValFunction(__typeRefForRefOfMapOfStringToValue, func(v types.Value) types.Value {
+		return NewRefOfMapOfStringToValue(v.(types.Ref).TargetRef())
+	})
+}
+
+func (r RefOfMapOfStringToValue) TargetValue(cs chunks.ChunkSource) MapOfStringToValue {
+	return types.ReadValue(r.target, cs).(MapOfStringToValue)
+}
+
+func (r RefOfMapOfStringToValue) SetTargetValue(val MapOfStringToValue, cs chunks.ChunkSink) RefOfMapOfStringToValue {
+	return NewRefOfMapOfStringToValue(types.WriteValue(val, cs))
 }
 
 // MapOfStringToValue

clients/util/rungen.go

@@ -0,0 +1,3 @@
+package util
+
+//go:generate go run ../../nomdl/codegen/codegen.go

clients/util/types.noms

@@ -0,0 +1 @@
+using List(Ref(Map(String, Value)))

clients/util/types.noms.go

@@ -0,0 +1,317 @@
+// This file was generated by nomdl/codegen.
+
+package util
+
+import (
+	"github.com/attic-labs/noms/chunks"
+	"github.com/attic-labs/noms/ref"
+	"github.com/attic-labs/noms/types"
+)
+
+// ListOfRefOfMapOfStringToValue
+
+type ListOfRefOfMapOfStringToValue struct {
+	l   types.List
+	ref *ref.Ref
+}
+
+func NewListOfRefOfMapOfStringToValue() ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{types.NewList(), &ref.Ref{}}
+}
+
+type ListOfRefOfMapOfStringToValueDef []ref.Ref
+
+func (def ListOfRefOfMapOfStringToValueDef) New() ListOfRefOfMapOfStringToValue {
+	l := make([]types.Value, len(def))
+	for i, d := range def {
+		l[i] = NewRefOfMapOfStringToValue(d)
+	}
+	return ListOfRefOfMapOfStringToValue{types.NewList(l...), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Def() ListOfRefOfMapOfStringToValueDef {
+	d := make([]ref.Ref, l.Len())
+	for i := uint64(0); i < l.Len(); i++ {
+		d[i] = l.l.Get(i).(RefOfMapOfStringToValue).TargetRef()
+	}
+	return d
+}
+
+func (l ListOfRefOfMapOfStringToValue) InternalImplementation() types.List {
+	return l.l
+}
+
+func (l ListOfRefOfMapOfStringToValue) Equals(other types.Value) bool {
+	return other != nil && __typeRefForListOfRefOfMapOfStringToValue.Equals(other.TypeRef()) && l.Ref() == other.Ref()
+}
+
+func (l ListOfRefOfMapOfStringToValue) Ref() ref.Ref {
+	return types.EnsureRef(l.ref, l)
+}
+
+func (l ListOfRefOfMapOfStringToValue) Chunks() (chunks []ref.Ref) {
+	chunks = append(chunks, l.TypeRef().Chunks()...)
+	chunks = append(chunks, l.l.Chunks()...)
+	return
+}
+
+// A Noms Value that describes ListOfRefOfMapOfStringToValue.
+var __typeRefForListOfRefOfMapOfStringToValue types.TypeRef
+
+func (m ListOfRefOfMapOfStringToValue) TypeRef() types.TypeRef {
+	return __typeRefForListOfRefOfMapOfStringToValue
+}
+
+func init() {
+	__typeRefForListOfRefOfMapOfStringToValue = types.MakeCompoundTypeRef(types.ListKind, types.MakeCompoundTypeRef(types.RefKind, types.MakeCompoundTypeRef(types.MapKind, types.MakePrimitiveTypeRef(types.StringKind), types.MakePrimitiveTypeRef(types.ValueKind))))
+	types.RegisterFromValFunction(__typeRefForListOfRefOfMapOfStringToValue, func(v types.Value) types.Value {
+		return ListOfRefOfMapOfStringToValue{v.(types.List), &ref.Ref{}}
+	})
+}
+
+func (l ListOfRefOfMapOfStringToValue) Len() uint64 {
+	return l.l.Len()
+}
+
+func (l ListOfRefOfMapOfStringToValue) Empty() bool {
+	return l.Len() == uint64(0)
+}
+
+func (l ListOfRefOfMapOfStringToValue) Get(i uint64) RefOfMapOfStringToValue {
+	return l.l.Get(i).(RefOfMapOfStringToValue)
+}
+
+func (l ListOfRefOfMapOfStringToValue) Slice(idx uint64, end uint64) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Slice(idx, end), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Set(i uint64, val RefOfMapOfStringToValue) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Set(i, val), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Append(v ...RefOfMapOfStringToValue) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Append(l.fromElemSlice(v)...), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Insert(idx uint64, v ...RefOfMapOfStringToValue) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Insert(idx, l.fromElemSlice(v)...), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) Remove(idx uint64, end uint64) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{l.l.Remove(idx, end), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) RemoveAt(idx uint64) ListOfRefOfMapOfStringToValue {
+	return ListOfRefOfMapOfStringToValue{(l.l.RemoveAt(idx)), &ref.Ref{}}
+}
+
+func (l ListOfRefOfMapOfStringToValue) fromElemSlice(p []RefOfMapOfStringToValue) []types.Value {
+	r := make([]types.Value, len(p))
+	for i, v := range p {
+		r[i] = v
+	}
+	return r
+}
+
+type ListOfRefOfMapOfStringToValueIterCallback func(v RefOfMapOfStringToValue, i uint64) (stop bool)
+
+func (l ListOfRefOfMapOfStringToValue) Iter(cb ListOfRefOfMapOfStringToValueIterCallback) {
+	l.l.Iter(func(v types.Value, i uint64) bool {
+		return cb(v.(RefOfMapOfStringToValue), i)
+	})
+}
+
+type ListOfRefOfMapOfStringToValueIterAllCallback func(v RefOfMapOfStringToValue, i uint64)
+
+func (l ListOfRefOfMapOfStringToValue) IterAll(cb ListOfRefOfMapOfStringToValueIterAllCallback) {
+	l.l.IterAll(func(v types.Value, i uint64) {
+		cb(v.(RefOfMapOfStringToValue), i)
+	})
+}
+
+type ListOfRefOfMapOfStringToValueFilterCallback func(v RefOfMapOfStringToValue, i uint64) (keep bool)
+
+func (l ListOfRefOfMapOfStringToValue) Filter(cb ListOfRefOfMapOfStringToValueFilterCallback) ListOfRefOfMapOfStringToValue {
+	nl := NewListOfRefOfMapOfStringToValue()
+	l.IterAll(func(v RefOfMapOfStringToValue, i uint64) {
+		if cb(v, i) {
+			nl = nl.Append(v)
+		}
+	})
+	return nl
+}
+
+// RefOfMapOfStringToValue
+
+type RefOfMapOfStringToValue struct {
+	target ref.Ref
+	ref    *ref.Ref
+}
+
+func NewRefOfMapOfStringToValue(target ref.Ref) RefOfMapOfStringToValue {
+	return RefOfMapOfStringToValue{target, &ref.Ref{}}
+}
+
+func (r RefOfMapOfStringToValue) TargetRef() ref.Ref {
+	return r.target
+}
+
+func (r RefOfMapOfStringToValue) Ref() ref.Ref {
+	return types.EnsureRef(r.ref, r)
+}
+
+func (r RefOfMapOfStringToValue) Equals(other types.Value) bool {
+	return other != nil && __typeRefForRefOfMapOfStringToValue.Equals(other.TypeRef()) && r.Ref() == other.Ref()
+}
+
+func (r RefOfMapOfStringToValue) Chunks() (chunks []ref.Ref) {
+	chunks = append(chunks, r.TypeRef().Chunks()...)
+	chunks = append(chunks, r.target)
+	return
+}
+
+// A Noms Value that describes RefOfMapOfStringToValue.
+var __typeRefForRefOfMapOfStringToValue types.TypeRef
+
+func (m RefOfMapOfStringToValue) TypeRef() types.TypeRef {
+	return __typeRefForRefOfMapOfStringToValue
+}
+
+func init() {
+	__typeRefForRefOfMapOfStringToValue = types.MakeCompoundTypeRef(types.RefKind, types.MakeCompoundTypeRef(types.MapKind, types.MakePrimitiveTypeRef(types.StringKind), types.MakePrimitiveTypeRef(types.ValueKind)))
+	types.RegisterFromValFunction(__typeRefForRefOfMapOfStringToValue, func(v types.Value) types.Value {
+		return NewRefOfMapOfStringToValue(v.(types.Ref).TargetRef())
+	})
+}
+
+func (r RefOfMapOfStringToValue) TargetValue(cs chunks.ChunkSource) MapOfStringToValue {
+	return types.ReadValue(r.target, cs).(MapOfStringToValue)
+}
+
+func (r RefOfMapOfStringToValue) SetTargetValue(val MapOfStringToValue, cs chunks.ChunkSink) RefOfMapOfStringToValue {
+	return NewRefOfMapOfStringToValue(types.WriteValue(val, cs))
+}
+
+// MapOfStringToValue
+
+type MapOfStringToValue struct {
+	m   types.Map
+	ref *ref.Ref
+}
+
+func NewMapOfStringToValue() MapOfStringToValue {
+	return MapOfStringToValue{types.NewMap(), &ref.Ref{}}
+}
+
+type MapOfStringToValueDef map[string]types.Value
+
+func (def MapOfStringToValueDef) New() MapOfStringToValue {
+	kv := make([]types.Value, 0, len(def)*2)
+	for k, v := range def {
+		kv = append(kv, types.NewString(k), v)
+	}
+	return MapOfStringToValue{types.NewMap(kv...), &ref.Ref{}}
+}
+
+func (m MapOfStringToValue) Def() MapOfStringToValueDef {
+	def := make(map[string]types.Value)
+	m.m.Iter(func(k, v types.Value) bool {
+		def[k.(types.String).String()] = v
+		return false
+	})
+	return def
+}
+
+func (m MapOfStringToValue) InternalImplementation() types.Map {
+	return m.m
+}
+
+func (m MapOfStringToValue) Equals(other types.Value) bool {
+	return other != nil && __typeRefForMapOfStringToValue.Equals(other.TypeRef()) && m.Ref() == other.Ref()
+}
+
+func (m MapOfStringToValue) Ref() ref.Ref {
+	return types.EnsureRef(m.ref, m)
+}
+
+func (m MapOfStringToValue) Chunks() (chunks []ref.Ref) {
+	chunks = append(chunks, m.TypeRef().Chunks()...)
+	chunks = append(chunks, m.m.Chunks()...)
+	return
+}
+
+// A Noms Value that describes MapOfStringToValue.
+var __typeRefForMapOfStringToValue types.TypeRef
+
+func (m MapOfStringToValue) TypeRef() types.TypeRef {
+	return __typeRefForMapOfStringToValue
+}
+
+func init() {
+	__typeRefForMapOfStringToValue = types.MakeCompoundTypeRef(types.MapKind, types.MakePrimitiveTypeRef(types.StringKind), types.MakePrimitiveTypeRef(types.ValueKind))
+	types.RegisterFromValFunction(__typeRefForMapOfStringToValue, func(v types.Value) types.Value {
+		return MapOfStringToValue{v.(types.Map), &ref.Ref{}}
+	})
+}
+
+func (m MapOfStringToValue) Empty() bool {
+	return m.m.Empty()
+}
+
+func (m MapOfStringToValue) Len() uint64 {
+	return m.m.Len()
+}
+
+func (m MapOfStringToValue) Has(p string) bool {
+	return m.m.Has(types.NewString(p))
+}
+
+func (m MapOfStringToValue) Get(p string) types.Value {
+	return m.m.Get(types.NewString(p))
+}
+
+func (m MapOfStringToValue) MaybeGet(p string) (types.Value, bool) {
+	v, ok := m.m.MaybeGet(types.NewString(p))
+	if !ok {
+		return types.Bool(false), false
+	}
+	return v, ok
+}
+
+func (m MapOfStringToValue) Set(k string, v types.Value) MapOfStringToValue {
+	return MapOfStringToValue{m.m.Set(types.NewString(k), v), &ref.Ref{}}
+}
+
+// TODO: Implement SetM?
+
+func (m MapOfStringToValue) Remove(p string) MapOfStringToValue {
+	return MapOfStringToValue{m.m.Remove(types.NewString(p)), &ref.Ref{}}
+}
+
+type MapOfStringToValueIterCallback func(k string, v types.Value) (stop bool)
+
+func (m MapOfStringToValue) Iter(cb MapOfStringToValueIterCallback) {
+	m.m.Iter(func(k, v types.Value) bool {
+		return cb(k.(types.String).String(), v)
+	})
+}
+
+type MapOfStringToValueIterAllCallback func(k string, v types.Value)
+
+func (m MapOfStringToValue) IterAll(cb MapOfStringToValueIterAllCallback) {
+	m.m.IterAll(func(k, v types.Value) {
+		cb(k.(types.String).String(), v)
+	})
+}
+
+type MapOfStringToValueFilterCallback func(k string, v types.Value) (keep bool)
+
+func (m MapOfStringToValue) Filter(cb MapOfStringToValueFilterCallback) MapOfStringToValue {
+	nm := NewMapOfStringToValue()
+	m.IterAll(func(k string, v types.Value) {
+		if cb(k, v) {
+			nm = nm.Set(k, v)
+		}
+	})
+	return nm
+}

clients/util/util.go

@@ -41,11 +41,11 @@ func NomsValueFromDecodedJSON(o interface{}) types.Value {
 		}
 		return out
 	case map[string]interface{}:
-		out := types.NewMap()
+		out := NewMapOfStringToValue()
 		for k, v := range o {
 			nv := NomsValueFromDecodedJSON(v)
 			if nv != nil {
-				out = out.Set(types.NewString(k), nv)
+				out = out.Set(k, nv)
 			}
 		}
 		return out

clients/util/util_test.go

@@ -38,10 +38,11 @@ func (suite *LibTestSuite) TestCompositeTypes() {
 	//  "list": [false true],
 	//  "map": {"nested": "string"}
 	// }
-	m := types.NewMap(
-		types.NewString("string"), types.NewString("string"),
-		types.NewString("list"), types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
-		types.NewString("map"), types.NewMap(types.NewString("nested"), types.NewString("string")))
+	m := MapOfStringToValueDef{
+		"string": types.NewString("string"),
+		"list":   types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
+		"map":    MapOfStringToValueDef{"nested": types.NewString("string")}.New(),
+	}.New()
 	suite.EqualValues(m, NomsValueFromDecodedJSON(map[string]interface{}{
 		"string": "string",
 		"list":   []interface{}{false, true},

clients/xml_importer/xml_importer.go

@@ -34,7 +34,7 @@ type fileIndex struct {
 }
 
 type refIndex struct {
-	ref   types.Ref
+	ref   ref.Ref
 	index int
 }
 
@@ -84,6 +84,7 @@ func main() {
 
 		wg := sync.WaitGroup{}
 		importXml := func() {
+			expectedType := util.NewMapOfStringToValue()
 			for f := range filesChan {
 				file, err := os.Open(f.path)
 				d.Exp.NoError(err, "Error getting XML")
@@ -94,10 +95,11 @@ func main() {
 				file.Close()
 
 				nomsObj := util.NomsValueFromDecodedJSON(object)
-				r := types.NewRef(ref.Ref{})
+				d.Chk.IsType(expectedType, nomsObj)
+				r := ref.Ref{}
 
 				if !*noIO {
-					r = types.NewRef(types.WriteValue(nomsObj, ds.Store()))
+					r = types.WriteValue(nomsObj, ds.Store())
 				}
 
 				refsChan <- refIndex{r, f.index}
@@ -122,15 +124,13 @@ func main() {
 		}
 		sort.Sort(refList)
 
-		refs := make([]types.Value, 0, len(refList))
-		for _, r := range refList {
-			refs = append(refs, r.ref)
+		refs := make(util.ListOfRefOfMapOfStringToValueDef, len(refList))
+		for idx, r := range refList {
+			refs[idx] = r.ref
 		}
 
-		list := types.NewList(refs...)
-
 		if !*noIO {
-			_, ok := ds.Commit(list)
+			_, ok := ds.Commit(refs.New())
 			d.Exp.True(ok, "Could not commit due to conflicting edit")
 		}
 

types/set.go

@@ -16,10 +16,6 @@ type Set struct {
 	ref  *ref.Ref
 }
 
-type setIterCallback func(v Value) bool
-type setIterAllCallback func(v Value)
-type setFilterCallback func(v Value) (keep bool)
-
 var setTypeRef = MakeCompoundTypeRef(SetKind, MakePrimitiveTypeRef(ValueKind))
 
 func NewSet(v ...Value) Set {
@@ -80,6 +76,8 @@ func (s Set) Subtract(others ...Set) Set {
 	return result
 }
 
+type setIterCallback func(v Value) bool
+
 func (s Set) Iter(cb setIterCallback) {
 	for _, v := range s.data {
 		if cb(v) {
@@ -88,6 +86,8 @@ func (s Set) Iter(cb setIterCallback) {
 	}
 }
 
+type setIterAllCallback func(v Value)
+
 func (s Set) IterAll(cb setIterAllCallback) {
 	for _, v := range s.data {
 		cb(v)
@@ -116,6 +116,8 @@ func (s Set) IterAllP(concurrency int, f setIterAllCallback) {
 	wg.Wait()
 }
 
+type setFilterCallback func(v Value) (keep bool)
+
 func (s Set) Filter(cb setFilterCallback) Set {
 	data := setData{}
 	for _, v := range s.data {