Allow xml_importer & pitchmap/index to be more streamy

clients/pitchmap/index/index.go

@@ -114,6 +114,10 @@ func getIndex(input types.List) MapOfStringToListOfPitch {
 	pitchers := NewMapOfStringToString()
 	for i := uint64(0); i < input.Len(); i++ {
 		m := input.Get(i).(types.Map)
+
+		// TODO: This really sucks
+		input.Release()
+
 		if key := types.NewString("inning"); m.Has(key) {
 			for idStr, p := range processInning(m.Get(key).(types.Map)) {
 				id := types.NewString(idStr)

clients/xml_importer/xml_importer.go

@@ -11,6 +11,8 @@ import (
 	"github.com/attic-labs/noms/clients/go"
 	"github.com/attic-labs/noms/datas"
 	"github.com/attic-labs/noms/dataset"
+	. "github.com/attic-labs/noms/dbg"
+	"github.com/attic-labs/noms/types"
 	"github.com/clbanning/mxj"
 )
 
@@ -45,7 +47,8 @@ func main() {
 		defer pprof.StopCPUProfile()
 	}
 
-	var objects []interface{}
+	list := types.NewList()
+
 	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			log.Fatalln("Cannot traverse directories: ", err)
@@ -62,16 +65,28 @@ func main() {
 		if err != nil {
 			log.Fatalln("Error decoding XML: ", err)
 		}
-		objects = append(objects, xmlObject.Old())
+		object := xmlObject.Old()
+
+		nomsObj := util.NomsValueFromDecodedJSON(object)
+		if *noIO {
+			return nil
+		}
+
+		ref, err := types.WriteValue(nomsObj, ds)
+		Chk.NoError(err)
+
+		// BUG 141
+		nomsObj, err = types.ReadValue(ref, ds)
+		Chk.NoError(err)
+
+		list = list.Append(nomsObj)
 		return nil
 	})
 
-	noms := util.NomsValueFromDecodedJSON(objects)
-
 	if !*noIO {
 		ds.Commit(datas.NewSetOfCommit().Insert(
 			datas.NewCommit().SetParents(
-				ds.Heads().NomsValue()).SetValue(noms)))
+				ds.Heads().NomsValue()).SetValue(list)))
 	}
 	if *memprofile != "" {
 		f, err := os.Create(*memprofile)

types/future.go

@@ -16,6 +16,9 @@ type Future interface {
 
 	// Fetch the Future value if necessary, then return it. Multiple calls to deref only result in one fetch.
 	Deref(cs chunks.ChunkSource) (Value, error)
+
+	// BUG 141. The lazy loading api is clearly not working.
+	Release()
 }
 
 func futuresEqual(f1, f2 Future) bool {

types/list.go

@@ -81,6 +81,13 @@ func (l List) Ref() ref.Ref {
 	return ensureRef(l.ref, l)
 }
 
+// BUG 141
+func (l List) Release() {
+	for _, f := range l.list {
+		f.Release()
+	}
+}
+
 func (l List) Equals(other Value) bool {
 	if other == nil {
 		return false

types/resolved_future.go

@@ -24,3 +24,6 @@ func (rf resolvedFuture) Val() Value {
 func (rf resolvedFuture) Deref(cs chunks.ChunkSource) (Value, error) {
 	return rf.val, nil
 }
+
+func (rf resolvedFuture) Release() {
+}

types/unresolved_future.go

@@ -35,3 +35,7 @@ func (f *unresolvedFuture) Deref(cs chunks.ChunkSource) (Value, error) {
 func (f *unresolvedFuture) Ref() ref.Ref {
 	return f.ref
 }
+
+func (f *unresolvedFuture) Release() {
+	f.val = nil
+}