Add pull command line tool, test for pulling an initial commit, address comments

In addition to putting in the 'pull' tool that I forgot to add in my initial PR,
I added an extra unit test to cover a case that we found to be buggy, as well
as addressing some comments by aa and arv.

1) Switched to io.Copy in CopyChunks
2) Added NewFlagsWithPrefix()
3) Cleaned up some error reporting

chunks/aws_store.go

@@ -194,9 +194,6 @@ type awsStoreFlags struct {
 }
 
 func awsFlags(prefix string) awsStoreFlags {
-	if prefix != "" {
-		prefix += "-"
-	}
 	return awsStoreFlags{
 		flag.String(prefix+"aws-store-bucket", "", "aws bucket to create an aws-based chunkstore in"),
 		flag.String(prefix+"aws-store-dynamo-table", "noms-root", "dynamodb table to store the root of the aws-based chunkstore in"),

chunks/chunk_store.go

@@ -39,7 +39,11 @@ type Flags struct {
 	nop    nopStoreFlags
 }
 
-func NewFlags(prefix string) Flags {
+func NewFlags() Flags {
+	return NewFlagsWithPrefix("")
+}
+
+func NewFlagsWithPrefix(prefix string) Flags {
 	return Flags{
 		awsFlags(prefix),
 		fileFlags(prefix),

chunks/file_store.go

@@ -151,9 +151,6 @@ type fileStoreFlags struct {
 }
 
 func fileFlags(prefix string) fileStoreFlags {
-	if prefix != "" {
-		prefix += "-"
-	}
 	return fileStoreFlags{
 		flag.String(prefix+"file-store", "", "directory to use for a file-based chunkstore"),
 		flag.String(prefix+"file-store-root", "root", "filename which holds the root ref in the filestore"),

chunks/memory_store.go

@@ -69,9 +69,6 @@ type memoryStoreFlags struct {
 }
 
 func memoryFlags(prefix string) memoryStoreFlags {
-	if prefix != "" {
-		prefix += "-"
-	}
 	return memoryStoreFlags{
 		flag.Bool(prefix+"memory-store", false, "use a memory-based (ephemeral, and private to this application) chunkstore"),
 	}

chunks/nop_store.go

@@ -41,9 +41,6 @@ type nopStoreFlags struct {
 }
 
 func nopFlags(prefix string) nopStoreFlags {
-	if prefix != "" {
-		prefix += "-"
-	}
 	return nopStoreFlags{
 		flag.Bool(prefix+"nop-store", false, "use a /dev/null-esque chunkstore"),
 	}

clients/counter/counter.go

@@ -10,7 +10,7 @@ import (
 )
 
 func main() {
-	dsFlags := dataset.Flags()
+	dsFlags := dataset.NewFlags()
 	flag.Parse()
 
 	ds := dsFlags.CreateDataset()

clients/csv_importer/csv_importer.go

@@ -13,7 +13,7 @@ import (
 )
 
 func main() {
-	dsFlags := dataset.Flags()
+	dsFlags := dataset.NewFlags()
 	flag.Parse()
 	ds := dsFlags.CreateDataset()
 	if ds == nil {

clients/flickr/flickr.go

@@ -33,7 +33,7 @@ type flickrCall struct {
 }
 
 func main() {
-	dsFlags := dataset.Flags()
+	dsFlags := dataset.NewFlags()
 	flag.Parse()
 
 	if *apiKeyFlag == "" || *apiKeySecretFlag == "" {

clients/json_importer/json_importer.go

@@ -12,7 +12,7 @@ import (
 )
 
 func main() {
-	dsFlags := dataset.Flags()
+	dsFlags := dataset.NewFlags()
 	flag.Parse()
 	ds := dsFlags.CreateDataset()
 	if ds == nil {

clients/noms-pull/.gitignore

@@ -0,0 +1 @@
+noms-pull

clients/noms-pull/noms-pull.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"runtime/pprof"
+
+	"github.com/attic-labs/noms/dataset"
+	"github.com/attic-labs/noms/dbg"
+	"github.com/attic-labs/noms/sync"
+)
+
+var (
+	cpuprofile    = flag.String("cpuprofile", "", "write cpu profile to file")
+	localDsFlags  = dataset.NewFlagsWithPrefix("local-")
+	remoteDsFlags = dataset.NewFlagsWithPrefix("remote-")
+)
+
+func main() {
+	flag.Parse()
+
+	source := remoteDsFlags.CreateDataset()
+	sink := localDsFlags.CreateDataset()
+	if source == nil || sink == nil {
+		flag.Usage()
+		return
+	}
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		dbg.Chk.NoError(err, "Can't create cpu profile file.")
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	newHead := source.Heads().Ref()
+	refs, err := sync.DiffHeadsByRef(sink.Heads().Ref(), newHead, source)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	err = sync.CopyChunks(refs, source, sink)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	_, err = sync.SetNewHeads(newHead, *sink)
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

clients/pitchmap/index/index.go

@@ -145,7 +145,7 @@ func getIndex(input types.List) MapOfStringToListOfPitch {
 }
 
 func main() {
-	csFlags := chunks.NewFlags("")
+	csFlags := chunks.NewFlags()
 	flag.Parse()
 
 	cs := csFlags.CreateStore()

clients/server/server.go

@@ -80,7 +80,7 @@ func (s server) handleGetDataset(w http.ResponseWriter, id string) {
 }
 
 func main() {
-	flags := chunks.NewFlags("")
+	flags := chunks.NewFlags()
 	flag.Parse()
 
 	cs := flags.CreateStore()

clients/xml_importer/xml_importer.go

@@ -27,7 +27,7 @@ var (
 )
 
 func main() {
-	dsFlags := dataset.Flags()
+	dsFlags := dataset.NewFlags()
 	flag.Usage = customUsage
 	flag.Parse()
 	ds := dsFlags.CreateDataset()

dataset/dataset.go

@@ -28,10 +28,14 @@ type datasetFlags struct {
 	datasetID *string
 }
 
-func Flags() datasetFlags {
+func NewFlags() datasetFlags {
+	return NewFlagsWithPrefix("")
+}
+
+func NewFlagsWithPrefix(prefix string) datasetFlags {
 	return datasetFlags{
-		chunks.NewFlags(""),
-		flag.String("dataset-id", "", "dataset id to store data for"),
+		chunks.NewFlagsWithPrefix(prefix),
+		flag.String(prefix+"dataset-id", "", "dataset id to store data for"),
 	}
 }
 

sync/pull.go

@@ -1,9 +1,8 @@
 package sync
 
 import (
-	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 
 	"github.com/attic-labs/noms/chunks"
 	"github.com/attic-labs/noms/datas"
@@ -16,12 +15,12 @@ import (
 func validateRefAsSetOfCommit(r ref.Ref, cs chunks.ChunkSource) (v types.Value, err error) {
 	v, err = types.ReadValue(r, cs)
 	if v == nil {
-		return nil, errors.New("BAH")
+		return nil, fmt.Errorf("%v cannot be found", r)
 	} else if err != nil {
 		return nil, err
 	}
 	// TODO: Replace this weird recover stuff below once we have a way to determine if
-	// a Value is an instance of a custom struct type.
+	// a Value is an instance of a custom struct type. BUG #133
 	err = fmt.Errorf("%+v is not a SetOfCommit", v)
 	defer func() { recover() }()
 	datas.SetOfCommitFromVal(v) // If this panics the return value will be the error above
@@ -49,20 +48,12 @@ func CopyChunks(refs []ref.Ref, src chunks.ChunkSource, sink chunks.ChunkSink) e
 		} else if err != nil {
 			return err
 		}
-		// It seems like there should be some better way to connect a reader and a writer.
-		data, err := ioutil.ReadAll(reader)
-		if err != nil {
-			return err
-		}
 		writer := sink.Put()
 		defer writer.Close()
-		n, err := writer.Write(data)
+		_, err = io.Copy(writer, reader)
 		if err != nil {
 			return err
 		}
-		if len(data) != n {
-			return fmt.Errorf("Should have read %d bytes; only read %d.", n, len(data))
-		}
 	}
 	return nil
 }

sync/pull_test.go

@@ -11,8 +11,8 @@ import (
 )
 
 func createTestDataset(name string) dataset.Dataset {
-	var t chunks.ChunkStore = &chunks.TestStore{}
-	return dataset.NewDataset(datas.NewDataStore(t, t.(chunks.RootTracker)), name)
+	t := &chunks.TestStore{}
+	return dataset.NewDataset(datas.NewDataStore(t, t), name)
 
 }
 
@@ -58,3 +58,31 @@ func TestPull(t *testing.T) {
 	assert.True(pullee.Heads().Equals(puller.Heads()))
 
 }
+
+func TestPullFirstCommit(t *testing.T) {
+	assert := assert.New(t)
+
+	puller := createTestDataset("puller")
+	pullee := createTestDataset("pullee")
+
+	commitValue := func(v types.Value, ds dataset.Dataset) dataset.Dataset {
+		return ds.Commit(
+			datas.NewSetOfCommit().Insert(
+				datas.NewCommit().SetParents(ds.Heads().NomsValue()).SetValue(v)))
+	}
+
+	initialValue := types.NewMap(
+		types.NewString("first"), types.NewList(),
+		types.NewString("second"), types.NewList(types.Int32(2)))
+
+	pullee = commitValue(initialValue, pullee)
+
+	refs, err := DiffHeadsByRef(puller.Heads().Ref(), pullee.Heads().Ref(), pullee)
+	assert.NoError(err)
+	assert.NoError(CopyChunks(refs, pullee, puller))
+	puller, err = SetNewHeads(pullee.Heads().Ref(), puller)
+	assert.NoError(err)
+	assert.Equal(pullee.Heads().Ref(), puller.Heads().Ref())
+	assert.True(pullee.Heads().Equals(puller.Heads()))
+
+}