diff --git a/clients/csv_importer/.gitignore b/clients/csv_importer/.gitignore new file mode 100644 index 0000000000..6e73a20b90 --- /dev/null +++ b/clients/csv_importer/.gitignore @@ -0,0 +1 @@ +csv_importer diff --git a/clients/csv_importer/README.md b/clients/csv_importer/README.md new file mode 100644 index 0000000000..3b0d98c846 --- /dev/null +++ b/clients/csv_importer/README.md @@ -0,0 +1,16 @@ +# CSV Importer + +Imports a CSV file as a List of Maps where the first row of the CSV file +describes the keys of the Map. + +## Usage + +To import a CSV file do: + +``` +$ go build csv_importer.go +$ ./csv_importer -file-store="/tmp/foo" +``` + +## List of CSV URLs + - https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv?accessType=DOWNLOAD diff --git a/clients/csv_importer/csv_importer.go b/clients/csv_importer/csv_importer.go new file mode 100644 index 0000000000..80f3d03092 --- /dev/null +++ b/clients/csv_importer/csv_importer.go @@ -0,0 +1,69 @@ +package main + +import ( + "encoding/csv" + "flag" + "io" + "log" + "net/http" + + "github.com/attic-labs/noms/chunks" + "github.com/attic-labs/noms/datas" + "github.com/attic-labs/noms/types" +) + +func main() { + flags := chunks.NewFlags() + flag.Parse() + cs := flags.CreateStore() + if cs == nil { + flag.Usage() + return + } + + url := flag.Arg(0) + if url == "" { + flag.Usage() + return + } + + res, err := http.Get(url) + defer res.Body.Close() + if err != nil { + log.Fatalf("Error fetching %s: %+v\n", url, err) + } else if res.StatusCode != 200 { + log.Fatalf("Error fetching %s: %s\n", url, res.Status) + } + + r := csv.NewReader(res.Body) + r.FieldsPerRecord = 0 // Let first row determine the number of fields. + + keys, err := r.Read() + if err != nil { + log.Fatalln("Error decoding CSV: ", err) + } + + value := types.NewList() + for { + row, err := r.Read() + if err == io.EOF { + break + } else if err != nil { + log.Fatalln("Error decoding CSV: ", err) + } + + m := types.NewMap() + for i, v := range row { + m = m.Set(types.NewString(keys[i]), types.NewString(v)) + } + value = value.Append(m) + } + + ds := datas.NewDataStore(cs) + roots := ds.Roots() + + ds.Commit(datas.NewRootSet().Insert( + datas.NewRoot().SetParents( + roots.NomsValue()).SetValue( + value))) +}