Add simple csv importer client

This creates a List of Maps where the first row in the CVS file is used
to determine the key names.

Fixes #29
This commit is contained in:
Erik Arvidsson
2015-07-13 11:39:21 -04:00
parent 26eb6aa7d9
commit acc70c703c
3 changed files with 86 additions and 0 deletions

1
clients/csv_importer/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
csv_importer

View File

@@ -0,0 +1,16 @@
# CSV Importer
Imports a CSV file as a List of Maps where the first row of the CSV file
describes the keys of the Map.
## Usage
To import a CSV file do:
```
$ go build csv_importer.go
$ ./csv_importer -file-store="/tmp/foo" <URL>
```
## List of CSV URLs
- https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv?accessType=DOWNLOAD

View File

@@ -0,0 +1,69 @@
package main
import (
"encoding/csv"
"flag"
"io"
"log"
"net/http"
"github.com/attic-labs/noms/chunks"
"github.com/attic-labs/noms/datas"
"github.com/attic-labs/noms/types"
)
func main() {
flags := chunks.NewFlags()
flag.Parse()
cs := flags.CreateStore()
if cs == nil {
flag.Usage()
return
}
url := flag.Arg(0)
if url == "" {
flag.Usage()
return
}
res, err := http.Get(url)
defer res.Body.Close()
if err != nil {
log.Fatalf("Error fetching %s: %+v\n", url, err)
} else if res.StatusCode != 200 {
log.Fatalf("Error fetching %s: %s\n", url, res.Status)
}
r := csv.NewReader(res.Body)
r.FieldsPerRecord = 0 // Let first row determine the number of fields.
keys, err := r.Read()
if err != nil {
log.Fatalln("Error decoding CSV: ", err)
}
value := types.NewList()
for {
row, err := r.Read()
if err == io.EOF {
break
} else if err != nil {
log.Fatalln("Error decoding CSV: ", err)
}
m := types.NewMap()
for i, v := range row {
m = m.Set(types.NewString(keys[i]), types.NewString(v))
}
value = value.Append(m)
}
ds := datas.NewDataStore(cs)
roots := ds.Roots()
ds.Commit(datas.NewRootSet().Insert(
datas.NewRoot().SetParents(
roots.NomsValue()).SetValue(
value)))
}