mirror of
https://github.com/dolthub/dolt.git
synced 2026-02-05 18:58:58 -06:00
Add simple csv importer client
This creates a List of Maps where the first row in the CVS file is used to determine the key names. Fixes #29
This commit is contained in:
1
clients/csv_importer/.gitignore
vendored
Normal file
1
clients/csv_importer/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
csv_importer
|
||||
16
clients/csv_importer/README.md
Normal file
16
clients/csv_importer/README.md
Normal file
@@ -0,0 +1,16 @@
|
||||
# CSV Importer
|
||||
|
||||
Imports a CSV file as a List of Maps where the first row of the CSV file
|
||||
describes the keys of the Map.
|
||||
|
||||
## Usage
|
||||
|
||||
To import a CSV file do:
|
||||
|
||||
```
|
||||
$ go build csv_importer.go
|
||||
$ ./csv_importer -file-store="/tmp/foo" <URL>
|
||||
```
|
||||
|
||||
## List of CSV URLs
|
||||
- https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv?accessType=DOWNLOAD
|
||||
69
clients/csv_importer/csv_importer.go
Normal file
69
clients/csv_importer/csv_importer.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/attic-labs/noms/chunks"
|
||||
"github.com/attic-labs/noms/datas"
|
||||
"github.com/attic-labs/noms/types"
|
||||
)
|
||||
|
||||
func main() {
|
||||
flags := chunks.NewFlags()
|
||||
flag.Parse()
|
||||
cs := flags.CreateStore()
|
||||
if cs == nil {
|
||||
flag.Usage()
|
||||
return
|
||||
}
|
||||
|
||||
url := flag.Arg(0)
|
||||
if url == "" {
|
||||
flag.Usage()
|
||||
return
|
||||
}
|
||||
|
||||
res, err := http.Get(url)
|
||||
defer res.Body.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("Error fetching %s: %+v\n", url, err)
|
||||
} else if res.StatusCode != 200 {
|
||||
log.Fatalf("Error fetching %s: %s\n", url, res.Status)
|
||||
}
|
||||
|
||||
r := csv.NewReader(res.Body)
|
||||
r.FieldsPerRecord = 0 // Let first row determine the number of fields.
|
||||
|
||||
keys, err := r.Read()
|
||||
if err != nil {
|
||||
log.Fatalln("Error decoding CSV: ", err)
|
||||
}
|
||||
|
||||
value := types.NewList()
|
||||
for {
|
||||
row, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
log.Fatalln("Error decoding CSV: ", err)
|
||||
}
|
||||
|
||||
m := types.NewMap()
|
||||
for i, v := range row {
|
||||
m = m.Set(types.NewString(keys[i]), types.NewString(v))
|
||||
}
|
||||
value = value.Append(m)
|
||||
}
|
||||
|
||||
ds := datas.NewDataStore(cs)
|
||||
roots := ds.Roots()
|
||||
|
||||
ds.Commit(datas.NewRootSet().Insert(
|
||||
datas.NewRoot().SetParents(
|
||||
roots.NomsValue()).SetValue(
|
||||
value)))
|
||||
}
|
||||
Reference in New Issue
Block a user