mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-06 19:35:18 -05:00
Code Issues Packages Projects Releases 160 Wiki Activity

dolt/go/libraries/doltcore: Remove references to Format_7_18.

Browse Source
This commit is contained in:
Aaron Son
2019-07-09 14:07:43 -07:00
committed by Brian Hendriks
parent 30b85b3177
commit 36b52f73d3
23 changed files with 77 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/cmd/dolt/commands/cnfcmds/cat.go
+2 -1
View File
@@ -2,6 +2,7 @@ package cnfcmds
import (
"context"
"github.com/liquidata-inc/ld/dolt/go/cmd/dolt/cli"
"github.com/liquidata-inc/ld/dolt/go/cmd/dolt/commands"
"github.com/liquidata-inc/ld/dolt/go/cmd/dolt/errhand"
@@ -109,7 +110,7 @@ func printConflicts(root *doltdb.RootValue, tblNames []string) errhand.VerboseEr
panic("")
})
p.InjectRow("fwt", untyped.NewRowFromTaggedStrings(cnfRd.GetSchema(), schema.ExtractAllColNames(cnfRd.GetSchema())))
p.InjectRow("fwt", untyped.NewRowFromTaggedStrings(tbl.Format(), cnfRd.GetSchema(), schema.ExtractAllColNames(cnfRd.GetSchema())))
p.Start()
p.Wait()
go/cmd/dolt/commands/diff.go
+3 -3
View File
@@ -362,10 +362,10 @@ func diffRows(newRows, oldRows types.Map, newSch, oldSch schema.Schema) errhand.
p := pipeline.NewAsyncPipeline(pipeline.ProcFuncForSourceFunc(src.NextDiff), sinkProcFunc, transforms, badRowCallback)
if schemasEqual {
p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(untypedUnionSch, newColNames))
p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(newRows.Format(), untypedUnionSch, newColNames))
} else {
p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(untypedUnionSch, oldColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedOld})
p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(untypedUnionSch, newColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedNew})
p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(newRows.Format(), untypedUnionSch, oldColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedOld})
p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(newRows.Format(), untypedUnionSch, newColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedNew})
}
p.Start()
go/cmd/dolt/commands/sql.go
+5 -4
View File
@@ -26,6 +26,7 @@ import (
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped/tabular"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/argparser"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/iohelp"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"vitess.io/vitess/go/vt/sqlparser"
)
@@ -298,7 +299,7 @@ func sqlShow(root *doltdb.RootValue, show *sqlparser.Show) error {
return err
}
return runPrintingPipeline(p, sch)
return runPrintingPipeline(root.VRW().Format(), p, sch)
}
// Executes a SQL select statement and prints the result to the CLI.
@@ -317,12 +318,12 @@ func sqlSelect(root *doltdb.RootValue, s *sqlparser.Select) error {
untypedSch, untypingTransform := newUntypingTransformer(resultSchema)
p.AddStage(untypingTransform)
return runPrintingPipeline(p, untypedSch)
return runPrintingPipeline(root.VRW().Format(), p, untypedSch)
}
// Adds some print-handling stages to the pipeline given and runs it, returning any error.
// Adds null-printing and fixed-width transformers. The schema given is assumed to be untyped (string-typed).
func runPrintingPipeline(p *pipeline.Pipeline, untypedSch schema.Schema) error {
func runPrintingPipeline(format *types.Format, p *pipeline.Pipeline, untypedSch schema.Schema) error {
nullPrinter := nullprinter.NewNullPrinter(untypedSch)
p.AddStage(pipeline.NewNamedTransform(nullprinter.NULL_PRINTING_STAGE, nullPrinter.ProcessRow))
@@ -344,7 +345,7 @@ func runPrintingPipeline(p *pipeline.Pipeline, untypedSch schema.Schema) error {
})
// Insert the table header row at the appropriate stage
p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(untypedSch, schema.ExtractAllColNames(untypedSch)))
p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(format, untypedSch, schema.ExtractAllColNames(untypedSch)))
p.Start()
if err := p.Wait(); err != nil {
go/cmd/dolt/commands/tblcmds/select.go
+1 -1
View File
@@ -240,7 +240,7 @@ func createPipeline(tbl *doltdb.Table, tblSch schema.Schema, outSch schema.Schem
p.RunAfter(func() { wr.Close(context.TODO()) })
// Insert the table header row at the appropriate stage
p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(outSch, colNames))
p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(tbl.Format(), outSch, colNames))
return p
}
go/libraries/doltcore/doltdb/table.go
+4
View File
@@ -52,6 +52,10 @@ func NewTable(ctx context.Context, vrw types.ValueReadWriter, schema types.Value
return &Table{vrw, tableStruct}
}
func (t *Table) Format() *types.Format {
return t.vrw.Format()
}
func (t *Table) SetConflicts(ctx context.Context, schemas Conflict, conflictData types.Map) *Table {
conflictsRef := writeValAndGetRef(ctx, t.vrw, conflictData)
go/libraries/doltcore/mvdata/data_loc.go
+3 -3
View File
@@ -137,15 +137,15 @@ func (dl *DataLocation) CreateReader(ctx context.Context, root *doltdb.RootValue
switch dl.Format {
case CsvFile:
rd, err := csv.OpenCSVReader(dl.Path, fs, csv.NewCSVInfo())
rd, err := csv.OpenCSVReader(root.VRW().Format(), dl.Path, fs, csv.NewCSVInfo())
return rd, false, err
case PsvFile:
rd, err := csv.OpenCSVReader(dl.Path, fs, csv.NewCSVInfo().SetDelim('|'))
rd, err := csv.OpenCSVReader(root.VRW().Format(), dl.Path, fs, csv.NewCSVInfo().SetDelim('|'))
return rd, false, err
case XlsxFile:
rd, err := xlsx.OpenXLSXReader(dl.Path, fs, xlsx.NewXLSXInfo(), tblName)
rd, err := xlsx.OpenXLSXReader(dl.Path, fs, xlsx.NewXLSXInfo(), root.VRW().Format(), tblName)
return rd, false, err
case JsonFile:
go/libraries/doltcore/sql/sqlselect.go
+1 -1
View File
@@ -514,7 +514,7 @@ func createSelectPipeline(ctx context.Context, root *doltdb.RootValue, selectStm
}
go func() {
defer close(cpChan)
selectStmt.intermediateRss.CrossProduct(results, cb)
selectStmt.intermediateRss.CrossProduct(root.VRW().Format(), results, cb)
}()
// TODO: we need to check errors in pipeline execution without blocking
go/libraries/doltcore/table/pipeline/errors_test.go
+2 -1
View File
@@ -3,12 +3,13 @@ package pipeline
import (
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/row"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"testing"
)
func TestTransformRowFailure(t *testing.T) {
_, sch := untyped.NewUntypedSchema("a", "b", "c")
r := untyped.NewRowFromStrings(sch, []string{"1", "2", "3"})
r := untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"1", "2", "3"})
var err error
err = &TransformRowFailure{r, "transform_name", "details"}
go/libraries/doltcore/table/pipeline/transform_test.go
+6 -6
View File
@@ -69,7 +69,7 @@ func TestPipeline(t *testing.T) {
func() {
csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true}
rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo)
rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo)
wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo)
tc := NewTransformCollection(
@@ -107,7 +107,7 @@ func TestAddingStages(t *testing.T) {
func() {
csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true}
rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo)
rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo)
wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo)
tc := NewTransformCollection(
@@ -177,7 +177,7 @@ Don,Beddoe,Bewitched (episode Humbug Not to Be Spoken Here - Season 4),1967,true
func() {
csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true}
rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo)
rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo)
wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo)
addedStages := []NamedTransform {
@@ -211,7 +211,7 @@ Don,Beddoe,Bewitched (episode Humbug Not to Be Spoken Here - Season 4),1967,true
4: "true",
5: "0",
}
injectedRow := untyped.NewRowFromTaggedStrings(schOut, injectedColumns)
injectedRow := untyped.NewRowFromTaggedStrings(types.Format_7_18, schOut, injectedColumns)
p.InjectRow("append", injectedRow)
//AnotherNew,Row,InAppendStage,3000,true,1
@@ -223,7 +223,7 @@ Don,Beddoe,Bewitched (episode Humbug Not to Be Spoken Here - Season 4),1967,true
4: "true",
5: "1",
}
injectedRow = untyped.NewRowFromTaggedStrings(schOut, injectedColumns)
injectedRow = untyped.NewRowFromTaggedStrings(types.Format_7_18, schOut, injectedColumns)
p.InjectRow("append", injectedRow)
p.RunAfter(func() { rd.Close(context.Background()) })
@@ -263,7 +263,7 @@ func TestAbort(t *testing.T) {
func() {
csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true}
rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo)
rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo)
wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo)
var wg = sync.WaitGroup{}
go/libraries/doltcore/table/typed/noms/noms_map_updater.go
+2 -3
View File
@@ -47,8 +47,7 @@ func NewNomsMapUpdater(ctx context.Context, vrw types.ValueReadWriter, m types.M
var totalStats types.AppliedEditStats
for edits := range mapChan {
var stats types.AppliedEditStats
// TODO(binformat)
m, stats = types.ApplyEdits(ctx, types.Format_7_18, edits, m)
m, stats = types.ApplyEdits(ctx, vrw.Format(), edits, m)
totalStats = totalStats.Add(stats)
if statsCB != nil {
@@ -59,7 +58,7 @@ func NewNomsMapUpdater(ctx context.Context, vrw types.ValueReadWriter, m types.M
resChan <- updateMapRes{m, nil}
}()
return &NomsMapUpdater{sch, vrw, 0, types.CreateEditAccForMapEdits(types.Format_7_18), mapChan, resChan, nil}
return &NomsMapUpdater{sch, vrw, 0, types.CreateEditAccForMapEdits(vrw.Format()), mapChan, resChan, nil}
}
// GetSchema gets the schema of the rows that this writer writes
go/libraries/doltcore/table/untyped/csv/reader.go
+6 -5
View File
@@ -28,22 +28,23 @@ type CSVReader struct {
info *CSVFileInfo
sch schema.Schema
isDone bool
format *types.Format
}
// OpenCSVReader opens a reader at a given path within a given filesys. The CSVFileInfo should describe the csv file
// being opened.
func OpenCSVReader(path string, fs filesys.ReadableFS, info *CSVFileInfo) (*CSVReader, error) {
func OpenCSVReader(format *types.Format, path string, fs filesys.ReadableFS, info *CSVFileInfo) (*CSVReader, error) {
r, err := fs.OpenForRead(path)
if err != nil {
return nil, err
}
return NewCSVReader(r, info)
return NewCSVReader(format, r, info)
}
// NewCSVReader creates a CSVReader from a given ReadCloser. The CSVFileInfo should describe the csv file being read.
func NewCSVReader(r io.ReadCloser, info *CSVFileInfo) (*CSVReader, error) {
func NewCSVReader(format *types.Format, r io.ReadCloser, info *CSVFileInfo) (*CSVReader, error) {
br := bufio.NewReaderSize(r, ReadBufSize)
colStrs, err := getColHeaders(br, info)
@@ -54,7 +55,7 @@ func NewCSVReader(r io.ReadCloser, info *CSVFileInfo) (*CSVReader, error) {
_, sch := untyped.NewUntypedSchema(colStrs...)
return &CSVReader{r, br, info, sch, false}, nil
return &CSVReader{r, br, info, sch, false, format}, nil
}
func getColHeaders(br *bufio.Reader, info *CSVFileInfo) ([]string, error) {
@@ -146,6 +147,6 @@ func (csvr *CSVReader) parseRow(line string) (row.Row, error) {
}
}
r := row.New(types.Format_7_18, sch, taggedVals)
r := row.New(csvr.format, sch, taggedVals)
return r, nil
}
go/libraries/doltcore/table/untyped/csv/reader_test.go
+7 -6
View File
@@ -6,6 +6,7 @@ import (
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"io"
"strings"
"testing"
@@ -40,13 +41,13 @@ func TestReader(t *testing.T) {
colNames := []string{"name", "age", "title"}
_, sch := untyped.NewUntypedSchema(colNames...)
goodExpectedRows := []row.Row{
untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}),
untyped.NewRowFromStrings(sch, []string{"John Johnson", "21", "Intern Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"John Johnson", "21", "Intern Dufus"}),
}
badExpectedRows := []row.Row{
untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}),
}
tests := []struct {
@@ -106,7 +107,7 @@ func readTestRows(t *testing.T, inputStr string, info *CSVFileInfo) ([]row.Row,
const path = "/file.csv"
fs := filesys.NewInMemFS(nil, map[string][]byte{path: []byte(inputStr)}, root)
csvR, err := OpenCSVReader(path, fs, info)
csvR, err := OpenCSVReader(types.Format_7_18, path, fs, info)
defer csvR.Close(context.Background())
if err != nil {
go/libraries/doltcore/table/untyped/fwt/fwt_transform.go
+1 -1
View File
@@ -93,6 +93,6 @@ func (fwtTr *FWTTransformer) Transform(r row.Row, props pipeline.ReadableMap) ([
destFields[tag] = types.String(buf)
}
r = row.New(types.Format_7_18, sch, destFields)
r = row.New(r.Format(), sch, destFields)
return []*pipeline.TransformedRowResult{{RowData: r}}, ""
}
go/libraries/doltcore/table/untyped/fwt/reader.go
+10 -7
View File
@@ -5,14 +5,16 @@ import (
"context"
"errors"
"fmt"
"io"
"strings"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/row"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/schema"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/iohelp"
"io"
"strings"
"github.com/liquidata-inc/ld/dolt/go/store/types"
)
// ReadBufSize is the size of the buffer used when reading the fwt file. It is set at the package level and all
@@ -26,25 +28,26 @@ type FWTReader struct {
fwtSch *FWTSchema
isDone bool
colSep string
format *types.Format
}
// OpenFWTReader opens a reader at a given path within a given filesys. The FWTSchema should describe the fwt file
// being opened and have the correct column widths.
func OpenFWTReader(path string, fs filesys.ReadableFS, fwtSch *FWTSchema, colSep string) (*FWTReader, error) {
func OpenFWTReader(path string, fs filesys.ReadableFS, format *types.Format, fwtSch *FWTSchema, colSep string) (*FWTReader, error) {
r, err := fs.OpenForRead(path)
if err != nil {
return nil, err
}
return NewFWTReader(r, fwtSch, colSep)
return NewFWTReader(format, r, fwtSch, colSep)
}
//
func NewFWTReader(r io.ReadCloser, fwtSch *FWTSchema, colSep string) (*FWTReader, error) {
func NewFWTReader(format *types.Format, r io.ReadCloser, fwtSch *FWTSchema, colSep string) (*FWTReader, error) {
br := bufio.NewReaderSize(r, ReadBufSize)
return &FWTReader{r, br, fwtSch, false, colSep}, nil
return &FWTReader{r, br, fwtSch, false, colSep, format}, nil
}
// ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and callin IsBadRow(err)
@@ -118,5 +121,5 @@ func (fwtRd *FWTReader) parseRow(lineBytes []byte) (row.Row, error) {
return false
})
return untyped.NewRowFromStrings(fwtRd.GetSchema(), fields), nil
return untyped.NewRowFromStrings(fwtRd.format, fwtRd.GetSchema(), fields), nil
}
go/libraries/doltcore/table/untyped/fwt/reader_test.go
+7 -6
View File
@@ -6,6 +6,7 @@ import (
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"io"
"strings"
"testing"
@@ -29,13 +30,13 @@ func TestReader(t *testing.T) {
colNames := []string{"name", "age", "title"}
_, sch := untyped.NewUntypedSchema(colNames...)
goodExpectedRows := []row.Row{
untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}),
untyped.NewRowFromStrings(sch, []string{"John Johnson", "21", "Intern Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"John Johnson", "21", "Intern Dufus"}),
}
badExpectedRows := []row.Row{
untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}),
untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}),
}
widths := map[string]int{
@@ -93,7 +94,7 @@ func readTestRows(t *testing.T, inputStr string, fwtSch *FWTSchema, sep string)
const path = "/file.csv"
fs := filesys.NewInMemFS(nil, map[string][]byte{path: []byte(inputStr)}, root)
fwtRd, err := OpenFWTReader(path, fs, fwtSch, sep)
fwtRd, err := OpenFWTReader(path, fs, types.Format_7_18, fwtSch, sep)
defer fwtRd.Close(context.Background())
if err != nil {
go/libraries/doltcore/table/untyped/nullprinter/nullprinter.go
+1 -1
View File
@@ -36,5 +36,5 @@ func (np *NullPrinter) ProcessRow(inRow row.Row, props pipeline.ReadableMap) (ro
return false
})
return []*pipeline.TransformedRowResult{{RowData: row.New(types.Format_7_18, np.Sch, taggedVals)}}, ""
return []*pipeline.TransformedRowResult{{RowData: row.New(inRow.Format(), np.Sch, taggedVals)}}, ""
}
go/libraries/doltcore/table/untyped/resultset/result_set.go
+2 -2
View File
@@ -234,13 +234,13 @@ type CrossProductRowCallback func(r row.Row)
// CrossProduct computes the cross-product of the table results given, calling the given callback once for each row in
// the result set. The resultant rows will have the schema of this result set, and will have (N * M * ... X) rows, one
// for every possible combination of entries in the table results given.
func (rss *ResultSetSchema) CrossProduct(tables []*TableResult, cb CrossProductRowCallback) {
func (rss *ResultSetSchema) CrossProduct(format *types.Format, tables []*TableResult, cb CrossProductRowCallback) {
// special case: no tables means no rows
if len(tables) == 0 {
return
}
emptyRow := RowWithSchema{row.New(types.Format_7_18, rss.destSch, row.TaggedValues{}), rss.destSch}
emptyRow := RowWithSchema{row.New(format, rss.destSch, row.TaggedValues{}), rss.destSch}
rss.cph(emptyRow, tables, cb)
}
go/libraries/doltcore/table/untyped/resultset/result_set_test.go
+1 -1
View File
@@ -393,7 +393,7 @@ func getCrossProduct(rss *ResultSetSchema, tables []*TableResult) []row.Row {
cb := func(r row.Row) {
result = append(result, r)
}
rss.CrossProduct(tables, cb)
rss.CrossProduct(types.Format_7_18, tables, cb)
return result
}
go/libraries/doltcore/table/untyped/untyped_rows.go
+4 -4
View File
@@ -33,7 +33,7 @@ func NewUntypedSchemaWithFirstTag(firstTag uint64, colNames ...string) (map[stri
// NewRowFromStrings is a utility method that takes a schema for an untyped row, and a slice of strings and uses the strings
// as the field values for the row by converting them to noms type.String
func NewRowFromStrings(sch schema.Schema, valStrs []string) row.Row {
func NewRowFromStrings(format *types.Format, sch schema.Schema, valStrs []string) row.Row {
allCols := sch.GetAllCols()
taggedVals := make(row.TaggedValues)
@@ -48,17 +48,17 @@ func NewRowFromStrings(sch schema.Schema, valStrs []string) row.Row {
taggedVals[tag] = types.String(valStr)
}
return row.New(types.Format_7_18, sch, taggedVals)
return row.New(format, sch, taggedVals)
}
// NewRowFromTaggedStrings takes an untyped schema and a map of column tag to string value and returns a row
func NewRowFromTaggedStrings(sch schema.Schema, taggedStrs map[uint64]string) row.Row {
func NewRowFromTaggedStrings(format *types.Format, sch schema.Schema, taggedStrs map[uint64]string) row.Row {
taggedVals := make(row.TaggedValues)
for tag, valStr := range taggedStrs {
taggedVals[tag] = types.String(valStr)
}
return row.New(types.Format_7_18, sch, taggedVals)
return row.New(format, sch, taggedVals)
}
// UntypeSchema takes a schema and returns a schema with the same columns, but with the types of each of those columns
go/libraries/doltcore/table/untyped/untyped_rows_test.go
+1 -1
View File
@@ -59,7 +59,7 @@ func TestNewUntypedSchema(t *testing.T) {
name := "Billy Bob"
city := "Fargo"
blurb := "Billy Bob is a scholar."
r := NewRowFromStrings(sch, []string{name, city, blurb})
r := NewRowFromStrings(types.Format_7_18, sch, []string{name, city, blurb})
nameVal, _ := r.GetColVal(nameToTag["name"])
go/libraries/doltcore/table/untyped/xlsx/marshaling.go
+2 -2
View File
@@ -26,7 +26,7 @@ func UnmarshalFromXLSX(path string) ([][][]string, error) {
return dataSlice, nil
}
func decodeXLSXRows(xlData [][][]string, sch schema.Schema) ([]row.Row, error) {
func decodeXLSXRows(format *types.Format, xlData [][][]string, sch schema.Schema) ([]row.Row, error) {
var rows []row.Row
var err error
@@ -52,7 +52,7 @@ func decodeXLSXRows(xlData [][][]string, sch schema.Schema) ([]row.Row, error) {
return nil, err
}
}
rows = append(rows, row.New(types.Format_7_18, sch, taggedVals))
rows = append(rows, row.New(format, sch, taggedVals))
fmt.Println(rows)
}
go/libraries/doltcore/table/untyped/xlsx/marshaling_test.go
+1 -1
View File
@@ -21,7 +21,7 @@ func TestDecodeXLSXRows(t *testing.T) {
first := [][]string{{"id", "first", "last", "age"}, {"1", "osheiza", "otori", "24"}}
second = append(second, first)
decoded, err := decodeXLSXRows(second, sch)
decoded, err := decodeXLSXRows(types.Format_7_18, second, sch)
if err != nil {
fmt.Println(err)
go/libraries/doltcore/table/untyped/xlsx/reader.go
+5 -4
View File
@@ -10,6 +10,7 @@ import (
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/schema"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/ld/dolt/go/store/types"
)
var ReadBufSize = 256 * 1024
@@ -22,17 +23,17 @@ type XLSXReader struct {
ind int
}
func OpenXLSXReader(path string, fs filesys.ReadableFS, info *XLSXFileInfo, tblName string) (*XLSXReader, error) {
func OpenXLSXReader(path string, fs filesys.ReadableFS, info *XLSXFileInfo, format *types.Format, tblName string) (*XLSXReader, error) {
r, err := fs.OpenForRead(path)
if err != nil {
return nil, err
}
return NewXLSXReader(r, info, fs, path, tblName)
return NewXLSXReader(r, info, fs, format, path, tblName)
}
func NewXLSXReader(r io.ReadCloser, info *XLSXFileInfo, fs filesys.ReadableFS, path string, tblName string) (*XLSXReader, error) {
func NewXLSXReader(r io.ReadCloser, info *XLSXFileInfo, fs filesys.ReadableFS, format *types.Format, path string, tblName string) (*XLSXReader, error) {
br := bufio.NewReaderSize(r, ReadBufSize)
colStrs, err := getColHeaders(path, tblName)
@@ -47,7 +48,7 @@ func NewXLSXReader(r io.ReadCloser, info *XLSXFileInfo, fs filesys.ReadableFS, p
_, sch := untyped.NewUntypedSchema(colStrs...)
decodedRows, err := decodeXLSXRows(data, sch)
decodedRows, err := decodeXLSXRows(format, data, sch)
if err != nil {
return nil, err
}