diff --git a/go/cmd/dolt/commands/cnfcmds/cat.go b/go/cmd/dolt/commands/cnfcmds/cat.go index 0b113a011f..982d18470c 100644 --- a/go/cmd/dolt/commands/cnfcmds/cat.go +++ b/go/cmd/dolt/commands/cnfcmds/cat.go @@ -2,6 +2,7 @@ package cnfcmds import ( "context" + "github.com/liquidata-inc/ld/dolt/go/cmd/dolt/cli" "github.com/liquidata-inc/ld/dolt/go/cmd/dolt/commands" "github.com/liquidata-inc/ld/dolt/go/cmd/dolt/errhand" @@ -109,7 +110,7 @@ func printConflicts(root *doltdb.RootValue, tblNames []string) errhand.VerboseEr panic("") }) - p.InjectRow("fwt", untyped.NewRowFromTaggedStrings(cnfRd.GetSchema(), schema.ExtractAllColNames(cnfRd.GetSchema()))) + p.InjectRow("fwt", untyped.NewRowFromTaggedStrings(tbl.Format(), cnfRd.GetSchema(), schema.ExtractAllColNames(cnfRd.GetSchema()))) p.Start() p.Wait() diff --git a/go/cmd/dolt/commands/diff.go b/go/cmd/dolt/commands/diff.go index 9e2c902d10..2179b225eb 100644 --- a/go/cmd/dolt/commands/diff.go +++ b/go/cmd/dolt/commands/diff.go @@ -362,10 +362,10 @@ func diffRows(newRows, oldRows types.Map, newSch, oldSch schema.Schema) errhand. p := pipeline.NewAsyncPipeline(pipeline.ProcFuncForSourceFunc(src.NextDiff), sinkProcFunc, transforms, badRowCallback) if schemasEqual { - p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(untypedUnionSch, newColNames)) + p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(newRows.Format(), untypedUnionSch, newColNames)) } else { - p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(untypedUnionSch, oldColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedOld}) - p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(untypedUnionSch, newColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedNew}) + p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(newRows.Format(), untypedUnionSch, oldColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedOld}) + p.InjectRowWithProps(fwtStageName, untyped.NewRowFromTaggedStrings(newRows.Format(), untypedUnionSch, newColNames), map[string]interface{}{diff.DiffTypeProp: diff.DiffModifiedNew}) } p.Start() diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index 38db624ca4..5e7467f064 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -26,6 +26,7 @@ import ( "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped/tabular" "github.com/liquidata-inc/ld/dolt/go/libraries/utils/argparser" "github.com/liquidata-inc/ld/dolt/go/libraries/utils/iohelp" + "github.com/liquidata-inc/ld/dolt/go/store/types" "vitess.io/vitess/go/vt/sqlparser" ) @@ -298,7 +299,7 @@ func sqlShow(root *doltdb.RootValue, show *sqlparser.Show) error { return err } - return runPrintingPipeline(p, sch) + return runPrintingPipeline(root.VRW().Format(), p, sch) } // Executes a SQL select statement and prints the result to the CLI. @@ -317,12 +318,12 @@ func sqlSelect(root *doltdb.RootValue, s *sqlparser.Select) error { untypedSch, untypingTransform := newUntypingTransformer(resultSchema) p.AddStage(untypingTransform) - return runPrintingPipeline(p, untypedSch) + return runPrintingPipeline(root.VRW().Format(), p, untypedSch) } // Adds some print-handling stages to the pipeline given and runs it, returning any error. // Adds null-printing and fixed-width transformers. The schema given is assumed to be untyped (string-typed). -func runPrintingPipeline(p *pipeline.Pipeline, untypedSch schema.Schema) error { +func runPrintingPipeline(format *types.Format, p *pipeline.Pipeline, untypedSch schema.Schema) error { nullPrinter := nullprinter.NewNullPrinter(untypedSch) p.AddStage(pipeline.NewNamedTransform(nullprinter.NULL_PRINTING_STAGE, nullPrinter.ProcessRow)) @@ -344,7 +345,7 @@ func runPrintingPipeline(p *pipeline.Pipeline, untypedSch schema.Schema) error { }) // Insert the table header row at the appropriate stage - p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(untypedSch, schema.ExtractAllColNames(untypedSch))) + p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(format, untypedSch, schema.ExtractAllColNames(untypedSch))) p.Start() if err := p.Wait(); err != nil { diff --git a/go/cmd/dolt/commands/tblcmds/select.go b/go/cmd/dolt/commands/tblcmds/select.go index 484670e85a..accb935093 100644 --- a/go/cmd/dolt/commands/tblcmds/select.go +++ b/go/cmd/dolt/commands/tblcmds/select.go @@ -240,7 +240,7 @@ func createPipeline(tbl *doltdb.Table, tblSch schema.Schema, outSch schema.Schem p.RunAfter(func() { wr.Close(context.TODO()) }) // Insert the table header row at the appropriate stage - p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(outSch, colNames)) + p.InjectRow(fwtStageName, untyped.NewRowFromTaggedStrings(tbl.Format(), outSch, colNames)) return p } diff --git a/go/libraries/doltcore/doltdb/table.go b/go/libraries/doltcore/doltdb/table.go index 6b85817428..aea67dafdf 100644 --- a/go/libraries/doltcore/doltdb/table.go +++ b/go/libraries/doltcore/doltdb/table.go @@ -52,6 +52,10 @@ func NewTable(ctx context.Context, vrw types.ValueReadWriter, schema types.Value return &Table{vrw, tableStruct} } +func (t *Table) Format() *types.Format { + return t.vrw.Format() +} + func (t *Table) SetConflicts(ctx context.Context, schemas Conflict, conflictData types.Map) *Table { conflictsRef := writeValAndGetRef(ctx, t.vrw, conflictData) diff --git a/go/libraries/doltcore/mvdata/data_loc.go b/go/libraries/doltcore/mvdata/data_loc.go index a3012a1570..3111de1501 100644 --- a/go/libraries/doltcore/mvdata/data_loc.go +++ b/go/libraries/doltcore/mvdata/data_loc.go @@ -137,15 +137,15 @@ func (dl *DataLocation) CreateReader(ctx context.Context, root *doltdb.RootValue switch dl.Format { case CsvFile: - rd, err := csv.OpenCSVReader(dl.Path, fs, csv.NewCSVInfo()) + rd, err := csv.OpenCSVReader(root.VRW().Format(), dl.Path, fs, csv.NewCSVInfo()) return rd, false, err case PsvFile: - rd, err := csv.OpenCSVReader(dl.Path, fs, csv.NewCSVInfo().SetDelim('|')) + rd, err := csv.OpenCSVReader(root.VRW().Format(), dl.Path, fs, csv.NewCSVInfo().SetDelim('|')) return rd, false, err case XlsxFile: - rd, err := xlsx.OpenXLSXReader(dl.Path, fs, xlsx.NewXLSXInfo(), tblName) + rd, err := xlsx.OpenXLSXReader(dl.Path, fs, xlsx.NewXLSXInfo(), root.VRW().Format(), tblName) return rd, false, err case JsonFile: diff --git a/go/libraries/doltcore/sql/sqlselect.go b/go/libraries/doltcore/sql/sqlselect.go index 09faa4e52d..990d029c32 100644 --- a/go/libraries/doltcore/sql/sqlselect.go +++ b/go/libraries/doltcore/sql/sqlselect.go @@ -514,7 +514,7 @@ func createSelectPipeline(ctx context.Context, root *doltdb.RootValue, selectStm } go func() { defer close(cpChan) - selectStmt.intermediateRss.CrossProduct(results, cb) + selectStmt.intermediateRss.CrossProduct(root.VRW().Format(), results, cb) }() // TODO: we need to check errors in pipeline execution without blocking diff --git a/go/libraries/doltcore/table/pipeline/errors_test.go b/go/libraries/doltcore/table/pipeline/errors_test.go index 6d97ec87b6..372555b222 100644 --- a/go/libraries/doltcore/table/pipeline/errors_test.go +++ b/go/libraries/doltcore/table/pipeline/errors_test.go @@ -3,12 +3,13 @@ package pipeline import ( "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/row" "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped" + "github.com/liquidata-inc/ld/dolt/go/store/types" "testing" ) func TestTransformRowFailure(t *testing.T) { _, sch := untyped.NewUntypedSchema("a", "b", "c") - r := untyped.NewRowFromStrings(sch, []string{"1", "2", "3"}) + r := untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"1", "2", "3"}) var err error err = &TransformRowFailure{r, "transform_name", "details"} diff --git a/go/libraries/doltcore/table/pipeline/transform_test.go b/go/libraries/doltcore/table/pipeline/transform_test.go index cc1dd76b8e..32eae9f089 100644 --- a/go/libraries/doltcore/table/pipeline/transform_test.go +++ b/go/libraries/doltcore/table/pipeline/transform_test.go @@ -69,7 +69,7 @@ func TestPipeline(t *testing.T) { func() { csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true} - rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo) + rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo) wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo) tc := NewTransformCollection( @@ -107,7 +107,7 @@ func TestAddingStages(t *testing.T) { func() { csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true} - rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo) + rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo) wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo) tc := NewTransformCollection( @@ -177,7 +177,7 @@ Don,Beddoe,Bewitched (episode Humbug Not to Be Spoken Here - Season 4),1967,true func() { csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true} - rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo) + rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo) wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo) addedStages := []NamedTransform { @@ -211,7 +211,7 @@ Don,Beddoe,Bewitched (episode Humbug Not to Be Spoken Here - Season 4),1967,true 4: "true", 5: "0", } - injectedRow := untyped.NewRowFromTaggedStrings(schOut, injectedColumns) + injectedRow := untyped.NewRowFromTaggedStrings(types.Format_7_18, schOut, injectedColumns) p.InjectRow("append", injectedRow) //AnotherNew,Row,InAppendStage,3000,true,1 @@ -223,7 +223,7 @@ Don,Beddoe,Bewitched (episode Humbug Not to Be Spoken Here - Season 4),1967,true 4: "true", 5: "1", } - injectedRow = untyped.NewRowFromTaggedStrings(schOut, injectedColumns) + injectedRow = untyped.NewRowFromTaggedStrings(types.Format_7_18, schOut, injectedColumns) p.InjectRow("append", injectedRow) p.RunAfter(func() { rd.Close(context.Background()) }) @@ -263,7 +263,7 @@ func TestAbort(t *testing.T) { func() { csvInfo := &csv.CSVFileInfo{Delim: ',', HasHeaderLine: true, Columns: nil, EscapeQuotes: true} - rd, _ := csv.NewCSVReader(ioutil.NopCloser(buf), csvInfo) + rd, _ := csv.NewCSVReader(types.Format_7_18, ioutil.NopCloser(buf), csvInfo) wr, _ := csv.NewCSVWriter(iohelp.NopWrCloser(outBuf), schOut, csvInfo) var wg = sync.WaitGroup{} diff --git a/go/libraries/doltcore/table/typed/noms/noms_map_updater.go b/go/libraries/doltcore/table/typed/noms/noms_map_updater.go index 63e26ff457..d5497fa0e7 100644 --- a/go/libraries/doltcore/table/typed/noms/noms_map_updater.go +++ b/go/libraries/doltcore/table/typed/noms/noms_map_updater.go @@ -47,8 +47,7 @@ func NewNomsMapUpdater(ctx context.Context, vrw types.ValueReadWriter, m types.M var totalStats types.AppliedEditStats for edits := range mapChan { var stats types.AppliedEditStats - // TODO(binformat) - m, stats = types.ApplyEdits(ctx, types.Format_7_18, edits, m) + m, stats = types.ApplyEdits(ctx, vrw.Format(), edits, m) totalStats = totalStats.Add(stats) if statsCB != nil { @@ -59,7 +58,7 @@ func NewNomsMapUpdater(ctx context.Context, vrw types.ValueReadWriter, m types.M resChan <- updateMapRes{m, nil} }() - return &NomsMapUpdater{sch, vrw, 0, types.CreateEditAccForMapEdits(types.Format_7_18), mapChan, resChan, nil} + return &NomsMapUpdater{sch, vrw, 0, types.CreateEditAccForMapEdits(vrw.Format()), mapChan, resChan, nil} } // GetSchema gets the schema of the rows that this writer writes diff --git a/go/libraries/doltcore/table/untyped/csv/reader.go b/go/libraries/doltcore/table/untyped/csv/reader.go index 7fe6ae4d25..574395e02e 100644 --- a/go/libraries/doltcore/table/untyped/csv/reader.go +++ b/go/libraries/doltcore/table/untyped/csv/reader.go @@ -28,22 +28,23 @@ type CSVReader struct { info *CSVFileInfo sch schema.Schema isDone bool + format *types.Format } // OpenCSVReader opens a reader at a given path within a given filesys. The CSVFileInfo should describe the csv file // being opened. -func OpenCSVReader(path string, fs filesys.ReadableFS, info *CSVFileInfo) (*CSVReader, error) { +func OpenCSVReader(format *types.Format, path string, fs filesys.ReadableFS, info *CSVFileInfo) (*CSVReader, error) { r, err := fs.OpenForRead(path) if err != nil { return nil, err } - return NewCSVReader(r, info) + return NewCSVReader(format, r, info) } // NewCSVReader creates a CSVReader from a given ReadCloser. The CSVFileInfo should describe the csv file being read. -func NewCSVReader(r io.ReadCloser, info *CSVFileInfo) (*CSVReader, error) { +func NewCSVReader(format *types.Format, r io.ReadCloser, info *CSVFileInfo) (*CSVReader, error) { br := bufio.NewReaderSize(r, ReadBufSize) colStrs, err := getColHeaders(br, info) @@ -54,7 +55,7 @@ func NewCSVReader(r io.ReadCloser, info *CSVFileInfo) (*CSVReader, error) { _, sch := untyped.NewUntypedSchema(colStrs...) - return &CSVReader{r, br, info, sch, false}, nil + return &CSVReader{r, br, info, sch, false, format}, nil } func getColHeaders(br *bufio.Reader, info *CSVFileInfo) ([]string, error) { @@ -146,6 +147,6 @@ func (csvr *CSVReader) parseRow(line string) (row.Row, error) { } } - r := row.New(types.Format_7_18, sch, taggedVals) + r := row.New(csvr.format, sch, taggedVals) return r, nil } diff --git a/go/libraries/doltcore/table/untyped/csv/reader_test.go b/go/libraries/doltcore/table/untyped/csv/reader_test.go index ff2314e298..d75815b7a1 100644 --- a/go/libraries/doltcore/table/untyped/csv/reader_test.go +++ b/go/libraries/doltcore/table/untyped/csv/reader_test.go @@ -6,6 +6,7 @@ import ( "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table" "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped" "github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys" + "github.com/liquidata-inc/ld/dolt/go/store/types" "io" "strings" "testing" @@ -40,13 +41,13 @@ func TestReader(t *testing.T) { colNames := []string{"name", "age", "title"} _, sch := untyped.NewUntypedSchema(colNames...) goodExpectedRows := []row.Row{ - untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}), - untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}), - untyped.NewRowFromStrings(sch, []string{"John Johnson", "21", "Intern Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"John Johnson", "21", "Intern Dufus"}), } badExpectedRows := []row.Row{ - untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}), - untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}), } tests := []struct { @@ -106,7 +107,7 @@ func readTestRows(t *testing.T, inputStr string, info *CSVFileInfo) ([]row.Row, const path = "/file.csv" fs := filesys.NewInMemFS(nil, map[string][]byte{path: []byte(inputStr)}, root) - csvR, err := OpenCSVReader(path, fs, info) + csvR, err := OpenCSVReader(types.Format_7_18, path, fs, info) defer csvR.Close(context.Background()) if err != nil { diff --git a/go/libraries/doltcore/table/untyped/fwt/fwt_transform.go b/go/libraries/doltcore/table/untyped/fwt/fwt_transform.go index 744bfe59a5..2b32959e3f 100644 --- a/go/libraries/doltcore/table/untyped/fwt/fwt_transform.go +++ b/go/libraries/doltcore/table/untyped/fwt/fwt_transform.go @@ -93,6 +93,6 @@ func (fwtTr *FWTTransformer) Transform(r row.Row, props pipeline.ReadableMap) ([ destFields[tag] = types.String(buf) } - r = row.New(types.Format_7_18, sch, destFields) + r = row.New(r.Format(), sch, destFields) return []*pipeline.TransformedRowResult{{RowData: r}}, "" } diff --git a/go/libraries/doltcore/table/untyped/fwt/reader.go b/go/libraries/doltcore/table/untyped/fwt/reader.go index 04e3474833..63bf745c0d 100644 --- a/go/libraries/doltcore/table/untyped/fwt/reader.go +++ b/go/libraries/doltcore/table/untyped/fwt/reader.go @@ -5,14 +5,16 @@ import ( "context" "errors" "fmt" + "io" + "strings" + "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/row" "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/schema" "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table" "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped" "github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys" "github.com/liquidata-inc/ld/dolt/go/libraries/utils/iohelp" - "io" - "strings" + "github.com/liquidata-inc/ld/dolt/go/store/types" ) // ReadBufSize is the size of the buffer used when reading the fwt file. It is set at the package level and all @@ -26,25 +28,26 @@ type FWTReader struct { fwtSch *FWTSchema isDone bool colSep string + format *types.Format } // OpenFWTReader opens a reader at a given path within a given filesys. The FWTSchema should describe the fwt file // being opened and have the correct column widths. -func OpenFWTReader(path string, fs filesys.ReadableFS, fwtSch *FWTSchema, colSep string) (*FWTReader, error) { +func OpenFWTReader(path string, fs filesys.ReadableFS, format *types.Format, fwtSch *FWTSchema, colSep string) (*FWTReader, error) { r, err := fs.OpenForRead(path) if err != nil { return nil, err } - return NewFWTReader(r, fwtSch, colSep) + return NewFWTReader(format, r, fwtSch, colSep) } // -func NewFWTReader(r io.ReadCloser, fwtSch *FWTSchema, colSep string) (*FWTReader, error) { +func NewFWTReader(format *types.Format, r io.ReadCloser, fwtSch *FWTSchema, colSep string) (*FWTReader, error) { br := bufio.NewReaderSize(r, ReadBufSize) - return &FWTReader{r, br, fwtSch, false, colSep}, nil + return &FWTReader{r, br, fwtSch, false, colSep, format}, nil } // ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and callin IsBadRow(err) @@ -118,5 +121,5 @@ func (fwtRd *FWTReader) parseRow(lineBytes []byte) (row.Row, error) { return false }) - return untyped.NewRowFromStrings(fwtRd.GetSchema(), fields), nil + return untyped.NewRowFromStrings(fwtRd.format, fwtRd.GetSchema(), fields), nil } diff --git a/go/libraries/doltcore/table/untyped/fwt/reader_test.go b/go/libraries/doltcore/table/untyped/fwt/reader_test.go index 9d492a5249..029f8b74ba 100644 --- a/go/libraries/doltcore/table/untyped/fwt/reader_test.go +++ b/go/libraries/doltcore/table/untyped/fwt/reader_test.go @@ -6,6 +6,7 @@ import ( "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table" "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped" "github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys" + "github.com/liquidata-inc/ld/dolt/go/store/types" "io" "strings" "testing" @@ -29,13 +30,13 @@ func TestReader(t *testing.T) { colNames := []string{"name", "age", "title"} _, sch := untyped.NewUntypedSchema(colNames...) goodExpectedRows := []row.Row{ - untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}), - untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}), - untyped.NewRowFromStrings(sch, []string{"John Johnson", "21", "Intern Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"John Johnson", "21", "Intern Dufus"}), } badExpectedRows := []row.Row{ - untyped.NewRowFromStrings(sch, []string{"Bill Billerson", "32", "Senior Dufus"}), - untyped.NewRowFromStrings(sch, []string{"Rob Robertson", "25", "Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Bill Billerson", "32", "Senior Dufus"}), + untyped.NewRowFromStrings(types.Format_7_18, sch, []string{"Rob Robertson", "25", "Dufus"}), } widths := map[string]int{ @@ -93,7 +94,7 @@ func readTestRows(t *testing.T, inputStr string, fwtSch *FWTSchema, sep string) const path = "/file.csv" fs := filesys.NewInMemFS(nil, map[string][]byte{path: []byte(inputStr)}, root) - fwtRd, err := OpenFWTReader(path, fs, fwtSch, sep) + fwtRd, err := OpenFWTReader(path, fs, types.Format_7_18, fwtSch, sep) defer fwtRd.Close(context.Background()) if err != nil { diff --git a/go/libraries/doltcore/table/untyped/nullprinter/nullprinter.go b/go/libraries/doltcore/table/untyped/nullprinter/nullprinter.go index f715209568..59ec1f0ca7 100644 --- a/go/libraries/doltcore/table/untyped/nullprinter/nullprinter.go +++ b/go/libraries/doltcore/table/untyped/nullprinter/nullprinter.go @@ -36,5 +36,5 @@ func (np *NullPrinter) ProcessRow(inRow row.Row, props pipeline.ReadableMap) (ro return false }) - return []*pipeline.TransformedRowResult{{RowData: row.New(types.Format_7_18, np.Sch, taggedVals)}}, "" + return []*pipeline.TransformedRowResult{{RowData: row.New(inRow.Format(), np.Sch, taggedVals)}}, "" } diff --git a/go/libraries/doltcore/table/untyped/resultset/result_set.go b/go/libraries/doltcore/table/untyped/resultset/result_set.go index b7d0184651..9ba8d144f2 100644 --- a/go/libraries/doltcore/table/untyped/resultset/result_set.go +++ b/go/libraries/doltcore/table/untyped/resultset/result_set.go @@ -234,13 +234,13 @@ type CrossProductRowCallback func(r row.Row) // CrossProduct computes the cross-product of the table results given, calling the given callback once for each row in // the result set. The resultant rows will have the schema of this result set, and will have (N * M * ... X) rows, one // for every possible combination of entries in the table results given. -func (rss *ResultSetSchema) CrossProduct(tables []*TableResult, cb CrossProductRowCallback) { +func (rss *ResultSetSchema) CrossProduct(format *types.Format, tables []*TableResult, cb CrossProductRowCallback) { // special case: no tables means no rows if len(tables) == 0 { return } - emptyRow := RowWithSchema{row.New(types.Format_7_18, rss.destSch, row.TaggedValues{}), rss.destSch} + emptyRow := RowWithSchema{row.New(format, rss.destSch, row.TaggedValues{}), rss.destSch} rss.cph(emptyRow, tables, cb) } diff --git a/go/libraries/doltcore/table/untyped/resultset/result_set_test.go b/go/libraries/doltcore/table/untyped/resultset/result_set_test.go index 1c4ca4e736..fceb69ff2d 100644 --- a/go/libraries/doltcore/table/untyped/resultset/result_set_test.go +++ b/go/libraries/doltcore/table/untyped/resultset/result_set_test.go @@ -393,7 +393,7 @@ func getCrossProduct(rss *ResultSetSchema, tables []*TableResult) []row.Row { cb := func(r row.Row) { result = append(result, r) } - rss.CrossProduct(tables, cb) + rss.CrossProduct(types.Format_7_18, tables, cb) return result } diff --git a/go/libraries/doltcore/table/untyped/untyped_rows.go b/go/libraries/doltcore/table/untyped/untyped_rows.go index 3538ac9bd9..51dc139d37 100644 --- a/go/libraries/doltcore/table/untyped/untyped_rows.go +++ b/go/libraries/doltcore/table/untyped/untyped_rows.go @@ -33,7 +33,7 @@ func NewUntypedSchemaWithFirstTag(firstTag uint64, colNames ...string) (map[stri // NewRowFromStrings is a utility method that takes a schema for an untyped row, and a slice of strings and uses the strings // as the field values for the row by converting them to noms type.String -func NewRowFromStrings(sch schema.Schema, valStrs []string) row.Row { +func NewRowFromStrings(format *types.Format, sch schema.Schema, valStrs []string) row.Row { allCols := sch.GetAllCols() taggedVals := make(row.TaggedValues) @@ -48,17 +48,17 @@ func NewRowFromStrings(sch schema.Schema, valStrs []string) row.Row { taggedVals[tag] = types.String(valStr) } - return row.New(types.Format_7_18, sch, taggedVals) + return row.New(format, sch, taggedVals) } // NewRowFromTaggedStrings takes an untyped schema and a map of column tag to string value and returns a row -func NewRowFromTaggedStrings(sch schema.Schema, taggedStrs map[uint64]string) row.Row { +func NewRowFromTaggedStrings(format *types.Format, sch schema.Schema, taggedStrs map[uint64]string) row.Row { taggedVals := make(row.TaggedValues) for tag, valStr := range taggedStrs { taggedVals[tag] = types.String(valStr) } - return row.New(types.Format_7_18, sch, taggedVals) + return row.New(format, sch, taggedVals) } // UntypeSchema takes a schema and returns a schema with the same columns, but with the types of each of those columns diff --git a/go/libraries/doltcore/table/untyped/untyped_rows_test.go b/go/libraries/doltcore/table/untyped/untyped_rows_test.go index 4e39cb9580..48458157e3 100644 --- a/go/libraries/doltcore/table/untyped/untyped_rows_test.go +++ b/go/libraries/doltcore/table/untyped/untyped_rows_test.go @@ -59,7 +59,7 @@ func TestNewUntypedSchema(t *testing.T) { name := "Billy Bob" city := "Fargo" blurb := "Billy Bob is a scholar." - r := NewRowFromStrings(sch, []string{name, city, blurb}) + r := NewRowFromStrings(types.Format_7_18, sch, []string{name, city, blurb}) nameVal, _ := r.GetColVal(nameToTag["name"]) diff --git a/go/libraries/doltcore/table/untyped/xlsx/marshaling.go b/go/libraries/doltcore/table/untyped/xlsx/marshaling.go index f0dbebd2cc..093c36eb52 100644 --- a/go/libraries/doltcore/table/untyped/xlsx/marshaling.go +++ b/go/libraries/doltcore/table/untyped/xlsx/marshaling.go @@ -26,7 +26,7 @@ func UnmarshalFromXLSX(path string) ([][][]string, error) { return dataSlice, nil } -func decodeXLSXRows(xlData [][][]string, sch schema.Schema) ([]row.Row, error) { +func decodeXLSXRows(format *types.Format, xlData [][][]string, sch schema.Schema) ([]row.Row, error) { var rows []row.Row var err error @@ -52,7 +52,7 @@ func decodeXLSXRows(xlData [][][]string, sch schema.Schema) ([]row.Row, error) { return nil, err } } - rows = append(rows, row.New(types.Format_7_18, sch, taggedVals)) + rows = append(rows, row.New(format, sch, taggedVals)) fmt.Println(rows) } diff --git a/go/libraries/doltcore/table/untyped/xlsx/marshaling_test.go b/go/libraries/doltcore/table/untyped/xlsx/marshaling_test.go index 59de7a3b59..6b8f771c25 100644 --- a/go/libraries/doltcore/table/untyped/xlsx/marshaling_test.go +++ b/go/libraries/doltcore/table/untyped/xlsx/marshaling_test.go @@ -21,7 +21,7 @@ func TestDecodeXLSXRows(t *testing.T) { first := [][]string{{"id", "first", "last", "age"}, {"1", "osheiza", "otori", "24"}} second = append(second, first) - decoded, err := decodeXLSXRows(second, sch) + decoded, err := decodeXLSXRows(types.Format_7_18, second, sch) if err != nil { fmt.Println(err) diff --git a/go/libraries/doltcore/table/untyped/xlsx/reader.go b/go/libraries/doltcore/table/untyped/xlsx/reader.go index 6096cafe02..b3d5dcb25f 100644 --- a/go/libraries/doltcore/table/untyped/xlsx/reader.go +++ b/go/libraries/doltcore/table/untyped/xlsx/reader.go @@ -10,6 +10,7 @@ import ( "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/schema" "github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/table/untyped" "github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys" + "github.com/liquidata-inc/ld/dolt/go/store/types" ) var ReadBufSize = 256 * 1024 @@ -22,17 +23,17 @@ type XLSXReader struct { ind int } -func OpenXLSXReader(path string, fs filesys.ReadableFS, info *XLSXFileInfo, tblName string) (*XLSXReader, error) { +func OpenXLSXReader(path string, fs filesys.ReadableFS, info *XLSXFileInfo, format *types.Format, tblName string) (*XLSXReader, error) { r, err := fs.OpenForRead(path) if err != nil { return nil, err } - return NewXLSXReader(r, info, fs, path, tblName) + return NewXLSXReader(r, info, fs, format, path, tblName) } -func NewXLSXReader(r io.ReadCloser, info *XLSXFileInfo, fs filesys.ReadableFS, path string, tblName string) (*XLSXReader, error) { +func NewXLSXReader(r io.ReadCloser, info *XLSXFileInfo, fs filesys.ReadableFS, format *types.Format, path string, tblName string) (*XLSXReader, error) { br := bufio.NewReaderSize(r, ReadBufSize) colStrs, err := getColHeaders(path, tblName) @@ -47,7 +48,7 @@ func NewXLSXReader(r io.ReadCloser, info *XLSXFileInfo, fs filesys.ReadableFS, p _, sch := untyped.NewUntypedSchema(colStrs...) - decodedRows, err := decodeXLSXRows(data, sch) + decodedRows, err := decodeXLSXRows(format, data, sch) if err != nil { return nil, err }