From 3cb68a213c90f7b49ff7d92378c7a9391e2752c0 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Fri, 2 Jul 2021 14:32:50 -0700 Subject: [PATCH 1/4] /integration-tests/bats/import-create-tables.bats: add skipped bats test reproducing error --- .../bats/import-create-tables.bats | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/integration-tests/bats/import-create-tables.bats b/integration-tests/bats/import-create-tables.bats index cfe41cd8ce..f40abe2880 100755 --- a/integration-tests/bats/import-create-tables.bats +++ b/integration-tests/bats/import-create-tables.bats @@ -45,6 +45,14 @@ CREATE TABLE test ( ); SQL + cat < people.csv +pk,first,last,age,street,city,state,zip,dollar,color,date +1,Oscar,Rodgers,38,Zapib View,Vervutce,OH,03020,$1200.09,RED,11/12/1928 +2,Estella,Cannon,33,Kubta Manor,Tocunuz,OH,04943,$1296.25,YELLOW,03/05/2016 +3,Dora,Stanley,27,Bidohe Boulevard,Siguhazep,CA,53768,$9744.06,WHITE,07/31/1993 +4,Brian,Newman,41,Koef Court,Abemivu,OH,44534,$3808.15,YELLOW,03/29/2064 +DELIM + } teardown() { @@ -99,6 +107,16 @@ teardown() { [ "${#lines[@]}" -eq 6 ] } +@test "import-create-tables: import data from csv and create the table different types" { + skip "unable to cast to uint64, parse error" + run dolt table import -c --pk=pk test people.csv + [ "$status" -eq 0 ] + [[ "$output" =~ "Import completed successfully." ]] || false + run dolt sql -q "select * from test" + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 8 ] +} + @test "import-create-tables: use -f to overwrite data in existing table" { cat < other.csv pk,c1,c2,c3,c4,c5 From fdd6d33293807ecbf60967ba5631fe6674ee63f8 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Wed, 7 Jul 2021 11:21:54 -0700 Subject: [PATCH 2/4] /{go,integration-tests}: fix parse int with leading zero issue --- go/libraries/doltcore/env/actions/infer_schema.go | 6 ++++++ integration-tests/bats/import-create-tables.bats | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/env/actions/infer_schema.go b/go/libraries/doltcore/env/actions/infer_schema.go index 4ba6527d24..accbde271e 100644 --- a/go/libraries/doltcore/env/actions/infer_schema.go +++ b/go/libraries/doltcore/env/actions/infer_schema.go @@ -224,6 +224,12 @@ func leastPermissiveNumericType(strVal string, floatThreshold float64) (ti typei if err != nil { return typeinfo.UnknownType } + + // handle leading zero case + if strVal[0] == '0' { + return typeinfo.StringDefaultType + } + if ui <= math.MaxUint32 { return typeinfo.Uint32Type } else { diff --git a/integration-tests/bats/import-create-tables.bats b/integration-tests/bats/import-create-tables.bats index f40abe2880..1f69342339 100755 --- a/integration-tests/bats/import-create-tables.bats +++ b/integration-tests/bats/import-create-tables.bats @@ -108,7 +108,6 @@ teardown() { } @test "import-create-tables: import data from csv and create the table different types" { - skip "unable to cast to uint64, parse error" run dolt table import -c --pk=pk test people.csv [ "$status" -eq 0 ] [[ "$output" =~ "Import completed successfully." ]] || false From 6d64c37891e7c4559392f2ac2ffc4e9ab1171790 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Wed, 7 Jul 2021 11:40:24 -0700 Subject: [PATCH 3/4] /go/libraries/doltcore/env/actions/{infer_schema}_test.go: fix tests --- go/libraries/doltcore/env/actions/infer_schema.go | 2 +- go/libraries/doltcore/env/actions/infer_schema_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/env/actions/infer_schema.go b/go/libraries/doltcore/env/actions/infer_schema.go index accbde271e..42f7739204 100644 --- a/go/libraries/doltcore/env/actions/infer_schema.go +++ b/go/libraries/doltcore/env/actions/infer_schema.go @@ -226,7 +226,7 @@ func leastPermissiveNumericType(strVal string, floatThreshold float64) (ti typei } // handle leading zero case - if strVal[0] == '0' { + if len(strVal) > 1 && strVal[0] == '0' { return typeinfo.StringDefaultType } diff --git a/go/libraries/doltcore/env/actions/infer_schema_test.go b/go/libraries/doltcore/env/actions/infer_schema_test.go index b38d4b52d0..4a7196a817 100644 --- a/go/libraries/doltcore/env/actions/infer_schema_test.go +++ b/go/libraries/doltcore/env/actions/infer_schema_test.go @@ -81,8 +81,8 @@ func TestLeastPermissiveNumericType(t *testing.T) { {"negative float", "-1.3451234", 0.0, typeinfo.Float32Type}, {"double decimal point", "0.00.0", 0.0, typeinfo.UnknownType}, {"zero float with high precision", "0.0000", 0.0, typeinfo.Float32Type}, - {"all zeroes", "0000", 0.0, typeinfo.Uint32Type}, - {"leading zeroes", "01", 0.0, typeinfo.Uint32Type}, + {"all zeroes", "0000", 0.0, typeinfo.StringDefaultType}, + {"leading zeroes", "01", 0.0, typeinfo.StringDefaultType}, {"negative int", "-1234", 0.0, typeinfo.Int32Type}, {"fits in uint64 but not int64", strconv.FormatUint(math.MaxUint64, 10), 0.0, typeinfo.Uint64Type}, {"negative less than math.MinInt64", "-" + strconv.FormatUint(math.MaxUint64, 10), 0.0, typeinfo.UnknownType}, From 90af72dee14dee55fe2256b94a720043dddad687 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Wed, 7 Jul 2021 12:09:53 -0700 Subject: [PATCH 4/4] /go/libraries/doltcore/env/actions/infer_schema_test.go: add test showing how we handle leading zero floats currently --- go/libraries/doltcore/env/actions/infer_schema_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/libraries/doltcore/env/actions/infer_schema_test.go b/go/libraries/doltcore/env/actions/infer_schema_test.go index 4a7196a817..26031dfd56 100644 --- a/go/libraries/doltcore/env/actions/infer_schema_test.go +++ b/go/libraries/doltcore/env/actions/infer_schema_test.go @@ -80,6 +80,7 @@ func TestLeastPermissiveNumericType(t *testing.T) { {"zero float with floatThreshold of 0.1", "0.0", 0.1, typeinfo.Int32Type}, {"negative float", "-1.3451234", 0.0, typeinfo.Float32Type}, {"double decimal point", "0.00.0", 0.0, typeinfo.UnknownType}, + {"leading zero floats", "05.78", 0.0, typeinfo.Float32Type}, {"zero float with high precision", "0.0000", 0.0, typeinfo.Float32Type}, {"all zeroes", "0000", 0.0, typeinfo.StringDefaultType}, {"leading zeroes", "01", 0.0, typeinfo.StringDefaultType},