From fcb5455fd0cd2aa0b5802d5fa2363bcdae40ace7 Mon Sep 17 00:00:00 2001 From: Tim Sehn Date: Thu, 18 May 2023 11:11:31 -0700 Subject: [PATCH 1/5] Stubs and a couple bats tests for load data bats tests improvements --- integration-tests/bats/sql-load-data.bats | 196 ++++++++++++++++++---- 1 file changed, 165 insertions(+), 31 deletions(-) diff --git a/integration-tests/bats/sql-load-data.bats b/integration-tests/bats/sql-load-data.bats index b513fa01f2..e089666bf5 100644 --- a/integration-tests/bats/sql-load-data.bats +++ b/integration-tests/bats/sql-load-data.bats @@ -1,11 +1,13 @@ #!/usr/bin/env bats load $BATS_TEST_DIRNAME/helper/common.bash +load $BATS_TEST_DIRNAME/helper/query-server-common.bash setup() { setup_common } teardown() { + stop_sql_server assert_feature_version teardown_common } @@ -17,13 +19,11 @@ pk||c1||c2||c3||c4||c5 1||1||2||3||4||5 DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk int primary key, c1 int, c2 int, c3 int, c4 int, c5 int); LOAD DATA INFILE '1pk5col-ints.csv' INTO TABLE test CHARACTER SET UTF8MB4 FIELDS TERMINATED BY '||' ESCAPED BY '' LINES TERMINATED BY '\n' IGNORE 1 LINES; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -56,13 +56,11 @@ pk||c1||c2||c3||c4||c5 "1"||"1"||"2"||"3"||"4"||"5" DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk int primary key, c1 int, c2 int, c3 int, c4 int, c5 int); LOAD DATA INFILE '1pk5col-ints.csv' INTO TABLE test CHARACTER SET UTF8MB4 FIELDS TERMINATED BY '||' ENCLOSED BY '"' ESCAPED BY '' LINES TERMINATED BY '\n' IGNORE 1 LINES; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -80,13 +78,11 @@ ignore me sssYo DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk longtext); LOAD DATA INFILE 'prefixed.txt' INTO TABLE test CHARACTER SET UTF8MB4 LINES STARTING BY 'sss' IGNORE 1 LINES; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test ORDER BY pk" [ "$status" -eq 0 ] @@ -103,13 +99,11 @@ pk,c1 1,1 DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk int primary key, c1 int, c2 int); LOAD DATA INFILE '1pk2col-ints.csv' INTO TABLE test FIELDS TERMINATED BY ',' IGNORE 1 LINES; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -126,13 +120,11 @@ pk c1 1 1 DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk int primary key, c1 int); LOAD DATA INFILE '1pk2col-ints.csv' INTO TABLE test FIELDS TERMINATED BY '\t' IGNORE 1 LINES; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -148,13 +140,11 @@ pk NULL DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk longtext); LOAD DATA INFILE '1pk2col-ints.csv' INTO TABLE test FIELDS IGNORE 1 LINES; SQL - [ "$status" -eq 0 ] - run dolt sql -q "select COUNT(*) from test WHERE pk IS NULL" [ "$status" -eq 0 ] [[ "$output" =~ "2" ]] || false @@ -169,13 +159,11 @@ pk,c1 "hello","2" DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk int, c1 longtext); LOAD DATA INFILE '1pk2col-ints.csv' INTO TABLE test FIELDS ENCLOSED BY '"' TERMINATED BY ',' IGNORE 1 LINES (c1,pk); SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -193,13 +181,11 @@ SQL 4,"a string containing a \", quote and comma",102.20 DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk int, c1 longtext, c2 float); LOAD DATA INFILE 'complex.csv' INTO TABLE test FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"'; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -219,13 +205,11 @@ SQL "new\ns" DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE loadtable(pk longtext); LOAD DATA INFILE './testdata/test5.txt' INTO TABLE loadtable FIELDS ENCLOSED BY '\"'; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -244,13 +228,16 @@ pk||c1||c2||c3||c4||c5 1||1||2||3||4||5||6 DELIM - run dolt sql << SQL + dolt sql << SQL CREATE TABLE test(pk int primary key, c1 int, c2 int, c3 int, c4 int, c5 int); -LOAD DATA INFILE '1pk5col-ints.csv' INTO TABLE test CHARACTER SET UTF8MB4 FIELDS TERMINATED BY '||' ESCAPED BY '' LINES TERMINATED BY '\n' IGNORE 1 LINES; +LOAD DATA INFILE '1pk5col-ints.csv' INTO TABLE test +CHARACTER SET UTF8MB4 +FIELDS TERMINATED BY '||' +ESCAPED BY '' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; SQL - [ "$status" -eq 0 ] - run dolt sql -r csv -q "select * from test" [ "$status" -eq 0 ] @@ -258,3 +245,150 @@ SQL [ "${lines[1]}" = "0,1,2,3,4,5" ] [ "${lines[2]}" = "1,1,2,3,4,5" ] } + +@test "sql-load-data: load data ignore" { + cat < in.csv +0,0,0 +1,1,1 +CSV + + dolt sql -q "create table t (pk int primary key, c1 int, c2 int)" + dolt sql -q "insert into t values (0,0,0)" + skip "load data ignore not supported" + run dolt sql < in.csv +0,0,1 +1,1,1 +CSV + + dolt sql -q "create table t (pk int primary key, c1 int, c2 int)" + dolt sql -q "insert into t values (0,0,0)" + skip "load data replace not supported" + run dolt sql < in.csv +0,0,0 +1,1,1 +CSV + + dolt sql -q "create table t (pk int primary key, c1 int, c2 int)" + + run dolt sql < in.csv +0,0,0 +1,1,1 +CSV + dolt sql -q "create table t (pk int primary key, c1 int, c2 int)" + + start_sql_server + + # File not found errors + run dolt sql-client -P $PORT -u dolt -q "load data infile 'foo.csv' into table t" + [ $status -ne 0 ] + [[ $output =~ "no such file or directory" ]] || false + + dolt sql-client -P $PORT -u dolt -q " +load data infile 'in.csv' into table t +fields terminated by ',' +lines terminated by '\n' +" + + stop_sql_server + + run dolt sql -r csv -q "select * from t" + [ $status -eq 0 ] + [[ $output =~ "0,0,0" ]] || false + [[ $output =~ "1,1,1" ]] || false +} From ce0d917c642517bf95ae23ee4a36209d7b48b9fa Mon Sep 17 00:00:00 2001 From: Tim Sehn Date: Thu, 18 May 2023 12:08:23 -0700 Subject: [PATCH 2/5] Load data testing progress --- integration-tests/bats/sql-load-data.bats | 154 ++++++++++++++++++++-- 1 file changed, 146 insertions(+), 8 deletions(-) diff --git a/integration-tests/bats/sql-load-data.bats b/integration-tests/bats/sql-load-data.bats index e089666bf5..a67dca575c 100644 --- a/integration-tests/bats/sql-load-data.bats +++ b/integration-tests/bats/sql-load-data.bats @@ -246,6 +246,46 @@ SQL [ "${lines[2]}" = "1,1,2,3,4,5" ] } +@test "sql-load-data: run twice it appends" { + cat < in.csv +0,0,0 +CSV + + dolt sql -q "create table t (pk int primary key, c1 int, c2 int)" + dolt sql < in.csv +1,1,1 +CSV + + dolt sql < in.csv 0,0,0 @@ -264,8 +304,8 @@ SQL run dolt sql -r csv -q "select * from t" [ $status -eq 0 ] - [[ $output =~ 0,0,0 ]] || false - [[ $output =~ 1,1,1 ]] || false + [[ $output =~ "0,0,0" ]] || false + [[ $output =~ "1,1,1" ]] || false } @@ -278,27 +318,125 @@ CSV dolt sql -q "create table t (pk int primary key, c1 int, c2 int)" dolt sql -q "insert into t values (0,0,0)" skip "load data replace not supported" - run dolt sql < in.csv +this,is,keyless +and,uses,strings +CSV + dolt sql -q "create table t (c1 varchar(10), c2 varchar(20), c3 varchar(30))" + + dolt sql < in.csv +0,a,a,this is text,0,0,0.01,a +CSV + dolt sql -q "create table t ( +pk int primary key, +c1 char(1), +c2 varchar(1), +c3 text, +c4 int, +c5 tinyint, +c6 double, +c7 enum('a','b') +)" + dolt sql < in.csv +1,a,a,this is text,0,5555555,0.01,a +CSV + run dolt sql < in.csv +0,2022-10-10 00:00:00,2022-10-10,00:00:00 +CSV + + dolt sql -q "create table t ( +pk int primary key, +c1 datetime, +c2 date, +c3 time)" + + dolt sql < in.csv +1,2022-10-10 00:00:00:00,2022-10-10,00:00:00 +CSV + + run dolt sql < Date: Thu, 18 May 2023 12:42:43 -0700 Subject: [PATCH 3/5] More load data tests --- integration-tests/bats/sql-load-data.bats | 84 +++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/integration-tests/bats/sql-load-data.bats b/integration-tests/bats/sql-load-data.bats index a67dca575c..bf6da95e84 100644 --- a/integration-tests/bats/sql-load-data.bats +++ b/integration-tests/bats/sql-load-data.bats @@ -437,15 +437,99 @@ SQL @test "sql-load-data: schema with not null constraints" { + cat < in.csv +0,0,0 +CSV + dolt sql -q "create table t (pk int primary key, c1 int not null, c2 int)" + dolt sql < in.csv +1,NULL,1 +CSV + + run dolt sql < in.csv +0,0,0 +CSV + dolt sql -q "create table t (pk int primary key, c1 int default 1, c2 int)" + dolt sql < in.csv +1,NULL,1 +CSV + + dolt sql < in.csv +0,0,0 +CSV + dolt sql -q "create table t (pk int primary key, c1 int, c2 int, check(c1 > 0))" + run dolt sql < in.csv +0,1,0 +CSV + + run dolt sql < Date: Thu, 18 May 2023 13:15:22 -0700 Subject: [PATCH 4/5] More thorough load data bats tests --- integration-tests/bats/sql-load-data.bats | 31 ++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/integration-tests/bats/sql-load-data.bats b/integration-tests/bats/sql-load-data.bats index bf6da95e84..c968b93fff 100644 --- a/integration-tests/bats/sql-load-data.bats +++ b/integration-tests/bats/sql-load-data.bats @@ -113,7 +113,7 @@ SQL } @test "sql-load-data: works with fields separated by tabs" { - skip "This needs to be fixed." + skip "This is a test problem with CI preserving tabs in this file. Tabs work locally." cat < 1pk2col-ints.csv pk c1 0 1 @@ -533,11 +533,36 @@ SQL } @test "sql-load-data: test schema with foreign keys" { + dolt sql -q "create table t1 (pk int primary key, c1 int, c2 int)" + dolt sql -q "create table t2 (pk int primary key, foreign key (pk) references t1(pk))" -} + dolt sql -q "insert into t1 values (0,0,0),(2,2,2)" -@test "sql-load-data: test load data defaults" { + cat < in.csv +0 +2 +CSV + dolt sql -q "load data infile 'in.csv' into table t2" + run dolt sql -r csv -q "select * from t2" + [ $status -eq 0 ] + [[ $output =~ "0" ]] || false + [[ $output =~ "2" ]] || false + + cat < in.csv +1 +CSV + + run dolt sql -q "load data infile 'in.csv' into table t2" + [ $status -ne 0 ] + [[ $output =~ "Foreign key violation" ]] || false + + dolt sql -q "set foreign_key_checks=0; load data infile 'in.csv' into table t2" + run dolt sql -r csv -q "select * from t2" + [ $status -eq 0 ] + [[ $output =~ "0" ]] || false + [[ $output =~ "1" ]] || false + [[ $output =~ "2" ]] || false } @test "sql-load-data: load data local" { From 3b43b06ceffb8c0a6143f201b438ded16c23f82b Mon Sep 17 00:00:00 2001 From: Tim Sehn Date: Thu, 18 May 2023 13:24:42 -0700 Subject: [PATCH 5/5] Remove extraneous spaces --- integration-tests/bats/sql-load-data.bats | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration-tests/bats/sql-load-data.bats b/integration-tests/bats/sql-load-data.bats index c968b93fff..a0e95d43f7 100644 --- a/integration-tests/bats/sql-load-data.bats +++ b/integration-tests/bats/sql-load-data.bats @@ -433,8 +433,6 @@ SQL [ $status -ne 0 ] [[ $output =~ "Incorrect datetime value" ]] || false } - - @test "sql-load-data: schema with not null constraints" { cat < in.csv