mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-09 10:38:10 -06:00
Code Issues Packages Projects Releases 127 Wiki Activity

add dump csv format

Browse Source
This commit is contained in:
jennifersp
2021-11-04 09:47:11 -07:00
parent 15e3498ba5
commit 51ed0fd79e
2 changed files with 229 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
go/cmd/dolt/commands/dump.go
View File

@@ -94,11 +94,13 @@ func (cmd DumpCmd) Exec(ctx context.Context, commandStr string, args []string, d
help, usage := cli.HelpAndUsagePrinters(cli.GetCommandDocumentation(commandStr, dumpDocs, ap))
apr := cli.ParseArgsOrDie(ap, args, help)
dumpOpts, verr := getDumpArgs(apr)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
if apr.NArg() > 0 {
return HandleVErrAndExitCode(errhand.BuildDError("too many arguments").SetPrintUsage().Build(), usage)
}
var fileName string
resultFormat, _ := apr.GetValue(FormatFlag)
root, verr := GetWorkingWithVErr(dEnv)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
@@ -106,27 +108,6 @@ func (cmd DumpCmd) Exec(ctx context.Context, commandStr string, args []string, d
force := apr.Contains(forceParam)
ow, err := checkOverwrite(ctx, root, dEnv.FS, force, dumpOpts.dest)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
if ow {
return HandleVErrAndExitCode(errhand.BuildDError("%s already exists. Use -f to overwrite.", dumpOpts.DumpDestName()).Build(), usage)
}
// create new file
err = dEnv.FS.MkDirs(filepath.Dir(dumpOpts.DumpDestName()))
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
filePath, err := dEnv.FS.Abs(dumpOpts.DumpDestName())
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
os.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
tblNames, err := doltdb.GetNonSystemTableNames(ctx, root)
if err != nil {
errhand.BuildDError("error: failed to get tables").AddCause(err).Build()
@@ -136,22 +117,44 @@ func (cmd DumpCmd) Exec(ctx context.Context, commandStr string, args []string, d
return 0
}
for _, tbl := range tblNames {
tblOpts := newTableArgs(tbl, dumpOpts.dest)
mover, verr := NewDumpDataMover(ctx, root, dEnv, tblOpts, importStatsCB, filePath)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
switch resultFormat {
case "", "sql", ".sql":
fileName = "doltdump.sql"
dumpOpts, err := getDumpArgs(fileName, resultFormat)
fPath, err := checkAndCreateOpenDestFile(ctx, root, dEnv, force, dumpOpts, fileName)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
skipped, verr := mvdata.MoveData(ctx, dEnv, mover, tblOpts)
if skipped > 0 {
cli.PrintErrln(color.YellowString("Lines skipped: %d", skipped))
for _, tbl := range tblNames {
tblOpts := newTableArgs(tbl, dumpOpts.dest)
mErr := dumpTable(ctx, root, dEnv, tblOpts, fPath)
if mErr != nil {
return HandleVErrAndExitCode(mErr, usage)
}
}
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
case "csv", ".csv":
fileName = "doltdump/"
for _, tbl := range tblNames {
fileName = "doltdump/" + tbl + ".csv"
dumpOpts, err := getDumpArgs(fileName, resultFormat)
fPath, err := checkAndCreateOpenDestFile(ctx, root, dEnv, force, dumpOpts, fileName)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
tblOpts := newTableArgs(tbl, dumpOpts.dest)
mErr := dumpTable(ctx, root, dEnv, tblOpts, fPath)
if mErr != nil {
return HandleVErrAndExitCode(mErr, usage)
}
}
default:
return HandleVErrAndExitCode(errhand.BuildDError("invalid result format").SetPrintUsage().Build(), usage)
}
cli.PrintErrln(color.CyanString("Successfully exported data."))
@@ -199,6 +202,50 @@ func (m dumpOptions) DumpDestName() string {
return m.dest.String()
}
// dumpTable dumps table in file given specific table and file location info
func dumpTable(ctx context.Context, root * doltdb.RootValue, dEnv *env.DoltEnv, tblOpts *tableOptions, filePath string) errhand.VerboseError {
mover, verr := NewDumpDataMover(ctx, root, dEnv, tblOpts, importStatsCB, filePath)
if verr != nil {
return verr
}
skipped, verr := mvdata.MoveData(ctx, dEnv, mover, tblOpts)
if skipped > 0 {
cli.PrintErrln(color.YellowString("Lines skipped: %d", skipped))
}
if verr != nil {
return verr
}
return nil
}
// checkAndCreateOpenDestFile returns filePath to created dest file after checking for any existing file and handles it
func checkAndCreateOpenDestFile(ctx context.Context, root *doltdb.RootValue, dEnv *env.DoltEnv, force bool, dumpOpts *dumpOptions, fileName string) (string, errhand.VerboseError) {
ow, err := checkOverwrite(ctx, root, dEnv.FS, force, dumpOpts.dest)
if err != nil {
return "", errhand.VerboseErrorFromError(err)
}
if ow {
return "", errhand.BuildDError("%s already exists. Use -f to overwrite.", fileName).Build()
}
// create new file
err = dEnv.FS.MkDirs(filepath.Dir(dumpOpts.DumpDestName()))
if err != nil {
return "", errhand.VerboseErrorFromError(err)
}
filePath, err := dEnv.FS.Abs(fileName)
if err != nil {
return "", errhand.VerboseErrorFromError(err)
}
os.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
return filePath, nil
}
// checkOverwrite returns TRUE if the file exists and force flag not given and
// FALSE if the file is stream data / file does not exist / file exists and force flag is given
func checkOverwrite(ctx context.Context, root *doltdb.RootValue, fs filesys.ReadableFS, force bool, dest mvdata.DataLocation) (bool, error) {
@@ -245,29 +292,11 @@ func getDumpDestination(path string) mvdata.DataLocation {
}
// getDumpArgs returns dumpOptions of result format and dest file location corresponding to the input parameters
func getDumpArgs(apr *argparser.ArgParseResults) (*dumpOptions, errhand.VerboseError) {
if apr.NArg() > 0 {
return nil, errhand.BuildDError("too many arguments").SetPrintUsage().Build()
}
var fileName string
resultFormat, _ := apr.GetValue(FormatFlag)
switch resultFormat {
case "", "sql", ".sql":
fileName = "doltdump.sql"
case "csv", ".csv":
//handle CSV filetype
//maybe create dir 'doltdump' and put all the csv dump files in it
default:
return nil, errhand.BuildDError("invalid result format").SetPrintUsage().Build()
}
func getDumpArgs(fileName string, rf string) (*dumpOptions, errhand.VerboseError) {
fileLoc := getDumpDestination(fileName)
return &dumpOptions{
format: resultFormat,
format: rf,
dest: fileLoc,
}, nil
}
@@ -324,3 +353,4 @@ func NewDumpDataMover(ctx context.Context, root *doltdb.RootValue, dEnv *env.Dol
return imp, nil
}
Z

159
integration-tests/bats/dump.bats
View File

@@ -10,9 +10,9 @@ teardown() {
teardown_common
}
@test "dump: dolt dump SQL export with multiple tables" {
dolt sql -q "CREATE TABLE mysqldump_table(pk int);"
dolt sql -q "INSERT INTO mysqldump_table VALUES (1);"
@test "dump: SQL type - dolt dump with multiple tables" {
dolt sql -q "CREATE TABLE new_table(pk int);"
dolt sql -q "INSERT INTO new_table VALUES (1);"
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
dolt sql -q "create table enums (a varchar(10) primary key, b enum('one','two','three'))"
@@ -42,14 +42,13 @@ teardown() {
run dolt sql < doltdump.sql
[ "$status" -eq 0 ]
[[ "$output" =~ "Rows inserted: 6 Rows updated: 0 Rows deleted: 0" ]] || false
}
@test "dump: compare tables in database with tables imported from doltdump.sql " {
@test "dump: SQL type - compare tables in database with tables imported file " {
dolt branch new_branch
dolt sql -q "CREATE TABLE mysqldump_table(pk int);"
dolt sql -q "INSERT INTO mysqldump_table VALUES (1);"
dolt sql -q "CREATE TABLE new_table(pk int);"
dolt sql -q "INSERT INTO new_table VALUES (1);"
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
dolt sql -q "CREATE TABLE keyless (c0 int, c1 int);"
@@ -72,9 +71,9 @@ teardown() {
[[ "$output" = "" ]] || false
}
@test "dump: dolt dump with Indexes" {
dolt sql -q "CREATE TABLE mysqldump_table(pk int);"
dolt sql -q "INSERT INTO mysqldump_table VALUES (1);"
@test "dump: SQL type - dolt dump with Indexes" {
dolt sql -q "CREATE TABLE new_table(pk int);"
dolt sql -q "INSERT INTO new_table VALUES (1);"
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
dolt sql -q "CREATE TABLE onepk (pk1 BIGINT PRIMARY KEY, v1 BIGINT, v2 BIGINT);"
@@ -94,10 +93,10 @@ teardown() {
[[ "$output" =~ 'KEY `idx_v1` (`v1`)' ]] || false
}
@test "dump: dolt dump with foreign key and import" {
@test "dump: SQL type - dolt dump with foreign key and import" {
skip "dolt dump foreign key option for import NOT implemented"
dolt sql -q "CREATE TABLE mysqldump_table(pk int);"
dolt sql -q "INSERT INTO mysqldump_table VALUES (1);"
dolt sql -q "CREATE TABLE new_table(pk int);"
dolt sql -q "INSERT INTO new_table VALUES (1);"
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
dolt sql -q "CREATE TABLE parent (id int PRIMARY KEY, pv1 int, pv2 int, INDEX v1 (pv1), INDEX v2 (pv2));"
@@ -114,7 +113,7 @@ teardown() {
[ "$status" -eq 0 ]
}
@test "dump: dolt dump with views/trigger" {
@test "dump: SQL type - dolt dump with views/trigger" {
skip "dolt dump views/trigger NOT implemented"
dolt sql -q "CREATE TABLE test(pk BIGINT PRIMARY KEY, v1 BIGINT);"
dolt sql -q "CREATE TRIGGER trigger1 BEFORE INSERT ON test FOR EACH ROW SET new.v1 = -new.v1;"
@@ -127,7 +126,11 @@ teardown() {
dolt sql -q "INSERT INTO a VALUES (2);"
}
@test "dump: dolt dump with keyless tables" {
@test "dump: SQL type - dolt dump with keyless tables" {
dolt sql -q "CREATE TABLE new_table(pk int);"
dolt sql -q "INSERT INTO new_table VALUES (1);"
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
dolt sql -q "CREATE TABLE keyless (c0 int, c1 int);"
dolt sql -q "INSERT INTO keyless VALUES (0,0),(2,2),(1,1),(1,1);"
dolt sql -q "ALTER TABLE keyless ADD INDEX (c1);"
@@ -155,5 +158,129 @@ teardown() {
[[ "${lines[2]}" = "1,1" ]] || false
[[ "${lines[3]}" = "1,1" ]] || false
[[ "${lines[4]}" = "4,2" ]] || false
}
@test "dump: SQL type - dolt dump with empty tables" {
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "CREATE TABLE keyless (c0 int, c1 int);"
dolt sql -q "CREATE TABLE test(pk BIGINT PRIMARY KEY, v1 BIGINT);"
dolt add .
dolt commit -m "create tables"
run dolt dump
[ "$status" -eq 0 ]
[ -f doltdump.sql ]
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
run dolt sql < doltdump.sql
[ "$status" -eq 0 ]
run grep CREATE doltdump.sql
[ "$status" -eq 0 ]
[ "${#lines[@]}" -eq 3 ]
run grep INSERT doltdump.sql
[ "$status" -eq 1 ]
[ "${#lines[@]}" -eq 0 ]
}
@test "dump: CSV type - dolt dump with multiple tables and check -f flag" {
dolt sql -q "CREATE TABLE new_table(pk int);"
dolt sql -q "INSERT INTO new_table VALUES (1);"
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
dolt sql -q "create table enums (a varchar(10) primary key, b enum('one','two','three'))"
dolt sql -q "insert into enums values ('abc', 'one'), ('def', 'two')"
run dolt dump -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "Successfully exported data." ]] || false
[ -f doltdump/new_table.csv ]
[ -f doltdump/warehouse.csv ]
[ -f doltdump/enums.csv ]
run dolt dump -r csv
[ "$status" -ne 0 ]
[[ "$output" =~ "new_table.csv already exists" ]] ||
[[ "$output" =~ "warehouse.csv already exists" ]] ||
[[ "$output" =~ "enums.csv already exists" ]] || false
run dolt dump -f -r csv
[ "$status" -eq 0 ]
[[ "$output" =~ "Successfully exported data." ]] || false
[ -f doltdump/new_table.csv ]
[ -f doltdump/warehouse.csv ]
[ -f doltdump/enums.csv ]
}
@test "dump: CSV type - compare tables in database with tables imported from corresponding files " {
dolt sql -q "CREATE TABLE new_table(pk int);"
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "CREATE TABLE keyless (c0 int, c1 int);"
dolt add .
dolt commit -m "create tables"
dolt branch new_branch
dolt sql -q "INSERT INTO new_table VALUES (1);"
dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
dolt sql -q "INSERT INTO keyless VALUES (0,0),(2,2),(1,1),(1,1);"
dolt add .
dolt commit -m "insert to tables"
run dolt dump -r csv
[ "$status" -eq 0 ]
[ -f doltdump/new_table.csv ]
[ -f doltdump/warehouse.csv ]
[ -f doltdump/keyless.csv ]
dolt checkout new_branch
dolt table import -r new_table doltdump/new_table.csv
dolt table import -r warehouse doltdump/warehouse.csv
dolt table import -r keyless doltdump/keyless.csv
dolt add .
dolt commit --allow-empty -m "create tables from doltdump"
run dolt diff --summary main new_branch
[ "$status" -eq 0 ]
[[ "$output" = "" ]] || false
}
@test "dump: CSV type - dolt dump with empty tables" {
dolt branch new_branch
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "CREATE TABLE keyless (c0 int, c1 int);"
dolt sql -q "CREATE TABLE test(pk BIGINT PRIMARY KEY, v1 BIGINT);"
dolt add .
dolt commit -m "create tables"
run dolt dump -r csv
[ "$status" -eq 0 ]
[ -f doltdump/warehouse.csv ]
[ -f doltdump/keyless.csv ]
[ -f doltdump/test.csv ]
dolt checkout new_branch
dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext);"
dolt sql -q "CREATE TABLE keyless (c0 int, c1 int);"
dolt sql -q "CREATE TABLE test(pk BIGINT PRIMARY KEY, v1 BIGINT);"
dolt table import -r warehouse doltdump/warehouse.csv
dolt table import -r keyless doltdump/keyless.csv
dolt table import -r test doltdump/test.csv
dolt add .
dolt commit --allow-empty -m "create tables from doltdump"
run dolt diff --summary main new_branch
[ "$status" -eq 0 ]
[[ "$output" = "" ]] || false
}