From 171c8a4807715e92cae3e1abda9fbfffed71c8b1 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 23 Jan 2024 16:09:48 -0800 Subject: [PATCH] dolt backup restore --force: Add a --force option to dolt backup restore, which allows restoring into an already existing database. --- go/cmd/dolt/cli/arg_parser_helpers.go | 1 + go/cmd/dolt/commands/backup.go | 96 ++++++++++++++++++--------- integration-tests/bats/backup.bats | 43 +++++++++++- 3 files changed, 109 insertions(+), 31 deletions(-) diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index 599e6b6543..be086558e9 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -250,6 +250,7 @@ func CreateBackupArgParser() *argparser.ArgParser { ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"creds-type", "credential type. Valid options are role, env, and file. See the help section for additional details."}) ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"profile", "AWS profile to use."}) ap.SupportsFlag(VerboseFlag, "v", "When printing the list of backups adds additional details.") + ap.SupportsFlag(ForceFlag, "f", "When restoring a backup, overwrite the contents of the existing database with the same name.") ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "") ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, dbfactory.AWSCredTypes)) ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file") diff --git a/go/cmd/dolt/commands/backup.go b/go/cmd/dolt/commands/backup.go index ab26abb7fc..632cc253ac 100644 --- a/go/cmd/dolt/commands/backup.go +++ b/go/cmd/dolt/commands/backup.go @@ -47,6 +47,7 @@ aws-creds-type specifies the means by which credentials should be retrieved in o role: Use the credentials installed for the current user env: Looks for environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY file: Uses the credentials file specified by the parameter aws-creds-file + GCP backup urls should be of the form gs://gcs-bucket/database and will use the credentials setup using the gcloud command line available from Google. @@ -56,10 +57,11 @@ The local filesystem can be used as a backup by providing a repository url in th Remove the backup named {{.LessThan}}name{{.GreaterThan}}. All configuration settings for the backup are removed. The contents of the backup are not affected. {{.EmphasisLeft}}restore{{.EmphasisRight}} -Restore a Dolt database from a given {{.LessThan}}url{{.GreaterThan}} into a specified directory {{.LessThan}}url{{.GreaterThan}}. +Restore a Dolt database from a given {{.LessThan}}url{{.GreaterThan}} into a specified directory {{.LessThan}}name{{.GreaterThan}}. This will fail if {{.LessThan}}name{{.GreaterThan}} is already a Dolt database unless '--force' is provided, in which case the existing database will be overwritten with the contents of the restored backup. {{.EmphasisLeft}}sync{{.EmphasisRight}} Snapshot the database and upload to the backup {{.LessThan}}name{{.GreaterThan}}. This includes branches, tags, working sets, and remote tracking refs. + {{.EmphasisLeft}}sync-url{{.EmphasisRight}} Snapshot the database and upload the backup to {{.LessThan}}url{{.GreaterThan}}. Like sync, this includes branches, tags, working sets, and remote tracking refs, but it does not require you to create a named backup`, @@ -68,7 +70,7 @@ Snapshot the database and upload the backup to {{.LessThan}}url{{.GreaterThan}}. "[-v | --verbose]", "add [--aws-region {{.LessThan}}region{{.GreaterThan}}] [--aws-creds-type {{.LessThan}}creds-type{{.GreaterThan}}] [--aws-creds-file {{.LessThan}}file{{.GreaterThan}}] [--aws-creds-profile {{.LessThan}}profile{{.GreaterThan}}] {{.LessThan}}name{{.GreaterThan}} {{.LessThan}}url{{.GreaterThan}}", "remove {{.LessThan}}name{{.GreaterThan}}", - "restore {{.LessThan}}url{{.GreaterThan}} {{.LessThan}}name{{.GreaterThan}}", + "restore [--force] {{.LessThan}}url{{.GreaterThan}} {{.LessThan}}name{{.GreaterThan}}", "sync {{.LessThan}}name{{.GreaterThan}}", "sync-url [--aws-region {{.LessThan}}region{{.GreaterThan}}] [--aws-creds-type {{.LessThan}}creds-type{{.GreaterThan}}] [--aws-creds-file {{.LessThan}}file{{.GreaterThan}}] [--aws-creds-profile {{.LessThan}}profile{{.GreaterThan}}] {{.LessThan}}url{{.GreaterThan}}", }, @@ -298,13 +300,15 @@ func restoreBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar return errhand.BuildDError("").SetPrintUsage().Build() } apr.Args = apr.Args[1:] - dir, urlStr, verr := parseArgs(apr) + restoredDB, urlStr, verr := parseArgs(apr) if verr != nil { return verr } - // second return value isDir is relevant but handled by library functions - userDirExists, _ := dEnv.FS.Exists(dir) + // For error recovery, record whether EnvForClone created the directory, or just `.dolt/noms` within the directory. + userDirExisted, _ := dEnv.FS.Exists(restoredDB) + + force := apr.Contains(cli.ForceFlag) scheme, remoteUrl, err := env.GetAbsRemoteUrl(dEnv.FS, dEnv.Config, urlStr) if err != nil { @@ -323,34 +327,66 @@ func restoreBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgPar return errhand.VerboseErrorFromError(err) } - // Create a new Dolt env for the clone; use env.NoRemote to avoid origin upstream - clonedEnv, err := actions.EnvForClone(ctx, srcDb.ValueReadWriter().Format(), env.NoRemote, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) + mrEnv, err := env.MultiEnvForDirectory(ctx, dEnv.Config.WriteableConfig(), dEnv.FS, dEnv.Version, dEnv) if err != nil { - return errhand.VerboseErrorFromError(err) + return errhand.BuildDError("error: Unable to list databases").AddCause(err).Build() } - - // Nil out the old Dolt env so we don't accidentally use the wrong database - dEnv = nil - - // still make empty repo state - _, err = env.CreateRepoState(clonedEnv.FS, env.DefaultInitBranch) - if err != nil { - return errhand.VerboseErrorFromError(err) - } - tmpDir, err := clonedEnv.TempTableFilesDir() - if err != nil { - return errhand.VerboseErrorFromError(err) - } - err = actions.SyncRoots(ctx, srcDb, clonedEnv.DoltDB, tmpDir, buildProgStarter(downloadLanguage), stopProgFuncs) - if err != nil { - // If we're cloning into a directory that already exists do not erase it. Otherwise - // make best effort to delete the directory we created. - if userDirExists { - _ = clonedEnv.FS.Delete(dbfactory.DoltDir, true) - } else { - _ = clonedEnv.FS.Delete(".", true) + var existingDEnv *env.DoltEnv + err = mrEnv.Iter(func(dbName string, dEnv *env.DoltEnv) (stop bool, err error) { + if dbName == restoredDB { + existingDEnv = dEnv + return true, nil + } + return false, nil + }) + if err != nil { + return errhand.BuildDError("error: Unable to list databases").AddCause(err).Build() + } + + if existingDEnv != nil { + if !force { + return errhand.BuildDError("error: cannot restore backup into " + restoredDB + ". A database with that name already exists. Did you mean to supply --force?").Build() + } + + tmpDir, err := existingDEnv.TempTableFilesDir() + if err != nil { + return errhand.VerboseErrorFromError(err) + } + + err = actions.SyncRoots(ctx, srcDb, existingDEnv.DoltDB, tmpDir, buildProgStarter(downloadLanguage), stopProgFuncs) + if err != nil { + return errhand.VerboseErrorFromError(err) + } + } else { + // Create a new Dolt env for the clone; use env.NoRemote to avoid origin upstream + clonedEnv, err := actions.EnvForClone(ctx, srcDb.ValueReadWriter().Format(), env.NoRemote, restoredDB, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) + if err != nil { + return errhand.VerboseErrorFromError(err) + } + + // Nil out the old Dolt env so we don't accidentally use the wrong database + dEnv = nil + + // still make empty repo state + _, err = env.CreateRepoState(clonedEnv.FS, env.DefaultInitBranch) + if err != nil { + return errhand.VerboseErrorFromError(err) + } + tmpDir, err := clonedEnv.TempTableFilesDir() + if err != nil { + return errhand.VerboseErrorFromError(err) + } + err = actions.SyncRoots(ctx, srcDb, clonedEnv.DoltDB, tmpDir, buildProgStarter(downloadLanguage), stopProgFuncs) + if err != nil { + // If we're cloning into a directory that already exists do not erase it. Otherwise + // make best effort to delete the directory we created. + if userDirExisted { + _ = clonedEnv.FS.Delete(dbfactory.DoltDir, true) + } else { + _ = clonedEnv.FS.Delete(".", true) + } + return errhand.VerboseErrorFromError(err) } - return errhand.VerboseErrorFromError(err) } return nil diff --git a/integration-tests/bats/backup.bats b/integration-tests/bats/backup.bats index bcf8bdc05f..6d447220c7 100644 --- a/integration-tests/bats/backup.bats +++ b/integration-tests/bats/backup.bats @@ -270,9 +270,50 @@ teardown() { [[ "$output" =~ "t1" ]] || false } +@test "backup: restore existing database fails" { + cd repo1 + dolt backup sync-url file://../bac1 + + cd .. + mkdir repo2 + cd repo2 + dolt init + + # Check in the ".dolt" is in my current directory case... + run dolt backup restore file://../bac1 repo2 + [ "$status" -ne 0 ] + [[ "$output" =~ "cannot restore backup into repo2. A database with that name already exists" ]] || false + + # Check in the ".dolt" is in a subdirectory case... + cd .. + run dolt backup restore file://../bac1 repo2 + [ "$status" -ne 0 ] + [[ "$output" =~ "cannot restore backup into repo2. A database with that name already exists" ]] || false +} + +@test "backup: restore existing database with --force succeeds" { + cd repo1 + dolt backup sync-url file://../bac1 + + cd .. + mkdir repo2 + cd repo2 + dolt init + + # Check in the ".dolt" is in my current directory case... + dolt backup restore --force file://../bac1 repo2 + + cd ../repo1 + dolt commit --allow-empty -m 'another commit' + dolt backup sync-url file://../bac1 + + # Check in the ".dolt" is in a subdirectory case... + cd .. + dolt backup restore --force file://./bac1 repo2 +} + @test "backup: sync-url in a non-dolt directory" { mkdir newdir && cd newdir run dolt backup sync-url file://../bac1 [ "$status" -ne 0 ] } -