mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-01-07 00:39:44 -06:00
Code Issues Packages Projects Releases 115 Wiki Activity

support 'add' and 'remove' options for dolt_backup() (#5660)

Browse Source
This commit is contained in:
jennifersp
2023-04-03 14:50:10 -07:00
committed by GitHub
parent 01f9ff9648
commit 5b11cd3583
4 changed files with 301 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
go/libraries/doltcore/env/environment.go vendored
View File

@@ -868,16 +868,16 @@ func (dEnv *DoltEnv) GetRemotes() (map[string]Remote, error) {
return dEnv.RepoState.Remotes, nil
}
// Check whether any backups or remotes share the given URL. Returns the first remote if multiple match.
// CheckRemoteAddressConflict checks whether any backups or remotes share the given URL. Returns the first remote if multiple match.
// Returns NoRemote and false if none match.
func checkRemoteAddressConflict(url string, remotes, backups map[string]Remote) (Remote, bool) {
func CheckRemoteAddressConflict(absUrl string, remotes, backups map[string]Remote) (Remote, bool) {
for _, r := range remotes {
if r.Url == url {
if r.Url == absUrl {
return r, true
}
}
for _, r := range backups {
if r.Url == url {
if r.Url == absUrl {
return r, true
}
}
@@ -899,7 +899,7 @@ func (dEnv *DoltEnv) AddRemote(r Remote) error {
}
// can have multiple remotes with the same address, but no conflicting backups
if rem, found := checkRemoteAddressConflict(absRemoteUrl, nil, dEnv.RepoState.Backups); found {
if rem, found := CheckRemoteAddressConflict(absRemoteUrl, nil, dEnv.RepoState.Backups); found {
return fmt.Errorf("%w: '%s' -> %s", ErrRemoteAddressConflict, rem.Name, rem.Url)
}
@@ -931,7 +931,7 @@ func (dEnv *DoltEnv) AddBackup(r Remote) error {
}
// no conflicting remote or backup addresses
if rem, found := checkRemoteAddressConflict(absRemoteUrl, dEnv.RepoState.Remotes, dEnv.RepoState.Backups); found {
if rem, found := CheckRemoteAddressConflict(absRemoteUrl, dEnv.RepoState.Remotes, dEnv.RepoState.Backups); found {
return fmt.Errorf("%w: '%s' -> %s", ErrRemoteAddressConflict, rem.Name, rem.Url)
}

227
go/libraries/doltcore/sqle/dprocedures/dolt_backup.go
View File

@@ -26,6 +26,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/datas/pull"
)
@@ -73,91 +74,179 @@ func doDoltBackup(ctx *sql.Context, args []string) (int, error) {
return statusErr, sql.ErrDatabaseNotFound.New(dbName)
}
var b env.Remote
switch {
case apr.NArg() == 0:
return statusErr, fmt.Errorf("listing existing backups endpoints in sql is unimplemented.")
case apr.Arg(0) == cli.AddBackupId:
return statusErr, fmt.Errorf("adding backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.RemoveBackupId:
return statusErr, fmt.Errorf("removing backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.RemoveBackupShortId:
return statusErr, fmt.Errorf("removing backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.RestoreBackupId:
if apr.NArg() == 0 {
return statusErr, fmt.Errorf("listing existing backup endpoints in sql is not currently implemented. Let us know if you need this by opening a GitHub issue: https://github.com/dolthub/dolt/issues")
}
switch apr.Arg(0) {
case cli.AddBackupId:
err = addBackup(ctx, dbData, apr)
if err != nil {
return statusErr, fmt.Errorf("error adding backup: %w", err)
}
case cli.RemoveBackupId, cli.RemoveBackupShortId:
err = removeBackup(ctx, dbData, apr)
if err != nil {
return statusErr, fmt.Errorf("error removing backup: %w", err)
}
case cli.RestoreBackupId:
return statusErr, fmt.Errorf("restoring backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.SyncBackupUrlId:
if apr.NArg() != 2 {
return statusErr, fmt.Errorf("usage: dolt_backup('sync-url', BACKUP_URL)")
}
backupUrl := strings.TrimSpace(apr.Arg(1))
cfg := loadConfig(ctx)
scheme, absBackupUrl, err := env.GetAbsRemoteUrl(filesys.LocalFS, cfg, backupUrl)
case cli.SyncBackupUrlId:
err = syncBackupViaUrl(ctx, dbData, sess, apr)
if err != nil {
return statusErr, fmt.Errorf("error: '%s' is not valid.", backupUrl)
} else if scheme == dbfactory.HTTPScheme || scheme == dbfactory.HTTPSScheme {
// not sure how to get the dialer so punting on this
return statusErr, fmt.Errorf("sync-url does not support http or https backup locations currently")
return statusErr, fmt.Errorf("error syncing backup url: %w", err)
}
params, err := cli.ProcessBackupArgs(apr, scheme, absBackupUrl)
case cli.SyncBackupId:
err = syncBackupViaName(ctx, dbData, sess, apr)
if err != nil {
return statusErr, err
return statusErr, fmt.Errorf("error syncing backup: %w", err)
}
credsFile, _ := sess.GetSessionVariable(ctx, dsess.AwsCredsFile)
credsFileStr, isStr := credsFile.(string)
if isStr && len(credsFileStr) > 0 {
params[dbfactory.AWSCredsFileParam] = credsFileStr
}
credsProfile, err := sess.GetSessionVariable(ctx, dsess.AwsCredsProfile)
profStr, isStr := credsProfile.(string)
if isStr && len(profStr) > 0 {
params[dbfactory.AWSCredsProfile] = profStr
}
credsRegion, err := sess.GetSessionVariable(ctx, dsess.AwsCredsRegion)
regionStr, isStr := credsRegion.(string)
if isStr && len(regionStr) > 0 {
params[dbfactory.AWSRegionParam] = regionStr
}
b = env.NewRemote("__temp__", backupUrl, params)
case apr.Arg(0) == cli.SyncBackupId:
if apr.NArg() != 2 {
return statusErr, fmt.Errorf("usage: dolt_backup('sync', BACKUP_NAME)")
}
backupName := strings.TrimSpace(apr.Arg(1))
backups, err := dbData.Rsr.GetBackups()
if err != nil {
return statusErr, err
}
b, ok = backups[backupName]
if !ok {
return statusErr, fmt.Errorf("error: unknown backup: '%s'; %v", backupName, backups)
}
default:
return statusErr, fmt.Errorf("unrecognized dolt_backup parameter: %s", apr.Arg(0))
}
destDb, err := sess.Provider().GetRemoteDB(ctx, dbData.Ddb.ValueReadWriter().Format(), b, true)
return statusOk, nil
}
func addBackup(ctx *sql.Context, dbData env.DbData, apr *argparser.ArgParseResults) error {
if apr.NArg() != 3 {
return fmt.Errorf("usage: dolt_backup('add', 'backup_name', 'backup-url')")
}
backupName := strings.TrimSpace(apr.Arg(1))
backupUrl := apr.Arg(2)
cfg := loadConfig(ctx)
scheme, absBackupUrl, err := env.GetAbsRemoteUrl(filesys.LocalFS, cfg, backupUrl)
if err != nil {
return statusErr, fmt.Errorf("error loading backup destination: %w", err)
return fmt.Errorf("error: '%s' is not valid, %s", backupUrl, err.Error())
} else if scheme == dbfactory.HTTPScheme || scheme == dbfactory.HTTPSScheme {
// not sure how to get the dialer so punting on this
return fmt.Errorf("sync-url does not support http or https backup locations currently")
}
params, err := cli.ProcessBackupArgs(apr, scheme, absBackupUrl)
if err != nil {
return err
}
r := env.NewRemote(backupName, absBackupUrl, params)
err = dbData.Rsw.AddBackup(r)
switch err {
case nil:
return nil
case env.ErrBackupAlreadyExists:
return fmt.Errorf("error: a backup named '%s' already exists, remove it before running this command again", r.Name)
case env.ErrBackupNotFound:
return fmt.Errorf("error: unknown backup: '%s' ", r.Name)
case env.ErrInvalidBackupURL:
return fmt.Errorf("error: '%s' is not valid, cause: %s", r.Url, err.Error())
case env.ErrInvalidBackupName:
return fmt.Errorf("error: invalid backup name: '%s'", r.Name)
default:
return fmt.Errorf("error: Unable to save changes, cause: %s", err.Error())
}
}
func removeBackup(ctx *sql.Context, dbData env.DbData, apr *argparser.ArgParseResults) error {
if apr.NArg() != 2 {
return fmt.Errorf("usage: dolt_backup('remove', 'backup_name'")
}
backupName := strings.TrimSpace(apr.Arg(1))
err := dbData.Rsw.RemoveBackup(ctx, backupName)
switch err {
case nil:
return nil
case env.ErrFailedToWriteRepoState:
return fmt.Errorf("error: failed to save change to repo state, cause: %s", err.Error())
case env.ErrFailedToDeleteBackup:
return fmt.Errorf("error: failed to delete backup tracking ref, cause: %s", err.Error())
case env.ErrFailedToReadFromDb:
return fmt.Errorf("error: failed to read from db, cause: %s", err.Error())
case env.ErrBackupNotFound:
return fmt.Errorf("error: unknown backup: '%s' ", backupName)
default:
return fmt.Errorf("error: unknown error, cause: %s", err.Error())
}
}
func syncBackupViaUrl(ctx *sql.Context, dbData env.DbData, sess *dsess.DoltSession, apr *argparser.ArgParseResults) error {
if apr.NArg() != 2 {
return fmt.Errorf("usage: dolt_backup('sync-url', BACKUP_URL)")
}
backupUrl := strings.TrimSpace(apr.Arg(1))
cfg := loadConfig(ctx)
scheme, absBackupUrl, err := env.GetAbsRemoteUrl(filesys.LocalFS, cfg, backupUrl)
if err != nil {
return fmt.Errorf("error: '%s' is not valid.", backupUrl)
} else if scheme == dbfactory.HTTPScheme || scheme == dbfactory.HTTPSScheme {
// not sure how to get the dialer so punting on this
return fmt.Errorf("sync-url does not support http or https backup locations currently")
}
params, err := cli.ProcessBackupArgs(apr, scheme, absBackupUrl)
if err != nil {
return err
}
credsFile, _ := sess.GetSessionVariable(ctx, dsess.AwsCredsFile)
credsFileStr, isStr := credsFile.(string)
if isStr && len(credsFileStr) > 0 {
params[dbfactory.AWSCredsFileParam] = credsFileStr
}
credsProfile, err := sess.GetSessionVariable(ctx, dsess.AwsCredsProfile)
profStr, isStr := credsProfile.(string)
if isStr && len(profStr) > 0 {
params[dbfactory.AWSCredsProfile] = profStr
}
credsRegion, err := sess.GetSessionVariable(ctx, dsess.AwsCredsRegion)
regionStr, isStr := credsRegion.(string)
if isStr && len(regionStr) > 0 {
params[dbfactory.AWSRegionParam] = regionStr
}
b := env.NewRemote("__temp__", backupUrl, params)
return syncRoots(ctx, dbData, sess, b)
}
func syncBackupViaName(ctx *sql.Context, dbData env.DbData, sess *dsess.DoltSession, apr *argparser.ArgParseResults) error {
if apr.NArg() != 2 {
return fmt.Errorf("usage: dolt_backup('sync', BACKUP_NAME)")
}
backupName := strings.TrimSpace(apr.Arg(1))
backups, err := dbData.Rsr.GetBackups()
if err != nil {
return err
}
b, ok := backups[backupName]
if !ok {
return fmt.Errorf("error: unknown backup: '%s'; %v", backupName, backups)
}
return syncRoots(ctx, dbData, sess, b)
}
func syncRoots(ctx *sql.Context, dbData env.DbData, sess *dsess.DoltSession, backup env.Remote) error {
destDb, err := sess.Provider().GetRemoteDB(ctx, dbData.Ddb.ValueReadWriter().Format(), backup, true)
if err != nil {
return fmt.Errorf("error loading backup destination: %w", err)
}
tmpDir, err := dbData.Rsw.TempTableFilesDir()
if err != nil {
return statusErr, err
return err
}
err = actions.SyncRoots(ctx, dbData.Ddb, destDb, tmpDir, runProgFuncs, stopProgFuncs)
if err != nil && err != pull.ErrDBUpToDate {
return 1, fmt.Errorf("error syncing backup: %w", err)
return fmt.Errorf("error syncing backup: %w", err)
}
return statusOk, nil
return nil
}

87
go/libraries/doltcore/sqle/dsess/session_state_adapter.go
View File

@@ -17,6 +17,7 @@ package dsess
import (
"context"
"fmt"
"strings"
"github.com/dolthub/go-mysql-server/sql"
@@ -173,7 +174,13 @@ func (s SessionStateAdapter) UpdateBranch(name string, new env.BranchConfig) err
}
func (s SessionStateAdapter) AddRemote(remote env.Remote) error {
s.remotes[remote.Name] = remote
if _, ok := s.remotes[remote.Name]; ok {
return env.ErrRemoteAlreadyExists
}
if strings.IndexAny(remote.Name, " \t\n\r./\\!@#$%^&*(){}[],.<>'\"?=+|") != -1 {
return env.ErrInvalidBackupName
}
fs, err := s.session.Provider().FileSystemForDatabase(s.dbName)
if err != nil {
@@ -184,16 +191,25 @@ func (s SessionStateAdapter) AddRemote(remote env.Remote) error {
if err != nil {
return err
}
// can have multiple remotes with the same address, but no conflicting backups
if rem, found := env.CheckRemoteAddressConflict(remote.Url, nil, repoState.Backups); found {
return fmt.Errorf("%w: '%s' -> %s", env.ErrRemoteAddressConflict, rem.Name, rem.Url)
}
s.remotes[remote.Name] = remote
repoState.AddRemote(remote)
return repoState.Save(fs)
}
func (s SessionStateAdapter) AddBackup(_ env.Remote) error {
return fmt.Errorf("cannot insert remote in an SQL session")
}
func (s SessionStateAdapter) AddBackup(backup env.Remote) error {
if _, ok := s.backups[backup.Name]; ok {
return env.ErrBackupAlreadyExists
}
func (s SessionStateAdapter) RemoveRemote(_ context.Context, name string) error {
delete(s.remotes, name)
if strings.IndexAny(backup.Name, " \t\n\r./\\!@#$%^&*(){}[],.<>'\"?=+|") != -1 {
return env.ErrInvalidBackupName
}
fs, err := s.session.Provider().FileSystemForDatabase(s.dbName)
if err != nil {
@@ -204,12 +220,67 @@ func (s SessionStateAdapter) RemoveRemote(_ context.Context, name string) error
if err != nil {
return err
}
// no conflicting remote or backup addresses
if bac, found := env.CheckRemoteAddressConflict(backup.Url, repoState.Remotes, repoState.Backups); found {
return fmt.Errorf("%w: '%s' -> %s", env.ErrRemoteAddressConflict, bac.Name, bac.Url)
}
s.backups[backup.Name] = backup
repoState.AddBackup(backup)
return repoState.Save(fs)
}
func (s SessionStateAdapter) RemoveRemote(_ context.Context, name string) error {
remote, ok := s.remotes[name]
if !ok {
return env.ErrRemoteNotFound
}
delete(s.remotes, remote.Name)
fs, err := s.session.Provider().FileSystemForDatabase(s.dbName)
if err != nil {
return err
}
repoState, err := env.LoadRepoState(fs)
if err != nil {
return err
}
remote, ok = repoState.Remotes[name]
if !ok {
// sanity check
return env.ErrRemoteNotFound
}
delete(repoState.Remotes, name)
return repoState.Save(fs)
}
func (s SessionStateAdapter) RemoveBackup(_ context.Context, _ string) error {
return fmt.Errorf("cannot delete remote in an SQL session")
func (s SessionStateAdapter) RemoveBackup(_ context.Context, name string) error {
backup, ok := s.backups[name]
if !ok {
return env.ErrBackupNotFound
}
delete(s.backups, backup.Name)
fs, err := s.session.Provider().FileSystemForDatabase(s.dbName)
if err != nil {
return err
}
repoState, err := env.LoadRepoState(fs)
if err != nil {
return err
}
backup, ok = repoState.Backups[name]
if !ok {
// sanity check
return env.ErrBackupNotFound
}
delete(repoState.Backups, name)
return repoState.Save(fs)
}
func (s SessionStateAdapter) TempTableFilesDir() (string, error) {

70
integration-tests/bats/sql-backup.bats
View File

@@ -17,21 +17,67 @@ teardown() {
}
@test "sql-backup: dolt_backup add" {
run dolt sql -q "call dolt_backup('add', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('add', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
mkdir the_backup
run dolt sql -q "call dolt_backup('add', 'hostedapidb-0', 'file://./the_backup')"
[ "$status" -eq 0 ]
run dolt backup -v
[ "$status" -eq 0 ]
[[ "$output" =~ "the_backup" ]] || false
}
@test "sql-backup: dolt_backup add cannot add remote with address of existing backup" {
mkdir bac1
dolt sql -q "call dolt_backup('add','bac1','file://./bac1')"
run dolt sql -q "call dolt_backup('add','rem1','file://./bac1')"
[ "$status" -eq 1 ]
[[ "$output" =~ "address conflict with a remote: 'bac1'" ]] || false
}
@test "sql-backup: dolt_backup add invalid https backup" {
mkdir bac1
run dolt sql -q "call dolt_backup('add', 'bac1', 'https://doltremoteapi.dolthub.com/Dolthub/non-existing-repo')"
[ "$status" -eq 1 ]
[[ "$output" =~ "sync-url does not support http or https backup locations currently" ]] || false
}
@test "sql-backup: dolt_backup remove" {
mkdir bac1
dolt sql -q "call dolt_backup('add', 'bac1', 'file://./bac1')"
run dolt backup -v
[ "$status" -eq 0 ]
[ "${#lines[@]}" -eq 1 ]
[[ "$output" =~ "bac1" ]] || false
dolt sql -q "call dolt_backup('remove','bac1')"
run dolt backup -v
[ "$status" -eq 0 ]
[ "${#lines[@]}" -eq 0 ]
}
@test "sql-backup: dolt_backup remove cannot remove non-existent backup" {
run dolt backup -v
[ "$status" -eq 0 ]
[ "${#lines[@]}" -eq 0 ]
[[ ! "$output" =~ "bac1" ]] || false
run dolt sql -q "call dolt_backup('remove','bac1')"
[ "$status" -eq 1 ]
[[ "$output" =~ "error: unknown backup: 'bac1'" ]] || false
}
@test "sql-backup: dolt_backup rm" {
run dolt sql -q "call dolt_backup('rm', 'hostedapidb-0')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('rm', 'hostedapidb-0')"
[ "$status" -ne 0 ]
run dolt sql -q "call dolt_backup('remove', 'hostedapidb-0')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('remove', 'hostedapidb-0')"
[ "$status" -ne 0 ]
mkdir bac1
dolt sql -q "call dolt_backup('add', 'bac1', 'file://./bac1')"
run dolt backup -v
[ "$status" -eq 0 ]
[ "${#lines[@]}" -eq 1 ]
[[ "$output" =~ "bac1" ]] || false
dolt sql -q "call dolt_backup('rm','bac1')"
run dolt backup -v
[ "$status" -eq 0 ]
[ "${#lines[@]}" -eq 0 ]
[[ ! "$output" =~ "bac1" ]] || false
}
@test "sql-backup: dolt_backup restore" {