Merge pull request #3441 from dolthub/aaron/dolt_backup-stored-procedure

go/libraries/doltcore/sqle/dfunctions: dolt_backup.go: Implement dolt_backup("sync", ...).
This commit is contained in:
Aaron Son
2022-05-18 10:19:49 -07:00
committed by GitHub
16 changed files with 303 additions and 35 deletions

View File

@@ -20,9 +20,12 @@ import (
"strings"
"time"
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
)
const VerboseFlag = "verbose"
// we are more permissive than what is documented.
var SupportedLayouts = []string{
"2006/01/02",
@@ -92,6 +95,14 @@ const (
DeleteForceFlag = "D"
)
const (
SyncBackupId = "sync"
RestoreBackupId = "restore"
AddBackupId = "add"
RemoveBackupId = "remove"
RemoveBackupShortId = "rm"
)
var mergeAbortDetails = `Abort the current conflict resolution process, and try to reconstruct the pre-merge state.
If there were uncommitted working set changes present when the merge started, {{.EmphasisLeft}}dolt merge --abort{{.EmphasisRight}} will be unable to reconstruct these changes. It is therefore recommended to always commit or stash your changes before running dolt merge.
@@ -186,3 +197,16 @@ func CreateBranchArgParser() *argparser.ArgParser {
return ap
}
func CreateBackupArgParser() *argparser.ArgParser {
ap := argparser.NewArgParser()
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"region", "cloud provider region associated with this backup."})
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"creds-type", "credential type. Valid options are role, env, and file. See the help section for additional details."})
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"profile", "AWS profile to use."})
ap.SupportsFlag(VerboseFlag, "v", "When printing the list of backups adds additional details.")
ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "")
ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, dbfactory.AWSCredTypes))
ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file")
ap.SupportsString(dbfactory.AWSCredsProfile, "", "profile", "AWS profile to use")
return ap
}

View File

@@ -73,14 +73,6 @@ Snapshot the database and upload to the backup {{.LessThan}}name{{.GreaterThan}}
type BackupCmd struct{}
const (
syncBackupId = "sync"
restoreBackupId = "restore"
addBackupId = "add"
removeBackupId = "remove"
removeBackupShortId = "rm"
)
// Name is returns the name of the Dolt cli command. This is what is used on the command line to invoke the command
func (cmd BackupCmd) Name() string {
return "backup"
@@ -102,16 +94,7 @@ func (cmd BackupCmd) CreateMarkdown(wr io.Writer, commandStr string) error {
}
func (cmd BackupCmd) ArgParser() *argparser.ArgParser {
ap := argparser.NewArgParser()
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"region", "cloud provider region associated with this backup."})
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"creds-type", "credential type. Valid options are role, env, and file. See the help section for additional details."})
ap.ArgListHelp = append(ap.ArgListHelp, [2]string{"profile", "AWS profile to use."})
ap.SupportsFlag(verboseFlag, "v", "When printing the list of backups adds additional details.")
ap.SupportsString(dbfactory.AWSRegionParam, "", "region", "")
ap.SupportsValidatedString(dbfactory.AWSCredsTypeParam, "", "creds-type", "", argparser.ValidatorFromStrList(dbfactory.AWSCredsTypeParam, credTypes))
ap.SupportsString(dbfactory.AWSCredsFileParam, "", "file", "AWS credentials file")
ap.SupportsString(dbfactory.AWSCredsProfile, "", "profile", "AWS profile to use")
return ap
return cli.CreateBackupArgParser()
}
// EventType returns the type of the event to log
@@ -130,15 +113,15 @@ func (cmd BackupCmd) Exec(ctx context.Context, commandStr string, args []string,
switch {
case apr.NArg() == 0:
verr = printBackups(dEnv, apr)
case apr.Arg(0) == addBackupId:
case apr.Arg(0) == cli.AddBackupId:
verr = addBackup(dEnv, apr)
case apr.Arg(0) == removeBackupId:
case apr.Arg(0) == cli.RemoveBackupId:
verr = removeBackup(ctx, dEnv, apr)
case apr.Arg(0) == removeBackupShortId:
case apr.Arg(0) == cli.RemoveBackupShortId:
verr = removeBackup(ctx, dEnv, apr)
case apr.Arg(0) == syncBackupId:
case apr.Arg(0) == cli.SyncBackupId:
verr = syncBackup(ctx, dEnv, apr)
case apr.Arg(0) == restoreBackupId:
case apr.Arg(0) == cli.RestoreBackupId:
verr = restoreBackup(ctx, dEnv, apr)
default:
verr = errhand.BuildDError("").SetPrintUsage().Build()
@@ -251,17 +234,25 @@ func syncBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseR
backupName := strings.TrimSpace(apr.Arg(1))
backups, err := dEnv.GetBackups()
if err != nil {
return errhand.BuildDError("Unable to get backups from the local directory").AddCause(err).Build()
}
b, ok := backups[backupName]
if !ok {
return errhand.BuildDError("error: unknown backup: '%s' ", backupName).Build()
}
destDb, err := b.GetRemoteDB(ctx, dEnv.DoltDB.ValueReadWriter().Format())
if err != nil {
return errhand.BuildDError("error: unable to open destination.").AddCause(err).Build()
}
err = actions.SyncRoots(ctx, dEnv.DoltDB, destDb, dEnv.TempTableFilesDir(), buildProgStarter(defaultLanguage), stopProgFuncs)
switch err {
case nil:
return nil
case pull.ErrDBUpToDate:
return nil
case env.ErrBackupAlreadyExists:
return errhand.BuildDError("error: a backup named '%s' already exists.", b.Name).AddDetails("remove it before running this command again").Build()
case env.ErrBackupNotFound:
@@ -270,8 +261,6 @@ func syncBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseR
return errhand.BuildDError("error: '%s' is not valid.", b.Url).AddCause(err).Build()
case env.ErrInvalidBackupName:
return errhand.BuildDError("error: invalid backup name: " + b.Name).Build()
case pull.ErrDBUpToDate:
return errhand.BuildDError("error: backup already up to date").Build()
default:
return errhand.BuildDError("error: Unable to save changes.").AddCause(err).Build()
}

View File

@@ -65,7 +65,7 @@ const (
moveFlag = "move"
deleteFlag = "delete"
deleteForceFlag = "D"
verboseFlag = "verbose"
verboseFlag = cli.VerboseFlag
allFlag = "all"
remoteFlag = "remote"
showCurrentFlag = "show-current"

View File

@@ -69,7 +69,7 @@ const (
)
var awsParams = []string{dbfactory.AWSRegionParam, dbfactory.AWSCredsTypeParam, dbfactory.AWSCredsFileParam, dbfactory.AWSCredsProfile}
var credTypes = []string{dbfactory.RoleCS.String(), dbfactory.EnvCS.String(), dbfactory.FileCS.String()}
var credTypes = dbfactory.AWSCredTypes
type RemoteCmd struct{}

View File

@@ -48,6 +48,8 @@ const (
AWSCredsProfile = "aws-creds-profile"
)
var AWSCredTypes = []string{RoleCS.String(), EnvCS.String(), FileCS.String()}
// AWSCredentialSource is an enum type representing the different credential sources (auto, role, env, file, or invalid)
type AWSCredentialSource int

View File

@@ -96,6 +96,7 @@ func repoStateLegacyFromRepoState(rs *RepoState) *repoStateLegacy {
return &repoStateLegacy{
Head: rs.Head,
Remotes: rs.Remotes,
Backups: rs.Backups,
Branches: rs.Branches,
Staged: rs.staged,
Working: rs.working,

View File

@@ -209,6 +209,11 @@ func GetInitialDBState(ctx context.Context, db SqlDatabase) (dsess.InitialDbStat
return dsess.InitialDbState{}, err
}
backups, err := rsr.GetBackups()
if err != nil {
return dsess.InitialDbState{}, err
}
branches, err := rsr.GetBranches()
if err != nil {
return dsess.InitialDbState{}, err
@@ -221,6 +226,7 @@ func GetInitialDBState(ctx context.Context, db SqlDatabase) (dsess.InitialDbStat
DbData: db.DbData(),
Remotes: remotes,
Branches: branches,
Backups: backups,
Err: retainedErr,
}, nil
}
@@ -364,7 +370,7 @@ func (db Database) GetTableInsensitiveWithRoot(ctx *sql.Context, root *doltdb.Ro
case doltdb.CommitAncestorsTableName:
dt, found = dtables.NewCommitAncestorsTable(ctx, db.ddb), true
case doltdb.StatusTableName:
dt, found = dtables.NewStatusTable(ctx, db.name, db.ddb, dsess.NewSessionStateAdapter(sess.Session, db.name, map[string]env.Remote{}, map[string]env.BranchConfig{}), db.drw), true
dt, found = dtables.NewStatusTable(ctx, db.name, db.ddb, dsess.NewSessionStateAdapter(sess.Session, db.name, map[string]env.Remote{}, map[string]env.BranchConfig{}, map[string]env.Remote{}), db.drw), true
}
if found {
return dt, found, nil

View File

@@ -0,0 +1,125 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dfunctions
import (
"fmt"
"strings"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/expression"
"github.com/dolthub/dolt/go/cmd/dolt/cli"
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/store/datas/pull"
)
const DoltBackupFuncName = "dolt_backup"
// Deprecated: please use the version in the dprocedures package
type DoltBackupFunc struct {
expression.NaryExpression
}
// Deprecated: please use the version in the dprocedures package
func NewDoltBackupFunc(args ...sql.Expression) (sql.Expression, error) {
return &DoltBackupFunc{expression.NaryExpression{ChildExpressions: args}}, nil
}
func (d DoltBackupFunc) String() string {
childrenStrings := make([]string, len(d.Children()))
for i, child := range d.Children() {
childrenStrings[i] = child.String()
}
return fmt.Sprintf("DOLT_BACKUP(%s)", strings.Join(childrenStrings, ","))
}
func (d DoltBackupFunc) Type() sql.Type {
return sql.Int8
}
func (d DoltBackupFunc) WithChildren(children ...sql.Expression) (sql.Expression, error) {
return NewDoltBackupFunc(children...)
}
func (d DoltBackupFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
args, err := getDoltArgs(ctx, row, d.Children())
if err != nil {
return 1, err
}
return DoDoltBackup(ctx, args)
}
func DoDoltBackup(ctx *sql.Context, args []string) (int, error) {
dbName := ctx.GetCurrentDatabase()
if len(dbName) == 0 {
return 1, fmt.Errorf("Empty database name.")
}
apr, err := cli.CreateBackupArgParser().Parse(args)
if err != nil {
return 1, err
}
switch {
case apr.NArg() == 0:
return 1, fmt.Errorf("listing existing backups endpoints in sql is unimplemented.")
case apr.Arg(0) == cli.AddBackupId:
return 1, fmt.Errorf("adding backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.RemoveBackupId:
return 1, fmt.Errorf("removing backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.RemoveBackupShortId:
return 1, fmt.Errorf("removing backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.RestoreBackupId:
return 1, fmt.Errorf("restoring backup endpoint in sql is unimplemented.")
case apr.Arg(0) == cli.SyncBackupId:
if apr.NArg() != 2 {
return 1, fmt.Errorf("usage: dolt_backup('sync', BACKUP_NAME)")
}
backupName := strings.TrimSpace(apr.Arg(1))
sess := dsess.DSessFromSess(ctx.Session)
dbData, ok := sess.GetDbData(ctx, dbName)
if !ok {
return 1, sql.ErrDatabaseNotFound.New(dbName)
}
backups, err := dbData.Rsr.GetBackups()
if err != nil {
return 1, err
}
b, ok := backups[backupName]
if !ok {
return 1, fmt.Errorf("error: unknown backup: '%s'; %v", backupName, backups)
}
destDb, err := b.GetRemoteDB(ctx, dbData.Ddb.ValueReadWriter().Format())
if err != nil {
return 1, fmt.Errorf("error loading backup destination: %w", err)
}
err = actions.SyncRoots(ctx, dbData.Ddb, destDb, dbData.Rsw.TempTableFilesDir(), runProgFuncs, stopProgFuncs)
if err != nil && err != pull.ErrDBUpToDate {
return 1, fmt.Errorf("error syncing backup: %w", err)
}
return 0, nil
default:
return 1, fmt.Errorf("unrecognized dolt_backup parameter: %s", apr.Arg(0))
}
}

View File

@@ -35,6 +35,7 @@ var DoltFunctions = []sql.Function{
sql.FunctionN{Name: DoltFetchFuncName, Fn: NewFetchFunc},
sql.FunctionN{Name: DoltPushFuncName, Fn: NewPushFunc},
sql.FunctionN{Name: DoltBranchFuncName, Fn: NewDoltBranchFunc},
sql.FunctionN{Name: DoltBackupFuncName, Fn: NewDoltBackupFunc},
}
// DolthubApiFunctions are the DoltFunctions that get exposed to Dolthub Api.

View File

@@ -0,0 +1,30 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dprocedures
import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions"
)
// doltBackup is the stored procedure version of the function `dolt_backup`.
func doltBackup(ctx *sql.Context, args ...string) (sql.RowIter, error) {
res, err := dfunctions.DoDoltBackup(ctx, args)
if err != nil {
return nil, err
}
return rowToIter(int64(res)), nil
}

View File

@@ -18,6 +18,7 @@ import "github.com/dolthub/go-mysql-server/sql"
var DoltProcedures = []sql.ExternalStoredProcedureDetails{
{Name: "dolt_add", Schema: int64Schema("status"), Function: doltAdd},
{Name: "dolt_backup", Schema: int64Schema("success"), Function: doltBackup},
{Name: "dolt_branch", Schema: int64Schema("status"), Function: doltBranch},
{Name: "dolt_checkout", Schema: int64Schema("status"), Function: doltCheckout},
{Name: "dolt_clean", Schema: int64Schema("status"), Function: doltClean},

View File

@@ -33,6 +33,7 @@ type InitialDbState struct {
ReadReplica *env.Remote
Remotes map[string]env.Remote
Branches map[string]env.BranchConfig
Backups map[string]env.Remote
// If err is set, this InitialDbState is partially invalid, but may be
// usable to initialize a database at a revision specifier, for

View File

@@ -853,7 +853,7 @@ func (sess *Session) AddDB(ctx *sql.Context, dbState InitialDbState) error {
// the writer with one that errors out
sessionState.dbData = dbState.DbData
sessionState.tmpFileDir = dbState.DbData.Rsw.TempTableFilesDir()
adapter := NewSessionStateAdapter(sess, db.Name(), dbState.Remotes, dbState.Branches)
adapter := NewSessionStateAdapter(sess, db.Name(), dbState.Remotes, dbState.Branches, dbState.Backups)
sessionState.dbData.Rsr = adapter
sessionState.dbData.Rsw = adapter
sessionState.readOnly, sessionState.readReplica = dbState.ReadOnly, dbState.ReadReplica

View File

@@ -75,11 +75,11 @@ var _ env.RepoStateReader = SessionStateAdapter{}
var _ env.RepoStateWriter = SessionStateAdapter{}
var _ env.RootsProvider = SessionStateAdapter{}
func NewSessionStateAdapter(session *Session, dbName string, remotes map[string]env.Remote, branches map[string]env.BranchConfig) SessionStateAdapter {
func NewSessionStateAdapter(session *Session, dbName string, remotes map[string]env.Remote, branches map[string]env.BranchConfig, backups map[string]env.Remote) SessionStateAdapter {
if branches == nil {
branches = make(map[string]env.BranchConfig)
}
return SessionStateAdapter{session: session, dbName: dbName, remotes: remotes, branches: branches}
return SessionStateAdapter{session: session, dbName: dbName, remotes: remotes, branches: branches, backups: backups}
}
func (s SessionStateAdapter) GetRoots(ctx context.Context) (doltdb.Roots, error) {

View File

@@ -167,10 +167,7 @@ teardown() {
cd repo1
dolt backup add bac1 file://../bac1
dolt backup sync bac1
run dolt backup sync bac1
[ "$status" -eq 1 ]
[[ ! "$output" =~ "panic" ]] || false
[[ "$output" =~ "backup already up to date" ]] || false
dolt backup sync bac1
}
@test "backup: no backup exists" {

View File

@@ -0,0 +1,91 @@
#!/usr/bin/env bats
load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
}
teardown() {
teardown_common
}
@test "sql-backup: dolt_backup no argument" {
run dolt sql -q "select dolt_backup()"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup()"
[ "$status" -ne 0 ]
}
@test "sql-backup: dolt_backup add" {
run dolt sql -q "select dolt_backup('add', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('add', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
}
@test "sql-backup: dolt_backup rm" {
run dolt sql -q "select dolt_backup('rm', 'hostedapidb-0')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('rm', 'hostedapidb-0')"
[ "$status" -ne 0 ]
run dolt sql -q "select dolt_backup('remove', 'hostedapidb-0')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('remove', 'hostedapidb-0')"
[ "$status" -ne 0 ]
}
@test "sql-backup: dolt_backup restore" {
run dolt sql -q "select dolt_backup('restore', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('restore', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
run dolt sql -q "select dolt_backup('restore', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('restore', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
}
@test "sql-backup: dolt_backup unrecognized" {
run dolt sql -q "select dolt_backup('unregonized', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('unrecognized', 'hostedapidb-0', 'file:///some_directory')"
[ "$status" -ne 0 ]
}
@test "sql-backup: dolt_backup sync wrong number of args" {
run dolt sql -q "select dolt_backup('sync')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('sync')"
[ "$status" -ne 0 ]
run dolt sql -q "select dolt_backup('sync', 'hostedapidb-0', 'too many')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('sync', 'hostedapidb-0', 'too many')"
[ "$status" -ne 0 ]
}
@test "sql-backup: dolt_backup no such backup" {
run dolt sql -q "select dolt_backup('sync', 'hostedapidb-0')"
[ "$status" -ne 0 ]
run dolt sql -q "CALL dolt_backup('sync', 'hostedapidb-0')"
[ "$status" -ne 0 ]
}
@test "sql-backup: dolt_backup sync to a backup" {
mkdir the_backup
dolt backup add hostedapidb-0 file://./the_backup
dolt backup -v
dolt sql -q "select dolt_backup('sync', 'hostedapidb-0')"
# Initial backup works.
dolt backup restore file://./the_backup the_restore
(cd the_restore && dolt status)
# Backup with nothing to push works.
dolt sql -q "select dolt_backup('sync', 'hostedapidb-0')"
rm -rf the_backup the_restore
mkdir the_backup
dolt sql -q "CALL dolt_backup('sync', 'hostedapidb-0')"
dolt backup restore file://./the_backup the_restore
(cd the_restore && dolt status)
dolt sql -q "CALL dolt_backup('sync', 'hostedapidb-0')"
}