diff --git a/go/cmd/dolt/commands/clone.go b/go/cmd/dolt/commands/clone.go index c9ecf082f8..8b44f56f9e 100644 --- a/go/cmd/dolt/commands/clone.go +++ b/go/cmd/dolt/commands/clone.go @@ -16,13 +16,8 @@ package commands import ( "context" - "errors" - "fmt" "os" "path" - "path/filepath" - "sort" - "sync" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/errhand" @@ -31,14 +26,11 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" - "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" "github.com/dolthub/dolt/go/libraries/events" "github.com/dolthub/dolt/go/libraries/utils/argparser" "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/libraries/utils/filesys" - "github.com/dolthub/dolt/go/libraries/utils/strhelp" - "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/types" ) @@ -109,58 +101,59 @@ func (cmd CloneCmd) Exec(ctx context.Context, commandStr string, args []string, remoteName := apr.GetValueOrDefault(remoteParam, "origin") branch := apr.GetValueOrDefault(branchParam, "") dir, urlStr, verr := parseArgs(apr) + if verr != nil { + return HandleVErrAndExitCode(verr, usage) + } + userDirExists, _ := dEnv.FS.Exists(dir) scheme, remoteUrl, err := env.GetAbsRemoteUrl(dEnv.FS, dEnv.Config, urlStr) if err != nil { - verr = errhand.BuildDError("error: '%s' is not valid.", urlStr).Build() + return HandleVErrAndExitCode(errhand.BuildDError("error: '%s' is not valid.", urlStr).Build(), usage) + } + var params map[string]string + params, verr = parseRemoteArgs(apr, scheme, remoteUrl) + if verr != nil { + return HandleVErrAndExitCode(verr, usage) } - if verr == nil { - var params map[string]string - params, verr = parseRemoteArgs(apr, scheme, remoteUrl) + var r env.Remote + var srcDB *doltdb.DoltDB + r, srcDB, verr = createRemote(ctx, remoteName, remoteUrl, params, dEnv) + if verr != nil { + return HandleVErrAndExitCode(verr, usage) + } - if verr == nil { - var r env.Remote - var srcDB *doltdb.DoltDB - r, srcDB, verr = createRemote(ctx, remoteName, remoteUrl, params, dEnv) + dEnv, err = actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) + if err != nil { + return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) + } - if verr == nil { - dEnv, verr = envForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version) + err = actions.CloneRemote(ctx, srcDB, remoteName, branch, dEnv) + if err != nil { + // If we're cloning into a directory that already exists do not erase it. Otherwise + // make best effort to delete the directory we created. + if userDirExists { + // Set the working dir to the parent of the .dolt folder so we can delete .dolt + _ = os.Chdir(dir) + _ = dEnv.FS.Delete(dbfactory.DoltDir, true) + } else { + _ = os.Chdir("../") + _ = dEnv.FS.Delete(dir, true) + } + return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) + } - if verr == nil { - verr = cloneRemote(ctx, srcDB, remoteName, branch, dEnv) - - if verr == nil { - evt := events.GetEventFromContext(ctx) - u, err := earl.Parse(remoteUrl) - - if err == nil { - if u.Scheme != "" { - evt.SetAttribute(eventsapi.AttributeID_REMOTE_URL_SCHEME, u.Scheme) - } - } - } - - if verr != nil { - // If we're cloning into a directory that already exists do not erase it. Otherwise - // make best effort to delete the directory we created. - if userDirExists { - // Set the working dir to the parent of the .dolt folder so we can delete .dolt - _ = os.Chdir(dir) - _ = dEnv.FS.Delete(dbfactory.DoltDir, true) - } else { - _ = os.Chdir("../") - _ = dEnv.FS.Delete(dir, true) - } - } - } - } + evt := events.GetEventFromContext(ctx) + u, err := earl.Parse(remoteUrl) + if err == nil { + if u.Scheme != "" { + evt.SetAttribute(eventsapi.AttributeID_REMOTE_URL_SCHEME, u.Scheme) } } - return HandleVErrAndExitCode(verr, usage) + return 0 } func parseArgs(apr *argparser.ArgParseResults) (string, string, errhand.VerboseError) { @@ -190,44 +183,6 @@ func parseArgs(apr *argparser.ArgParseResults) (string, string, errhand.VerboseE return dir, urlStr, nil } -func envForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, dir string, fs filesys.Filesys, version string) (*env.DoltEnv, errhand.VerboseError) { - exists, _ := fs.Exists(filepath.Join(dir, dbfactory.DoltDir)) - - if exists { - return nil, errhand.BuildDError("error: data repository already exists at " + dir).Build() - } - - err := fs.MkDirs(dir) - - if err != nil { - return nil, errhand.BuildDError("error: unable to create directories: " + dir).Build() - } - - err = os.Chdir(dir) - - if err != nil { - return nil, errhand.BuildDError("error: unable to access directory " + dir).Build() - } - - dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, version) - err = dEnv.InitRepoWithNoData(ctx, nbf) - - if err != nil { - return nil, errhand.BuildDError("error: unable to initialize repo without data").AddCause(err).Build() - } - - dEnv.RSLoadErr = nil - if !env.IsEmptyRemote(r) { - dEnv.RepoState, err = env.CloneRepoState(dEnv.FS, r) - - if err != nil { - return nil, errhand.BuildDError("error: unable to create repo state with remote " + r.Name).AddCause(err).Build() - } - } - - return dEnv, nil -} - func createRemote(ctx context.Context, remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv) (env.Remote, *doltdb.DoltDB, errhand.VerboseError) { cli.Printf("cloning %s\n", remoteUrl) @@ -248,202 +203,3 @@ func createRemote(ctx context.Context, remoteName, remoteUrl string, params map[ return r, ddb, nil } - -func cloneProg(eventCh <-chan datas.TableFileEvent) { - var ( - chunks int64 - chunksDownloading int64 - chunksDownloaded int64 - cliPos int - ) - - cliPos = cli.DeleteAndPrint(cliPos, "Retrieving remote information.") - for tblFEvt := range eventCh { - switch tblFEvt.EventType { - case datas.Listed: - for _, tf := range tblFEvt.TableFiles { - chunks += int64(tf.NumChunks()) - } - case datas.DownloadStart: - for _, tf := range tblFEvt.TableFiles { - chunksDownloading += int64(tf.NumChunks()) - } - case datas.DownloadSuccess: - for _, tf := range tblFEvt.TableFiles { - chunksDownloading -= int64(tf.NumChunks()) - chunksDownloaded += int64(tf.NumChunks()) - } - case datas.DownloadFailed: - // Ignore for now and output errors on the main thread - } - - str := fmt.Sprintf("%s of %s chunks complete. %s chunks being downloaded currently.", strhelp.CommaIfy(chunksDownloaded), strhelp.CommaIfy(chunks), strhelp.CommaIfy(chunksDownloading)) - cliPos = cli.DeleteAndPrint(cliPos, str) - } - - cli.Println() -} - -func cloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch string, dEnv *env.DoltEnv) errhand.VerboseError { - eventCh := make(chan datas.TableFileEvent, 128) - - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() - cloneProg(eventCh) - }() - - err := actions.Clone(ctx, srcDB, dEnv.DoltDB, eventCh) - close(eventCh) - - wg.Wait() - - if err != nil { - if err == datas.ErrNoData { - err = errors.New("remote at that url contains no Dolt data") - } - - return errhand.BuildDError("error: clone failed").AddCause(err).Build() - } - - branches, err := dEnv.DoltDB.GetBranches(ctx) - if err != nil { - return errhand.BuildDError("error: failed to list branches").AddCause(err).Build() - } - - if branch == "" { - branch = GetDefaultBranch(dEnv, branches) - } - - // If we couldn't find a branch but the repo cloned successfully, it's empty. Initialize it instead of pulling from - // the remote. - performPull := true - if branch == "" { - err = initEmptyClonedRepo(ctx, dEnv) - if err != nil { - return nil - } - - branch = env.GetDefaultInitBranch(dEnv.Config) - performPull = false - } - - cs, _ := doltdb.NewCommitSpec(branch) - cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil) - - if err != nil { - return errhand.BuildDError("error: could not get " + branch).AddCause(err).Build() - } - - rootVal, err := cm.GetRootValue() - if err != nil { - return errhand.BuildDError("error: could not get the root value of " + branch).AddCause(err).Build() - } - - // After actions.Clone, we have repository with a local branch for - // every branch in the remote. What we want is a remote branch ref for - // every branch in the remote. We iterate through local branches and - // create remote refs corresponding to each of them. We delete all of - // the local branches except for the one corresponding to |branch|. - for _, brnch := range branches { - cs, _ := doltdb.NewCommitSpec(brnch.GetPath()) - cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil) - if err != nil { - return errhand.BuildDError("error: could not resolve branch ref at " + brnch.String()).AddCause(err).Build() - } - - remoteRef := ref.NewRemoteRef(remoteName, brnch.GetPath()) - err = dEnv.DoltDB.SetHeadToCommit(ctx, remoteRef, cm) - if err != nil { - return errhand.BuildDError("error: could not create remote ref at " + remoteRef.String()).AddCause(err).Build() - } - - if brnch.GetPath() != branch { - err := dEnv.DoltDB.DeleteBranch(ctx, brnch) - if err != nil { - return errhand.BuildDError("error: could not delete local branch " + brnch.String() + " after clone.").AddCause(err).Build() - } - } - } - - if performPull { - err = actions.SaveDocsFromRoot(ctx, rootVal, dEnv) - if err != nil { - return errhand.BuildDError("error: failed to update docs on the filesystem").AddCause(err).Build() - } - } - - // TODO: make this interface take a DoltRef and marshal it automatically - err = dEnv.RepoStateWriter().SetCWBHeadRef(ctx, ref.MarshalableRef{Ref: ref.NewBranchRef(branch)}) - if err != nil { - return errhand.VerboseErrorFromError(err) - } - - wsRef, err := ref.WorkingSetRefForHead(ref.NewBranchRef(branch)) - if err != nil { - return errhand.VerboseErrorFromError(err) - } - - ws := doltdb.EmptyWorkingSet(wsRef) - err = dEnv.UpdateWorkingSet(ctx, ws.WithWorkingRoot(rootVal).WithStagedRoot(rootVal)) - if err != nil { - return errhand.VerboseErrorFromError(err) - } - - return nil -} - -// Inits an empty, newly cloned repo. This would be unnecessary if we properly initialized the storage for a repository -// when we created it on dolthub. If we do that, this code can be removed. -func initEmptyClonedRepo(ctx context.Context, dEnv *env.DoltEnv) error { - name := dEnv.Config.GetStringOrDefault(env.UserNameKey, "") - email := dEnv.Config.GetStringOrDefault(env.UserEmailKey, "") - initBranch := env.GetDefaultInitBranch(dEnv.Config) - - if *name == "" { - return errhand.BuildDError(fmt.Sprintf("error: could not determine user name. run dolt config --global --add %[1]s", env.UserNameKey)).Build() - } else if *email == "" { - return errhand.BuildDError("error: could not determine email. run dolt config --global --add %[1]s", env.UserEmailKey).Build() - } - - err := dEnv.InitDBWithTime(ctx, types.Format_Default, *name, *email, initBranch, doltdb.CommitNowFunc()) - if err != nil { - return errhand.BuildDError("error: could not initialize repository").AddCause(err).Build() - } - - return nil -} - -// GetDefaultBranch returns the default branch from among the branches given, returning -// the configs default config branch first, then init branch main, then the old init branch master, -// and finally the first lexicographical branch if none of the others are found -func GetDefaultBranch(dEnv *env.DoltEnv, branches []ref.DoltRef) string { - if len(branches) == 0 { - return env.DefaultInitBranch - } - - sort.Slice(branches, func(i, j int) bool { - return branches[i].GetPath() < branches[j].GetPath() - }) - - branchMap := make(map[string]ref.DoltRef) - for _, b := range branches { - branchMap[b.GetPath()] = b - } - - if _, ok := branchMap[env.DefaultInitBranch]; ok { - return env.DefaultInitBranch - } - if _, ok := branchMap["master"]; ok { - return "master" - } - - // todo: do we care about this during clone? - defaultOrMain := env.GetDefaultInitBranch(dEnv.Config) - if _, ok := branchMap[defaultOrMain]; ok { - return defaultOrMain - } - - return branches[0].GetPath() -} diff --git a/go/cmd/dolt/commands/commit.go b/go/cmd/dolt/commands/commit.go index bcb400d7dc..32985e0033 100644 --- a/go/cmd/dolt/commands/commit.go +++ b/go/cmd/dolt/commands/commit.go @@ -226,7 +226,7 @@ func getCommitMessageFromEditor(ctx context.Context, dEnv *env.DoltEnv) (string, editorStr := dEnv.Config.GetStringOrDefault(env.DoltEditor, backupEd) cli.ExecuteWithStdioRestored(func() { - commitMsg, _ := editor.OpenCommitEditor(*editorStr, initialMsg) + commitMsg, _ := editor.OpenCommitEditor(editorStr, initialMsg) finalMsg = parseCommitMessage(commitMsg) }) return finalMsg, nil diff --git a/go/cmd/dolt/commands/credcmds/check.go b/go/cmd/dolt/commands/credcmds/check.go index 4a6f9bdf0b..1dd36e094d 100644 --- a/go/cmd/dolt/commands/credcmds/check.go +++ b/go/cmd/dolt/commands/credcmds/check.go @@ -104,7 +104,7 @@ func loadEndpoint(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) string { host := dEnv.Config.GetStringOrDefault(env.RemotesApiHostKey, env.DefaultRemotesApiHost) port := dEnv.Config.GetStringOrDefault(env.RemotesApiHostPortKey, env.DefaultRemotesApiPort) - return fmt.Sprintf("%s:%s", *host, *port) + return fmt.Sprintf("%s:%s", host, port) } func loadCred(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) (creds.DoltCreds, errhand.VerboseError) { diff --git a/go/cmd/dolt/commands/credcmds/import.go b/go/cmd/dolt/commands/credcmds/import.go index c4143a4ee7..180b332ae7 100644 --- a/go/cmd/dolt/commands/credcmds/import.go +++ b/go/cmd/dolt/commands/credcmds/import.go @@ -162,7 +162,7 @@ func updateProfileWithCredentials(ctx context.Context, dEnv *env.DoltEnv, c cred host := dEnv.Config.GetStringOrDefault(env.RemotesApiHostKey, env.DefaultRemotesApiHost) port := dEnv.Config.GetStringOrDefault(env.RemotesApiHostPortKey, env.DefaultRemotesApiPort) - hostAndPort := fmt.Sprintf("%s:%s", *host, *port) + hostAndPort := fmt.Sprintf("%s:%s", host, port) endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{ Endpoint: hostAndPort, Creds: c, diff --git a/go/cmd/dolt/commands/login.go b/go/cmd/dolt/commands/login.go index d72fb58ace..d13f213597 100644 --- a/go/cmd/dolt/commands/login.go +++ b/go/cmd/dolt/commands/login.go @@ -192,7 +192,7 @@ func loginWithCreds(ctx context.Context, dEnv *env.DoltEnv, dc creds.DoltCreds, func openBrowserForCredsAdd(dEnv *env.DoltEnv, dc creds.DoltCreds) { loginUrl := dEnv.Config.GetStringOrDefault(env.AddCredsUrlKey, env.DefaultLoginUrl) - url := fmt.Sprintf("%s#%s", *loginUrl, dc.PubKeyBase32Str()) + url := fmt.Sprintf("%s#%s", loginUrl, dc.PubKeyBase32Str()) cli.Printf("Opening a browser to:\n\t%s\nPlease associate your key with your account.\n", url) open.Start(url) } @@ -201,7 +201,7 @@ func getCredentialsClient(dEnv *env.DoltEnv, dc creds.DoltCreds) (remotesapi.Cre host := dEnv.Config.GetStringOrDefault(env.RemotesApiHostKey, env.DefaultRemotesApiHost) port := dEnv.Config.GetStringOrDefault(env.RemotesApiHostPortKey, env.DefaultRemotesApiPort) endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{ - Endpoint: fmt.Sprintf("%s:%s", *host, *port), + Endpoint: fmt.Sprintf("%s:%s", host, port), Creds: dc, }) if err != nil { diff --git a/go/cmd/dolt/commands/read_tables.go b/go/cmd/dolt/commands/read_tables.go index 34ec55ff9c..cc5d50faa1 100644 --- a/go/cmd/dolt/commands/read_tables.go +++ b/go/cmd/dolt/commands/read_tables.go @@ -18,15 +18,15 @@ import ( "context" "path" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/utils/earl" - "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/errhand" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/utils/argparser" + "github.com/dolthub/dolt/go/libraries/utils/earl" "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/types" ) var readTablesDocs = cli.CommandDocumentationContent{ @@ -125,7 +125,7 @@ func (cmd ReadTablesCmd) Exec(ctx context.Context, commandStr string, args []str BuildVerrAndExit("Failed to get remote branches", err) } - dEnv, verr = initializeShallowCloneRepo(ctx, dEnv, srcDB.Format(), dir, GetDefaultBranch(dEnv, branches)) + dEnv, verr = initializeShallowCloneRepo(ctx, dEnv, srcDB.Format(), dir, env.GetDefaultBranch(dEnv, branches)) if verr != nil { return HandleVErrAndExitCode(verr, usage) } @@ -223,20 +223,19 @@ func getRemoteDBAtCommit(ctx context.Context, remoteUrl string, remoteUrlParams } func initializeShallowCloneRepo(ctx context.Context, dEnv *env.DoltEnv, nbf *types.NomsBinFormat, dir, branchName string) (*env.DoltEnv, errhand.VerboseError) { - var verr errhand.VerboseError - dEnv, verr = envForClone(ctx, nbf, env.NoRemote, dir, dEnv.FS, dEnv.Version) + var err error + dEnv, err = actions.EnvForClone(ctx, nbf, env.NoRemote, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir) - if verr != nil { - return nil, verr + if err != nil { + return nil, errhand.VerboseErrorFromError(err) } - err := initEmptyClonedRepo(ctx, dEnv) + err = actions.InitEmptyClonedRepo(ctx, dEnv) if err != nil { return nil, errhand.BuildDError("Unable to initialize repo.").AddCause(err).Build() } err = dEnv.InitializeRepoState(ctx, branchName) - if err != nil { return nil, errhand.BuildDError("Unable to initialize repo.").AddCause(err).Build() } diff --git a/go/cmd/dolt/commands/send_metrics.go b/go/cmd/dolt/commands/send_metrics.go index 765d99133d..2b315bfc2e 100644 --- a/go/cmd/dolt/commands/send_metrics.go +++ b/go/cmd/dolt/commands/send_metrics.go @@ -79,7 +79,7 @@ func (cmd SendMetricsCmd) Exec(ctx context.Context, commandStr string, args []st metricsDisabled := dEnv.Config.GetStringOrDefault(env.MetricsDisabled, "false") - disabled, err := strconv.ParseBool(*metricsDisabled) + disabled, err := strconv.ParseBool(metricsDisabled) if err != nil { // log.Print(err) return 1 @@ -134,20 +134,20 @@ func getGRPCEmitter(dEnv *env.DoltEnv) *events.GrpcEmitter { portStr := dEnv.Config.GetStringOrDefault(env.MetricsPort, env.DefaultMetricsPort) insecureStr := dEnv.Config.GetStringOrDefault(env.MetricsInsecure, "false") - port, err := strconv.ParseUint(*portStr, 10, 16) + port, err := strconv.ParseUint(portStr, 10, 16) if err != nil { - log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid port.", env.MetricsPort, *portStr)) + log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid port.", env.MetricsPort, portStr)) return nil } - insecure, err := strconv.ParseBool(*insecureStr) + insecure, err := strconv.ParseBool(insecureStr) if err != nil { - log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid true/false value", env.MetricsInsecure, *insecureStr)) + log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid true/false value", env.MetricsInsecure, insecureStr)) } - hostAndPort := fmt.Sprintf("%s:%d", *host, port) + hostAndPort := fmt.Sprintf("%s:%d", host, port) endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{ Endpoint: hostAndPort, Insecure: insecure, diff --git a/go/cmd/dolt/commands/sql.go b/go/cmd/dolt/commands/sql.go index 299e0f2188..c51913ccb8 100644 --- a/go/cmd/dolt/commands/sql.go +++ b/go/cmd/dolt/commands/sql.go @@ -520,8 +520,8 @@ func CollectDBs(ctx context.Context, mrEnv env.MultiRepoEnv) ([]dsqle.SqlDatabas var db dsqle.SqlDatabase err := mrEnv.Iter(func(name string, dEnv *env.DoltEnv) (stop bool, err error) { db = newDatabase(name, dEnv) - if remoteName := os.Getenv(dsqle.DoltReadReplicaKey); remoteName != "" { - db, err = dsqle.NewReadReplicaDatabase(ctx, db.(dsqle.Database), remoteName, dEnv.RepoStateReader(), dEnv.TempTableFilesDir(), dEnv.NewWorkingSetMeta("read replica update")) + if remoteName := dEnv.Config.GetStringOrDefault(dsqle.DoltReadReplicaKey, ""); remoteName != "" { + db, err = dsqle.NewReadReplicaDatabase(ctx, db.(dsqle.Database), remoteName, dEnv.RepoStateReader(), dEnv.TempTableFilesDir(), doltdb.TodoWorkingSetMeta()) if err != nil { return true, err } @@ -1474,9 +1474,7 @@ func newSqlEngine( } // TODO: not having user and email for this command should probably be an error or warning, it disables certain functionality - username := *dEnv.Config.GetStringOrDefault(env.UserNameKey, "") - email := *dEnv.Config.GetStringOrDefault(env.UserEmailKey, "") - sess, err := dsess.NewSession(sql.NewEmptyContext(), sql.NewBaseSession(), pro, username, email, dbStates...) + sess, err := dsess.NewSession(sql.NewEmptyContext(), sql.NewBaseSession(), pro, dEnv.Config, dbStates...) // TODO: this should just be the session default like it is with MySQL err = sess.SetSessionVariable(sql.NewContext(ctx), sql.AutoCommitSessionVar, true) diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index aceffae712..6f6443a7b1 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -40,22 +40,6 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/tracing" ) -const DoltDefaultBranchKey = "dolt_default_branch" - -func init() { - sql.SystemVariables.AddSystemVariables([]sql.SystemVariable{ - { - Name: DoltDefaultBranchKey, - Scope: sql.SystemVariableScope_Global, - Dynamic: true, - SetVarHintApplies: false, - Type: sql.NewSystemStringType(DoltDefaultBranchKey), - Default: "", - }, - }) - -} - // Serve starts a MySQL-compatible server. Returns any errors that were encountered. func Serve(ctx context.Context, version string, serverConfig ServerConfig, serverController *ServerController, dEnv *env.DoltEnv) (startError error, closeError error) { if serverConfig == nil { @@ -102,8 +86,6 @@ func Serve(ctx context.Context, version string, serverConfig ServerConfig, serve userAuth := auth.NewNativeSingle(serverConfig.User(), serverConfig.Password(), permissions) - var username string - var email string var mrEnv env.MultiRepoEnv dbNamesAndPaths := serverConfig.DatabaseNamesAndPaths() if len(dbNamesAndPaths) == 0 { @@ -112,9 +94,6 @@ func Serve(ctx context.Context, version string, serverConfig ServerConfig, serve if err != nil { return err, nil } - - username = *dEnv.Config.GetStringOrDefault(env.UserNameKey, "") - email = *dEnv.Config.GetStringOrDefault(env.UserEmailKey, "") } else { var err error mrEnv, err = env.LoadMultiEnv(ctx, env.GetCurrentUserHomeDir, dEnv.FS, version, dbNamesAndPaths...) @@ -164,7 +143,7 @@ func Serve(ctx context.Context, version string, serverConfig ServerConfig, serve // to the value of mysql that we support. }, sqlEngine, - newSessionBuilder(sqlEngine, username, email, pro, mrEnv, serverConfig.AutoCommit()), + newSessionBuilder(sqlEngine, dEnv.Config, pro, mrEnv, serverConfig.AutoCommit()), ) if startError != nil { @@ -191,7 +170,7 @@ func portInUse(hostPort string) bool { return false } -func newSessionBuilder(sqlEngine *sqle.Engine, username string, email string, pro dsqle.DoltDatabaseProvider, mrEnv env.MultiRepoEnv, autocommit bool) server.SessionBuilder { +func newSessionBuilder(sqlEngine *sqle.Engine, dConf *env.DoltCliConfig, pro dsqle.DoltDatabaseProvider, mrEnv env.MultiRepoEnv, autocommit bool) server.SessionBuilder { return func(ctx context.Context, conn *mysql.Conn, host string) (sql.Session, *sql.IndexRegistry, *sql.ViewRegistry, error) { tmpSqlCtx := sql.NewEmptyContext() @@ -203,7 +182,7 @@ func newSessionBuilder(sqlEngine *sqle.Engine, username string, email string, pr return nil, nil, nil, err } - doltSess, err := dsess.NewSession(tmpSqlCtx, mysqlSess, pro, username, email, dbStates...) + doltSess, err := dsess.NewSession(tmpSqlCtx, mysqlSess, pro, dConf, dbStates...) if err != nil { return nil, nil, nil, err } @@ -249,7 +228,7 @@ func getDbStates(ctx context.Context, dbs []dsqle.SqlDatabase) ([]dsess.InitialD var init dsess.InitialDbState var err error - _, val, ok := sql.SystemVariables.GetGlobal(DoltDefaultBranchKey) + _, val, ok := sql.SystemVariables.GetGlobal(dsess.DoltDefaultBranchKey) if ok && val != "" { init, err = GetInitialDBStateWithDefaultBranch(ctx, db, val.(string)) } else { diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index f88ef0f50e..55a5c8336e 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -326,6 +326,13 @@ func TestServerSetDefaultBranch(t *testing.T) { }, } + defer func(sess *dbr.Session) { + var res []struct { + int + } + sess.SelectBySql("set GLOBAL dolt_default_branch = ''").LoadContext(context.Background(), &res) + }(sess) + for _, test := range tests { t.Run(test.query.Query, func(t *testing.T) { var branch []testBranch @@ -360,11 +367,14 @@ func TestServerSetDefaultBranch(t *testing.T) { assert.ElementsMatch(t, branch, test.expectedRes) }) } + + var res []struct { + int + } + sess.SelectBySql("set GLOBAL dolt_default_branch = ''").LoadContext(context.Background(), &res) } func TestReadReplica(t *testing.T) { - t.Skip("this fails on a query from the previous test suite if run as a file") - var err error cwd, err := os.Getwd() if err != nil { @@ -372,11 +382,22 @@ func TestReadReplica(t *testing.T) { } defer os.Chdir(cwd) - multiSetup := testcommands.CreateMultiEnvWithRemote() + multiSetup := testcommands.NewMultiRepoTestSetup(t) defer os.RemoveAll(multiSetup.Root) - readOnlyDbName := multiSetup.DbNames[0] - sourceDbName := multiSetup.DbNames[0] + multiSetup.NewDB("read_replica") + multiSetup.NewRemote("remote1") + multiSetup.PushToRemote("read_replica", "remote1") + multiSetup.CloneDB("remote1", "source_db") + + readReplicaDbName := multiSetup.DbNames[0] + sourceDbName := multiSetup.DbNames[1] + + localCfg, ok := multiSetup.MrEnv[readReplicaDbName].Config.GetConfig(env.LocalConfig) + if !ok { + t.Fatal("local config does not exist") + } + localCfg.SetStrings(map[string]string{dsqle.DoltReadReplicaKey: "remote1"}) // start server as read replica sc := CreateServerController() @@ -384,10 +405,10 @@ func TestReadReplica(t *testing.T) { func() { err = os.Setenv(dsqle.DoltReadReplicaKey, "remote1") - os.Chdir(multiSetup.DbPaths[readOnlyDbName]) + os.Chdir(multiSetup.DbPaths[readReplicaDbName]) go func() { - _, _ = Serve(context.Background(), "", serverConfig, sc, multiSetup.MrEnv[readOnlyDbName]) + _, _ = Serve(context.Background(), "", serverConfig, sc, multiSetup.MrEnv[readReplicaDbName]) }() err = sc.WaitForStart() require.NoError(t, err) @@ -395,20 +416,19 @@ func TestReadReplica(t *testing.T) { defer sc.StopServer() defer os.Unsetenv(dsqle.DoltReadReplicaKey) - conn, err := dbr.Open("mysql", ConnectionString(serverConfig)+readOnlyDbName, nil) + conn, err := dbr.Open("mysql", ConnectionString(serverConfig)+readReplicaDbName, nil) defer conn.Close() require.NoError(t, err) sess := conn.NewSession(nil) - // TODO: why doesn't this throw a "no common ancestor" error? t.Run("push common new commit", func(t *testing.T) { var res []string replicatedTable := "new_table" - multiSetup.CreateTable(t, sourceDbName, replicatedTable) - multiSetup.AddAll(t, sourceDbName) + multiSetup.CreateTable(sourceDbName, replicatedTable) + multiSetup.StageAll(sourceDbName) _ = multiSetup.CommitWithWorkingSet(sourceDbName) - multiSetup.PushToRemote(t, sourceDbName) + multiSetup.PushToRemote(sourceDbName, "remote1") q := sess.SelectBySql("show tables") _, err := q.LoadContext(context.Background(), &res) diff --git a/go/cmd/dolt/dolt.go b/go/cmd/dolt/dolt.go index d2241ef98a..95fc2e70ca 100644 --- a/go/cmd/dolt/dolt.go +++ b/go/cmd/dolt/dolt.go @@ -266,7 +266,7 @@ func runMain() int { metricsDisabled := dEnv.Config.GetStringOrDefault(env.MetricsDisabled, "false") - disabled, err := strconv.ParseBool(*metricsDisabled) + disabled, err := strconv.ParseBool(metricsDisabled) if err != nil { // log.Print(err) return diff --git a/go/libraries/doltcore/doltdb/commit_hooks.go b/go/libraries/doltcore/doltdb/commit_hooks.go index 94e0e4597d..d0fcbd00a6 100644 --- a/go/libraries/doltcore/doltdb/commit_hooks.go +++ b/go/libraries/doltcore/doltdb/commit_hooks.go @@ -24,7 +24,7 @@ import ( "github.com/dolthub/dolt/go/store/datas" ) -const BackupToRemoteKey = "DOLT_BACKUP_TO_REMOTE" +const BackupToRemoteKey = "dolt_backup_to_remote" type ReplicateHook struct { destDB datas.Database diff --git a/go/libraries/doltcore/doltdb/commit_hooks_test.go b/go/libraries/doltcore/doltdb/commit_hooks_test.go index f9b11ff789..a18da714e8 100644 --- a/go/libraries/doltcore/doltdb/commit_hooks_test.go +++ b/go/libraries/doltcore/doltdb/commit_hooks_test.go @@ -31,6 +31,8 @@ import ( "github.com/stretchr/testify/assert" ) +const defaultBranch = "main" + func TestReplicateHook(t *testing.T) { ctx := context.Background() @@ -123,12 +125,12 @@ func TestReplicateHook(t *testing.T) { ddb.SetCommitHooks(ctx, []datas.CommitHook{hook}) t.Run("replicate to backup remote", func(t *testing.T) { - srcCommit, err := ddb.Commit(context.Background(), valHash, ref.NewBranchRef("master"), meta) - ds, err := ddb.db.GetDataset(ctx, "refs/heads/master") + srcCommit, err := ddb.Commit(context.Background(), valHash, ref.NewBranchRef(defaultBranch), meta) + ds, err := ddb.db.GetDataset(ctx, "refs/heads/main") err = hook.Execute(ctx, ds, ddb.db) assert.NoError(t, err) - cs, _ = NewCommitSpec("master") + cs, _ = NewCommitSpec(defaultBranch) destCommit, err := destDB.Resolve(context.Background(), cs, nil) srcHash, _ := srcCommit.HashOf() diff --git a/go/libraries/doltcore/dtestutils/testcommands/multienv.go b/go/libraries/doltcore/dtestutils/testcommands/multienv.go index b6bd5f443a..fc18afbc85 100644 --- a/go/libraries/doltcore/dtestutils/testcommands/multienv.go +++ b/go/libraries/doltcore/dtestutils/testcommands/multienv.go @@ -18,13 +18,11 @@ import ( "context" "fmt" "io/ioutil" - "log" "os" "path/filepath" "testing" "github.com/dolthub/dolt/go/cmd/dolt/cli" - "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils" "github.com/dolthub/dolt/go/libraries/doltcore/env" @@ -48,6 +46,8 @@ type MultiRepoTestSetup struct { Root string DbPaths map[string]string Home string + Remotes map[string]env.Remote + T *testing.T } const ( @@ -56,94 +56,146 @@ const ( defaultBranch = "main" ) -func CreateMultiEnvWithRemote() *MultiRepoTestSetup { - ctx := context.Background() - cwd, err := os.Getwd() - if err != nil { - log.Fatal(err) - } - defer os.Chdir(cwd) - +// TODO this is not a proper builder, dbs need to be added before remotes +func NewMultiRepoTestSetup(t *testing.T) *MultiRepoTestSetup { dir, err := ioutil.TempDir("", "") if err != nil { - log.Fatal(err) + t.Fatal(err) } homeDir, err := ioutil.TempDir(dir, homePrefix) if err != nil { - log.Fatal(err) - } - homeProv := func() (string, error) { - return homeDir, nil - } - - remote, err := ioutil.TempDir(dir, remotePrefix) - if err != nil { - log.Fatal(err) - } - - repoCnt := 2 - mrEnv := env.MultiRepoEnv{} - dbs := make(map[string]*doltdb.DoltDB, repoCnt) - dbNames := make([]string, repoCnt) - dbPaths := make(map[string]string, repoCnt) - for i := 0; i < repoCnt; i++ { - repo, err := ioutil.TempDir(dir, repoPrefix) - if err != nil { - log.Fatal(err) - } - - err = os.Chdir(repo) - if err != nil { - log.Fatal(err) - } - - dbName := filepath.Base(repo) - dbPaths[dbName] = repo - repoPath := fmt.Sprintf("file://%s", repo) - - // TODO sometimes tempfiles scrubber is racy with tempfolder deleter - dEnv := env.Load(context.Background(), homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test") - if err != nil { - panic("Failed to initialize environment:" + err.Error()) - } - cfg, _ := dEnv.Config.GetConfig(env.GlobalConfig) - cfg.SetStrings(map[string]string{ - env.UserNameKey: name, - env.UserEmailKey: email, - }) - err = dEnv.InitRepo(context.Background(), types.Format_Default, name, email, defaultBranch) - if err != nil { - panic("Failed to initialize environment:" + err.Error()) - } - - ddb, err := doltdb.LoadDoltDB(ctx, types.Format_Default, filepath.Join(repoPath, dbfactory.DoltDir), filesys.LocalFS) - if err != nil { - panic("Failed to initialize environment:" + err.Error()) - } - - remotePath := fmt.Sprintf("file://%s", remote) - rem := env.NewRemote("remote1", remotePath, nil, dEnv) - dEnv.RepoState.AddRemote(rem) - dEnv.RepoState.Save(filesys.LocalFS) - dEnv = env.Load(context.Background(), homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test") - - mrEnv.AddEnv(dbName, dEnv) - dbs[dbName] = ddb - dbNames[i] = dbName + t.Fatal(err) } return &MultiRepoTestSetup{ - MrEnv: mrEnv, - Remote: fmt.Sprintf("file://%s", remote), - DoltDBs: dbs, - DbNames: dbNames, + MrEnv: env.MultiRepoEnv{}, + Remotes: make(map[string]env.Remote), + DoltDBs: make(map[string]*doltdb.DoltDB, 0), + DbNames: make([]string, 0), Root: dir, Home: homeDir, - DbPaths: dbPaths, + DbPaths: make(map[string]string, 0), + T: t, } } +func (mr *MultiRepoTestSetup) homeProv() (string, error) { + return mr.Home, nil +} + +func (mr *MultiRepoTestSetup) Cleanup(dbName string) { + os.RemoveAll(mr.Root) +} + +func (mr *MultiRepoTestSetup) NewDB(dbName string) { + ctx := context.Background() + + repo := filepath.Join(mr.Root, dbName) + os.Mkdir(repo, os.ModePerm) + + err := os.Chdir(repo) + if err != nil { + mr.T.Fatal(err) + } + + // TODO sometimes tempfiles scrubber is racy with tempfolder deleter + dEnv := env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test") + if err != nil { + mr.T.Fatal("Failed to initialize environment:" + err.Error()) + } + cfg, _ := dEnv.Config.GetConfig(env.GlobalConfig) + cfg.SetStrings(map[string]string{ + env.UserNameKey: name, + env.UserEmailKey: email, + }) + err = dEnv.InitRepo(context.Background(), types.Format_Default, name, email, defaultBranch) + if err != nil { + mr.T.Fatal("Failed to initialize environment:" + err.Error()) + } + + ddb, err := doltdb.LoadDoltDB(ctx, types.Format_Default, doltdb.LocalDirDoltDB, filesys.LocalFS) + if err != nil { + mr.T.Fatal("Failed to initialize environment:" + err.Error()) + } + + dEnv = env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test") + + mr.MrEnv.AddEnv(dbName, dEnv) + mr.DoltDBs[dbName] = ddb + mr.DbNames = append(mr.DbNames, dbName) + mr.DbPaths[dbName] = repo +} + +func (mr *MultiRepoTestSetup) NewRemote(remoteName string) { + remote := filepath.Join(mr.Root, remoteName) + os.Mkdir(remote, os.ModePerm) + remotePath := fmt.Sprintf("file:///%s", remote) + + dEnv := mr.MrEnv[mr.DbNames[0]] + rem := env.NewRemote(remoteName, remotePath, nil, dEnv) + + for _, dEnv := range mr.MrEnv { + dEnv.RepoState.AddRemote(rem) + dEnv.RepoState.Save(filesys.LocalFS) + } + + mr.Remotes[remoteName] = rem +} + +func (mr *MultiRepoTestSetup) CloneDB(fromRemote, dbName string) { + ctx := context.Background() + cloneDir := filepath.Join(mr.Root, dbName) + + r := mr.GetRemote(fromRemote) + srcDB, err := r.GetRemoteDB(ctx, types.Format_Default) + + dEnv := env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test") + dEnv, err = actions.EnvForClone(ctx, srcDB.Format(), r, cloneDir, dEnv.FS, dEnv.Version, mr.homeProv) + if err != nil { + mr.T.Fatal(err) + } + + err = actions.CloneRemote(ctx, srcDB, r.Name, "", dEnv) + if err != nil { + mr.T.Fatal(err) + } + + wd, err := os.Getwd() + if err != nil { + mr.T.Fatal(err) + } + os.Chdir(cloneDir) + defer os.Chdir(wd) + ddb, err := doltdb.LoadDoltDB(ctx, types.Format_Default, doltdb.LocalDirDoltDB, filesys.LocalFS) + if err != nil { + mr.T.Fatal("Failed to initialize environment:" + err.Error()) + } + + dEnv = env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test") + + mr.MrEnv.AddEnv(dbName, dEnv) + mr.DoltDBs[dbName] = ddb + mr.DbNames = append(mr.DbNames, dbName) + mr.DbPaths[dbName] = cloneDir +} + +func (mr *MultiRepoTestSetup) GetRemote(remoteName string) env.Remote { + rem, ok := mr.Remotes[remoteName] + if !ok { + mr.T.Fatal("remote not found") + } + return rem +} + +func (mr *MultiRepoTestSetup) GetDB(dbName string) *doltdb.DoltDB { + db, ok := mr.DoltDBs[dbName] + if !ok { + mr.T.Fatal("db not found") + } + return db +} + func (mr *MultiRepoTestSetup) CommitWithWorkingSet(dbName string) *doltdb.Commit { ctx := context.Background() dEnv, ok := mr.MrEnv[dbName] @@ -187,7 +239,7 @@ func (mr *MultiRepoTestSetup) CommitWithWorkingSet(dbName string) *doltdb.Commit pendingCommit, ws.WithStagedRoot(pendingCommit.Roots.Staged).WithWorkingRoot(pendingCommit.Roots.Working).ClearMerge(), prevHash, - dEnv.NewWorkingSetMeta(fmt.Sprintf("Updated by test")), + doltdb.TodoWorkingSetMeta(), ) if err != nil { panic("couldn't commit: " + err.Error()) @@ -195,10 +247,10 @@ func (mr *MultiRepoTestSetup) CommitWithWorkingSet(dbName string) *doltdb.Commit return commit } -func (mr *MultiRepoTestSetup) CreateTable(t *testing.T, dbName, tblName string) { +func (mr *MultiRepoTestSetup) CreateTable(dbName, tblName string) { dEnv, ok := mr.MrEnv[dbName] if !ok { - t.Fatalf("Failed to find db: %s", dbName) + mr.T.Fatalf("Failed to find db: %s", dbName) } imt, sch := dtestutils.CreateTestDataTable(true) @@ -206,49 +258,49 @@ func (mr *MultiRepoTestSetup) CreateTable(t *testing.T, dbName, tblName string) for i := 0; i < imt.NumRows(); i++ { r, err := imt.GetRow(i) if err != nil { - t.Fatalf("Failed to create table: %s", err.Error()) + mr.T.Fatalf("Failed to create table: %s", err.Error()) } rows[i] = r } - dtestutils.CreateTestTable(t, dEnv, tblName, sch, rows...) + dtestutils.CreateTestTable(mr.T, dEnv, tblName, sch, rows...) } -func (mr *MultiRepoTestSetup) AddAll(t *testing.T, dbName string) { +func (mr *MultiRepoTestSetup) StageAll(dbName string) { dEnv, ok := mr.MrEnv[dbName] if !ok { - t.Fatalf("Failed to find db: %s", dbName) + mr.T.Fatalf("Failed to find db: %s", dbName) } ctx := context.Background() roots, err := dEnv.Roots(ctx) if !ok { - t.Fatalf("Failed to get roots: %s", dbName) + mr.T.Fatalf("Failed to get roots: %s", dbName) } roots, err = actions.StageAllTables(ctx, roots, dEnv.Docs) err = dEnv.UpdateRoots(ctx, roots) if err != nil { - t.Fatalf("Failed to update roots: %s", dbName) + mr.T.Fatalf("Failed to update roots: %s", dbName) } } -func (mr *MultiRepoTestSetup) PushToRemote(t *testing.T, dbName string) { +func (mr *MultiRepoTestSetup) PushToRemote(dbName, remoteName string) { ctx := context.Background() dEnv, ok := mr.MrEnv[dbName] if !ok { - t.Fatalf("Failed to find db: %s", dbName) + mr.T.Fatalf("Failed to find db: %s", dbName) } ap := cli.CreatePushArgParser() - apr, err := ap.Parse([]string{"remote1", defaultBranch}) + apr, err := ap.Parse([]string{remoteName, defaultBranch}) if err != nil { - t.Fatalf("Failed to push remote: %s", err.Error()) + mr.T.Fatalf("Failed to push remote: %s", err.Error()) } opts, err := env.NewParseOpts(ctx, apr, dEnv.RepoStateReader(), dEnv.DoltDB, false, false) if err != nil { - t.Fatalf("Failed to push remote: %s", err.Error()) + mr.T.Fatalf("Failed to push remote: %s", err.Error()) } err = actions.DoPush(ctx, dEnv.RepoStateReader(), dEnv.RepoStateWriter(), dEnv.DoltDB, dEnv.TempTableFilesDir(), opts, actions.NoopRunProgFuncs, actions.NoopStopProgFuncs) if err != nil { - t.Fatalf("Failed to push remote: %s", err.Error()) + mr.T.Fatalf("Failed to push remote: %s", err.Error()) } } diff --git a/go/libraries/doltcore/env/actions/clone.go b/go/libraries/doltcore/env/actions/clone.go new file mode 100644 index 0000000000..d6480e9f43 --- /dev/null +++ b/go/libraries/doltcore/env/actions/clone.go @@ -0,0 +1,257 @@ +// Copyright 2021 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package actions + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/dolthub/dolt/go/cmd/dolt/cli" + "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/doltcore/ref" + "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/libraries/utils/strhelp" + "github.com/dolthub/dolt/go/store/datas" + "github.com/dolthub/dolt/go/store/types" +) + +var ErrRepositoryExists = errors.New("data repository already exists") +var ErrFailedToInitRepo = errors.New("") +var ErrFailedToCreateDirectory = errors.New("unable to create directories") +var ErrFailedToAccessDir = errors.New("unable to access directories") +var ErrFailedToCreateRepoStateWithRemote = errors.New("unable to create repo state with remote") +var ErrNoDataAtRemote = errors.New("remote at that url contains no Dolt data") +var ErrFailedToListBranches = errors.New("failed to list branches") +var ErrFailedToGetBranch = errors.New("could not get branch") +var ErrFailedToGetRootValue = errors.New("could not find root value") +var ErrFailedToResolveBranchRef = errors.New("could not resole branch ref") +var ErrFailedToCreateRemoteRef = errors.New("could not create remote ref") +var ErrFailedToDeleteBranch = errors.New("could not delete local branch after clone") +var ErrFailedToUpdateDocs = errors.New("failed to update docs on the filesystem") +var ErrUserNotFound = errors.New("could not determine user name. run dolt config --global --add user.name") +var ErrEmailNotFound = errors.New("could not determine email. run dolt config --global --add user.email") +var ErrCloneFailed = errors.New("clone failed") + +func EnvForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, dir string, fs filesys.Filesys, version string, homeProvider env.HomeDirProvider) (*env.DoltEnv, error) { + exists, _ := fs.Exists(filepath.Join(dir, dbfactory.DoltDir)) + + if exists { + return nil, fmt.Errorf("%w: %s", ErrRepositoryExists, dir) + } + + err := fs.MkDirs(dir) + + if err != nil { + return nil, fmt.Errorf("%w: %s; %s", ErrFailedToCreateDirectory, dir, err.Error()) + } + + err = os.Chdir(dir) + + if err != nil { + return nil, fmt.Errorf("%w: %s; %s", ErrFailedToAccessDir, dir, err.Error()) + } + + dEnv := env.Load(ctx, homeProvider, fs, doltdb.LocalDirDoltDB, version) + err = dEnv.InitRepoWithNoData(ctx, nbf) + + if err != nil { + return nil, fmt.Errorf("%w; %s", ErrFailedToInitRepo, err.Error()) + } + + dEnv.RSLoadErr = nil + if !env.IsEmptyRemote(r) { + dEnv.RepoState, err = env.CloneRepoState(dEnv.FS, r) + + if err != nil { + return nil, fmt.Errorf("%w: %s; %s", ErrFailedToCreateRepoStateWithRemote, r.Name, err.Error()) + } + } + + return dEnv, nil +} + +func cloneProg(eventCh <-chan datas.TableFileEvent) { + var ( + chunks int64 + chunksDownloading int64 + chunksDownloaded int64 + cliPos int + ) + + cliPos = cli.DeleteAndPrint(cliPos, "Retrieving remote information.") + for tblFEvt := range eventCh { + switch tblFEvt.EventType { + case datas.Listed: + for _, tf := range tblFEvt.TableFiles { + chunks += int64(tf.NumChunks()) + } + case datas.DownloadStart: + for _, tf := range tblFEvt.TableFiles { + chunksDownloading += int64(tf.NumChunks()) + } + case datas.DownloadSuccess: + for _, tf := range tblFEvt.TableFiles { + chunksDownloading -= int64(tf.NumChunks()) + chunksDownloaded += int64(tf.NumChunks()) + } + case datas.DownloadFailed: + // Ignore for now and output errors on the main thread + } + + str := fmt.Sprintf("%s of %s chunks complete. %s chunks being downloaded currently.", strhelp.CommaIfy(chunksDownloaded), strhelp.CommaIfy(chunks), strhelp.CommaIfy(chunksDownloading)) + cliPos = cli.DeleteAndPrint(cliPos, str) + } + + cli.Println() +} + +func CloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch string, dEnv *env.DoltEnv) error { + eventCh := make(chan datas.TableFileEvent, 128) + + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + cloneProg(eventCh) + }() + + err := Clone(ctx, srcDB, dEnv.DoltDB, eventCh) + close(eventCh) + + wg.Wait() + + if err != nil { + if err == datas.ErrNoData { + err = ErrNoDataAtRemote + } + return fmt.Errorf("%w; %s", ErrCloneFailed, err.Error()) + } + + branches, err := dEnv.DoltDB.GetBranches(ctx) + if err != nil { + return fmt.Errorf("%w; %s", ErrFailedToListBranches, err.Error()) + } + + if branch == "" { + branch = env.GetDefaultBranch(dEnv, branches) + } + + // If we couldn't find a branch but the repo cloned successfully, it's empty. Initialize it instead of pulling from + // the remote. + performPull := true + if branch == "" { + err = InitEmptyClonedRepo(ctx, dEnv) + if err != nil { + return nil + } + + branch = env.GetDefaultInitBranch(dEnv.Config) + performPull = false + } + + cs, _ := doltdb.NewCommitSpec(branch) + cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil) + + if err != nil { + return fmt.Errorf("%w: %s; %s", ErrFailedToGetBranch, branch, err.Error()) + + } + + rootVal, err := cm.GetRootValue() + if err != nil { + return fmt.Errorf("%w: %s; %s", ErrFailedToGetRootValue, branch, err.Error()) + } + + // After actions.Clone, we have repository with a local branch for + // every branch in the remote. What we want is a remote branch ref for + // every branch in the remote. We iterate through local branches and + // create remote refs corresponding to each of them. We delete all of + // the local branches except for the one corresponding to |branch|. + for _, brnch := range branches { + cs, _ := doltdb.NewCommitSpec(brnch.GetPath()) + cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil) + if err != nil { + return fmt.Errorf("%w: %s; %s", ErrFailedToResolveBranchRef, brnch.String(), err.Error()) + + } + + remoteRef := ref.NewRemoteRef(remoteName, brnch.GetPath()) + err = dEnv.DoltDB.SetHeadToCommit(ctx, remoteRef, cm) + if err != nil { + return fmt.Errorf("%w: %s; %s", ErrFailedToCreateRemoteRef, remoteRef.String(), err.Error()) + + } + + if brnch.GetPath() != branch { + err := dEnv.DoltDB.DeleteBranch(ctx, brnch) + if err != nil { + return fmt.Errorf("%w: %s; %s", ErrFailedToDeleteBranch, brnch.String(), err.Error()) + } + } + } + + if performPull { + err = SaveDocsFromRoot(ctx, rootVal, dEnv) + if err != nil { + return ErrFailedToUpdateDocs + } + } + + // TODO: make this interface take a DoltRef and marshal it automatically + err = dEnv.RepoStateWriter().SetCWBHeadRef(ctx, ref.MarshalableRef{Ref: ref.NewBranchRef(branch)}) + if err != nil { + return err + } + + wsRef, err := ref.WorkingSetRefForHead(ref.NewBranchRef(branch)) + if err != nil { + return err + } + + ws := doltdb.EmptyWorkingSet(wsRef) + err = dEnv.UpdateWorkingSet(ctx, ws.WithWorkingRoot(rootVal).WithStagedRoot(rootVal)) + if err != nil { + return err + } + + return nil +} + +// Inits an empty, newly cloned repo. This would be unnecessary if we properly initialized the storage for a repository +// when we created it on dolthub. If we do that, this code can be removed. +func InitEmptyClonedRepo(ctx context.Context, dEnv *env.DoltEnv) error { + name := dEnv.Config.GetStringOrDefault(env.UserNameKey, "") + email := dEnv.Config.GetStringOrDefault(env.UserEmailKey, "") + initBranch := env.GetDefaultInitBranch(dEnv.Config) + + if name == "" { + return ErrUserNotFound + } else if email == "" { + return ErrEmailNotFound + } + + err := dEnv.InitDBWithTime(ctx, types.Format_Default, name, email, initBranch, doltdb.CommitNowFunc()) + if err != nil { + return ErrFailedToInitRepo + } + + return nil +} diff --git a/go/libraries/doltcore/env/config.go b/go/libraries/doltcore/env/config.go index bd7cdf77b7..37cd784ecb 100644 --- a/go/libraries/doltcore/env/config.go +++ b/go/libraries/doltcore/env/config.go @@ -86,6 +86,8 @@ type DoltCliConfig struct { fs filesys.ReadWriteFS } +var _ config.ReadableConfig = &DoltCliConfig{} + func loadDoltCliConfig(hdp HomeDirProvider, fs filesys.ReadWriteFS) (*DoltCliConfig, error) { ch := config.NewConfigHierarchy() @@ -159,7 +161,7 @@ func (dcc *DoltCliConfig) GetConfig(element DoltConfigElement) (config.ReadWrite // GetStringOrDefault retrieves a string from the config hierarchy and returns it if available. Otherwise it returns // the default string value -func (dcc *DoltCliConfig) GetStringOrDefault(key, defStr string) *string { +func (dcc *DoltCliConfig) GetStringOrDefault(key, defStr string) string { return GetStringOrDefault(dcc.ch, key, defStr) } @@ -180,14 +182,14 @@ func (dcc *DoltCliConfig) IfEmptyUseConfig(val, key string) string { return cfgVal } -func GetStringOrDefault(cfg config.ReadableConfig, key, defStr string) *string { +func GetStringOrDefault(cfg config.ReadableConfig, key, defStr string) string { val, err := cfg.GetString(key) if err != nil { - return &defStr + return defStr } - return &val + return val } // GetNameAndEmail returns the name and email from the supplied config diff --git a/go/libraries/doltcore/env/config_test.go b/go/libraries/doltcore/env/config_test.go index 772c0fef3f..309d165c4d 100644 --- a/go/libraries/doltcore/env/config_test.go +++ b/go/libraries/doltcore/env/config_test.go @@ -30,11 +30,11 @@ func TestConfig(t *testing.T) { lCfg.SetStrings(map[string]string{UserEmailKey: email}) gCfg.SetStrings(map[string]string{UserNameKey: name}) - if *dEnv.Config.GetStringOrDefault(UserEmailKey, "no") != email { + if dEnv.Config.GetStringOrDefault(UserEmailKey, "no") != email { t.Error("Should return", email) } - if *dEnv.Config.GetStringOrDefault("bad_key", "yes") != "yes" { + if dEnv.Config.GetStringOrDefault("bad_key", "yes") != "yes" { t.Error("Should return default value of yes") } diff --git a/go/libraries/doltcore/env/environment.go b/go/libraries/doltcore/env/environment.go index 553eb2c8b6..9d71671afa 100644 --- a/go/libraries/doltcore/env/environment.go +++ b/go/libraries/doltcore/env/environment.go @@ -19,7 +19,6 @@ import ( "crypto/tls" "errors" "fmt" - "os" "path/filepath" "runtime" "strings" @@ -61,7 +60,7 @@ const ( func getCommitHooks(ctx context.Context, dEnv *DoltEnv) ([]datas.CommitHook, error) { postCommitHooks := make([]datas.CommitHook, 0) - backupName := os.Getenv(doltdb.BackupToRemoteKey) + backupName := dEnv.Config.GetStringOrDefault(doltdb.BackupToRemoteKey, "") if backupName != "" { remotes, err := dEnv.GetRemotes() if err != nil { @@ -161,7 +160,7 @@ func Load(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, urlStr, // The process will not wait for this to finish so this may not always complete. go func() { // TODO dEnv.HasDoltTempTableDir() true but dEnv.TempTableFileDir() panics - tmpTableDir, err := dEnv.FS.Abs(filepath.Join(dEnv.urlStr, dEnv.GetDoltDir(), tempTablesDir)) + tmpTableDir, err := dEnv.FS.Abs(filepath.Join(dEnv.urlStr, dbfactory.DoltDir, tempTablesDir)) if err != nil { return } @@ -206,8 +205,7 @@ func Load(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, urlStr, } func GetDefaultInitBranch(cfg config.ReadableConfig) string { - s := GetStringOrDefault(cfg, InitBranchName, DefaultInitBranch) - return *s + return GetStringOrDefault(cfg, InitBranchName, DefaultInitBranch) } // initWorkingSetFromRepoState sets the working set for the env's head to mirror the contents of the repo state file. @@ -758,8 +756,8 @@ func (dEnv *DoltEnv) workingSetMeta() *doltdb.WorkingSetMeta { func (dEnv *DoltEnv) NewWorkingSetMeta(message string) *doltdb.WorkingSetMeta { return &doltdb.WorkingSetMeta{ - User: *dEnv.Config.GetStringOrDefault(UserNameKey, ""), - Email: *dEnv.Config.GetStringOrDefault(UserEmailKey, ""), + User: dEnv.Config.GetStringOrDefault(UserNameKey, ""), + Email: dEnv.Config.GetStringOrDefault(UserEmailKey, ""), Timestamp: uint64(time.Now().Unix()), Description: message, } diff --git a/go/libraries/doltcore/env/remotes.go b/go/libraries/doltcore/env/remotes.go index 7dd28fb2b9..dcbdd1714a 100644 --- a/go/libraries/doltcore/env/remotes.go +++ b/go/libraries/doltcore/env/remotes.go @@ -20,6 +20,7 @@ import ( "fmt" "path" "path/filepath" + "sort" "strings" "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" @@ -460,3 +461,36 @@ func getAbsFileRemoteUrl(urlStr string, fs filesys2.Filesys) (string, error) { } return dbfactory.FileScheme + "://" + urlStr, nil } + +// GetDefaultBranch returns the default branch from among the branches given, returning +// the configs default config branch first, then init branch main, then the old init branch master, +// and finally the first lexicographical branch if none of the others are found +func GetDefaultBranch(dEnv *DoltEnv, branches []ref.DoltRef) string { + if len(branches) == 0 { + return DefaultInitBranch + } + + sort.Slice(branches, func(i, j int) bool { + return branches[i].GetPath() < branches[j].GetPath() + }) + + branchMap := make(map[string]ref.DoltRef) + for _, b := range branches { + branchMap[b.GetPath()] = b + } + + if _, ok := branchMap[DefaultInitBranch]; ok { + return DefaultInitBranch + } + if _, ok := branchMap["master"]; ok { + return "master" + } + + // todo: do we care about this during clone? + defaultOrMain := GetDefaultInitBranch(dEnv.Config) + if _, ok := branchMap[defaultOrMain]; ok { + return defaultOrMain + } + + return branches[0].GetPath() +} diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index a220209a51..b31b9fcf1e 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -46,7 +46,7 @@ var ErrInvalidTableName = errors.NewKind("Invalid table name %s. Table names mus var ErrReservedTableName = errors.NewKind("Invalid table name %s. Table names beginning with `dolt_` are reserved for internal use") var ErrSystemTableAlter = errors.NewKind("Cannot alter table %s: system tables cannot be dropped or altered") -const DoltReadReplicaKey = "DOLT_READ_REPLICA_REMOTE" +const DoltReadReplicaKey = "dolt_read_replica_remote" type SqlDatabase interface { sql.Database @@ -68,8 +68,6 @@ func DbsAsDSQLDBs(dbs []sql.Database) []SqlDatabase { continue } switch v := sqlDb.(type) { - //case ReadReplicaDatabase, *ReadReplicaDatabase: - // dsqlDBs = append(dsqlDBs, v) case ReadReplicaDatabase, Database: dsqlDBs = append(dsqlDBs, v) default: diff --git a/go/libraries/doltcore/sqle/dsess/session.go b/go/libraries/doltcore/sqle/dsess/session.go index 3ead3e3ae5..9426ad8ab0 100644 --- a/go/libraries/doltcore/sqle/dsess/session.go +++ b/go/libraries/doltcore/sqle/dsess/session.go @@ -27,6 +27,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" + "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/store/hash" ) @@ -42,6 +43,7 @@ const ( DoltCommitOnTransactionCommit = "dolt_transaction_commit" TransactionsDisabledSysVar = "dolt_transactions_disabled" ForceTransactionCommit = "dolt_force_transaction_commit" + DoltDefaultBranchKey = "dolt_default_branch" ) var transactionMergeStomp = false @@ -133,6 +135,7 @@ type Session struct { BatchMode batchMode Username string Email string + Config config.ReadableConfig dbStates map[string]*DatabaseSessionState provider RevisionDatabaseProvider } @@ -194,11 +197,15 @@ type InitialDbState struct { } // NewSession creates a Session object from a standard sql.Session and 0 or more Database objects. -func NewSession(ctx *sql.Context, sqlSess sql.Session, pro RevisionDatabaseProvider, username, email string, dbs ...InitialDbState) (*Session, error) { +func NewSession(ctx *sql.Context, sqlSess sql.Session, pro RevisionDatabaseProvider, conf config.ReadableConfig, dbs ...InitialDbState) (*Session, error) { + username := conf.GetStringOrDefault(env.UserNameKey, "") + email := conf.GetStringOrDefault(env.UserEmailKey, "") + sess := &Session{ Session: sqlSess, Username: username, Email: email, + Config: conf, dbStates: make(map[string]*DatabaseSessionState), provider: pro, } @@ -1207,6 +1214,14 @@ func defineSystemVariables(name string) { Type: sql.NewSystemStringType(StagedKey(name)), Default: "", }, + { + Name: DoltDefaultBranchKey, + Scope: sql.SystemVariableScope_Global, + Dynamic: true, + SetVarHintApplies: false, + Type: sql.NewSystemStringType(DoltDefaultBranchKey), + Default: "", + }, }) } } diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go index f62f5a3193..2e6e0a7dae 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go @@ -58,7 +58,7 @@ var _ enginetest.ReadOnlyDatabaseHarness = (*DoltHarness)(nil) func newDoltHarness(t *testing.T) *DoltHarness { dEnv := dtestutils.CreateTestEnv() pro := sqle.NewDoltDatabaseProvider(dEnv.Config) - session, err := dsess.NewSession(sql.NewEmptyContext(), enginetest.NewBaseSession(), pro, "test", "email@test.com") + session, err := dsess.NewSession(sql.NewEmptyContext(), enginetest.NewBaseSession(), pro, dEnv.Config) require.NoError(t, err) return &DoltHarness{ t: t, @@ -137,7 +137,7 @@ func (d *DoltHarness) NewSession() *sql.Context { enginetest.NewContext(d), enginetest.NewBaseSession(), pro.(dsess.RevisionDatabaseProvider), - user, email, + d.env.Config, states..., ) require.NoError(d.t, err) diff --git a/go/libraries/utils/config/config.go b/go/libraries/utils/config/config.go index b38c77a050..023a92cead 100644 --- a/go/libraries/utils/config/config.go +++ b/go/libraries/utils/config/config.go @@ -30,6 +30,10 @@ type ReadableConfig interface { // ReadableConfig implementation. GetString(key string) (value string, err error) + // GetStringOrDefault retrieves a string from the config hierarchy and returns it if available. Otherwise it returns + // the default string value + GetStringOrDefault(key, defStr string) string + // Iter will perform a callback for each value in a config until all values have been exhausted or until the // callback returns true indicating that it should stop. Iter(func(string, string) (stop bool)) diff --git a/go/libraries/utils/config/config_hierarchy.go b/go/libraries/utils/config/config_hierarchy.go index a8a5ee2c3c..4de77774ad 100644 --- a/go/libraries/utils/config/config_hierarchy.go +++ b/go/libraries/utils/config/config_hierarchy.go @@ -31,6 +31,10 @@ type ConfigHierarchy struct { nameToConfig map[string]ReadWriteConfig } +var _ ReadableConfig = &ConfigHierarchy{} +var _ WritableConfig = &ConfigHierarchy{} +var _ ReadWriteConfig = &ConfigHierarchy{} + // NewConfigHierarchy creates an empty ConfigurationHierarchy func NewConfigHierarchy() *ConfigHierarchy { return &ConfigHierarchy{[]ReadWriteConfig{}, map[string]ReadWriteConfig{}} @@ -84,6 +88,13 @@ func (ch *ConfigHierarchy) GetString(k string) (string, error) { return "", ErrConfigParamNotFound } +func (ch *ConfigHierarchy) GetStringOrDefault(key, defStr string) string { + if val, err := ch.GetString(key); err == nil { + return val + } + return defStr +} + // SetStrings will set the value of configuration parameters in memory, and persist any changes to the backing file. // For ConfigHierarchies update parameter names must be of the format config_name::param_name func (ch *ConfigHierarchy) SetStrings(updates map[string]string) error { diff --git a/go/libraries/utils/config/file_config.go b/go/libraries/utils/config/file_config.go index 0d4fec56bb..28c873a40d 100644 --- a/go/libraries/utils/config/file_config.go +++ b/go/libraries/utils/config/file_config.go @@ -31,6 +31,10 @@ type FileConfig struct { properties map[string]string } +var _ ReadableConfig = &FileConfig{} +var _ WritableConfig = &FileConfig{} +var _ ReadWriteConfig = &FileConfig{} + // NewFileConfig creates a new empty config and writes it to a newly created file. If a file already exists at this // location it will be overwritten. If a directory does not exist where this file should live, it will be created. func NewFileConfig(path string, fs filesys.ReadWriteFS, properties map[string]string) (*FileConfig, error) { @@ -79,6 +83,15 @@ func (fc *FileConfig) GetString(k string) (string, error) { return "", ErrConfigParamNotFound } +// GetString retrieves a string from the cached config state +func (fc *FileConfig) GetStringOrDefault(k, defStr string) string { + if val, ok := fc.properties[k]; ok { + return val + } + + return defStr +} + // SetStrings will set the value of configuration parameters in memory, and persist any changes to the backing file. func (fc *FileConfig) SetStrings(updates map[string]string) error { modified := false diff --git a/go/libraries/utils/config/map_config.go b/go/libraries/utils/config/map_config.go index ba613bdead..8ca13ba8cb 100644 --- a/go/libraries/utils/config/map_config.go +++ b/go/libraries/utils/config/map_config.go @@ -21,6 +21,10 @@ type MapConfig struct { properties map[string]string } +var _ ReadableConfig = &MapConfig{} +var _ WritableConfig = &MapConfig{} +var _ ReadWriteConfig = &MapConfig{} + // NewMapConfig creates a config from a map. func NewMapConfig(properties map[string]string) *MapConfig { return &MapConfig{properties} @@ -35,6 +39,13 @@ func (mc *MapConfig) GetString(k string) (string, error) { return "", ErrConfigParamNotFound } +func (mc *MapConfig) GetStringOrDefault(key, defStr string) string { + if val, err := mc.GetString(key); err == nil { + return val + } + return defStr +} + // SetString sets the values for a map of updates. func (mc *MapConfig) SetStrings(updates map[string]string) error { for k, v := range updates { diff --git a/integration-tests/bats/aws-remotes.bats b/integration-tests/bats/aws-remotes.bats index 16d2c617e9..3ea45b229e 100644 --- a/integration-tests/bats/aws-remotes.bats +++ b/integration-tests/bats/aws-remotes.bats @@ -55,8 +55,8 @@ skip_if_no_aws_tests() { random_repo=`openssl rand -hex 32` run dolt clone 'aws://['"$DOLT_BATS_AWS_TABLE"':'"$DOLT_BATS_AWS_BUCKET"']/'"$random_repo" [ "$status" -eq 1 ] - [[ "$output" =~ "error: clone failed" ]] || false - [[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false + [[ "$output" =~ "clone failed" ]] || false + [[ "$output" =~ "remote at that url contains no Dolt data" ]] || false } @test "aws-remotes: can push to new remote" { diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index 6c2189251e..732d23cc84 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -335,8 +335,8 @@ SQL @test "remotes: clone an empty remote" { run dolt clone http://localhost:50051/test-org/empty [ "$status" -eq 1 ] - [[ "$output" =~ "error: clone failed" ]] || false - [[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false + [[ "$output" =~ "clone failed" ]] || false + [[ "$output" =~ "remote at that url contains no Dolt data" ]] || false } @test "remotes: clone a non-existent remote" { @@ -344,8 +344,8 @@ SQL cd "dolt-repo-clones" run dolt clone http://localhost:50051/foo/bar [ "$status" -eq 1 ] - [[ "$output" =~ "error: clone failed" ]] || false - [[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false + [[ "$output" =~ "clone failed" ]] || false + [[ "$output" =~ "remote at that url contains no Dolt data" ]] || false } @test "remotes: clone a different branch than main" { @@ -1142,7 +1142,7 @@ setup_ref_test() { run dolt clone "file://../clone_root" . [ "$status" -eq 1 ] - [[ "$output" =~ "error: clone failed" ]] || false + [[ "$output" =~ "clone failed" ]] || false # Validates that the directory exists run ls $testdir @@ -1158,7 +1158,7 @@ setup_ref_test() { cd .. run dolt clone "file://./clone_root" dest/ [ "$status" -eq 1 ] - [[ "$output" =~ "error: clone failed" ]] || false + [[ "$output" =~ "clone failed" ]] || false run ls $testdir [ "$status" -eq 0 ] diff --git a/integration-tests/bats/replication.bats b/integration-tests/bats/replication.bats index 46ff712b3c..308e04b0a8 100644 --- a/integration-tests/bats/replication.bats +++ b/integration-tests/bats/replication.bats @@ -33,15 +33,15 @@ teardown() { } @test "replication: push on commit" { - export DOLT_BACKUP_TO_REMOTE=backup1 cd repo1 + dolt config --local --add DOLT_BACKUP_TO_REMOTE backup1 + dolt config --list dolt remote -v dolt sql -q "create table t1 (a int primary key)" dolt commit -am "cm" cd .. dolt clone file://./bac1 repo2 - export DOLT_BACKUP_TO_REMOTE= cd repo2 run dolt ls [ "$status" -eq 0 ] @@ -50,8 +50,8 @@ teardown() { } @test "replication: no tags" { - export DOLT_BACKUP_TO_REMOTE=backup1 cd repo1 + dolt config --local --add DOLT_BACKUP_TO_REMOTE backup1 dolt tag [ ! -d "../bac1/.dolt" ] || false @@ -70,7 +70,7 @@ teardown() { [ "${#lines[@]}" -eq 1 ] [[ ! "$output" =~ "t1" ]] || false - export DOLT_READ_REPLICA_REMOTE=remote1 + dolt config --local --add DOLT_READ_REPLICA_REMOTE remote1 run dolt sql -q "show tables" -r csv [ "$status" -eq 0 ] [ "${#lines[@]}" -eq 2 ] diff --git a/integration-tests/bats/sql-server.bats b/integration-tests/bats/sql-server.bats index 73c3ecb320..eb61d043a6 100644 --- a/integration-tests/bats/sql-server.bats +++ b/integration-tests/bats/sql-server.bats @@ -1055,7 +1055,7 @@ while True: mkdir bac1 cd repo1 dolt remote add backup1 file://../bac1 - export DOLT_BACKUP_TO_REMOTE=backup1 + dolt config --local --add DOLT_BACKUP_TO_REMOTE backup1 start_sql_server repo1 multi_query repo1 1 " @@ -1098,7 +1098,7 @@ while True: dolt push -u remote1 main cd ../repo1 - export DOLT_READ_REPLICA_REMOTE=remote1 + dolt config --local --add DOLT_READ_REPLICA_REMOTE remote1 start_sql_server repo1 server_query repo1 1 "show tables" "Table\ntest"