mirror of
https://github.com/dolthub/dolt.git
synced 2026-03-14 19:20:44 -05:00
config, clone, backups refactor (#2196)
* shittiest read replica imaginable is kind of working * import cycle progress * delete unecessary files and fix db type switch bug * Add bats test * delete comments * fix working set updates for cli * [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh * clean comments * comment racy server test * move env variables to local dolt config * refactor clone, fix multienv test * cleanup comments * missing copyright * brian's comments, add compile time checks for config interfaces * format * fix windows filepaths issue * file:/// with three slashes should work on windows * more windows problems * three slashes didn't work for clone, do chdir to ref local dolt db Co-authored-by: max-hoffman <max-hoffman@users.noreply.github.com>
This commit is contained in:
Maximilian Hoffman
committed by
GitHub
GitHub
parent
96e7adf387
commit
8a5f3f54be
@@ -16,13 +16,8 @@ package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/dolthub/dolt/go/cmd/dolt/cli"
|
||||
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
|
||||
@@ -31,14 +26,11 @@ import (
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/env"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/remotestorage"
|
||||
"github.com/dolthub/dolt/go/libraries/events"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/argparser"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/earl"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/filesys"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/strhelp"
|
||||
"github.com/dolthub/dolt/go/store/datas"
|
||||
"github.com/dolthub/dolt/go/store/types"
|
||||
)
|
||||
|
||||
@@ -109,58 +101,59 @@ func (cmd CloneCmd) Exec(ctx context.Context, commandStr string, args []string,
|
||||
remoteName := apr.GetValueOrDefault(remoteParam, "origin")
|
||||
branch := apr.GetValueOrDefault(branchParam, "")
|
||||
dir, urlStr, verr := parseArgs(apr)
|
||||
if verr != nil {
|
||||
return HandleVErrAndExitCode(verr, usage)
|
||||
}
|
||||
|
||||
userDirExists, _ := dEnv.FS.Exists(dir)
|
||||
|
||||
scheme, remoteUrl, err := env.GetAbsRemoteUrl(dEnv.FS, dEnv.Config, urlStr)
|
||||
|
||||
if err != nil {
|
||||
verr = errhand.BuildDError("error: '%s' is not valid.", urlStr).Build()
|
||||
return HandleVErrAndExitCode(errhand.BuildDError("error: '%s' is not valid.", urlStr).Build(), usage)
|
||||
}
|
||||
var params map[string]string
|
||||
params, verr = parseRemoteArgs(apr, scheme, remoteUrl)
|
||||
if verr != nil {
|
||||
return HandleVErrAndExitCode(verr, usage)
|
||||
}
|
||||
|
||||
if verr == nil {
|
||||
var params map[string]string
|
||||
params, verr = parseRemoteArgs(apr, scheme, remoteUrl)
|
||||
var r env.Remote
|
||||
var srcDB *doltdb.DoltDB
|
||||
r, srcDB, verr = createRemote(ctx, remoteName, remoteUrl, params, dEnv)
|
||||
if verr != nil {
|
||||
return HandleVErrAndExitCode(verr, usage)
|
||||
}
|
||||
|
||||
if verr == nil {
|
||||
var r env.Remote
|
||||
var srcDB *doltdb.DoltDB
|
||||
r, srcDB, verr = createRemote(ctx, remoteName, remoteUrl, params, dEnv)
|
||||
dEnv, err = actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir)
|
||||
if err != nil {
|
||||
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
|
||||
}
|
||||
|
||||
if verr == nil {
|
||||
dEnv, verr = envForClone(ctx, srcDB.ValueReadWriter().Format(), r, dir, dEnv.FS, dEnv.Version)
|
||||
err = actions.CloneRemote(ctx, srcDB, remoteName, branch, dEnv)
|
||||
if err != nil {
|
||||
// If we're cloning into a directory that already exists do not erase it. Otherwise
|
||||
// make best effort to delete the directory we created.
|
||||
if userDirExists {
|
||||
// Set the working dir to the parent of the .dolt folder so we can delete .dolt
|
||||
_ = os.Chdir(dir)
|
||||
_ = dEnv.FS.Delete(dbfactory.DoltDir, true)
|
||||
} else {
|
||||
_ = os.Chdir("../")
|
||||
_ = dEnv.FS.Delete(dir, true)
|
||||
}
|
||||
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
|
||||
}
|
||||
|
||||
if verr == nil {
|
||||
verr = cloneRemote(ctx, srcDB, remoteName, branch, dEnv)
|
||||
|
||||
if verr == nil {
|
||||
evt := events.GetEventFromContext(ctx)
|
||||
u, err := earl.Parse(remoteUrl)
|
||||
|
||||
if err == nil {
|
||||
if u.Scheme != "" {
|
||||
evt.SetAttribute(eventsapi.AttributeID_REMOTE_URL_SCHEME, u.Scheme)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if verr != nil {
|
||||
// If we're cloning into a directory that already exists do not erase it. Otherwise
|
||||
// make best effort to delete the directory we created.
|
||||
if userDirExists {
|
||||
// Set the working dir to the parent of the .dolt folder so we can delete .dolt
|
||||
_ = os.Chdir(dir)
|
||||
_ = dEnv.FS.Delete(dbfactory.DoltDir, true)
|
||||
} else {
|
||||
_ = os.Chdir("../")
|
||||
_ = dEnv.FS.Delete(dir, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
evt := events.GetEventFromContext(ctx)
|
||||
u, err := earl.Parse(remoteUrl)
|
||||
if err == nil {
|
||||
if u.Scheme != "" {
|
||||
evt.SetAttribute(eventsapi.AttributeID_REMOTE_URL_SCHEME, u.Scheme)
|
||||
}
|
||||
}
|
||||
|
||||
return HandleVErrAndExitCode(verr, usage)
|
||||
return 0
|
||||
}
|
||||
|
||||
func parseArgs(apr *argparser.ArgParseResults) (string, string, errhand.VerboseError) {
|
||||
@@ -190,44 +183,6 @@ func parseArgs(apr *argparser.ArgParseResults) (string, string, errhand.VerboseE
|
||||
return dir, urlStr, nil
|
||||
}
|
||||
|
||||
func envForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, dir string, fs filesys.Filesys, version string) (*env.DoltEnv, errhand.VerboseError) {
|
||||
exists, _ := fs.Exists(filepath.Join(dir, dbfactory.DoltDir))
|
||||
|
||||
if exists {
|
||||
return nil, errhand.BuildDError("error: data repository already exists at " + dir).Build()
|
||||
}
|
||||
|
||||
err := fs.MkDirs(dir)
|
||||
|
||||
if err != nil {
|
||||
return nil, errhand.BuildDError("error: unable to create directories: " + dir).Build()
|
||||
}
|
||||
|
||||
err = os.Chdir(dir)
|
||||
|
||||
if err != nil {
|
||||
return nil, errhand.BuildDError("error: unable to access directory " + dir).Build()
|
||||
}
|
||||
|
||||
dEnv := env.Load(ctx, env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, version)
|
||||
err = dEnv.InitRepoWithNoData(ctx, nbf)
|
||||
|
||||
if err != nil {
|
||||
return nil, errhand.BuildDError("error: unable to initialize repo without data").AddCause(err).Build()
|
||||
}
|
||||
|
||||
dEnv.RSLoadErr = nil
|
||||
if !env.IsEmptyRemote(r) {
|
||||
dEnv.RepoState, err = env.CloneRepoState(dEnv.FS, r)
|
||||
|
||||
if err != nil {
|
||||
return nil, errhand.BuildDError("error: unable to create repo state with remote " + r.Name).AddCause(err).Build()
|
||||
}
|
||||
}
|
||||
|
||||
return dEnv, nil
|
||||
}
|
||||
|
||||
func createRemote(ctx context.Context, remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv) (env.Remote, *doltdb.DoltDB, errhand.VerboseError) {
|
||||
cli.Printf("cloning %s\n", remoteUrl)
|
||||
|
||||
@@ -248,202 +203,3 @@ func createRemote(ctx context.Context, remoteName, remoteUrl string, params map[
|
||||
|
||||
return r, ddb, nil
|
||||
}
|
||||
|
||||
func cloneProg(eventCh <-chan datas.TableFileEvent) {
|
||||
var (
|
||||
chunks int64
|
||||
chunksDownloading int64
|
||||
chunksDownloaded int64
|
||||
cliPos int
|
||||
)
|
||||
|
||||
cliPos = cli.DeleteAndPrint(cliPos, "Retrieving remote information.")
|
||||
for tblFEvt := range eventCh {
|
||||
switch tblFEvt.EventType {
|
||||
case datas.Listed:
|
||||
for _, tf := range tblFEvt.TableFiles {
|
||||
chunks += int64(tf.NumChunks())
|
||||
}
|
||||
case datas.DownloadStart:
|
||||
for _, tf := range tblFEvt.TableFiles {
|
||||
chunksDownloading += int64(tf.NumChunks())
|
||||
}
|
||||
case datas.DownloadSuccess:
|
||||
for _, tf := range tblFEvt.TableFiles {
|
||||
chunksDownloading -= int64(tf.NumChunks())
|
||||
chunksDownloaded += int64(tf.NumChunks())
|
||||
}
|
||||
case datas.DownloadFailed:
|
||||
// Ignore for now and output errors on the main thread
|
||||
}
|
||||
|
||||
str := fmt.Sprintf("%s of %s chunks complete. %s chunks being downloaded currently.", strhelp.CommaIfy(chunksDownloaded), strhelp.CommaIfy(chunks), strhelp.CommaIfy(chunksDownloading))
|
||||
cliPos = cli.DeleteAndPrint(cliPos, str)
|
||||
}
|
||||
|
||||
cli.Println()
|
||||
}
|
||||
|
||||
func cloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch string, dEnv *env.DoltEnv) errhand.VerboseError {
|
||||
eventCh := make(chan datas.TableFileEvent, 128)
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cloneProg(eventCh)
|
||||
}()
|
||||
|
||||
err := actions.Clone(ctx, srcDB, dEnv.DoltDB, eventCh)
|
||||
close(eventCh)
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if err != nil {
|
||||
if err == datas.ErrNoData {
|
||||
err = errors.New("remote at that url contains no Dolt data")
|
||||
}
|
||||
|
||||
return errhand.BuildDError("error: clone failed").AddCause(err).Build()
|
||||
}
|
||||
|
||||
branches, err := dEnv.DoltDB.GetBranches(ctx)
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: failed to list branches").AddCause(err).Build()
|
||||
}
|
||||
|
||||
if branch == "" {
|
||||
branch = GetDefaultBranch(dEnv, branches)
|
||||
}
|
||||
|
||||
// If we couldn't find a branch but the repo cloned successfully, it's empty. Initialize it instead of pulling from
|
||||
// the remote.
|
||||
performPull := true
|
||||
if branch == "" {
|
||||
err = initEmptyClonedRepo(ctx, dEnv)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
branch = env.GetDefaultInitBranch(dEnv.Config)
|
||||
performPull = false
|
||||
}
|
||||
|
||||
cs, _ := doltdb.NewCommitSpec(branch)
|
||||
cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil)
|
||||
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: could not get " + branch).AddCause(err).Build()
|
||||
}
|
||||
|
||||
rootVal, err := cm.GetRootValue()
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: could not get the root value of " + branch).AddCause(err).Build()
|
||||
}
|
||||
|
||||
// After actions.Clone, we have repository with a local branch for
|
||||
// every branch in the remote. What we want is a remote branch ref for
|
||||
// every branch in the remote. We iterate through local branches and
|
||||
// create remote refs corresponding to each of them. We delete all of
|
||||
// the local branches except for the one corresponding to |branch|.
|
||||
for _, brnch := range branches {
|
||||
cs, _ := doltdb.NewCommitSpec(brnch.GetPath())
|
||||
cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil)
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: could not resolve branch ref at " + brnch.String()).AddCause(err).Build()
|
||||
}
|
||||
|
||||
remoteRef := ref.NewRemoteRef(remoteName, brnch.GetPath())
|
||||
err = dEnv.DoltDB.SetHeadToCommit(ctx, remoteRef, cm)
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: could not create remote ref at " + remoteRef.String()).AddCause(err).Build()
|
||||
}
|
||||
|
||||
if brnch.GetPath() != branch {
|
||||
err := dEnv.DoltDB.DeleteBranch(ctx, brnch)
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: could not delete local branch " + brnch.String() + " after clone.").AddCause(err).Build()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if performPull {
|
||||
err = actions.SaveDocsFromRoot(ctx, rootVal, dEnv)
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: failed to update docs on the filesystem").AddCause(err).Build()
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: make this interface take a DoltRef and marshal it automatically
|
||||
err = dEnv.RepoStateWriter().SetCWBHeadRef(ctx, ref.MarshalableRef{Ref: ref.NewBranchRef(branch)})
|
||||
if err != nil {
|
||||
return errhand.VerboseErrorFromError(err)
|
||||
}
|
||||
|
||||
wsRef, err := ref.WorkingSetRefForHead(ref.NewBranchRef(branch))
|
||||
if err != nil {
|
||||
return errhand.VerboseErrorFromError(err)
|
||||
}
|
||||
|
||||
ws := doltdb.EmptyWorkingSet(wsRef)
|
||||
err = dEnv.UpdateWorkingSet(ctx, ws.WithWorkingRoot(rootVal).WithStagedRoot(rootVal))
|
||||
if err != nil {
|
||||
return errhand.VerboseErrorFromError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Inits an empty, newly cloned repo. This would be unnecessary if we properly initialized the storage for a repository
|
||||
// when we created it on dolthub. If we do that, this code can be removed.
|
||||
func initEmptyClonedRepo(ctx context.Context, dEnv *env.DoltEnv) error {
|
||||
name := dEnv.Config.GetStringOrDefault(env.UserNameKey, "")
|
||||
email := dEnv.Config.GetStringOrDefault(env.UserEmailKey, "")
|
||||
initBranch := env.GetDefaultInitBranch(dEnv.Config)
|
||||
|
||||
if *name == "" {
|
||||
return errhand.BuildDError(fmt.Sprintf("error: could not determine user name. run dolt config --global --add %[1]s", env.UserNameKey)).Build()
|
||||
} else if *email == "" {
|
||||
return errhand.BuildDError("error: could not determine email. run dolt config --global --add %[1]s", env.UserEmailKey).Build()
|
||||
}
|
||||
|
||||
err := dEnv.InitDBWithTime(ctx, types.Format_Default, *name, *email, initBranch, doltdb.CommitNowFunc())
|
||||
if err != nil {
|
||||
return errhand.BuildDError("error: could not initialize repository").AddCause(err).Build()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetDefaultBranch returns the default branch from among the branches given, returning
|
||||
// the configs default config branch first, then init branch main, then the old init branch master,
|
||||
// and finally the first lexicographical branch if none of the others are found
|
||||
func GetDefaultBranch(dEnv *env.DoltEnv, branches []ref.DoltRef) string {
|
||||
if len(branches) == 0 {
|
||||
return env.DefaultInitBranch
|
||||
}
|
||||
|
||||
sort.Slice(branches, func(i, j int) bool {
|
||||
return branches[i].GetPath() < branches[j].GetPath()
|
||||
})
|
||||
|
||||
branchMap := make(map[string]ref.DoltRef)
|
||||
for _, b := range branches {
|
||||
branchMap[b.GetPath()] = b
|
||||
}
|
||||
|
||||
if _, ok := branchMap[env.DefaultInitBranch]; ok {
|
||||
return env.DefaultInitBranch
|
||||
}
|
||||
if _, ok := branchMap["master"]; ok {
|
||||
return "master"
|
||||
}
|
||||
|
||||
// todo: do we care about this during clone?
|
||||
defaultOrMain := env.GetDefaultInitBranch(dEnv.Config)
|
||||
if _, ok := branchMap[defaultOrMain]; ok {
|
||||
return defaultOrMain
|
||||
}
|
||||
|
||||
return branches[0].GetPath()
|
||||
}
|
||||
|
||||
@@ -226,7 +226,7 @@ func getCommitMessageFromEditor(ctx context.Context, dEnv *env.DoltEnv) (string,
|
||||
editorStr := dEnv.Config.GetStringOrDefault(env.DoltEditor, backupEd)
|
||||
|
||||
cli.ExecuteWithStdioRestored(func() {
|
||||
commitMsg, _ := editor.OpenCommitEditor(*editorStr, initialMsg)
|
||||
commitMsg, _ := editor.OpenCommitEditor(editorStr, initialMsg)
|
||||
finalMsg = parseCommitMessage(commitMsg)
|
||||
})
|
||||
return finalMsg, nil
|
||||
|
||||
@@ -104,7 +104,7 @@ func loadEndpoint(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) string {
|
||||
|
||||
host := dEnv.Config.GetStringOrDefault(env.RemotesApiHostKey, env.DefaultRemotesApiHost)
|
||||
port := dEnv.Config.GetStringOrDefault(env.RemotesApiHostPortKey, env.DefaultRemotesApiPort)
|
||||
return fmt.Sprintf("%s:%s", *host, *port)
|
||||
return fmt.Sprintf("%s:%s", host, port)
|
||||
}
|
||||
|
||||
func loadCred(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) (creds.DoltCreds, errhand.VerboseError) {
|
||||
|
||||
@@ -162,7 +162,7 @@ func updateProfileWithCredentials(ctx context.Context, dEnv *env.DoltEnv, c cred
|
||||
|
||||
host := dEnv.Config.GetStringOrDefault(env.RemotesApiHostKey, env.DefaultRemotesApiHost)
|
||||
port := dEnv.Config.GetStringOrDefault(env.RemotesApiHostPortKey, env.DefaultRemotesApiPort)
|
||||
hostAndPort := fmt.Sprintf("%s:%s", *host, *port)
|
||||
hostAndPort := fmt.Sprintf("%s:%s", host, port)
|
||||
endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
|
||||
Endpoint: hostAndPort,
|
||||
Creds: c,
|
||||
|
||||
@@ -192,7 +192,7 @@ func loginWithCreds(ctx context.Context, dEnv *env.DoltEnv, dc creds.DoltCreds,
|
||||
|
||||
func openBrowserForCredsAdd(dEnv *env.DoltEnv, dc creds.DoltCreds) {
|
||||
loginUrl := dEnv.Config.GetStringOrDefault(env.AddCredsUrlKey, env.DefaultLoginUrl)
|
||||
url := fmt.Sprintf("%s#%s", *loginUrl, dc.PubKeyBase32Str())
|
||||
url := fmt.Sprintf("%s#%s", loginUrl, dc.PubKeyBase32Str())
|
||||
cli.Printf("Opening a browser to:\n\t%s\nPlease associate your key with your account.\n", url)
|
||||
open.Start(url)
|
||||
}
|
||||
@@ -201,7 +201,7 @@ func getCredentialsClient(dEnv *env.DoltEnv, dc creds.DoltCreds) (remotesapi.Cre
|
||||
host := dEnv.Config.GetStringOrDefault(env.RemotesApiHostKey, env.DefaultRemotesApiHost)
|
||||
port := dEnv.Config.GetStringOrDefault(env.RemotesApiHostPortKey, env.DefaultRemotesApiPort)
|
||||
endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
|
||||
Endpoint: fmt.Sprintf("%s:%s", *host, *port),
|
||||
Endpoint: fmt.Sprintf("%s:%s", host, port),
|
||||
Creds: dc,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -18,15 +18,15 @@ import (
|
||||
"context"
|
||||
"path"
|
||||
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/earl"
|
||||
"github.com/dolthub/dolt/go/store/types"
|
||||
|
||||
"github.com/dolthub/dolt/go/cmd/dolt/cli"
|
||||
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/env"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/argparser"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/earl"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/filesys"
|
||||
"github.com/dolthub/dolt/go/store/types"
|
||||
)
|
||||
|
||||
var readTablesDocs = cli.CommandDocumentationContent{
|
||||
@@ -125,7 +125,7 @@ func (cmd ReadTablesCmd) Exec(ctx context.Context, commandStr string, args []str
|
||||
BuildVerrAndExit("Failed to get remote branches", err)
|
||||
}
|
||||
|
||||
dEnv, verr = initializeShallowCloneRepo(ctx, dEnv, srcDB.Format(), dir, GetDefaultBranch(dEnv, branches))
|
||||
dEnv, verr = initializeShallowCloneRepo(ctx, dEnv, srcDB.Format(), dir, env.GetDefaultBranch(dEnv, branches))
|
||||
if verr != nil {
|
||||
return HandleVErrAndExitCode(verr, usage)
|
||||
}
|
||||
@@ -223,20 +223,19 @@ func getRemoteDBAtCommit(ctx context.Context, remoteUrl string, remoteUrlParams
|
||||
}
|
||||
|
||||
func initializeShallowCloneRepo(ctx context.Context, dEnv *env.DoltEnv, nbf *types.NomsBinFormat, dir, branchName string) (*env.DoltEnv, errhand.VerboseError) {
|
||||
var verr errhand.VerboseError
|
||||
dEnv, verr = envForClone(ctx, nbf, env.NoRemote, dir, dEnv.FS, dEnv.Version)
|
||||
var err error
|
||||
dEnv, err = actions.EnvForClone(ctx, nbf, env.NoRemote, dir, dEnv.FS, dEnv.Version, env.GetCurrentUserHomeDir)
|
||||
|
||||
if verr != nil {
|
||||
return nil, verr
|
||||
if err != nil {
|
||||
return nil, errhand.VerboseErrorFromError(err)
|
||||
}
|
||||
|
||||
err := initEmptyClonedRepo(ctx, dEnv)
|
||||
err = actions.InitEmptyClonedRepo(ctx, dEnv)
|
||||
if err != nil {
|
||||
return nil, errhand.BuildDError("Unable to initialize repo.").AddCause(err).Build()
|
||||
}
|
||||
|
||||
err = dEnv.InitializeRepoState(ctx, branchName)
|
||||
|
||||
if err != nil {
|
||||
return nil, errhand.BuildDError("Unable to initialize repo.").AddCause(err).Build()
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ func (cmd SendMetricsCmd) Exec(ctx context.Context, commandStr string, args []st
|
||||
|
||||
metricsDisabled := dEnv.Config.GetStringOrDefault(env.MetricsDisabled, "false")
|
||||
|
||||
disabled, err := strconv.ParseBool(*metricsDisabled)
|
||||
disabled, err := strconv.ParseBool(metricsDisabled)
|
||||
if err != nil {
|
||||
// log.Print(err)
|
||||
return 1
|
||||
@@ -134,20 +134,20 @@ func getGRPCEmitter(dEnv *env.DoltEnv) *events.GrpcEmitter {
|
||||
portStr := dEnv.Config.GetStringOrDefault(env.MetricsPort, env.DefaultMetricsPort)
|
||||
insecureStr := dEnv.Config.GetStringOrDefault(env.MetricsInsecure, "false")
|
||||
|
||||
port, err := strconv.ParseUint(*portStr, 10, 16)
|
||||
port, err := strconv.ParseUint(portStr, 10, 16)
|
||||
|
||||
if err != nil {
|
||||
log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid port.", env.MetricsPort, *portStr))
|
||||
log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid port.", env.MetricsPort, portStr))
|
||||
return nil
|
||||
}
|
||||
|
||||
insecure, err := strconv.ParseBool(*insecureStr)
|
||||
insecure, err := strconv.ParseBool(insecureStr)
|
||||
|
||||
if err != nil {
|
||||
log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid true/false value", env.MetricsInsecure, *insecureStr))
|
||||
log.Println(color.YellowString("The config value of '%s' is '%s' which is not a valid true/false value", env.MetricsInsecure, insecureStr))
|
||||
}
|
||||
|
||||
hostAndPort := fmt.Sprintf("%s:%d", *host, port)
|
||||
hostAndPort := fmt.Sprintf("%s:%d", host, port)
|
||||
endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
|
||||
Endpoint: hostAndPort,
|
||||
Insecure: insecure,
|
||||
|
||||
@@ -520,8 +520,8 @@ func CollectDBs(ctx context.Context, mrEnv env.MultiRepoEnv) ([]dsqle.SqlDatabas
|
||||
var db dsqle.SqlDatabase
|
||||
err := mrEnv.Iter(func(name string, dEnv *env.DoltEnv) (stop bool, err error) {
|
||||
db = newDatabase(name, dEnv)
|
||||
if remoteName := os.Getenv(dsqle.DoltReadReplicaKey); remoteName != "" {
|
||||
db, err = dsqle.NewReadReplicaDatabase(ctx, db.(dsqle.Database), remoteName, dEnv.RepoStateReader(), dEnv.TempTableFilesDir(), dEnv.NewWorkingSetMeta("read replica update"))
|
||||
if remoteName := dEnv.Config.GetStringOrDefault(dsqle.DoltReadReplicaKey, ""); remoteName != "" {
|
||||
db, err = dsqle.NewReadReplicaDatabase(ctx, db.(dsqle.Database), remoteName, dEnv.RepoStateReader(), dEnv.TempTableFilesDir(), doltdb.TodoWorkingSetMeta())
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
@@ -1474,9 +1474,7 @@ func newSqlEngine(
|
||||
}
|
||||
|
||||
// TODO: not having user and email for this command should probably be an error or warning, it disables certain functionality
|
||||
username := *dEnv.Config.GetStringOrDefault(env.UserNameKey, "")
|
||||
email := *dEnv.Config.GetStringOrDefault(env.UserEmailKey, "")
|
||||
sess, err := dsess.NewSession(sql.NewEmptyContext(), sql.NewBaseSession(), pro, username, email, dbStates...)
|
||||
sess, err := dsess.NewSession(sql.NewEmptyContext(), sql.NewBaseSession(), pro, dEnv.Config, dbStates...)
|
||||
|
||||
// TODO: this should just be the session default like it is with MySQL
|
||||
err = sess.SetSessionVariable(sql.NewContext(ctx), sql.AutoCommitSessionVar, true)
|
||||
|
||||
@@ -40,22 +40,6 @@ import (
|
||||
"github.com/dolthub/dolt/go/libraries/utils/tracing"
|
||||
)
|
||||
|
||||
const DoltDefaultBranchKey = "dolt_default_branch"
|
||||
|
||||
func init() {
|
||||
sql.SystemVariables.AddSystemVariables([]sql.SystemVariable{
|
||||
{
|
||||
Name: DoltDefaultBranchKey,
|
||||
Scope: sql.SystemVariableScope_Global,
|
||||
Dynamic: true,
|
||||
SetVarHintApplies: false,
|
||||
Type: sql.NewSystemStringType(DoltDefaultBranchKey),
|
||||
Default: "",
|
||||
},
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// Serve starts a MySQL-compatible server. Returns any errors that were encountered.
|
||||
func Serve(ctx context.Context, version string, serverConfig ServerConfig, serverController *ServerController, dEnv *env.DoltEnv) (startError error, closeError error) {
|
||||
if serverConfig == nil {
|
||||
@@ -102,8 +86,6 @@ func Serve(ctx context.Context, version string, serverConfig ServerConfig, serve
|
||||
|
||||
userAuth := auth.NewNativeSingle(serverConfig.User(), serverConfig.Password(), permissions)
|
||||
|
||||
var username string
|
||||
var email string
|
||||
var mrEnv env.MultiRepoEnv
|
||||
dbNamesAndPaths := serverConfig.DatabaseNamesAndPaths()
|
||||
if len(dbNamesAndPaths) == 0 {
|
||||
@@ -112,9 +94,6 @@ func Serve(ctx context.Context, version string, serverConfig ServerConfig, serve
|
||||
if err != nil {
|
||||
return err, nil
|
||||
}
|
||||
|
||||
username = *dEnv.Config.GetStringOrDefault(env.UserNameKey, "")
|
||||
email = *dEnv.Config.GetStringOrDefault(env.UserEmailKey, "")
|
||||
} else {
|
||||
var err error
|
||||
mrEnv, err = env.LoadMultiEnv(ctx, env.GetCurrentUserHomeDir, dEnv.FS, version, dbNamesAndPaths...)
|
||||
@@ -164,7 +143,7 @@ func Serve(ctx context.Context, version string, serverConfig ServerConfig, serve
|
||||
// to the value of mysql that we support.
|
||||
},
|
||||
sqlEngine,
|
||||
newSessionBuilder(sqlEngine, username, email, pro, mrEnv, serverConfig.AutoCommit()),
|
||||
newSessionBuilder(sqlEngine, dEnv.Config, pro, mrEnv, serverConfig.AutoCommit()),
|
||||
)
|
||||
|
||||
if startError != nil {
|
||||
@@ -191,7 +170,7 @@ func portInUse(hostPort string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func newSessionBuilder(sqlEngine *sqle.Engine, username string, email string, pro dsqle.DoltDatabaseProvider, mrEnv env.MultiRepoEnv, autocommit bool) server.SessionBuilder {
|
||||
func newSessionBuilder(sqlEngine *sqle.Engine, dConf *env.DoltCliConfig, pro dsqle.DoltDatabaseProvider, mrEnv env.MultiRepoEnv, autocommit bool) server.SessionBuilder {
|
||||
return func(ctx context.Context, conn *mysql.Conn, host string) (sql.Session, *sql.IndexRegistry, *sql.ViewRegistry, error) {
|
||||
tmpSqlCtx := sql.NewEmptyContext()
|
||||
|
||||
@@ -203,7 +182,7 @@ func newSessionBuilder(sqlEngine *sqle.Engine, username string, email string, pr
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
doltSess, err := dsess.NewSession(tmpSqlCtx, mysqlSess, pro, username, email, dbStates...)
|
||||
doltSess, err := dsess.NewSession(tmpSqlCtx, mysqlSess, pro, dConf, dbStates...)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
@@ -249,7 +228,7 @@ func getDbStates(ctx context.Context, dbs []dsqle.SqlDatabase) ([]dsess.InitialD
|
||||
var init dsess.InitialDbState
|
||||
var err error
|
||||
|
||||
_, val, ok := sql.SystemVariables.GetGlobal(DoltDefaultBranchKey)
|
||||
_, val, ok := sql.SystemVariables.GetGlobal(dsess.DoltDefaultBranchKey)
|
||||
if ok && val != "" {
|
||||
init, err = GetInitialDBStateWithDefaultBranch(ctx, db, val.(string))
|
||||
} else {
|
||||
|
||||
@@ -326,6 +326,13 @@ func TestServerSetDefaultBranch(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
defer func(sess *dbr.Session) {
|
||||
var res []struct {
|
||||
int
|
||||
}
|
||||
sess.SelectBySql("set GLOBAL dolt_default_branch = ''").LoadContext(context.Background(), &res)
|
||||
}(sess)
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.query.Query, func(t *testing.T) {
|
||||
var branch []testBranch
|
||||
@@ -360,11 +367,14 @@ func TestServerSetDefaultBranch(t *testing.T) {
|
||||
assert.ElementsMatch(t, branch, test.expectedRes)
|
||||
})
|
||||
}
|
||||
|
||||
var res []struct {
|
||||
int
|
||||
}
|
||||
sess.SelectBySql("set GLOBAL dolt_default_branch = ''").LoadContext(context.Background(), &res)
|
||||
}
|
||||
|
||||
func TestReadReplica(t *testing.T) {
|
||||
t.Skip("this fails on a query from the previous test suite if run as a file")
|
||||
|
||||
var err error
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
@@ -372,11 +382,22 @@ func TestReadReplica(t *testing.T) {
|
||||
}
|
||||
defer os.Chdir(cwd)
|
||||
|
||||
multiSetup := testcommands.CreateMultiEnvWithRemote()
|
||||
multiSetup := testcommands.NewMultiRepoTestSetup(t)
|
||||
defer os.RemoveAll(multiSetup.Root)
|
||||
|
||||
readOnlyDbName := multiSetup.DbNames[0]
|
||||
sourceDbName := multiSetup.DbNames[0]
|
||||
multiSetup.NewDB("read_replica")
|
||||
multiSetup.NewRemote("remote1")
|
||||
multiSetup.PushToRemote("read_replica", "remote1")
|
||||
multiSetup.CloneDB("remote1", "source_db")
|
||||
|
||||
readReplicaDbName := multiSetup.DbNames[0]
|
||||
sourceDbName := multiSetup.DbNames[1]
|
||||
|
||||
localCfg, ok := multiSetup.MrEnv[readReplicaDbName].Config.GetConfig(env.LocalConfig)
|
||||
if !ok {
|
||||
t.Fatal("local config does not exist")
|
||||
}
|
||||
localCfg.SetStrings(map[string]string{dsqle.DoltReadReplicaKey: "remote1"})
|
||||
|
||||
// start server as read replica
|
||||
sc := CreateServerController()
|
||||
@@ -384,10 +405,10 @@ func TestReadReplica(t *testing.T) {
|
||||
|
||||
func() {
|
||||
err = os.Setenv(dsqle.DoltReadReplicaKey, "remote1")
|
||||
os.Chdir(multiSetup.DbPaths[readOnlyDbName])
|
||||
os.Chdir(multiSetup.DbPaths[readReplicaDbName])
|
||||
|
||||
go func() {
|
||||
_, _ = Serve(context.Background(), "", serverConfig, sc, multiSetup.MrEnv[readOnlyDbName])
|
||||
_, _ = Serve(context.Background(), "", serverConfig, sc, multiSetup.MrEnv[readReplicaDbName])
|
||||
}()
|
||||
err = sc.WaitForStart()
|
||||
require.NoError(t, err)
|
||||
@@ -395,20 +416,19 @@ func TestReadReplica(t *testing.T) {
|
||||
defer sc.StopServer()
|
||||
defer os.Unsetenv(dsqle.DoltReadReplicaKey)
|
||||
|
||||
conn, err := dbr.Open("mysql", ConnectionString(serverConfig)+readOnlyDbName, nil)
|
||||
conn, err := dbr.Open("mysql", ConnectionString(serverConfig)+readReplicaDbName, nil)
|
||||
defer conn.Close()
|
||||
|
||||
require.NoError(t, err)
|
||||
sess := conn.NewSession(nil)
|
||||
|
||||
// TODO: why doesn't this throw a "no common ancestor" error?
|
||||
t.Run("push common new commit", func(t *testing.T) {
|
||||
var res []string
|
||||
replicatedTable := "new_table"
|
||||
multiSetup.CreateTable(t, sourceDbName, replicatedTable)
|
||||
multiSetup.AddAll(t, sourceDbName)
|
||||
multiSetup.CreateTable(sourceDbName, replicatedTable)
|
||||
multiSetup.StageAll(sourceDbName)
|
||||
_ = multiSetup.CommitWithWorkingSet(sourceDbName)
|
||||
multiSetup.PushToRemote(t, sourceDbName)
|
||||
multiSetup.PushToRemote(sourceDbName, "remote1")
|
||||
|
||||
q := sess.SelectBySql("show tables")
|
||||
_, err := q.LoadContext(context.Background(), &res)
|
||||
|
||||
@@ -266,7 +266,7 @@ func runMain() int {
|
||||
|
||||
metricsDisabled := dEnv.Config.GetStringOrDefault(env.MetricsDisabled, "false")
|
||||
|
||||
disabled, err := strconv.ParseBool(*metricsDisabled)
|
||||
disabled, err := strconv.ParseBool(metricsDisabled)
|
||||
if err != nil {
|
||||
// log.Print(err)
|
||||
return
|
||||
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
"github.com/dolthub/dolt/go/store/datas"
|
||||
)
|
||||
|
||||
const BackupToRemoteKey = "DOLT_BACKUP_TO_REMOTE"
|
||||
const BackupToRemoteKey = "dolt_backup_to_remote"
|
||||
|
||||
type ReplicateHook struct {
|
||||
destDB datas.Database
|
||||
|
||||
@@ -31,6 +31,8 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const defaultBranch = "main"
|
||||
|
||||
func TestReplicateHook(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -123,12 +125,12 @@ func TestReplicateHook(t *testing.T) {
|
||||
ddb.SetCommitHooks(ctx, []datas.CommitHook{hook})
|
||||
|
||||
t.Run("replicate to backup remote", func(t *testing.T) {
|
||||
srcCommit, err := ddb.Commit(context.Background(), valHash, ref.NewBranchRef("master"), meta)
|
||||
ds, err := ddb.db.GetDataset(ctx, "refs/heads/master")
|
||||
srcCommit, err := ddb.Commit(context.Background(), valHash, ref.NewBranchRef(defaultBranch), meta)
|
||||
ds, err := ddb.db.GetDataset(ctx, "refs/heads/main")
|
||||
err = hook.Execute(ctx, ds, ddb.db)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cs, _ = NewCommitSpec("master")
|
||||
cs, _ = NewCommitSpec(defaultBranch)
|
||||
destCommit, err := destDB.Resolve(context.Background(), cs, nil)
|
||||
|
||||
srcHash, _ := srcCommit.HashOf()
|
||||
|
||||
@@ -18,13 +18,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/dolthub/dolt/go/cmd/dolt/cli"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/env"
|
||||
@@ -48,6 +46,8 @@ type MultiRepoTestSetup struct {
|
||||
Root string
|
||||
DbPaths map[string]string
|
||||
Home string
|
||||
Remotes map[string]env.Remote
|
||||
T *testing.T
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -56,94 +56,146 @@ const (
|
||||
defaultBranch = "main"
|
||||
)
|
||||
|
||||
func CreateMultiEnvWithRemote() *MultiRepoTestSetup {
|
||||
ctx := context.Background()
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer os.Chdir(cwd)
|
||||
|
||||
// TODO this is not a proper builder, dbs need to be added before remotes
|
||||
func NewMultiRepoTestSetup(t *testing.T) *MultiRepoTestSetup {
|
||||
dir, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
homeDir, err := ioutil.TempDir(dir, homePrefix)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
homeProv := func() (string, error) {
|
||||
return homeDir, nil
|
||||
}
|
||||
|
||||
remote, err := ioutil.TempDir(dir, remotePrefix)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
repoCnt := 2
|
||||
mrEnv := env.MultiRepoEnv{}
|
||||
dbs := make(map[string]*doltdb.DoltDB, repoCnt)
|
||||
dbNames := make([]string, repoCnt)
|
||||
dbPaths := make(map[string]string, repoCnt)
|
||||
for i := 0; i < repoCnt; i++ {
|
||||
repo, err := ioutil.TempDir(dir, repoPrefix)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = os.Chdir(repo)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
dbName := filepath.Base(repo)
|
||||
dbPaths[dbName] = repo
|
||||
repoPath := fmt.Sprintf("file://%s", repo)
|
||||
|
||||
// TODO sometimes tempfiles scrubber is racy with tempfolder deleter
|
||||
dEnv := env.Load(context.Background(), homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
||||
if err != nil {
|
||||
panic("Failed to initialize environment:" + err.Error())
|
||||
}
|
||||
cfg, _ := dEnv.Config.GetConfig(env.GlobalConfig)
|
||||
cfg.SetStrings(map[string]string{
|
||||
env.UserNameKey: name,
|
||||
env.UserEmailKey: email,
|
||||
})
|
||||
err = dEnv.InitRepo(context.Background(), types.Format_Default, name, email, defaultBranch)
|
||||
if err != nil {
|
||||
panic("Failed to initialize environment:" + err.Error())
|
||||
}
|
||||
|
||||
ddb, err := doltdb.LoadDoltDB(ctx, types.Format_Default, filepath.Join(repoPath, dbfactory.DoltDir), filesys.LocalFS)
|
||||
if err != nil {
|
||||
panic("Failed to initialize environment:" + err.Error())
|
||||
}
|
||||
|
||||
remotePath := fmt.Sprintf("file://%s", remote)
|
||||
rem := env.NewRemote("remote1", remotePath, nil, dEnv)
|
||||
dEnv.RepoState.AddRemote(rem)
|
||||
dEnv.RepoState.Save(filesys.LocalFS)
|
||||
dEnv = env.Load(context.Background(), homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
||||
|
||||
mrEnv.AddEnv(dbName, dEnv)
|
||||
dbs[dbName] = ddb
|
||||
dbNames[i] = dbName
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return &MultiRepoTestSetup{
|
||||
MrEnv: mrEnv,
|
||||
Remote: fmt.Sprintf("file://%s", remote),
|
||||
DoltDBs: dbs,
|
||||
DbNames: dbNames,
|
||||
MrEnv: env.MultiRepoEnv{},
|
||||
Remotes: make(map[string]env.Remote),
|
||||
DoltDBs: make(map[string]*doltdb.DoltDB, 0),
|
||||
DbNames: make([]string, 0),
|
||||
Root: dir,
|
||||
Home: homeDir,
|
||||
DbPaths: dbPaths,
|
||||
DbPaths: make(map[string]string, 0),
|
||||
T: t,
|
||||
}
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) homeProv() (string, error) {
|
||||
return mr.Home, nil
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) Cleanup(dbName string) {
|
||||
os.RemoveAll(mr.Root)
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) NewDB(dbName string) {
|
||||
ctx := context.Background()
|
||||
|
||||
repo := filepath.Join(mr.Root, dbName)
|
||||
os.Mkdir(repo, os.ModePerm)
|
||||
|
||||
err := os.Chdir(repo)
|
||||
if err != nil {
|
||||
mr.T.Fatal(err)
|
||||
}
|
||||
|
||||
// TODO sometimes tempfiles scrubber is racy with tempfolder deleter
|
||||
dEnv := env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
||||
if err != nil {
|
||||
mr.T.Fatal("Failed to initialize environment:" + err.Error())
|
||||
}
|
||||
cfg, _ := dEnv.Config.GetConfig(env.GlobalConfig)
|
||||
cfg.SetStrings(map[string]string{
|
||||
env.UserNameKey: name,
|
||||
env.UserEmailKey: email,
|
||||
})
|
||||
err = dEnv.InitRepo(context.Background(), types.Format_Default, name, email, defaultBranch)
|
||||
if err != nil {
|
||||
mr.T.Fatal("Failed to initialize environment:" + err.Error())
|
||||
}
|
||||
|
||||
ddb, err := doltdb.LoadDoltDB(ctx, types.Format_Default, doltdb.LocalDirDoltDB, filesys.LocalFS)
|
||||
if err != nil {
|
||||
mr.T.Fatal("Failed to initialize environment:" + err.Error())
|
||||
}
|
||||
|
||||
dEnv = env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
||||
|
||||
mr.MrEnv.AddEnv(dbName, dEnv)
|
||||
mr.DoltDBs[dbName] = ddb
|
||||
mr.DbNames = append(mr.DbNames, dbName)
|
||||
mr.DbPaths[dbName] = repo
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) NewRemote(remoteName string) {
|
||||
remote := filepath.Join(mr.Root, remoteName)
|
||||
os.Mkdir(remote, os.ModePerm)
|
||||
remotePath := fmt.Sprintf("file:///%s", remote)
|
||||
|
||||
dEnv := mr.MrEnv[mr.DbNames[0]]
|
||||
rem := env.NewRemote(remoteName, remotePath, nil, dEnv)
|
||||
|
||||
for _, dEnv := range mr.MrEnv {
|
||||
dEnv.RepoState.AddRemote(rem)
|
||||
dEnv.RepoState.Save(filesys.LocalFS)
|
||||
}
|
||||
|
||||
mr.Remotes[remoteName] = rem
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) CloneDB(fromRemote, dbName string) {
|
||||
ctx := context.Background()
|
||||
cloneDir := filepath.Join(mr.Root, dbName)
|
||||
|
||||
r := mr.GetRemote(fromRemote)
|
||||
srcDB, err := r.GetRemoteDB(ctx, types.Format_Default)
|
||||
|
||||
dEnv := env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
||||
dEnv, err = actions.EnvForClone(ctx, srcDB.Format(), r, cloneDir, dEnv.FS, dEnv.Version, mr.homeProv)
|
||||
if err != nil {
|
||||
mr.T.Fatal(err)
|
||||
}
|
||||
|
||||
err = actions.CloneRemote(ctx, srcDB, r.Name, "", dEnv)
|
||||
if err != nil {
|
||||
mr.T.Fatal(err)
|
||||
}
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
mr.T.Fatal(err)
|
||||
}
|
||||
os.Chdir(cloneDir)
|
||||
defer os.Chdir(wd)
|
||||
ddb, err := doltdb.LoadDoltDB(ctx, types.Format_Default, doltdb.LocalDirDoltDB, filesys.LocalFS)
|
||||
if err != nil {
|
||||
mr.T.Fatal("Failed to initialize environment:" + err.Error())
|
||||
}
|
||||
|
||||
dEnv = env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
||||
|
||||
mr.MrEnv.AddEnv(dbName, dEnv)
|
||||
mr.DoltDBs[dbName] = ddb
|
||||
mr.DbNames = append(mr.DbNames, dbName)
|
||||
mr.DbPaths[dbName] = cloneDir
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) GetRemote(remoteName string) env.Remote {
|
||||
rem, ok := mr.Remotes[remoteName]
|
||||
if !ok {
|
||||
mr.T.Fatal("remote not found")
|
||||
}
|
||||
return rem
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) GetDB(dbName string) *doltdb.DoltDB {
|
||||
db, ok := mr.DoltDBs[dbName]
|
||||
if !ok {
|
||||
mr.T.Fatal("db not found")
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) CommitWithWorkingSet(dbName string) *doltdb.Commit {
|
||||
ctx := context.Background()
|
||||
dEnv, ok := mr.MrEnv[dbName]
|
||||
@@ -187,7 +239,7 @@ func (mr *MultiRepoTestSetup) CommitWithWorkingSet(dbName string) *doltdb.Commit
|
||||
pendingCommit,
|
||||
ws.WithStagedRoot(pendingCommit.Roots.Staged).WithWorkingRoot(pendingCommit.Roots.Working).ClearMerge(),
|
||||
prevHash,
|
||||
dEnv.NewWorkingSetMeta(fmt.Sprintf("Updated by test")),
|
||||
doltdb.TodoWorkingSetMeta(),
|
||||
)
|
||||
if err != nil {
|
||||
panic("couldn't commit: " + err.Error())
|
||||
@@ -195,10 +247,10 @@ func (mr *MultiRepoTestSetup) CommitWithWorkingSet(dbName string) *doltdb.Commit
|
||||
return commit
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) CreateTable(t *testing.T, dbName, tblName string) {
|
||||
func (mr *MultiRepoTestSetup) CreateTable(dbName, tblName string) {
|
||||
dEnv, ok := mr.MrEnv[dbName]
|
||||
if !ok {
|
||||
t.Fatalf("Failed to find db: %s", dbName)
|
||||
mr.T.Fatalf("Failed to find db: %s", dbName)
|
||||
}
|
||||
|
||||
imt, sch := dtestutils.CreateTestDataTable(true)
|
||||
@@ -206,49 +258,49 @@ func (mr *MultiRepoTestSetup) CreateTable(t *testing.T, dbName, tblName string)
|
||||
for i := 0; i < imt.NumRows(); i++ {
|
||||
r, err := imt.GetRow(i)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create table: %s", err.Error())
|
||||
mr.T.Fatalf("Failed to create table: %s", err.Error())
|
||||
}
|
||||
rows[i] = r
|
||||
}
|
||||
dtestutils.CreateTestTable(t, dEnv, tblName, sch, rows...)
|
||||
dtestutils.CreateTestTable(mr.T, dEnv, tblName, sch, rows...)
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) AddAll(t *testing.T, dbName string) {
|
||||
func (mr *MultiRepoTestSetup) StageAll(dbName string) {
|
||||
dEnv, ok := mr.MrEnv[dbName]
|
||||
if !ok {
|
||||
t.Fatalf("Failed to find db: %s", dbName)
|
||||
mr.T.Fatalf("Failed to find db: %s", dbName)
|
||||
}
|
||||
ctx := context.Background()
|
||||
roots, err := dEnv.Roots(ctx)
|
||||
if !ok {
|
||||
t.Fatalf("Failed to get roots: %s", dbName)
|
||||
mr.T.Fatalf("Failed to get roots: %s", dbName)
|
||||
}
|
||||
|
||||
roots, err = actions.StageAllTables(ctx, roots, dEnv.Docs)
|
||||
err = dEnv.UpdateRoots(ctx, roots)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to update roots: %s", dbName)
|
||||
mr.T.Fatalf("Failed to update roots: %s", dbName)
|
||||
}
|
||||
}
|
||||
|
||||
func (mr *MultiRepoTestSetup) PushToRemote(t *testing.T, dbName string) {
|
||||
func (mr *MultiRepoTestSetup) PushToRemote(dbName, remoteName string) {
|
||||
ctx := context.Background()
|
||||
dEnv, ok := mr.MrEnv[dbName]
|
||||
if !ok {
|
||||
t.Fatalf("Failed to find db: %s", dbName)
|
||||
mr.T.Fatalf("Failed to find db: %s", dbName)
|
||||
}
|
||||
|
||||
ap := cli.CreatePushArgParser()
|
||||
apr, err := ap.Parse([]string{"remote1", defaultBranch})
|
||||
apr, err := ap.Parse([]string{remoteName, defaultBranch})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to push remote: %s", err.Error())
|
||||
mr.T.Fatalf("Failed to push remote: %s", err.Error())
|
||||
}
|
||||
opts, err := env.NewParseOpts(ctx, apr, dEnv.RepoStateReader(), dEnv.DoltDB, false, false)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to push remote: %s", err.Error())
|
||||
mr.T.Fatalf("Failed to push remote: %s", err.Error())
|
||||
}
|
||||
err = actions.DoPush(ctx, dEnv.RepoStateReader(), dEnv.RepoStateWriter(), dEnv.DoltDB, dEnv.TempTableFilesDir(), opts, actions.NoopRunProgFuncs, actions.NoopStopProgFuncs)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to push remote: %s", err.Error())
|
||||
mr.T.Fatalf("Failed to push remote: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
257
go/libraries/doltcore/env/actions/clone.go
vendored
Normal file
257
go/libraries/doltcore/env/actions/clone.go
vendored
Normal file
@@ -0,0 +1,257 @@
|
||||
// Copyright 2021 Dolthub, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package actions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/dolthub/dolt/go/cmd/dolt/cli"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/env"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/filesys"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/strhelp"
|
||||
"github.com/dolthub/dolt/go/store/datas"
|
||||
"github.com/dolthub/dolt/go/store/types"
|
||||
)
|
||||
|
||||
var ErrRepositoryExists = errors.New("data repository already exists")
|
||||
var ErrFailedToInitRepo = errors.New("")
|
||||
var ErrFailedToCreateDirectory = errors.New("unable to create directories")
|
||||
var ErrFailedToAccessDir = errors.New("unable to access directories")
|
||||
var ErrFailedToCreateRepoStateWithRemote = errors.New("unable to create repo state with remote")
|
||||
var ErrNoDataAtRemote = errors.New("remote at that url contains no Dolt data")
|
||||
var ErrFailedToListBranches = errors.New("failed to list branches")
|
||||
var ErrFailedToGetBranch = errors.New("could not get branch")
|
||||
var ErrFailedToGetRootValue = errors.New("could not find root value")
|
||||
var ErrFailedToResolveBranchRef = errors.New("could not resole branch ref")
|
||||
var ErrFailedToCreateRemoteRef = errors.New("could not create remote ref")
|
||||
var ErrFailedToDeleteBranch = errors.New("could not delete local branch after clone")
|
||||
var ErrFailedToUpdateDocs = errors.New("failed to update docs on the filesystem")
|
||||
var ErrUserNotFound = errors.New("could not determine user name. run dolt config --global --add user.name")
|
||||
var ErrEmailNotFound = errors.New("could not determine email. run dolt config --global --add user.email")
|
||||
var ErrCloneFailed = errors.New("clone failed")
|
||||
|
||||
func EnvForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, dir string, fs filesys.Filesys, version string, homeProvider env.HomeDirProvider) (*env.DoltEnv, error) {
|
||||
exists, _ := fs.Exists(filepath.Join(dir, dbfactory.DoltDir))
|
||||
|
||||
if exists {
|
||||
return nil, fmt.Errorf("%w: %s", ErrRepositoryExists, dir)
|
||||
}
|
||||
|
||||
err := fs.MkDirs(dir)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s; %s", ErrFailedToCreateDirectory, dir, err.Error())
|
||||
}
|
||||
|
||||
err = os.Chdir(dir)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s; %s", ErrFailedToAccessDir, dir, err.Error())
|
||||
}
|
||||
|
||||
dEnv := env.Load(ctx, homeProvider, fs, doltdb.LocalDirDoltDB, version)
|
||||
err = dEnv.InitRepoWithNoData(ctx, nbf)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w; %s", ErrFailedToInitRepo, err.Error())
|
||||
}
|
||||
|
||||
dEnv.RSLoadErr = nil
|
||||
if !env.IsEmptyRemote(r) {
|
||||
dEnv.RepoState, err = env.CloneRepoState(dEnv.FS, r)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s; %s", ErrFailedToCreateRepoStateWithRemote, r.Name, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return dEnv, nil
|
||||
}
|
||||
|
||||
func cloneProg(eventCh <-chan datas.TableFileEvent) {
|
||||
var (
|
||||
chunks int64
|
||||
chunksDownloading int64
|
||||
chunksDownloaded int64
|
||||
cliPos int
|
||||
)
|
||||
|
||||
cliPos = cli.DeleteAndPrint(cliPos, "Retrieving remote information.")
|
||||
for tblFEvt := range eventCh {
|
||||
switch tblFEvt.EventType {
|
||||
case datas.Listed:
|
||||
for _, tf := range tblFEvt.TableFiles {
|
||||
chunks += int64(tf.NumChunks())
|
||||
}
|
||||
case datas.DownloadStart:
|
||||
for _, tf := range tblFEvt.TableFiles {
|
||||
chunksDownloading += int64(tf.NumChunks())
|
||||
}
|
||||
case datas.DownloadSuccess:
|
||||
for _, tf := range tblFEvt.TableFiles {
|
||||
chunksDownloading -= int64(tf.NumChunks())
|
||||
chunksDownloaded += int64(tf.NumChunks())
|
||||
}
|
||||
case datas.DownloadFailed:
|
||||
// Ignore for now and output errors on the main thread
|
||||
}
|
||||
|
||||
str := fmt.Sprintf("%s of %s chunks complete. %s chunks being downloaded currently.", strhelp.CommaIfy(chunksDownloaded), strhelp.CommaIfy(chunks), strhelp.CommaIfy(chunksDownloading))
|
||||
cliPos = cli.DeleteAndPrint(cliPos, str)
|
||||
}
|
||||
|
||||
cli.Println()
|
||||
}
|
||||
|
||||
func CloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch string, dEnv *env.DoltEnv) error {
|
||||
eventCh := make(chan datas.TableFileEvent, 128)
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cloneProg(eventCh)
|
||||
}()
|
||||
|
||||
err := Clone(ctx, srcDB, dEnv.DoltDB, eventCh)
|
||||
close(eventCh)
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if err != nil {
|
||||
if err == datas.ErrNoData {
|
||||
err = ErrNoDataAtRemote
|
||||
}
|
||||
return fmt.Errorf("%w; %s", ErrCloneFailed, err.Error())
|
||||
}
|
||||
|
||||
branches, err := dEnv.DoltDB.GetBranches(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w; %s", ErrFailedToListBranches, err.Error())
|
||||
}
|
||||
|
||||
if branch == "" {
|
||||
branch = env.GetDefaultBranch(dEnv, branches)
|
||||
}
|
||||
|
||||
// If we couldn't find a branch but the repo cloned successfully, it's empty. Initialize it instead of pulling from
|
||||
// the remote.
|
||||
performPull := true
|
||||
if branch == "" {
|
||||
err = InitEmptyClonedRepo(ctx, dEnv)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
branch = env.GetDefaultInitBranch(dEnv.Config)
|
||||
performPull = false
|
||||
}
|
||||
|
||||
cs, _ := doltdb.NewCommitSpec(branch)
|
||||
cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s; %s", ErrFailedToGetBranch, branch, err.Error())
|
||||
|
||||
}
|
||||
|
||||
rootVal, err := cm.GetRootValue()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s; %s", ErrFailedToGetRootValue, branch, err.Error())
|
||||
}
|
||||
|
||||
// After actions.Clone, we have repository with a local branch for
|
||||
// every branch in the remote. What we want is a remote branch ref for
|
||||
// every branch in the remote. We iterate through local branches and
|
||||
// create remote refs corresponding to each of them. We delete all of
|
||||
// the local branches except for the one corresponding to |branch|.
|
||||
for _, brnch := range branches {
|
||||
cs, _ := doltdb.NewCommitSpec(brnch.GetPath())
|
||||
cm, err := dEnv.DoltDB.Resolve(ctx, cs, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s; %s", ErrFailedToResolveBranchRef, brnch.String(), err.Error())
|
||||
|
||||
}
|
||||
|
||||
remoteRef := ref.NewRemoteRef(remoteName, brnch.GetPath())
|
||||
err = dEnv.DoltDB.SetHeadToCommit(ctx, remoteRef, cm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s; %s", ErrFailedToCreateRemoteRef, remoteRef.String(), err.Error())
|
||||
|
||||
}
|
||||
|
||||
if brnch.GetPath() != branch {
|
||||
err := dEnv.DoltDB.DeleteBranch(ctx, brnch)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s; %s", ErrFailedToDeleteBranch, brnch.String(), err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if performPull {
|
||||
err = SaveDocsFromRoot(ctx, rootVal, dEnv)
|
||||
if err != nil {
|
||||
return ErrFailedToUpdateDocs
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: make this interface take a DoltRef and marshal it automatically
|
||||
err = dEnv.RepoStateWriter().SetCWBHeadRef(ctx, ref.MarshalableRef{Ref: ref.NewBranchRef(branch)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
wsRef, err := ref.WorkingSetRefForHead(ref.NewBranchRef(branch))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ws := doltdb.EmptyWorkingSet(wsRef)
|
||||
err = dEnv.UpdateWorkingSet(ctx, ws.WithWorkingRoot(rootVal).WithStagedRoot(rootVal))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Inits an empty, newly cloned repo. This would be unnecessary if we properly initialized the storage for a repository
|
||||
// when we created it on dolthub. If we do that, this code can be removed.
|
||||
func InitEmptyClonedRepo(ctx context.Context, dEnv *env.DoltEnv) error {
|
||||
name := dEnv.Config.GetStringOrDefault(env.UserNameKey, "")
|
||||
email := dEnv.Config.GetStringOrDefault(env.UserEmailKey, "")
|
||||
initBranch := env.GetDefaultInitBranch(dEnv.Config)
|
||||
|
||||
if name == "" {
|
||||
return ErrUserNotFound
|
||||
} else if email == "" {
|
||||
return ErrEmailNotFound
|
||||
}
|
||||
|
||||
err := dEnv.InitDBWithTime(ctx, types.Format_Default, name, email, initBranch, doltdb.CommitNowFunc())
|
||||
if err != nil {
|
||||
return ErrFailedToInitRepo
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
10
go/libraries/doltcore/env/config.go
vendored
10
go/libraries/doltcore/env/config.go
vendored
@@ -86,6 +86,8 @@ type DoltCliConfig struct {
|
||||
fs filesys.ReadWriteFS
|
||||
}
|
||||
|
||||
var _ config.ReadableConfig = &DoltCliConfig{}
|
||||
|
||||
func loadDoltCliConfig(hdp HomeDirProvider, fs filesys.ReadWriteFS) (*DoltCliConfig, error) {
|
||||
ch := config.NewConfigHierarchy()
|
||||
|
||||
@@ -159,7 +161,7 @@ func (dcc *DoltCliConfig) GetConfig(element DoltConfigElement) (config.ReadWrite
|
||||
|
||||
// GetStringOrDefault retrieves a string from the config hierarchy and returns it if available. Otherwise it returns
|
||||
// the default string value
|
||||
func (dcc *DoltCliConfig) GetStringOrDefault(key, defStr string) *string {
|
||||
func (dcc *DoltCliConfig) GetStringOrDefault(key, defStr string) string {
|
||||
return GetStringOrDefault(dcc.ch, key, defStr)
|
||||
}
|
||||
|
||||
@@ -180,14 +182,14 @@ func (dcc *DoltCliConfig) IfEmptyUseConfig(val, key string) string {
|
||||
return cfgVal
|
||||
}
|
||||
|
||||
func GetStringOrDefault(cfg config.ReadableConfig, key, defStr string) *string {
|
||||
func GetStringOrDefault(cfg config.ReadableConfig, key, defStr string) string {
|
||||
val, err := cfg.GetString(key)
|
||||
|
||||
if err != nil {
|
||||
return &defStr
|
||||
return defStr
|
||||
}
|
||||
|
||||
return &val
|
||||
return val
|
||||
}
|
||||
|
||||
// GetNameAndEmail returns the name and email from the supplied config
|
||||
|
||||
4
go/libraries/doltcore/env/config_test.go
vendored
4
go/libraries/doltcore/env/config_test.go
vendored
@@ -30,11 +30,11 @@ func TestConfig(t *testing.T) {
|
||||
lCfg.SetStrings(map[string]string{UserEmailKey: email})
|
||||
gCfg.SetStrings(map[string]string{UserNameKey: name})
|
||||
|
||||
if *dEnv.Config.GetStringOrDefault(UserEmailKey, "no") != email {
|
||||
if dEnv.Config.GetStringOrDefault(UserEmailKey, "no") != email {
|
||||
t.Error("Should return", email)
|
||||
}
|
||||
|
||||
if *dEnv.Config.GetStringOrDefault("bad_key", "yes") != "yes" {
|
||||
if dEnv.Config.GetStringOrDefault("bad_key", "yes") != "yes" {
|
||||
t.Error("Should return default value of yes")
|
||||
}
|
||||
|
||||
|
||||
12
go/libraries/doltcore/env/environment.go
vendored
12
go/libraries/doltcore/env/environment.go
vendored
@@ -19,7 +19,6 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
@@ -61,7 +60,7 @@ const (
|
||||
func getCommitHooks(ctx context.Context, dEnv *DoltEnv) ([]datas.CommitHook, error) {
|
||||
postCommitHooks := make([]datas.CommitHook, 0)
|
||||
|
||||
backupName := os.Getenv(doltdb.BackupToRemoteKey)
|
||||
backupName := dEnv.Config.GetStringOrDefault(doltdb.BackupToRemoteKey, "")
|
||||
if backupName != "" {
|
||||
remotes, err := dEnv.GetRemotes()
|
||||
if err != nil {
|
||||
@@ -161,7 +160,7 @@ func Load(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, urlStr,
|
||||
// The process will not wait for this to finish so this may not always complete.
|
||||
go func() {
|
||||
// TODO dEnv.HasDoltTempTableDir() true but dEnv.TempTableFileDir() panics
|
||||
tmpTableDir, err := dEnv.FS.Abs(filepath.Join(dEnv.urlStr, dEnv.GetDoltDir(), tempTablesDir))
|
||||
tmpTableDir, err := dEnv.FS.Abs(filepath.Join(dEnv.urlStr, dbfactory.DoltDir, tempTablesDir))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -206,8 +205,7 @@ func Load(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, urlStr,
|
||||
}
|
||||
|
||||
func GetDefaultInitBranch(cfg config.ReadableConfig) string {
|
||||
s := GetStringOrDefault(cfg, InitBranchName, DefaultInitBranch)
|
||||
return *s
|
||||
return GetStringOrDefault(cfg, InitBranchName, DefaultInitBranch)
|
||||
}
|
||||
|
||||
// initWorkingSetFromRepoState sets the working set for the env's head to mirror the contents of the repo state file.
|
||||
@@ -758,8 +756,8 @@ func (dEnv *DoltEnv) workingSetMeta() *doltdb.WorkingSetMeta {
|
||||
|
||||
func (dEnv *DoltEnv) NewWorkingSetMeta(message string) *doltdb.WorkingSetMeta {
|
||||
return &doltdb.WorkingSetMeta{
|
||||
User: *dEnv.Config.GetStringOrDefault(UserNameKey, ""),
|
||||
Email: *dEnv.Config.GetStringOrDefault(UserEmailKey, ""),
|
||||
User: dEnv.Config.GetStringOrDefault(UserNameKey, ""),
|
||||
Email: dEnv.Config.GetStringOrDefault(UserEmailKey, ""),
|
||||
Timestamp: uint64(time.Now().Unix()),
|
||||
Description: message,
|
||||
}
|
||||
|
||||
34
go/libraries/doltcore/env/remotes.go
vendored
34
go/libraries/doltcore/env/remotes.go
vendored
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
|
||||
@@ -460,3 +461,36 @@ func getAbsFileRemoteUrl(urlStr string, fs filesys2.Filesys) (string, error) {
|
||||
}
|
||||
return dbfactory.FileScheme + "://" + urlStr, nil
|
||||
}
|
||||
|
||||
// GetDefaultBranch returns the default branch from among the branches given, returning
|
||||
// the configs default config branch first, then init branch main, then the old init branch master,
|
||||
// and finally the first lexicographical branch if none of the others are found
|
||||
func GetDefaultBranch(dEnv *DoltEnv, branches []ref.DoltRef) string {
|
||||
if len(branches) == 0 {
|
||||
return DefaultInitBranch
|
||||
}
|
||||
|
||||
sort.Slice(branches, func(i, j int) bool {
|
||||
return branches[i].GetPath() < branches[j].GetPath()
|
||||
})
|
||||
|
||||
branchMap := make(map[string]ref.DoltRef)
|
||||
for _, b := range branches {
|
||||
branchMap[b.GetPath()] = b
|
||||
}
|
||||
|
||||
if _, ok := branchMap[DefaultInitBranch]; ok {
|
||||
return DefaultInitBranch
|
||||
}
|
||||
if _, ok := branchMap["master"]; ok {
|
||||
return "master"
|
||||
}
|
||||
|
||||
// todo: do we care about this during clone?
|
||||
defaultOrMain := GetDefaultInitBranch(dEnv.Config)
|
||||
if _, ok := branchMap[defaultOrMain]; ok {
|
||||
return defaultOrMain
|
||||
}
|
||||
|
||||
return branches[0].GetPath()
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ var ErrInvalidTableName = errors.NewKind("Invalid table name %s. Table names mus
|
||||
var ErrReservedTableName = errors.NewKind("Invalid table name %s. Table names beginning with `dolt_` are reserved for internal use")
|
||||
var ErrSystemTableAlter = errors.NewKind("Cannot alter table %s: system tables cannot be dropped or altered")
|
||||
|
||||
const DoltReadReplicaKey = "DOLT_READ_REPLICA_REMOTE"
|
||||
const DoltReadReplicaKey = "dolt_read_replica_remote"
|
||||
|
||||
type SqlDatabase interface {
|
||||
sql.Database
|
||||
@@ -68,8 +68,6 @@ func DbsAsDSQLDBs(dbs []sql.Database) []SqlDatabase {
|
||||
continue
|
||||
}
|
||||
switch v := sqlDb.(type) {
|
||||
//case ReadReplicaDatabase, *ReadReplicaDatabase:
|
||||
// dsqlDBs = append(dsqlDBs, v)
|
||||
case ReadReplicaDatabase, Database:
|
||||
dsqlDBs = append(dsqlDBs, v)
|
||||
default:
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
|
||||
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
|
||||
"github.com/dolthub/dolt/go/libraries/utils/config"
|
||||
"github.com/dolthub/dolt/go/store/hash"
|
||||
)
|
||||
|
||||
@@ -42,6 +43,7 @@ const (
|
||||
DoltCommitOnTransactionCommit = "dolt_transaction_commit"
|
||||
TransactionsDisabledSysVar = "dolt_transactions_disabled"
|
||||
ForceTransactionCommit = "dolt_force_transaction_commit"
|
||||
DoltDefaultBranchKey = "dolt_default_branch"
|
||||
)
|
||||
|
||||
var transactionMergeStomp = false
|
||||
@@ -133,6 +135,7 @@ type Session struct {
|
||||
BatchMode batchMode
|
||||
Username string
|
||||
Email string
|
||||
Config config.ReadableConfig
|
||||
dbStates map[string]*DatabaseSessionState
|
||||
provider RevisionDatabaseProvider
|
||||
}
|
||||
@@ -194,11 +197,15 @@ type InitialDbState struct {
|
||||
}
|
||||
|
||||
// NewSession creates a Session object from a standard sql.Session and 0 or more Database objects.
|
||||
func NewSession(ctx *sql.Context, sqlSess sql.Session, pro RevisionDatabaseProvider, username, email string, dbs ...InitialDbState) (*Session, error) {
|
||||
func NewSession(ctx *sql.Context, sqlSess sql.Session, pro RevisionDatabaseProvider, conf config.ReadableConfig, dbs ...InitialDbState) (*Session, error) {
|
||||
username := conf.GetStringOrDefault(env.UserNameKey, "")
|
||||
email := conf.GetStringOrDefault(env.UserEmailKey, "")
|
||||
|
||||
sess := &Session{
|
||||
Session: sqlSess,
|
||||
Username: username,
|
||||
Email: email,
|
||||
Config: conf,
|
||||
dbStates: make(map[string]*DatabaseSessionState),
|
||||
provider: pro,
|
||||
}
|
||||
@@ -1207,6 +1214,14 @@ func defineSystemVariables(name string) {
|
||||
Type: sql.NewSystemStringType(StagedKey(name)),
|
||||
Default: "",
|
||||
},
|
||||
{
|
||||
Name: DoltDefaultBranchKey,
|
||||
Scope: sql.SystemVariableScope_Global,
|
||||
Dynamic: true,
|
||||
SetVarHintApplies: false,
|
||||
Type: sql.NewSystemStringType(DoltDefaultBranchKey),
|
||||
Default: "",
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ var _ enginetest.ReadOnlyDatabaseHarness = (*DoltHarness)(nil)
|
||||
func newDoltHarness(t *testing.T) *DoltHarness {
|
||||
dEnv := dtestutils.CreateTestEnv()
|
||||
pro := sqle.NewDoltDatabaseProvider(dEnv.Config)
|
||||
session, err := dsess.NewSession(sql.NewEmptyContext(), enginetest.NewBaseSession(), pro, "test", "email@test.com")
|
||||
session, err := dsess.NewSession(sql.NewEmptyContext(), enginetest.NewBaseSession(), pro, dEnv.Config)
|
||||
require.NoError(t, err)
|
||||
return &DoltHarness{
|
||||
t: t,
|
||||
@@ -137,7 +137,7 @@ func (d *DoltHarness) NewSession() *sql.Context {
|
||||
enginetest.NewContext(d),
|
||||
enginetest.NewBaseSession(),
|
||||
pro.(dsess.RevisionDatabaseProvider),
|
||||
user, email,
|
||||
d.env.Config,
|
||||
states...,
|
||||
)
|
||||
require.NoError(d.t, err)
|
||||
|
||||
@@ -30,6 +30,10 @@ type ReadableConfig interface {
|
||||
// ReadableConfig implementation.
|
||||
GetString(key string) (value string, err error)
|
||||
|
||||
// GetStringOrDefault retrieves a string from the config hierarchy and returns it if available. Otherwise it returns
|
||||
// the default string value
|
||||
GetStringOrDefault(key, defStr string) string
|
||||
|
||||
// Iter will perform a callback for each value in a config until all values have been exhausted or until the
|
||||
// callback returns true indicating that it should stop.
|
||||
Iter(func(string, string) (stop bool))
|
||||
|
||||
@@ -31,6 +31,10 @@ type ConfigHierarchy struct {
|
||||
nameToConfig map[string]ReadWriteConfig
|
||||
}
|
||||
|
||||
var _ ReadableConfig = &ConfigHierarchy{}
|
||||
var _ WritableConfig = &ConfigHierarchy{}
|
||||
var _ ReadWriteConfig = &ConfigHierarchy{}
|
||||
|
||||
// NewConfigHierarchy creates an empty ConfigurationHierarchy
|
||||
func NewConfigHierarchy() *ConfigHierarchy {
|
||||
return &ConfigHierarchy{[]ReadWriteConfig{}, map[string]ReadWriteConfig{}}
|
||||
@@ -84,6 +88,13 @@ func (ch *ConfigHierarchy) GetString(k string) (string, error) {
|
||||
return "", ErrConfigParamNotFound
|
||||
}
|
||||
|
||||
func (ch *ConfigHierarchy) GetStringOrDefault(key, defStr string) string {
|
||||
if val, err := ch.GetString(key); err == nil {
|
||||
return val
|
||||
}
|
||||
return defStr
|
||||
}
|
||||
|
||||
// SetStrings will set the value of configuration parameters in memory, and persist any changes to the backing file.
|
||||
// For ConfigHierarchies update parameter names must be of the format config_name::param_name
|
||||
func (ch *ConfigHierarchy) SetStrings(updates map[string]string) error {
|
||||
|
||||
@@ -31,6 +31,10 @@ type FileConfig struct {
|
||||
properties map[string]string
|
||||
}
|
||||
|
||||
var _ ReadableConfig = &FileConfig{}
|
||||
var _ WritableConfig = &FileConfig{}
|
||||
var _ ReadWriteConfig = &FileConfig{}
|
||||
|
||||
// NewFileConfig creates a new empty config and writes it to a newly created file. If a file already exists at this
|
||||
// location it will be overwritten. If a directory does not exist where this file should live, it will be created.
|
||||
func NewFileConfig(path string, fs filesys.ReadWriteFS, properties map[string]string) (*FileConfig, error) {
|
||||
@@ -79,6 +83,15 @@ func (fc *FileConfig) GetString(k string) (string, error) {
|
||||
return "", ErrConfigParamNotFound
|
||||
}
|
||||
|
||||
// GetString retrieves a string from the cached config state
|
||||
func (fc *FileConfig) GetStringOrDefault(k, defStr string) string {
|
||||
if val, ok := fc.properties[k]; ok {
|
||||
return val
|
||||
}
|
||||
|
||||
return defStr
|
||||
}
|
||||
|
||||
// SetStrings will set the value of configuration parameters in memory, and persist any changes to the backing file.
|
||||
func (fc *FileConfig) SetStrings(updates map[string]string) error {
|
||||
modified := false
|
||||
|
||||
@@ -21,6 +21,10 @@ type MapConfig struct {
|
||||
properties map[string]string
|
||||
}
|
||||
|
||||
var _ ReadableConfig = &MapConfig{}
|
||||
var _ WritableConfig = &MapConfig{}
|
||||
var _ ReadWriteConfig = &MapConfig{}
|
||||
|
||||
// NewMapConfig creates a config from a map.
|
||||
func NewMapConfig(properties map[string]string) *MapConfig {
|
||||
return &MapConfig{properties}
|
||||
@@ -35,6 +39,13 @@ func (mc *MapConfig) GetString(k string) (string, error) {
|
||||
return "", ErrConfigParamNotFound
|
||||
}
|
||||
|
||||
func (mc *MapConfig) GetStringOrDefault(key, defStr string) string {
|
||||
if val, err := mc.GetString(key); err == nil {
|
||||
return val
|
||||
}
|
||||
return defStr
|
||||
}
|
||||
|
||||
// SetString sets the values for a map of updates.
|
||||
func (mc *MapConfig) SetStrings(updates map[string]string) error {
|
||||
for k, v := range updates {
|
||||
|
||||
@@ -55,8 +55,8 @@ skip_if_no_aws_tests() {
|
||||
random_repo=`openssl rand -hex 32`
|
||||
run dolt clone 'aws://['"$DOLT_BATS_AWS_TABLE"':'"$DOLT_BATS_AWS_BUCKET"']/'"$random_repo"
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" =~ "error: clone failed" ]] || false
|
||||
[[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false
|
||||
[[ "$output" =~ "clone failed" ]] || false
|
||||
[[ "$output" =~ "remote at that url contains no Dolt data" ]] || false
|
||||
}
|
||||
|
||||
@test "aws-remotes: can push to new remote" {
|
||||
|
||||
@@ -335,8 +335,8 @@ SQL
|
||||
@test "remotes: clone an empty remote" {
|
||||
run dolt clone http://localhost:50051/test-org/empty
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" =~ "error: clone failed" ]] || false
|
||||
[[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false
|
||||
[[ "$output" =~ "clone failed" ]] || false
|
||||
[[ "$output" =~ "remote at that url contains no Dolt data" ]] || false
|
||||
}
|
||||
|
||||
@test "remotes: clone a non-existent remote" {
|
||||
@@ -344,8 +344,8 @@ SQL
|
||||
cd "dolt-repo-clones"
|
||||
run dolt clone http://localhost:50051/foo/bar
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" =~ "error: clone failed" ]] || false
|
||||
[[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false
|
||||
[[ "$output" =~ "clone failed" ]] || false
|
||||
[[ "$output" =~ "remote at that url contains no Dolt data" ]] || false
|
||||
}
|
||||
|
||||
@test "remotes: clone a different branch than main" {
|
||||
@@ -1142,7 +1142,7 @@ setup_ref_test() {
|
||||
|
||||
run dolt clone "file://../clone_root" .
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" =~ "error: clone failed" ]] || false
|
||||
[[ "$output" =~ "clone failed" ]] || false
|
||||
|
||||
# Validates that the directory exists
|
||||
run ls $testdir
|
||||
@@ -1158,7 +1158,7 @@ setup_ref_test() {
|
||||
cd ..
|
||||
run dolt clone "file://./clone_root" dest/
|
||||
[ "$status" -eq 1 ]
|
||||
[[ "$output" =~ "error: clone failed" ]] || false
|
||||
[[ "$output" =~ "clone failed" ]] || false
|
||||
|
||||
run ls $testdir
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
@@ -33,15 +33,15 @@ teardown() {
|
||||
}
|
||||
|
||||
@test "replication: push on commit" {
|
||||
export DOLT_BACKUP_TO_REMOTE=backup1
|
||||
cd repo1
|
||||
dolt config --local --add DOLT_BACKUP_TO_REMOTE backup1
|
||||
dolt config --list
|
||||
dolt remote -v
|
||||
dolt sql -q "create table t1 (a int primary key)"
|
||||
dolt commit -am "cm"
|
||||
|
||||
cd ..
|
||||
dolt clone file://./bac1 repo2
|
||||
export DOLT_BACKUP_TO_REMOTE=
|
||||
cd repo2
|
||||
run dolt ls
|
||||
[ "$status" -eq 0 ]
|
||||
@@ -50,8 +50,8 @@ teardown() {
|
||||
}
|
||||
|
||||
@test "replication: no tags" {
|
||||
export DOLT_BACKUP_TO_REMOTE=backup1
|
||||
cd repo1
|
||||
dolt config --local --add DOLT_BACKUP_TO_REMOTE backup1
|
||||
dolt tag
|
||||
|
||||
[ ! -d "../bac1/.dolt" ] || false
|
||||
@@ -70,7 +70,7 @@ teardown() {
|
||||
[ "${#lines[@]}" -eq 1 ]
|
||||
[[ ! "$output" =~ "t1" ]] || false
|
||||
|
||||
export DOLT_READ_REPLICA_REMOTE=remote1
|
||||
dolt config --local --add DOLT_READ_REPLICA_REMOTE remote1
|
||||
run dolt sql -q "show tables" -r csv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "${#lines[@]}" -eq 2 ]
|
||||
|
||||
@@ -1055,7 +1055,7 @@ while True:
|
||||
mkdir bac1
|
||||
cd repo1
|
||||
dolt remote add backup1 file://../bac1
|
||||
export DOLT_BACKUP_TO_REMOTE=backup1
|
||||
dolt config --local --add DOLT_BACKUP_TO_REMOTE backup1
|
||||
start_sql_server repo1
|
||||
|
||||
multi_query repo1 1 "
|
||||
@@ -1098,7 +1098,7 @@ while True:
|
||||
dolt push -u remote1 main
|
||||
|
||||
cd ../repo1
|
||||
export DOLT_READ_REPLICA_REMOTE=remote1
|
||||
dolt config --local --add DOLT_READ_REPLICA_REMOTE remote1
|
||||
start_sql_server repo1
|
||||
|
||||
server_query repo1 1 "show tables" "Table\ntest"
|
||||
|
||||
Reference in New Issue
Block a user