diff --git a/go/cmd/dolt/commands/clone.go b/go/cmd/dolt/commands/clone.go index 686748e16e..5009ca6389 100644 --- a/go/cmd/dolt/commands/clone.go +++ b/go/cmd/dolt/commands/clone.go @@ -49,6 +49,18 @@ This default configuration is achieved by creating references to the remote bran }, } +type remoteDialerWithGitCacheRoot struct { + dbfactory.GRPCDialProvider + root string +} + +func (d remoteDialerWithGitCacheRoot) GitCacheRoot() (string, bool) { + if strings.TrimSpace(d.root) == "" { + return "", false + } + return d.root, true +} + type CloneCmd struct{} // Name is returns the name of the Dolt cli command. This is what is used on the command line to invoke the command @@ -130,7 +142,11 @@ func clone(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env.DoltEn var r env.Remote var srcDB *doltdb.DoltDB - r, srcDB, verr = createRemote(ctx, remoteName, remoteUrl, params, dEnv) + cloneRoot, err := dEnv.FS.Abs(dir) + if err != nil { + return errhand.VerboseErrorFromError(err) + } + r, srcDB, verr = createRemote(ctx, remoteName, remoteUrl, params, dEnv, cloneRoot) if verr != nil { return verr } @@ -211,11 +227,15 @@ func parseArgs(apr *argparser.ArgParseResults) (string, string, errhand.VerboseE return dir, urlStr, nil } -func createRemote(ctx context.Context, remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv) (env.Remote, *doltdb.DoltDB, errhand.VerboseError) { +func createRemote(ctx context.Context, remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv, cloneRoot string) (env.Remote, *doltdb.DoltDB, errhand.VerboseError) { cli.Printf("cloning %s\n", remoteUrl) r := env.NewRemote(remoteName, remoteUrl, params) - ddb, err := r.GetRemoteDB(ctx, types.Format_Default, dEnv) + dialer := dbfactory.GRPCDialProvider(dEnv) + if strings.TrimSpace(cloneRoot) != "" { + dialer = remoteDialerWithGitCacheRoot{GRPCDialProvider: dEnv, root: cloneRoot} + } + ddb, err := r.GetRemoteDB(ctx, types.Format_Default, dialer) if err != nil { bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err) return env.NoRemote, nil, bdr.Build() diff --git a/go/libraries/doltcore/dbfactory/git_remote.go b/go/libraries/doltcore/dbfactory/git_remote.go index 75a45a3787..b4ef58c4d6 100644 --- a/go/libraries/doltcore/dbfactory/git_remote.go +++ b/go/libraries/doltcore/dbfactory/git_remote.go @@ -34,12 +34,21 @@ import ( ) const ( + // GitCacheRootParam is the absolute path to the local Dolt repository root (the directory that contains `.dolt/`). + // When set for git remotes, callers can choose a per-repo cache location under `.dolt/`. + GitCacheRootParam = "git_cache_root" GitRefParam = "git_ref" GitRemoteNameParam = "git_remote_name" defaultGitRef = "refs/dolt/data" defaultGitRemoteName = "origin" ) +// GitCacheRootProvider provides the local Dolt repo root for per-repo git remote caches. +// Implementations should return ok=false when no repo root is available. +type GitCacheRootProvider interface { + GitCacheRoot() (string, bool) +} + // GitRemoteFactory opens a Dolt database backed by a Git remote, using a local bare // repository as an object cache and remote configuration store. // @@ -83,10 +92,14 @@ func (fact GitRemoteFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFor return nil, nil, nil, err } - cacheBase, err := defaultGitCacheBase() + cacheRoot, ok, err := resolveGitCacheRoot(params) if err != nil { return nil, nil, nil, err } + if !ok { + return nil, nil, nil, fmt.Errorf("%s is required for git remotes", GitCacheRootParam) + } + cacheBase := filepath.Join(cacheRoot, DoltDir, "git-remote-cache") cacheRepo, err := cacheRepoPath(cacheBase, remoteURL.String(), ref) if err != nil { @@ -166,12 +179,24 @@ func resolveGitRemoteName(params map[string]interface{}) string { return defaultGitRemoteName } -func defaultGitCacheBase() (string, error) { - base, err := os.UserCacheDir() - if err != nil { - return "", err +// resolveGitCacheRoot parses and validates the optional GitCacheRootParam. +// It returns ok=false when the param is not present. +func resolveGitCacheRoot(params map[string]interface{}) (root string, ok bool, err error) { + if params == nil { + return "", false, nil } - return filepath.Join(base, "dolt", "git-remote-cache"), nil + v, ok := params[GitCacheRootParam] + if !ok || v == nil { + return "", false, nil + } + s, ok := v.(string) + if !ok { + return "", false, fmt.Errorf("%s must be a string", GitCacheRootParam) + } + if strings.TrimSpace(s) == "" { + return "", false, fmt.Errorf("%s cannot be empty", GitCacheRootParam) + } + return s, true, nil } func cacheRepoPath(cacheBase, remoteURL, ref string) (string, error) { diff --git a/go/libraries/doltcore/dbfactory/git_remote_test.go b/go/libraries/doltcore/dbfactory/git_remote_test.go index e0dbf93ed2..a46c538710 100644 --- a/go/libraries/doltcore/dbfactory/git_remote_test.go +++ b/go/libraries/doltcore/dbfactory/git_remote_test.go @@ -49,7 +49,14 @@ func shortTempDir(t *testing.T) string { return dir } -func TestGitRemoteFactory_GitFile_UsesDefaultCacheDirAndCanWrite(t *testing.T) { +func TestGitRemoteFactory_GitFile_RequiresGitCacheRootParam(t *testing.T) { + ctx := context.Background() + _, _, _, err := CreateDB(ctx, types.Format_Default, "git+file:///tmp/remote.git", map[string]interface{}{}) + require.Error(t, err) + require.Contains(t, err.Error(), GitCacheRootParam) +} + +func TestGitRemoteFactory_GitFile_CachesUnderRepoDoltDirAndCanWrite(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git not found on PATH") } @@ -58,20 +65,22 @@ func TestGitRemoteFactory_GitFile_UsesDefaultCacheDirAndCanWrite(t *testing.T) { remoteRepo, err := gitrepo.InitBare(ctx, filepath.Join(shortTempDir(t), "remote.git")) require.NoError(t, err) + localRepoRoot := shortTempDir(t) + remotePath := filepath.ToSlash(remoteRepo.GitDir) remoteURL := "file://" + remotePath urlStr := "git+file://" + remotePath - params := map[string]interface{}{} + params := map[string]interface{}{ + GitCacheRootParam: localRepoRoot, + } db, vrw, _, err := CreateDB(ctx, types.Format_Default, urlStr, params) require.NoError(t, err) require.NotNil(t, db) require.NotNil(t, vrw) - // Ensure cache repo created under default cache dir. - base, err := os.UserCacheDir() - require.NoError(t, err) - cacheBase := filepath.Join(base, "dolt", "git-remote-cache") + // Ensure cache repo created under /.dolt/git-remote-cache. + cacheBase := filepath.Join(localRepoRoot, DoltDir, "git-remote-cache") sum := sha256.Sum256([]byte(remoteURL + "|" + "refs/dolt/data")) h := hex.EncodeToString(sum[:]) @@ -120,8 +129,10 @@ func TestGitRemoteFactory_TwoClientsDistinctCacheDirsRoundtrip(t *testing.T) { return func(context.Context, hash.HashSet, chunks.PendingRefExists) error { return nil } } - open := func() (db datas.Database, cs chunks.ChunkStore) { - params := map[string]interface{}{} + open := func(cacheRoot string) (db datas.Database, cs chunks.ChunkStore) { + params := map[string]interface{}{ + GitCacheRootParam: cacheRoot, + } d, vrw, _, err := CreateDB(ctx, types.Format_Default, urlStr, params) require.NoError(t, err) require.NotNil(t, d) @@ -132,8 +143,11 @@ func TestGitRemoteFactory_TwoClientsDistinctCacheDirsRoundtrip(t *testing.T) { return d, vs.ChunkStore() } + cacheA := shortTempDir(t) + cacheB := shortTempDir(t) + // Client A writes a root pointing at chunk A. - dbA, csA := open() + dbA, csA := open(cacheA) cA := chunks.NewChunk([]byte("clientA\n")) require.NoError(t, csA.Put(ctx, cA, noopGetAddrs)) lastA, err := csA.Root(ctx) @@ -144,7 +158,7 @@ func TestGitRemoteFactory_TwoClientsDistinctCacheDirsRoundtrip(t *testing.T) { require.NoError(t, dbA.Close()) // Client B reads chunk A, then writes chunk B and updates the root. - dbB, csB := open() + dbB, csB := open(cacheB) require.NoError(t, csB.Rebase(ctx)) rootB, err := csB.Root(ctx) require.NoError(t, err) @@ -161,7 +175,7 @@ func TestGitRemoteFactory_TwoClientsDistinctCacheDirsRoundtrip(t *testing.T) { require.NoError(t, dbB.Close()) // Client A re-opens and should see B's update. - dbA2, csA2 := open() + dbA2, csA2 := open(cacheA) require.NoError(t, csA2.Rebase(ctx)) rootA2, err := csA2.Root(ctx) require.NoError(t, err) diff --git a/go/libraries/doltcore/env/environment.go b/go/libraries/doltcore/env/environment.go index 618f21b22c..06e59d8e4c 100644 --- a/go/libraries/doltcore/env/environment.go +++ b/go/libraries/doltcore/env/environment.go @@ -184,6 +184,16 @@ func (dEnv *DoltEnv) UrlStr() string { return dEnv.urlStr } +// GitCacheRoot returns the absolute path to the local Dolt repository root (the directory that contains `.dolt/`). +// It is used to place git-remote caches under `/.dolt/...`. +func (dEnv *DoltEnv) GitCacheRoot() (string, bool) { + doltDir := dEnv.GetDoltDir() + if doltDir == "" { + return "", false + } + return filepath.Dir(doltDir), true +} + func createRepoState(fs filesys.Filesys) (*RepoState, error) { repoState, rsErr := LoadRepoState(fs) diff --git a/go/libraries/doltcore/env/remotes.go b/go/libraries/doltcore/env/remotes.go index 7e819fa9f6..a4c3d64c76 100644 --- a/go/libraries/doltcore/env/remotes.go +++ b/go/libraries/doltcore/env/remotes.go @@ -106,6 +106,11 @@ func (r *Remote) GetRemoteDB(ctx context.Context, nbf *types.NomsBinFormat, dial params[dbfactory.GRPCDialProviderParam] = dialer if u, err := earl.Parse(r.Url); err == nil && u != nil && strings.HasPrefix(strings.ToLower(u.Scheme), "git+") { params[dbfactory.GitRemoteNameParam] = r.Name + if p, ok := dialer.(dbfactory.GitCacheRootProvider); ok { + if root, ok := p.GitCacheRoot(); ok { + params[dbfactory.GitCacheRootParam] = root + } + } } return doltdb.LoadDoltDBWithParams(ctx, nbf, r.Url, filesys2.LocalFS, params) @@ -122,6 +127,11 @@ func (r *Remote) Prepare(ctx context.Context, nbf *types.NomsBinFormat, dialer d params[dbfactory.GRPCDialProviderParam] = dialer if u, err := earl.Parse(r.Url); err == nil && u != nil && strings.HasPrefix(strings.ToLower(u.Scheme), "git+") { params[dbfactory.GitRemoteNameParam] = r.Name + if p, ok := dialer.(dbfactory.GitCacheRootProvider); ok { + if root, ok := p.GitCacheRoot(); ok { + params[dbfactory.GitCacheRootParam] = root + } + } } return dbfactory.PrepareDB(ctx, nbf, r.Url, params) @@ -136,6 +146,11 @@ func (r *Remote) GetRemoteDBWithoutCaching(ctx context.Context, nbf *types.NomsB params[dbfactory.GRPCDialProviderParam] = dialer if u, err := earl.Parse(r.Url); err == nil && u != nil && strings.HasPrefix(strings.ToLower(u.Scheme), "git+") { params[dbfactory.GitRemoteNameParam] = r.Name + if p, ok := dialer.(dbfactory.GitCacheRootProvider); ok { + if root, ok := p.GitCacheRoot(); ok { + params[dbfactory.GitCacheRootParam] = root + } + } } return doltdb.LoadDoltDBWithParams(ctx, nbf, r.Url, filesys2.LocalFS, params) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 5d7cca5d6d..69a4401be7 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -77,6 +77,18 @@ type DoltDatabaseProvider struct { InitDatabaseHooks []InitDatabaseHook } +type remoteDialerWithGitCacheRoot struct { + dbfactory.GRPCDialProvider + root string +} + +func (d remoteDialerWithGitCacheRoot) GitCacheRoot() (string, bool) { + if strings.TrimSpace(d.root) == "" { + return "", false + } + return d.root, true +} + var _ sql.DatabaseProvider = (*DoltDatabaseProvider)(nil) var _ sql.FunctionProvider = (*DoltDatabaseProvider)(nil) var _ sql.MutableDatabaseProvider = (*DoltDatabaseProvider)(nil) @@ -502,10 +514,26 @@ func (p *DoltDatabaseProvider) allRevisionDbs(ctx *sql.Context, db dsess.SqlData } func (p *DoltDatabaseProvider) GetRemoteDB(ctx context.Context, format *types.NomsBinFormat, r env.Remote, withCaching bool) (*doltdb.DoltDB, error) { - if withCaching { - return r.GetRemoteDB(ctx, format, p.remoteDialer) + // For git remotes, thread through the initiating database's repo root so git caches can be located under + // `/.dolt/...` instead of a user-global cache dir. + dialer := p.remoteDialer + if sqlCtx, ok := ctx.(*sql.Context); ok { + baseName, _ := doltdb.SplitRevisionDbName(sqlCtx.GetCurrentDatabase()) + dbKey := strings.ToLower(baseName) + p.mu.RLock() + dbLoc, ok := p.dbLocations[dbKey] + p.mu.RUnlock() + if ok && dbLoc != nil { + if root, err := dbLoc.Abs("."); err == nil && strings.TrimSpace(root) != "" { + dialer = remoteDialerWithGitCacheRoot{GRPCDialProvider: p.remoteDialer, root: root} + } + } } - return r.GetRemoteDBWithoutCaching(ctx, format, p.remoteDialer) + + if withCaching { + return r.GetRemoteDB(ctx, format, dialer) + } + return r.GetRemoteDBWithoutCaching(ctx, format, dialer) } func (p *DoltDatabaseProvider) CreateDatabase(ctx *sql.Context, name string) error { @@ -814,7 +842,11 @@ func (p *DoltDatabaseProvider) cloneDatabaseFromRemote( } r := env.NewRemote(remoteName, remoteUrl, remoteParams) - srcDB, err := r.GetRemoteDB(ctx, types.Format_Default, p.remoteDialer) + destRoot, err := p.fs.Abs(dbName) + if err != nil { + return err + } + srcDB, err := r.GetRemoteDB(ctx, types.Format_Default, remoteDialerWithGitCacheRoot{GRPCDialProvider: p.remoteDialer, root: destRoot}) if err != nil { return err } diff --git a/integration-tests/bats/remotes-git.bats b/integration-tests/bats/remotes-git.bats index af38f9e419..58d0449b41 100644 --- a/integration-tests/bats/remotes-git.bats +++ b/integration-tests/bats/remotes-git.bats @@ -8,15 +8,11 @@ setup() { cd $BATS_TMPDIR cd dolt-repo-$$ mkdir "dolt-repo-clones" - - # Keep auto-selected git cache dir inside this test's sandbox. - export XDG_CACHE_HOME="$(mktemp -d)" } teardown() { assert_feature_version teardown_common - rm -rf "$XDG_CACHE_HOME" } @test "remotes-git: smoke push/clone/push-back/pull" { diff --git a/integration-tests/bats/sql-remotes-git.bats b/integration-tests/bats/sql-remotes-git.bats index a1ebe2ab8a..05fbe71fcf 100644 --- a/integration-tests/bats/sql-remotes-git.bats +++ b/integration-tests/bats/sql-remotes-git.bats @@ -7,15 +7,11 @@ setup() { setup_common cd $BATS_TMPDIR cd dolt-repo-$$ - - # Keep auto-selected git cache dir inside this test's sandbox. - export XDG_CACHE_HOME="$(mktemp -d)" } teardown() { assert_feature_version teardown_common - rm -rf "$XDG_CACHE_HOME" } @test "sql-remotes-git: dolt_remote add supports --ref for git remotes" {