diff --git a/go/store/blobstore/git_blobstore_helpers_test.go b/go/store/blobstore/git_blobstore_helpers_test.go index 8080f4e838..be763240d1 100644 --- a/go/store/blobstore/git_blobstore_helpers_test.go +++ b/go/store/blobstore/git_blobstore_helpers_test.go @@ -33,6 +33,8 @@ type fakeGitAPI struct { listTree func(ctx context.Context, commit git.OID, treePath string) ([]git.TreeEntry, error) blobSize func(ctx context.Context, oid git.OID) (int64, error) blobReader func(ctx context.Context, oid git.OID) (io.ReadCloser, error) + fetchRef func(ctx context.Context, remote string, srcRef string, dstRef string) error + pushRefWithLease func(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID git.OID) error } func (f fakeGitAPI) TryResolveRefCommit(ctx context.Context, ref string) (git.OID, bool, error) { @@ -86,6 +88,27 @@ func (f fakeGitAPI) UpdateRefCAS(ctx context.Context, ref string, newOID git.OID func (f fakeGitAPI) UpdateRef(ctx context.Context, ref string, newOID git.OID, msg string) error { panic("unexpected call") } +func (f fakeGitAPI) FetchRef(ctx context.Context, remote string, srcRef string, dstRef string) error { + if f.fetchRef == nil { + panic("unexpected call") + } + return f.fetchRef(ctx, remote, srcRef, dstRef) +} +func (f fakeGitAPI) PushRefWithLease(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID git.OID) error { + if f.pushRefWithLease == nil { + panic("unexpected call") + } + return f.pushRefWithLease(ctx, remote, srcRef, dstRef, expectedDstOID) +} +func (f fakeGitAPI) MergeBase(ctx context.Context, a git.OID, b git.OID) (git.OID, bool, error) { + panic("unexpected call") +} +func (f fakeGitAPI) ListTreeRecursive(ctx context.Context, commit git.OID, treePath string) ([]git.TreeEntry, error) { + panic("unexpected call") +} +func (f fakeGitAPI) CommitTreeWithParents(ctx context.Context, tree git.OID, parents []git.OID, message string, author *git.Identity) (git.OID, error) { + panic("unexpected call") +} func TestGitBlobstoreHelpers_resolveCommitForGet(t *testing.T) { ctx := context.Background() diff --git a/go/store/blobstore/internal/git/api.go b/go/store/blobstore/internal/git/api.go index cabda14d60..ea1ae76fa6 100644 --- a/go/store/blobstore/internal/git/api.go +++ b/go/store/blobstore/internal/git/api.go @@ -108,6 +108,34 @@ type GitAPI interface { // Equivalent plumbing: // GIT_DIR=... git update-ref -m UpdateRef(ctx context.Context, ref string, newOID OID, msg string) error + + // FetchRef fetches |srcRef| from |remote| and updates |dstRef| in the local repo. + // It is expected to be a forced update to keep local tracking refs in sync with remote truth. + // Equivalent plumbing: + // GIT_DIR=... git fetch +: + FetchRef(ctx context.Context, remote string, srcRef string, dstRef string) error + + // PushRefWithLease pushes |srcRef| to |dstRef| on |remote|, but only if the remote's |dstRef| + // currently equals |expectedDstOID| (i.e. force-with-lease semantics). + // Equivalent plumbing: + // GIT_DIR=... git push --force-with-lease=: : + PushRefWithLease(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID OID) error + + // MergeBase returns the merge-base OID for |a| and |b|, if one exists. + // Equivalent plumbing: + // GIT_DIR=... git merge-base + MergeBase(ctx context.Context, a OID, b OID) (oid OID, ok bool, err error) + + // ListTreeRecursive lists entries of the tree at |treePath| within |commit| recursively. + // For entries under a subdirectory, Name contains the relative path (e.g. "sub/x.txt"). + // Equivalent plumbing: + // GIT_DIR=... git ls-tree -r -t : + ListTreeRecursive(ctx context.Context, commit OID, treePath string) ([]TreeEntry, error) + + // CommitTreeWithParents creates a commit object from |tree| with zero or more |parents| and returns its oid. + // Equivalent plumbing: + // GIT_DIR=... git commit-tree [-p ...] -m + CommitTreeWithParents(ctx context.Context, tree OID, parents []OID, message string, author *Identity) (OID, error) } // TreeEntry describes one entry in a git tree listing. diff --git a/go/store/blobstore/internal/git/impl.go b/go/store/blobstore/internal/git/impl.go index 8e78ccfaf6..13cdc14f4a 100644 --- a/go/store/blobstore/internal/git/impl.go +++ b/go/store/blobstore/internal/git/impl.go @@ -144,6 +144,39 @@ func (a *GitAPIImpl) ListTree(ctx context.Context, commit OID, treePath string) return entries, nil } +func (a *GitAPIImpl) ListTreeRecursive(ctx context.Context, commit OID, treePath string) ([]TreeEntry, error) { + spec := commit.String() + if treePath != "" { + spec = spec + ":" + treePath + } else { + spec = spec + "^{tree}" + } + + out, err := a.r.Run(ctx, RunOptions{}, "ls-tree", "-r", "-t", spec) + if err != nil { + if isPathNotFoundErr(err) && treePath != "" { + return nil, &PathNotFoundError{Commit: commit.String(), Path: treePath} + } + return nil, err + } + lines := strings.Split(strings.TrimRight(string(out), "\n"), "\n") + if len(lines) == 1 && strings.TrimSpace(lines[0]) == "" { + return nil, nil + } + entries := make([]TreeEntry, 0, len(lines)) + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + e, err := parseLsTreeLine(line) + if err != nil { + return nil, err + } + entries = append(entries, e) + } + return entries, nil +} + func (a *GitAPIImpl) CatFileType(ctx context.Context, oid OID) (string, error) { out, err := a.r.Run(ctx, RunOptions{}, "cat-file", "-t", oid.String()) if err != nil { @@ -262,6 +295,63 @@ func (a *GitAPIImpl) CommitTree(ctx context.Context, tree OID, parent *OID, mess return OID(oid), nil } +func (a *GitAPIImpl) CommitTreeWithParents(ctx context.Context, tree OID, parents []OID, message string, author *Identity) (OID, error) { + args := []string{"commit-tree", tree.String(), "-m", message} + for _, p := range parents { + if p == "" { + continue + } + args = append(args, "-p", p.String()) + } + + var env []string + if author != nil { + if author.Name != "" { + env = append(env, + "GIT_AUTHOR_NAME="+author.Name, + "GIT_COMMITTER_NAME="+author.Name, + ) + } + if author.Email != "" { + env = append(env, + "GIT_AUTHOR_EMAIL="+author.Email, + "GIT_COMMITTER_EMAIL="+author.Email, + ) + } + } + + out, err := a.r.Run(ctx, RunOptions{Env: env}, args...) + if err != nil { + return "", err + } + oid := strings.TrimSpace(string(out)) + if oid == "" { + return "", fmt.Errorf("git commit-tree returned empty oid") + } + return OID(oid), nil +} + +func (a *GitAPIImpl) MergeBase(ctx context.Context, aOID OID, bOID OID) (oid OID, ok bool, err error) { + if aOID == "" { + return "", false, fmt.Errorf("git merge-base: oid a is required") + } + if bOID == "" { + return "", false, fmt.Errorf("git merge-base: oid b is required") + } + out, err := a.r.Run(ctx, RunOptions{}, "merge-base", aOID.String(), bOID.String()) + if err != nil { + if isNoMergeBaseErr(err) { + return "", false, nil + } + return "", false, err + } + s := strings.TrimSpace(string(out)) + if s == "" { + return "", false, fmt.Errorf("git merge-base returned empty oid") + } + return OID(s), true, nil +} + func (a *GitAPIImpl) UpdateRefCAS(ctx context.Context, ref string, newOID OID, oldOID OID, msg string) error { args := []string{"update-ref"} if msg != "" { @@ -282,6 +372,43 @@ func (a *GitAPIImpl) UpdateRef(ctx context.Context, ref string, newOID OID, msg return err } +func (a *GitAPIImpl) FetchRef(ctx context.Context, remote string, srcRef string, dstRef string) error { + if remote == "" { + return fmt.Errorf("git fetch: remote is required") + } + if srcRef == "" { + return fmt.Errorf("git fetch: src ref is required") + } + if dstRef == "" { + return fmt.Errorf("git fetch: dst ref is required") + } + // Forced refspec to keep tracking refs in sync with remote truth. + srcRef = strings.TrimPrefix(srcRef, "+") + refspec := "+" + srcRef + ":" + dstRef + _, err := a.r.Run(ctx, RunOptions{}, "fetch", "--no-tags", remote, refspec) + return err +} + +func (a *GitAPIImpl) PushRefWithLease(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID OID) error { + if remote == "" { + return fmt.Errorf("git push: remote is required") + } + if srcRef == "" { + return fmt.Errorf("git push: src ref is required") + } + if dstRef == "" { + return fmt.Errorf("git push: dst ref is required") + } + if expectedDstOID == "" { + return fmt.Errorf("git push: expected dst oid is required") + } + srcRef = strings.TrimPrefix(srcRef, "+") + refspec := srcRef + ":" + dstRef + lease := "--force-with-lease=" + dstRef + ":" + expectedDstOID.String() + _, err := a.r.Run(ctx, RunOptions{}, "push", "--porcelain", lease, remote, refspec) + return err +} + func isRefNotFoundErr(err error) bool { ce, ok := err.(*CmdError) if !ok { @@ -298,6 +425,19 @@ func isRefNotFoundErr(err error) bool { strings.Contains(msg, "not a valid object name") } +func isNoMergeBaseErr(err error) bool { + ce, ok := err.(*CmdError) + if !ok { + return false + } + // `git merge-base ` returns exit 1 when there is no merge base. + if ce.ExitCode == 1 { + msg := strings.ToLower(string(bytes.TrimSpace(ce.Output))) + return msg == "" || strings.Contains(msg, "no merge base") + } + return false +} + func isPathNotFoundErr(err error) bool { ce, ok := err.(*CmdError) if !ok { diff --git a/go/store/blobstore/internal/git/impl_test.go b/go/store/blobstore/internal/git/impl_test.go index 216cb880c7..0bcf770817 100644 --- a/go/store/blobstore/internal/git/impl_test.go +++ b/go/store/blobstore/internal/git/impl_test.go @@ -773,3 +773,388 @@ func TestGitAPIImpl_UpdateRef_And_CAS(t *testing.T) { t.Fatalf("ref changed unexpectedly: ok=%v got=%q want=%q", ok, got, c2) } } + +func TestGitAPIImpl_FetchRef_ForcedUpdatesTrackingRef(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + remoteRepo, err := gitrepo.InitBareTemp(ctx, "") + if err != nil { + t.Fatal(err) + } + remoteRunner, err := NewRunner(remoteRepo.GitDir) + if err != nil { + t.Fatal(err) + } + remoteAPI := NewGitAPIImpl(remoteRunner) + + // Create two commits on the same tree in the remote. + indexFile := tempIndexFile(t) + if err := remoteAPI.ReadTreeEmpty(ctx, indexFile); err != nil { + t.Fatal(err) + } + treeOID, err := remoteAPI.WriteTree(ctx, indexFile) + if err != nil { + t.Fatal(err) + } + c1, err := remoteAPI.CommitTree(ctx, treeOID, nil, "c1", testAuthor()) + if err != nil { + t.Fatal(err) + } + c2, err := remoteAPI.CommitTree(ctx, treeOID, nil, "c2", testAuthor()) + if err != nil { + t.Fatal(err) + } + if c1 == c2 { + c2, err = remoteAPI.CommitTree(ctx, treeOID, nil, "c2b", testAuthor()) + if err != nil { + t.Fatal(err) + } + if c1 == c2 { + t.Fatalf("expected distinct commit oids") + } + } + + remoteDataRef := "refs/dolt/data" + if err := remoteAPI.UpdateRef(ctx, remoteDataRef, c2, "seed remote"); err != nil { + t.Fatal(err) + } + + localRepo, err := gitrepo.InitBareTemp(ctx, "") + if err != nil { + t.Fatal(err) + } + localRunner, err := NewRunner(localRepo.GitDir) + if err != nil { + t.Fatal(err) + } + _, err = localRunner.Run(ctx, RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir) + if err != nil { + t.Fatal(err) + } + localAPI := NewGitAPIImpl(localRunner) + + dstRef := "refs/dolt/remotes/origin/data" + if err := localAPI.FetchRef(ctx, "origin", remoteDataRef, dstRef); err != nil { + t.Fatal(err) + } + got, err := localAPI.ResolveRefCommit(ctx, dstRef) + if err != nil { + t.Fatal(err) + } + if got != c2 { + t.Fatalf("tracking ref mismatch: got %q, want %q", got, c2) + } + + // Rewind the remote ref to c1 and ensure a subsequent fetch forces the tracking ref backwards. + if err := remoteAPI.UpdateRef(ctx, remoteDataRef, c1, "rewind remote"); err != nil { + t.Fatal(err) + } + if err := localAPI.FetchRef(ctx, "origin", remoteDataRef, dstRef); err != nil { + t.Fatal(err) + } + got, err = localAPI.ResolveRefCommit(ctx, dstRef) + if err != nil { + t.Fatal(err) + } + if got != c1 { + t.Fatalf("tracking ref mismatch after rewind: got %q, want %q", got, c1) + } +} + +func TestGitAPIImpl_PushRefWithLease_SucceedsThenRejectsStaleLease(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + remoteRepo, err := gitrepo.InitBareTemp(ctx, "") + if err != nil { + t.Fatal(err) + } + remoteRunner, err := NewRunner(remoteRepo.GitDir) + if err != nil { + t.Fatal(err) + } + remoteAPI := NewGitAPIImpl(remoteRunner) + + // Seed remote ref with r1, then later advance to r2. + indexFile := tempIndexFile(t) + if err := remoteAPI.ReadTreeEmpty(ctx, indexFile); err != nil { + t.Fatal(err) + } + treeOID, err := remoteAPI.WriteTree(ctx, indexFile) + if err != nil { + t.Fatal(err) + } + r1, err := remoteAPI.CommitTree(ctx, treeOID, nil, "r1", testAuthor()) + if err != nil { + t.Fatal(err) + } + r2, err := remoteAPI.CommitTree(ctx, treeOID, nil, "r2", testAuthor()) + if err != nil { + t.Fatal(err) + } + if r1 == r2 { + r2, err = remoteAPI.CommitTree(ctx, treeOID, nil, "r2b", testAuthor()) + if err != nil { + t.Fatal(err) + } + if r1 == r2 { + t.Fatalf("expected distinct commit oids") + } + } + + remoteDataRef := "refs/dolt/data" + if err := remoteAPI.UpdateRef(ctx, remoteDataRef, r1, "seed remote"); err != nil { + t.Fatal(err) + } + + localRepo, err := gitrepo.InitBareTemp(ctx, "") + if err != nil { + t.Fatal(err) + } + localRunner, err := NewRunner(localRepo.GitDir) + if err != nil { + t.Fatal(err) + } + _, err = localRunner.Run(ctx, RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir) + if err != nil { + t.Fatal(err) + } + localAPI := NewGitAPIImpl(localRunner) + + // Create a local commit l1 and set local refs/dolt/data to it (src ref for push). + localIndex := tempIndexFile(t) + if err := localAPI.ReadTreeEmpty(ctx, localIndex); err != nil { + t.Fatal(err) + } + localTree, err := localAPI.WriteTree(ctx, localIndex) + if err != nil { + t.Fatal(err) + } + l1, err := localAPI.CommitTree(ctx, localTree, nil, "l1", testAuthor()) + if err != nil { + t.Fatal(err) + } + if err := localAPI.UpdateRef(ctx, remoteDataRef, l1, "set local src"); err != nil { + t.Fatal(err) + } + + // Lease matches remote (r1) -> push should succeed and overwrite remoteDataRef to l1. + if err := localAPI.PushRefWithLease(ctx, "origin", remoteDataRef, remoteDataRef, r1); err != nil { + t.Fatal(err) + } + got, err := remoteAPI.ResolveRefCommit(ctx, remoteDataRef) + if err != nil { + t.Fatal(err) + } + if got != l1 { + t.Fatalf("remote ref mismatch after push: got %q, want %q", got, l1) + } + + // Advance remote to r2, then attempt a stale-lease push expecting r1 -> should fail and not clobber r2. + if err := remoteAPI.UpdateRef(ctx, remoteDataRef, r2, "advance remote"); err != nil { + t.Fatal(err) + } + err = localAPI.PushRefWithLease(ctx, "origin", remoteDataRef, remoteDataRef, r1) + if err == nil { + t.Fatalf("expected stale lease push to fail") + } + got, err = remoteAPI.ResolveRefCommit(ctx, remoteDataRef) + if err != nil { + t.Fatal(err) + } + if got != r2 { + t.Fatalf("remote ref changed unexpectedly on stale lease: got %q, want %q", got, r2) + } +} + +func TestGitAPIImpl_MergeBase_OkAndNoMergeBase(t *testing.T) { + t.Parallel() + + ctx := context.Background() + repo, err := gitrepo.InitBareTemp(ctx, "") + if err != nil { + t.Fatal(err) + } + r, err := NewRunner(repo.GitDir) + if err != nil { + t.Fatal(err) + } + api := NewGitAPIImpl(r) + + // Base commit (empty tree). + indexFile := tempIndexFile(t) + if err := api.ReadTreeEmpty(ctx, indexFile); err != nil { + t.Fatal(err) + } + treeOID, err := api.WriteTree(ctx, indexFile) + if err != nil { + t.Fatal(err) + } + base, err := api.CommitTree(ctx, treeOID, nil, "base", testAuthor()) + if err != nil { + t.Fatal(err) + } + + // Two children of base. + p := base + a, err := api.CommitTree(ctx, treeOID, &p, "a", testAuthor()) + if err != nil { + t.Fatal(err) + } + p = base + b, err := api.CommitTree(ctx, treeOID, &p, "b", testAuthor()) + if err != nil { + t.Fatal(err) + } + + mb, ok, err := api.MergeBase(ctx, a, b) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatalf("expected merge base") + } + if mb != base { + t.Fatalf("merge base mismatch: got %q, want %q", mb, base) + } + + // No merge base: two unrelated root commits in the same repo. + c1, err := api.CommitTree(ctx, treeOID, nil, "root1", testAuthor()) + if err != nil { + t.Fatal(err) + } + c2, err := api.CommitTree(ctx, treeOID, nil, "root2", testAuthor()) + if err != nil { + t.Fatal(err) + } + _, ok, err = api.MergeBase(ctx, c1, c2) + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatalf("expected no merge base") + } +} + +func TestGitAPIImpl_ListTreeRecursive(t *testing.T) { + t.Parallel() + + ctx := context.Background() + repo, err := gitrepo.InitBareTemp(ctx, "") + if err != nil { + t.Fatal(err) + } + r, err := NewRunner(repo.GitDir) + if err != nil { + t.Fatal(err) + } + api := NewGitAPIImpl(r) + + indexFile := tempIndexFile(t) + if err := api.ReadTreeEmpty(ctx, indexFile); err != nil { + t.Fatal(err) + } + + oidA, err := api.HashObject(ctx, bytes.NewReader([]byte("a\n"))) + if err != nil { + t.Fatal(err) + } + oidX, err := api.HashObject(ctx, bytes.NewReader([]byte("x\n"))) + if err != nil { + t.Fatal(err) + } + if err := api.UpdateIndexCacheInfo(ctx, indexFile, "100644", oidA, "dir/a.txt"); err != nil { + t.Fatal(err) + } + if err := api.UpdateIndexCacheInfo(ctx, indexFile, "100644", oidX, "dir/sub/x.txt"); err != nil { + t.Fatal(err) + } + treeOID, err := api.WriteTree(ctx, indexFile) + if err != nil { + t.Fatal(err) + } + commitOID, err := api.CommitTree(ctx, treeOID, nil, "seed", testAuthor()) + if err != nil { + t.Fatal(err) + } + + entries, err := api.ListTreeRecursive(ctx, commitOID, "dir") + if err != nil { + t.Fatal(err) + } + got := map[string]TreeEntry{} + for _, e := range entries { + got[e.Name] = e + } + if e, ok := got["a.txt"]; !ok || e.Type != ObjectTypeBlob || e.OID != oidA { + t.Fatalf("missing or unexpected a.txt: ok=%v entry=%+v", ok, e) + } + if e, ok := got["sub"]; !ok || e.Type != ObjectTypeTree || e.OID == "" { + t.Fatalf("missing or unexpected sub tree: ok=%v entry=%+v", ok, e) + } + if e, ok := got["sub/x.txt"]; !ok || e.Type != ObjectTypeBlob || e.OID != oidX { + t.Fatalf("missing or unexpected sub/x.txt: ok=%v entry=%+v", ok, e) + } +} + +func TestGitAPIImpl_CommitTreeWithParents(t *testing.T) { + t.Parallel() + + ctx := context.Background() + repo, err := gitrepo.InitBareTemp(ctx, "") + if err != nil { + t.Fatal(err) + } + r, err := NewRunner(repo.GitDir) + if err != nil { + t.Fatal(err) + } + api := NewGitAPIImpl(r) + + indexFile := tempIndexFile(t) + if err := api.ReadTreeEmpty(ctx, indexFile); err != nil { + t.Fatal(err) + } + treeOID, err := api.WriteTree(ctx, indexFile) + if err != nil { + t.Fatal(err) + } + base, err := api.CommitTree(ctx, treeOID, nil, "base", testAuthor()) + if err != nil { + t.Fatal(err) + } + + p := base + a, err := api.CommitTree(ctx, treeOID, &p, "a", testAuthor()) + if err != nil { + t.Fatal(err) + } + p = base + b, err := api.CommitTree(ctx, treeOID, &p, "b", testAuthor()) + if err != nil { + t.Fatal(err) + } + + merge, err := api.CommitTreeWithParents(ctx, treeOID, []OID{a, b}, "merge", testAuthor()) + if err != nil { + t.Fatal(err) + } + + out, err := r.Run(ctx, RunOptions{}, "rev-parse", merge.String()+"^1") + if err != nil { + t.Fatal(err) + } + if gotP1 := string(bytes.TrimSpace(out)); gotP1 != a.String() { + t.Fatalf("parent1 mismatch: got %q, want %q", gotP1, a.String()) + } + out, err = r.Run(ctx, RunOptions{}, "rev-parse", merge.String()+"^2") + if err != nil { + t.Fatal(err) + } + if gotP2 := string(bytes.TrimSpace(out)); gotP2 != b.String() { + t.Fatalf("parent2 mismatch: got %q, want %q", gotP2, b.String()) + } +}