/go/store/blobstore: add git fetch push and merge primitives

This commit is contained in:
coffeegoddd☕️✨
2026-02-09 14:44:15 -08:00
parent f729910f99
commit a8b2fdddbf
4 changed files with 576 additions and 0 deletions

View File

@@ -33,6 +33,8 @@ type fakeGitAPI struct {
listTree func(ctx context.Context, commit git.OID, treePath string) ([]git.TreeEntry, error)
blobSize func(ctx context.Context, oid git.OID) (int64, error)
blobReader func(ctx context.Context, oid git.OID) (io.ReadCloser, error)
fetchRef func(ctx context.Context, remote string, srcRef string, dstRef string) error
pushRefWithLease func(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID git.OID) error
}
func (f fakeGitAPI) TryResolveRefCommit(ctx context.Context, ref string) (git.OID, bool, error) {
@@ -86,6 +88,27 @@ func (f fakeGitAPI) UpdateRefCAS(ctx context.Context, ref string, newOID git.OID
func (f fakeGitAPI) UpdateRef(ctx context.Context, ref string, newOID git.OID, msg string) error {
panic("unexpected call")
}
func (f fakeGitAPI) FetchRef(ctx context.Context, remote string, srcRef string, dstRef string) error {
if f.fetchRef == nil {
panic("unexpected call")
}
return f.fetchRef(ctx, remote, srcRef, dstRef)
}
func (f fakeGitAPI) PushRefWithLease(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID git.OID) error {
if f.pushRefWithLease == nil {
panic("unexpected call")
}
return f.pushRefWithLease(ctx, remote, srcRef, dstRef, expectedDstOID)
}
func (f fakeGitAPI) MergeBase(ctx context.Context, a git.OID, b git.OID) (git.OID, bool, error) {
panic("unexpected call")
}
func (f fakeGitAPI) ListTreeRecursive(ctx context.Context, commit git.OID, treePath string) ([]git.TreeEntry, error) {
panic("unexpected call")
}
func (f fakeGitAPI) CommitTreeWithParents(ctx context.Context, tree git.OID, parents []git.OID, message string, author *git.Identity) (git.OID, error) {
panic("unexpected call")
}
func TestGitBlobstoreHelpers_resolveCommitForGet(t *testing.T) {
ctx := context.Background()

View File

@@ -108,6 +108,34 @@ type GitAPI interface {
// Equivalent plumbing:
// GIT_DIR=... git update-ref -m <msg> <ref> <new>
UpdateRef(ctx context.Context, ref string, newOID OID, msg string) error
// FetchRef fetches |srcRef| from |remote| and updates |dstRef| in the local repo.
// It is expected to be a forced update to keep local tracking refs in sync with remote truth.
// Equivalent plumbing:
// GIT_DIR=... git fetch <remote> +<srcRef>:<dstRef>
FetchRef(ctx context.Context, remote string, srcRef string, dstRef string) error
// PushRefWithLease pushes |srcRef| to |dstRef| on |remote|, but only if the remote's |dstRef|
// currently equals |expectedDstOID| (i.e. force-with-lease semantics).
// Equivalent plumbing:
// GIT_DIR=... git push --force-with-lease=<dstRef>:<expectedDstOID> <remote> <srcRef>:<dstRef>
PushRefWithLease(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID OID) error
// MergeBase returns the merge-base OID for |a| and |b|, if one exists.
// Equivalent plumbing:
// GIT_DIR=... git merge-base <a> <b>
MergeBase(ctx context.Context, a OID, b OID) (oid OID, ok bool, err error)
// ListTreeRecursive lists entries of the tree at |treePath| within |commit| recursively.
// For entries under a subdirectory, Name contains the relative path (e.g. "sub/x.txt").
// Equivalent plumbing:
// GIT_DIR=... git ls-tree -r -t <commit>:<treePath>
ListTreeRecursive(ctx context.Context, commit OID, treePath string) ([]TreeEntry, error)
// CommitTreeWithParents creates a commit object from |tree| with zero or more |parents| and returns its oid.
// Equivalent plumbing:
// GIT_DIR=... git commit-tree <tree> [-p <parent> ...] -m <message>
CommitTreeWithParents(ctx context.Context, tree OID, parents []OID, message string, author *Identity) (OID, error)
}
// TreeEntry describes one entry in a git tree listing.

View File

@@ -144,6 +144,39 @@ func (a *GitAPIImpl) ListTree(ctx context.Context, commit OID, treePath string)
return entries, nil
}
func (a *GitAPIImpl) ListTreeRecursive(ctx context.Context, commit OID, treePath string) ([]TreeEntry, error) {
spec := commit.String()
if treePath != "" {
spec = spec + ":" + treePath
} else {
spec = spec + "^{tree}"
}
out, err := a.r.Run(ctx, RunOptions{}, "ls-tree", "-r", "-t", spec)
if err != nil {
if isPathNotFoundErr(err) && treePath != "" {
return nil, &PathNotFoundError{Commit: commit.String(), Path: treePath}
}
return nil, err
}
lines := strings.Split(strings.TrimRight(string(out), "\n"), "\n")
if len(lines) == 1 && strings.TrimSpace(lines[0]) == "" {
return nil, nil
}
entries := make([]TreeEntry, 0, len(lines))
for _, line := range lines {
if strings.TrimSpace(line) == "" {
continue
}
e, err := parseLsTreeLine(line)
if err != nil {
return nil, err
}
entries = append(entries, e)
}
return entries, nil
}
func (a *GitAPIImpl) CatFileType(ctx context.Context, oid OID) (string, error) {
out, err := a.r.Run(ctx, RunOptions{}, "cat-file", "-t", oid.String())
if err != nil {
@@ -262,6 +295,63 @@ func (a *GitAPIImpl) CommitTree(ctx context.Context, tree OID, parent *OID, mess
return OID(oid), nil
}
func (a *GitAPIImpl) CommitTreeWithParents(ctx context.Context, tree OID, parents []OID, message string, author *Identity) (OID, error) {
args := []string{"commit-tree", tree.String(), "-m", message}
for _, p := range parents {
if p == "" {
continue
}
args = append(args, "-p", p.String())
}
var env []string
if author != nil {
if author.Name != "" {
env = append(env,
"GIT_AUTHOR_NAME="+author.Name,
"GIT_COMMITTER_NAME="+author.Name,
)
}
if author.Email != "" {
env = append(env,
"GIT_AUTHOR_EMAIL="+author.Email,
"GIT_COMMITTER_EMAIL="+author.Email,
)
}
}
out, err := a.r.Run(ctx, RunOptions{Env: env}, args...)
if err != nil {
return "", err
}
oid := strings.TrimSpace(string(out))
if oid == "" {
return "", fmt.Errorf("git commit-tree returned empty oid")
}
return OID(oid), nil
}
func (a *GitAPIImpl) MergeBase(ctx context.Context, aOID OID, bOID OID) (oid OID, ok bool, err error) {
if aOID == "" {
return "", false, fmt.Errorf("git merge-base: oid a is required")
}
if bOID == "" {
return "", false, fmt.Errorf("git merge-base: oid b is required")
}
out, err := a.r.Run(ctx, RunOptions{}, "merge-base", aOID.String(), bOID.String())
if err != nil {
if isNoMergeBaseErr(err) {
return "", false, nil
}
return "", false, err
}
s := strings.TrimSpace(string(out))
if s == "" {
return "", false, fmt.Errorf("git merge-base returned empty oid")
}
return OID(s), true, nil
}
func (a *GitAPIImpl) UpdateRefCAS(ctx context.Context, ref string, newOID OID, oldOID OID, msg string) error {
args := []string{"update-ref"}
if msg != "" {
@@ -282,6 +372,43 @@ func (a *GitAPIImpl) UpdateRef(ctx context.Context, ref string, newOID OID, msg
return err
}
func (a *GitAPIImpl) FetchRef(ctx context.Context, remote string, srcRef string, dstRef string) error {
if remote == "" {
return fmt.Errorf("git fetch: remote is required")
}
if srcRef == "" {
return fmt.Errorf("git fetch: src ref is required")
}
if dstRef == "" {
return fmt.Errorf("git fetch: dst ref is required")
}
// Forced refspec to keep tracking refs in sync with remote truth.
srcRef = strings.TrimPrefix(srcRef, "+")
refspec := "+" + srcRef + ":" + dstRef
_, err := a.r.Run(ctx, RunOptions{}, "fetch", "--no-tags", remote, refspec)
return err
}
func (a *GitAPIImpl) PushRefWithLease(ctx context.Context, remote string, srcRef string, dstRef string, expectedDstOID OID) error {
if remote == "" {
return fmt.Errorf("git push: remote is required")
}
if srcRef == "" {
return fmt.Errorf("git push: src ref is required")
}
if dstRef == "" {
return fmt.Errorf("git push: dst ref is required")
}
if expectedDstOID == "" {
return fmt.Errorf("git push: expected dst oid is required")
}
srcRef = strings.TrimPrefix(srcRef, "+")
refspec := srcRef + ":" + dstRef
lease := "--force-with-lease=" + dstRef + ":" + expectedDstOID.String()
_, err := a.r.Run(ctx, RunOptions{}, "push", "--porcelain", lease, remote, refspec)
return err
}
func isRefNotFoundErr(err error) bool {
ce, ok := err.(*CmdError)
if !ok {
@@ -298,6 +425,19 @@ func isRefNotFoundErr(err error) bool {
strings.Contains(msg, "not a valid object name")
}
func isNoMergeBaseErr(err error) bool {
ce, ok := err.(*CmdError)
if !ok {
return false
}
// `git merge-base <a> <b>` returns exit 1 when there is no merge base.
if ce.ExitCode == 1 {
msg := strings.ToLower(string(bytes.TrimSpace(ce.Output)))
return msg == "" || strings.Contains(msg, "no merge base")
}
return false
}
func isPathNotFoundErr(err error) bool {
ce, ok := err.(*CmdError)
if !ok {

View File

@@ -773,3 +773,388 @@ func TestGitAPIImpl_UpdateRef_And_CAS(t *testing.T) {
t.Fatalf("ref changed unexpectedly: ok=%v got=%q want=%q", ok, got, c2)
}
}
func TestGitAPIImpl_FetchRef_ForcedUpdatesTrackingRef(t *testing.T) {
t.Parallel()
ctx := context.Background()
remoteRepo, err := gitrepo.InitBareTemp(ctx, "")
if err != nil {
t.Fatal(err)
}
remoteRunner, err := NewRunner(remoteRepo.GitDir)
if err != nil {
t.Fatal(err)
}
remoteAPI := NewGitAPIImpl(remoteRunner)
// Create two commits on the same tree in the remote.
indexFile := tempIndexFile(t)
if err := remoteAPI.ReadTreeEmpty(ctx, indexFile); err != nil {
t.Fatal(err)
}
treeOID, err := remoteAPI.WriteTree(ctx, indexFile)
if err != nil {
t.Fatal(err)
}
c1, err := remoteAPI.CommitTree(ctx, treeOID, nil, "c1", testAuthor())
if err != nil {
t.Fatal(err)
}
c2, err := remoteAPI.CommitTree(ctx, treeOID, nil, "c2", testAuthor())
if err != nil {
t.Fatal(err)
}
if c1 == c2 {
c2, err = remoteAPI.CommitTree(ctx, treeOID, nil, "c2b", testAuthor())
if err != nil {
t.Fatal(err)
}
if c1 == c2 {
t.Fatalf("expected distinct commit oids")
}
}
remoteDataRef := "refs/dolt/data"
if err := remoteAPI.UpdateRef(ctx, remoteDataRef, c2, "seed remote"); err != nil {
t.Fatal(err)
}
localRepo, err := gitrepo.InitBareTemp(ctx, "")
if err != nil {
t.Fatal(err)
}
localRunner, err := NewRunner(localRepo.GitDir)
if err != nil {
t.Fatal(err)
}
_, err = localRunner.Run(ctx, RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
if err != nil {
t.Fatal(err)
}
localAPI := NewGitAPIImpl(localRunner)
dstRef := "refs/dolt/remotes/origin/data"
if err := localAPI.FetchRef(ctx, "origin", remoteDataRef, dstRef); err != nil {
t.Fatal(err)
}
got, err := localAPI.ResolveRefCommit(ctx, dstRef)
if err != nil {
t.Fatal(err)
}
if got != c2 {
t.Fatalf("tracking ref mismatch: got %q, want %q", got, c2)
}
// Rewind the remote ref to c1 and ensure a subsequent fetch forces the tracking ref backwards.
if err := remoteAPI.UpdateRef(ctx, remoteDataRef, c1, "rewind remote"); err != nil {
t.Fatal(err)
}
if err := localAPI.FetchRef(ctx, "origin", remoteDataRef, dstRef); err != nil {
t.Fatal(err)
}
got, err = localAPI.ResolveRefCommit(ctx, dstRef)
if err != nil {
t.Fatal(err)
}
if got != c1 {
t.Fatalf("tracking ref mismatch after rewind: got %q, want %q", got, c1)
}
}
func TestGitAPIImpl_PushRefWithLease_SucceedsThenRejectsStaleLease(t *testing.T) {
t.Parallel()
ctx := context.Background()
remoteRepo, err := gitrepo.InitBareTemp(ctx, "")
if err != nil {
t.Fatal(err)
}
remoteRunner, err := NewRunner(remoteRepo.GitDir)
if err != nil {
t.Fatal(err)
}
remoteAPI := NewGitAPIImpl(remoteRunner)
// Seed remote ref with r1, then later advance to r2.
indexFile := tempIndexFile(t)
if err := remoteAPI.ReadTreeEmpty(ctx, indexFile); err != nil {
t.Fatal(err)
}
treeOID, err := remoteAPI.WriteTree(ctx, indexFile)
if err != nil {
t.Fatal(err)
}
r1, err := remoteAPI.CommitTree(ctx, treeOID, nil, "r1", testAuthor())
if err != nil {
t.Fatal(err)
}
r2, err := remoteAPI.CommitTree(ctx, treeOID, nil, "r2", testAuthor())
if err != nil {
t.Fatal(err)
}
if r1 == r2 {
r2, err = remoteAPI.CommitTree(ctx, treeOID, nil, "r2b", testAuthor())
if err != nil {
t.Fatal(err)
}
if r1 == r2 {
t.Fatalf("expected distinct commit oids")
}
}
remoteDataRef := "refs/dolt/data"
if err := remoteAPI.UpdateRef(ctx, remoteDataRef, r1, "seed remote"); err != nil {
t.Fatal(err)
}
localRepo, err := gitrepo.InitBareTemp(ctx, "")
if err != nil {
t.Fatal(err)
}
localRunner, err := NewRunner(localRepo.GitDir)
if err != nil {
t.Fatal(err)
}
_, err = localRunner.Run(ctx, RunOptions{}, "remote", "add", "origin", remoteRepo.GitDir)
if err != nil {
t.Fatal(err)
}
localAPI := NewGitAPIImpl(localRunner)
// Create a local commit l1 and set local refs/dolt/data to it (src ref for push).
localIndex := tempIndexFile(t)
if err := localAPI.ReadTreeEmpty(ctx, localIndex); err != nil {
t.Fatal(err)
}
localTree, err := localAPI.WriteTree(ctx, localIndex)
if err != nil {
t.Fatal(err)
}
l1, err := localAPI.CommitTree(ctx, localTree, nil, "l1", testAuthor())
if err != nil {
t.Fatal(err)
}
if err := localAPI.UpdateRef(ctx, remoteDataRef, l1, "set local src"); err != nil {
t.Fatal(err)
}
// Lease matches remote (r1) -> push should succeed and overwrite remoteDataRef to l1.
if err := localAPI.PushRefWithLease(ctx, "origin", remoteDataRef, remoteDataRef, r1); err != nil {
t.Fatal(err)
}
got, err := remoteAPI.ResolveRefCommit(ctx, remoteDataRef)
if err != nil {
t.Fatal(err)
}
if got != l1 {
t.Fatalf("remote ref mismatch after push: got %q, want %q", got, l1)
}
// Advance remote to r2, then attempt a stale-lease push expecting r1 -> should fail and not clobber r2.
if err := remoteAPI.UpdateRef(ctx, remoteDataRef, r2, "advance remote"); err != nil {
t.Fatal(err)
}
err = localAPI.PushRefWithLease(ctx, "origin", remoteDataRef, remoteDataRef, r1)
if err == nil {
t.Fatalf("expected stale lease push to fail")
}
got, err = remoteAPI.ResolveRefCommit(ctx, remoteDataRef)
if err != nil {
t.Fatal(err)
}
if got != r2 {
t.Fatalf("remote ref changed unexpectedly on stale lease: got %q, want %q", got, r2)
}
}
func TestGitAPIImpl_MergeBase_OkAndNoMergeBase(t *testing.T) {
t.Parallel()
ctx := context.Background()
repo, err := gitrepo.InitBareTemp(ctx, "")
if err != nil {
t.Fatal(err)
}
r, err := NewRunner(repo.GitDir)
if err != nil {
t.Fatal(err)
}
api := NewGitAPIImpl(r)
// Base commit (empty tree).
indexFile := tempIndexFile(t)
if err := api.ReadTreeEmpty(ctx, indexFile); err != nil {
t.Fatal(err)
}
treeOID, err := api.WriteTree(ctx, indexFile)
if err != nil {
t.Fatal(err)
}
base, err := api.CommitTree(ctx, treeOID, nil, "base", testAuthor())
if err != nil {
t.Fatal(err)
}
// Two children of base.
p := base
a, err := api.CommitTree(ctx, treeOID, &p, "a", testAuthor())
if err != nil {
t.Fatal(err)
}
p = base
b, err := api.CommitTree(ctx, treeOID, &p, "b", testAuthor())
if err != nil {
t.Fatal(err)
}
mb, ok, err := api.MergeBase(ctx, a, b)
if err != nil {
t.Fatal(err)
}
if !ok {
t.Fatalf("expected merge base")
}
if mb != base {
t.Fatalf("merge base mismatch: got %q, want %q", mb, base)
}
// No merge base: two unrelated root commits in the same repo.
c1, err := api.CommitTree(ctx, treeOID, nil, "root1", testAuthor())
if err != nil {
t.Fatal(err)
}
c2, err := api.CommitTree(ctx, treeOID, nil, "root2", testAuthor())
if err != nil {
t.Fatal(err)
}
_, ok, err = api.MergeBase(ctx, c1, c2)
if err != nil {
t.Fatal(err)
}
if ok {
t.Fatalf("expected no merge base")
}
}
func TestGitAPIImpl_ListTreeRecursive(t *testing.T) {
t.Parallel()
ctx := context.Background()
repo, err := gitrepo.InitBareTemp(ctx, "")
if err != nil {
t.Fatal(err)
}
r, err := NewRunner(repo.GitDir)
if err != nil {
t.Fatal(err)
}
api := NewGitAPIImpl(r)
indexFile := tempIndexFile(t)
if err := api.ReadTreeEmpty(ctx, indexFile); err != nil {
t.Fatal(err)
}
oidA, err := api.HashObject(ctx, bytes.NewReader([]byte("a\n")))
if err != nil {
t.Fatal(err)
}
oidX, err := api.HashObject(ctx, bytes.NewReader([]byte("x\n")))
if err != nil {
t.Fatal(err)
}
if err := api.UpdateIndexCacheInfo(ctx, indexFile, "100644", oidA, "dir/a.txt"); err != nil {
t.Fatal(err)
}
if err := api.UpdateIndexCacheInfo(ctx, indexFile, "100644", oidX, "dir/sub/x.txt"); err != nil {
t.Fatal(err)
}
treeOID, err := api.WriteTree(ctx, indexFile)
if err != nil {
t.Fatal(err)
}
commitOID, err := api.CommitTree(ctx, treeOID, nil, "seed", testAuthor())
if err != nil {
t.Fatal(err)
}
entries, err := api.ListTreeRecursive(ctx, commitOID, "dir")
if err != nil {
t.Fatal(err)
}
got := map[string]TreeEntry{}
for _, e := range entries {
got[e.Name] = e
}
if e, ok := got["a.txt"]; !ok || e.Type != ObjectTypeBlob || e.OID != oidA {
t.Fatalf("missing or unexpected a.txt: ok=%v entry=%+v", ok, e)
}
if e, ok := got["sub"]; !ok || e.Type != ObjectTypeTree || e.OID == "" {
t.Fatalf("missing or unexpected sub tree: ok=%v entry=%+v", ok, e)
}
if e, ok := got["sub/x.txt"]; !ok || e.Type != ObjectTypeBlob || e.OID != oidX {
t.Fatalf("missing or unexpected sub/x.txt: ok=%v entry=%+v", ok, e)
}
}
func TestGitAPIImpl_CommitTreeWithParents(t *testing.T) {
t.Parallel()
ctx := context.Background()
repo, err := gitrepo.InitBareTemp(ctx, "")
if err != nil {
t.Fatal(err)
}
r, err := NewRunner(repo.GitDir)
if err != nil {
t.Fatal(err)
}
api := NewGitAPIImpl(r)
indexFile := tempIndexFile(t)
if err := api.ReadTreeEmpty(ctx, indexFile); err != nil {
t.Fatal(err)
}
treeOID, err := api.WriteTree(ctx, indexFile)
if err != nil {
t.Fatal(err)
}
base, err := api.CommitTree(ctx, treeOID, nil, "base", testAuthor())
if err != nil {
t.Fatal(err)
}
p := base
a, err := api.CommitTree(ctx, treeOID, &p, "a", testAuthor())
if err != nil {
t.Fatal(err)
}
p = base
b, err := api.CommitTree(ctx, treeOID, &p, "b", testAuthor())
if err != nil {
t.Fatal(err)
}
merge, err := api.CommitTreeWithParents(ctx, treeOID, []OID{a, b}, "merge", testAuthor())
if err != nil {
t.Fatal(err)
}
out, err := r.Run(ctx, RunOptions{}, "rev-parse", merge.String()+"^1")
if err != nil {
t.Fatal(err)
}
if gotP1 := string(bytes.TrimSpace(out)); gotP1 != a.String() {
t.Fatalf("parent1 mismatch: got %q, want %q", gotP1, a.String())
}
out, err = r.Run(ctx, RunOptions{}, "rev-parse", merge.String()+"^2")
if err != nil {
t.Fatal(err)
}
if gotP2 := string(bytes.TrimSpace(out)); gotP2 != b.String() {
t.Fatalf("parent2 mismatch: got %q, want %q", gotP2, b.String())
}
}