Add more shallow clone specific tests

Get golang tests working again

go/cmd/dolt/cli/arg_parser_helpers.go

@@ -183,7 +183,6 @@ func CreateFetchArgParser() *argparser.ArgParser {
 	ap.SupportsString(UserFlag, "", "user", "User name to use when authenticating with the remote. Gets password from the environment variable {{.EmphasisLeft}}DOLT_REMOTE_PASSWORD{{.EmphasisRight}}.")
 	ap.SupportsFlag(PruneFlag, "p", "After fetching, remove any remote-tracking references that don't exist on the remote.")
 	ap.SupportsFlag(SilentFlag, "", "Suppress progress information.")
-	ap.SupportsInt("depth", "", "depth", "Limit fetching to the specified number of commits from the tip of each remote branch history.") // NM4.
 	return ap
 }
 

go/cmd/dolt/commands/diff.go

@@ -16,6 +16,7 @@ package commands
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"strconv"
@@ -433,10 +434,8 @@ func (dArgs *diffArgs) applyDiffRoots(queryist cli.Queryist, sqlCtx *sql.Context
 	fromRef := args[0]
 	// treat the first arg as a ref spec
 	_, err := getTableNamesAtRef(queryist, sqlCtx, fromRef)
-	if err == doltdb.ErrUnexpectedGhostCommit {
-		// NM4 - this isn't sustainable.
-		return nil, errhand.BuildDError("Shallow Clone prevents loading requested commit. Perform a full clone and try again.").Build()
-
+	if errors.Is(err, doltdb.ErrUnexpectedGhostCommit) {
+		return nil, err
 	}
 	// if it doesn't resolve, treat it as a table name
 	if err != nil {

go/cmd/dolt/commands/read_tables.go

@@ -223,7 +223,7 @@ func getRemoteDBAtCommit(ctx context.Context, remoteUrl string, remoteUrlParams
 	}
 	cm, ok := optCmt.ToCommit()
 	if !ok {
-		return nil, nil, errhand.BuildDError(doltdb.ErrUnexpectedGhostCommit.Error()).Build() // NM4 - TEST??
+		return nil, nil, errhand.BuildDError(doltdb.ErrUnexpectedGhostCommit.Error()).Build()
 	}
 
 	srcRoot, err := cm.GetRootValue(ctx)

go/cmd/dolt/commands/show.go

@@ -284,7 +284,7 @@ func printObjects(ctx context.Context, dEnv *env.DoltEnv, opts *showOpts) error
 		}
 		commit, ok := optCmt.ToCommit()
 		if !ok {
-			return doltdb.ErrUnexpectedGhostCommit // NM4 - TEST.
+			return doltdb.ErrUnexpectedGhostCommit
 		}
 
 		value := commit.Value()

go/cmd/dolt/commands/stashcmds/pop.go

@@ -148,7 +148,9 @@ func applyStashAtIdx(ctx *sql.Context, dEnv *env.DoltEnv, curWorkingRoot *doltdb
 	}
 	parentCommit, ok := optCmt.ToCommit()
 	if !ok {
-		return false, doltdb.ErrUnexpectedGhostCommit // NM4 - TEST!
+		// Should not be possible to get into this situation. The parent of the stashed commit
+		// Must have been present at the time it was created
+		return false, doltdb.ErrUnexpectedGhostCommit
 	}
 
 	parentRoot, err := parentCommit.GetRootValue(ctx)

go/libraries/doltcore/cherry_pick/cherry_pick.go

@@ -210,7 +210,7 @@ func cherryPick(ctx *sql.Context, dSess *dsess.DoltSession, roots doltdb.Roots,
 	}
 	cherryCommit, ok := optCmt.ToCommit()
 	if err != nil {
-		return nil, "", doltdb.ErrUnexpectedGhostCommit // NM4 - TEST. Surely need a better error.
+		return nil, "", doltdb.ErrUnexpectedGhostCommit
 	}
 
 	if len(cherryCommit.DatasParents()) > 1 {
@@ -233,7 +233,7 @@ func cherryPick(ctx *sql.Context, dSess *dsess.DoltSession, roots doltdb.Roots,
 	}
 	parentCommit, ok := optCmt.ToCommit()
 	if !ok {
-		return nil, "", doltdb.ErrUnexpectedGhostCommit // NM4 - TEST. Surely need a better error.
+		return nil, "", doltdb.ErrUnexpectedGhostCommit
 	}
 
 	parentRoot, err := parentCommit.GetRootValue(ctx)

go/libraries/doltcore/diff/diffsplitter.go

@@ -214,36 +214,3 @@ func (ds DiffSplitter) SplitDiffResultRow(row sql.Row) (from, to RowDiff, err er
 	}
 	return
 }
-
-/*
-// MaybeResolveRoot returns a root value and true if the a commit exists for given spec string; nil and false if it does not exist.
-// todo: distinguish between non-existent CommitSpec and other errors, don't assume non-existent
-func MaybeResolveRoot(ctx context.Context, rsr env.RepoStateReader, doltDB *doltdb.DoltDB, spec string) (*doltdb.RootValue, bool) {
-	cs, err := doltdb.NewCommitSpec(spec)
-	if err != nil {
-		// it's non-existent CommitSpec
-		return nil, false
-	}
-
-	headRef, err := rsr.CWBHeadRef()
-	if err != nil {
-		return nil, false
-	}
-	optCmt, err := doltDB.Resolve(ctx, cs, headRef)
-	if err != nil {
-		return nil, false
-	}
-
-	cm, ok := optCmt.ToCommit()
-	if !ok {
-		return nil, false // NM4 - We should return an error here, gotta change the interface
-	}
-
-	root, err := cm.GetRootValue(ctx)
-	if err != nil {
-		return nil, false
-	}
-
-	return root, true
-}
-*/

go/libraries/doltcore/doltdb/commit.go

@@ -147,9 +147,6 @@ func (c *Commit) GetParent(ctx context.Context, idx int) (*OptionalCommit, error
 	if err != nil {
 		return nil, err
 	}
-
-	// NM4 - Looks like this method can return nil - we check in places. Should we return an error instead?
-
 	return &OptionalCommit{cmt, parent.Addr()}, nil
 }
 

go/libraries/doltcore/doltdb/commit_hooks.go

@@ -61,7 +61,7 @@ func pushDataset(ctx context.Context, destDB, srcDB datas.Database, ds datas.Dat
 	}
 
 	// NM4 tmp to compile.
-	err := pullHash(ctx, destDB, srcDB, nil, nil, []hash.Hash{addr}, tmpDir, nil, nil)
+	err := pullHash(ctx, destDB, srcDB, []hash.Hash{addr}, tmpDir, nil, nil)
 	if err != nil {
 		return err
 	}

go/libraries/doltcore/doltdb/commit_hooks_test.go

@@ -88,8 +88,8 @@ func TestPushOnWriteHook(t *testing.T) {
 	if err != nil {
 		t.Fatal("Couldn't find commit")
 	}
-	commit, err := optCmt.ToCommit()
-	assert.NoError(t, err)
+	commit, ok := optCmt.ToCommit()
+	assert.True(t, ok)
 
 	meta, err := commit.GetCommitMeta(context.Background())
 	assert.NoError(t, err)
@@ -145,8 +145,8 @@ func TestPushOnWriteHook(t *testing.T) {
 		cs, _ = NewCommitSpec(defaultBranch)
 		optCmt, err := destDB.Resolve(context.Background(), cs, nil)
 		require.NoError(t, err)
-		destCommit, err := optCmt.ToCommit()
-		require.NoError(t, err)
+		destCommit, ok := optCmt.ToCommit()
+		require.True(t, ok)
 
 		srcHash, _ := srcCommit.HashOf()
 		destHash, _ := destCommit.HashOf()
@@ -236,8 +236,8 @@ func TestAsyncPushOnWrite(t *testing.T) {
 			if err != nil {
 				t.Fatal("Couldn't find commit")
 			}
-			commit, err := optCmt.ToCommit()
-			assert.NoError(t, err)
+			commit, ok := optCmt.ToCommit()
+			assert.True(t, ok)
 
 			meta, err := commit.GetCommitMeta(context.Background())
 			assert.NoError(t, err)

go/libraries/doltcore/doltdb/doltdb.go

@@ -38,7 +38,6 @@ import (
 	"github.com/dolthub/dolt/go/store/types"
 	"github.com/dolthub/dolt/go/store/types/edits"
 	"github.com/dolthub/go-mysql-server/sql"
-	"github.com/sirupsen/logrus"
 )
 
 func init() {
@@ -407,7 +406,6 @@ func (ddb *DoltDB) Resolve(ctx context.Context, cs *CommitSpec, cwb ref.DoltRef)
 	}
 
 	if commitValue.IsGhost() {
-		// NM4 - not sure about the accestor walk here.....
 		return &OptionalCommit{nil, *hash}, nil
 	}
 
@@ -416,7 +414,6 @@ func (ddb *DoltDB) Resolve(ctx context.Context, cs *CommitSpec, cwb ref.DoltRef)
 		return nil, err
 	}
 
-	// This is a little messy. If someone callse HEAD~4, the ghost commit may be in the middle of the range.
 	return commit.GetAncestor(ctx, cs.aSpec)
 }
 
@@ -439,12 +436,9 @@ func (ddb *DoltDB) BootstrapShallowResolve(ctx context.Context, cs *CommitSpec,
 	}
 
 	if commitValue.IsGhost() {
-		return prolly.CommitClosure{}, fmt.Errorf("Commit requested as shallow clone tip could not be found. Incomplete clone likely, please reclone")
+		return prolly.CommitClosure{}, ErrUnexpectedGhostCommit
 	}
 
-	// NM4 - What about the ancestor spec? Not clear if we can get it in a shallow clone. We either need a specific
-	// branch or commit id. ~5 requires walking the history.
-
 	return getCommitClosure(ctx, commitValue, ddb.vrw, ddb.ns)
 }
 
@@ -1628,13 +1622,12 @@ func (ddb *DoltDB) PullChunks(
 	statsCh chan pull.Stats,
 	skipHashes hash.HashSet,
 ) error {
-	return pullHash(ctx, ddb.db, srcDB.db, ddb.vrw, srcDB.vrw, targetHashes, tempDir, statsCh, skipHashes)
+	return pullHash(ctx, ddb.db, srcDB.db, targetHashes, tempDir, statsCh, skipHashes)
 }
 
 func pullHash(
 	ctx context.Context,
 	destDB, srcDB datas.Database,
-	destVRW, srcVRW types.ValueReadWriter,
 	targetHashes []hash.Hash,
 	tempDir string,
 	statsCh chan pull.Stats,
@@ -1642,17 +1635,7 @@ func pullHash(
 ) error {
 	srcCS := datas.ChunkStoreFromDatabase(srcDB)
 	destCS := datas.ChunkStoreFromDatabase(destDB)
-
-	// Address Walker to rule them all NM4.
-	waf := types.WalkAddrsForMacneale(srcDB.Format(), func(r types.Ref) bool {
-		hsh := r.TargetHash()
-
-		if skipHashes != nil && skipHashes.Has(hsh) {
-			logrus.Info("Skipping hash: ", hsh.String())
-			return true
-		}
-		return false
-	})
+	waf := types.WalkAddrsForNBF(srcDB.Format(), skipHashes)
 
 	if datas.CanUsePuller(srcDB) && datas.CanUsePuller(destDB) {
 		puller, err := pull.NewPuller(ctx, tempDir, defaultChunksPerTF, srcCS, destCS, waf, targetHashes, statsCh)

go/libraries/doltcore/doltdb/doltdb_test.go

@@ -205,8 +205,8 @@ func TestEmptyInMemoryRepoCreation(t *testing.T) {
 	if err != nil {
 		t.Fatal("Could not find commit")
 	}
-	commit, err := optCmt.ToCommit()
-	assert.NoError(t, err)
+	commit, ok := optCmt.ToCommit()
+	assert.True(t, ok)
 
 	h, err := commit.HashOf()
 	assert.NoError(t, err)
@@ -281,8 +281,8 @@ func TestLDNoms(t *testing.T) {
 		if err != nil {
 			t.Fatal("Couldn't find commit")
 		}
-		commit, err := optCmt.ToCommit()
-		assert.NoError(t, err)
+		commit, ok := optCmt.ToCommit()
+		assert.True(t, ok)
 
 		meta, err := commit.GetCommitMeta(context.Background())
 		assert.NoError(t, err)

go/libraries/doltcore/dtestutils/testcommands/multienv.go

@@ -196,7 +196,7 @@ func (mr *MultiRepoTestSetup) CloneDB(fromRemote, dbName string) {
 		mr.Errhand(err)
 	}
 
-	err = actions.CloneRemote(ctx, srcDB, r.Name, "", false, dEnv)
+	err = actions.CloneRemote(ctx, srcDB, r.Name, "", false, -1, dEnv)
 	if err != nil {
 		mr.Errhand(err)
 	}

go/libraries/doltcore/env/actions/clone.go

@@ -330,7 +330,6 @@ func fullClone(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch str
 		clonePrint(eventCh)
 	}()
 
-	// was Clone()
 	err := srcDB.Clone(ctx, dEnv.DoltDB, eventCh)
 
 	close(eventCh)
@@ -341,63 +340,24 @@ func fullClone(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch str
 }
 
 func shallowClone(ctx context.Context, destData env.DbData, srcDB *doltdb.DoltDB, remoteName, branch string, depth int) error {
-	/*
-		specs, err := env.GetRefSpecs(destData.Rsr, remoteName)
-		if err != nil {
-			return err
-		}
-
-	*/
-	// What should the specs looks like? Verify there is only one? NM4
-
-	// Get the remote
 	remotes, err := destData.Rsr.GetRemotes()
 	if err != nil {
 		return err
 	}
-
 	remote, ok := remotes.Get(remoteName)
 	if !ok {
+		// By the time we get to this point, the remote should be created, so this should never happen.
 		return fmt.Errorf("remote %s not found", remoteName)
 	}
 
 	if branch == "" {
-		branch = "main"
+		branch = env.DefaultInitBranch
 	}
 	specs, err := env.ParseRefSpecs([]string{branch}, destData.Rsr, remote)
 
 	return ShallowFetchRefSpec(ctx, destData, srcDB, specs[0], &remote, depth)
 }
 
-func guessSingleBranch(ctx context.Context, dEnv *env.DoltEnv) (string, error) {
-	// Get all the refs from the remote. These branch refs will be translated to remote branch refs, tags will
-	// be preserved, and all other refs will be ignored.
-	srcRefHashes, err := dEnv.DoltDB.GetRefsWithHashes(ctx)
-	if err != nil {
-		return "", fmt.Errorf("%w; %s", ErrCloneFailed, err.Error())
-	}
-
-	branches := make([]ref.DoltRef, 0, len(srcRefHashes))
-	for _, refHash := range srcRefHashes {
-		if refHash.Ref.GetType() == ref.BranchRefType {
-			br := refHash.Ref.(ref.BranchRef)
-			branches = append(branches, br)
-		}
-	}
-
-	branch := env.GetDefaultBranch(dEnv, branches)
-
-	// If we couldn't find a branch but the repo cloned successfully, it's empty. Initialize it instead of pulling from
-	// the remote.
-	if branch == "" {
-		if err = InitEmptyClonedRepo(ctx, dEnv); err != nil {
-			return "", err
-		}
-		branch = env.GetDefaultInitBranch(dEnv.Config)
-	}
-	return branch, nil
-}
-
 // InitEmptyClonedRepo inits an empty, newly cloned repo. This would be unnecessary if we properly initialized the
 // storage for a repository when we created it on dolthub. If we do that, this code can be removed.
 func InitEmptyClonedRepo(ctx context.Context, dEnv *env.DoltEnv) error {

go/libraries/doltcore/env/actions/commitwalk/commitwalk.go

@@ -151,7 +151,6 @@ func (q *q) Get(ctx context.Context, ddb *doltdb.DoltDB, id hash.Hash) (*c, erro
 
 	commit, ok := optCmt.ToCommit()
 	if !ok {
-		// Preserve the Ghost commit. NM4 - TEST THIS PATH.
 		return &c{ddb: ddb, commit: optCmt, hash: id}, nil
 	}
 
@@ -204,17 +203,6 @@ func GetDotDotRevisions(ctx context.Context, includedDB *doltdb.DoltDB, included
 	return commitList, nil
 }
 
-/*
-NM4. Kill this?
-
-// GetTopologicalOrderCommits returns the commits reachable from the commits in `startCommitHashes`
-// in reverse topological order, with tiebreaking done by the height of the commit graph -- higher commits
-// appear first. Remaining ties are broken by timestamp; newer commits appear first.
-func GetTopologicalOrderCommits(ctx context.Context, ddb *doltdb.DoltDB, startCommitHashes []hash.Hash) ([]*doltdb.OptionalCommit, error) {
-	return GetTopNTopoOrderedCommitsMatching(ctx, ddb, startCommitHashes, -1, nil)
-}
-*/
-
 // GetTopologicalOrderCommitIterator returns an iterator for commits generated with the same semantics as
 // GetTopologicalOrderCommits
 func GetTopologicalOrderIterator(ctx context.Context, ddb *doltdb.DoltDB, startCommitHashes []hash.Hash, matchFn func(*doltdb.OptionalCommit) (bool, error)) (doltdb.CommitItr, error) {
@@ -296,35 +284,6 @@ func (i *commiterator) Reset(ctx context.Context) error {
 	return nil
 }
 
-/* NM4 - Kill this?
-
-// GetTopNTopoOrderedCommitsMatching returns the first N commits (If N <= 0 then all commits) reachable from the commits in
-// `startCommitHashes` in reverse topological order, with tiebreaking done by the height of the commit graph -- higher
-// commits appear first. Remaining ties are broken by timestamp; newer commits appear first.
-func GetTopNTopoOrderedCommitsMatching(ctx context.Context, ddb *doltdb.DoltDB, startCommitHashes []hash.Hash, n int, matchFn func(*doltdb.OptionalCommit) (bool, error)) ([]*doltdb.OptionalCommit, error) {
-	// NM4 - This is interesting. marking this for later.
-	// This appears to be dead code?!
-	itr, err := GetTopologicalOrderIterator(ctx, ddb, startCommitHashes, matchFn)
-	if err != nil {
-		return nil, err
-	}
-
-	var commitList []*doltdb.OptionalCommit
-	for n < 0 || len(commitList) < n {
-		_, commit, err := itr.Next(ctx)
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			return nil, err
-		}
-
-		commitList = append(commitList, commit)
-	}
-
-	return commitList, nil
-}
-*/
-
 // GetDotDotRevisionsIterator returns an iterator for commits generated with the same semantics as
 // GetDotDotRevisions
 func GetDotDotRevisionsIterator(ctx context.Context, includedDdb *doltdb.DoltDB, startCommitHashes []hash.Hash, excludedDdb *doltdb.DoltDB, excludingCommitHashes []hash.Hash, matchFn func(*doltdb.OptionalCommit) (bool, error)) (doltdb.CommitItr, error) {

go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go

@@ -70,8 +70,8 @@ func TestGetDotDotRevisions(t *testing.T) {
 	require.NoError(t, err)
 	opt, err := dEnv.DoltDB.Resolve(context.Background(), cs, nil)
 	require.NoError(t, err)
-	commit, err := opt.ToCommit()
-	require.NoError(t, err)
+	commit, ok := opt.ToCommit()
+	require.True(t, ok)
 
 	rv, err := commit.GetRootValue(context.Background())
 	require.NoError(t, err)
@@ -241,21 +241,20 @@ func TestGetDotDotRevisions(t *testing.T) {
 }
 
 func assertEqualHashes(t *testing.T, lc, rc interface{}) {
-	var err error
 	leftCm, ok := lc.(*doltdb.Commit)
 	if !ok {
 		opt, ok := lc.(*doltdb.OptionalCommit)
 		require.True(t, ok)
-		leftCm, err = opt.ToCommit()
-		require.NoError(t, err)
+		leftCm, ok = opt.ToCommit()
+		require.True(t, ok)
 	}
 
 	rightCm, ok := rc.(*doltdb.Commit)
 	if !ok {
 		opt, ok := rc.(*doltdb.OptionalCommit)
 		require.True(t, ok)
-		rightCm, err = opt.ToCommit()
-		require.NoError(t, err)
+		rightCm, ok = opt.ToCommit()
+		require.True(t, ok)
 	}
 
 	assert.Equal(t, mustGetHash(t, leftCm), mustGetHash(t, rightCm))

go/libraries/doltcore/merge/merge_test.go

@@ -718,8 +718,8 @@ func buildLeftRightAncCommitsAndBranches(t *testing.T, ddb *doltdb.DoltDB, rootT
 	mainHeadSpec, _ := doltdb.NewCommitSpec(env.DefaultInitBranch)
 	optCmt, err := ddb.Resolve(context.Background(), mainHeadSpec, nil)
 	require.NoError(t, err)
-	mainHead, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	mainHead, ok := optCmt.ToCommit()
+	require.True(t, ok)
 
 	mRoot, err := mainHead.GetRootValue(context.Background())
 	require.NoError(t, err)
@@ -760,8 +760,8 @@ func buildLeftRightAncCommitsAndBranches(t *testing.T, ddb *doltdb.DoltDB, rootT
 
 	optCmt, err = doltdb.GetCommitAncestor(context.Background(), commit, mergeCommit)
 	require.NoError(t, err)
-	ancCm, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	ancCm, ok := optCmt.ToCommit()
+	require.True(t, ok)
 
 	ancRoot, err := ancCm.GetRootValue(context.Background())
 	require.NoError(t, err)

go/libraries/doltcore/sqle/dsess/dolt_session_test.go

@@ -307,7 +307,7 @@ func (e emptyRevisionDatabaseProvider) FileSystemForDatabase(dbname string) (fil
 	return nil, nil
 }
 
-func (e emptyRevisionDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error {
+func (e emptyRevisionDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, depth int, remoteParams map[string]string) error {
 	return nil
 }
 

go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go

@@ -176,8 +176,8 @@ func TestDoltTransactionCommitOneClient(t *testing.T) {
 	require.NoError(t, err)
 	optCmt, err := db.Resolve(context.Background(), cs, headRefs[0])
 	require.NoError(t, err)
-	commit, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	commit, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	cm, err := commit.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Contains(t, cm.Description, "Commit Message 42")
@@ -188,8 +188,8 @@ func TestDoltTransactionCommitOneClient(t *testing.T) {
 	require.NoError(t, err)
 	optCmt, err = db.Resolve(context.Background(), cs, headRefs[0])
 	require.NoError(t, err)
-	commit, err = optCmt.ToCommit()
-	require.NoError(t, err)
+	commit, ok = optCmt.ToCommit()
+	require.True(t, ok)
 	cm, err = commit.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Contains(t, cm.Description, "Transaction commit")
@@ -198,8 +198,8 @@ func TestDoltTransactionCommitOneClient(t *testing.T) {
 	require.NoError(t, err)
 	optCmt, err = commit.GetAncestor(context.Background(), as)
 	require.NoError(t, err)
-	initialCommit, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	initialCommit, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	icm, err := initialCommit.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Equal(t, "checkpoint enginetest database mydb", icm.Description)
@@ -355,8 +355,8 @@ func TestDoltTransactionCommitTwoClients(t *testing.T) {
 	require.NoError(t, err)
 	optCmt, err := db.Resolve(context.Background(), cs, headRefs[0])
 	require.NoError(t, err)
-	commit2, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	commit2, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	cm2, err := commit2.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Contains(t, cm2.Description, "ClientA Commit")
@@ -365,16 +365,16 @@ func TestDoltTransactionCommitTwoClients(t *testing.T) {
 	require.NoError(t, err)
 	optCmt, err = commit2.GetAncestor(context.Background(), as)
 	require.NoError(t, err)
-	commit1, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	commit1, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	cm1, err := commit1.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Contains(t, cm1.Description, "ClientB Commit")
 
 	optCmt, err = commit1.GetAncestor(context.Background(), as)
 	require.NoError(t, err)
-	commit0, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	commit0, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	cm0, err := commit0.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Equal(t, "checkpoint enginetest database mydb", cm0.Description)
@@ -441,8 +441,8 @@ func TestDoltTransactionCommitAutocommit(t *testing.T) {
 	require.NoError(t, err)
 	optCmt, err := db.Resolve(context.Background(), headSpec, headRefs[0])
 	require.NoError(t, err)
-	head, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	head, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	headMeta, err := head.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Contains(t, headMeta.Description, "ClientB Commit")
@@ -451,16 +451,16 @@ func TestDoltTransactionCommitAutocommit(t *testing.T) {
 	require.NoError(t, err)
 	optCmt, err = head.GetAncestor(context.Background(), ancestorSpec)
 	require.NoError(t, err)
-	parent, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	parent, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	parentMeta, err := parent.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Contains(t, parentMeta.Description, "Transaction commit")
 
 	optCmt, err = parent.GetAncestor(context.Background(), ancestorSpec)
 	require.NoError(t, err)
-	grandParent, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	grandParent, ok := optCmt.ToCommit()
+	require.True(t, ok)
 	grandparentMeta, err := grandParent.GetCommitMeta(context.Background())
 	require.NoError(t, err)
 	require.Equal(t, "checkpoint enginetest database mydb", grandparentMeta.Description)

go/libraries/doltcore/sqle/index/mergeable_indexes_setup_test.go

@@ -183,8 +183,8 @@ func getDbState(t *testing.T, db sql.Database, dEnv *env.DoltEnv) (dsess.Initial
 	optCmt, err := dEnv.DoltDB.Resolve(ctx, headSpec, headRef)
 	require.NoError(t, err)
 
-	headCommit, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	headCommit, ok := optCmt.ToCommit()
+	require.True(t, ok)
 
 	ws, err := dEnv.WorkingSet(ctx)
 	require.NoError(t, err)

go/libraries/doltcore/sqle/sqlselect_test.go

@@ -1564,8 +1564,8 @@ func testSelectDiffQuery(t *testing.T, test SelectTest) {
 	optCmt, err := dEnv.DoltDB.Resolve(ctx, cs, nil)
 	require.NoError(t, err)
 
-	cm, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	cm, ok := optCmt.ToCommit()
+	require.True(t, ok)
 
 	root, err := cm.GetRootValue(ctx)
 	require.NoError(t, err)
@@ -1680,8 +1680,8 @@ func initializeWithHistory(t *testing.T, ctx context.Context, dEnv *env.DoltEnv,
 		optCmt, err := dEnv.DoltDB.Resolve(ctx, cs, nil)
 		require.NoError(t, err)
 
-		cm, err := optCmt.ToCommit()
-		require.NoError(t, err)
+		cm, ok := optCmt.ToCommit()
+		require.True(t, ok)
 
 		processNode(t, ctx, dEnv, node, cm)
 	}
@@ -1703,8 +1703,8 @@ func processNode(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, node Hist
 	optCmt, err := dEnv.DoltDB.Resolve(ctx, cs, nil)
 	require.NoError(t, err)
 
-	cm, err := optCmt.ToCommit()
-	require.NoError(t, err)
+	cm, ok := optCmt.ToCommit()
+	require.True(t, ok)
 
 	root, err := cm.GetRootValue(ctx)
 	require.NoError(t, err)

go/store/cmd/noms/noms_cat.go

@@ -161,7 +161,7 @@ func runCat(ctx context.Context, args []string) int {
 		//Want a clean db every loop
 		sp, _ := spec.ForDatabase("mem")
 		vrw := sp.GetVRW(ctx)
-		waf := types.WalkAddrsForNBF(vrw.Format())
+		waf := types.WalkAddrsForNBF(vrw.Format(), nil)
 
 		fmt.Printf("        chunk[%d].raw.len:     %d\n", cidx, len(currCD.compressed))
 

go/store/nbs/benchmarks/file_block_store.go

@@ -96,3 +96,9 @@ func (fb fileBlockStore) Commit(ctx context.Context, current, last hash.Hash) (b
 	err := fb.bw.Flush()
 	return true, err
 }
+
+// NM4
+func (fb fileBlockStore) GhostTheseRefsBrah(ctx context.Context, refs hash.HashSet) error {
+	panic("not impl")
+	return nil
+}

go/store/nbs/benchmarks/null_block_store.go

@@ -86,3 +86,8 @@ func (nb nullBlockStore) Root(ctx context.Context) (hash.Hash, error) {
 func (nb nullBlockStore) Commit(ctx context.Context, current, last hash.Hash) (bool, error) {
 	return true, nil
 }
+
+// NM4
+func (nb nullBlockStore) GhostTheseRefsBrah(ctx context.Context, refs hash.HashSet) error {
+	panic("not impl")
+}

go/store/types/ref.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/hash"
-	"github.com/sirupsen/logrus"
 )
 
 type Ref struct {
@@ -238,22 +237,13 @@ func WalkAddrsForChunkStore(cs chunks.ChunkStore) (func(chunks.Chunk, func(h has
 	if err != nil {
 		return nil, fmt.Errorf("could not find binary format corresponding to %s. try upgrading dolt.", cs.Version())
 	}
-	return WalkAddrsForNBF(nbf), nil
+	return WalkAddrsForNBF(nbf, nil), nil
 }
 
-func WalkAddrsForNBF(nbf *NomsBinFormat) func(chunks.Chunk, func(h hash.Hash, isleaf bool) error) error {
+func WalkAddrsForNBF(nbf *NomsBinFormat, skipAddrs hash.HashSet) func(chunks.Chunk, func(h hash.Hash, isleaf bool) error) error {
 	return func(c chunks.Chunk, cb func(h hash.Hash, isleaf bool) error) error {
 		return walkRefs(c.Data(), nbf, func(r Ref) error {
-			return cb(r.TargetHash(), r.Height() == 1)
-		})
-	}
-}
-
-func WalkAddrsForMacneale(nbf *NomsBinFormat, isIgnoredCommit func(r Ref) bool) func(chunks.Chunk, func(h hash.Hash, isleaf bool) error) error {
-	return func(c chunks.Chunk, cb func(h hash.Hash, isleaf bool) error) error {
-		return walkRefs(c.Data(), nbf, func(r Ref) error {
-			if isIgnoredCommit(r) {
-				logrus.Info("Walk discovered this Ignored commit ID:" + r.TargetHash().String())
+			if skipAddrs != nil && skipAddrs.Has(r.TargetHash()) {
 				return nil
 			}
 

go/store/valuefile/file_value_store.go

@@ -287,3 +287,8 @@ func (f *FileValueStore) iterChunks(cb func(ch chunks.Chunk) error) error {
 
 	return nil
 }
+
+// NM4
+func (f *FileValueStore) GhostTheseRefsBrah(ctx context.Context, refs hash.HashSet) error {
+	panic("not implemented")
+}

integration-tests/bats/shallow-clone.bats

@@ -21,6 +21,7 @@ teardown() {
 stop_remotesrv() {
     if [ -n "$remotesrv_pid" ]; then
         kill $remotesrv_pid || :
+        remotesrv_pid=""
     fi
 }
 
@@ -45,6 +46,80 @@ seed_and_start_serial_remote() {
     cd ..
 }
 
+@test "shallow-clone: dolt_clone depth 1" {
+    seed_and_start_serial_remote
+
+    mkdir clones
+    cd clones
+    run dolt sql -q "call dolt_clone('--depth', '1','http://localhost:50051/test-org/test-repo')"
+
+    [ "$status" -eq 0 ]
+
+    run dolt log --oneline --decorate=no
+    [ "$status" -eq 0 ]
+    [ "${#lines[@]}" -eq 1 ]
+
+    run dolt sql -q "select count(*) = 1 from dolt_log()"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "true" ]] || false
+
+    # Verify that the table is complete.
+    run dolt sql -q "select sum(i) from vals"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "15" ]] || false # 1+2+3+4+5 = 15.
+}
+
+@test "shallow-clone: dolt_clone depth 2" {
+    seed_and_start_serial_remote
+
+    mkdir clones
+    cd clones
+    run dolt sql -q "call dolt_clone('--depth', '2','http://localhost:50051/test-org/test-repo')"
+    [ "$status" -eq 0 ]
+
+    run dolt log --oneline --decorate=no
+    [ "$status" -eq 0 ]
+    [ "${#lines[@]}" -eq 2 ]
+
+    run dolt sql -q "select count(*) = 2 from dolt_log()"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "true" ]] || false
+
+    # Verify that the table is complete.
+    run dolt sql -q "select sum(i) from vals"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "15" ]] || false # 1+2+3+4+5 = 15.
+}
+
+@test "shallow-clone: shallow clone with a file path" {
+    seed_and_start_serial_remote
+    stop_remotesrv
+    cd remote
+    dolt remote add origin file://../file-remote
+    dolt push origin main
+    cd ..
+
+    mkdir clones
+    cd clones
+    run dolt sql -q "call dolt_clone('--depth', '1','file://../file-remote')"
+    [ "$status" -eq 0 ]
+
+    run dolt log --oneline --decorate=no
+    [ "$status" -eq 0 ]
+    [ "${#lines[@]}" -eq 1 ]
+
+    run dolt sql -q "select count(*) = 1 from dolt_log()"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "true" ]] || false
+
+    # Verify that the table is complete.
+    run dolt sql -q "select sum(i) from vals"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "15" ]] || false # 1+2+3+4+5 = 15.
+
+
+}
+
 @test "shallow-clone: depth 3 clone of serial history" {
     seed_and_start_serial_remote
 
@@ -52,7 +127,6 @@ seed_and_start_serial_remote() {
     cd clones
 
     run dolt clone --depth 3 http://localhost:50051/test-org/test-repo
-
     [ "$status" -eq 0 ]
 
     cd test-repo
@@ -172,6 +246,13 @@ seed_and_start_serial_remote() {
     [ "$status" -eq 1 ]
     [[ "$output" =~ "Commit not found. You are using a shallow clone" ]] || false
 
+    run dolt revert HEAD~1
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "Commit not found. You are using a shallow clone" ]] || false
+
+    run dolt cherry-pick HEAD~1
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "Commit not found. You are using a shallow clone" ]] || false
 }
 
 @test "shallow-clone: shallow clone can push" {
@@ -209,6 +290,120 @@ seed_and_start_serial_remote() {
     [[ "$output" =~ "57" ]] || false # 1+2+3+4+5+42 = 57.
 }
 
+@test "shallow-clone: fetch new changes after shallow clone" {
+    seed_and_start_serial_remote
+
+    mkdir clones
+    cd clones
+
+    # initial clone
+    dolt clone --depth 2 http://localhost:50051/test-org/test-repo
+
+    # clone another copy, and push to remote srv.
+    dolt clone http://localhost:50051/test-org/test-repo full-clone
+    cd full-clone
+
+    dolt sql -q "insert into vals (i,s) values (23, \"val 23\")"
+    dolt commit -a -m "Added Val: 23"
+    run dolt push origin main
+    [ "$status" -eq 0 ]
+
+    # Go to out of date clone, and fetch.
+    cd ../test-repo
+    run dolt pull
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Fast-forward" ]] || false
+
+    run dolt log --oneline --decorate=no
+    [ "$status" -eq 0 ]
+    [ "${#lines[@]}" -eq 3 ]
+
+    dolt show
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Added Val: 23" ]] || false
+}
+
+@test "shallow-clone: fetch connected new branch works after shallow clone" {
+  seed_and_start_serial_remote
+
+  mkdir clones
+  cd clones
+
+  # initial clone
+  dolt clone --depth 2 http://localhost:50051/test-org/test-repo
+
+  # clone another copy, and push new branch to remote srv.
+  dolt clone http://localhost:50051/test-org/test-repo full-clone
+  cd full-clone
+
+  # Create two new commits on top of commit which exists in shallow clone.
+  dolt sql -q "insert into vals (i,s) values (23, \"val 23\")"
+  dolt commit -a -m "Added Val: 23"
+  dolt sql -q "insert into vals (i,s) values (42, \"val 42\")"
+  dolt commit -a -m "Added Val: 42"
+  dolt push origin HEAD:refs/heads/brch
+
+  cd ../test-repo
+
+  dolt fetch # Should pull new branch, and it's history should be length 4.
+  run dolt branch -a
+  [ "$status" -eq 0 ]
+  [[ "$output" =~ "remotes/origin/brch" ]] || false
+
+  run dolt log --oneline --decorate=no origin/brch
+  [ "$status" -eq 0 ]
+  [ "${#lines[@]}" -eq 4 ]
+
+  run dolt show origin/brch
+  [ "$status" -eq 0 ]
+  [[ "$output" =~ "Added Val: 42" ]] || false
+
+  run dolt show origin/brch~1
+  [ "$status" -eq 0 ]
+  [[ "$output" =~ "Added Val: 23" ]] || falses
+}
+
+@test "shallow-clone: fetch disconnected new branch works after shallow clone" {
+    seed_and_start_serial_remote
+
+    mkdir clones
+    cd clones
+
+    # initial clone
+    dolt clone --depth 2 http://localhost:50051/test-org/test-repo
+
+    # clone another copy, and push new branch to remote srv.
+    dolt clone http://localhost:50051/test-org/test-repo full-clone
+    cd full-clone
+    # Create two new commits rooted from a commit which doesn't exist in the
+    dolt reset --hard HEAD~3 # HEAD~3 == (val 2)
+    dolt sql -q "insert into vals (i,s) values (13, \"val 13\")"
+    dolt commit -a -m "Added Val: 13"
+    dolt sql -q "insert into vals (i,s) values (11, \"val 11\")"
+    dolt commit -a -m "Added Val: 11"
+    dolt push origin HEAD:refs/heads/brch
+
+    cd ../test-repo
+
+    dolt fetch # Should pull new branch, and it's history should be length 2.
+    run dolt branch -a
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "remotes/origin/brch" ]] || false
+
+    run dolt log --oneline --decorate=no origin/brch
+    [ "$status" -eq 0 ]
+    [ "${#lines[@]}" -eq 2 ]
+
+    run dolt show origin/brch
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Added Val: 11" ]] || false
+
+    # Verify that the table is complete.
+    run dolt sql -q "select sum(i) from vals as of 'origin/brch'"
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "27" ]] # 1+2+11+13 = 27.
+}
+
 # complex repository is 14 commits with the following dag:
 #
 # (init) <- (table create) <- (val 1) <- (val 2) <- (val 3) <- (val 4) <- (val 5) <- (merge 2)                [main]
@@ -258,7 +453,6 @@ seed_and_start_complex_remote() {
     cd test-repo
 
     run dolt log --oneline --decorate=no
-
     [ "$status" -eq 0 ]
     [ "${#lines[@]}" -eq 1 ]
 
@@ -376,8 +570,6 @@ seed_and_start_complex_remote() {
 
 
 # Tests to write:
-# - Make a simple change, and push it successfully.
-#   - Cloning the new change works as expected.
 # - Fetch after initial clone
 #   - Fetch when no changes have happened.
 #   - Fetch when there are remote changes on main