Bh/remotes bat fixes (#701)

bats/1pk5col-ints.bats

@@ -676,9 +676,10 @@ DELIM
     dolt commit -m "added row to test"
     dolt checkout test-branch-m
     run dolt merge test-branch
+    echo $output
     [ "$status" -eq 0 ]
-    [ "${lines[1]}" = "test | 0 " ]
-    skip "Row addition not totalled correctly" [ "${lines[2]}" = "1 tables changed, 1 rows added(+), 0 rows modified(*), 0 rows deleted(-)" ]
+    [ "${lines[1]}" = "test | 1 +" ]
+    [ "${lines[2]}" = "1 tables changed, 1 rows added(+), 0 rows modified(*), 0 rows deleted(-)" ]
 }
 
 @test "checkout table with branch of same name" {

bats/aws-remotes.bats

@@ -48,17 +48,15 @@ skip_if_no_aws_tests() {
     dolt sql -q 'show tables'
 }
 
-# TODO: Is this what we want?
-@test "clone empty aws repository gets new initial repository with configured remote" {
+# Matches behavior of other remote types
+@test "clone empty aws remote fails" {
     skip_if_no_aws_tests
     rm -rf .dolt
     random_repo=`openssl rand -hex 32`
-    dolt clone 'aws://['"$DOLT_BATS_AWS_TABLE"':'"$DOLT_BATS_AWS_BUCKET"']/'"$random_repo"
-    cd "$random_repo"
-    run dolt remote -v
-    [ "$status" -eq 0 ]
-    [ "${#lines[@]}" -eq 1 ]
-    [ "${lines[0]}" == 'origin aws://['"$DOLT_BATS_AWS_TABLE"':'"$DOLT_BATS_AWS_BUCKET"']/'"$random_repo"' ' ]
+    run dolt clone 'aws://['"$DOLT_BATS_AWS_TABLE"':'"$DOLT_BATS_AWS_BUCKET"']/'"$random_repo"
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "error: clone failed" ]] || false
+    [[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false
 }
 
 @test "can push to new remote" {

bats/remotes-file-system.bats

@@ -22,10 +22,9 @@ teardown() {
 }
 
 @test "Add a file system remote with a bad path" {
-    skip "Bad error message"
     run dolt remote add origin file:///poop/
     [ $status -ne 0 ]
-    [[ ! "$output" =~ "'' is not valid" ]] || false
+    [[ "$output" =~ "'file:///poop/' is not valid" ]] || false
 }
 
 @test "push, pull, and clone file based remotes" {
@@ -160,39 +159,16 @@ SQL
 
     run dolt branch
     [ "$status" -eq 0 ]
-    [[ "${lines[0]}" =~ "master" ]] || false
-    [[ "${lines[1]}" =~ "tester" ]] || false
+    [[ "$output" =~ "master" ]] || false
+    [[ "$output" =~ "tester" ]] || false
 
     # delete tester branch
     dolt branch -d -f tester
 
     run dolt branch
     [ "$status" -eq 0 ]
-    [[ "${lines[0]}" =~ "master" ]] || false
-    [[ "${lines[1]}" != "tester" ]] || false
-
-    # fetch branches from remote
-    dolt fetch
-
-    # should display both branches fetched from remote
-    run dolt branch
-    [ "$status" -eq 0 ]
-    [[ "${lines[0]}" =~ "master" ]] || false
-    skip "Fetched branch not displayed in branch list" [[ "${lines[1]}" =~ "tester" ]] || false
-
-    # should not be able to create branch with same
-    # name as remote branch
-    run dolt checkout -b tester
-    [ "$status" -eq 1 ]
-    [[ "${lines[0]}" =~ "fatal: A branch named 'tester' already exists." ]] || false
-
-    # delete tester branch
-    dolt branch -d -f tester
-    
-    # should not be able to checkout branch that is not displayed
-    run dolt checkout tester
-    [ "$status" -eq 1 ]
-    [[ "${lines[0]}" =~ "error: could not find tester" ]] || false
+    [[ "$output" =~ "master" ]] || false
+    [[ ! "$output" =~ "tester" ]] || false
 }
 
 @test "branches can be deleted after fetch" {
@@ -255,21 +231,6 @@ SQL
     run dolt branch
     [ "$status" -eq 0 ]
     [[ "${lines[0]}" =~ "master" ]] || false
-    skip "Fetched branch not displayed in branch list" [[ "${lines[1]}" =~ "tester" ]] || false
-
-    # delete fetched branch
-    run dolt branch -d -f tester
-    [ "$status" -eq 0 ]
-
-    # attempt to delete branch again
-    run dolt branch -d -f tester
-    [ "$status" -eq 1 ]
-    [[ "${lines[0]}" =~ "fatal: branch 'tester' not found" ]] || false
-
-    # should not be able to checkout branch that is deleted
-    run dolt checkout tester
-    [ "$status" -eq 1 ]
-    [[ "${lines[0]}" =~ "error: could not find tester" ]] || false
 }
 
 @test "clone from a directory that is not a dolt repo" {
@@ -277,10 +238,7 @@ SQL
     touch remotedir/some-junk
 
     cd dolt-repo-clones
-    dolt clone file://../remotedir test-repo
-    cd test-repo
-    skip "Cloning any directory partially initializes a dolt repo"
-    [ ! -d .dolt ]
-    [ ! -f LICENSE.md ]
-    [ ! -f README.md ]
-}
+    run dolt clone file://../remotedir test-repo
+    [ "$status" -eq 1 ]
+    [ ! -d test-repo ]
+}

bats/remotes.bats

@@ -61,9 +61,9 @@ teardown() {
     dolt remote add test-remote http://localhost:50051/test-org/test-repo
     run dolt push test-remote
     [ "$status" -eq 1 ]
-    skip "Bad error message for only one command to push"
-    [[ !"$output" =~ "unable to find" ]] || false
-    [[ "$output" =~ "must specify remote and branch" ]] || false
+    [[ "$output" =~ "fatal: The current branch master has no upstream branch." ]] || false
+    [[ "$output" =~ "To push the current branch and set the remote as upstream, use" ]] || false
+    [[ "$output" =~ "dolt push --set-upstream test-remote master" ]] || false
 }
 
 @test "push and pull master branch from a remote" {
@@ -73,8 +73,7 @@ teardown() {
     [ -d "$BATS_TMPDIR/remotes-$$/test-org/test-repo" ]
     run dolt pull test-remote
     [ "$status" -eq 0 ]
-    skip "Should say Already up to date not fast forward"
-    [[ "$output" = "up to date" ]] || false
+    [[ "$output" =~ "Everything up-to-date" ]] || false
 }
 
 @test "push and pull non-master branch from remote" {
@@ -84,8 +83,7 @@ teardown() {
     [ "$status" -eq 0 ]
     run dolt pull test-remote
     [ "$status" -eq 0 ]
-    skip "Should say up to date not fast forward"
-    [[ "$output" = "up to date" ]] || false
+    [[ "$output" =~ "Everything up-to-date" ]] || false
 }
 
 @test "push and pull from non-master branch and use --set-upstream" {
@@ -205,27 +203,18 @@ SQL
 
 @test "clone an empty remote" {
     run dolt clone http://localhost:50051/test-org/empty
-    [ "$status" -eq 0 ]
-    cd empty
-    run dolt status
-    [ "$status" -eq 0 ]
-    run ls
-    [[ ! "$output" =~ "LICENSE.md" ]] || false
-    [[ ! "$output" =~ "README.md" ]] || false
-    run dolt remote -v
-    [ "$status" -eq 0 ]
-    [[ "$output" =~ "origin" ]] || false
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "error: clone failed" ]] || false
+    [[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false
 }
 
 @test "clone a non-existent remote" {
     dolt remote add test-remote http://localhost:50051/test-org/test-repo
     cd "dolt-repo-clones"
-    run dolt clone foo/bar
+    run dolt clone http://localhost:50051/foo/bar
     [ "$status" -eq 1 ]
-    skip "Cloning a non-existant repository fails weirdly and leaves trash"
-    [ "$output" = "fatal: repository 'foo/bar' does not exist" ]
-    [[ ! "$output" =~ "permission denied" ]] || false
-    [ ! -d bar ]
+    [[ "$output" =~ "error: clone failed" ]] || false
+    [[ "$output" =~ "cause: remote at that url contains no Dolt data" ]] || false
 }
 
 @test "clone a different branch than master" {
@@ -300,7 +289,6 @@ SQL
     [ "$output" = "" ]
     run dolt fetch poop refs/heads/master:refs/remotes/poop/master
     [ "$status" -eq 1 ]
-    echo $output
     [[ "$output" =~ "unknown remote" ]] || false
     run dolt fetch test-remote refs/heads/master:refs/remotes/test-remote/poop
     [ "$status" -eq 0 ]
@@ -422,12 +410,11 @@ SQL
     dolt commit -m "test commit"
     dolt push test-remote master
     cd "dolt-repo-clones/test-repo"
-    skip "This remotes/origin/master syntax no longer works but works on git"
     run dolt merge remotes/origin/master
     [ "$status" -eq 0 ]
     # This needs to say up-to-date like the skipped test above
-    # [[ "$output" =~ "up to date" ]]
-    dolt fetch origin master
+    [[ "$output" =~ "Everything up-to-date" ]]
+    run dolt fetch origin master
     [ "$status" -eq 0 ]
     run dolt merge remotes/origin/master
     [ "$status" -eq 0 ]

go/cmd/dolt/commands/clone.go

@@ -16,6 +16,7 @@ package commands
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -288,6 +289,10 @@ func cloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch s
 	wg.Wait()
 
 	if err != nil {
+		if err == datas.ErrNoData {
+			err = errors.New("remote at that url contains no Dolt data")
+		}
+
 		return errhand.BuildDError("error: clone failed").AddCause(err).Build()
 	}
 

go/cmd/dolt/commands/fetch.go

@@ -119,7 +119,7 @@ func getRefSpecs(args []string, dEnv *env.DoltEnv, remotes map[string]env.Remote
 	var rs []ref.RemoteRefSpec
 	var verr errhand.VerboseError
 	if len(args) != 0 {
-		rs, verr = parseRSFromArgs(args)
+		rs, verr = parseRSFromArgs(remName, args)
 	} else {
 		rs, verr = dEnv.GetRefSpecs(remName)
 	}
@@ -131,7 +131,7 @@ func getRefSpecs(args []string, dEnv *env.DoltEnv, remotes map[string]env.Remote
 	return remote, rs, verr
 }
 
-func parseRSFromArgs(args []string) ([]ref.RemoteRefSpec, errhand.VerboseError) {
+func parseRSFromArgs(remName string, args []string) ([]ref.RemoteRefSpec, errhand.VerboseError) {
 	var refSpecs []ref.RemoteRefSpec
 	for i := 0; i < len(args); i++ {
 		rsStr := args[i]
@@ -141,6 +141,16 @@ func parseRSFromArgs(args []string) ([]ref.RemoteRefSpec, errhand.VerboseError)
 			return nil, errhand.BuildDError("error: '%s' is not a valid refspec.", rsStr).SetPrintUsage().Build()
 		}
 
+		if _, ok := rs.(ref.BranchToBranchRefSpec); ok {
+			local := "refs/heads/" + rsStr
+			remTracking := "remotes/" + remName + "/" + rsStr
+			rs2, err := ref.ParseRefSpec(local + ":" + remTracking)
+
+			if err == nil {
+				rs = rs2
+			}
+		}
+
 		if rrs, ok := rs.(ref.RemoteRefSpec); !ok {
 			return nil, errhand.BuildDError("error: '%s' is not a valid refspec referring to a remote tracking branch", rsStr).Build()
 		} else {

go/cmd/dolt/commands/push.go

@@ -102,23 +102,31 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
 	}
 
 	remoteName := "origin"
-	remote, remoteOK := remotes[remoteName]
 
+	args = apr.Args()
+	if len(args) == 1 {
+		if _, ok := remotes[args[0]]; ok {
+			remoteName = args[0]
+			args = []string{}
+		}
+	}
+
+	remote, remoteOK := remotes[remoteName]
 	currentBranch := dEnv.RepoState.CWBHeadRef()
 	upstream, hasUpstream := dEnv.RepoState.Branches[currentBranch.GetPath()]
 
 	var refSpec ref.RefSpec
 	var verr errhand.VerboseError
-	if remoteOK && apr.NArg() == 1 {
-		refSpecStr := apr.Arg(0)
+	if remoteOK && len(args) == 1 {
+		refSpecStr := args[0]
 		refSpec, err = ref.ParseRefSpec(refSpecStr)
 
 		if err != nil {
 			verr = errhand.BuildDError("error: invalid refspec '%s'", refSpecStr).AddCause(err).Build()
 		}
-	} else if apr.NArg() == 2 {
-		remoteName = apr.Arg(0)
-		refSpecStr := apr.Arg(1)
+	} else if len(args) == 2 {
+		remoteName = args[0]
+		refSpecStr := args[1]
 		refSpec, err = ref.ParseRefSpec(refSpecStr)
 
 		if err != nil {
@@ -127,7 +135,7 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
 	} else if apr.Contains(SetUpstreamFlag) {
 		verr = errhand.BuildDError("error: --set-upstream requires <remote> and <refspec> params.").SetPrintUsage().Build()
 	} else if hasUpstream {
-		if apr.NArg() > 0 {
+		if len(args) > 0 {
 			cli.PrintErrf("fatal: upstream branch set for '%s'.  Use 'dolt push' without arguments to push.\n", currentBranch)
 			return 1
 		}
@@ -148,7 +156,7 @@ func (cmd PushCmd) Exec(ctx context.Context, commandStr string, args []string, d
 		remoteName = upstream.Remote
 		refSpec, _ = ref.NewBranchToBranchRefSpec(currentBranch.(ref.BranchRef), upstream.Merge.Ref.(ref.BranchRef))
 	} else {
-		if apr.NArg() == 0 {
+		if len(args) == 0 {
 			remoteName = "<remote>"
 			if defRemote, verr := dEnv.GetDefaultRemote(); verr == nil {
 				remoteName = defRemote.Name
@@ -273,7 +281,7 @@ func pushToRemoteBranch(ctx context.Context, dEnv *env.DoltEnv, mode ref.RefUpda
 	cm, err := localDB.Resolve(ctx, cs)
 
 	if err != nil {
-		return errhand.BuildDError("error: unable to find %v", srcRef.GetPath()).Build()
+		return errhand.BuildDError("error: refspec '%v' not found.", srcRef.GetPath()).Build()
 	} else {
 		wg, progChan, pullerEventCh := runProgFuncs()
 		err = actions.Push(ctx, dEnv, mode, destRef.(ref.BranchRef), remoteRef.(ref.RemoteRef), localDB, remoteDB, cm, progChan, pullerEventCh)

go/cmd/dolt/commands/remote.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"os"
 	"path"
 	"path/filepath"
 	"strings"
@@ -231,7 +230,7 @@ func getAbsFileRemoteUrl(urlStr string, fs filesys.Filesys) (string, error) {
 	exists, isDir := fs.Exists(urlStr)
 
 	if !exists {
-		return "", os.ErrNotExist
+		return "", filesys.ErrDirNotExist
 	} else if !isDir {
 		return "", filesys.ErrIsFile
 	}
@@ -259,19 +258,19 @@ func addRemote(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) errhand.Verbos
 	}
 
 	remoteUrl := apr.Arg(2)
-	scheme, remoteUrl, err := getAbsRemoteUrl(dEnv.FS, dEnv.Config, remoteUrl)
+	scheme, absRemoteUrl, err := getAbsRemoteUrl(dEnv.FS, dEnv.Config, remoteUrl)
 
 	if err != nil {
-		return errhand.BuildDError("error: '%s' is not valid.", remoteUrl).Build()
+		return errhand.BuildDError("error: '%s' is not valid.", remoteUrl).AddCause(err).Build()
 	}
 
-	params, verr := parseRemoteArgs(apr, scheme, remoteUrl)
+	params, verr := parseRemoteArgs(apr, scheme, absRemoteUrl)
 
 	if verr != nil {
 		return verr
 	}
 
-	r := env.NewRemote(remoteName, remoteUrl, params)
+	r := env.NewRemote(remoteName, absRemoteUrl, params)
 	dEnv.RepoState.AddRemote(r)
 	err = dEnv.RepoState.Save(dEnv.FS)
 

go/libraries/doltcore/env/actions/commit.go

@@ -193,6 +193,10 @@ func AddCommits(ctx context.Context, ddb *doltdb.DoltDB, commit *doltdb.Commit,
 		return err
 	}
 
+	if _, ok := hashToCommit[hash]; ok {
+		return nil
+	}
+
 	hashToCommit[hash] = commit
 
 	numParents, err := commit.NumParents()

go/libraries/doltcore/env/environment.go

@@ -703,6 +703,10 @@ func (dEnv *DoltEnv) FindRef(ctx context.Context, refStr string) (ref.DoltRef, e
 	} else if hasRef {
 		return localRef, nil
 	} else {
+		if strings.HasPrefix(refStr, "remotes/") {
+			refStr = refStr[len("remotes/"):]
+		}
+
 		slashIdx := strings.IndexRune(refStr, '/')
 		if slashIdx > 0 {
 			remoteName := refStr[:slashIdx]

go/libraries/doltcore/merge/merge.go

@@ -111,7 +111,12 @@ func (merger *Merger) MergeTable(ctx context.Context, tblName string) (*doltdb.T
 	}
 
 	if h == anch {
-		return mergeTbl, &MergeStats{Operation: TableModified}, nil
+		ms := MergeStats{Operation: TableModified}
+		if h != mh {
+			ms, err = calcTableMergeStats(ctx, tbl, mergeTbl)
+		}
+
+		return mergeTbl, &ms, nil
 	} else if mh == anch {
 		return tbl, &MergeStats{Operation: TableUnmodified}, nil
 	}
@@ -211,6 +216,46 @@ func (merger *Merger) MergeTable(ctx context.Context, tblName string) (*doltdb.T
 	return mergedTable, stats, nil
 }
 
+func calcTableMergeStats(ctx context.Context, tbl *doltdb.Table, mergeTbl *doltdb.Table) (MergeStats, error) {
+	rows, err := tbl.GetRowData(ctx)
+
+	if err != nil {
+		return MergeStats{}, err
+	}
+
+	mergeRows, err := mergeTbl.GetRowData(ctx)
+
+	if err != nil {
+		return MergeStats{}, err
+	}
+
+	ae := atomicerr.New()
+	ch := make(chan diff.DiffSummaryProgress)
+	go func() {
+		defer close(ch)
+		err := diff.Summary(ctx, ch, mergeRows, rows)
+
+		ae.SetIfError(err)
+	}()
+
+	ms := MergeStats{Operation: TableModified}
+	for p := range ch {
+		if ae.IsSet() {
+			break
+		}
+
+		ms.Adds += int(p.Adds)
+		ms.Deletes += int(p.Removes)
+		ms.Modifications += int(p.Changes)
+	}
+
+	if err := ae.Get(); err != nil {
+		return MergeStats{}, err
+	}
+
+	return ms, nil
+}
+
 func stopAndDrain(stop chan<- struct{}, drain <-chan types.ValueChanged) {
 	close(stop)
 	for range drain {

go/libraries/utils/filesys/fs.go

@@ -24,6 +24,7 @@ import (
 
 var ErrIsDir = errors.New("operation not valid on a directory")
 var ErrIsFile = errors.New("operation not valid on a file")
+var ErrDirNotExist = errors.New("directory does not exist")
 
 // ReadableFS is an interface providing read access to objs in a filesystem
 type ReadableFS interface {

go/store/datas/pull.go

@@ -48,6 +48,8 @@ const (
 	defaultBatchSize       = 1 << 12 // 4096 chunks
 )
 
+var ErrNoData = errors.New("no data")
+
 func makeProgTrack(progressCh chan PullProgress) func(moreDone, moreKnown, moreApproxBytesWritten uint64) {
 	var doneCount, knownCount, approxBytesWritten uint64
 	return func(moreDone, moreKnown, moreApproxBytesWritten uint64) {
@@ -69,6 +71,16 @@ func Clone(ctx context.Context, srcDB, sinkDB Database, eventCh chan<- TableFile
 		return errors.New("src db is not a Table File Store")
 	}
 
+	size, err := srcTS.Size(ctx)
+
+	if err != nil {
+		return err
+	}
+
+	if size == 0 {
+		return ErrNoData
+	}
+
 	sinkTS, sinkOK := sinkCS.(nbs.TableFileStore)
 
 	if !sinkOK {

go/utils/remotesrv/grpc.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
+	"path/filepath"
 	"sync/atomic"
 
 	"google.golang.org/grpc/codes"
@@ -278,9 +280,28 @@ func (rs *RemoteChunkStore) GetRepoMetadata(ctx context.Context, req *remotesapi
 		return nil, status.Error(codes.Internal, "Could not get chunkstore")
 	}
 
+	_, tfs, err := cs.Sources(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var size uint64
+	for _, tf := range tfs {
+		path := filepath.Join(req.RepoId.Org, req.RepoId.RepoName, tf.FileID())
+		info, err := os.Stat(path)
+
+		if err != nil {
+			return nil, err
+		}
+
+		size += uint64(info.Size())
+	}
+
 	return &remotesapi.GetRepoMetadataResponse{
-		NbfVersion: cs.Version(),
-		NbsVersion: nbs.StorageVersion,
+		NbfVersion:  cs.Version(),
+		NbsVersion:  nbs.StorageVersion,
+		StorageSize: size,
 	}, nil
 }