dolt/go/cmd/git-dolt{,-smudge}: Add git smudge filter for git-dolt (#1651)

* dolt/go/cmd/git-dolt/config/config.go: Add package comment

* dolt/go/cmd/git-dolt: Factor CloneToRevision function out of Fetch command

* dolt/go/cmd/git-dolt: Factor AppendToFile out of Link command

* dolt/go/cmd/git-dolt: Add install command

* dolt/go/cmd/git-dolt-smudge: Create git-dolt-smudge command to use as a git smudge filter

* dolt/go/cmd/git-dolt/git-dolt.go: Cleanup

* dolt/bats/git-dolt.bats: Add bats test for git dolt install

* Jenkinsfile: Add git-dolt-smudge to dolt/bats stage

* dolt/bats/git-dolt.bats: Add test for git-dolt smudge filter

* dolt/bats/git-dolt.bats: Configure fake name and email in test git repos so that committing works on Jenkins

* dolt/{bats,go}: Improve git-dolt cloning behavior (especially streaming output from dolt clone); refactor

* dolt/{bats,go}: Small tweaks to git-dolt output

bats/git-dolt.bats

@@ -28,6 +28,22 @@ teardown() {
 	rm -rf $BATS_TMPDIR/remotes-$$
 }
 
+@test "git dolt install sets up a smudge filter in the current git repository" {
+	init_git_repo
+	run git dolt install
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" =~ "Installed git-dolt smudge filter" ]] || false
+	[[ "${lines[1]}" =~ "commit the changes to .gitattributes" ]] || false
+
+	run cat .gitattributes
+	[ "${lines[0]}" = "*.git-dolt filter=git-dolt" ]
+
+	run cat .git/config
+	len=${#lines[@]}
+	[ "${lines[len-2]}" = "[filter \"git-dolt\"]" ]
+	[[ "${lines[len-1]}" =~ "smudge = git-dolt-smudge" ]] || false
+}
+
 @test "git dolt link takes a remote url (and an optional revspec and destination directory), clones the repo, and outputs a pointer file" {
 	init_git_repo
 	run git dolt link $REMOTE
@@ -49,13 +65,30 @@ teardown() {
 	[[ "${lines[0]}" =~ "test-repo" ]] || false
 }
 
+@test "git dolt's smudge filter automatically clones dolt repositories referenced in checked out git-dolt pointer files" {
+	init_git_repo
+	git dolt install
+	git dolt link $REMOTE
+	git add .
+	git commit -m "set up git-dolt integration"
+	rm -rf test-repo test-repo.git-dolt
+
+	run git checkout -- test-repo.git-dolt
+	[[ "$output" =~ "Found git-dolt pointer file" ]] || false
+	[[ "$output" =~ "Cloning remote $REMOTE" ]] || false
+	[ -d test-repo ]
+
+	cd test-repo
+	[ `get_head_commit` = "$DOLT_HEAD_COMMIT" ]
+}
+
 @test "git dolt fetch takes the name of a git-dolt pointer file and clones the repo to the specified revision if it doesn't exist" {
 	init_git_repo
 	create_test_pointer
 
 	run git dolt fetch test-repo
 	[ "$status" -eq 0 ]
-	[ "${lines[0]}" = "Dolt repository cloned from remote $REMOTE to directory test-repo at revision $DOLT_HEAD_COMMIT" ]
+	[[ "$output" =~ "Dolt repository cloned from remote $REMOTE to directory test-repo at revision $DOLT_HEAD_COMMIT" ]] || false
 	[ -d test-repo ]
 
 	cd test-repo
@@ -108,6 +141,8 @@ init_git_repo() {
 	mkdir ../git-repo-$$
 	cd ../git-repo-$$
 	git init
+	git config user.email "foo@bar.com"
+	git config user.name "Foo User"
 }
 
 create_test_pointer() {

go/cmd/git-dolt-smudge/git-dolt-smudge.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"io/ioutil"
+	"bufio"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/config"
+	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/doltops"
+	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/utils"
+)
+
+func main() {
+	// Because this is a git smudge filter, the pointer file contents
+	// are read through stdin.
+	r := bufio.NewReader(os.Stdin)
+	bs, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Print the pointer file contents right back to stdout; the smudge filter
+	// uses this output to replace the contents of the smudged file. In this case,
+	// no changes to the file are desired (though this may change).
+	fmt.Printf("%s", bs)
+
+	cfg, err := config.Parse(string(bs))
+	if err != nil {
+		log.Fatalf("error parsing config: %v", err)
+	}
+
+	dirname := utils.LastSegment(cfg.Remote)
+
+	// We send output intended for the console to stderr instead of stdout
+	// or else it will end up in the pointer file.
+	fmt.Fprintf(os.Stderr, "Found git-dolt pointer file. Cloning remote %s to revision %s in directory %s...", cfg.Remote, cfg.Revision, dirname)
+
+	if err := doltops.CloneToRevisionSilent(cfg.Remote, cfg.Revision); err != nil {
+		log.Fatalf("error cloning repository: %v", err)
+	}
+
+	fmt.Fprintln(os.Stderr, "done.")
+}

go/cmd/git-dolt/commands/fetch.go

@@ -2,9 +2,9 @@ package commands
 
 import (
 	"fmt"
-	"os/exec"
 
 	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/config"
+	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/doltops"
 	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/utils"
 )
 
@@ -16,17 +16,10 @@ func Fetch(ptrFname string) error {
 		return err
 	}
 
-	if _, err := exec.Command("dolt", "clone", config.Remote).Output(); err != nil {
-		return fmt.Errorf("error cloning remote repository at %s: %v", config.Remote, err)
+	if err := doltops.CloneToRevision(config.Remote, config.Revision); err != nil {
+		return err
 	}
 
-	dirname := utils.LastSegment(config.Remote)
-	checkoutCmd := exec.Command("dolt", "checkout", "-b", "git-dolt-pinned", config.Revision)
-	checkoutCmd.Dir = dirname
-	if _, err := checkoutCmd.Output(); err != nil {
-		return fmt.Errorf("error checking out revision %s in directory %s: %v", config.Revision, dirname, err)
-	}
-
-	fmt.Printf("Dolt repository cloned from remote %s to directory %s at revision %s\n", config.Remote, dirname, config.Revision)
+	fmt.Printf("Dolt repository cloned from remote %s to directory %s at revision %s\n", config.Remote, utils.LastSegment(config.Remote), config.Revision)
 	return nil
 }

go/cmd/git-dolt/commands/install.go

@@ -0,0 +1,28 @@
+package commands
+
+import (
+	"fmt"
+	"os/exec"
+
+	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/utils"
+)
+
+// Install configures this git repository for use with git-dolt; specifically, it sets up the 
+// smudge filter that automatically clones dolt repos when git-dolt pointer files are checked out.
+func Install() error {
+	if _, err := exec.LookPath("git-dolt-smudge"); err != nil {
+		return fmt.Errorf("can't find git-dolt-smudge in PATH")
+	}
+
+	if err := utils.AppendToFile(".gitattributes", "*.git-dolt filter=git-dolt"); err != nil {
+		return err
+	}
+
+	if err := utils.AppendToFile(".git/config", "[filter \"git-dolt\"]\n\tsmudge = git-dolt-smudge"); err != nil {
+		return err
+	} 
+
+	fmt.Println("Installed git-dolt smudge filter. When git-dolt pointer files are checked out in this git repository, the corresponding Dolt repositories will be automatically cloned.")
+	fmt.Println("\nYou should git commit the changes to .gitattributes.")
+	return nil
+}

go/cmd/git-dolt/commands/link.go

@@ -2,10 +2,9 @@ package commands
 
 import (
 	"fmt"
-	"os"
-	"os/exec"
 
 	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/config"
+	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/doltops"
 	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/env"
 	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/utils"
 )
@@ -13,11 +12,11 @@ import (
 // Link creates a git-dolt pointer file linking the given dolt remote
 // to the current git repository.
 func Link(remote string) error {
-	dirname := utils.LastSegment(remote)
-	if _, err := exec.Command("dolt", "clone", remote).Output(); err != nil {
-		return fmt.Errorf("error cloning remote repository at %s: %v", remote, err)
+	if err := doltops.Clone(remote); err != nil {
+		return err
 	}
-
+	
+	dirname := utils.LastSegment(remote)
 	revision, err := utils.CurrentRevision(dirname)
 	if err != nil {
 		return err
@@ -28,18 +27,12 @@ func Link(remote string) error {
 		return err
 	}
 
-	giFile, err := os.OpenFile(".gitignore", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
-	if err != nil {
-		return fmt.Errorf("error opening .gitignore: %v", err)
-	}
-	defer giFile.Close()
-
-	if _, err = giFile.WriteString(fmt.Sprintf("%s\n", dirname)); err != nil {
-		return fmt.Errorf("error writing to .gitignore at %v", err)
+	if err := utils.AppendToFile(".gitignore", dirname); err != nil {
+		return err
 	}
 
-	fmt.Printf("\nDolt repository successfully linked!\n\n")
-	fmt.Printf("* Dolt repository cloned to %s at revision %s\n", dirname, revision)
+	fmt.Printf("\nDolt repository linked!\n\n")
+	fmt.Printf("* Repository cloned to %s at revision %s\n", dirname, revision)
 	fmt.Printf("* Pointer file created at %s.git-dolt\n", dirname)
 	fmt.Printf("* %s added to .gitignore\n\n", dirname)
 	fmt.Println("You should git commit these results.")

go/cmd/git-dolt/config/config.go

@@ -1,3 +1,5 @@
+// Package config contains types and functions for dealing with
+// git-dolt configuration, including config file I/O.
 package config
 
 import (

go/cmd/git-dolt/doltops/doltops.go

@@ -0,0 +1,83 @@
+// Package doltops contains functions for performing dolt operations
+// using the CLI.
+package doltops
+
+import (
+	"bufio"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/liquidata-inc/ld/dolt/go/cmd/git-dolt/utils"
+)
+
+// Clone clones the specified dolt remote, streaming the output from dolt clone to stdout.
+func Clone(remote string) error {
+	cmd := exec.Command("dolt", "clone", remote)
+	if err := runAndStreamOutput(cmd, "dolt clone"); err != nil {
+		return err
+	}
+	return nil
+}
+
+// CloneToRevision clones the specified dolt remote and checks it out to the specified revision.
+// It streams the output from dolt clone and dolt checkout to stdout.
+func CloneToRevision(remote string, revision string) error {
+	if err := Clone(remote); err != nil {
+		return err
+	}
+
+	dirname := utils.LastSegment(remote)
+	checkoutCmd := exec.Command("dolt", "checkout", "-b", "git-dolt-pinned", revision)
+	checkoutCmd.Dir = dirname
+	if err := runAndStreamOutput(checkoutCmd, "dolt checkout"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func runAndStreamOutput(cmd *exec.Cmd, name string) error {
+	cmdReader, err := cmd.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("error creating StdoutPipe for %s: %v", name, err)
+	}
+
+	scanner := bufio.NewScanner(cmdReader)
+	go func() {
+		for scanner.Scan() {
+			text := scanner.Text()
+			// TODO: Remove this hacky conditional
+			if !strings.HasPrefix(text, "Switched to branch") {
+				fmt.Println(scanner.Text())
+			}
+		}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("error starting %s: %v", name, err)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("error waiting for %s: %v", name, err)
+	}
+
+	return nil
+}
+
+// CloneToRevisionSilent clones the specified dolt remote and checks it out to the specified revision,
+// suppressing all output from dolt clone and dolt checkout.
+func CloneToRevisionSilent(remote string, revision string) error {
+	if err := exec.Command("dolt", "clone", remote).Run(); err != nil {
+		return fmt.Errorf("error cloning remote repository at %s: %v", remote, err)
+	}
+
+	dirname := utils.LastSegment(remote)
+	checkoutCmd := exec.Command("dolt", "checkout", "-b", "git-dolt-pinned", revision)
+	checkoutCmd.Dir = dirname
+	if err := checkoutCmd.Run(); err != nil {
+		return fmt.Errorf("error checking out revision %s in directory %s: %v", revision, dirname, err)
+	}
+
+	return nil
+}

go/cmd/git-dolt/git-dolt.go

@@ -10,18 +10,21 @@ import (
 
 func main() {
 	if _, err := exec.LookPath("dolt"); err != nil {
-		die("It looks like Dolt is not installed on your system. Make sure that the `dolt` binary is in your PATH before attempting to run git-dolt commands.")
+		fmt.Println("It looks like Dolt is not installed on your system. Make sure that the `dolt` binary is in your PATH before attempting to run git-dolt commands.")
+		os.Exit(1)
 	}
 
 	if len(os.Args) == 1 {
 		fmt.Println("Dolt: It's Git for Data.")
-		fmt.Println("Usage")
+		printUsage()
 		return
 	}
 
 	var err error
 
 	switch cmd := os.Args[1]; cmd {
+	case "install":
+		err = commands.Install()
 	case "link":
 		remote := os.Args[2]
 		err = commands.Link(remote)
@@ -33,15 +36,17 @@ func main() {
 		revision := os.Args[3]
 		err = commands.Update(ptrFname, revision)
 	default:
-		die("Unknown command " + cmd)
+		fmt.Printf("Unknown command %s\n", cmd)
+		printUsage()
+		os.Exit(1)
 	}
 
 	if err != nil {
-		die(err)
+		fmt.Printf("Error: %v\n", err)
+		os.Exit(1)
 	}
 }
 
-func die(reason interface{}) {
-	fmt.Printf("Fatal: %v\n", reason)
-	os.Exit(1)
+func printUsage() {
+	fmt.Println("Usage")
 }

go/cmd/git-dolt/utils/append_to_file.go

@@ -0,0 +1,22 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+)
+
+// AppendToFile appends the given string to the given file in the current
+// directory, creating it if it does not exist.
+func AppendToFile(f string, s string) error {
+	giFile, err := os.OpenFile(f, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("error opening %s: %v", f, err)
+	}
+	defer giFile.Close()
+
+	if _, err = giFile.WriteString(fmt.Sprintf("%s\n", s)); err != nil {
+		return fmt.Errorf("error writing to %s at %v", f, err)
+	}
+
+	return nil
+}