Merge branch 'main' into james/prevent-spatial-keys

go/cmd/dolt/commands/backup.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
-	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 	"github.com/dolthub/dolt/go/store/types"
 
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
@@ -270,7 +270,7 @@ func syncBackup(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseR
 		return errhand.BuildDError("error: '%s' is not valid.", b.Url).AddCause(err).Build()
 	case env.ErrInvalidBackupName:
 		return errhand.BuildDError("error: invalid backup name: " + b.Name).Build()
-	case datas.ErrDBUpToDate:
+	case pull.ErrDBUpToDate:
 		return errhand.BuildDError("error: backup already up to date").Build()
 	default:
 		return errhand.BuildDError("error: Unable to save changes.").AddCause(err).Build()

go/cmd/dolt/commands/push.go

@@ -35,6 +35,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/remotestorage"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
 	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 )
 
 var pushDocs = cli.CommandDocumentationContent{
@@ -174,7 +175,7 @@ func (ts *TextSpinner) next() string {
 	return string([]rune{spinnerSeq[ts.seqPos]})
 }
 
-func pullerProgFunc(ctx context.Context, pullerEventCh chan datas.PullerEvent) {
+func pullerProgFunc(ctx context.Context, pullerEventCh chan pull.PullerEvent) {
 	var pos int
 	var currentTreeLevel int
 	var percentBuffered float64
@@ -189,17 +190,17 @@ func pullerProgFunc(ctx context.Context, pullerEventCh chan datas.PullerEvent) {
 			return
 		}
 		switch evt.EventType {
-		case datas.NewLevelTWEvent:
+		case pull.NewLevelTWEvent:
 			if evt.TWEventDetails.TreeLevel != 1 {
 				currentTreeLevel = evt.TWEventDetails.TreeLevel
 				percentBuffered = 0
 			}
-		case datas.DestDBHasTWEvent:
+		case pull.DestDBHasTWEvent:
 			if evt.TWEventDetails.TreeLevel != -1 {
 				currentTreeLevel = evt.TWEventDetails.TreeLevel
 			}
 
-		case datas.LevelUpdateTWEvent:
+		case pull.LevelUpdateTWEvent:
 			if evt.TWEventDetails.TreeLevel != -1 {
 				currentTreeLevel = evt.TWEventDetails.TreeLevel
 				toBuffer := evt.TWEventDetails.ChunksInLevel - evt.TWEventDetails.ChunksAlreadyHad
@@ -209,18 +210,18 @@ func pullerProgFunc(ctx context.Context, pullerEventCh chan datas.PullerEvent) {
 				}
 			}
 
-		case datas.LevelDoneTWEvent:
+		case pull.LevelDoneTWEvent:
 
-		case datas.TableFileClosedEvent:
+		case pull.TableFileClosedEvent:
 			tableFilesBuffered += 1
 
-		case datas.StartUploadTableFileEvent:
+		case pull.StartUploadTableFileEvent:
 
-		case datas.UploadTableFileUpdateEvent:
+		case pull.UploadTableFileUpdateEvent:
 			bps := float64(evt.TFEventDetails.Stats.Read) / evt.TFEventDetails.Stats.Elapsed.Seconds()
 			uploadRate = humanize.Bytes(uint64(bps)) + "/s"
 
-		case datas.EndUploadTableFileEvent:
+		case pull.EndUploadTableFileEvent:
 			filesUploaded += 1
 		}
 
@@ -239,8 +240,8 @@ func pullerProgFunc(ctx context.Context, pullerEventCh chan datas.PullerEvent) {
 	}
 }
 
-func progFunc(ctx context.Context, progChan chan datas.PullProgress) {
-	var latest datas.PullProgress
+func progFunc(ctx context.Context, progChan chan pull.PullProgress) {
+	var latest pull.PullProgress
 	last := time.Now().UnixNano() - 1
 	lenPrinted := 0
 	done := false
@@ -277,9 +278,9 @@ func progFunc(ctx context.Context, progChan chan datas.PullProgress) {
 	}
 }
 
-func runProgFuncs(ctx context.Context) (*sync.WaitGroup, chan datas.PullProgress, chan datas.PullerEvent) {
-	pullerEventCh := make(chan datas.PullerEvent, 128)
-	progChan := make(chan datas.PullProgress, 128)
+func runProgFuncs(ctx context.Context) (*sync.WaitGroup, chan pull.PullProgress, chan pull.PullerEvent) {
+	pullerEventCh := make(chan pull.PullerEvent, 128)
+	progChan := make(chan pull.PullProgress, 128)
 	wg := &sync.WaitGroup{}
 
 	wg.Add(1)
@@ -297,7 +298,7 @@ func runProgFuncs(ctx context.Context) (*sync.WaitGroup, chan datas.PullProgress
 	return wg, progChan, pullerEventCh
 }
 
-func stopProgFuncs(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) {
+func stopProgFuncs(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) {
 	cancel()
 	close(progChan)
 	close(pullerEventCh)

go/cmd/dolt/commands/read_tables.go

@@ -183,7 +183,7 @@ func pullTableValue(ctx context.Context, dEnv *env.DoltEnv, srcDB *doltdb.DoltDB
 	newCtx, cancelFunc := context.WithCancel(ctx)
 	cli.Println("Retrieving", tblName)
 	wg, progChan, pullerEventCh := runProgFuncs(newCtx)
-	err = dEnv.DoltDB.PushChunksForRefHash(ctx, dEnv.TempTableFilesDir(), srcDB, tblHash, pullerEventCh)
+	err = dEnv.DoltDB.PullChunks(ctx, dEnv.TempTableFilesDir(), srcDB, tblHash, progChan, pullerEventCh)
 	stopProgFuncs(cancelFunc, wg, progChan, pullerEventCh)
 	if err != nil {
 		return nil, errhand.BuildDError("Failed reading chunks for remote table '%s' at '%s'", tblName, commitStr).AddCause(err).Build()

go/cmd/dolt/dolt.go

@@ -51,7 +51,7 @@ import (
 )
 
 const (
-	Version = "0.35.10"
+	Version = "0.36.0"
 )
 
 var dumpDocsCommand = &commands.DumpDocsCmd{}

go/go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20201005193433-3ee972b1d078
 	github.com/dolthub/fslock v0.0.3
-	github.com/dolthub/go-mysql-server v0.11.1-0.20220128120832-8e12c8568cc0
+	github.com/dolthub/go-mysql-server v0.11.1-0.20220131205155-2c7d2f75cb60
 	github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371
 	github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66
 	github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81

go/go.sum

@@ -174,8 +174,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
-github.com/dolthub/go-mysql-server v0.11.1-0.20220128120832-8e12c8568cc0 h1:gcOmnYNoLwlakoxLVRk0Vh2RzK4SONdIXnp8kB9mxV4=
-github.com/dolthub/go-mysql-server v0.11.1-0.20220128120832-8e12c8568cc0/go.mod h1:46f9AwcXU81JvYIl1nnq0iuS18D5WTcWHKJbXaK7Stw=
+github.com/dolthub/go-mysql-server v0.11.1-0.20220131205155-2c7d2f75cb60 h1:9WsF5q0rVo7p0ysq/JHSgYBQ8EUJb95kUldZhHg70kI=
+github.com/dolthub/go-mysql-server v0.11.1-0.20220131205155-2c7d2f75cb60/go.mod h1:46f9AwcXU81JvYIl1nnq0iuS18D5WTcWHKJbXaK7Stw=
 github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g=
 github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms=
 github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=

go/libraries/doltcore/doltdb/commit_hooks.go

@@ -24,7 +24,9 @@ import (
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/ref"
 	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 	"github.com/dolthub/dolt/go/store/hash"
+	"github.com/dolthub/dolt/go/store/types"
 )
 
 type PushOnWriteHook struct {
@@ -76,8 +78,15 @@ func pushDataset(ctx context.Context, destDB, srcDB datas.Database, tempTableDir
 		return err
 	}
 
-	puller, err := datas.NewPuller(ctx, tempTableDir, defaultChunksPerTF, srcDB, destDB, stRef.TargetHash(), nil)
-	if err == datas.ErrDBUpToDate {
+	srcCS := datas.ChunkStoreFromDatabase(srcDB)
+	destCS := datas.ChunkStoreFromDatabase(destDB)
+	wrf, err := types.WalkRefsForChunkStore(srcCS)
+	if err != nil {
+		return err
+	}
+
+	puller, err := pull.NewPuller(ctx, tempTableDir, defaultChunksPerTF, srcCS, destCS, wrf, stRef.TargetHash(), nil)
+	if err == pull.ErrDBUpToDate {
 		return nil
 	} else if err != nil {
 		return err

go/libraries/doltcore/doltdb/doltdb.go

@@ -28,6 +28,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/spec"
 	"github.com/dolthub/dolt/go/store/types"
@@ -1239,13 +1240,20 @@ func (ddb *DoltDB) pruneUnreferencedDatasets(ctx context.Context) error {
 	return nil
 }
 
-// PushChunks initiates a push into a database from the source database given, at the Value ref given. Pull progress is
-// communicated over the provided channel.
-func (ddb *DoltDB) PushChunks(ctx context.Context, tempDir string, srcDB *DoltDB, rf types.Ref, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
-	if datas.CanUsePuller(srcDB.db) && datas.CanUsePuller(ddb.db) {
-		puller, err := datas.NewPuller(ctx, tempDir, defaultChunksPerTF, srcDB.db, ddb.db, rf.TargetHash(), pullerEventCh)
+// PullChunks initiates a pull into this database from the source database
+// given, pulling all chunks reachable from the given targetHash. Pull progress
+// is communicated over the provided channel.
+func (ddb *DoltDB) PullChunks(ctx context.Context, tempDir string, srcDB *DoltDB, targetHash hash.Hash, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) error {
+	srcCS := datas.ChunkStoreFromDatabase(srcDB.db)
+	destCS := datas.ChunkStoreFromDatabase(ddb.db)
+	wrf, err := types.WalkRefsForChunkStore(srcCS)
+	if err != nil {
+		return err
+	}
 
-		if err == datas.ErrDBUpToDate {
+	if datas.CanUsePuller(srcDB.db) && datas.CanUsePuller(ddb.db) {
+		puller, err := pull.NewPuller(ctx, tempDir, defaultChunksPerTF, srcCS, destCS, wrf, targetHash, pullerEventCh)
+		if err == pull.ErrDBUpToDate {
 			return nil
 		} else if err != nil {
 			return err
@@ -1253,43 +1261,12 @@ func (ddb *DoltDB) PushChunks(ctx context.Context, tempDir string, srcDB *DoltDB
 
 		return puller.Pull(ctx)
 	} else {
-		return datas.Pull(ctx, srcDB.db, ddb.db, rf, progChan)
+		return pull.Pull(ctx, srcCS, destCS, wrf, targetHash, progChan)
 	}
 }
 
-func (ddb *DoltDB) PushChunksForRefHash(ctx context.Context, tempDir string, srcDB *DoltDB, h hash.Hash, pullerEventCh chan datas.PullerEvent) error {
-	if datas.CanUsePuller(srcDB.db) && datas.CanUsePuller(ddb.db) {
-		puller, err := datas.NewPuller(ctx, tempDir, defaultChunksPerTF, srcDB.db, ddb.db, h, pullerEventCh)
-
-		if err == datas.ErrDBUpToDate {
-			return nil
-		} else if err != nil {
-			return err
-		}
-
-		return puller.Pull(ctx)
-	} else {
-		return errors.New("this type of chunk store does not support this operation")
-	}
-}
-
-// PullChunks initiates a pull into a database from the source database given, at the commit given. Progress is
-// communicated over the provided channel.
-func (ddb *DoltDB) PullChunks(ctx context.Context, tempDir string, srcDB *DoltDB, stRef types.Ref, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
-	if datas.CanUsePuller(srcDB.db) && datas.CanUsePuller(ddb.db) {
-		puller, err := datas.NewPuller(ctx, tempDir, defaultChunksPerTF, srcDB.db, ddb.db, stRef.TargetHash(), pullerEventCh)
-		if err != nil {
-			return err
-		}
-
-		return puller.Pull(ctx)
-	} else {
-		return datas.PullWithoutBatching(ctx, srcDB.db, ddb.db, stRef, progChan)
-	}
-}
-
-func (ddb *DoltDB) Clone(ctx context.Context, destDB *DoltDB, eventCh chan<- datas.TableFileEvent) error {
-	return datas.Clone(ctx, ddb.db, destDB.db, eventCh)
+func (ddb *DoltDB) Clone(ctx context.Context, destDB *DoltDB, eventCh chan<- pull.TableFileEvent) error {
+	return pull.Clone(ctx, datas.ChunkStoreFromDatabase(ddb.db), datas.ChunkStoreFromDatabase(destDB.db), eventCh)
 }
 
 func (ddb *DoltDB) SetCommitHooks(ctx context.Context, postHooks []datas.CommitHook) *DoltDB {

go/libraries/doltcore/dtestutils/data.go

@@ -111,6 +111,15 @@ func init() {
 	if err != nil {
 		panic(err)
 	}
+
+	_, err = TypedSchema.Checks().AddCheck("test-check", "age < 123", true)
+	if err != nil {
+		panic(err)
+	}
+	_, err = UntypedSchema.Checks().AddCheck("test-check", "age < 123", true)
+	if err != nil {
+		panic(err)
+	}
 }
 
 func NewTypedRow(id uuid.UUID, name string, age uint, isMarried bool, title *string) row.Row {

go/libraries/doltcore/env/actions/clone.go

@@ -29,7 +29,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/ref"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 	"github.com/dolthub/dolt/go/libraries/utils/strhelp"
-	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 	"github.com/dolthub/dolt/go/store/types"
 )
 
@@ -88,7 +88,7 @@ func EnvForClone(ctx context.Context, nbf *types.NomsBinFormat, r env.Remote, di
 	return dEnv, nil
 }
 
-func cloneProg(eventCh <-chan datas.TableFileEvent) {
+func cloneProg(eventCh <-chan pull.TableFileEvent) {
 	var (
 		chunks            int64
 		chunksDownloading int64
@@ -99,20 +99,20 @@ func cloneProg(eventCh <-chan datas.TableFileEvent) {
 	cliPos = cli.DeleteAndPrint(cliPos, "Retrieving remote information.")
 	for tblFEvt := range eventCh {
 		switch tblFEvt.EventType {
-		case datas.Listed:
+		case pull.Listed:
 			for _, tf := range tblFEvt.TableFiles {
 				chunks += int64(tf.NumChunks())
 			}
-		case datas.DownloadStart:
+		case pull.DownloadStart:
 			for _, tf := range tblFEvt.TableFiles {
 				chunksDownloading += int64(tf.NumChunks())
 			}
-		case datas.DownloadSuccess:
+		case pull.DownloadSuccess:
 			for _, tf := range tblFEvt.TableFiles {
 				chunksDownloading -= int64(tf.NumChunks())
 				chunksDownloaded += int64(tf.NumChunks())
 			}
-		case datas.DownloadFailed:
+		case pull.DownloadFailed:
 			// Ignore for now and output errors on the main thread
 		}
 
@@ -124,7 +124,7 @@ func cloneProg(eventCh <-chan datas.TableFileEvent) {
 }
 
 func CloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch string, dEnv *env.DoltEnv) error {
-	eventCh := make(chan datas.TableFileEvent, 128)
+	eventCh := make(chan pull.TableFileEvent, 128)
 
 	wg := &sync.WaitGroup{}
 	wg.Add(1)
@@ -139,7 +139,7 @@ func CloneRemote(ctx context.Context, srcDB *doltdb.DoltDB, remoteName, branch s
 	wg.Wait()
 
 	if err != nil {
-		if err == datas.ErrNoData {
+		if err == pull.ErrNoData {
 			err = ErrNoDataAtRemote
 		}
 		return fmt.Errorf("%w; %s", ErrCloneFailed, err.Error())

go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/ref"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
-	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/types"
 )
@@ -220,8 +220,8 @@ func mustForkDB(t *testing.T, fromDB *doltdb.DoltDB, bn string, cm *doltdb.Commi
 	forkEnv := createUninitializedEnv()
 	err = forkEnv.InitRepo(context.Background(), types.Format_Default, "Bill Billerson", "bill@billerson.com", env.DefaultInitBranch)
 	require.NoError(t, err)
-	p1 := make(chan datas.PullProgress)
-	p2 := make(chan datas.PullerEvent)
+	p1 := make(chan pull.PullProgress)
+	p2 := make(chan pull.PullerEvent)
 	go func() {
 		for range p1 {
 		}
@@ -230,8 +230,8 @@ func mustForkDB(t *testing.T, fromDB *doltdb.DoltDB, bn string, cm *doltdb.Commi
 		for range p2 {
 		}
 	}()
-	err = forkEnv.DoltDB.PullChunks(context.Background(), "", fromDB, stref, p1, p2)
-	if err == datas.ErrDBUpToDate {
+	err = forkEnv.DoltDB.PullChunks(context.Background(), "", fromDB, stref.TargetHash(), p1, p2)
+	if err == pull.ErrDBUpToDate {
 		err = nil
 	}
 	require.NoError(t, err)

go/libraries/doltcore/env/actions/prog_handlers.go

@@ -18,10 +18,10 @@ import (
 	"context"
 	"sync"
 
-	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 )
 
-func pullerProgFunc(ctx context.Context, pullerEventCh <-chan datas.PullerEvent) {
+func pullerProgFunc(ctx context.Context, pullerEventCh <-chan pull.PullerEvent) {
 	for {
 		select {
 		case <-ctx.Done():
@@ -37,7 +37,7 @@ func pullerProgFunc(ctx context.Context, pullerEventCh <-chan datas.PullerEvent)
 	}
 }
 
-func progFunc(ctx context.Context, progChan <-chan datas.PullProgress) {
+func progFunc(ctx context.Context, progChan <-chan pull.PullProgress) {
 	for {
 		select {
 		case <-ctx.Done():
@@ -53,9 +53,9 @@ func progFunc(ctx context.Context, progChan <-chan datas.PullProgress) {
 	}
 }
 
-func NoopRunProgFuncs(ctx context.Context) (*sync.WaitGroup, chan datas.PullProgress, chan datas.PullerEvent) {
-	pullerEventCh := make(chan datas.PullerEvent)
-	progChan := make(chan datas.PullProgress)
+func NoopRunProgFuncs(ctx context.Context) (*sync.WaitGroup, chan pull.PullProgress, chan pull.PullerEvent) {
+	pullerEventCh := make(chan pull.PullerEvent)
+	progChan := make(chan pull.PullProgress)
 	wg := &sync.WaitGroup{}
 
 	wg.Add(1)
@@ -73,7 +73,7 @@ func NoopRunProgFuncs(ctx context.Context) (*sync.WaitGroup, chan datas.PullProg
 	return wg, progChan, pullerEventCh
 }
 
-func NoopStopProgFuncs(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) {
+func NoopStopProgFuncs(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) {
 	cancel()
 	close(progChan)
 	close(pullerEventCh)

go/libraries/doltcore/env/actions/remotes.go

@@ -30,6 +30,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/events"
 	"github.com/dolthub/dolt/go/libraries/utils/earl"
 	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 )
 
 var ErrCantFF = errors.New("can't fast forward merge")
@@ -42,15 +43,15 @@ var ErrFailedToDeleteBackup = errors.New("failed to delete backup")
 var ErrFailedToGetBackupDb = errors.New("failed to get backup db")
 var ErrUnknownPushErr = errors.New("unknown push error")
 
-type ProgStarter func(ctx context.Context) (*sync.WaitGroup, chan datas.PullProgress, chan datas.PullerEvent)
-type ProgStopper func(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent)
+type ProgStarter func(ctx context.Context) (*sync.WaitGroup, chan pull.PullProgress, chan pull.PullerEvent)
+type ProgStopper func(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent)
 
 // Push will update a destination branch, in a given destination database if it can be done as a fast forward merge.
 // This is accomplished first by verifying that the remote tracking reference for the source database can be updated to
 // the given commit via a fast forward merge.  If this is the case, an attempt will be made to update the branch in the
 // destination db to the given commit via fast forward move.  If that succeeds the tracking branch is updated in the
 // source db.
-func Push(ctx context.Context, tempTableDir string, mode ref.UpdateMode, destRef ref.BranchRef, remoteRef ref.RemoteRef, srcDB, destDB *doltdb.DoltDB, commit *doltdb.Commit, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
+func Push(ctx context.Context, tempTableDir string, mode ref.UpdateMode, destRef ref.BranchRef, remoteRef ref.RemoteRef, srcDB, destDB *doltdb.DoltDB, commit *doltdb.Commit, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) error {
 	var err error
 	if mode == ref.FastForwardOnly {
 		canFF, err := srcDB.CanFastForward(ctx, remoteRef, commit)
@@ -68,7 +69,7 @@ func Push(ctx context.Context, tempTableDir string, mode ref.UpdateMode, destRef
 		return err
 	}
 
-	err = destDB.PushChunks(ctx, tempTableDir, srcDB, rf, progChan, pullerEventCh)
+	err = destDB.PullChunks(ctx, tempTableDir, srcDB, rf.TargetHash(), progChan, pullerEventCh)
 
 	if err != nil {
 		return err
@@ -139,7 +140,7 @@ func DoPush(ctx context.Context, rsr env.RepoStateReader, rsw env.RepoStateWrite
 }
 
 // PushTag pushes a commit tag and all underlying data from a local source database to a remote destination database.
-func PushTag(ctx context.Context, tempTableDir string, destRef ref.TagRef, srcDB, destDB *doltdb.DoltDB, tag *doltdb.Tag, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
+func PushTag(ctx context.Context, tempTableDir string, destRef ref.TagRef, srcDB, destDB *doltdb.DoltDB, tag *doltdb.Tag, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) error {
 	var err error
 
 	rf, err := tag.GetStRef()
@@ -148,7 +149,7 @@ func PushTag(ctx context.Context, tempTableDir string, destRef ref.TagRef, srcDB
 		return err
 	}
 
-	err = destDB.PushChunks(ctx, tempTableDir, srcDB, rf, progChan, pullerEventCh)
+	err = destDB.PullChunks(ctx, tempTableDir, srcDB, rf.TargetHash(), progChan, pullerEventCh)
 
 	if err != nil {
 		return err
@@ -250,29 +251,29 @@ func DeleteRemoteBranch(ctx context.Context, targetRef ref.BranchRef, remoteRef
 }
 
 // FetchCommit takes a fetches a commit and all underlying data from a remote source database to the local destination database.
-func FetchCommit(ctx context.Context, tempTablesDir string, srcDB, destDB *doltdb.DoltDB, srcDBCommit *doltdb.Commit, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
+func FetchCommit(ctx context.Context, tempTablesDir string, srcDB, destDB *doltdb.DoltDB, srcDBCommit *doltdb.Commit, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) error {
 	stRef, err := srcDBCommit.GetStRef()
 
 	if err != nil {
 		return err
 	}
 
-	return destDB.PullChunks(ctx, tempTablesDir, srcDB, stRef, progChan, pullerEventCh)
+	return destDB.PullChunks(ctx, tempTablesDir, srcDB, stRef.TargetHash(), progChan, pullerEventCh)
 }
 
 // FetchTag takes a fetches a commit tag and all underlying data from a remote source database to the local destination database.
-func FetchTag(ctx context.Context, tempTableDir string, srcDB, destDB *doltdb.DoltDB, srcDBTag *doltdb.Tag, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error {
+func FetchTag(ctx context.Context, tempTableDir string, srcDB, destDB *doltdb.DoltDB, srcDBTag *doltdb.Tag, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) error {
 	stRef, err := srcDBTag.GetStRef()
 
 	if err != nil {
 		return err
 	}
 
-	return destDB.PullChunks(ctx, tempTableDir, srcDB, stRef, progChan, pullerEventCh)
+	return destDB.PullChunks(ctx, tempTableDir, srcDB, stRef.TargetHash(), progChan, pullerEventCh)
 }
 
 // Clone pulls all data from a remote source database to a local destination database.
-func Clone(ctx context.Context, srcDB, destDB *doltdb.DoltDB, eventCh chan<- datas.TableFileEvent) error {
+func Clone(ctx context.Context, srcDB, destDB *doltdb.DoltDB, eventCh chan<- pull.TableFileEvent) error {
 	return srcDB.Clone(ctx, destDB, eventCh)
 }
 
@@ -317,7 +318,7 @@ func FetchFollowTags(ctx context.Context, tempTableDir string, srcDB, destDB *do
 		progStopper(cancelFunc, wg, progChan, pullerEventCh)
 		if err == nil {
 			cli.Println()
-		} else if err == datas.ErrDBUpToDate {
+		} else if err == pull.ErrDBUpToDate {
 			err = nil
 		}
 
@@ -361,7 +362,7 @@ func FetchRemoteBranch(ctx context.Context, tempTablesDir string, rem env.Remote
 	progStopper(cancelFunc, wg, progChan, pullerEventCh)
 	if err == nil {
 		cli.Println()
-	} else if err == datas.ErrDBUpToDate {
+	} else if err == pull.ErrDBUpToDate {
 		err = nil
 	}
 
@@ -457,7 +458,7 @@ func SyncRoots(ctx context.Context, srcDb, destDb *doltdb.DoltDB, tempTableDir s
 	}
 
 	if srcRoot == destRoot {
-		return datas.ErrDBUpToDate
+		return pull.ErrDBUpToDate
 	}
 
 	newCtx, cancelFunc := context.WithCancel(ctx)
@@ -469,7 +470,7 @@ func SyncRoots(ctx context.Context, srcDb, destDb *doltdb.DoltDB, tempTableDir s
 		}
 	}()
 
-	err = destDb.PushChunksForRefHash(ctx, tempTableDir, srcDb, srcRoot, pullerEventCh)
+	err = destDb.PullChunks(ctx, tempTableDir, srcDb, srcRoot, progChan, pullerEventCh)
 	if err != nil {
 		return err
 	}

go/libraries/doltcore/schema/alterschema/addcolumn.go

@@ -123,6 +123,14 @@ func addColumnToSchema(
 	}
 	newSch.Indexes().AddIndex(sch.Indexes().AllIndexes()...)
 
+	// Copy over all checks from the old schema
+	for _, check := range sch.Checks().AllChecks() {
+		_, err := newSch.Checks().AddCheck(check.Name(), check.Expression(), check.Enforced())
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	pkOrds, err := modifyPkOrdinals(sch, newSch)
 	if err != nil {
 		return nil, err

go/libraries/doltcore/schema/alterschema/addcolumn_test.go

@@ -157,6 +157,7 @@ func TestAddColumnToTable(t *testing.T) {
 			index := sch.Indexes().GetByName(dtestutils.IndexName)
 			assert.NotNil(t, index)
 			tt.expectedSchema.Indexes().AddIndex(index)
+			tt.expectedSchema.Checks().AddCheck("test-check", "age < 123", true)
 			require.Equal(t, tt.expectedSchema, sch)
 
 			rowData, err := updatedTable.GetNomsRowData(ctx)

go/libraries/doltcore/schema/alterschema/addpk.go

@@ -119,6 +119,14 @@ func AddPrimaryKeyToTable(ctx context.Context, table *doltdb.Table, tableName st
 		return nil, err
 	}
 
+	// Copy over all checks from the old schema
+	for _, check := range sch.Checks().AllChecks() {
+		_, err := newSchema.Checks().AddCheck(check.Name(), check.Expression(), check.Enforced())
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	newSchema.Indexes().AddIndex(sch.Indexes().AllIndexes()...)
 	err = newSchema.SetPkOrdinals(pkOrdinals)
 	if err != nil {

go/libraries/doltcore/schema/alterschema/addpk_test.go

@@ -61,7 +61,10 @@ func TestAddPk(t *testing.T) {
 		assert.NoError(t, err)
 		assert.False(t, originalMap.Empty())
 
-		exitCode := commands.SqlCmd{}.Exec(ctx, "sql", []string{"-q", "ALTER TABLE test ADD PRIMARY KEY(id)"}, dEnv)
+		exitCode := commands.SqlCmd{}.Exec(ctx, "sql", []string{"-q", "ALTER TABLE test ADD constraint test_check CHECK (c1 > 0)"}, dEnv)
+		require.Equal(t, 0, exitCode)
+
+		exitCode = commands.SqlCmd{}.Exec(ctx, "sql", []string{"-q", "ALTER TABLE test ADD PRIMARY KEY(id)"}, dEnv)
 		require.Equal(t, 0, exitCode)
 
 		table, err = getTable(ctx, dEnv, "test")
@@ -70,6 +73,9 @@ func TestAddPk(t *testing.T) {
 		sch, err := table.GetSchema(ctx)
 		assert.NoError(t, err)
 
+		assert.Equal(t, 1, sch.Checks().Count())
+		assert.Equal(t, "test_check", sch.Checks().AllChecks()[0].Name())
+
 		// Assert the new index map is not empty
 		newMap, err := table.GetNomsIndexRowData(ctx, indexName)
 		assert.NoError(t, err)

go/libraries/doltcore/schema/alterschema/dropcolumn.go

@@ -88,6 +88,14 @@ func DropColumn(ctx context.Context, tbl *doltdb.Table, colName string, foreignK
 	}
 	newSch.Indexes().AddIndex(sch.Indexes().AllIndexes()...)
 
+	// Copy over all checks from the old schema
+	for _, check := range sch.Checks().AllChecks() {
+		_, err := newSch.Checks().AddCheck(check.Name(), check.Expression(), check.Enforced())
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	pkOrds, err := modifyPkOrdinals(sch, newSch)
 	if err != nil {
 		return nil, err

go/libraries/doltcore/schema/alterschema/dropcolumn_test.go

@@ -84,6 +84,7 @@ func TestDropColumn(t *testing.T) {
 			require.NoError(t, err)
 			index := originalSch.Indexes().GetByName(dtestutils.IndexName)
 			tt.expectedSchema.Indexes().AddIndex(index)
+			tt.expectedSchema.Checks().AddCheck("test-check", "age < 123", true)
 			require.Equal(t, tt.expectedSchema, sch)
 
 			rowData, err := updatedTable.GetNomsRowData(ctx)
@@ -151,6 +152,8 @@ func TestDropColumnUsedByIndex(t *testing.T) {
 			require.NoError(t, err)
 			originalSch, err := tbl.GetSchema(ctx)
 			require.NoError(t, err)
+			tt.expectedSchema.Checks().AddCheck("test-check", "age < 123", true)
+
 			index := originalSch.Indexes().GetByName(dtestutils.IndexName)
 			assert.NotNil(t, index)
 			if tt.expectedIndex {

go/libraries/doltcore/schema/alterschema/droppk.go

@@ -49,6 +49,14 @@ func DropPrimaryKeyFromTable(ctx context.Context, table *doltdb.Table, nbf *type
 
 	newSchema.Indexes().AddIndex(sch.Indexes().AllIndexes()...)
 
+	// Copy over all checks from the old schema
+	for _, check := range sch.Checks().AllChecks() {
+		_, err := newSchema.Checks().AddCheck(check.Name(), check.Expression(), check.Enforced())
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	table, err = table.UpdateSchema(ctx, newSchema)
 	if err != nil {
 		return nil, err

go/libraries/doltcore/schema/alterschema/modifycolumn.go

@@ -350,6 +350,14 @@ func replaceColumnInSchema(sch schema.Schema, oldCol schema.Column, newCol schem
 		}
 	}
 
+	// Copy over all checks from the old schema
+	for _, check := range sch.Checks().AllChecks() {
+		_, err := newSch.Checks().AddCheck(check.Name(), check.Expression(), check.Enforced())
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	pkOrds, err := modifyPkOrdinals(sch, newSch)
 	if err != nil {
 		return nil, err

go/libraries/doltcore/schema/alterschema/modifycolumn_test.go

@@ -193,6 +193,7 @@ func TestModifyColumn(t *testing.T) {
 			assert.NotNil(t, index)
 			tt.expectedSchema.Indexes().AddIndex(index)
 			tt.expectedSchema.SetPkOrdinals(sch.GetPkOrdinals())
+			tt.expectedSchema.Checks().AddCheck("test-check", "age < 123", true)
 			require.Equal(t, tt.expectedSchema, sch)
 
 			rowData, err := updatedTable.GetNomsRowData(ctx)

go/libraries/doltcore/sqle/dfunctions/dolt_pull.go

@@ -29,7 +29,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
-	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 )
 
 const DoltPullFuncName = "dolt_pull"
@@ -153,7 +153,7 @@ func (d DoltPullFunc) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
 	return noConflicts, nil
 }
 
-func pullerProgFunc(ctx context.Context, pullerEventCh <-chan datas.PullerEvent) {
+func pullerProgFunc(ctx context.Context, pullerEventCh <-chan pull.PullerEvent) {
 	for {
 		if ctx.Err() != nil {
 			return
@@ -167,7 +167,7 @@ func pullerProgFunc(ctx context.Context, pullerEventCh <-chan datas.PullerEvent)
 	}
 }
 
-func progFunc(ctx context.Context, progChan <-chan datas.PullProgress) {
+func progFunc(ctx context.Context, progChan <-chan pull.PullProgress) {
 	for {
 		if ctx.Err() != nil {
 			return
@@ -181,9 +181,9 @@ func progFunc(ctx context.Context, progChan <-chan datas.PullProgress) {
 	}
 }
 
-func runProgFuncs(ctx context.Context) (*sync.WaitGroup, chan datas.PullProgress, chan datas.PullerEvent) {
-	pullerEventCh := make(chan datas.PullerEvent)
-	progChan := make(chan datas.PullProgress)
+func runProgFuncs(ctx context.Context) (*sync.WaitGroup, chan pull.PullProgress, chan pull.PullerEvent) {
+	pullerEventCh := make(chan pull.PullerEvent)
+	progChan := make(chan pull.PullProgress)
 	wg := &sync.WaitGroup{}
 
 	wg.Add(1)
@@ -201,7 +201,7 @@ func runProgFuncs(ctx context.Context) (*sync.WaitGroup, chan datas.PullProgress
 	return wg, progChan, pullerEventCh
 }
 
-func stopProgFuncs(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) {
+func stopProgFuncs(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan pull.PullProgress, pullerEventCh chan pull.PullerEvent) {
 	cancel()
 	close(progChan)
 	close(pullerEventCh)

go/libraries/doltcore/sqle/index/dolt_index.go

@@ -454,7 +454,6 @@ func prollyRangeFromSqlRange(sqlRange sql.Range, tb *val.TupleBuilder) (rng prol
 		Start:   start,
 		Stop:    stop,
 		KeyDesc: tb.Desc,
-		Reverse: false,
 	}, nil
 }
 

go/libraries/doltcore/sqle/logictest/dolt/doltharness.go

@@ -27,7 +27,6 @@ import (
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/sqllogictest/go/logictest"
 	"github.com/dolthub/vitess/go/vt/proto/query"
-	"github.com/dolthub/vitess/go/vt/sqlparser"
 	"github.com/shopspring/decimal"
 
 	"github.com/dolthub/dolt/go/cmd/dolt/commands"
@@ -68,8 +67,6 @@ func (h *DoltHarness) ExecuteStatement(statement string) error {
 		sql.WithPid(rand.Uint64()),
 		sql.WithSession(h.sess))
 
-	statement = normalizeStatement(statement)
-
 	_, rowIter, err := h.engine.Query(ctx, statement)
 	if err != nil {
 		return err
@@ -189,37 +186,6 @@ func getDbState(db sql.Database, dEnv *env.DoltEnv) dsess.InitialDbState {
 	}
 }
 
-// We cheat a little at these tests. A great many of them use tables without primary keys, which we don't currently
-// support. Until we do, we just make every column in such tables part of the primary key.
-func normalizeStatement(statement string) string {
-	if !strings.Contains(statement, "CREATE TABLE") {
-		return statement
-	}
-	if strings.Contains(statement, "PRIMARY KEY") {
-		return statement
-	}
-
-	stmt, err := sqlparser.Parse(statement)
-	if err != nil {
-		panic(err)
-	}
-	create, ok := stmt.(*sqlparser.DDL)
-	if !ok {
-		panic("Expected CREATE TABLE statement")
-	}
-
-	lastParen := strings.LastIndex(statement, ")")
-	normalized := statement[:lastParen] + ", PRIMARY KEY ("
-	for i, column := range create.TableSpec.Columns {
-		normalized += column.Name.String()
-		if i != len(create.TableSpec.Columns)-1 {
-			normalized += ", "
-		}
-	}
-	normalized += "))"
-	return normalized
-}
-
 func drainIterator(ctx *sql.Context, iter sql.RowIter) error {
 	if iter == nil {
 		return nil

go/libraries/doltcore/sqle/logictest/dolt/doltharness_test.go

@@ -62,7 +62,7 @@ func TestDoltHarness(t *testing.T) {
 			expErr:    nil,
 		},
 		{
-			statement: "INSERT INTO t1(e,c,b,d,a) VALUES(103,102,100,101,104);",
+			statement: "INSERT INTO t1(e,c,b,d,a) VALUES(NULL,102,NULL,101,104);",
 			expErr:    nil,
 		},
 		{
@@ -76,19 +76,19 @@ func TestDoltHarness(t *testing.T) {
 			query:      "SELECT a,c,e FROM t1;",
 			expErr:     nil,
 			expSchema:  "III",
-			expResults: []string{"104", "102", "103", "107", "106", "109"},
+			expResults: []string{"104", "102", "NULL", "107", "106", "109"},
 		},
 		{
 			query:      "SELECT b,d FROM t1;",
 			expErr:     nil,
 			expSchema:  "II",
-			expResults: []string{"100", "101", "105", "108"},
+			expResults: []string{"NULL", "101", "105", "108"},
 		},
 		{
 			query:      "SELECT * FROM t1 WHERE d < 107;",
 			expErr:     nil,
 			expSchema:  "IIIII",
-			expResults: []string{"104", "100", "102", "101", "103"},
+			expResults: []string{"104", "NULL", "102", "101", "NULL"},
 		},
 		{
 			query:      "SELECT * FROM t1 WHERE d > 102;",

go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go

@@ -44,7 +44,7 @@ func TestEndToEnd(t *testing.T) {
 	id := uuid.MustParse("00000000-0000-0000-0000-000000000000")
 	tableName := "people"
 
-	dropCreateStatement := sqlfmt.DropTableIfExistsStmt(tableName) + "\nCREATE TABLE `people` (\n  `id` char(36) character set ascii collate ascii_bin NOT NULL,\n  `name` varchar(16383) NOT NULL,\n  `age` bigint unsigned NOT NULL,\n  `is_married` bit(1) NOT NULL,\n  `title` varchar(16383),\n  PRIMARY KEY (`id`),\n  KEY `idx_name` (`name`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"
+	dropCreateStatement := sqlfmt.DropTableIfExistsStmt(tableName) + "\nCREATE TABLE `people` (\n  `id` char(36) character set ascii collate ascii_bin NOT NULL,\n  `name` varchar(16383) NOT NULL,\n  `age` bigint unsigned NOT NULL,\n  `is_married` bit(1) NOT NULL,\n  `title` varchar(16383),\n  PRIMARY KEY (`id`),\n  KEY `idx_name` (`name`),\n  CONSTRAINT `test-check` CHECK ((`age` < 123))\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"
 
 	type test struct {
 		name           string

go/store/cmd/noms/noms_sync.go

@@ -33,6 +33,7 @@ import (
 	"github.com/dolthub/dolt/go/store/cmd/noms/util"
 	"github.com/dolthub/dolt/go/store/config"
 	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/datas/pull"
 	"github.com/dolthub/dolt/go/store/types"
 	"github.com/dolthub/dolt/go/store/util/profile"
 	"github.com/dolthub/dolt/go/store/util/status"
@@ -75,11 +76,11 @@ func runSync(ctx context.Context, args []string) int {
 	defer sinkDB.Close()
 
 	start := time.Now()
-	progressCh := make(chan datas.PullProgress)
-	lastProgressCh := make(chan datas.PullProgress)
+	progressCh := make(chan pull.PullProgress)
+	lastProgressCh := make(chan pull.PullProgress)
 
 	go func() {
-		var last datas.PullProgress
+		var last pull.PullProgress
 
 		for info := range progressCh {
 			last = info
@@ -101,9 +102,13 @@ func runSync(ctx context.Context, args []string) int {
 	sinkRef, sinkExists, err := sinkDataset.MaybeHeadRef()
 	util.CheckError(err)
 	nonFF := false
+	srcCS := datas.ChunkStoreFromDatabase(sourceStore)
+	sinkCS := datas.ChunkStoreFromDatabase(sinkDB)
+	wrf, err := types.WalkRefsForChunkStore(srcCS)
+	util.CheckError(err)
 	f := func() error {
 		defer profile.MaybeStartProfile().Stop()
-		err := datas.Pull(ctx, sourceStore, sinkDB, sourceRef, progressCh)
+		err := pull.Pull(ctx, srcCS, sinkCS, wrf, sourceRef.TargetHash(), progressCh)
 
 		if err != nil {
 			return err

go/store/datas/database.go

@@ -189,7 +189,11 @@ type Database interface {
 }
 
 func NewDatabase(cs chunks.ChunkStore) Database {
-	return newDatabase(cs)
+	return newDatabase(types.NewValueStore(cs))
+}
+
+func NewTypesDatabase(vs *types.ValueStore) Database {
+	return newDatabase(vs)
 }
 
 // GarbageCollector provides a method to

go/store/datas/database_common.go

@@ -64,9 +64,7 @@ type rootTracker interface {
 	Commit(ctx context.Context, current, last hash.Hash) (bool, error)
 }
 
-func newDatabase(cs chunks.ChunkStore) *database {
-	vs := types.NewValueStore(cs)
-
+func newDatabase(vs *types.ValueStore) *database {
 	return &database{
 		ValueStore: vs, // ValueStore is responsible for closing |cs|
 		rt:         vs,

go/store/datas/pull.go

@@ -1,436 +0,0 @@
-// Copyright 2019 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// This file incorporates work covered by the following copyright and
-// permission notice:
-//
-// Copyright 2016 Attic Labs, Inc. All rights reserved.
-// Licensed under the Apache License, version 2.0:
-// http://www.apache.org/licenses/LICENSE-2.0
-
-package datas
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-	"math/rand"
-	"sync"
-
-	"github.com/cenkalti/backoff"
-	"github.com/golang/snappy"
-	"golang.org/x/sync/errgroup"
-	"golang.org/x/sync/semaphore"
-
-	"github.com/dolthub/dolt/go/store/chunks"
-	"github.com/dolthub/dolt/go/store/hash"
-	"github.com/dolthub/dolt/go/store/nbs"
-	"github.com/dolthub/dolt/go/store/types"
-)
-
-type PullProgress struct {
-	DoneCount, KnownCount, ApproxWrittenBytes uint64
-}
-
-const (
-	bytesWrittenSampleRate = .10
-	defaultBatchSize       = 1 << 12 // 4096 chunks
-)
-
-var ErrNoData = errors.New("no data")
-
-func makeProgTrack(progressCh chan PullProgress) func(moreDone, moreKnown, moreApproxBytesWritten uint64) {
-	var doneCount, knownCount, approxBytesWritten uint64
-	return func(moreDone, moreKnown, moreApproxBytesWritten uint64) {
-		if progressCh == nil {
-			return
-		}
-		doneCount, knownCount, approxBytesWritten = doneCount+moreDone, knownCount+moreKnown, approxBytesWritten+moreApproxBytesWritten
-		progressCh <- PullProgress{doneCount, knownCount, approxBytesWritten}
-	}
-}
-
-func Clone(ctx context.Context, srcDB, sinkDB Database, eventCh chan<- TableFileEvent) error {
-
-	srcCS := srcDB.chunkStore().(interface{})
-	sinkCS := sinkDB.chunkStore().(interface{})
-
-	srcTS, srcOK := srcCS.(nbs.TableFileStore)
-
-	if !srcOK {
-		return errors.New("src db is not a Table File Store")
-	}
-
-	size, err := srcTS.Size(ctx)
-
-	if err != nil {
-		return err
-	}
-
-	if size == 0 {
-		return ErrNoData
-	}
-
-	sinkTS, sinkOK := sinkCS.(nbs.TableFileStore)
-
-	if !sinkOK {
-		return errors.New("sink db is not a Table File Store")
-	}
-
-	return clone(ctx, srcTS, sinkTS, eventCh)
-}
-
-type CloneTableFileEvent int
-
-const (
-	Listed = iota
-	DownloadStart
-	DownloadSuccess
-	DownloadFailed
-)
-
-type TableFileEvent struct {
-	EventType  CloneTableFileEvent
-	TableFiles []nbs.TableFile
-}
-
-// mapTableFiles returns the list of all fileIDs for the table files, and a map from fileID to nbs.TableFile
-func mapTableFiles(tblFiles []nbs.TableFile) ([]string, map[string]nbs.TableFile, map[string]int) {
-	fileIds := make([]string, len(tblFiles))
-	fileIDtoTblFile := make(map[string]nbs.TableFile)
-	fileIDtoNumChunks := make(map[string]int)
-
-	for i, tblFile := range tblFiles {
-		fileIDtoTblFile[tblFile.FileID()] = tblFile
-		fileIds[i] = tblFile.FileID()
-		fileIDtoNumChunks[tblFile.FileID()] = tblFile.NumChunks()
-	}
-
-	return fileIds, fileIDtoTblFile, fileIDtoNumChunks
-}
-
-func CloseWithErr(c io.Closer, err *error) {
-	e := c.Close()
-	if *err == nil && e != nil {
-		*err = e
-	}
-}
-
-const concurrentTableFileDownloads = 3
-
-func clone(ctx context.Context, srcTS, sinkTS nbs.TableFileStore, eventCh chan<- TableFileEvent) error {
-	root, sourceFiles, appendixFiles, err := srcTS.Sources(ctx)
-	if err != nil {
-		return err
-	}
-
-	tblFiles := filterAppendicesFromSourceFiles(appendixFiles, sourceFiles)
-	report := func(e TableFileEvent) {
-		if eventCh != nil {
-			eventCh <- e
-		}
-	}
-
-	// Initializes the list of fileIDs we are going to download, and the map of fileIDToTF.  If this clone takes a long
-	// time some of the urls within the nbs.TableFiles will expire and fail to download.  At that point we will retrieve
-	// the sources again, and update the fileIDToTF map with updated info, but not change the files we are downloading.
-	desiredFiles, fileIDToTF, fileIDToNumChunks := mapTableFiles(tblFiles)
-	completed := make([]bool, len(desiredFiles))
-
-	report(TableFileEvent{Listed, tblFiles})
-
-	download := func(ctx context.Context) error {
-		sem := semaphore.NewWeighted(concurrentTableFileDownloads)
-		eg, ctx := errgroup.WithContext(ctx)
-		for i := 0; i < len(desiredFiles); i++ {
-			if completed[i] {
-				continue
-			}
-			if err := sem.Acquire(ctx, 1); err != nil {
-				// The errgroup ctx has been canceled. We will
-				// return the error from wg.Wait() below.
-				break
-			}
-			idx := i
-			eg.Go(func() (err error) {
-				defer sem.Release(1)
-
-				fileID := desiredFiles[idx]
-				tblFile, ok := fileIDToTF[fileID]
-				if !ok {
-					// conjoin happened during clone
-					return backoff.Permanent(errors.New("table file not found. please try again"))
-				}
-
-				var rd io.ReadCloser
-				if rd, err = tblFile.Open(ctx); err != nil {
-					return err
-				}
-				defer CloseWithErr(rd, &err)
-
-				report(TableFileEvent{DownloadStart, []nbs.TableFile{tblFile}})
-				err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), rd, 0, nil)
-				if err != nil {
-					report(TableFileEvent{DownloadFailed, []nbs.TableFile{tblFile}})
-					return err
-				}
-
-				report(TableFileEvent{DownloadSuccess, []nbs.TableFile{tblFile}})
-				completed[idx] = true
-				return nil
-			})
-		}
-
-		return eg.Wait()
-	}
-
-	const maxAttempts = 3
-	previousCompletedCnt := 0
-	failureCount := 0
-
-	madeProgress := func() bool {
-		currentCompletedCnt := 0
-		for _, b := range completed {
-			if b {
-				currentCompletedCnt++
-			}
-		}
-		if currentCompletedCnt == previousCompletedCnt {
-			return false
-		} else {
-			previousCompletedCnt = currentCompletedCnt
-			return true
-		}
-	}
-
-	// keep going as long as progress is being made.  If progress is not made retry up to maxAttempts times.
-	for {
-		err = download(ctx)
-		if err == nil {
-			break
-		}
-		if permanent, ok := err.(*backoff.PermanentError); ok {
-			return permanent.Err
-		} else if madeProgress() {
-			failureCount = 0
-		} else {
-			failureCount++
-		}
-		if failureCount >= maxAttempts {
-			return err
-		}
-		if _, sourceFiles, appendixFiles, err = srcTS.Sources(ctx); err != nil {
-			return err
-		} else {
-			tblFiles = filterAppendicesFromSourceFiles(appendixFiles, sourceFiles)
-			_, fileIDToTF, _ = mapTableFiles(tblFiles)
-		}
-	}
-
-	sinkTS.AddTableFilesToManifest(ctx, fileIDToNumChunks)
-	return sinkTS.SetRootChunk(ctx, root, hash.Hash{})
-}
-
-func filterAppendicesFromSourceFiles(appendixFiles []nbs.TableFile, sourceFiles []nbs.TableFile) []nbs.TableFile {
-	if len(appendixFiles) == 0 {
-		return sourceFiles
-	}
-	tblFiles := make([]nbs.TableFile, 0)
-	_, appendixMap, _ := mapTableFiles(appendixFiles)
-	for _, sf := range sourceFiles {
-		if _, ok := appendixMap[sf.FileID()]; !ok {
-			tblFiles = append(tblFiles, sf)
-		}
-	}
-	return tblFiles
-}
-
-// Pull objects that descend from sourceRef from srcDB to sinkDB.
-func Pull(ctx context.Context, srcDB, sinkDB Database, sourceRef types.Ref, progressCh chan PullProgress) error {
-	return pull(ctx, srcDB, sinkDB, sourceRef.TargetHash(), progressCh, defaultBatchSize)
-}
-
-func pull(ctx context.Context, srcDB, sinkDB Database, sourceHash hash.Hash, progressCh chan PullProgress, batchSize int) error {
-	// Sanity Check
-	exists, err := srcDB.chunkStore().Has(ctx, sourceHash)
-
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		return errors.New("not found")
-	}
-
-	exists, err = sinkDB.chunkStore().Has(ctx, sourceHash)
-
-	if err != nil {
-		return err
-	}
-
-	if exists {
-		return nil // already up to date
-	}
-
-	if srcDB.chunkStore().Version() != sinkDB.chunkStore().Version() {
-		return fmt.Errorf("cannot pull from src to sink; src version is %v and sink version is %v", srcDB.chunkStore().Version(), sinkDB.chunkStore().Version())
-	}
-
-	var sampleSize, sampleCount uint64
-	updateProgress := makeProgTrack(progressCh)
-
-	// TODO: This batches based on limiting the _number_ of chunks processed at the same time. We really want to batch based on the _amount_ of chunk data being processed simultaneously. We also want to consider the chunks in a particular order, however, and the current GetMany() interface doesn't provide any ordering guarantees. Once BUG 3750 is fixed, we should be able to revisit this and do a better job.
-	absent := hash.HashSlice{sourceHash}
-	for absentCount := len(absent); absentCount != 0; absentCount = len(absent) {
-		updateProgress(0, uint64(absentCount), 0)
-
-		// For gathering up the hashes in the next level of the tree
-		nextLevel := hash.HashSet{}
-		uniqueOrdered := hash.HashSlice{}
-
-		// Process all absent chunks in this level of the tree in quanta of at most |batchSize|
-		for start, end := 0, batchSize; start < absentCount; start, end = end, end+batchSize {
-			if end > absentCount {
-				end = absentCount
-			}
-			batch := absent[start:end]
-
-			neededChunks, err := getChunks(ctx, srcDB, batch, sampleSize, sampleCount, updateProgress)
-
-			if err != nil {
-				return err
-			}
-
-			uniqueOrdered, err = putChunks(ctx, sinkDB, batch, neededChunks, nextLevel, uniqueOrdered)
-
-			if err != nil {
-				return err
-			}
-		}
-
-		absent, err = nextLevelMissingChunks(ctx, sinkDB, nextLevel, absent, uniqueOrdered)
-
-		if err != nil {
-			return err
-		}
-	}
-
-	err = persistChunks(ctx, sinkDB.chunkStore())
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func persistChunks(ctx context.Context, cs chunks.ChunkStore) error {
-	// todo: there is no call to rebase on an unsuccessful Commit()
-	// will  this loop forever?
-	var success bool
-	for !success {
-		r, err := cs.Root(ctx)
-
-		if err != nil {
-			return err
-		}
-
-		success, err = cs.Commit(ctx, r, r)
-
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// PullWithoutBatching effectively removes the batching of chunk retrieval done on each level of the tree.  This means
-// all chunks from one level of the tree will be retrieved from the underlying chunk store in one call, which pushes the
-// optimization problem down to the chunk store which can make smarter decisions.
-func PullWithoutBatching(ctx context.Context, srcDB, sinkDB Database, sourceRef types.Ref, progressCh chan PullProgress) error {
-	// by increasing the batch size to MaxInt32 we effectively remove batching here.
-	return pull(ctx, srcDB, sinkDB, sourceRef.TargetHash(), progressCh, math.MaxInt32)
-}
-
-// concurrently pull all chunks from this batch that the sink is missing out of the source
-func getChunks(ctx context.Context, srcDB Database, batch hash.HashSlice, sampleSize uint64, sampleCount uint64, updateProgress func(moreDone uint64, moreKnown uint64, moreApproxBytesWritten uint64)) (map[hash.Hash]*chunks.Chunk, error) {
-	mu := &sync.Mutex{}
-	neededChunks := map[hash.Hash]*chunks.Chunk{}
-	err := srcDB.chunkStore().GetMany(ctx, batch.HashSet(), func(ctx context.Context, c *chunks.Chunk) {
-		mu.Lock()
-		defer mu.Unlock()
-		neededChunks[c.Hash()] = c
-
-		// Randomly sample amount of data written
-		if rand.Float64() < bytesWrittenSampleRate {
-			sampleSize += uint64(len(snappy.Encode(nil, c.Data())))
-			sampleCount++
-		}
-		updateProgress(1, 0, sampleSize/uint64(math.Max(1, float64(sampleCount))))
-	})
-	if err != nil {
-		return nil, err
-	}
-	return neededChunks, nil
-}
-
-// put the chunks that were downloaded into the sink IN ORDER and at the same time gather up an ordered, uniquified list
-// of all the children of the chunks and add them to the list of the next level tree chunks.
-func putChunks(ctx context.Context, sinkDB Database, hashes hash.HashSlice, neededChunks map[hash.Hash]*chunks.Chunk, nextLevel hash.HashSet, uniqueOrdered hash.HashSlice) (hash.HashSlice, error) {
-	for _, h := range hashes {
-		c := neededChunks[h]
-		err := sinkDB.chunkStore().Put(ctx, *c)
-
-		if err != nil {
-			return hash.HashSlice{}, err
-		}
-
-		err = types.WalkRefs(*c, sinkDB.Format(), func(r types.Ref) error {
-			if !nextLevel.Has(r.TargetHash()) {
-				uniqueOrdered = append(uniqueOrdered, r.TargetHash())
-				nextLevel.Insert(r.TargetHash())
-			}
-
-			return nil
-		})
-
-		if err != nil {
-			return hash.HashSlice{}, err
-		}
-	}
-
-	return uniqueOrdered, nil
-}
-
-// ask sinkDB which of the next level's hashes it doesn't have, and add those chunks to the absent list which will need
-// to be retrieved.
-func nextLevelMissingChunks(ctx context.Context, sinkDB Database, nextLevel hash.HashSet, absent hash.HashSlice, uniqueOrdered hash.HashSlice) (hash.HashSlice, error) {
-	missingFromSink, err := sinkDB.chunkStore().HasMany(ctx, nextLevel)
-
-	if err != nil {
-		return hash.HashSlice{}, err
-	}
-
-	absent = absent[:0]
-	for _, h := range uniqueOrdered {
-		if missingFromSink.Has(h) {
-			absent = append(absent, h)
-		}
-	}
-
-	return absent, nil
-}

go/store/datas/pull/clone.go

@@ -0,0 +1,220 @@
+// Copyright 2019 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pull
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"github.com/cenkalti/backoff"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/semaphore"
+
+	"github.com/dolthub/dolt/go/store/chunks"
+	"github.com/dolthub/dolt/go/store/hash"
+	"github.com/dolthub/dolt/go/store/nbs"
+)
+
+func Clone(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, eventCh chan<- TableFileEvent) error {
+	srcTS, srcOK := srcCS.(nbs.TableFileStore)
+
+	if !srcOK {
+		return errors.New("src db is not a Table File Store")
+	}
+
+	size, err := srcTS.Size(ctx)
+
+	if err != nil {
+		return err
+	}
+
+	if size == 0 {
+		return ErrNoData
+	}
+
+	sinkTS, sinkOK := sinkCS.(nbs.TableFileStore)
+
+	if !sinkOK {
+		return errors.New("sink db is not a Table File Store")
+	}
+
+	return clone(ctx, srcTS, sinkTS, eventCh)
+}
+
+type CloneTableFileEvent int
+
+const (
+	Listed = iota
+	DownloadStart
+	DownloadSuccess
+	DownloadFailed
+)
+
+type TableFileEvent struct {
+	EventType  CloneTableFileEvent
+	TableFiles []nbs.TableFile
+}
+
+// mapTableFiles returns the list of all fileIDs for the table files, and a map from fileID to nbs.TableFile
+func mapTableFiles(tblFiles []nbs.TableFile) ([]string, map[string]nbs.TableFile, map[string]int) {
+	fileIds := make([]string, len(tblFiles))
+	fileIDtoTblFile := make(map[string]nbs.TableFile)
+	fileIDtoNumChunks := make(map[string]int)
+
+	for i, tblFile := range tblFiles {
+		fileIDtoTblFile[tblFile.FileID()] = tblFile
+		fileIds[i] = tblFile.FileID()
+		fileIDtoNumChunks[tblFile.FileID()] = tblFile.NumChunks()
+	}
+
+	return fileIds, fileIDtoTblFile, fileIDtoNumChunks
+}
+
+func CloseWithErr(c io.Closer, err *error) {
+	e := c.Close()
+	if *err == nil && e != nil {
+		*err = e
+	}
+}
+
+const concurrentTableFileDownloads = 3
+
+func clone(ctx context.Context, srcTS, sinkTS nbs.TableFileStore, eventCh chan<- TableFileEvent) error {
+	root, sourceFiles, appendixFiles, err := srcTS.Sources(ctx)
+	if err != nil {
+		return err
+	}
+
+	tblFiles := filterAppendicesFromSourceFiles(appendixFiles, sourceFiles)
+	report := func(e TableFileEvent) {
+		if eventCh != nil {
+			eventCh <- e
+		}
+	}
+
+	// Initializes the list of fileIDs we are going to download, and the map of fileIDToTF.  If this clone takes a long
+	// time some of the urls within the nbs.TableFiles will expire and fail to download.  At that point we will retrieve
+	// the sources again, and update the fileIDToTF map with updated info, but not change the files we are downloading.
+	desiredFiles, fileIDToTF, fileIDToNumChunks := mapTableFiles(tblFiles)
+	completed := make([]bool, len(desiredFiles))
+
+	report(TableFileEvent{Listed, tblFiles})
+
+	download := func(ctx context.Context) error {
+		sem := semaphore.NewWeighted(concurrentTableFileDownloads)
+		eg, ctx := errgroup.WithContext(ctx)
+		for i := 0; i < len(desiredFiles); i++ {
+			if completed[i] {
+				continue
+			}
+			if err := sem.Acquire(ctx, 1); err != nil {
+				// The errgroup ctx has been canceled. We will
+				// return the error from wg.Wait() below.
+				break
+			}
+			idx := i
+			eg.Go(func() (err error) {
+				defer sem.Release(1)
+
+				fileID := desiredFiles[idx]
+				tblFile, ok := fileIDToTF[fileID]
+				if !ok {
+					// conjoin happened during clone
+					return backoff.Permanent(errors.New("table file not found. please try again"))
+				}
+
+				var rd io.ReadCloser
+				if rd, err = tblFile.Open(ctx); err != nil {
+					return err
+				}
+				defer CloseWithErr(rd, &err)
+
+				report(TableFileEvent{DownloadStart, []nbs.TableFile{tblFile}})
+				err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), rd, 0, nil)
+				if err != nil {
+					report(TableFileEvent{DownloadFailed, []nbs.TableFile{tblFile}})
+					return err
+				}
+
+				report(TableFileEvent{DownloadSuccess, []nbs.TableFile{tblFile}})
+				completed[idx] = true
+				return nil
+			})
+		}
+
+		return eg.Wait()
+	}
+
+	const maxAttempts = 3
+	previousCompletedCnt := 0
+	failureCount := 0
+
+	madeProgress := func() bool {
+		currentCompletedCnt := 0
+		for _, b := range completed {
+			if b {
+				currentCompletedCnt++
+			}
+		}
+		if currentCompletedCnt == previousCompletedCnt {
+			return false
+		} else {
+			previousCompletedCnt = currentCompletedCnt
+			return true
+		}
+	}
+
+	// keep going as long as progress is being made.  If progress is not made retry up to maxAttempts times.
+	for {
+		err = download(ctx)
+		if err == nil {
+			break
+		}
+		if permanent, ok := err.(*backoff.PermanentError); ok {
+			return permanent.Err
+		} else if madeProgress() {
+			failureCount = 0
+		} else {
+			failureCount++
+		}
+		if failureCount >= maxAttempts {
+			return err
+		}
+		if _, sourceFiles, appendixFiles, err = srcTS.Sources(ctx); err != nil {
+			return err
+		} else {
+			tblFiles = filterAppendicesFromSourceFiles(appendixFiles, sourceFiles)
+			_, fileIDToTF, _ = mapTableFiles(tblFiles)
+		}
+	}
+
+	sinkTS.AddTableFilesToManifest(ctx, fileIDToNumChunks)
+	return sinkTS.SetRootChunk(ctx, root, hash.Hash{})
+}
+
+func filterAppendicesFromSourceFiles(appendixFiles []nbs.TableFile, sourceFiles []nbs.TableFile) []nbs.TableFile {
+	if len(appendixFiles) == 0 {
+		return sourceFiles
+	}
+	tblFiles := make([]nbs.TableFile, 0)
+	_, appendixMap, _ := mapTableFiles(appendixFiles)
+	for _, sf := range sourceFiles {
+		if _, ok := appendixMap[sf.FileID()]; !ok {
+			tblFiles = append(tblFiles, sf)
+		}
+	}
+	return tblFiles
+}

go/store/datas/pull/pull.go

@@ -0,0 +1,236 @@
+// Copyright 2019 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// This file incorporates work covered by the following copyright and
+// permission notice:
+//
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package pull
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"math/rand"
+	"sync"
+
+	"github.com/golang/snappy"
+
+	"github.com/dolthub/dolt/go/store/chunks"
+	"github.com/dolthub/dolt/go/store/hash"
+)
+
+type PullProgress struct {
+	DoneCount, KnownCount, ApproxWrittenBytes uint64
+}
+
+const (
+	bytesWrittenSampleRate = .10
+	defaultBatchSize       = 1 << 12 // 4096 chunks
+)
+
+var ErrNoData = errors.New("no data")
+
+func makeProgTrack(progressCh chan PullProgress) func(moreDone, moreKnown, moreApproxBytesWritten uint64) {
+	var doneCount, knownCount, approxBytesWritten uint64
+	return func(moreDone, moreKnown, moreApproxBytesWritten uint64) {
+		if progressCh == nil {
+			return
+		}
+		doneCount, knownCount, approxBytesWritten = doneCount+moreDone, knownCount+moreKnown, approxBytesWritten+moreApproxBytesWritten
+		progressCh <- PullProgress{doneCount, knownCount, approxBytesWritten}
+	}
+}
+
+// Pull objects that descend from sourceHash from srcDB to sinkDB.
+func Pull(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkRefs WalkRefs, sourceHash hash.Hash, progressCh chan PullProgress) error {
+	return pull(ctx, srcCS, sinkCS, walkRefs, sourceHash, progressCh, defaultBatchSize)
+}
+
+func pull(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkRefs WalkRefs, sourceHash hash.Hash, progressCh chan PullProgress, batchSize int) error {
+	// Sanity Check
+	exists, err := srcCS.Has(ctx, sourceHash)
+
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		return errors.New("not found")
+	}
+
+	exists, err = sinkCS.Has(ctx, sourceHash)
+
+	if err != nil {
+		return err
+	}
+
+	if exists {
+		return nil // already up to date
+	}
+
+	if srcCS.Version() != sinkCS.Version() {
+		return fmt.Errorf("cannot pull from src to sink; src version is %v and sink version is %v", srcCS.Version(), sinkCS.Version())
+	}
+
+	var sampleSize, sampleCount uint64
+	updateProgress := makeProgTrack(progressCh)
+
+	// TODO: This batches based on limiting the _number_ of chunks processed at the same time. We really want to batch based on the _amount_ of chunk data being processed simultaneously. We also want to consider the chunks in a particular order, however, and the current GetMany() interface doesn't provide any ordering guarantees. Once BUG 3750 is fixed, we should be able to revisit this and do a better job.
+	absent := hash.HashSlice{sourceHash}
+	for absentCount := len(absent); absentCount != 0; absentCount = len(absent) {
+		updateProgress(0, uint64(absentCount), 0)
+
+		// For gathering up the hashes in the next level of the tree
+		nextLevel := hash.HashSet{}
+		uniqueOrdered := hash.HashSlice{}
+
+		// Process all absent chunks in this level of the tree in quanta of at most |batchSize|
+		for start, end := 0, batchSize; start < absentCount; start, end = end, end+batchSize {
+			if end > absentCount {
+				end = absentCount
+			}
+			batch := absent[start:end]
+
+			neededChunks, err := getChunks(ctx, srcCS, batch, sampleSize, sampleCount, updateProgress)
+
+			if err != nil {
+				return err
+			}
+
+			uniqueOrdered, err = putChunks(ctx, walkRefs, sinkCS, batch, neededChunks, nextLevel, uniqueOrdered)
+
+			if err != nil {
+				return err
+			}
+		}
+
+		absent, err = nextLevelMissingChunks(ctx, sinkCS, nextLevel, absent, uniqueOrdered)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	err = persistChunks(ctx, sinkCS)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func persistChunks(ctx context.Context, cs chunks.ChunkStore) error {
+	// todo: there is no call to rebase on an unsuccessful Commit()
+	// will  this loop forever?
+	var success bool
+	for !success {
+		r, err := cs.Root(ctx)
+
+		if err != nil {
+			return err
+		}
+
+		success, err = cs.Commit(ctx, r, r)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PullWithoutBatching effectively removes the batching of chunk retrieval done on each level of the tree.  This means
+// all chunks from one level of the tree will be retrieved from the underlying chunk store in one call, which pushes the
+// optimization problem down to the chunk store which can make smarter decisions.
+func PullWithoutBatching(ctx context.Context, srcCS, sinkCS chunks.ChunkStore, walkRefs WalkRefs, sourceHash hash.Hash, progressCh chan PullProgress) error {
+	// by increasing the batch size to MaxInt32 we effectively remove batching here.
+	return pull(ctx, srcCS, sinkCS, walkRefs, sourceHash, progressCh, math.MaxInt32)
+}
+
+// concurrently pull all chunks from this batch that the sink is missing out of the source
+func getChunks(ctx context.Context, srcCS chunks.ChunkStore, batch hash.HashSlice, sampleSize uint64, sampleCount uint64, updateProgress func(moreDone uint64, moreKnown uint64, moreApproxBytesWritten uint64)) (map[hash.Hash]*chunks.Chunk, error) {
+	mu := &sync.Mutex{}
+	neededChunks := map[hash.Hash]*chunks.Chunk{}
+	err := srcCS.GetMany(ctx, batch.HashSet(), func(ctx context.Context, c *chunks.Chunk) {
+		mu.Lock()
+		defer mu.Unlock()
+		neededChunks[c.Hash()] = c
+
+		// Randomly sample amount of data written
+		if rand.Float64() < bytesWrittenSampleRate {
+			sampleSize += uint64(len(snappy.Encode(nil, c.Data())))
+			sampleCount++
+		}
+		updateProgress(1, 0, sampleSize/uint64(math.Max(1, float64(sampleCount))))
+	})
+	if err != nil {
+		return nil, err
+	}
+	return neededChunks, nil
+}
+
+type WalkRefs func(chunks.Chunk, func(hash.Hash, uint64) error) error
+
+// put the chunks that were downloaded into the sink IN ORDER and at the same time gather up an ordered, uniquified list
+// of all the children of the chunks and add them to the list of the next level tree chunks.
+func putChunks(ctx context.Context, wrh WalkRefs, sinkCS chunks.ChunkStore, hashes hash.HashSlice, neededChunks map[hash.Hash]*chunks.Chunk, nextLevel hash.HashSet, uniqueOrdered hash.HashSlice) (hash.HashSlice, error) {
+	for _, h := range hashes {
+		c := neededChunks[h]
+		err := sinkCS.Put(ctx, *c)
+
+		if err != nil {
+			return hash.HashSlice{}, err
+		}
+
+		err = wrh(*c, func(h hash.Hash, height uint64) error {
+			if !nextLevel.Has(h) {
+				uniqueOrdered = append(uniqueOrdered, h)
+				nextLevel.Insert(h)
+			}
+			return nil
+		})
+
+		if err != nil {
+			return hash.HashSlice{}, err
+		}
+	}
+
+	return uniqueOrdered, nil
+}
+
+// ask sinkDB which of the next level's hashes it doesn't have, and add those chunks to the absent list which will need
+// to be retrieved.
+func nextLevelMissingChunks(ctx context.Context, sinkCS chunks.ChunkStore, nextLevel hash.HashSet, absent hash.HashSlice, uniqueOrdered hash.HashSlice) (hash.HashSlice, error) {
+	missingFromSink, err := sinkCS.HasMany(ctx, nextLevel)
+
+	if err != nil {
+		return hash.HashSlice{}, err
+	}
+
+	absent = absent[:0]
+	for _, h := range uniqueOrdered {
+		if missingFromSink.Has(h) {
+			absent = append(absent, h)
+		}
+	}
+
+	return absent, nil
+}

go/store/datas/pull/pull_test.go

@@ -19,7 +19,7 @@
 // Licensed under the Apache License, version 2.0:
 // http://www.apache.org/licenses/LICENSE-2.0
 
-package datas
+package pull
 
 import (
 	"bytes"
@@ -36,6 +36,7 @@ import (
 
 	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/d"
+	"github.com/dolthub/dolt/go/store/datas"
 	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/nbs"
 	"github.com/dolthub/dolt/go/store/types"
@@ -63,8 +64,8 @@ type PullSuite struct {
 	suite.Suite
 	sinkCS      *chunks.TestStoreView
 	sourceCS    *chunks.TestStoreView
-	sink        Database
-	source      Database
+	sinkVRW     types.ValueReadWriter
+	sourceVRW   types.ValueReadWriter
 	commitReads int // The number of reads triggered by commit differs across chunk store impls
 }
 
@@ -79,8 +80,7 @@ type LocalToLocalSuite struct {
 
 func (suite *LocalToLocalSuite) SetupTest() {
 	suite.sinkCS, suite.sourceCS = makeTestStoreViews()
-	suite.sink = NewDatabase(suite.sinkCS)
-	suite.source = NewDatabase(suite.sourceCS)
+	suite.sinkVRW, suite.sourceVRW = types.NewValueStore(suite.sinkCS), types.NewValueStore(suite.sourceCS)
 }
 
 type RemoteToLocalSuite struct {
@@ -89,8 +89,7 @@ type RemoteToLocalSuite struct {
 
 func (suite *RemoteToLocalSuite) SetupTest() {
 	suite.sinkCS, suite.sourceCS = makeTestStoreViews()
-	suite.sink = NewDatabase(suite.sinkCS)
-	suite.source = makeRemoteDb(suite.sourceCS)
+	suite.sinkVRW, suite.sourceVRW = types.NewValueStore(suite.sinkCS), types.NewValueStore(suite.sourceCS)
 }
 
 type LocalToRemoteSuite struct {
@@ -99,8 +98,7 @@ type LocalToRemoteSuite struct {
 
 func (suite *LocalToRemoteSuite) SetupTest() {
 	suite.sinkCS, suite.sourceCS = makeTestStoreViews()
-	suite.sink = makeRemoteDb(suite.sinkCS)
-	suite.source = NewDatabase(suite.sourceCS)
+	suite.sinkVRW, suite.sourceVRW = types.NewValueStore(suite.sinkCS), types.NewValueStore(suite.sourceCS)
 	suite.commitReads = 1
 }
 
@@ -110,18 +108,11 @@ type RemoteToRemoteSuite struct {
 
 func (suite *RemoteToRemoteSuite) SetupTest() {
 	suite.sinkCS, suite.sourceCS = makeTestStoreViews()
-	suite.sink = makeRemoteDb(suite.sinkCS)
-	suite.source = makeRemoteDb(suite.sourceCS)
+	suite.sinkVRW, suite.sourceVRW = types.NewValueStore(suite.sinkCS), types.NewValueStore(suite.sourceCS)
 	suite.commitReads = 1
 }
 
-func makeRemoteDb(cs chunks.ChunkStore) Database {
-	return NewDatabase(cs)
-}
-
 func (suite *PullSuite) TearDownTest() {
-	suite.sink.Close()
-	suite.source.Close()
 	suite.sinkCS.Close()
 	suite.sourceCS.Close()
 }
@@ -177,18 +168,20 @@ func (pt *progressTracker) Validate(suite *PullSuite) {
 func (suite *PullSuite) TestPullEverything() {
 	expectedReads := suite.sinkCS.Reads()
 
-	l := buildListOfHeight(2, suite.source)
-	sourceRef := suite.commitToSource(l, mustList(types.NewList(context.Background(), suite.source)))
+	l := buildListOfHeight(2, suite.sourceVRW)
+	sourceRef := suite.commitToSource(l, mustList(types.NewList(context.Background(), suite.sourceVRW)))
 	pt := startProgressTracker()
 
-	err := Pull(context.Background(), suite.source, suite.sink, sourceRef, pt.Ch)
+	wrf, err := types.WalkRefsForChunkStore(suite.sourceCS)
+	suite.NoError(err)
+	err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, wrf, sourceRef.TargetHash(), pt.Ch)
 	suite.NoError(err)
 	suite.True(expectedReads-suite.sinkCS.Reads() <= suite.commitReads)
 	pt.Validate(suite)
 
-	v := mustValue(suite.sink.ReadValue(context.Background(), sourceRef.TargetHash())).(types.Struct)
+	v := mustValue(suite.sinkVRW.ReadValue(context.Background(), sourceRef.TargetHash())).(types.Struct)
 	suite.NotNil(v)
-	suite.True(l.Equals(mustGetValue(v.MaybeGet(ValueField))))
+	suite.True(l.Equals(mustGetValue(v.MaybeGet(datas.ValueField))))
 }
 
 // Source: -6-> C3(L5) -1-> N
@@ -211,29 +204,31 @@ func (suite *PullSuite) TestPullEverything() {
 //                \  -2-> L1 -1-> N
 //                         \ -1-> L0
 func (suite *PullSuite) TestPullMultiGeneration() {
-	sinkL := buildListOfHeight(2, suite.sink)
-	suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sink)))
+	sinkL := buildListOfHeight(2, suite.sinkVRW)
+	suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sinkVRW)))
 	expectedReads := suite.sinkCS.Reads()
 
-	srcL := buildListOfHeight(2, suite.source)
-	sourceRef := suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.source)))
-	srcL = buildListOfHeight(4, suite.source)
-	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.source, sourceRef)))
-	srcL = buildListOfHeight(5, suite.source)
-	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.source, sourceRef)))
+	srcL := buildListOfHeight(2, suite.sourceVRW)
+	sourceRef := suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.sourceVRW)))
+	srcL = buildListOfHeight(4, suite.sourceVRW)
+	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.sourceVRW, sourceRef)))
+	srcL = buildListOfHeight(5, suite.sourceVRW)
+	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.sourceVRW, sourceRef)))
 
 	pt := startProgressTracker()
 
-	err := Pull(context.Background(), suite.source, suite.sink, sourceRef, pt.Ch)
+	wrf, err := types.WalkRefsForChunkStore(suite.sourceCS)
+	suite.NoError(err)
+	err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, wrf, sourceRef.TargetHash(), pt.Ch)
 	suite.NoError(err)
 
 	suite.True(expectedReads-suite.sinkCS.Reads() <= suite.commitReads)
 	pt.Validate(suite)
 
-	v, err := suite.sink.ReadValue(context.Background(), sourceRef.TargetHash())
+	v, err := suite.sinkVRW.ReadValue(context.Background(), sourceRef.TargetHash())
 	suite.NoError(err)
 	suite.NotNil(v)
-	suite.True(srcL.Equals(mustGetValue(v.(types.Struct).MaybeGet(ValueField))))
+	suite.True(srcL.Equals(mustGetValue(v.(types.Struct).MaybeGet(datas.ValueField))))
 }
 
 // Source: -6-> C2(L5) -1-> N
@@ -259,32 +254,34 @@ func (suite *PullSuite) TestPullMultiGeneration() {
 //                             \ -2-> L1 -1-> N
 //                                     \ -1-> L0
 func (suite *PullSuite) TestPullDivergentHistory() {
-	sinkL := buildListOfHeight(3, suite.sink)
-	sinkRef := suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sink)))
-	srcL := buildListOfHeight(3, suite.source)
-	sourceRef := suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.source)))
+	sinkL := buildListOfHeight(3, suite.sinkVRW)
+	sinkRef := suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sinkVRW)))
+	srcL := buildListOfHeight(3, suite.sourceVRW)
+	sourceRef := suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.sourceVRW)))
 
 	var err error
 	sinkL, err = sinkL.Edit().Append(types.String("oy!")).List(context.Background())
 	suite.NoError(err)
-	sinkRef = suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sink, sinkRef)))
-	srcL, err = srcL.Edit().Set(1, buildListOfHeight(5, suite.source)).List(context.Background())
+	sinkRef = suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sinkVRW, sinkRef)))
+	srcL, err = srcL.Edit().Set(1, buildListOfHeight(5, suite.sourceVRW)).List(context.Background())
 	suite.NoError(err)
-	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.source, sourceRef)))
+	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.sourceVRW, sourceRef)))
 	preReads := suite.sinkCS.Reads()
 
 	pt := startProgressTracker()
 
-	err = Pull(context.Background(), suite.source, suite.sink, sourceRef, pt.Ch)
+	wrf, err := types.WalkRefsForChunkStore(suite.sourceCS)
+	suite.NoError(err)
+	err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, wrf, sourceRef.TargetHash(), pt.Ch)
 	suite.NoError(err)
 
 	suite.True(preReads-suite.sinkCS.Reads() <= suite.commitReads)
 	pt.Validate(suite)
 
-	v, err := suite.sink.ReadValue(context.Background(), sourceRef.TargetHash())
+	v, err := suite.sinkVRW.ReadValue(context.Background(), sourceRef.TargetHash())
 	suite.NoError(err)
 	suite.NotNil(v)
-	suite.True(srcL.Equals(mustGetValue(v.(types.Struct).MaybeGet(ValueField))))
+	suite.True(srcL.Equals(mustGetValue(v.(types.Struct).MaybeGet(datas.ValueField))))
 }
 
 // Source: -6-> C2(L4) -1-> N
@@ -304,51 +301,55 @@ func (suite *PullSuite) TestPullDivergentHistory() {
 //                                 \ -2-> L1 -1-> N
 //                                         \ -1-> L0
 func (suite *PullSuite) TestPullUpdates() {
-	sinkL := buildListOfHeight(4, suite.sink)
-	suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sink)))
+	sinkL := buildListOfHeight(4, suite.sinkVRW)
+	suite.commitToSink(sinkL, mustList(types.NewList(context.Background(), suite.sinkVRW)))
 	expectedReads := suite.sinkCS.Reads()
 
-	srcL := buildListOfHeight(4, suite.source)
-	sourceRef := suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.source)))
-	L3 := mustValue(mustValue(srcL.Get(context.Background(), 1)).(types.Ref).TargetValue(context.Background(), suite.source)).(types.List)
-	L2 := mustValue(mustValue(L3.Get(context.Background(), 1)).(types.Ref).TargetValue(context.Background(), suite.source)).(types.List)
-	L2Ed := L2.Edit().Append(mustRef(suite.source.WriteValue(context.Background(), types.String("oy!"))))
+	srcL := buildListOfHeight(4, suite.sourceVRW)
+	sourceRef := suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.sourceVRW)))
+	L3 := mustValue(mustValue(srcL.Get(context.Background(), 1)).(types.Ref).TargetValue(context.Background(), suite.sourceVRW)).(types.List)
+	L2 := mustValue(mustValue(L3.Get(context.Background(), 1)).(types.Ref).TargetValue(context.Background(), suite.sourceVRW)).(types.List)
+	L2Ed := L2.Edit().Append(mustRef(suite.sourceVRW.WriteValue(context.Background(), types.String("oy!"))))
 	L2, err := L2Ed.List(context.Background())
 	suite.NoError(err)
-	L3Ed := L3.Edit().Set(1, mustRef(suite.source.WriteValue(context.Background(), L2)))
+	L3Ed := L3.Edit().Set(1, mustRef(suite.sourceVRW.WriteValue(context.Background(), L2)))
 	L3, err = L3Ed.List(context.Background())
 	suite.NoError(err)
-	srcLEd := srcL.Edit().Set(1, mustRef(suite.source.WriteValue(context.Background(), L3)))
+	srcLEd := srcL.Edit().Set(1, mustRef(suite.sourceVRW.WriteValue(context.Background(), L3)))
 	srcL, err = srcLEd.List(context.Background())
 	suite.NoError(err)
-	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.source, sourceRef)))
+	sourceRef = suite.commitToSource(srcL, mustList(types.NewList(context.Background(), suite.sourceVRW, sourceRef)))
 
 	pt := startProgressTracker()
 
-	err = Pull(context.Background(), suite.source, suite.sink, sourceRef, pt.Ch)
+	wrf, err := types.WalkRefsForChunkStore(suite.sourceCS)
+	suite.NoError(err)
+	err = Pull(context.Background(), suite.sourceCS, suite.sinkCS, wrf, sourceRef.TargetHash(), pt.Ch)
 	suite.NoError(err)
 
 	suite.True(expectedReads-suite.sinkCS.Reads() <= suite.commitReads)
 	pt.Validate(suite)
 
-	v, err := suite.sink.ReadValue(context.Background(), sourceRef.TargetHash())
+	v, err := suite.sinkVRW.ReadValue(context.Background(), sourceRef.TargetHash())
 	suite.NoError(err)
 	suite.NotNil(v)
-	suite.True(srcL.Equals(mustGetValue(v.(types.Struct).MaybeGet(ValueField))))
+	suite.True(srcL.Equals(mustGetValue(v.(types.Struct).MaybeGet(datas.ValueField))))
 }
 
 func (suite *PullSuite) commitToSource(v types.Value, p types.List) types.Ref {
-	ds, err := suite.source.GetDataset(context.Background(), datasetID)
+	db := datas.NewTypesDatabase(suite.sourceVRW.(*types.ValueStore))
+	ds, err := db.GetDataset(context.Background(), datasetID)
 	suite.NoError(err)
-	ds, err = suite.source.Commit(context.Background(), ds, v, CommitOptions{ParentsList: p})
+	ds, err = db.Commit(context.Background(), ds, v, datas.CommitOptions{ParentsList: p})
 	suite.NoError(err)
 	return mustHeadRef(ds)
 }
 
 func (suite *PullSuite) commitToSink(v types.Value, p types.List) types.Ref {
-	ds, err := suite.sink.GetDataset(context.Background(), datasetID)
+	db := datas.NewTypesDatabase(suite.sinkVRW.(*types.ValueStore))
+	ds, err := db.GetDataset(context.Background(), datasetID)
 	suite.NoError(err)
-	ds, err = suite.sink.Commit(context.Background(), ds, v, CommitOptions{ParentsList: p})
+	ds, err = db.Commit(context.Background(), ds, v, datas.CommitOptions{ParentsList: p})
 	suite.NoError(err)
 	return mustHeadRef(ds)
 }
@@ -573,3 +574,38 @@ func TestClone(t *testing.T) {
 		assert.True(t, reflect.DeepEqual(flakeySrc.TestTableFileStore, dest))
 	})
 }
+
+func mustList(l types.List, err error) types.List {
+	d.PanicIfError(err)
+	return l
+}
+
+func mustValue(val types.Value, err error) types.Value {
+	d.PanicIfError(err)
+	return val
+}
+
+func mustGetValue(v types.Value, found bool, err error) types.Value {
+	d.PanicIfError(err)
+	d.PanicIfFalse(found)
+	return v
+}
+
+func mustRef(ref types.Ref, err error) types.Ref {
+	d.PanicIfError(err)
+	return ref
+}
+
+func mustHeadRef(ds datas.Dataset) types.Ref {
+	hr, ok, err := ds.MaybeHeadRef()
+
+	if err != nil {
+		panic("error getting head")
+	}
+
+	if !ok {
+		panic("no head")
+	}
+
+	return hr
+}

go/store/datas/pull/puller.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package datas
+package pull
 
 import (
 	"context"
@@ -32,7 +32,6 @@ import (
 	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/nbs"
-	"github.com/dolthub/dolt/go/store/types"
 )
 
 // ErrDBUpToDate is the error code returned from NewPuller in the event that there is no work to do.
@@ -61,9 +60,8 @@ type CmpChnkAndRefs struct {
 
 // Puller is used to sync data between to Databases
 type Puller struct {
-	fmt *types.NomsBinFormat
+	wrf WalkRefs
 
-	srcDB         Database
 	srcChunkStore nbs.NBSCompressedChunkStore
 	sinkDBCS      chunks.ChunkStore
 	rootChunkHash hash.Hash
@@ -121,9 +119,9 @@ func NewTFPullerEvent(et PullerEventType, details *TableFileEventDetails) Puller
 
 // NewPuller creates a new Puller instance to do the syncing.  If a nil puller is returned without error that means
 // that there is nothing to pull and the sinkDB is already up to date.
-func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcDB, sinkDB Database, rootChunkHash hash.Hash, eventCh chan PullerEvent) (*Puller, error) {
+func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcCS, sinkCS chunks.ChunkStore, walkRefs WalkRefs, rootChunkHash hash.Hash, eventCh chan PullerEvent) (*Puller, error) {
 	// Sanity Check
-	exists, err := srcDB.chunkStore().Has(ctx, rootChunkHash)
+	exists, err := srcCS.Has(ctx, rootChunkHash)
 
 	if err != nil {
 		return nil, err
@@ -133,9 +131,7 @@ func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcDB, sink
 		return nil, errors.New("not found")
 	}
 
-	sinkDBCS := sinkDB.chunkStore()
-
-	exists, err = sinkDBCS.Has(ctx, rootChunkHash)
+	exists, err = sinkCS.Has(ctx, rootChunkHash)
 
 	if err != nil {
 		return nil, err
@@ -145,11 +141,11 @@ func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcDB, sink
 		return nil, ErrDBUpToDate
 	}
 
-	if srcDB.chunkStore().Version() != sinkDB.chunkStore().Version() {
-		return nil, fmt.Errorf("cannot pull from src to sink; src version is %v and sink version is %v", srcDB.chunkStore().Version(), sinkDB.chunkStore().Version())
+	if srcCS.Version() != sinkCS.Version() {
+		return nil, fmt.Errorf("cannot pull from src to sink; src version is %v and sink version is %v", srcCS.Version(), sinkCS.Version())
 	}
 
-	srcChunkStore, ok := srcDB.chunkStore().(nbs.NBSCompressedChunkStore)
+	srcChunkStore, ok := srcCS.(nbs.NBSCompressedChunkStore)
 	if !ok {
 		return nil, ErrIncompatibleSourceChunkStore
 	}
@@ -171,10 +167,9 @@ func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcDB, sink
 	}
 
 	p := &Puller{
-		fmt:           srcDB.Format(),
-		srcDB:         srcDB,
+		wrf:           walkRefs,
 		srcChunkStore: srcChunkStore,
-		sinkDBCS:      sinkDBCS,
+		sinkDBCS:      sinkCS,
 		rootChunkHash: rootChunkHash,
 		downloaded:    hash.HashSet{},
 		tablefileSema: semaphore.NewWeighted(outstandingTableFiles),
@@ -185,7 +180,7 @@ func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcDB, sink
 		pushLog:       pushLogger,
 	}
 
-	if lcs, ok := sinkDBCS.(chunks.LoggingChunkStore); ok {
+	if lcs, ok := sinkCS.(chunks.LoggingChunkStore); ok {
 		lcs.SetLogger(p)
 	}
 
@@ -407,8 +402,8 @@ func (p *Puller) getCmp(ctx context.Context, twDetails *TreeWalkEventDetails, le
 				}
 
 				refs := make(map[hash.Hash]int)
-				if err := types.WalkRefs(chnk, p.fmt, func(r types.Ref) error {
-					refs[r.TargetHash()] = int(r.Height())
+				if err := p.wrf(chnk, func(h hash.Hash, height uint64) error {
+					refs[h] = int(height)
 					return nil
 				}); ae.SetIfError(err) {
 					return

go/store/datas/pull/puller_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package datas
+package pull
 
 import (
 	"context"
@@ -27,6 +27,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/dolthub/dolt/go/store/d"
+	"github.com/dolthub/dolt/go/store/datas"
 	"github.com/dolthub/dolt/go/store/nbs"
 	"github.com/dolthub/dolt/go/store/types"
 	"github.com/dolthub/dolt/go/store/util/clienttest"
@@ -120,21 +122,22 @@ func deleteTableValues(ctx context.Context, vrw types.ValueReadWriter, m types.M
 	return me.Map(ctx)
 }
 
-func tempDirDB(ctx context.Context) (Database, error) {
+func tempDirDB(ctx context.Context) (types.ValueReadWriter, datas.Database, error) {
 	dir := filepath.Join(os.TempDir(), uuid.New().String())
 	err := os.MkdirAll(dir, os.ModePerm)
 
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	st, err := nbs.NewLocalStore(ctx, types.Format_Default.VersionString(), dir, clienttest.DefaultMemTableSize)
-
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	return NewDatabase(st), nil
+	vs := types.NewValueStore(st)
+
+	return vs, datas.NewTypesDatabase(vs), nil
 }
 
 func TestPuller(t *testing.T) {
@@ -226,24 +229,24 @@ func TestPuller(t *testing.T) {
 	}
 
 	ctx := context.Background()
-	db, err := tempDirDB(ctx)
+	vs, db, err := tempDirDB(ctx)
 	require.NoError(t, err)
 	ds, err := db.GetDataset(ctx, "ds")
 	require.NoError(t, err)
-	rootMap, err := types.NewMap(ctx, db)
+	rootMap, err := types.NewMap(ctx, vs)
 	require.NoError(t, err)
 
-	parent, err := types.NewList(ctx, db)
+	parent, err := types.NewList(ctx, vs)
 	require.NoError(t, err)
 	states := map[string]types.Ref{}
 	for _, delta := range deltas {
 		for tbl, sets := range delta.sets {
-			rootMap, err = addTableValues(ctx, db, rootMap, tbl, sets...)
+			rootMap, err = addTableValues(ctx, vs, rootMap, tbl, sets...)
 			require.NoError(t, err)
 		}
 
 		for tbl, dels := range delta.deletes {
-			rootMap, err = deleteTableValues(ctx, db, rootMap, tbl, dels...)
+			rootMap, err = deleteTableValues(ctx, vs, rootMap, tbl, dels...)
 			require.NoError(t, err)
 		}
 
@@ -254,7 +257,7 @@ func TestPuller(t *testing.T) {
 		rootMap, err = me.Map(ctx)
 		require.NoError(t, err)
 
-		commitOpts := CommitOptions{ParentsList: parent}
+		commitOpts := datas.CommitOptions{ParentsList: parent}
 		ds, err = db.Commit(ctx, ds, rootMap, commitOpts)
 		require.NoError(t, err)
 
@@ -262,16 +265,16 @@ func TestPuller(t *testing.T) {
 		require.NoError(t, err)
 		require.True(t, ok)
 
-		parent, err = types.NewList(ctx, db, r)
+		parent, err = types.NewList(ctx, vs, r)
 		require.NoError(t, err)
 
 		states[delta.name] = r
 	}
 
-	tbl, err := makeABigTable(ctx, db)
+	tbl, err := makeABigTable(ctx, vs)
 	require.NoError(t, err)
 
-	tblRef, err := writeValAndGetRef(ctx, db, tbl)
+	tblRef, err := writeValAndGetRef(ctx, vs, tbl)
 	require.NoError(t, err)
 
 	me := rootMap.Edit()
@@ -279,7 +282,7 @@ func TestPuller(t *testing.T) {
 	rootMap, err = me.Map(ctx)
 	require.NoError(t, err)
 
-	commitOpts := CommitOptions{ParentsList: parent}
+	commitOpts := datas.CommitOptions{ParentsList: parent}
 	ds, err = db.Commit(ctx, ds, rootMap, commitOpts)
 	require.NoError(t, err)
 
@@ -313,13 +316,15 @@ func TestPuller(t *testing.T) {
 				}
 			}()
 
-			sinkdb, err := tempDirDB(ctx)
+			sinkvs, sinkdb, err := tempDirDB(ctx)
 			require.NoError(t, err)
 
 			tmpDir := filepath.Join(os.TempDir(), uuid.New().String())
 			err = os.MkdirAll(tmpDir, os.ModePerm)
 			require.NoError(t, err)
-			plr, err := NewPuller(ctx, tmpDir, 128, db, sinkdb, rootRef.TargetHash(), eventCh)
+			wrf, err := types.WalkRefsForChunkStore(datas.ChunkStoreFromDatabase(db))
+			require.NoError(t, err)
+			plr, err := NewPuller(ctx, tmpDir, 128, datas.ChunkStoreFromDatabase(db), datas.ChunkStoreFromDatabase(sinkdb), wrf, rootRef.TargetHash(), eventCh)
 			require.NoError(t, err)
 
 			err = plr.Pull(ctx)
@@ -337,7 +342,7 @@ func TestPuller(t *testing.T) {
 			require.NoError(t, err)
 			require.True(t, ok)
 
-			eq, err := pullerRefEquality(ctx, rootRef, sinkRootRef, db, sinkdb)
+			eq, err := pullerRefEquality(ctx, rootRef, sinkRootRef, vs, sinkvs)
 			require.NoError(t, err)
 			assert.True(t, eq)
 
@@ -345,8 +350,8 @@ func TestPuller(t *testing.T) {
 	}
 }
 
-func makeABigTable(ctx context.Context, db Database) (types.Map, error) {
-	m, err := types.NewMap(ctx, db)
+func makeABigTable(ctx context.Context, vrw types.ValueReadWriter) (types.Map, error) {
+	m, err := types.NewMap(ctx, vrw)
 
 	if err != nil {
 		return types.EmptyMap, nil
@@ -355,7 +360,7 @@ func makeABigTable(ctx context.Context, db Database) (types.Map, error) {
 	me := m.Edit()
 
 	for i := 0; i < 256*1024; i++ {
-		tpl, err := types.NewTuple(db.Format(), types.UUID(uuid.New()), types.String(uuid.New().String()), types.Float(float64(i)))
+		tpl, err := types.NewTuple(vrw.Format(), types.UUID(uuid.New()), types.String(uuid.New().String()), types.Float(float64(i)))
 
 		if err != nil {
 			return types.EmptyMap, err
@@ -367,14 +372,14 @@ func makeABigTable(ctx context.Context, db Database) (types.Map, error) {
 	return me.Map(ctx)
 }
 
-func pullerRefEquality(ctx context.Context, expectad, actual types.Ref, srcDB, sinkDB Database) (bool, error) {
-	expectedVal, err := expectad.TargetValue(ctx, srcDB)
+func pullerRefEquality(ctx context.Context, expectad, actual types.Ref, src, sink types.ValueReadWriter) (bool, error) {
+	expectedVal, err := expectad.TargetValue(ctx, src)
 
 	if err != nil {
 		return false, err
 	}
 
-	actualVal, err := actual.TargetValue(ctx, sinkDB)
+	actualVal, err := actual.TargetValue(ctx, sink)
 	if err != nil {
 		return false, err
 	}
@@ -404,13 +409,13 @@ func pullerRefEquality(ctx context.Context, expectad, actual types.Ref, srcDB, s
 			return errors.New("Missing table " + string(key.(types.String)))
 		}
 
-		exMapVal, err := exVal.(types.Ref).TargetValue(ctx, srcDB)
+		exMapVal, err := exVal.(types.Ref).TargetValue(ctx, src)
 
 		if err != nil {
 			return err
 		}
 
-		actMapVal, err := actVal.(types.Ref).TargetValue(ctx, sinkDB)
+		actMapVal, err := actVal.(types.Ref).TargetValue(ctx, sink)
 
 		if err != nil {
 			return err
@@ -475,7 +480,7 @@ func errIfNotEqual(ctx context.Context, ex, act types.Map) error {
 }
 
 func parentsAndTables(cm types.Struct) (types.List, types.Map, error) {
-	ps, ok, err := cm.MaybeGet(ParentsListField)
+	ps, ok, err := cm.MaybeGet(datas.ParentsListField)
 
 	if err != nil {
 		return types.EmptyList, types.EmptyMap, err
@@ -521,3 +526,8 @@ func writeValAndGetRef(ctx context.Context, vrw types.ValueReadWriter, val types
 
 	return valRef, err
 }
+
+func mustTuple(val types.Tuple, err error) types.Tuple {
+	d.PanicIfError(err)
+	return val
+}

go/store/datas/serialize_hashes.go

@@ -1,89 +0,0 @@
-// Copyright 2019 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// This file incorporates work covered by the following copyright and
-// permission notice:
-//
-// Copyright 2016 Attic Labs, Inc. All rights reserved.
-// Licensed under the Apache License, version 2.0:
-// http://www.apache.org/licenses/LICENSE-2.0
-
-package datas
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-
-	"github.com/dolthub/dolt/go/store/chunks"
-	"github.com/dolthub/dolt/go/store/hash"
-)
-
-func serializeHashes(w io.Writer, batch chunks.ReadBatch) error {
-	err := binary.Write(w, binary.BigEndian, uint32(len(batch))) // 4 billion hashes is probably absurd. Maybe this should be smaller?
-
-	if err != nil {
-		return err
-	}
-
-	for h := range batch {
-		err = serializeHash(w, h)
-
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func serializeHash(w io.Writer, h hash.Hash) error {
-	_, err := w.Write(h[:])
-
-	return err
-}
-
-func deserializeHashes(reader io.Reader) (hash.HashSlice, error) {
-	count := uint32(0)
-	err := binary.Read(reader, binary.BigEndian, &count)
-
-	if err != nil {
-		return hash.HashSlice{}, err
-	}
-
-	hashes := make(hash.HashSlice, count)
-	for i := range hashes {
-		hashes[i], err = deserializeHash(reader)
-
-		if err != nil {
-			return hash.HashSlice{}, err
-		}
-	}
-	return hashes, nil
-}
-
-func deserializeHash(reader io.Reader) (hash.Hash, error) {
-	h := hash.Hash{}
-	n, err := io.ReadFull(reader, h[:])
-
-	if err != nil {
-		return hash.Hash{}, err
-	}
-
-	if int(hash.ByteLen) != n {
-		return hash.Hash{}, errors.New("failed to read all data")
-	}
-
-	return h, nil
-}

go/store/datas/serialize_hashes_test.go

@@ -1,53 +0,0 @@
-// Copyright 2019 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// This file incorporates work covered by the following copyright and
-// permission notice:
-//
-// Copyright 2016 Attic Labs, Inc. All rights reserved.
-// Licensed under the Apache License, version 2.0:
-// http://www.apache.org/licenses/LICENSE-2.0
-
-package datas
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/dolthub/dolt/go/store/chunks"
-	"github.com/dolthub/dolt/go/store/hash"
-)
-
-func TestHashRoundTrip(t *testing.T) {
-	b := &bytes.Buffer{}
-	input := chunks.ReadBatch{
-		hash.Parse("00000000000000000000000000000000"): nil,
-		hash.Parse("00000000000000000000000000000001"): nil,
-		hash.Parse("00000000000000000000000000000002"): nil,
-		hash.Parse("00000000000000000000000000000003"): nil,
-	}
-	defer input.Close()
-
-	err := serializeHashes(b, input)
-	assert.NoError(t, err)
-	output, err := deserializeHashes(b)
-	assert.NoError(t, err)
-	assert.Len(t, output, len(input), "Output has different number of elements than input: %v, %v", output, input)
-	for _, h := range output {
-		_, present := input[h]
-		assert.True(t, present, "%s is in output but not in input", h)
-	}
-}

go/store/nbs/store.go

@@ -1105,6 +1105,8 @@ func (nbs *NomsBlockStore) updateManifest(ctx context.Context, current, last has
 }
 
 func (nbs *NomsBlockStore) Version() string {
+	nbs.mu.RLock()
+	defer nbs.mu.RUnlock()
 	return nbs.upstream.nbfVers
 }
 

go/store/prolly/map.go

@@ -17,6 +17,7 @@ package prolly
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	"github.com/dolthub/dolt/go/store/hash"
 	"github.com/dolthub/dolt/go/store/types"
@@ -80,14 +81,17 @@ func (m Map) Count() uint64 {
 	return m.root.cumulativeCount() / 2
 }
 
+// HashOf returns the Hash of this Map.
 func (m Map) HashOf() hash.Hash {
 	return hash.Of(m.root)
 }
 
+// Format returns the NomsBinFormat of this Map.
 func (m Map) Format() *types.NomsBinFormat {
 	return m.ns.Format()
 }
 
+// Descriptors returns the TupleDesc's from this Map.
 func (m Map) Descriptors() (val.TupleDesc, val.TupleDesc) {
 	return m.keyDesc, m.valDesc
 }
@@ -136,50 +140,97 @@ func (m Map) IterAll(ctx context.Context) (MapRangeIter, error) {
 		Start:   RangeCut{Unbound: true},
 		Stop:    RangeCut{Unbound: true},
 		KeyDesc: m.keyDesc,
-		Reverse: false,
 	}
 	return m.IterRange(ctx, rng)
 }
 
-// IterValueRange returns a MapRangeIter that iterates over a Range.
+// IterRange returns a MapRangeIter that iterates over a Range.
 func (m Map) IterRange(ctx context.Context, rng Range) (MapRangeIter, error) {
-	var cur *nodeCursor
-	var err error
-
-	if rng.Start.Unbound {
-		if rng.Reverse {
-			cur, err = m.cursorAtEnd(ctx)
-		} else {
-			cur, err = m.cursorAtStart(ctx)
-		}
-	} else {
-		cur, err = m.cursorAtkey(ctx, rng.Start.Key)
-	}
+	iter, err := m.iterFromRange(ctx, rng)
 	if err != nil {
 		return MapRangeIter{}, err
 	}
-	proCur := mapTupleCursor{cur: cur}
 
-	return NewMapRangeIter(ctx, nil, proCur, rng)
+	return NewMapRangeIter(nil, iter, rng), nil
 }
 
-func (m Map) cursorAtStart(ctx context.Context) (*nodeCursor, error) {
-	return newCursorAtStart(ctx, m.ns, m.root)
-}
+func (m Map) iterFromRange(ctx context.Context, rng Range) (*prollyRangeIter, error) {
+	var (
+		err   error
+		start *nodeCursor
+		stop  *nodeCursor
+	)
 
-func (m Map) cursorAtEnd(ctx context.Context) (*nodeCursor, error) {
-	return newCursorAtEnd(ctx, m.ns, m.root)
-}
-
-func (m Map) cursorAtkey(ctx context.Context, key val.Tuple) (*nodeCursor, error) {
-	cur, err := newCursorAtItem(ctx, m.ns, m.root, nodeItem(key), m.searchNode)
-	if err == nil {
-		cur.keepInBounds()
+	startSearch := m.rangeStartSearchFn(rng)
+	if rng.Start.Unbound {
+		start, err = newCursorAtStart(ctx, m.ns, m.root)
+	} else {
+		start, err = newCursorAtTuple(ctx, m.ns, m.root, rng.Start.Key, startSearch)
 	}
-	return cur, err
+	if err != nil {
+		return nil, err
+	}
+
+	stopSearch := m.rangeStopSearchFn(rng)
+	if rng.Stop.Unbound {
+		stop, err = newCursorPastEnd(ctx, m.ns, m.root)
+	} else {
+		stop, err = newCursorAtTuple(ctx, m.ns, m.root, rng.Stop.Key, stopSearch)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if start.compare(stop) >= 0 {
+		start = nil // empty range
+	}
+
+	return &prollyRangeIter{
+		curr: start,
+		stop: stop,
+	}, nil
 }
 
-// searchNode is a searchFn for a Map, adapted from search.Sort.
+func (m Map) rangeStartSearchFn(rng Range) searchFn {
+	// todo(andy): inline sort.Search()
+	return func(query nodeItem, nd Node) int {
+		i := sort.Search(nd.nodeCount()/stride, func(i int) bool {
+			q := val.Tuple(query)
+			t := val.Tuple(nd.getItem(i * stride))
+
+			// compare using the range's tuple descriptor.
+			cmp := rng.KeyDesc.Compare(q, t)
+			if rng.Start.Inclusive {
+				return cmp <= 0
+			} else {
+				return cmp < 0
+			}
+		})
+		return i * stride
+	}
+}
+
+func (m Map) rangeStopSearchFn(rng Range) searchFn {
+	// todo(andy): inline sort.Search()
+	return func(query nodeItem, nd Node) int {
+		i := sort.Search(nd.nodeCount()/stride, func(i int) bool {
+			q := val.Tuple(query)
+			t := val.Tuple(nd.getItem(i * stride))
+
+			// compare using the range's tuple descriptor.
+			cmp := rng.KeyDesc.Compare(q, t)
+			if rng.Stop.Inclusive {
+				return cmp < 0
+			} else {
+				return cmp <= 0
+			}
+		})
+		return i * stride
+	}
+}
+
+// searchNode returns the smallest index where nd[i] >= query
+// Adapted from search.Sort to inline comparison.
 func (m Map) searchNode(query nodeItem, nd Node) int {
 	n := nd.nodeCount() / stride
 	// Define f(-1) == false and f(n) == true.
@@ -200,6 +251,8 @@ func (m Map) searchNode(query nodeItem, nd Node) int {
 	return i * stride
 }
 
+var _ searchFn = Map{}.searchNode
+
 // compareItems is a compareFn.
 func (m Map) compareItems(left, right nodeItem) int {
 	l, r := val.Tuple(left), val.Tuple(right)
@@ -210,26 +263,34 @@ func (m Map) compareKeys(left, right val.Tuple) int {
 	return int(m.keyDesc.Compare(left, right))
 }
 
-type mapTupleCursor struct {
-	cur *nodeCursor
+type prollyRangeIter struct {
+	// current tuple location
+	curr *nodeCursor
+	// non-inclusive range stop
+	stop *nodeCursor
 }
 
-var _ tupleCursor = mapTupleCursor{}
+var _ rangeIter = &prollyRangeIter{}
 
-func (cur mapTupleCursor) current() (key, value val.Tuple) {
-	if cur.cur.valid() {
-		pair := cur.cur.currentPair()
-		key, value = val.Tuple(pair.key()), val.Tuple(pair.value())
+func (it *prollyRangeIter) current() (key, value val.Tuple) {
+	// |it.curr| is set to nil when its range is exhausted
+	if it.curr != nil && it.curr.valid() {
+		p := it.curr.currentPair()
+		return val.Tuple(p.key()), val.Tuple(p.value())
 	}
 	return
 }
 
-func (cur mapTupleCursor) advance(ctx context.Context) (err error) {
-	_, err = cur.cur.advance(ctx)
-	return
-}
+func (it *prollyRangeIter) iterate(ctx context.Context) (err error) {
+	_, err = it.curr.advance(ctx)
+	if err != nil {
+		return err
+	}
+
+	if it.curr.compare(it.stop) >= 0 {
+		// past the end of the range
+		it.curr = nil
+	}
 
-func (cur mapTupleCursor) retreat(ctx context.Context) (err error) {
-	_, err = cur.cur.retreat(ctx)
 	return
 }

go/store/prolly/map_test.go

@@ -18,10 +18,7 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"math"
 	"math/rand"
-	"sort"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -41,35 +38,34 @@ func TestMap(t *testing.T) {
 	}
 
 	for _, s := range scales {
-		name := fmt.Sprintf("test proCur map at scale %d", s)
+		name := fmt.Sprintf("test prolly map at scale %d", s)
 		t.Run(name, func(t *testing.T) {
 			prollyMap, tuples := makeProllyMap(t, s)
 
 			t.Run("get item from map", func(t *testing.T) {
-				testOrderedMapGet(t, prollyMap, tuples)
+				testGet(t, prollyMap, tuples)
 			})
 			t.Run("iter all from map", func(t *testing.T) {
-				testOrderedMapIterAll(t, prollyMap, tuples)
+				testIterAll(t, prollyMap, tuples)
 			})
-			t.Run("iter all backwards from map", func(t *testing.T) {
-				testOrderedMapIterAllBackward(t, prollyMap, tuples)
+			t.Run("iter range", func(t *testing.T) {
+				testIterRange(t, prollyMap, tuples)
 			})
-			t.Run("iter value range", func(t *testing.T) {
-				testOrderedMapIterValueRange(t, prollyMap, tuples)
+
+			indexMap, tuples2 := makeProllySecondaryIndex(t, s)
+			t.Run("iter prefix range", func(t *testing.T) {
+				testIterPrefixRange(t, indexMap, tuples2)
 			})
 
 			pm := prollyMap.(Map)
 			t.Run("item exists in map", func(t *testing.T) {
-				testProllyMapHas(t, pm, tuples)
+				testHas(t, pm, tuples)
 			})
 		})
 	}
 }
 
 func makeProllyMap(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
-	ctx := context.Background()
-	ns := newTestNodeStore()
-
 	kd := val.NewTupleDescriptor(
 		val.Type{Enc: val.Uint32Enc, Nullable: false},
 	)
@@ -80,6 +76,27 @@ func makeProllyMap(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
 	)
 
 	tuples := randomTuplePairs(count, kd, vd)
+	om := prollyMapFromTuple(t, count, kd, vd, tuples)
+
+	return om, tuples
+}
+
+func makeProllySecondaryIndex(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
+	kd := val.NewTupleDescriptor(
+		val.Type{Enc: val.Uint32Enc, Nullable: true},
+		val.Type{Enc: val.Uint32Enc, Nullable: false},
+	)
+	vd := val.NewTupleDescriptor()
+
+	tuples := randomCompositeTuplePairs(count, kd, vd)
+	om := prollyMapFromTuple(t, count, kd, vd, tuples)
+
+	return om, tuples
+}
+
+func prollyMapFromTuple(t *testing.T, count int, kd, vd val.TupleDesc, tuples [][2]val.Tuple) orderedMap {
+	ctx := context.Background()
+	ns := newTestNodeStore()
 
 	chunker, err := newEmptyTreeChunker(ctx, ns, newDefaultNodeSplitter)
 	require.NoError(t, err)
@@ -98,33 +115,10 @@ func makeProllyMap(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
 		ns:      ns,
 	}
 
-	return m, tuples
+	return m
 }
 
-type orderedMap interface {
-	Get(ctx context.Context, key val.Tuple, cb KeyValueFn) (err error)
-	IterAll(ctx context.Context) (MapRangeIter, error)
-	IterRange(ctx context.Context, rng Range) (MapRangeIter, error)
-}
-
-var _ orderedMap = Map{}
-var _ orderedMap = MutableMap{}
-var _ orderedMap = memoryMap{}
-
-func getKeyDesc(om orderedMap) val.TupleDesc {
-	switch m := om.(type) {
-	case Map:
-		return m.keyDesc
-	case MutableMap:
-		return m.m.keyDesc
-	case memoryMap:
-		return m.keyDesc
-	default:
-		panic("unknown ordered map")
-	}
-}
-
-func testOrderedMapGet(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
+func testGet(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
 	ctx := context.Background()
 	for _, kv := range tuples {
 		err := om.Get(ctx, kv[0], func(key, val val.Tuple) (err error) {
@@ -137,7 +131,7 @@ func testOrderedMapGet(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
 	}
 }
 
-func testProllyMapHas(t *testing.T, om Map, tuples [][2]val.Tuple) {
+func testHas(t *testing.T, om Map, tuples [][2]val.Tuple) {
 	ctx := context.Background()
 	for _, kv := range tuples {
 		ok, err := om.Has(ctx, kv[0])
@@ -146,7 +140,7 @@ func testProllyMapHas(t *testing.T, om Map, tuples [][2]val.Tuple) {
 	}
 }
 
-func testOrderedMapIterAll(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
+func testIterAll(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
 	ctx := context.Background()
 	iter, err := om.IterAll(ctx)
 	require.NoError(t, err)
@@ -166,271 +160,8 @@ func testOrderedMapIterAll(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
 
 	assert.Equal(t, len(tuples), idx)
 	for i, kv := range actual {
-		require.NoError(t, err)
+		require.True(t, i < len(tuples))
 		assert.Equal(t, tuples[i][0], kv[0])
 		assert.Equal(t, tuples[i][1], kv[1])
 	}
 }
-
-func testOrderedMapIterAllBackward(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
-	desc := getKeyDesc(om)
-	rng := Range{
-		Start:   RangeCut{Unbound: true},
-		Stop:    RangeCut{Unbound: true},
-		KeyDesc: desc,
-		Reverse: true,
-	}
-
-	ctx := context.Background()
-	iter, err := om.IterRange(ctx, rng)
-	require.NoError(t, err)
-
-	idx := len(tuples) - 1
-	for {
-		key, value, err := iter.Next(ctx)
-		if err == io.EOF {
-			break
-		}
-		require.NoError(t, err)
-		assert.Equal(t, tuples[idx][0], key)
-		assert.Equal(t, tuples[idx][1], value)
-		idx--
-	}
-	assert.Equal(t, -1, idx)
-}
-
-type rangeTest struct {
-	name      string
-	testRange Range
-	expCount  int
-}
-
-func testOrderedMapIterValueRange(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
-	ctx := context.Background()
-	desc := getKeyDesc(om)
-
-	for i := 0; i < 100; i++ {
-
-		cnt := len(tuples)
-		a, z := testRand.Intn(cnt), testRand.Intn(cnt)
-		if a > z {
-			a, z = z, a
-		}
-		start, stop := tuples[a][0], tuples[z][0]
-
-		tests := []rangeTest{
-			// two-sided ranges
-			{
-				name:      "OpenRange",
-				testRange: OpenRange(start, stop, desc),
-				expCount:  nonNegative((z - a) - 1),
-			},
-			{
-				name:      "OpenStartRange",
-				testRange: OpenStartRange(start, stop, desc),
-				expCount:  z - a,
-			},
-			{
-				name:      "OpenStopRange",
-				testRange: OpenStopRange(start, stop, desc),
-				expCount:  z - a,
-			},
-			{
-				name:      "ClosedRange",
-				testRange: ClosedRange(start, stop, desc),
-				expCount:  (z - a) + 1,
-			},
-
-			// put it down flip it and reverse it
-			{
-				name:      "OpenRange",
-				testRange: OpenRange(stop, start, desc),
-				expCount:  nonNegative((z - a) - 1),
-			},
-			{
-				name:      "OpenStartRange",
-				testRange: OpenStartRange(stop, start, desc),
-				expCount:  z - a,
-			},
-			{
-				name:      "OpenStopRange",
-				testRange: OpenStopRange(stop, start, desc),
-				expCount:  z - a,
-			},
-			{
-				name:      "ClosedRange",
-				testRange: ClosedRange(stop, start, desc),
-				expCount:  (z - a) + 1,
-			},
-
-			// one-sided ranges
-			{
-				name:      "GreaterRange",
-				testRange: GreaterRange(start, desc),
-				expCount:  nonNegative(cnt - a - 1),
-			},
-			{
-				name:      "GreaterOrEqualRange",
-				testRange: GreaterOrEqualRange(start, desc),
-				expCount:  cnt - a,
-			},
-			{
-				name:      "LesserRange",
-				testRange: LesserRange(stop, desc),
-				expCount:  z,
-			},
-			{
-				name:      "LesserOrEqualRange",
-				testRange: LesserOrEqualRange(stop, desc),
-				expCount:  z + 1,
-			},
-		}
-
-		for _, test := range tests {
-			iter, err := om.IterRange(ctx, test.testRange)
-			require.NoError(t, err)
-
-			key, _, err := iter.Next(ctx)
-			actCount := 0
-			for err != io.EOF {
-				actCount++
-				prev := key
-				key, _, err = iter.Next(ctx)
-
-				if key != nil {
-					if test.testRange.Reverse {
-						assert.True(t, desc.Compare(prev, key) > 0)
-					} else {
-						assert.True(t, desc.Compare(prev, key) < 0)
-					}
-				}
-			}
-			assert.Equal(t, io.EOF, err)
-			assert.Equal(t, test.expCount, actCount)
-			//fmt.Printf("a: %d \t z: %d cnt: %d", a, z, cnt)
-		}
-	}
-}
-
-func randomTuplePairs(count int, keyDesc, valDesc val.TupleDesc) (items [][2]val.Tuple) {
-	keyBuilder := val.NewTupleBuilder(keyDesc)
-	valBuilder := val.NewTupleBuilder(valDesc)
-
-	items = make([][2]val.Tuple, count)
-	for i := range items {
-		items[i][0] = randomTuple(keyBuilder)
-		items[i][1] = randomTuple(valBuilder)
-	}
-
-	sortTuplePairs(items, keyDesc)
-
-	for i := range items {
-		if i == 0 {
-			continue
-		}
-		if keyDesc.Compare(items[i][0], items[i-1][0]) == 0 {
-			panic("duplicate key, unlucky!")
-		}
-	}
-	return
-}
-
-func randomTuple(tb *val.TupleBuilder) (tup val.Tuple) {
-	for i, typ := range tb.Desc.Types {
-		randomField(tb, i, typ)
-	}
-	return tb.Build(sharedPool)
-}
-
-func cloneRandomTuples(items [][2]val.Tuple) (clone [][2]val.Tuple) {
-	clone = make([][2]val.Tuple, len(items))
-	for i := range clone {
-		clone[i] = items[i]
-	}
-	return
-}
-
-func sortTuplePairs(items [][2]val.Tuple, keyDesc val.TupleDesc) {
-	sort.Slice(items, func(i, j int) bool {
-		return keyDesc.Compare(items[i][0], items[j][0]) < 0
-	})
-}
-
-func shuffleTuplePairs(items [][2]val.Tuple) {
-	testRand.Shuffle(len(items), func(i, j int) {
-		items[i], items[j] = items[j], items[i]
-	})
-}
-
-func randomField(tb *val.TupleBuilder, idx int, typ val.Type) {
-	// todo(andy): add NULLs
-
-	neg := -1
-	if testRand.Int()%2 == 1 {
-		neg = 1
-	}
-
-	switch typ.Enc {
-	case val.Int8Enc:
-		v := int8(testRand.Intn(math.MaxInt8) * neg)
-		tb.PutInt8(idx, v)
-	case val.Uint8Enc:
-		v := uint8(testRand.Intn(math.MaxUint8))
-		tb.PutUint8(idx, v)
-	case val.Int16Enc:
-		v := int16(testRand.Intn(math.MaxInt16) * neg)
-		tb.PutInt16(idx, v)
-	case val.Uint16Enc:
-		v := uint16(testRand.Intn(math.MaxUint16))
-		tb.PutUint16(idx, v)
-	case val.Int32Enc:
-		v := int32(testRand.Intn(math.MaxInt32) * neg)
-		tb.PutInt32(idx, v)
-	case val.Uint32Enc:
-		v := uint32(testRand.Intn(math.MaxUint32))
-		tb.PutUint32(idx, v)
-	case val.Int64Enc:
-		v := int64(testRand.Intn(math.MaxInt64) * neg)
-		tb.PutInt64(idx, v)
-	case val.Uint64Enc:
-		v := uint64(testRand.Uint64())
-		tb.PutUint64(idx, v)
-	case val.Float32Enc:
-		tb.PutFloat32(idx, testRand.Float32())
-	case val.Float64Enc:
-		tb.PutFloat64(idx, testRand.Float64())
-	case val.StringEnc:
-		buf := make([]byte, (testRand.Int63()%40)+10)
-		testRand.Read(buf)
-		tb.PutString(idx, string(buf))
-	case val.BytesEnc:
-		buf := make([]byte, (testRand.Int63()%40)+10)
-		testRand.Read(buf)
-		tb.PutBytes(idx, buf)
-	default:
-		panic("unknown encoding")
-	}
-}
-
-func nonNegative(x int) int {
-	if x < 0 {
-		x = 0
-	}
-	return x
-}
-
-func fmtTupleList(tuples [][2]val.Tuple, kd, vd val.TupleDesc) string {
-	var sb strings.Builder
-	sb.WriteString("{ ")
-	for _, kv := range tuples {
-		if kv[0] == nil || kv[1] == nil {
-			break
-		}
-		sb.WriteString(kd.Format(kv[0]))
-		sb.WriteString(": ")
-		sb.WriteString(vd.Format(kv[1]))
-		sb.WriteString(", ")
-	}
-	sb.WriteString("}")
-	return sb.String()
-}

go/store/prolly/memory_map.go

@@ -56,6 +56,11 @@ func (mm memoryMap) Put(key, val val.Tuple) {
 	mm.list.Put(key, val)
 }
 
+// Delete deletes the Tuple pair keyed by |key|.
+func (mm memoryMap) Delete(key val.Tuple) {
+	mm.list.Put(key, nil)
+}
+
 // Get fetches the Tuple pair keyed by |key|, if it exists, and passes it to |cb|.
 // If the |key| is not present in the memoryMap, a nil Tuple pair is passed to |cb|.
 func (mm memoryMap) Get(_ context.Context, key val.Tuple, cb KeyValueFn) error {
@@ -75,41 +80,98 @@ func (mm memoryMap) IterAll(ctx context.Context) (MapRangeIter, error) {
 		Start:   RangeCut{Unbound: true},
 		Stop:    RangeCut{Unbound: true},
 		KeyDesc: mm.keyDesc,
-		Reverse: false,
 	}
 	return mm.IterRange(ctx, rng)
 }
 
 // IterValueRange returns a MapRangeIter that iterates over a Range.
 func (mm memoryMap) IterRange(ctx context.Context, rng Range) (MapRangeIter, error) {
+	memIter := mm.iterFromRange(rng)
+	return NewMapRangeIter(memIter, nil, rng), nil
+}
+
+func (mm memoryMap) iterFromRange(rng Range) *memRangeIter {
 	var iter *skip.ListIter
 	if rng.Start.Unbound {
-		if rng.Reverse {
-			iter = mm.list.IterAtEnd()
-		} else {
-			iter = mm.list.IterAtStart()
-		}
+		iter = mm.list.IterAtStart()
 	} else {
-		iter = mm.list.IterAt(rng.Start.Key)
+		vc := valueCmpForRange(rng)
+		iter = mm.list.GetIterAtWithFn(rng.Start.Key, vc)
 	}
-	memCur := memTupleCursor{iter: iter}
 
-	return NewMapRangeIter(ctx, memCur, nil, rng)
+	// enforce range start
+	var key val.Tuple
+	for {
+		key, _ = iter.Current()
+		if key == nil || rng.insideStart(key) {
+			break // |i| inside |rng|
+		}
+		iter.Advance()
+	}
+
+	// enforce range end
+	if key == nil || !rng.insideStop(key) {
+		iter = nil
+	}
+
+	return &memRangeIter{
+		iter: iter,
+		rng:  rng,
+	}
+}
+
+func valueCmpForRange(rng Range) skip.ValueCmp {
+	return func(left, right []byte) int {
+		l, r := val.Tuple(left), val.Tuple(right)
+		return rng.KeyDesc.Compare(l, r)
+	}
 }
 
 func (mm memoryMap) mutations() mutationIter {
-	return memTupleCursor{iter: mm.list.IterAtStart()}
+	return &memRangeIter{
+		iter: mm.list.IterAtStart(),
+		rng: Range{
+			Start:   RangeCut{Unbound: true},
+			Stop:    RangeCut{Unbound: true},
+			KeyDesc: mm.keyDesc,
+		},
+	}
 }
 
-type memTupleCursor struct {
-	iter    *skip.ListIter
-	reverse bool
+type memRangeIter struct {
+	iter *skip.ListIter
+	rng  Range
 }
 
-var _ tupleCursor = memTupleCursor{}
-var _ mutationIter = memTupleCursor{}
+var _ rangeIter = &memRangeIter{}
+var _ mutationIter = &memRangeIter{}
 
-func (it memTupleCursor) nextMutation() (key, value val.Tuple) {
+// current returns the iter's current Tuple pair, or nil Tuples
+// if the iter has exhausted its range, it will
+func (it *memRangeIter) current() (key, value val.Tuple) {
+	// |it.iter| is set to nil when its range is exhausted
+	if it.iter != nil {
+		key, value = it.iter.Current()
+	}
+	return
+}
+
+// iterate progresses the iter inside its range, skipping
+// over pending deletes in the memoryMap.
+func (it *memRangeIter) iterate(context.Context) (err error) {
+	for {
+		it.iter.Advance()
+
+		k, _ := it.current()
+		if k == nil || !it.rng.insideStop(k) {
+			it.iter = nil // range exhausted
+		}
+
+		return
+	}
+}
+
+func (it *memRangeIter) nextMutation() (key, value val.Tuple) {
 	key, value = it.iter.Current()
 	if key == nil {
 		return
@@ -118,24 +180,10 @@ func (it memTupleCursor) nextMutation() (key, value val.Tuple) {
 	return
 }
 
-func (it memTupleCursor) current() (key, value val.Tuple) {
-	return it.iter.Current()
-}
-
-func (it memTupleCursor) advance(context.Context) (err error) {
-	it.iter.Advance()
-	return
-}
-
-func (it memTupleCursor) retreat(context.Context) (err error) {
-	it.iter.Retreat()
-	return
-}
-
-func (it memTupleCursor) count() int {
+func (it *memRangeIter) count() int {
 	return it.iter.Count()
 }
 
-func (it memTupleCursor) close() error {
+func (it *memRangeIter) close() error {
 	return nil
 }

go/store/prolly/memory_map_test.go

@@ -17,7 +17,6 @@ package prolly
 import (
 	"context"
 	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -35,50 +34,54 @@ func TestMemMap(t *testing.T) {
 	}
 
 	for _, s := range scales {
-		name := fmt.Sprintf("test memCur map at scale %d", s)
+		name := fmt.Sprintf("test memory map at scale %d", s)
 		t.Run(name, func(t *testing.T) {
 
 			memMap, tuples := makeMemMap(t, s)
 			t.Run("get item from map", func(t *testing.T) {
-				testOrderedMapGet(t, memMap, tuples)
+				testGet(t, memMap, tuples)
 			})
 			t.Run("iter all from map", func(t *testing.T) {
-				testOrderedMapIterAll(t, memMap, tuples)
+				testIterAll(t, memMap, tuples)
 			})
-			t.Run("iter all backwards from map", func(t *testing.T) {
-				testOrderedMapIterAllBackward(t, memMap, tuples)
-			})
-			t.Run("iter value range", func(t *testing.T) {
-				testOrderedMapIterValueRange(t, memMap, tuples)
+			t.Run("iter range", func(t *testing.T) {
+				testIterRange(t, memMap, tuples)
 			})
 
-			memMap2, tuples2, deletes := makeMemMapWithDeletes(t, s)
+			memIndex, idxTuples := makeMemSecondaryIndex(t, s)
+			t.Run("iter prefix range", func(t *testing.T) {
+				testIterPrefixRange(t, memIndex, idxTuples)
+			})
+
+			memMap2, tuples2, deletes := deleteFromMemoryMap(memMap.(memoryMap), tuples)
 			t.Run("get item from map with deletes", func(t *testing.T) {
 				testMemoryMapGetAndHas(t, memMap2, tuples2, deletes)
 			})
 			t.Run("iter all from map with deletes", func(t *testing.T) {
-				testOrderedMapIterAll(t, memMap2, tuples2)
+				testIterAll(t, memMap2, tuples2)
 			})
-			t.Run("iter all backwards from map", func(t *testing.T) {
-				testOrderedMapIterAllBackward(t, memMap2, tuples2)
+			t.Run("iter range", func(t *testing.T) {
+				testIterRange(t, memMap2, tuples2)
 			})
-			t.Run("iter value range", func(t *testing.T) {
-				testOrderedMapIterValueRange(t, memMap2, tuples2)
+
+			memIndex, idxTuples2, _ := deleteFromMemoryMap(memIndex.(memoryMap), idxTuples)
+			t.Run("iter prefix range", func(t *testing.T) {
+				testIterPrefixRange(t, memIndex, idxTuples2)
 			})
 		})
 	}
 }
 
-var memKeyDesc = val.NewTupleDescriptor(
-	val.Type{Enc: val.Uint32Enc, Nullable: false},
-)
-var memValueDesc = val.NewTupleDescriptor(
-	val.Type{Enc: val.Uint32Enc, Nullable: true},
-	val.Type{Enc: val.Uint32Enc, Nullable: true},
-	val.Type{Enc: val.Uint32Enc, Nullable: true},
-)
-
 func makeMemMap(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
+	memKeyDesc := val.NewTupleDescriptor(
+		val.Type{Enc: val.Uint32Enc, Nullable: false},
+	)
+	memValueDesc := val.NewTupleDescriptor(
+		val.Type{Enc: val.Uint32Enc, Nullable: true},
+		val.Type{Enc: val.Uint32Enc, Nullable: true},
+		val.Type{Enc: val.Uint32Enc, Nullable: true},
+	)
+
 	tuples := randomTuplePairs(count, memKeyDesc, memValueDesc)
 	mm := newMemoryMap(memKeyDesc)
 	for _, pair := range tuples {
@@ -88,27 +91,42 @@ func makeMemMap(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
 	return mm, tuples
 }
 
-func makeMemMapWithDeletes(t *testing.T, count int) (mut memoryMap, tuples, deletes [][2]val.Tuple) {
-	om, tuples := makeMemMap(t, count)
-	mut = om.(memoryMap)
+func makeMemSecondaryIndex(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
+	memKeyDesc := val.NewTupleDescriptor(
+		val.Type{Enc: val.Uint32Enc, Nullable: false},
+		val.Type{Enc: val.Uint32Enc, Nullable: true},
+	)
+	memValueDesc := val.NewTupleDescriptor()
 
+	tuples := randomCompositeTuplePairs(count, memKeyDesc, memValueDesc)
+
+	mm := newMemoryMap(memKeyDesc)
+	for _, pair := range tuples {
+		mm.Put(pair[0], pair[1])
+	}
+
+	return mm, tuples
+}
+
+func deleteFromMemoryMap(mm memoryMap, tt [][2]val.Tuple) (memoryMap, [][2]val.Tuple, [][2]val.Tuple) {
+	count := len(tt)
 	testRand.Shuffle(count, func(i, j int) {
-		tuples[i], tuples[j] = tuples[j], tuples[i]
+		tt[i], tt[j] = tt[j], tt[i]
 	})
 
 	// delete 1/4 of tuples
-	deletes = tuples[:count/4]
+	deletes := tt[:count/4]
 
 	// re-sort the remaining tuples
-	tuples = tuples[count/4:]
-	desc := getKeyDesc(om)
-	sortTuplePairs(tuples, desc)
+	remaining := tt[count/4:]
+	desc := keyDescFromMap(mm)
+	sortTuplePairs(remaining, desc)
 
 	for _, kv := range deletes {
-		mut.Put(kv[0], nil)
+		mm.Put(kv[0], nil)
 	}
 
-	return mut, tuples, deletes
+	return mm, remaining, deletes
 }
 
 func testMemoryMapGetAndHas(t *testing.T, mem memoryMap, tuples, deletes [][2]val.Tuple) {
@@ -133,28 +151,3 @@ func testMemoryMapGetAndHas(t *testing.T, mem memoryMap, tuples, deletes [][2]va
 		require.NoError(t, err)
 	}
 }
-
-func debugFmt(tup val.Tuple, desc val.TupleDesc) (s string) {
-	if tup == nil {
-		s = "[ nil ]"
-	} else {
-		s = desc.Format(tup)
-	}
-	return
-}
-
-func fmtMany(tuples, deletes [][2]val.Tuple) string {
-	tuples = append(tuples, deletes...)
-	sortTuplePairs(tuples, memKeyDesc)
-
-	var sb strings.Builder
-	sb.WriteString("{ ")
-	for _, kv := range tuples {
-		sb.WriteString(debugFmt(kv[0], memKeyDesc))
-		sb.WriteString(": ")
-		sb.WriteString(debugFmt(kv[1], memValueDesc))
-		sb.WriteString(", ")
-	}
-	sb.WriteString("}")
-	return sb.String()
-}

go/store/prolly/mutable_map.go

@@ -17,8 +17,6 @@ package prolly
 import (
 	"context"
 
-	"github.com/dolthub/dolt/go/store/skip"
-
 	"github.com/dolthub/dolt/go/store/val"
 )
 
@@ -27,20 +25,20 @@ const (
 )
 
 type MutableMap struct {
-	m       Map
+	prolly  Map
 	overlay memoryMap
 }
 
 func newMutableMap(m Map) MutableMap {
 	return MutableMap{
-		m:       m,
+		prolly:  m,
 		overlay: newMemoryMap(m.keyDesc),
 	}
 }
 
 // Map materializes the pending mutations in the MutableMap.
 func (mut MutableMap) Map(ctx context.Context) (Map, error) {
-	return materializeMutations(ctx, mut.m, mut.overlay.mutations())
+	return materializeMutations(ctx, mut.prolly, mut.overlay.mutations())
 }
 
 // Put adds the Tuple pair |key|, |value| to the MutableMap.
@@ -51,7 +49,7 @@ func (mut MutableMap) Put(_ context.Context, key, value val.Tuple) error {
 
 // Delete deletes the pair keyed by |key| from the MutableMap.
 func (mut MutableMap) Delete(_ context.Context, key val.Tuple) error {
-	mut.overlay.Put(key, nil)
+	mut.overlay.Delete(key)
 	return nil
 }
 
@@ -67,7 +65,7 @@ func (mut MutableMap) Get(ctx context.Context, key val.Tuple, cb KeyValueFn) (er
 		return cb(key, value)
 	}
 
-	return mut.m.Get(ctx, key, cb)
+	return mut.prolly.Get(ctx, key, cb)
 }
 
 // Has returns true if |key| is present in the MutableMap.
@@ -84,41 +82,18 @@ func (mut MutableMap) IterAll(ctx context.Context) (MapRangeIter, error) {
 	rng := Range{
 		Start:   RangeCut{Unbound: true},
 		Stop:    RangeCut{Unbound: true},
-		KeyDesc: mut.m.keyDesc,
-		Reverse: false,
+		KeyDesc: mut.prolly.keyDesc,
 	}
 	return mut.IterRange(ctx, rng)
 }
 
 // IterValueRange returns a MapRangeIter that iterates over a Range.
 func (mut MutableMap) IterRange(ctx context.Context, rng Range) (MapRangeIter, error) {
-	var iter *skip.ListIter
-	if rng.Start.Unbound {
-		if rng.Reverse {
-			iter = mut.overlay.list.IterAtEnd()
-		} else {
-			iter = mut.overlay.list.IterAtStart()
-		}
-	} else {
-		iter = mut.overlay.list.IterAt(rng.Start.Key)
-	}
-	memCur := memTupleCursor{iter: iter}
-
-	var err error
-	var cur *nodeCursor
-	if rng.Start.Unbound {
-		if rng.Reverse {
-			cur, err = mut.m.cursorAtEnd(ctx)
-		} else {
-			cur, err = mut.m.cursorAtStart(ctx)
-		}
-	} else {
-		cur, err = mut.m.cursorAtkey(ctx, rng.Start.Key)
-	}
+	proIter, err := mut.prolly.iterFromRange(ctx, rng)
 	if err != nil {
 		return MapRangeIter{}, err
 	}
-	proCur := mapTupleCursor{cur: cur}
+	memIter := mut.overlay.iterFromRange(rng)
 
-	return NewMapRangeIter(ctx, memCur, proCur, rng)
+	return NewMapRangeIter(memIter, proIter, rng), nil
 }

go/store/prolly/mutable_map_read_test.go

@@ -39,45 +39,55 @@ func TestMutableMapReads(t *testing.T) {
 
 			mutableMap, tuples := makeMutableMap(t, s)
 			t.Run("get item from map", func(t *testing.T) {
-				testOrderedMapGet(t, mutableMap, tuples)
+				testGet(t, mutableMap, tuples)
 			})
 			t.Run("iter all from map", func(t *testing.T) {
-				testOrderedMapIterAll(t, mutableMap, tuples)
+				testIterAll(t, mutableMap, tuples)
 			})
-			t.Run("iter all backwards from map", func(t *testing.T) {
-				testOrderedMapIterAllBackward(t, mutableMap, tuples)
+			t.Run("iter range", func(t *testing.T) {
+				testIterRange(t, mutableMap, tuples)
 			})
-			t.Run("iter value range", func(t *testing.T) {
-				testOrderedMapIterValueRange(t, mutableMap, tuples)
+			t.Run("iter prefix range", func(t *testing.T) {
+				testIterPrefixRange(t, mutableMap, tuples)
 			})
 
-			mutableMap2, tuples2, deletes := makeMutableMapWithDeletes(t, s)
+			mutableIndex, idxTuples := makeMutableSecondaryIndex(t, s)
+			t.Run("iter prefix range", func(t *testing.T) {
+				testIterPrefixRange(t, mutableIndex, idxTuples)
+			})
+
+			mutableMap2, tuples2, deletes := deleteFromMutableMap(mutableMap.(MutableMap), tuples)
 			t.Run("get item from map with deletes", func(t *testing.T) {
 				testMutableMapGetAndHas(t, mutableMap2, tuples2, deletes)
 			})
 			t.Run("iter all from map with deletes", func(t *testing.T) {
-				testOrderedMapIterAll(t, mutableMap2, tuples2)
+				testIterAll(t, mutableMap2, tuples2)
 			})
-			t.Run("iter all backwards from map", func(t *testing.T) {
-				testOrderedMapIterAllBackward(t, mutableMap2, tuples2)
+			t.Run("iter range with pending deletes", func(t *testing.T) {
+				testIterRange(t, mutableMap2, tuples2)
 			})
-			t.Run("iter value range", func(t *testing.T) {
-				testOrderedMapIterValueRange(t, mutableMap2, tuples2)
+
+			mutableIndex2, idxTuples2, _ := deleteFromMutableMap(mutableIndex.(MutableMap), idxTuples)
+			t.Run("iter prefix range", func(t *testing.T) {
+				testIterPrefixRange(t, mutableIndex, idxTuples2)
 			})
 
 			prollyMap, err := mutableMap2.Map(context.Background())
 			require.NoError(t, err)
-			t.Run("get item from map with deletes", func(t *testing.T) {
-				testProllyMapHas(t, prollyMap, tuples2)
+			t.Run("get item from map after deletes applied", func(t *testing.T) {
+				testHas(t, prollyMap, tuples2)
 			})
-			t.Run("iter all from map with deletes", func(t *testing.T) {
-				testOrderedMapIterAll(t, prollyMap, tuples2)
+			t.Run("iter all from map after deletes applied", func(t *testing.T) {
+				testIterAll(t, prollyMap, tuples2)
 			})
-			t.Run("iter all backwards from map", func(t *testing.T) {
-				testOrderedMapIterAllBackward(t, prollyMap, tuples2)
+			t.Run("iter range after deletes applied", func(t *testing.T) {
+				testIterRange(t, prollyMap, tuples2)
 			})
-			t.Run("iter value range", func(t *testing.T) {
-				testOrderedMapIterValueRange(t, prollyMap, tuples2)
+
+			prollyIndex, err := mutableIndex2.Map(context.Background())
+			require.NoError(t, err)
+			t.Run("iter prefix range", func(t *testing.T) {
+				testIterPrefixRange(t, prollyIndex, idxTuples2)
 			})
 		})
 	}
@@ -118,7 +128,7 @@ func makeMutableMap(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
 	require.NoError(t, err)
 
 	mut := MutableMap{
-		m: Map{
+		prolly: Map{
 			root:    root,
 			keyDesc: kd,
 			valDesc: vd,
@@ -135,29 +145,33 @@ func makeMutableMap(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
 	return mut, tuples
 }
 
-func makeMutableMapWithDeletes(t *testing.T, count int) (mut MutableMap, tuples, deletes [][2]val.Tuple) {
-	ctx := context.Background()
-	om, tuples := makeMutableMap(t, count)
-	mut = om.(MutableMap)
+func makeMutableSecondaryIndex(t *testing.T, count int) (orderedMap, [][2]val.Tuple) {
+	m, tuples := makeProllySecondaryIndex(t, count)
+	return newMutableMap(m.(Map)), tuples
+}
 
+func deleteFromMutableMap(mut MutableMap, tt [][2]val.Tuple) (MutableMap, [][2]val.Tuple, [][2]val.Tuple) {
+	count := len(tt)
 	testRand.Shuffle(count, func(i, j int) {
-		tuples[i], tuples[j] = tuples[j], tuples[i]
+		tt[i], tt[j] = tt[j], tt[i]
 	})
 
 	// delete 1/4 of tuples
-	deletes = tuples[:count/4]
+	deletes := tt[:count/4]
 
 	// re-sort the remaining tuples
-	tuples = tuples[count/4:]
-	desc := getKeyDesc(om)
-	sortTuplePairs(tuples, desc)
+	remaining := tt[count/4:]
+	desc := keyDescFromMap(mut)
+	sortTuplePairs(remaining, desc)
 
+	ctx := context.Background()
 	for _, kv := range deletes {
-		err := mut.Put(ctx, kv[0], nil)
-		require.NoError(t, err)
+		if err := mut.Delete(ctx, kv[0]); err != nil {
+			panic(err)
+		}
 	}
 
-	return mut, tuples, deletes
+	return mut, remaining, deletes
 }
 
 func testMutableMapGetAndHas(t *testing.T, mut MutableMap, tuples, deletes [][2]val.Tuple) {

go/store/prolly/mutable_map_write_test.go

@@ -483,7 +483,7 @@ func materializeMap(t *testing.T, mut MutableMap) Map {
 		if next == nil {
 			break
 		}
-		cmp := mut.m.compareKeys(prev, next)
+		cmp := mut.prolly.compareKeys(prev, next)
 		assert.True(t, cmp < 0)
 		prev = next
 	}

go/store/prolly/mutator.go

@@ -26,7 +26,7 @@ type mutationIter interface {
 	close() error
 }
 
-var _ mutationIter = memTupleCursor{}
+var _ mutationIter = &memRangeIter{}
 
 func materializeMutations(ctx context.Context, m Map, edits mutationIter) (Map, error) {
 	var err error

go/store/prolly/node_cursor.go

@@ -24,6 +24,8 @@ package prolly
 import (
 	"context"
 	"sort"
+
+	"github.com/dolthub/dolt/go/store/val"
 )
 
 const (
@@ -58,7 +60,7 @@ func newCursorAtStart(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCur
 	return
 }
 
-func newCursorAtEnd(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCursor, err error) {
+func newCursorPastEnd(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCursor, err error) {
 	cur = &nodeCursor{nd: nd, nrw: nrw}
 	cur.skipToNodeEnd()
 
@@ -73,9 +75,23 @@ func newCursorAtEnd(ctx context.Context, nrw NodeStore, nd Node) (cur *nodeCurso
 		cur = &nodeCursor{nd: nd, parent: parent, nrw: nrw}
 		cur.skipToNodeEnd()
 	}
+
+	// advance |cur| past the end
+	ok, err := cur.advance(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		panic("expected |ok| to be  false")
+	}
+
 	return
 }
 
+func newCursorAtTuple(ctx context.Context, nrw NodeStore, nd Node, tup val.Tuple, search searchFn) (cur *nodeCursor, err error) {
+	return newCursorAtItem(ctx, nrw, nd, nodeItem(tup), search)
+}
+
 func newCursorAtItem(ctx context.Context, nrw NodeStore, nd Node, item nodeItem, search searchFn) (cur *nodeCursor, err error) {
 	cur = &nodeCursor{nd: nd, nrw: nrw}
 
@@ -122,36 +138,6 @@ func newLeafCursorAtItem(ctx context.Context, nrw NodeStore, nd Node, item nodeI
 	return cur, nil
 }
 
-func newCursorAtIndex(ctx context.Context, nrw NodeStore, nd Node, idx uint64) (cur nodeCursor, err error) {
-	cur = nodeCursor{nd: nd, nrw: nrw}
-
-	distance := idx
-	for !cur.isLeaf() {
-
-		mv := metaValue(cur.currentPair().value())
-		for distance >= mv.GetCumulativeCount() {
-
-			if _, err = cur.advance(ctx); err != nil {
-				return nodeCursor{}, err
-			}
-
-			distance -= mv.GetCumulativeCount()
-			mv = metaValue(cur.currentPair().value())
-		}
-
-		nd, err = fetchChild(ctx, nrw, mv)
-		if err != nil {
-			return nodeCursor{}, err
-		}
-
-		parent := cur
-		cur = nodeCursor{nd: nd, parent: &parent, nrw: nrw}
-	}
-
-	cur.idx = int(distance)
-	return
-}
-
 func (cur *nodeCursor) valid() bool {
 	if cur.nd == nil {
 		return false

go/store/prolly/range.go

@@ -32,7 +32,6 @@ type RangeCut struct {
 type Range struct {
 	Start, Stop RangeCut
 	KeyDesc     val.TupleDesc
-	Reverse     bool
 }
 
 func (r Range) insideStart(key val.Tuple) bool {
@@ -44,9 +43,6 @@ func (r Range) insideStart(key val.Tuple) bool {
 	if cmp == 0 {
 		return r.Start.Inclusive
 	}
-	if r.Reverse {
-		cmp = -cmp
-	}
 	return cmp > 0
 }
 
@@ -59,213 +55,86 @@ func (r Range) insideStop(key val.Tuple) bool {
 	if cmp == 0 {
 		return r.Stop.Inclusive
 	}
-	if r.Reverse {
-		cmp = -cmp
-	}
 	return cmp < 0
 }
 
-// GreaterRange defines a Range of Tuples greater than |start|.
-func GreaterRange(start val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Key:       start,
-			Inclusive: false,
-		},
-		Stop: RangeCut{
-			Unbound: true,
-		},
-		KeyDesc: desc,
-		Reverse: false,
+func NewMapRangeIter(memory, prolly rangeIter, rng Range) MapRangeIter {
+	if memory == nil {
+		memory = emptyIter{}
 	}
-}
-
-// GreaterOrEqualRange defines a Range of Tuples greater than or equal to |start|.
-func GreaterOrEqualRange(start val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Key:       start,
-			Inclusive: true,
-		},
-		Stop: RangeCut{
-			Unbound: true,
-		},
-		KeyDesc: desc,
-		Reverse: false,
+	if prolly == nil {
+		prolly = emptyIter{}
 	}
-}
 
-// LesserRange defines a Range of Tuples less than |stop|.
-func LesserRange(stop val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Unbound: true,
-		},
-		Stop: RangeCut{
-			Key:       stop,
-			Inclusive: false,
-		},
-		KeyDesc: desc,
-		Reverse: false,
-	}
-}
-
-// LesserOrEqualRange defines a Range of Tuples less than or equal to |stop|.
-func LesserOrEqualRange(stop val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Unbound: true,
-		},
-		Stop: RangeCut{
-			Key:       stop,
-			Inclusive: true,
-		},
-		KeyDesc: desc,
-		Reverse: false,
-	}
-}
-
-// todo(andy): reverse ranges for GT, GTE, LT, and LTE?
-
-// OpenRange defines a non-inclusive Range of Tuples from |start| to |stop|.
-func OpenRange(start, stop val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Key:       start,
-			Inclusive: false,
-		},
-		Stop: RangeCut{
-			Key:       stop,
-			Inclusive: false,
-		},
-		KeyDesc: desc,
-		Reverse: desc.Compare(start, stop) > 0,
-	}
-}
-
-// OpenStartRange defines a half-open Range of Tuples from |start| to |stop|.
-func OpenStartRange(start, stop val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Key:       start,
-			Inclusive: false,
-		},
-		Stop: RangeCut{
-			Key:       stop,
-			Inclusive: true,
-		},
-		KeyDesc: desc,
-		Reverse: desc.Compare(start, stop) > 0,
-	}
-}
-
-// OpenStopRange defines a half-open Range of Tuples from |start| to |stop|.
-func OpenStopRange(start, stop val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Key:       start,
-			Inclusive: true,
-		},
-		Stop: RangeCut{
-			Key:       stop,
-			Inclusive: false,
-		},
-		KeyDesc: desc,
-		Reverse: desc.Compare(start, stop) > 0,
-	}
-}
-
-// ClosedRange defines an inclusive Range of Tuples from |start| to |stop|.
-func ClosedRange(start, stop val.Tuple, desc val.TupleDesc) Range {
-	return Range{
-		Start: RangeCut{
-			Key:       start,
-			Inclusive: true,
-		},
-		Stop: RangeCut{
-			Key:       stop,
-			Inclusive: true,
-		},
-		KeyDesc: desc,
-		Reverse: desc.Compare(start, stop) > 0,
+	return MapRangeIter{
+		memory: memory,
+		prolly: prolly,
+		rng:    rng,
 	}
 }
 
 // MapRangeIter iterates over a Range of Tuples.
 type MapRangeIter struct {
-	memCur, proCur tupleCursor
-	rng            Range
+	memory rangeIter
+	prolly rangeIter
+	rng    Range
 }
 
-func NewMapRangeIter(ctx context.Context, memCur, proCur tupleCursor, rng Range) (MapRangeIter, error) {
-	mri := MapRangeIter{
-		memCur: memCur,
-		proCur: proCur,
-		rng:    rng,
-	}
-
-	err := startInRange(ctx, mri)
-	if err != nil {
-		return MapRangeIter{}, err
-	}
-
-	return mri, nil
-}
-
-type tupleCursor interface {
+type rangeIter interface {
+	iterate(ctx context.Context) error
 	current() (key, value val.Tuple)
-	advance(ctx context.Context) error
-	retreat(ctx context.Context) error
 }
 
 // Next returns the next pair of Tuples in the Range, or io.EOF if the iter is done.
 func (it MapRangeIter) Next(ctx context.Context) (key, value val.Tuple, err error) {
-	for value == nil {
-		key, value = it.current()
+	for {
+		mk, mv := it.memory.current()
+		pk, pv := it.prolly.current()
 
-		if key == nil {
-			// |key| == nil is exhausted |iter|
+		if mk == nil && pk == nil {
+			// range is exhausted
 			return nil, nil, io.EOF
 		}
 
-		// |value| == nil is a pending delete
+		cmp := it.compareKeys(pk, mk)
+		switch {
+		case cmp < 0:
+			key, value = pk, pv
+			if err = it.prolly.iterate(ctx); err != nil {
+				return nil, nil, err
+			}
 
-		if err = it.progress(ctx); err != nil {
-			return nil, nil, err
+		case cmp > 0:
+			key, value = mk, mv
+			if err = it.memory.iterate(ctx); err != nil {
+				return nil, nil, err
+			}
+
+		case cmp == 0:
+			// |it.memory| wins ties
+			key, value = mk, mv
+			if err = it.memory.iterate(ctx); err != nil {
+				return nil, nil, err
+			}
+			if err = it.prolly.iterate(ctx); err != nil {
+				return nil, nil, err
+			}
 		}
+
+		if key != nil && value == nil {
+			continue // pending delete
+		}
+
+		return key, value, nil
 	}
-
-	return key, value, nil
-}
-
-func (it MapRangeIter) current() (key, value val.Tuple) {
-	memKey, proKey := it.currentKeys()
-
-	if memKey == nil && proKey == nil {
-		// cursors exhausted
-		return nil, nil
-	}
-
-	if it.compareKeys(memKey, proKey) > 0 {
-		key, value = it.proCur.current()
-	} else {
-		// |memCur| wins ties
-		key, value = it.memCur.current()
-	}
-
-	if !it.rng.insideStop(key) {
-		return nil, nil
-	}
-
-	return
 }
 
 func (it MapRangeIter) currentKeys() (memKey, proKey val.Tuple) {
-	if it.memCur != nil {
-		memKey, _ = it.memCur.current()
+	if it.memory != nil {
+		memKey, _ = it.memory.current()
 	}
-	if it.proCur != nil {
-		proKey, _ = it.proCur.current()
+	if it.prolly != nil {
+		proKey, _ = it.prolly.current()
 	}
 	return
 }
@@ -277,68 +146,133 @@ func (it MapRangeIter) compareKeys(memKey, proKey val.Tuple) int {
 	if proKey == nil {
 		return -1
 	}
-
-	cmp := it.rng.KeyDesc.Compare(memKey, proKey)
-	if it.rng.Reverse {
-		cmp = -cmp
-	}
-	return cmp
+	return it.rng.KeyDesc.Compare(memKey, proKey)
 }
 
-func (it MapRangeIter) progress(ctx context.Context) (err error) {
-	memKey, proKey := it.currentKeys()
+type emptyIter struct{}
 
-	if memKey == nil && proKey == nil {
-		return nil // can't progress
-	}
+var _ rangeIter = emptyIter{}
 
-	cmp := it.compareKeys(memKey, proKey)
-
-	if cmp >= 0 {
-		if err = it.moveCursor(ctx, it.proCur); err != nil {
-			return err
-		}
-	}
-	if cmp <= 0 {
-		if err = it.moveCursor(ctx, it.memCur); err != nil {
-			return err
-		}
-	}
+func (e emptyIter) iterate(ctx context.Context) (err error) {
 	return
 }
 
-func (it MapRangeIter) moveCursor(ctx context.Context, cur tupleCursor) error {
-	if it.rng.Reverse {
-		return cur.retreat(ctx)
-	} else {
-		return cur.advance(ctx)
+func (e emptyIter) current() (key, value val.Tuple) {
+	return
+}
+
+// GreaterRange defines a Range of Tuples greater than |lo|.
+func GreaterRange(start val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Key:       start,
+			Inclusive: false,
+		},
+		Stop: RangeCut{
+			Unbound: true,
+		},
+		KeyDesc: desc,
 	}
 }
 
-func startInRange(ctx context.Context, iter MapRangeIter) error {
-	if iter.rng.Start.Unbound {
-		return nil
+// GreaterOrEqualRange defines a Range of Tuples greater than or equal to |lo|.
+func GreaterOrEqualRange(start val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Key:       start,
+			Inclusive: true,
+		},
+		Stop: RangeCut{
+			Unbound: true,
+		},
+		KeyDesc: desc,
+	}
+}
+
+// LesserRange defines a Range of Tuples less than |last|.
+func LesserRange(stop val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Unbound: true,
+		},
+		Stop: RangeCut{
+			Key:       stop,
+			Inclusive: false,
+		},
+		KeyDesc: desc,
+	}
+}
+
+// LesserOrEqualRange defines a Range of Tuples less than or equal to |last|.
+func LesserOrEqualRange(stop val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Unbound: true,
+		},
+		Stop: RangeCut{
+			Key:       stop,
+			Inclusive: true,
+		},
+		KeyDesc: desc,
+	}
+}
+
+// OpenRange defines a non-inclusive Range of Tuples from |lo| to |last|.
+func OpenRange(start, stop val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Key:       start,
+			Inclusive: false,
+		},
+		Stop: RangeCut{
+			Key:       stop,
+			Inclusive: false,
+		},
+		KeyDesc: desc,
+	}
+}
+
+// OpenStartRange defines a half-open Range of Tuples from |lo| to |last|.
+func OpenStartRange(start, stop val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Key:       start,
+			Inclusive: false,
+		},
+		Stop: RangeCut{
+			Key:       stop,
+			Inclusive: true,
+		},
+		KeyDesc: desc,
+	}
+}
+
+// OpenStopRange defines a half-open Range of Tuples from |lo| to |last|.
+func OpenStopRange(start, stop val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Key:       start,
+			Inclusive: true,
+		},
+		Stop: RangeCut{
+			Key:       stop,
+			Inclusive: false,
+		},
+		KeyDesc: desc,
+	}
+}
+
+// ClosedRange defines an inclusive Range of Tuples from |lo| to |last|.
+func ClosedRange(start, stop val.Tuple, desc val.TupleDesc) Range {
+	return Range{
+		Start: RangeCut{
+			Key:       start,
+			Inclusive: true,
+		},
+		Stop: RangeCut{
+			Key:       stop,
+			Inclusive: true,
+		},
+		KeyDesc: desc,
 	}
-
-	key, value := iter.current()
-	if key == nil {
-		// |key| == nil is exhausted iter
-		return nil
-	}
-
-	// |value| == nil is a pending delete
-
-	for !iter.rng.insideStart(key) || value == nil {
-		if err := iter.progress(ctx); err != nil {
-			return err
-		}
-
-		key, value = iter.current()
-		if key == nil {
-			// |key| == nil is exhausted iter
-			return nil
-		}
-	}
-
-	return nil
 }

go/store/prolly/range_iter_test.go

@@ -0,0 +1,443 @@
+// Copyright 2021 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prolly
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/dolthub/dolt/go/store/val"
+)
+
+type rangeTest struct {
+	name      string
+	testRange Range
+	expCount  int
+}
+
+func testIterRange(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
+	ctx := context.Background()
+	desc := keyDescFromMap(om)
+
+	for i := 0; i < 100; i++ {
+
+		cnt := len(tuples)
+		a, z := testRand.Intn(cnt), testRand.Intn(cnt)
+		if a > z {
+			a, z = z, a
+		}
+		start, stop := tuples[a][0], tuples[z][0]
+
+		tests := []rangeTest{
+			// two-sided ranges
+			{
+				name:      "OpenRange",
+				testRange: OpenRange(start, stop, desc),
+				expCount:  nonNegative((z - a) - 1),
+			},
+			{
+				name:      "OpenStartRange",
+				testRange: OpenStartRange(start, stop, desc),
+				expCount:  z - a,
+			},
+			{
+				name:      "OpenStopRange",
+				testRange: OpenStopRange(start, stop, desc),
+				expCount:  z - a,
+			},
+			{
+				name:      "ClosedRange",
+				testRange: ClosedRange(start, stop, desc),
+				expCount:  (z - a) + 1,
+			},
+
+			// one-sided ranges
+			{
+				name:      "GreaterRange",
+				testRange: GreaterRange(start, desc),
+				expCount:  nonNegative(cnt - a - 1),
+			},
+			{
+				name:      "GreaterOrEqualRange",
+				testRange: GreaterOrEqualRange(start, desc),
+				expCount:  cnt - a,
+			},
+			{
+				name:      "LesserRange",
+				testRange: LesserRange(stop, desc),
+				expCount:  z,
+			},
+			{
+				name:      "LesserOrEqualRange",
+				testRange: LesserOrEqualRange(stop, desc),
+				expCount:  z + 1,
+			},
+		}
+
+		for _, test := range tests {
+			//s := fmt.Sprintf(test.testRange.format())
+			//fmt.Println(s)
+
+			iter, err := om.IterRange(ctx, test.testRange)
+			require.NoError(t, err)
+
+			key, _, err := iter.Next(ctx)
+			actCount := 0
+			for err != io.EOF {
+				actCount++
+				prev := key
+				key, _, err = iter.Next(ctx)
+
+				if key != nil {
+					assert.True(t, desc.Compare(prev, key) < 0)
+				}
+			}
+			assert.Equal(t, io.EOF, err)
+			assert.Equal(t, test.expCount, actCount)
+			//fmt.Printf("a: %d \t z: %d cnt: %d", a, z, cnt)
+		}
+	}
+}
+
+func nonNegative(x int) int {
+	if x < 0 {
+		x = 0
+	}
+	return x
+}
+
+type prefixRangeTest struct {
+	name      string
+	testRange Range
+}
+
+func testIterPrefixRange(t *testing.T, om orderedMap, tuples [][2]val.Tuple) {
+	ctx := context.Background()
+	prefixDesc := getDescPrefix(keyDescFromMap(om), 1)
+
+	for i := 0; i < 100; i++ {
+
+		cnt := len(tuples)
+		a, z := testRand.Intn(cnt), testRand.Intn(cnt)
+		if a > z {
+			a, z = z, a
+		}
+		start := getKeyPrefix(tuples[a][0], prefixDesc)
+		stop := getKeyPrefix(tuples[z][0], prefixDesc)
+
+		tests := []prefixRangeTest{
+			// two-sided ranges
+			{
+				name:      "OpenRange",
+				testRange: OpenRange(start, stop, prefixDesc),
+			},
+			{
+				name:      "OpenStartRange",
+				testRange: OpenStartRange(start, stop, prefixDesc),
+			},
+			{
+				name:      "OpenStopRange",
+				testRange: OpenStopRange(start, stop, prefixDesc),
+			},
+			{
+				name:      "ClosedRange",
+				testRange: ClosedRange(start, stop, prefixDesc),
+			},
+
+			// one-sided ranges
+			{
+				name:      "GreaterRange",
+				testRange: GreaterRange(start, prefixDesc),
+			},
+			{
+				name:      "GreaterOrEqualRange",
+				testRange: GreaterOrEqualRange(start, prefixDesc),
+			},
+			{
+				name:      "LesserRange",
+				testRange: LesserRange(stop, prefixDesc),
+			},
+			{
+				name:      "LesserOrEqualRange",
+				testRange: LesserOrEqualRange(stop, prefixDesc),
+			},
+		}
+
+		for _, test := range tests {
+			//s := fmt.Sprintf(test.testRange.format())
+			//fmt.Println(s)
+
+			iter, err := om.IterRange(ctx, test.testRange)
+			require.NoError(t, err)
+
+			key, _, err := iter.Next(ctx)
+			actCount := 0
+			for err != io.EOF {
+				actCount++
+				prev := key
+				key, _, err = iter.Next(ctx)
+
+				if key != nil {
+					assert.True(t, prefixDesc.Compare(prev, key) < 0)
+				}
+			}
+			assert.Equal(t, io.EOF, err)
+
+			expCount := getExpectedRangeSize(test.testRange, tuples)
+			assert.Equal(t, expCount, actCount)
+		}
+	}
+}
+
+func getDescPrefix(desc val.TupleDesc, sz int) val.TupleDesc {
+	return val.NewTupleDescriptor(desc.Types[:sz]...)
+}
+
+func getKeyPrefix(key val.Tuple, desc val.TupleDesc) (partial val.Tuple) {
+	tb := val.NewTupleBuilder(desc)
+	for i := range desc.Types {
+		tb.PutRaw(i, key.GetField(i))
+	}
+	return tb.Build(sharedPool)
+}
+
+// computes expected range on full tuples set
+func getExpectedRangeSize(rng Range, tuples [][2]val.Tuple) (sz int) {
+	for i := range tuples {
+		k := tuples[i][0]
+		if rng.insideStart(k) && rng.insideStop(k) {
+			sz++
+		}
+	}
+	return
+}
+
+func TestMapIterRange(t *testing.T) {
+	ctx := context.Background()
+	ns := newTestNodeStore()
+	kd := val.NewTupleDescriptor(
+		val.Type{Enc: val.Int32Enc},
+		val.Type{Enc: val.Int32Enc},
+	)
+	vd := val.NewTupleDescriptor()
+
+	tuples := []val.Tuple{
+		intTuple(1, 1), intTuple(),
+		intTuple(1, 2), intTuple(),
+		intTuple(1, 3), intTuple(),
+		intTuple(2, 1), intTuple(),
+		intTuple(2, 2), intTuple(),
+		intTuple(2, 3), intTuple(),
+		intTuple(3, 1), intTuple(),
+		intTuple(3, 2), intTuple(),
+		intTuple(3, 3), intTuple(),
+		intTuple(4, 1), intTuple(),
+		intTuple(4, 2), intTuple(),
+		intTuple(4, 3), intTuple(),
+	}
+	require.Equal(t, 24, len(tuples))
+
+	index, err := NewMapFromTuples(ctx, ns, kd, vd, tuples...)
+	require.NoError(t, err)
+	require.Equal(t, uint64(12), index.Count())
+
+	partialDesc := val.NewTupleDescriptor(
+		val.Type{Enc: val.Int32Enc},
+	)
+	fullDesc := val.NewTupleDescriptor(
+		val.Type{Enc: val.Int32Enc},
+		val.Type{Enc: val.Int32Enc},
+	)
+
+	tests := []mapRangeTest{
+		// partial-key range scan
+		{
+			name: "range [1:4]",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1),
+					Inclusive: true,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4),
+					Inclusive: true,
+				},
+				KeyDesc: partialDesc,
+			},
+			exp: tuples[:],
+		},
+		{
+			name: "range (1:4]",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1),
+					Inclusive: false,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4),
+					Inclusive: true,
+				},
+				KeyDesc: partialDesc,
+			},
+			exp: tuples[6:],
+		},
+		{
+			name: "range [1:4)",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1),
+					Inclusive: true,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4),
+					Inclusive: false,
+				},
+				KeyDesc: partialDesc,
+			},
+			exp: tuples[:18],
+		},
+		{
+			name: "range (1:4)",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1),
+					Inclusive: false,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4),
+					Inclusive: false,
+				},
+				KeyDesc: partialDesc,
+			},
+			exp: tuples[6:18],
+		},
+
+		// full-key range scan
+		{
+			name: "range [1,2:4,2]",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1, 2),
+					Inclusive: true,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4, 2),
+					Inclusive: true,
+				},
+				KeyDesc: fullDesc,
+			},
+			exp: tuples[2:22],
+		},
+		{
+			name: "range (1,2:4,2]",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1, 2),
+					Inclusive: false,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4, 2),
+					Inclusive: true,
+				},
+				KeyDesc: fullDesc,
+			},
+			exp: tuples[4:22],
+		},
+		{
+			name: "range [1,2:4,2)",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1, 2),
+					Inclusive: true,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4, 2),
+					Inclusive: false,
+				},
+				KeyDesc: fullDesc,
+			},
+			exp: tuples[2:20],
+		},
+		{
+			name: "range (1,2:4,2)",
+			rng: Range{
+				Start: RangeCut{
+					Key:       intTuple(1, 2),
+					Inclusive: false,
+				},
+				Stop: RangeCut{
+					Key:       intTuple(4, 2),
+					Inclusive: false,
+				},
+				KeyDesc: fullDesc,
+			},
+			exp: tuples[4:20], // 🌲
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testMapRange(t, index, test)
+		})
+	}
+}
+
+type mapRangeTest struct {
+	name string
+	rng  Range
+	exp  []val.Tuple
+}
+
+func testMapRange(t *testing.T, idx Map, test mapRangeTest) {
+	ctx := context.Background()
+
+	iter, err := idx.IterRange(ctx, test.rng)
+	require.NoError(t, err)
+
+	var k, v val.Tuple
+	act := make([]val.Tuple, 0, len(test.exp))
+	for {
+		k, v, err = iter.Next(ctx)
+		if err == io.EOF {
+			break
+		}
+		assert.NoError(t, err)
+		act = append(act, k, v)
+	}
+	assert.Error(t, io.EOF, err)
+
+	require.Equal(t, len(test.exp), len(act))
+	for i := range test.exp {
+		assert.Equal(t, test.exp[i], act[i])
+	}
+}
+
+func intTuple(ints ...int32) val.Tuple {
+	types := make([]val.Type, len(ints))
+	for i := range types {
+		types[i] = val.Type{Enc: val.Int32Enc}
+	}
+
+	desc := val.NewTupleDescriptor(types...)
+	tb := val.NewTupleBuilder(desc)
+	for i := range ints {
+		tb.PutInt32(i, ints[i])
+	}
+	return tb.Build(sharedPool)
+}

go/store/prolly/utils_test.go

@@ -0,0 +1,214 @@
+// Copyright 2021 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prolly
+
+import (
+	"context"
+	"math"
+	"math/rand"
+	"sort"
+	"strings"
+
+	"github.com/dolthub/dolt/go/store/val"
+)
+
+// orderedMap is a utility type that allows us to create a common test
+// harness for Map, memoryMap, and MutableMap.
+type orderedMap interface {
+	Get(ctx context.Context, key val.Tuple, cb KeyValueFn) (err error)
+	IterAll(ctx context.Context) (MapRangeIter, error)
+	IterRange(ctx context.Context, rng Range) (MapRangeIter, error)
+}
+
+var _ orderedMap = Map{}
+var _ orderedMap = MutableMap{}
+var _ orderedMap = memoryMap{}
+
+func keyDescFromMap(om orderedMap) val.TupleDesc {
+	switch m := om.(type) {
+	case Map:
+		return m.keyDesc
+	case MutableMap:
+		return m.prolly.keyDesc
+	case memoryMap:
+		return m.keyDesc
+	default:
+		panic("unknown ordered map")
+	}
+}
+
+func randomTuplePairs(count int, keyDesc, valDesc val.TupleDesc) (items [][2]val.Tuple) {
+	keyBuilder := val.NewTupleBuilder(keyDesc)
+	valBuilder := val.NewTupleBuilder(valDesc)
+
+	items = make([][2]val.Tuple, count)
+	for i := range items {
+		items[i][0] = randomTuple(keyBuilder)
+		items[i][1] = randomTuple(valBuilder)
+	}
+
+	sortTuplePairs(items, keyDesc)
+
+	for i := range items {
+		if i == 0 {
+			continue
+		}
+		if keyDesc.Compare(items[i][0], items[i-1][0]) == 0 {
+			panic("duplicate key, unlucky!")
+		}
+	}
+	return
+}
+
+func randomCompositeTuplePairs(count int, keyDesc, valDesc val.TupleDesc) (items [][2]val.Tuple) {
+	// preconditions
+	if count%5 != 0 {
+		panic("expected count divisible by 5")
+	}
+	if len(keyDesc.Types) < 2 {
+		panic("expected composite key")
+	}
+
+	tt := randomTuplePairs(count, keyDesc, valDesc)
+
+	tuples := make([][2]val.Tuple, len(tt)*3)
+	for i := range tuples {
+		j := i % len(tt)
+		tuples[i] = tt[j]
+	}
+
+	// permute the second column
+	swap := make([]byte, len(tuples[0][0].GetField(1)))
+	rand.Shuffle(len(tuples), func(i, j int) {
+		f1 := tuples[i][0].GetField(1)
+		f2 := tuples[i][0].GetField(1)
+		copy(swap, f1)
+		copy(f1, f2)
+		copy(f2, swap)
+	})
+
+	sortTuplePairs(tuples, keyDesc)
+
+	tuples = deduplicateTuples(keyDesc, tuples)
+
+	return tuples[:count]
+}
+
+// assumes a sorted list
+func deduplicateTuples(desc val.TupleDesc, tups [][2]val.Tuple) (uniq [][2]val.Tuple) {
+	uniq = make([][2]val.Tuple, 1, len(tups))
+	uniq[0] = tups[0]
+
+	for i := 1; i < len(tups); i++ {
+		cmp := desc.Compare(tups[i-1][0], tups[i][0])
+		if cmp < 0 {
+			uniq = append(uniq, tups[i])
+		}
+	}
+	return
+}
+
+func randomTuple(tb *val.TupleBuilder) (tup val.Tuple) {
+	for i, typ := range tb.Desc.Types {
+		randomField(tb, i, typ)
+	}
+	return tb.Build(sharedPool)
+}
+
+func cloneRandomTuples(items [][2]val.Tuple) (clone [][2]val.Tuple) {
+	clone = make([][2]val.Tuple, len(items))
+	for i := range clone {
+		clone[i] = items[i]
+	}
+	return
+}
+
+func sortTuplePairs(items [][2]val.Tuple, keyDesc val.TupleDesc) {
+	sort.Slice(items, func(i, j int) bool {
+		return keyDesc.Compare(items[i][0], items[j][0]) < 0
+	})
+}
+
+func shuffleTuplePairs(items [][2]val.Tuple) {
+	testRand.Shuffle(len(items), func(i, j int) {
+		items[i], items[j] = items[j], items[i]
+	})
+}
+
+func randomField(tb *val.TupleBuilder, idx int, typ val.Type) {
+	// todo(andy): add NULLs
+
+	neg := -1
+	if testRand.Int()%2 == 1 {
+		neg = 1
+	}
+
+	switch typ.Enc {
+	case val.Int8Enc:
+		v := int8(testRand.Intn(math.MaxInt8) * neg)
+		tb.PutInt8(idx, v)
+	case val.Uint8Enc:
+		v := uint8(testRand.Intn(math.MaxUint8))
+		tb.PutUint8(idx, v)
+	case val.Int16Enc:
+		v := int16(testRand.Intn(math.MaxInt16) * neg)
+		tb.PutInt16(idx, v)
+	case val.Uint16Enc:
+		v := uint16(testRand.Intn(math.MaxUint16))
+		tb.PutUint16(idx, v)
+	case val.Int32Enc:
+		v := int32(testRand.Intn(math.MaxInt32) * neg)
+		tb.PutInt32(idx, v)
+	case val.Uint32Enc:
+		v := uint32(testRand.Intn(math.MaxUint32))
+		tb.PutUint32(idx, v)
+	case val.Int64Enc:
+		v := int64(testRand.Intn(math.MaxInt64) * neg)
+		tb.PutInt64(idx, v)
+	case val.Uint64Enc:
+		v := uint64(testRand.Uint64())
+		tb.PutUint64(idx, v)
+	case val.Float32Enc:
+		tb.PutFloat32(idx, testRand.Float32())
+	case val.Float64Enc:
+		tb.PutFloat64(idx, testRand.Float64())
+	case val.StringEnc:
+		buf := make([]byte, (testRand.Int63()%40)+10)
+		testRand.Read(buf)
+		tb.PutString(idx, string(buf))
+	case val.BytesEnc:
+		buf := make([]byte, (testRand.Int63()%40)+10)
+		testRand.Read(buf)
+		tb.PutBytes(idx, buf)
+	default:
+		panic("unknown encoding")
+	}
+}
+
+func fmtTupleList(tuples [][2]val.Tuple, kd, vd val.TupleDesc) string {
+	var sb strings.Builder
+	sb.WriteString("{ ")
+	for _, kv := range tuples {
+		if kv[0] == nil || kv[1] == nil {
+			break
+		}
+		sb.WriteString(kd.Format(kv[0]))
+		sb.WriteString(": ")
+		sb.WriteString(vd.Format(kv[1]))
+		sb.WriteString(", ")
+	}
+	sb.WriteString("}")
+	return sb.String()
+}

go/store/skip/list.go

@@ -176,9 +176,13 @@ func (it *ListIter) Retreat() {
 	return
 }
 
-func (l *List) IterAt(key []byte) (it *ListIter) {
+func (l *List) GetIterAt(key []byte) (it *ListIter) {
+	return l.GetIterAtWithFn(key, l.cmp)
+}
+
+func (l *List) GetIterAtWithFn(key []byte, cmp ValueCmp) (it *ListIter) {
 	it = &ListIter{
-		curr: l.seek(key),
+		curr: l.seekWithFn(key, cmp),
 		list: l,
 	}
 
@@ -206,11 +210,15 @@ func (l *List) IterAtEnd() *ListIter {
 }
 
 // seek returns the skipNode with the smallest key >= |key|.
-func (l *List) seek(key []byte) (node skipNode) {
+func (l *List) seek(key []byte) skipNode {
+	return l.seekWithFn(key, l.cmp)
+}
+
+func (l *List) seekWithFn(key []byte, cmp ValueCmp) (node skipNode) {
 	ptr := l.head
 	for h := int64(highest); h >= 0; h-- {
 		node = l.getNode(ptr[h])
-		for l.compareKeys(key, node.key) > 0 {
+		for l.compareKeysWithFn(key, node.key, cmp) > 0 {
 			ptr = node.next
 			node = l.getNode(ptr[h])
 		}
@@ -240,11 +248,14 @@ func (l *List) compare(left, right skipNode) int {
 }
 
 func (l *List) compareKeys(left, right []byte) int {
+	return l.compareKeysWithFn(left, right, l.cmp)
+}
+
+func (l *List) compareKeysWithFn(left, right []byte, cmp ValueCmp) int {
 	if right == nil {
-		// |right| is sentinel key
-		return -1
+		return -1 // |right| is sentinel key
 	}
-	return l.cmp(left, right)
+	return cmp(left, right)
 }
 
 func (l *List) makeNode(key, val []byte) (n skipNode) {

go/store/skip/list_test.go

@@ -168,7 +168,7 @@ func testSkipListIterBackward(t *testing.T, list *List, vals ...[]byte) {
 }
 
 func validateIterForwardFrom(t *testing.T, l *List, key []byte) (count int) {
-	iter := l.IterAt(key)
+	iter := l.GetIterAt(key)
 	k, _ := iter.Current()
 	for k != nil {
 		count++
@@ -181,7 +181,7 @@ func validateIterForwardFrom(t *testing.T, l *List, key []byte) (count int) {
 }
 
 func validateIterBackwardFrom(t *testing.T, l *List, key []byte) (count int) {
-	iter := l.IterAt(key)
+	iter := l.GetIterAt(key)
 	k, _ := iter.Current()
 	for k != nil {
 		count++

go/store/types/ref.go

@@ -24,7 +24,9 @@ package types
 import (
 	"bytes"
 	"context"
+	"fmt"
 
+	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/hash"
 )
 
@@ -220,3 +222,19 @@ func (r Ref) String() string {
 func (r Ref) HumanReadableString() string {
 	panic("unreachable")
 }
+
+// Returns a function that can be used to walk the hash and height of all the
+// Refs of a given Chunk.  This function is meant to decouple callers from the
+// types package itself, and so the callback itself does not take |types.Ref|
+// values.
+func WalkRefsForChunkStore(cs chunks.ChunkStore) (func(chunks.Chunk, func(h hash.Hash, height uint64) error) error, error) {
+	nbf, err := GetFormatForVersionString(cs.Version())
+	if err != nil {
+		return nil, fmt.Errorf("could not find binary format corresponding to %s. try upgrading dolt.", cs.Version())
+	}
+	return func(c chunks.Chunk, cb func(h hash.Hash, height uint64) error) error {
+		return WalkRefs(c, nbf, func(r Ref) error {
+			return cb(r.TargetHash(), r.Height())
+		})
+	}, nil
+}

go/store/val/tuple_builder.go

@@ -218,6 +218,19 @@ func (tb *TupleBuilder) PutJSON(i int, v interface{}) {
 	tb.pos += sz
 }
 
+// PutRaw writes a []byte to the ith field of the Tuple being built.
+func (tb *TupleBuilder) PutRaw(i int, buf []byte) {
+	if buf == nil {
+		// todo(andy): does it make senes to
+		//  allow/expect nulls here?
+		return
+	}
+	sz := ByteSize(len(buf))
+	tb.fields[i] = tb.buf[tb.pos : tb.pos+sz]
+	writeBytes(tb.fields[i], buf, tb.Desc.Types[i].Coll)
+	tb.pos += sz
+}
+
 // PutField writes an interface{} to the ith field of the Tuple being built.
 func (tb *TupleBuilder) PutField(i int, v interface{}) {
 	if v == nil {

go/store/val/tuple_descriptor.go

@@ -363,8 +363,12 @@ func (td TupleDesc) expectEncoding(i int, encodings ...Encoding) {
 
 // Format prints a Tuple as a string.
 func (td TupleDesc) Format(tup Tuple) string {
+	if tup == nil || tup.Count() == 0 {
+		return "( )"
+	}
+
 	var sb strings.Builder
-	sb.WriteString("[ ")
+	sb.WriteString("( ")
 
 	seenOne := false
 	for i, typ := range td.Types {
@@ -420,6 +424,6 @@ func (td TupleDesc) Format(tup Tuple) string {
 			sb.Write(tup.GetField(i))
 		}
 	}
-	sb.WriteString(" ]")
+	sb.WriteString(" )")
 	return sb.String()
 }

go/utils/copyrightshdrs/main.go

@@ -146,10 +146,8 @@ var CopiedNomsFiles []CopiedNomsFile = []CopiedNomsFile{
 	{Path: "store/datas/database_test.go", NomsPath: "go/datas/database_test.go", HadCopyrightNotice: true},
 	{Path: "store/datas/dataset.go", NomsPath: "go/datas/dataset.go", HadCopyrightNotice: true},
 	{Path: "store/datas/dataset_test.go", NomsPath: "go/datas/dataset_test.go", HadCopyrightNotice: true},
-	{Path: "store/datas/pull.go", NomsPath: "go/datas/pull.go", HadCopyrightNotice: true},
-	{Path: "store/datas/pull_test.go", NomsPath: "go/datas/pull_test.go", HadCopyrightNotice: true},
-	{Path: "store/datas/serialize_hashes.go", NomsPath: "go/datas/serialize_hashes.go", HadCopyrightNotice: true},
-	{Path: "store/datas/serialize_hashes_test.go", NomsPath: "go/datas/serialize_hashes_test.go", HadCopyrightNotice: true},
+	{Path: "store/datas/pull/pull.go", NomsPath: "go/datas/pull.go", HadCopyrightNotice: true},
+	{Path: "store/datas/pull/pull_test.go", NomsPath: "go/datas/pull_test.go", HadCopyrightNotice: true},
 	{Path: "store/diff/apply_patch.go", NomsPath: "go/diff/apply_patch.go", HadCopyrightNotice: true},
 	{Path: "store/diff/apply_patch_test.go", NomsPath: "go/diff/apply_patch_test.go", HadCopyrightNotice: true},
 	{Path: "store/diff/diff.go", NomsPath: "go/diff/diff.go", HadCopyrightNotice: true},

integration-tests/bats/schema-changes.bats

@@ -37,6 +37,14 @@ teardown() {
     [[ ! "$ouput" =~ "Merge failed" ]] || false
 }
 
+@test "schema-changes: dolt schema alter column preserves table checks" {
+    dolt sql -q "alter table test add constraint test_check CHECK (c2 < 12345);"
+    dolt sql -q "alter table test rename column c1 to c0;"
+    run dolt schema show test
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "CONSTRAINT \`test_check\` CHECK ((\`c2\` < 12345))" ]] || false
+}
+
 @test "schema-changes: dolt schema rename column" {
     dolt sql -q 'insert into test values (1,1,1,1,1,1)'
     run dolt sql -q "alter table test rename column c1 to c0"
@@ -56,6 +64,16 @@ teardown() {
     dolt sql -q "select * from test"
 }
 
+@test "schema-changes: dolt schema rename column fails when column is used in table check" {
+    dolt sql -q "alter table test add constraint test_check CHECK (c2 < 12345);"
+    dolt sql -q "alter table test rename column c2 to c0"
+
+    # TODO: This test can be enabled/finished once the go-mysql-server validation for checks
+    #       has been merged in:
+    #       https://github.com/dolthub/go-mysql-server/pull/774
+    skip "dolt incorrectly allows a column used in a table check expression to be renamed/removed"
+}
+
 @test "schema-changes: dolt schema delete column" {
     dolt sql -q 'insert into test values (1,1,1,1,1,1)'
     run dolt sql -q "alter table test drop column c1"

integration-tests/bats/sql-check-constraints.bats

@@ -64,20 +64,125 @@ SQL
     [[ "$output" =~ "constraint" ]] || false
 }
 
-@test "sql-check-constraints: check constraints survive adding a primary key" {
-    dolt sql <<SQL 
+@test "sql-check-constraints: check constraints survive adding a new column" {
+    dolt sql <<SQL
 create table foo (
        pk int,
-       c1 int
-       CHECK (c1 > 3)
-);      
-ALTER TABLE foo ADD PRIMARY KEY(pk);
+       c1 int,
+       CHECK (c1 > 3),
+       PRIMARY KEY (pk)
+);
+ALTER TABLE foo ADD COLUMN j int;
 SQL
-    skip "Alter tables kill all constraints now"
     run dolt schema show
     [ $status -eq 0 ]
+    echo $output
     [[ "$output" =~ "CHECK" ]] || false
-    [[ "$output" =~ "c1 > c3" ]] || false
-    
+    [[ "$output" =~ "`c1` > 3" ]] || false
 }
 
+@test "sql-check-constraints: check constraints survive renaming a column" {
+    dolt sql <<SQL
+create table foo (
+       pk int,
+       c1 int,
+       j int,
+       CHECK (c1 > 3),
+       PRIMARY KEY (pk)
+);
+ALTER TABLE foo RENAME COLUMN j to j2;
+SQL
+    run dolt schema show
+    [ $status -eq 0 ]
+    echo $output
+    [[ "$output" =~ "CHECK" ]] || false
+    [[ "$output" =~ "`c1` > 3" ]] || false
+}
+
+@test "sql-check-constraints: check constraints survive modifying a column" {
+    dolt sql <<SQL
+create table foo (
+       pk int,
+       c1 int,
+       j int,
+       CHECK (c1 > 3),
+       PRIMARY KEY (pk)
+);
+ALTER TABLE foo MODIFY COLUMN j int COMMENT 'j column';
+SQL
+    run dolt schema show
+    [ $status -eq 0 ]
+    echo $output
+    [[ "$output" =~ "CHECK" ]] || false
+    [[ "$output" =~ "`c1` > 3" ]] || false
+}
+
+@test "sql-check-constraints: check constraints survive dropping a column" {
+    dolt sql <<SQL
+create table foo (
+       pk int,
+       c1 int,
+       j int,
+       CHECK (c1 > 3),
+       PRIMARY KEY (pk)
+);
+ALTER TABLE foo DROP COLUMN j;
+SQL
+    run dolt schema show
+    [ $status -eq 0 ]
+    echo $output
+    [[ "$output" =~ "CHECK" ]] || false
+    [[ "$output" =~ "`c1` > 3" ]] || false
+}
+
+@test "sql-check-constraints: check constraints survive adding a primary key" {
+    dolt sql <<SQL
+create table foo (
+       pk int,
+       c1 int,
+       CHECK (c1 > 3)
+);
+ALTER TABLE foo ADD PRIMARY KEY(pk);
+SQL
+    run dolt schema show
+    [ $status -eq 0 ]
+    echo $output
+    [[ "$output" =~ "CHECK" ]] || false
+    [[ "$output" =~ "`c1` > 3" ]] || false
+}
+
+@test "sql-check-constraints: check constraints survive dropping a primary key" {
+    dolt sql <<SQL
+create table foo (
+       pk int,
+       c1 int,
+       CHECK (c1 > 3),
+       PRIMARY KEY (pk)
+);
+ALTER TABLE foo DROP PRIMARY KEY;
+SQL
+    run dolt schema show
+    [ $status -eq 0 ]
+    echo $output
+    [[ "$output" =~ "CHECK" ]] || false
+    [[ "$output" =~ "`c1` > 3" ]] || false
+}
+
+@test "sql-check-constraints: check constraints survive renaming a table" {
+    dolt sql <<SQL
+create table foo (
+       pk int,
+       c1 int,
+       CHECK (c1 > 3),
+       PRIMARY KEY (pk)
+);
+RENAME TABLE foo to foo2;
+SQL
+    run dolt schema show
+    [ $status -eq 0 ]
+    echo $output
+    [[ "$output" =~ "CHECK" ]] || false
+    [[ "$output" =~ "`c1` > 3" ]] || false
+}
+
+