go/utils/remotesrv: Make remotesrv capable of serving as a name-agnostic remote for a single dolt repository made with dolt CLI.

go/store/nbs/generational_chunk_store.go

@@ -331,7 +331,7 @@ func (gcs *GenerationalNBS) GetChunkLocationsWithPaths(hashes hash.HashSet) (map
 			return nil, err
 		}
 		for k, v := range toadd {
-			res[prefix + "/" + k] = v
+			res[filepath.ToSlash(filepath.Join(prefix, k))] = v
 		}
 	}
 	return res, nil

go/utils/remotesrv/cscache.go

@@ -40,22 +40,26 @@ type store interface {
 var _ store = &nbs.NomsBlockStore{}
 var _ store = &nbs.GenerationalNBS{}
 
-type DBCache struct {
+type LocalCSCache struct {
 	mu  *sync.Mutex
 	dbs map[string]store
 
 	fs filesys.Filesys
 }
 
-func NewLocalCSCache(filesys filesys.Filesys) *DBCache {
-	return &DBCache{
+func NewLocalCSCache(filesys filesys.Filesys) *LocalCSCache {
+	return &LocalCSCache{
 		&sync.Mutex{},
 		make(map[string]store),
 		filesys,
 	}
 }
 
-func (cache *DBCache) Get(org, repo, nbfVerStr string) (store, error) {
+type DBCache interface {
+	Get(org, repo, nbfVerStr string) (store, error)
+}
+
+func (cache *LocalCSCache) Get(org, repo, nbfVerStr string) (store, error) {
 	cache.mu.Lock()
 	defer cache.mu.Unlock()
 
@@ -87,3 +91,11 @@ func (cache *DBCache) Get(org, repo, nbfVerStr string) (store, error) {
 
 	return newCS, nil
 }
+
+type SingletonCSCache struct {
+	s store
+}
+
+func (cache SingletonCSCache) Get(org, repo, nbfVerStr string) (store, error) {
+	return cache.s, nil
+}

go/utils/remotesrv/grpc.go

@@ -36,14 +36,14 @@ import (
 
 type RemoteChunkStore struct {
 	HttpHost      string
-	csCache       *DBCache
+	csCache       DBCache
 	bucket        string
 	expectedFiles fileDetails
 	fs            filesys.Filesys
 	remotesapi.UnimplementedChunkStoreServiceServer
 }
 
-func NewHttpFSBackedChunkStore(httpHost string, csCache *DBCache, expectedFiles fileDetails, fs filesys.Filesys) *RemoteChunkStore {
+func NewHttpFSBackedChunkStore(httpHost string, csCache DBCache, expectedFiles fileDetails, fs filesys.Filesys) *RemoteChunkStore {
 	return &RemoteChunkStore{
 		HttpHost:      httpHost,
 		csCache:       csCache,
@@ -369,6 +369,11 @@ func (rs *RemoteChunkStore) GetRepoMetadata(ctx context.Context, req *remotesapi
 		return nil, status.Error(codes.Internal, "Could not get chunkstore")
 	}
 
+	err := cs.Rebase(ctx)
+	if err != nil {
+		return nil, err
+	}
+
 	size, err := cs.Size(ctx)
 	if err != nil {
 		return nil, err

go/utils/remotesrv/http.go

@@ -62,12 +62,12 @@ func newFileDetails() fileDetails {
 }
 
 type filehandler struct {
-	dbCache       *DBCache
+	dbCache       DBCache
 	expectedFiles fileDetails
 	fs            filesys.Filesys
 }
 
-func newFileHandler(dbCache *DBCache, expectedFiles fileDetails, fs filesys.Filesys) filehandler {
+func newFileHandler(dbCache DBCache, expectedFiles fileDetails, fs filesys.Filesys) filehandler {
 	return filehandler{dbCache, expectedFiles, fs}
 }
 
@@ -111,6 +111,7 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
 		if len(tokens) != 3 {
 			logger(fmt.Sprintf("response to: %v method: %v http response code: %v", req.RequestURI, req.Method, http.StatusNotFound))
 			respWr.WriteHeader(http.StatusNotFound)
+			return
 		}
 
 		org := tokens[0]
@@ -178,7 +179,7 @@ func readTableFile(logger func(string), path string, respWr http.ResponseWriter,
 
 	logger(fmt.Sprintf("wrote %d bytes", n))
 
-	return http.StatusOK
+	return -1
 }
 
 type uploadreader struct {
@@ -216,7 +217,7 @@ func (u *uploadreader) Close() error {
 	return nil
 }
 
-func writeTableFile(ctx context.Context, logger func(string), dbCache *DBCache, expectedFiles fileDetails, org, repo, fileId string, request *http.Request) int {
+func writeTableFile(ctx context.Context, logger func(string), dbCache DBCache, expectedFiles fileDetails, org, repo, fileId string, request *http.Request) int {
 	_, ok := hash.MaybeParse(fileId)
 
 	if !ok {

go/utils/remotesrv/main.go

@@ -28,10 +28,14 @@ import (
 	"google.golang.org/grpc"
 
 	remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1"
+	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
+	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
+	"github.com/dolthub/dolt/go/store/datas"
 )
 
 func main() {
+	repoModeParam := flag.Bool("repo-mode", false, "act as a remote for a dolt directory, instead of stand alone")
 	dirParam := flag.String("dir", "", "root directory that this command will run in.")
 	grpcPortParam := flag.Int("grpc-port", -1, "root directory that this command will run in.")
 	httpPortParam := flag.Int("http-port", -1, "root directory that this command will run in.")
@@ -62,7 +66,25 @@ func main() {
 		log.Println("'grpc-port' parameter not provided. Using default port 50051")
 	}
 
-	stopChan, wg := startServer(*httpHostParam, *httpPortParam, *grpcPortParam)
+	fs, err := filesys.LocalFilesysWithWorkingDir(".")
+	if err != nil {
+		log.Fatalln("could not get cwd path:", err.Error())
+	}
+
+	var dbCache DBCache
+	if *repoModeParam {
+		dEnv := env.Load(context.Background(), env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, "remotesrv")
+		if !dEnv.Valid() {
+			log.Fatalln("repo-mode failed to load repository")
+		}
+		db := doltdb.HackDatasDatabaseFromDoltDB(dEnv.DoltDB)
+		cs := datas.ChunkStoreFromDatabase(db)
+		dbCache = SingletonCSCache{cs.(store)}
+	} else {
+		dbCache = NewLocalCSCache(fs)
+	}
+
+	stopChan, wg := startServer(*httpHostParam, *httpPortParam, *grpcPortParam, fs, dbCache)
 	waitForSignal()
 
 	close(stopChan)
@@ -75,13 +97,7 @@ func waitForSignal() {
 	<-c
 }
 
-func startServer(httpHost string, httpPort, grpcPort int) (chan interface{}, *sync.WaitGroup) {
-	fs, err := filesys.LocalFilesysWithWorkingDir(".")
-	if err != nil {
-		log.Fatalln("could not get cwd path:", err.Error())
-	}
-
-	dbCache := NewLocalCSCache(fs)
+func startServer(httpHost string, httpPort, grpcPort int, fs filesys.Filesys, dbCache DBCache) (chan interface{}, *sync.WaitGroup) {
 	expectedFiles := newFileDetails()
 
 	wg := sync.WaitGroup{}
@@ -102,7 +118,7 @@ func startServer(httpHost string, httpPort, grpcPort int) (chan interface{}, *sy
 	return stopChan, &wg
 }
 
-func grpcServer(dbCache *DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpHost string, grpcPort int, stopChan chan interface{}) {
+func grpcServer(dbCache DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpHost string, grpcPort int, stopChan chan interface{}) {
 	defer func() {
 		log.Println("exiting grpc Server go routine")
 	}()
@@ -127,7 +143,7 @@ func grpcServer(dbCache *DBCache, fs filesys.Filesys, expectedFiles fileDetails,
 	grpcServer.GracefulStop()
 }
 
-func httpServer(dbCache *DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpPort int, stopChan chan interface{}) {
+func httpServer(dbCache DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpPort int, stopChan chan interface{}) {
 	defer func() {
 		log.Println("exiting http Server go routine")
 	}()

integration-tests/bats/remotesrv.bats

@@ -0,0 +1,76 @@
+#!/usr/bin/env bats
+#
+# Tests for remotesrv itself. remotes.bats also uses remotesrv as a testing
+# dependency, but it is not technically the unit under test there. This test
+# uses dolt remotestorage, clone, pull, fetch, etc., as a dependency but is
+# more concerned with covering remotesrv functionality.
+
+load $BATS_TEST_DIRNAME/helper/common.bash
+
+remotesrv_pid=
+setup() {
+    skiponwindows "tests are flaky on Windows"
+    setup_common
+}
+
+teardown() {
+    teardown_common
+    if [ -n "$remotesrv_pid" ]; then
+        kill $remotesrv_pid
+    fi
+}
+
+@test "remotesrv: can read from remotesrv in repo-mode" {
+    mkdir remote
+    cd remote
+    dolt init
+    dolt sql -q 'create table vals (i int);'
+    dolt sql -q 'insert into vals (i) values (1), (2), (3), (4), (5);'
+    dolt add vals
+    dolt commit -m 'initial vals.'
+
+    remotesrv --http-port 1234 --repo-mode &
+    remotesrv_pid=$!
+
+    cd ../
+    dolt clone http://localhost:50051/test-org/test-repo repo1
+    cd repo1
+    run dolt ls
+    [[ "$output" =~ "vals" ]] || false
+    run dolt sql -q 'select count(*) from vals'
+    [[ "$output" =~ "5" ]] || false
+
+    cd ../remote
+    dolt sql -q 'insert into vals (i) values (6), (7), (8), (9), (10);'
+    dolt commit -am 'add some vals'
+
+    cd ../repo1
+    dolt pull
+    run dolt sql -q 'select count(*) from vals;'
+    [[ "$output" =~ "10" ]] || false
+}
+
+@test "remotesrv: can write to remotesrv in repo-mode" {
+    mkdir remote
+    cd remote
+    dolt init
+    dolt sql -q 'create table vals (i int);'
+    dolt add vals
+    dolt commit -m 'create vals table.'
+
+    remotesrv --http-port 1234 --repo-mode &
+    remotesrv_pid=$!
+
+    cd ../
+    dolt clone http://localhost:50051/test-org/test-repo repo1
+    cd repo1
+    dolt sql -q 'insert into vals values (1), (2), (3), (4), (5);'
+    dolt commit -am 'insert some values'
+    dolt push origin main:main
+
+    cd ../remote
+    # Have to reset the working set, which was not updated by the push...
+    dolt reset --hard
+    run dolt sql -q 'select count(*) from vals;'
+    [[ "$output" =~ "5" ]] || false
+}