diff --git a/go/store/nbs/generational_chunk_store.go b/go/store/nbs/generational_chunk_store.go index 4d6e4f5137..2b63cb65ce 100644 --- a/go/store/nbs/generational_chunk_store.go +++ b/go/store/nbs/generational_chunk_store.go @@ -331,7 +331,7 @@ func (gcs *GenerationalNBS) GetChunkLocationsWithPaths(hashes hash.HashSet) (map return nil, err } for k, v := range toadd { - res[prefix + "/" + k] = v + res[filepath.ToSlash(filepath.Join(prefix, k))] = v } } return res, nil diff --git a/go/utils/remotesrv/cscache.go b/go/utils/remotesrv/cscache.go index 4b26c49119..9c58be5a4b 100644 --- a/go/utils/remotesrv/cscache.go +++ b/go/utils/remotesrv/cscache.go @@ -40,22 +40,26 @@ type store interface { var _ store = &nbs.NomsBlockStore{} var _ store = &nbs.GenerationalNBS{} -type DBCache struct { +type LocalCSCache struct { mu *sync.Mutex dbs map[string]store fs filesys.Filesys } -func NewLocalCSCache(filesys filesys.Filesys) *DBCache { - return &DBCache{ +func NewLocalCSCache(filesys filesys.Filesys) *LocalCSCache { + return &LocalCSCache{ &sync.Mutex{}, make(map[string]store), filesys, } } -func (cache *DBCache) Get(org, repo, nbfVerStr string) (store, error) { +type DBCache interface { + Get(org, repo, nbfVerStr string) (store, error) +} + +func (cache *LocalCSCache) Get(org, repo, nbfVerStr string) (store, error) { cache.mu.Lock() defer cache.mu.Unlock() @@ -87,3 +91,11 @@ func (cache *DBCache) Get(org, repo, nbfVerStr string) (store, error) { return newCS, nil } + +type SingletonCSCache struct { + s store +} + +func (cache SingletonCSCache) Get(org, repo, nbfVerStr string) (store, error) { + return cache.s, nil +} diff --git a/go/utils/remotesrv/grpc.go b/go/utils/remotesrv/grpc.go index cf7d21d9dd..da8c635daa 100644 --- a/go/utils/remotesrv/grpc.go +++ b/go/utils/remotesrv/grpc.go @@ -36,14 +36,14 @@ import ( type RemoteChunkStore struct { HttpHost string - csCache *DBCache + csCache DBCache bucket string expectedFiles fileDetails fs filesys.Filesys remotesapi.UnimplementedChunkStoreServiceServer } -func NewHttpFSBackedChunkStore(httpHost string, csCache *DBCache, expectedFiles fileDetails, fs filesys.Filesys) *RemoteChunkStore { +func NewHttpFSBackedChunkStore(httpHost string, csCache DBCache, expectedFiles fileDetails, fs filesys.Filesys) *RemoteChunkStore { return &RemoteChunkStore{ HttpHost: httpHost, csCache: csCache, @@ -369,6 +369,11 @@ func (rs *RemoteChunkStore) GetRepoMetadata(ctx context.Context, req *remotesapi return nil, status.Error(codes.Internal, "Could not get chunkstore") } + err := cs.Rebase(ctx) + if err != nil { + return nil, err + } + size, err := cs.Size(ctx) if err != nil { return nil, err diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index 4341f23ddf..8cb223d746 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -62,12 +62,12 @@ func newFileDetails() fileDetails { } type filehandler struct { - dbCache *DBCache + dbCache DBCache expectedFiles fileDetails fs filesys.Filesys } -func newFileHandler(dbCache *DBCache, expectedFiles fileDetails, fs filesys.Filesys) filehandler { +func newFileHandler(dbCache DBCache, expectedFiles fileDetails, fs filesys.Filesys) filehandler { return filehandler{dbCache, expectedFiles, fs} } @@ -111,6 +111,7 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) { if len(tokens) != 3 { logger(fmt.Sprintf("response to: %v method: %v http response code: %v", req.RequestURI, req.Method, http.StatusNotFound)) respWr.WriteHeader(http.StatusNotFound) + return } org := tokens[0] @@ -178,7 +179,7 @@ func readTableFile(logger func(string), path string, respWr http.ResponseWriter, logger(fmt.Sprintf("wrote %d bytes", n)) - return http.StatusOK + return -1 } type uploadreader struct { @@ -216,7 +217,7 @@ func (u *uploadreader) Close() error { return nil } -func writeTableFile(ctx context.Context, logger func(string), dbCache *DBCache, expectedFiles fileDetails, org, repo, fileId string, request *http.Request) int { +func writeTableFile(ctx context.Context, logger func(string), dbCache DBCache, expectedFiles fileDetails, org, repo, fileId string, request *http.Request) int { _, ok := hash.MaybeParse(fileId) if !ok { diff --git a/go/utils/remotesrv/main.go b/go/utils/remotesrv/main.go index 9fbac172a5..3085e24d01 100644 --- a/go/utils/remotesrv/main.go +++ b/go/utils/remotesrv/main.go @@ -28,10 +28,14 @@ import ( "google.golang.org/grpc" remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/datas" ) func main() { + repoModeParam := flag.Bool("repo-mode", false, "act as a remote for a dolt directory, instead of stand alone") dirParam := flag.String("dir", "", "root directory that this command will run in.") grpcPortParam := flag.Int("grpc-port", -1, "root directory that this command will run in.") httpPortParam := flag.Int("http-port", -1, "root directory that this command will run in.") @@ -62,7 +66,25 @@ func main() { log.Println("'grpc-port' parameter not provided. Using default port 50051") } - stopChan, wg := startServer(*httpHostParam, *httpPortParam, *grpcPortParam) + fs, err := filesys.LocalFilesysWithWorkingDir(".") + if err != nil { + log.Fatalln("could not get cwd path:", err.Error()) + } + + var dbCache DBCache + if *repoModeParam { + dEnv := env.Load(context.Background(), env.GetCurrentUserHomeDir, fs, doltdb.LocalDirDoltDB, "remotesrv") + if !dEnv.Valid() { + log.Fatalln("repo-mode failed to load repository") + } + db := doltdb.HackDatasDatabaseFromDoltDB(dEnv.DoltDB) + cs := datas.ChunkStoreFromDatabase(db) + dbCache = SingletonCSCache{cs.(store)} + } else { + dbCache = NewLocalCSCache(fs) + } + + stopChan, wg := startServer(*httpHostParam, *httpPortParam, *grpcPortParam, fs, dbCache) waitForSignal() close(stopChan) @@ -75,13 +97,7 @@ func waitForSignal() { <-c } -func startServer(httpHost string, httpPort, grpcPort int) (chan interface{}, *sync.WaitGroup) { - fs, err := filesys.LocalFilesysWithWorkingDir(".") - if err != nil { - log.Fatalln("could not get cwd path:", err.Error()) - } - - dbCache := NewLocalCSCache(fs) +func startServer(httpHost string, httpPort, grpcPort int, fs filesys.Filesys, dbCache DBCache) (chan interface{}, *sync.WaitGroup) { expectedFiles := newFileDetails() wg := sync.WaitGroup{} @@ -102,7 +118,7 @@ func startServer(httpHost string, httpPort, grpcPort int) (chan interface{}, *sy return stopChan, &wg } -func grpcServer(dbCache *DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpHost string, grpcPort int, stopChan chan interface{}) { +func grpcServer(dbCache DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpHost string, grpcPort int, stopChan chan interface{}) { defer func() { log.Println("exiting grpc Server go routine") }() @@ -127,7 +143,7 @@ func grpcServer(dbCache *DBCache, fs filesys.Filesys, expectedFiles fileDetails, grpcServer.GracefulStop() } -func httpServer(dbCache *DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpPort int, stopChan chan interface{}) { +func httpServer(dbCache DBCache, fs filesys.Filesys, expectedFiles fileDetails, httpPort int, stopChan chan interface{}) { defer func() { log.Println("exiting http Server go routine") }() diff --git a/integration-tests/bats/remotesrv.bats b/integration-tests/bats/remotesrv.bats new file mode 100644 index 0000000000..a746365785 --- /dev/null +++ b/integration-tests/bats/remotesrv.bats @@ -0,0 +1,76 @@ +#!/usr/bin/env bats +# +# Tests for remotesrv itself. remotes.bats also uses remotesrv as a testing +# dependency, but it is not technically the unit under test there. This test +# uses dolt remotestorage, clone, pull, fetch, etc., as a dependency but is +# more concerned with covering remotesrv functionality. + +load $BATS_TEST_DIRNAME/helper/common.bash + +remotesrv_pid= +setup() { + skiponwindows "tests are flaky on Windows" + setup_common +} + +teardown() { + teardown_common + if [ -n "$remotesrv_pid" ]; then + kill $remotesrv_pid + fi +} + +@test "remotesrv: can read from remotesrv in repo-mode" { + mkdir remote + cd remote + dolt init + dolt sql -q 'create table vals (i int);' + dolt sql -q 'insert into vals (i) values (1), (2), (3), (4), (5);' + dolt add vals + dolt commit -m 'initial vals.' + + remotesrv --http-port 1234 --repo-mode & + remotesrv_pid=$! + + cd ../ + dolt clone http://localhost:50051/test-org/test-repo repo1 + cd repo1 + run dolt ls + [[ "$output" =~ "vals" ]] || false + run dolt sql -q 'select count(*) from vals' + [[ "$output" =~ "5" ]] || false + + cd ../remote + dolt sql -q 'insert into vals (i) values (6), (7), (8), (9), (10);' + dolt commit -am 'add some vals' + + cd ../repo1 + dolt pull + run dolt sql -q 'select count(*) from vals;' + [[ "$output" =~ "10" ]] || false +} + +@test "remotesrv: can write to remotesrv in repo-mode" { + mkdir remote + cd remote + dolt init + dolt sql -q 'create table vals (i int);' + dolt add vals + dolt commit -m 'create vals table.' + + remotesrv --http-port 1234 --repo-mode & + remotesrv_pid=$! + + cd ../ + dolt clone http://localhost:50051/test-org/test-repo repo1 + cd repo1 + dolt sql -q 'insert into vals values (1), (2), (3), (4), (5);' + dolt commit -am 'insert some values' + dolt push origin main:main + + cd ../remote + # Have to reset the working set, which was not updated by the push... + dolt reset --hard + run dolt sql -q 'select count(*) from vals;' + [[ "$output" =~ "5" ]] || false +}