go/cmd/dolt/commnds/sqlserver: Make it possible to run a read-only remotesapi service exposing the databases in the sql-server.

Supply `--remotesapi-port` with a port number on which to expose the service.

The endpoint can be used as a remote from the dolt client:

`dolt clone http://localhost:50051/this_does_not_matter/my_dolt_database`

will clone my_dolt_database from the sql-server that was run as `dolt
sql-server --remotesapi-port 50051`.

go/cmd/dolt/commands/sqlserver/server.go

@@ -32,6 +32,8 @@ import (
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
 	"github.com/dolthub/dolt/go/cmd/dolt/commands/engine"
 	"github.com/dolthub/dolt/go/libraries/doltcore/env"
+	"github.com/dolthub/dolt/go/libraries/doltcore/remotesrv"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	_ "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqlserver"
 )
@@ -201,6 +203,21 @@ func Serve(
 		}()
 	}
 
+	var remoteSrv *remotesrv.Server
+	if serverConfig.RemotesapiPort() != nil {
+		if remoteSrvSqlCtx, err := sqlEngine.NewContext(context.Background()); err == nil {
+			remoteSrv = sqle.NewRemoteSrvServer(remoteSrvSqlCtx, *serverConfig.RemotesapiPort())
+			go func() {
+				err := remoteSrv.Serve()
+				if err != nil {
+					lgr.Warnf("error starting remotesapi server on port %d: %v\n", *serverConfig.RemotesapiPort(), err)
+				}
+			}()
+		} else {
+			lgr.Warnf("error creating SQL engine context for remotesapi server; remotesapi will not be available: %v\n", err)
+		}
+	}
+
 	if ok, f := mrEnv.IsLocked(); ok {
 		startError = env.ErrActiveServerLock.New(f)
 		return
@@ -214,6 +231,9 @@ func Serve(
 		if metSrv != nil {
 			metSrv.Close()
 		}
+		if remoteSrv != nil {
+			remoteSrv.GracefulStop()
+		}
 
 		return mySQLServer.Close()
 	})

go/cmd/dolt/commands/sqlserver/serverconfig.go

@@ -141,6 +141,11 @@ type ServerConfig interface {
 	AllowCleartextPasswords() bool
 	// Socket is a path to the unix socket file
 	Socket() string
+	// RemotesapiPort is the port to use for serving a remotesapi interface with this sql-server instance.
+	// A remotesapi interface will allow this sql-server process to be used
+	// as a dolt remote for things like `clone`, `fetch` and read
+	// replication.
+	RemotesapiPort() *int
 }
 
 type commandLineServerConfig struct {
@@ -164,6 +169,7 @@ type commandLineServerConfig struct {
 	privilegeFilePath       string
 	allowCleartextPasswords bool
 	socket                  string
+	remotesapiPort          *int
 }
 
 var _ ServerConfig = (*commandLineServerConfig)(nil)
@@ -263,6 +269,10 @@ func (cfg *commandLineServerConfig) MetricsPort() int {
 	return defaultMetricsPort
 }
 
+func (cfg *commandLineServerConfig) RemotesapiPort() *int {
+	return cfg.remotesapiPort
+}
+
 // PrivilegeFilePath returns the path to the file which contains all needed privilege information in the form of a
 // JSON string.
 func (cfg *commandLineServerConfig) PrivilegeFilePath() string {
@@ -400,6 +410,12 @@ func (cfg *commandLineServerConfig) WithSocket(sockFilePath string) *commandLine
 	return cfg
 }
 
+// WithRemotesapiPort sets the remotesapi port to use.
+func (cfg *commandLineServerConfig) WithRemotesapiPort(port *int) *commandLineServerConfig {
+	cfg.remotesapiPort = port
+	return cfg
+}
+
 // DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values.
 func DefaultServerConfig() *commandLineServerConfig {
 	return &commandLineServerConfig{

go/cmd/dolt/commands/sqlserver/sqlserver.go

@@ -47,6 +47,7 @@ const (
 	persistenceBehaviorFlag     = "persistence-behavior"
 	allowCleartextPasswordsFlag = "allow-cleartext-passwords"
 	socketFlag                  = "socket"
+	remotesapiPortFlag          = "remotesapi-port"
 )
 
 func indentLines(s string) string {
@@ -148,6 +149,7 @@ func (cmd SqlServerCmd) ArgParser() *argparser.ArgParser {
 	ap.SupportsString(commands.PrivsFilePathFlag, "", "privilege file", "Path to a file to load and store users and grants. Defaults to `$doltcfg-dir/privileges.db`. Will only be created if there is a change to privileges.")
 	ap.SupportsString(allowCleartextPasswordsFlag, "", "allow-cleartext-passwords", "Allows use of cleartext passwords. Defaults to false.")
 	ap.SupportsOptionalString(socketFlag, "", "socket file", "Path for the unix socket file. Defaults to '/tmp/mysql.sock'.")
+	ap.SupportsUint(remotesapiPortFlag, "", "remotesapi port", "Sets the port for a server which can expose the databases in this sql-server over remotesapi.")
 	return ap
 }
 
@@ -335,6 +337,10 @@ func getCommandLineServerConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResult
 		serverConfig.withPassword(password)
 	}
 
+	if port, ok := apr.GetInt(remotesapiPortFlag); ok {
+		serverConfig.WithRemotesapiPort(&port)
+	}
+
 	if persistenceBehavior, ok := apr.GetValue(persistenceBehaviorFlag); ok {
 		serverConfig.withPersistenceBehavior(persistenceBehavior)
 	}

go/cmd/dolt/commands/sqlserver/yaml_config.go

@@ -113,6 +113,10 @@ type MetricsYAMLConfig struct {
 	Port   *int              `yaml:"port"`
 }
 
+type RemotesapiYAMLConfig struct {
+	Port *int `yaml:"port"`
+}
+
 type UserSessionVars struct {
 	Name string            `yaml:"name"`
 	Vars map[string]string `yaml:"vars"`
@@ -129,6 +133,7 @@ type YAMLConfig struct {
 	DataDirStr        *string               `yaml:"data_dir"`
 	CfgDirStr         *string               `yaml:"cfg_dir"`
 	MetricsConfig     MetricsYAMLConfig     `yaml:"metrics"`
+	RemotesapiConfig  RemotesapiYAMLConfig  `yaml:"remotesapi"`
 	PrivilegeFile     *string               `yaml:"privilege_file"`
 	Vars              []UserSessionVars     `yaml:"user_session_vars"`
 	Jwks              []engine.JwksConfig   `yaml:"jwks"`
@@ -334,6 +339,10 @@ func (cfg YAMLConfig) MetricsPort() int {
 	return *cfg.MetricsConfig.Port
 }
 
+func (cfg YAMLConfig) RemotesapiPort() *int {
+	return cfg.RemotesapiConfig.Port
+}
+
 // PrivilegeFilePath returns the path to the file which contains all needed privilege information in the form of a
 // JSON string.
 func (cfg YAMLConfig) PrivilegeFilePath() string {

go/cmd/dolt/commands/sqlserver/yaml_config_test.go

@@ -204,6 +204,7 @@ func TestYAMLConfigDefaults(t *testing.T) {
 	assert.Equal(t, defaultMetricsPort, cfg.MetricsPort())
 	assert.Nil(t, cfg.MetricsConfig.Labels)
 	assert.Equal(t, defaultAllowCleartextPasswords, cfg.AllowCleartextPasswords())
+	assert.Nil(t, cfg.RemotesapiPort())
 
 	c, err := LoadTLSConfig(cfg)
 	assert.NoError(t, err)

go/libraries/doltcore/remotesrv/server.go

@@ -92,12 +92,24 @@ func grpcMultiplexHandler(grpcSrv *grpc.Server, handler http.Handler) http.Handl
 func (s *Server) Serve() error {
 	httpListener, err := net.Listen("tcp", fmt.Sprintf(":%d", s.httpPort))
 	if err != nil {
+		// Cleanup s.wg so that callers who have spawned Serve() in a
+		// goroutine can safely call GracefulStop() from outside of it.
+		s.wg.Done()
+		s.wg.Done()
+		if s.grpcPort != s.httpPort {
+			s.wg.Done()
+			s.wg.Done()
+		}
 		return err
 	}
 
 	if s.grpcPort != s.httpPort {
 		grpcListener, err := net.Listen("tcp", fmt.Sprintf(":%d", s.grpcPort))
 		if err != nil {
+			s.wg.Done()
+			s.wg.Done()
+			s.wg.Done()
+			s.wg.Done()
 			httpListener.Close()
 			return err
 		}

go/libraries/doltcore/sqle/remotesrv.go

@@ -0,0 +1,56 @@
+// Copyright 2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqle
+
+import (
+	"errors"
+
+	"github.com/dolthub/go-mysql-server/sql"
+
+	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
+	"github.com/dolthub/dolt/go/libraries/doltcore/remotesrv"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
+	"github.com/dolthub/dolt/go/store/datas"
+)
+
+type remotesrvStore struct {
+	ctx *sql.Context
+}
+
+var _ remotesrv.DBCache = remotesrvStore{}
+
+func (s remotesrvStore) Get(org, repo, nbfVerStr string) (remotesrv.RemoteSrvStore, error) {
+	sess := dsess.DSessFromSess(s.ctx.Session)
+	db, err := sess.Provider().Database(s.ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+	ddb, ok := db.(Database)
+	if !ok {
+		return nil, errors.New("unimplemented")
+	}
+	datasdb := doltdb.HackDatasDatabaseFromDoltDB(ddb.DbData().Ddb)
+	cs := datas.ChunkStoreFromDatabase(datasdb)
+	rss, ok := cs.(remotesrv.RemoteSrvStore)
+	if !ok {
+		return nil, errors.New("unimplemented")
+	}
+	return rss, nil
+}
+
+func NewRemoteSrvServer(ctx *sql.Context, port int) *remotesrv.Server {
+	sess := dsess.DSessFromSess(ctx.Session)
+	return remotesrv.NewServer("", port, port, sess.Provider().FileSystem(), remotesrvStore{ctx}, true)
+}