From b6cd726af301564e012da1837cf1893f47a1292a Mon Sep 17 00:00:00 2001 From: Brian Hendriks Date: Wed, 15 May 2024 10:27:16 -0700 Subject: [PATCH] move config --- go/cmd/dolt/commands/engine/jwtplugin.go | 8 +- .../commands/sqlserver/command_line_config.go | 100 +++++---- .../dolt/commands/sqlserver/queryist_utils.go | 3 +- go/cmd/dolt/commands/sqlserver/server.go | 57 ++--- go/cmd/dolt/commands/sqlserver/server_test.go | 57 ++--- go/cmd/dolt/commands/sqlserver/sqlserver.go | 30 +-- .../doltcore/servercfg}/minver_test.go | 2 +- .../doltcore/servercfg}/serverconfig.go | 203 ++++++++++++------ .../servercfg}/testdata/chain_cert.pem | 0 .../servercfg}/testdata/chain_key.pem | 0 .../servercfg}/testdata/minver_validation.txt | 24 +-- .../servercfg}/testdata/selfsigned_cert.pem | 0 .../servercfg}/testdata/selfsigned_key.pem | 0 .../doltcore/servercfg}/yaml_config.go | 148 ++++++------- .../doltcore/servercfg}/yaml_config_test.go | 44 ++-- go/libraries/doltcore/sqle/cluster/config.go | 37 ---- .../doltcore/sqle/cluster/controller.go | 7 +- .../sqle/enginetest/dolt_server_test.go | 25 +-- go/libraries/utils/minver/minvertesting.go | 2 +- .../replicationbench/async_replica_test.go | 8 +- .../replicationbench/replica_test.go | 13 +- go/performance/serverbench/bench_test.go | 13 +- go/utils/genminver_validation/main.go | 4 +- 23 files changed, 401 insertions(+), 384 deletions(-) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/minver_test.go (98%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/serverconfig.go (68%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/testdata/chain_cert.pem (100%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/testdata/chain_key.pem (100%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/testdata/minver_validation.txt (75%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/testdata/selfsigned_cert.pem (100%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/testdata/selfsigned_key.pem (100%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/yaml_config.go (85%) rename go/{cmd/dolt/commands/sqlserver => libraries/doltcore/servercfg}/yaml_config_test.go (89%) delete mode 100644 go/libraries/doltcore/sqle/cluster/config.go diff --git a/go/cmd/dolt/commands/engine/jwtplugin.go b/go/cmd/dolt/commands/engine/jwtplugin.go index 4e9e012091..489f760218 100644 --- a/go/cmd/dolt/commands/engine/jwtplugin.go +++ b/go/cmd/dolt/commands/engine/jwtplugin.go @@ -23,15 +23,11 @@ import ( "github.com/dolthub/go-mysql-server/sql/mysql_db" "github.com/sirupsen/logrus" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "github.com/dolthub/dolt/go/libraries/utils/jwtauth" ) -type JwksConfig struct { - Name string `yaml:"name"` - LocationUrl string `yaml:"location_url"` - Claims map[string]string `yaml:"claims"` - FieldsToLog []string `yaml:"fields_to_log"` -} +type JwksConfig servercfg.ServerJwksConfig // authenticateDoltJWTPlugin is used to authenticate plaintext user plugins type authenticateDoltJWTPlugin struct { diff --git a/go/cmd/dolt/commands/sqlserver/command_line_config.go b/go/cmd/dolt/commands/sqlserver/command_line_config.go index b932dbd2b7..0a55770823 100755 --- a/go/cmd/dolt/commands/sqlserver/command_line_config.go +++ b/go/cmd/dolt/commands/sqlserver/command_line_config.go @@ -16,14 +16,14 @@ package sqlserver import ( "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" + "github.com/dolthub/dolt/go/libraries/utils/filesys" "path/filepath" "strconv" "strings" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" - "github.com/dolthub/dolt/go/cmd/dolt/commands/engine" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/cluster" "github.com/dolthub/dolt/go/libraries/utils/argparser" ) @@ -34,7 +34,7 @@ type commandLineServerConfig struct { password string timeout uint64 readOnly bool - logLevel LogLevel + logLevel servercfg.LogLevel dataDir string cfgDir string autoCommit bool @@ -58,39 +58,39 @@ type commandLineServerConfig struct { valuesSet map[string]struct{} } -var _ ServerConfig = (*commandLineServerConfig)(nil) +var _ servercfg.ServerConfig = (*commandLineServerConfig)(nil) -// DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values. -func DefaultServerConfig() *commandLineServerConfig { +// DefaultCommandLineServerConfig creates a `*ServerConfig` that has all of the options set to their default values. +func DefaultCommandLineServerConfig() *commandLineServerConfig { return &commandLineServerConfig{ - host: defaultHost, - port: defaultPort, - password: defaultPass, - timeout: defaultTimeout, - readOnly: defaultReadOnly, - logLevel: defaultLogLevel, - autoCommit: defaultAutoCommit, - maxConnections: defaultMaxConnections, - queryParallelism: defaultQueryParallelism, - persistenceBehavior: defaultPersistenceBahavior, - dataDir: defaultDataDir, - cfgDir: filepath.Join(defaultDataDir, defaultCfgDir), - privilegeFilePath: filepath.Join(defaultDataDir, defaultCfgDir, defaultPrivilegeFilePath), - branchControlFilePath: filepath.Join(defaultDataDir, defaultCfgDir, defaultBranchControlFilePath), - allowCleartextPasswords: defaultAllowCleartextPasswords, - maxLoggedQueryLen: defaultMaxLoggedQueryLen, + host: servercfg.DefaultHost, + port: servercfg.DefaultPort, + password: servercfg.DefaultPass, + timeout: servercfg.DefaultTimeout, + readOnly: servercfg.DefaultReadOnly, + logLevel: servercfg.DefaultLogLevel, + autoCommit: servercfg.DefaultAutoCommit, + maxConnections: servercfg.DefaultMaxConnections, + queryParallelism: servercfg.DefaultQueryParallelism, + persistenceBehavior: servercfg.DefaultPersistenceBahavior, + dataDir: servercfg.DefaultDataDir, + cfgDir: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir), + privilegeFilePath: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir, servercfg.DefaultPrivilegeFilePath), + branchControlFilePath: filepath.Join(servercfg.DefaultDataDir, servercfg.DefaultCfgDir, servercfg.DefaultBranchControlFilePath), + allowCleartextPasswords: servercfg.DefaultAllowCleartextPasswords, + maxLoggedQueryLen: servercfg.DefaultMaxLoggedQueryLen, valuesSet: map[string]struct{}{}, } } // NewCommandLineConfig returns server config based on the credentials and command line arguments given. -func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResults) (ServerConfig, error) { - config := DefaultServerConfig() +func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResults) (servercfg.ServerConfig, error) { + config := DefaultCommandLineServerConfig() if sock, ok := apr.GetValue(socketFlag); ok { // defined without value gets default if sock == "" { - sock = defaultUnixSocketFilePath + sock = servercfg.DefaultUnixSocketFilePath } config.WithSocket(sock) } @@ -144,7 +144,7 @@ func NewCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult } if logLevel, ok := apr.GetValue(logLevelFlag); ok { - config.withLogLevel(LogLevel(strings.ToLower(logLevel))) + config.withLogLevel(servercfg.LogLevel(strings.ToLower(logLevel))) } if dataDir, ok := apr.GetValue(commands.MultiDBDirFlag); ok { @@ -215,7 +215,7 @@ func (cfg *commandLineServerConfig) ReadOnly() bool { } // LogLevel returns the level of logging that the server will use. -func (cfg *commandLineServerConfig) LogLevel() LogLevel { +func (cfg *commandLineServerConfig) LogLevel() servercfg.LogLevel { return cfg.logLevel } @@ -287,11 +287,11 @@ func (cfg *commandLineServerConfig) MetricsLabels() map[string]string { } func (cfg *commandLineServerConfig) MetricsHost() string { - return defaultMetricsHost + return servercfg.DefaultMetricsHost } func (cfg *commandLineServerConfig) MetricsPort() int { - return defaultMetricsPort + return servercfg.DefaultMetricsPort } func (cfg *commandLineServerConfig) RemotesapiPort() *int { @@ -302,7 +302,7 @@ func (cfg *commandLineServerConfig) RemotesapiReadOnly() *bool { return cfg.remotesapiReadOnly } -func (cfg *commandLineServerConfig) ClusterConfig() cluster.Config { +func (cfg *commandLineServerConfig) ClusterConfig() servercfg.ClusterConfig { return nil } @@ -318,15 +318,15 @@ func (cfg *commandLineServerConfig) BranchControlFilePath() string { } // UserVars is an array containing user specific session variables. -func (cfg *commandLineServerConfig) UserVars() []UserSessionVars { +func (cfg *commandLineServerConfig) UserVars() []servercfg.UserSessionVars { return nil } -func (cfg *commandLineServerConfig) SystemVars() engine.SystemVariables { +func (cfg *commandLineServerConfig) SystemVars() map[string]interface{} { return nil } -func (cfg *commandLineServerConfig) JwksConfig() []engine.JwksConfig { +func (cfg *commandLineServerConfig) JwksConfig() []servercfg.ServerJwksConfig { return nil } @@ -376,8 +376,8 @@ func (cfg *commandLineServerConfig) withPassword(password string) *commandLineSe // withTimeout updates the timeout and returns the called `*commandLineServerConfig`, which is useful for chaining calls. func (cfg *commandLineServerConfig) withTimeout(timeout uint64) *commandLineServerConfig { cfg.timeout = timeout - cfg.valuesSet[readTimeoutKey] = struct{}{} - cfg.valuesSet[writeTimeoutKey] = struct{}{} + cfg.valuesSet[servercfg.ReadTimeoutKey] = struct{}{} + cfg.valuesSet[servercfg.WriteTimeoutKey] = struct{}{} return cfg } @@ -388,7 +388,7 @@ func (cfg *commandLineServerConfig) withReadOnly(readonly bool) *commandLineServ } // withLogLevel updates the log level and returns the called `*commandLineServerConfig`, which is useful for chaining calls. -func (cfg *commandLineServerConfig) withLogLevel(loglevel LogLevel) *commandLineServerConfig { +func (cfg *commandLineServerConfig) withLogLevel(loglevel servercfg.LogLevel) *commandLineServerConfig { cfg.logLevel = loglevel return cfg } @@ -397,7 +397,7 @@ func (cfg *commandLineServerConfig) withLogLevel(loglevel LogLevel) *commandLine // `*commandLineServerConfig`, which is useful for chaining calls. func (cfg *commandLineServerConfig) withMaxConnections(maxConnections uint64) *commandLineServerConfig { cfg.maxConnections = maxConnections - cfg.valuesSet[maxConnectionsKey] = struct{}{} + cfg.valuesSet[servercfg.MaxConnectionsKey] = struct{}{} return cfg } @@ -459,7 +459,7 @@ func (cfg *commandLineServerConfig) WithRemotesapiReadOnly(readonly *bool) *comm return cfg } -func (cfg *commandLineServerConfig) goldenMysqlConnectionString() string { +func (cfg *commandLineServerConfig) GoldenMysqlConnectionString() string { return cfg.goldenMysqlConn } @@ -481,7 +481,7 @@ func (cfg *commandLineServerConfig) EventSchedulerStatus() string { func (cfg *commandLineServerConfig) withEventScheduler(es string) *commandLineServerConfig { cfg.eventSchedulerStatus = es - cfg.valuesSet[eventSchedulerKey] = struct{}{} + cfg.valuesSet[servercfg.EventSchedulerKey] = struct{}{} return cfg } @@ -489,3 +489,25 @@ func (cfg *commandLineServerConfig) ValueSet(value string) bool { _, ok := cfg.valuesSet[value] return ok } + +// DoltServerConfigReader is the default implementation of ServerConfigReader suitable for parsing Dolt config files +// and command line options. +type DoltServerConfigReader struct{} + +// ServerConfigReader is an interface for reading a ServerConfig from a file or command line arguments. +type ServerConfigReader interface { + // ReadConfigFile reads a config file and returns a ServerConfig for it + ReadConfigFile(cwdFS filesys.Filesys, file string) (servercfg.ServerConfig, error) + // ReadConfigArgs reads command line arguments and returns a ServerConfig for them + ReadConfigArgs(args *argparser.ArgParseResults) (servercfg.ServerConfig, error) +} + +var _ ServerConfigReader = DoltServerConfigReader{} + +func (d DoltServerConfigReader) ReadConfigFile(cwdFS filesys.Filesys, file string) (servercfg.ServerConfig, error) { + return servercfg.YamlConfigFromFile(cwdFS, file) +} + +func (d DoltServerConfigReader) ReadConfigArgs(args *argparser.ArgParseResults) (servercfg.ServerConfig, error) { + return NewCommandLineConfig(nil, args) +} diff --git a/go/cmd/dolt/commands/sqlserver/queryist_utils.go b/go/cmd/dolt/commands/sqlserver/queryist_utils.go index c552735344..e5532889ff 100644 --- a/go/cmd/dolt/commands/sqlserver/queryist_utils.go +++ b/go/cmd/dolt/commands/sqlserver/queryist_utils.go @@ -18,6 +18,7 @@ import ( "context" sql2 "database/sql" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "io" "github.com/dolthub/go-mysql-server/sql" @@ -42,7 +43,7 @@ func BuildConnectionStringQueryist(ctx context.Context, cwdFS filesys.Filesys, c // ParseDSN currently doesn't support `/` in the db name dbName, _ := dsess.SplitRevisionDbName(dbRev) - parsedMySQLConfig, err := mysql.ParseDSN(ConnectionString(clientConfig, dbName)) + parsedMySQLConfig, err := mysql.ParseDSN(servercfg.ConnectionString(clientConfig, dbName)) if err != nil { return nil, err } diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 0e029ca06f..e40b643b9f 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -19,10 +19,10 @@ import ( "crypto/tls" "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "net" "net/http" "os" - "runtime" "strconv" "strings" "time" @@ -76,7 +76,7 @@ var ErrCouldNotLockDatabase = goerrors.NewKind("database \"%s\" is locked by ano func Serve( ctx context.Context, version string, - serverConfig ServerConfig, + serverConfig servercfg.ServerConfig, controller *svcs.Controller, dEnv *env.DoltEnv, ) (startError error, closeError error) { @@ -96,14 +96,14 @@ func Serve( } func ConfigureServices( - serverConfig ServerConfig, + serverConfig servercfg.ServerConfig, controller *svcs.Controller, version string, dEnv *env.DoltEnv, ) { ValidateConfigStep := &svcs.AnonService{ InitF: func(context.Context) error { - return ValidateConfig(serverConfig) + return servercfg.ValidateConfig(serverConfig) }, } controller.Register(ValidateConfigStep) @@ -219,6 +219,12 @@ func ConfigureServices( } controller.Register(LoadServerConfig) + serverJwksConfigs := serverConfig.JwksConfig() + jwksConfigs := make([]engine.JwksConfig, len(serverJwksConfigs)) + for i := range serverJwksConfigs { + jwksConfigs[i] = engine.JwksConfig(serverJwksConfigs[i]) + } + // Create SQL Engine with users var config *engine.SqlEngineConfig InitSqlEngineConfig := &svcs.AnonService{ @@ -233,7 +239,7 @@ func ConfigureServices( ServerHost: serverConfig.Host(), Autocommit: serverConfig.AutoCommit(), DoltTransactionCommit: serverConfig.DoltTransactionCommit(), - JwksConfig: serverConfig.JwksConfig(), + JwksConfig: jwksConfigs, SystemVariables: serverConfig.SystemVars(), ClusterController: clusterController, BinlogReplicaController: binlogreplication.DoltBinlogReplicaController, @@ -289,7 +295,7 @@ func ConfigureServices( mysqlDb.AddSuperUser(ed, config.ServerUser, "%", config.ServerPass) } } else if !privsExist { - mysqlDb.AddSuperUser(ed, defaultUser, "%", defaultPass) + mysqlDb.AddSuperUser(ed, servercfg.DefaultUser, "%", servercfg.DefaultPass) } ed.Close() @@ -514,15 +520,15 @@ func ConfigureServices( var mySQLServer *server.Server InitSQLServer := &svcs.AnonService{ InitF: func(context.Context) (err error) { - v, ok := serverConfig.(validatingServerConfig) - if ok && v.goldenMysqlConnectionString() != "" { + v, ok := serverConfig.(servercfg.ValidatingServerConfig) + if ok && v.GoldenMysqlConnectionString() != "" { mySQLServer, err = server.NewServerWithHandler( serverConf, sqlEngine.GetUnderlyingEngine(), newSessionBuilder(sqlEngine, serverConfig), metListener, func(h mysql.Handler) (mysql.Handler, error) { - return golden.NewValidatingHandler(h, v.goldenMysqlConnectionString(), logrus.StandardLogger()) + return golden.NewValidatingHandler(h, v.GoldenMysqlConnectionString(), logrus.StandardLogger()) }, ) } else { @@ -756,7 +762,7 @@ func (r *remotesapiAuth) ApiAuthorize(ctx context.Context, superUserRequired boo return true, nil } -func LoadClusterTLSConfig(cfg cluster.Config) (*tls.Config, error) { +func LoadClusterTLSConfig(cfg servercfg.ClusterConfig) (*tls.Config, error) { rcfg := cfg.RemotesAPIConfig() if rcfg.TLSKey() == "" && rcfg.TLSCert() == "" { return nil, nil @@ -782,7 +788,7 @@ func portInUse(hostPort string) bool { return false } -func newSessionBuilder(se *engine.SqlEngine, config ServerConfig) server.SessionBuilder { +func newSessionBuilder(se *engine.SqlEngine, config servercfg.ServerConfig) server.SessionBuilder { userToSessionVars := make(map[string]map[string]string) userVars := config.UserVars() for _, curr := range userVars { @@ -820,7 +826,7 @@ func newSessionBuilder(se *engine.SqlEngine, config ServerConfig) server.Session } // getConfigFromServerConfig processes ServerConfig and returns server.Config for sql-server. -func getConfigFromServerConfig(serverConfig ServerConfig) (server.Config, error) { +func getConfigFromServerConfig(serverConfig servercfg.ServerConfig) (server.Config, error) { serverConf, err := handleProtocolAndAddress(serverConfig) if err != nil { return server.Config{}, err @@ -831,14 +837,14 @@ func getConfigFromServerConfig(serverConfig ServerConfig) (server.Config, error) readTimeout := time.Duration(serverConfig.ReadTimeout()) * time.Millisecond writeTimeout := time.Duration(serverConfig.WriteTimeout()) * time.Millisecond - tlsConfig, err := LoadTLSConfig(serverConfig) + tlsConfig, err := servercfg.LoadTLSConfig(serverConfig) if err != nil { return server.Config{}, err } // if persist is 'load' we use currently set persisted global variable, // else if 'ignore' we set persisted global variable to current value from serverConfig - if serverConfig.PersistenceBehavior() == loadPerisistentGlobals { + if serverConfig.PersistenceBehavior() == servercfg.LoadPerisistentGlobals { serverConf, err = serverConf.NewConfig() if err != nil { return server.Config{}, err @@ -864,7 +870,7 @@ func getConfigFromServerConfig(serverConfig ServerConfig) (server.Config, error) } // handleProtocolAndAddress returns new server.Config object with only Protocol and Address defined. -func handleProtocolAndAddress(serverConfig ServerConfig) (server.Config, error) { +func handleProtocolAndAddress(serverConfig servercfg.ServerConfig) (server.Config, error) { serverConf := server.Config{Protocol: "tcp"} portAsString := strconv.Itoa(serverConfig.Port()) @@ -875,7 +881,7 @@ func handleProtocolAndAddress(serverConfig ServerConfig) (server.Config, error) } serverConf.Address = hostPort - sock, useSock, err := checkForUnixSocket(serverConfig) + sock, useSock, err := servercfg.CheckForUnixSocket(serverConfig) if err != nil { return server.Config{}, err } @@ -886,25 +892,6 @@ func handleProtocolAndAddress(serverConfig ServerConfig) (server.Config, error) return serverConf, nil } -// checkForUnixSocket evaluates ServerConfig for whether the unix socket is to be used or not. -// If user defined socket flag or host is 'localhost', it returns the unix socket file location -// either user-defined or the default if it was not defined. -func checkForUnixSocket(config ServerConfig) (string, bool, error) { - if config.Socket() != "" { - if runtime.GOOS == "windows" { - return "", false, fmt.Errorf("cannot define unix socket file on Windows") - } - return config.Socket(), true, nil - } else { - // if host is undefined or defined as "localhost" -> unix - if runtime.GOOS != "windows" && config.Host() == "localhost" { - return defaultUnixSocketFilePath, true, nil - } - } - - return "", false, nil -} - func getEventSchedulerStatus(status string) (eventscheduler.SchedulerStatus, error) { switch strings.ToLower(status) { case "on", "1": diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index 26b1298d9b..63675e7d6b 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -15,6 +15,7 @@ package sqlserver import ( + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "net/http" "os" "strings" @@ -166,31 +167,31 @@ func TestServerGoodParams(t *testing.T) { assert.NoError(t, env.DoltDB.Close()) }() - tests := []ServerConfig{ - DefaultServerConfig(), - DefaultServerConfig().WithHost("127.0.0.1").WithPort(15400), - DefaultServerConfig().WithHost("localhost").WithPort(15401), - //DefaultServerConfig().WithHost("::1").WithPort(15402), // Fails on Jenkins, assuming no IPv6 support - DefaultServerConfig().withUser("testusername").WithPort(15403), - DefaultServerConfig().withPassword("hunter2").WithPort(15404), - DefaultServerConfig().withTimeout(0).WithPort(15405), - DefaultServerConfig().withTimeout(5).WithPort(15406), - DefaultServerConfig().withLogLevel(LogLevel_Debug).WithPort(15407), - DefaultServerConfig().withLogLevel(LogLevel_Info).WithPort(15408), - DefaultServerConfig().withReadOnly(true).WithPort(15409), - DefaultServerConfig().withUser("testusernamE").withPassword("hunter2").withTimeout(4).WithPort(15410), - DefaultServerConfig().withAllowCleartextPasswords(true), + tests := []servercfg.ServerConfig{ + DefaultCommandLineServerConfig(), + DefaultCommandLineServerConfig().WithHost("127.0.0.1").WithPort(15400), + DefaultCommandLineServerConfig().WithHost("localhost").WithPort(15401), + //DefaultCommandLineServerConfig().WithHost("::1").WithPort(15402), // Fails on Jenkins, assuming no IPv6 support + DefaultCommandLineServerConfig().withUser("testusername").WithPort(15403), + DefaultCommandLineServerConfig().withPassword("hunter2").WithPort(15404), + DefaultCommandLineServerConfig().withTimeout(0).WithPort(15405), + DefaultCommandLineServerConfig().withTimeout(5).WithPort(15406), + DefaultCommandLineServerConfig().withLogLevel(servercfg.LogLevel_Debug).WithPort(15407), + DefaultCommandLineServerConfig().withLogLevel(servercfg.LogLevel_Info).WithPort(15408), + DefaultCommandLineServerConfig().withReadOnly(true).WithPort(15409), + DefaultCommandLineServerConfig().withUser("testusernamE").withPassword("hunter2").withTimeout(4).WithPort(15410), + DefaultCommandLineServerConfig().withAllowCleartextPasswords(true), } for _, test := range tests { - t.Run(ConfigInfo(test), func(t *testing.T) { + t.Run(servercfg.ConfigInfo(test), func(t *testing.T) { sc := svcs.NewController() - go func(config ServerConfig, sc *svcs.Controller) { + go func(config servercfg.ServerConfig, sc *svcs.Controller) { _, _ = Serve(context.Background(), "0.0.0", config, sc, env) }(test, sc) err := sc.WaitForStart() require.NoError(t, err) - conn, err := dbr.Open("mysql", ConnectionString(test, "dbname"), nil) + conn, err := dbr.Open("mysql", servercfg.ConnectionString(test, "dbname"), nil) require.NoError(t, err) err = conn.Close() require.NoError(t, err) @@ -208,7 +209,7 @@ func TestServerSelect(t *testing.T) { assert.NoError(t, env.DoltDB.Close()) }() - serverConfig := DefaultServerConfig().withLogLevel(LogLevel_Fatal).WithPort(15300) + serverConfig := DefaultCommandLineServerConfig().withLogLevel(servercfg.LogLevel_Fatal).WithPort(15300) sc := svcs.NewController() defer sc.Stop() @@ -219,7 +220,7 @@ func TestServerSelect(t *testing.T) { require.NoError(t, err) const dbName = "dolt" - conn, err := dbr.Open("mysql", ConnectionString(serverConfig, dbName), nil) + conn, err := dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) defer conn.Close() sess := conn.NewSession(nil) @@ -307,7 +308,7 @@ func TestServerSetDefaultBranch(t *testing.T) { assert.NoError(t, dEnv.DoltDB.Close()) }() - serverConfig := DefaultServerConfig().withLogLevel(LogLevel_Fatal).WithPort(15302) + serverConfig := DefaultCommandLineServerConfig().withLogLevel(servercfg.LogLevel_Fatal).WithPort(15302) sc := svcs.NewController() defer sc.Stop() @@ -321,7 +322,7 @@ func TestServerSetDefaultBranch(t *testing.T) { defaultBranch := env.DefaultInitBranch - conn, err := dbr.Open("mysql", ConnectionString(serverConfig, dbName), nil) + conn, err := dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) sess := conn.NewSession(nil) @@ -342,7 +343,7 @@ func TestServerSetDefaultBranch(t *testing.T) { runDefaultBranchTests(t, tests, conn) - conn, err = dbr.Open("mysql", ConnectionString(serverConfig, dbName), nil) + conn, err = dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) sess = conn.NewSession(nil) @@ -367,7 +368,7 @@ func TestServerSetDefaultBranch(t *testing.T) { runDefaultBranchTests(t, tests, conn) - conn, err = dbr.Open("mysql", ConnectionString(serverConfig, dbName), nil) + conn, err = dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) sess = conn.NewSession(nil) @@ -384,7 +385,7 @@ func TestServerSetDefaultBranch(t *testing.T) { runDefaultBranchTests(t, tests, conn) - conn, err = dbr.Open("mysql", ConnectionString(serverConfig, dbName), nil) + conn, err = dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) sess = conn.NewSession(nil) @@ -397,7 +398,7 @@ func TestServerSetDefaultBranch(t *testing.T) { runDefaultBranchTests(t, tests, conn) - conn, err = dbr.Open("mysql", ConnectionString(serverConfig, dbName), nil) + conn, err = dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) sess = conn.NewSession(nil) @@ -410,7 +411,7 @@ func TestServerSetDefaultBranch(t *testing.T) { runDefaultBranchTests(t, tests, conn) - conn, err = dbr.Open("mysql", ConnectionString(serverConfig, dbName), nil) + conn, err = dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) sess = conn.NewSession(nil) @@ -469,7 +470,7 @@ func TestReadReplica(t *testing.T) { // start server as read replica sc := svcs.NewController() - serverConfig := DefaultServerConfig().withLogLevel(LogLevel_Fatal).WithPort(15303) + serverConfig := DefaultCommandLineServerConfig().withLogLevel(servercfg.LogLevel_Fatal).WithPort(15303) // set socket to nil to force tcp serverConfig = serverConfig.WithHost("127.0.0.1").WithSocket("") @@ -489,7 +490,7 @@ func TestReadReplica(t *testing.T) { multiSetup.PushToRemote(sourceDbName, "remote1", "main") t.Run("read replica pulls multiple branches", func(t *testing.T) { - conn, err := dbr.Open("mysql", ConnectionString(serverConfig, readReplicaDbName), nil) + conn, err := dbr.Open("mysql", servercfg.ConnectionString(serverConfig, readReplicaDbName), nil) defer conn.Close() require.NoError(t, err) sess := conn.NewSession(nil) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index ed42c8b24c..6906f70336 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -17,6 +17,8 @@ package sqlserver import ( "context" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" + "github.com/dolthub/go-mysql-server/sql" "path/filepath" "strings" @@ -71,7 +73,7 @@ var sqlServerDocs = cli.CommandDocumentationContent{ "the server directly on the command line. If {{.EmphasisLeft}}--config {{.EmphasisRight}} is provided all" + " other command line arguments are ignored.\n\nThis is an example yaml configuration file showing all supported" + " items and their default values:\n\n" + - indentLines(ServerConfigAsYAMLConfig(DefaultServerConfig()).String()) + "\n\n" + ` + indentLines(servercfg.ServerConfigAsYAMLConfig(DefaultCommandLineServerConfig()).String()) + "\n\n" + ` SUPPORTED CONFIG FILE FIELDS: {{.EmphasisLeft}}data_dir{{.EmphasisRight}}: A directory where the server will load dolt databases to serve, and create new ones. Defaults to the current directory. @@ -147,7 +149,7 @@ func (cmd SqlServerCmd) ArgParser() *argparser.ArgParser { } func (cmd SqlServerCmd) ArgParserWithName(name string) *argparser.ArgParser { - serverConfig := DefaultServerConfig() + serverConfig := DefaultCommandLineServerConfig() ap := argparser.NewArgParserWithVariableArgs(name) ap.SupportsString(configFileFlag, "", "file", "When provided configuration is taken from the yaml config file and all command line parameters are ignored.") @@ -241,12 +243,12 @@ func StartServer(ctx context.Context, versionStr, commandStr string, args []stri return err } - err = ApplySystemVariables(serverConfig) + err = servercfg.ApplySystemVariables(serverConfig, sql.SystemVariables) if err != nil { return err } - cli.PrintErrf("Starting server with Config %v\n", ConfigInfo(serverConfig)) + cli.PrintErrf("Starting server with Config %v\n", servercfg.ConfigInfo(serverConfig)) startError, closeError := Serve(ctx, versionStr, serverConfig, controller, dEnv) if startError != nil { @@ -260,7 +262,7 @@ func StartServer(ctx context.Context, versionStr, commandStr string, args []stri } // ServerConfigFromArgs returns a ServerConfig from the given args -func ServerConfigFromArgs(ap *argparser.ArgParser, help cli.UsagePrinter, args []string, dEnv *env.DoltEnv) (ServerConfig, error) { +func ServerConfigFromArgs(ap *argparser.ArgParser, help cli.UsagePrinter, args []string, dEnv *env.DoltEnv) (servercfg.ServerConfig, error) { return ServerConfigFromArgsWithReader(ap, help, args, dEnv, DoltServerConfigReader{}) } @@ -271,7 +273,7 @@ func ServerConfigFromArgsWithReader( args []string, dEnv *env.DoltEnv, reader ServerConfigReader, -) (ServerConfig, error) { +) (servercfg.ServerConfig, error) { apr := cli.ParseArgsOrDie(ap, args, help) if err := validateSqlServerArgs(apr); err != nil { cli.PrintErrln(color.RedString(err.Error())) @@ -292,7 +294,7 @@ func ServerConfigFromArgsWithReader( // getServerConfig returns ServerConfig that is set either from yaml file if given, if not it is set with values defined // on command line. Server config variables not defined are set to default values. -func getServerConfig(cwdFS filesys.Filesys, apr *argparser.ArgParseResults, reader ServerConfigReader) (ServerConfig, error) { +func getServerConfig(cwdFS filesys.Filesys, apr *argparser.ArgParseResults, reader ServerConfigReader) (servercfg.ServerConfig, error) { cfgFile, ok := apr.GetValue(configFileFlag) if !ok { return reader.ReadConfigArgs(apr) @@ -305,7 +307,7 @@ func getServerConfig(cwdFS filesys.Filesys, apr *argparser.ArgParseResults, read // if command line user argument was given, override the config file's user and password if user, hasUser := apr.GetValue(commands.UserFlag); hasUser { - if wcfg, ok := cfg.(WritableServerConfig); ok { + if wcfg, ok := cfg.(servercfg.WritableServerConfig); ok { pass, _ := apr.GetValue(passwordFlag) wcfg.SetUserName(user) wcfg.SetPassword(pass) @@ -313,7 +315,7 @@ func getServerConfig(cwdFS filesys.Filesys, apr *argparser.ArgParseResults, read } if connStr, ok := apr.GetValue(goldenMysqlConn); ok { - if yamlCfg, ok := cfg.(YAMLConfig); ok { + if yamlCfg, ok := cfg.(servercfg.YAMLConfig); ok { cli.Println(connStr) yamlCfg.GoldenMysqlConn = &connStr } @@ -326,19 +328,19 @@ func getServerConfig(cwdFS filesys.Filesys, apr *argparser.ArgParseResults, read // is a little confusing, but it is because the client and server use the same configuration struct. The main difference // between this method and getServerConfig is that this method required a cli.UserPassword argument. It is created by // prompting the user, and we don't want the server to follow that code path. -func GetClientConfig(cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argparser.ArgParseResults) (ServerConfig, error) { +func GetClientConfig(cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argparser.ArgParseResults) (servercfg.ServerConfig, error) { cfgFile, hasCfgFile := apr.GetValue(configFileFlag) if !hasCfgFile { return NewCommandLineConfig(creds, apr) } - var yamlCfg YAMLConfig - cfg, err := YamlConfigFromFile(cwdFS, cfgFile) + var yamlCfg servercfg.YAMLConfig + cfg, err := servercfg.YamlConfigFromFile(cwdFS, cfgFile) if err != nil { return nil, err } - yamlCfg = cfg.(YAMLConfig) + yamlCfg = cfg.(servercfg.YAMLConfig) // if command line user argument was given, replace yaml's user and password if creds.Specified { @@ -355,7 +357,7 @@ func GetClientConfig(cwdFS filesys.Filesys, creds *cli.UserPassword, apr *argpar } // setupDoltConfig updates the given server config with where to create .doltcfg directory -func setupDoltConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResults, config ServerConfig) error { +func setupDoltConfig(dEnv *env.DoltEnv, apr *argparser.ArgParseResults, config servercfg.ServerConfig) error { if _, ok := apr.GetValue(configFileFlag); ok { return nil } diff --git a/go/cmd/dolt/commands/sqlserver/minver_test.go b/go/libraries/doltcore/servercfg/minver_test.go similarity index 98% rename from go/cmd/dolt/commands/sqlserver/minver_test.go rename to go/libraries/doltcore/servercfg/minver_test.go index 4a5c7529da..fa00c5fa1a 100644 --- a/go/cmd/dolt/commands/sqlserver/minver_test.go +++ b/go/libraries/doltcore/servercfg/minver_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqlserver +package servercfg import ( "testing" diff --git a/go/cmd/dolt/commands/sqlserver/serverconfig.go b/go/libraries/doltcore/servercfg/serverconfig.go similarity index 68% rename from go/cmd/dolt/commands/sqlserver/serverconfig.go rename to go/libraries/doltcore/servercfg/serverconfig.go index 5c4da2c600..999a03cc7b 100644 --- a/go/cmd/dolt/commands/sqlserver/serverconfig.go +++ b/go/libraries/doltcore/servercfg/serverconfig.go @@ -12,21 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqlserver +package servercfg import ( "crypto/tls" "errors" "fmt" "net" + "path/filepath" + "runtime" "strings" - - "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/cmd/dolt/commands/engine" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/cluster" - "github.com/dolthub/dolt/go/libraries/utils/argparser" - "github.com/dolthub/dolt/go/libraries/utils/filesys" ) // LogLevel defines the available levels of logging for the server. @@ -42,35 +37,39 @@ const ( ) const ( - defaultHost = "localhost" - defaultPort = 3306 - defaultUser = "root" - defaultPass = "" - defaultTimeout = 8 * 60 * 60 * 1000 // 8 hours, same as MySQL - defaultReadOnly = false - defaultLogLevel = LogLevel_Info - defaultAutoCommit = true - defaultDoltTransactionCommit = false - defaultMaxConnections = 100 - defaultQueryParallelism = 0 - defaultPersistenceBahavior = loadPerisistentGlobals - defaultDataDir = "." - defaultCfgDir = ".doltcfg" - defaultPrivilegeFilePath = "privileges.db" - defaultBranchControlFilePath = "branch_control.db" - defaultMetricsHost = "" - defaultMetricsPort = -1 - defaultAllowCleartextPasswords = false - defaultUnixSocketFilePath = "/tmp/mysql.sock" - defaultMaxLoggedQueryLen = 0 - defaultEncodeLoggedQuery = false + DefaultHost = "localhost" + DefaultPort = 3306 + DefaultUser = "root" + DefaultPass = "" + DefaultTimeout = 8 * 60 * 60 * 1000 // 8 hours, same as MySQL + DefaultReadOnly = false + DefaultLogLevel = LogLevel_Info + DefaultAutoCommit = true + DefaultDoltTransactionCommit = false + DefaultMaxConnections = 100 + DefaultQueryParallelism = 0 + DefaultPersistenceBahavior = LoadPerisistentGlobals + DefaultDataDir = "." + DefaultCfgDir = ".doltcfg" + DefaultPrivilegeFilePath = "privileges.db" + DefaultBranchControlFilePath = "branch_control.db" + DefaultMetricsHost = "" + DefaultMetricsPort = -1 + DefaultAllowCleartextPasswords = false + DefaultUnixSocketFilePath = "/tmp/mysql.sock" + DefaultMaxLoggedQueryLen = 0 + DefaultEncodeLoggedQuery = false ) const ( - ignorePeristentGlobals = "ignore" - loadPerisistentGlobals = "load" + IgnorePeristentGlobals = "ignore" + LoadPerisistentGlobals = "load" ) +func ptr[T any](t T) *T { + return &t +} + // String returns the string representation of the log level. func (level LogLevel) String() string { switch level { @@ -91,6 +90,35 @@ func (level LogLevel) String() string { } } +type ClusterConfig interface { + StandbyRemotes() []ClusterStandbyRemoteConfig + BootstrapRole() string + BootstrapEpoch() int + RemotesAPIConfig() ClusterRemotesAPIConfig +} + +type ClusterRemotesAPIConfig interface { + Address() string + Port() int + TLSKey() string + TLSCert() string + TLSCA() string + ServerNameURLMatches() []string + ServerNameDNSMatches() []string +} + +type ClusterStandbyRemoteConfig interface { + Name() string + RemoteURLTemplate() string +} + +type ServerJwksConfig struct { + Name string `yaml:"name"` + LocationUrl string `yaml:"location_url"` + Claims map[string]string `yaml:"claims"` + FieldsToLog []string `yaml:"fields_to_log"` +} + // ServerConfig contains all of the configurable options for the MySQL-compatible server. type ServerConfig interface { // Host returns the domain that the server will run on. Accepts an IPv4 or IPv6 address, in addition to localhost. @@ -154,9 +182,9 @@ type ServerConfig interface { // UserVars is an array containing user specific session variables UserVars() []UserSessionVars // SystemVars is a map setting global SQL system variables. For example, `secure_file_priv`. - SystemVars() engine.SystemVariables + SystemVars() map[string]interface{} // JwksConfig is an array containing jwks config - JwksConfig() []engine.JwksConfig + JwksConfig() []ServerJwksConfig // AllowCleartextPasswords is true if the server should accept cleartext passwords. AllowCleartextPasswords() bool // Socket is a path to the unix socket file @@ -169,13 +197,45 @@ type ServerConfig interface { // RemotesapiReadOnly is true if the remotesapi interface should be read only. RemotesapiReadOnly() *bool // ClusterConfig is the configuration for clustering in this sql-server. - ClusterConfig() cluster.Config + ClusterConfig() ClusterConfig // EventSchedulerStatus is the configuration for enabling or disabling the event scheduler in this server. EventSchedulerStatus() string // ValueSet returns whether the value string provided was explicitly set in the config ValueSet(value string) bool } +// DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values. +func DefaultServerConfig() ServerConfig { + return &YAMLConfig{ + LogLevelStr: ptr(string(DefaultLogLevel)), + MaxQueryLenInLogs: ptr(DefaultMaxLoggedQueryLen), + EncodeLoggedQuery: ptr(DefaultEncodeLoggedQuery), + BehaviorConfig: BehaviorYAMLConfig{ + ReadOnly: ptr(DefaultReadOnly), + AutoCommit: ptr(DefaultAutoCommit), + PersistenceBehavior: ptr(DefaultPersistenceBahavior), + DoltTransactionCommit: ptr(DefaultDoltTransactionCommit), + }, + UserConfig: UserYAMLConfig{ + Name: ptr(""), + Password: ptr(""), + }, + ListenerConfig: ListenerYAMLConfig{ + HostStr: ptr(DefaultHost), + PortNumber: ptr(DefaultPort), + MaxConnections: ptr(uint64(DefaultMaxConnections)), + ReadTimeoutMillis: ptr(uint64(DefaultTimeout)), + WriteTimeoutMillis: ptr(uint64(DefaultTimeout)), + AllowCleartextPasswords: ptr(DefaultAllowCleartextPasswords), + }, + PerformanceConfig: PerformanceYAMLConfig{QueryParallelism: ptr(DefaultQueryParallelism)}, + DataDirStr: ptr(DefaultDataDir), + CfgDirStr: ptr(filepath.Join(DefaultDataDir, DefaultCfgDir)), + PrivilegeFile: ptr(filepath.Join(DefaultDataDir, DefaultCfgDir, DefaultPrivilegeFilePath)), + BranchControlFile: ptr(filepath.Join(DefaultDataDir, DefaultCfgDir, DefaultBranchControlFilePath)), + } +} + // WritableServerConfig is a ServerConfig that support overwriting certain values. type WritableServerConfig interface { ServerConfig @@ -185,19 +245,11 @@ type WritableServerConfig interface { SetPassword(string) } -type validatingServerConfig interface { +type ValidatingServerConfig interface { ServerConfig // goldenMysqlConnectionString returns a connection string for a mysql // instance that can be used to validate query results - goldenMysqlConnectionString() string -} - -// ServerConfigReader is an interface for reading a ServerConfig from a file or command line arguments. -type ServerConfigReader interface { - // ReadConfigFile reads a config file and returns a ServerConfig for it - ReadConfigFile(cwdFS filesys.Filesys, file string) (ServerConfig, error) - // ReadConfigArgs reads command line arguments and returns a ServerConfig for them - ReadConfigArgs(args *argparser.ArgParseResults) (ServerConfig, error) + GoldenMysqlConnectionString() string } // ValidateConfig returns an `error` if any field is not valid. @@ -221,37 +273,41 @@ func ValidateConfig(config ServerConfig) error { } const ( - maxConnectionsKey = "max_connections" - readTimeoutKey = "net_read_timeout" - writeTimeoutKey = "net_write_timeout" - eventSchedulerKey = "event_scheduler" + MaxConnectionsKey = "max_connections" + ReadTimeoutKey = "net_read_timeout" + WriteTimeoutKey = "net_write_timeout" + EventSchedulerKey = "event_scheduler" ) +type SystemVariableTarget interface { + SetGlobal(name string, value interface{}) error +} + // ApplySystemVariables sets the global system variables based on the given `ServerConfig`. -func ApplySystemVariables(cfg ServerConfig) error { - if cfg.ValueSet(maxConnectionsKey) { - err := sql.SystemVariables.SetGlobal("max_connections", cfg.MaxConnections()) +func ApplySystemVariables(cfg ServerConfig, sysVarTarget SystemVariableTarget) error { + if cfg.ValueSet(MaxConnectionsKey) { + err := sysVarTarget.SetGlobal("max_connections", cfg.MaxConnections()) if err != nil { return err } } - if cfg.ValueSet(readTimeoutKey) { - err := sql.SystemVariables.SetGlobal("net_read_timeout", cfg.ReadTimeout()) + if cfg.ValueSet(ReadTimeoutKey) { + err := sysVarTarget.SetGlobal("net_read_timeout", cfg.ReadTimeout()) if err != nil { return err } } - if cfg.ValueSet(writeTimeoutKey) { - err := sql.SystemVariables.SetGlobal("net_write_timeout", cfg.WriteTimeout()) + if cfg.ValueSet(WriteTimeoutKey) { + err := sysVarTarget.SetGlobal("net_write_timeout", cfg.WriteTimeout()) if err != nil { return err } } - if cfg.ValueSet(eventSchedulerKey) { - err := sql.SystemVariables.SetGlobal("event_scheduler", cfg.EventSchedulerStatus()) + if cfg.ValueSet(EventSchedulerKey) { + err := sysVarTarget.SetGlobal("event_scheduler", cfg.EventSchedulerStatus()) if err != nil { return err } @@ -260,7 +316,7 @@ func ApplySystemVariables(cfg ServerConfig) error { return nil } -func ValidateClusterConfig(config cluster.Config) error { +func ValidateClusterConfig(config ClusterConfig) error { if config == nil { return nil } @@ -316,7 +372,7 @@ func ConnectionString(config ServerConfig, database string) string { // ConfigInfo returns a summary of some of the config which contains some of the more important information func ConfigInfo(config ServerConfig) string { socket := "" - sock, useSock, err := checkForUnixSocket(config) + sock, useSock, err := CheckForUnixSocket(config) if err != nil { panic(err) } @@ -344,16 +400,21 @@ func LoadTLSConfig(cfg ServerConfig) (*tls.Config, error) { }, nil } -// DoltServerConfigReader is the default implementation of ServerConfigReader suitable for parsing Dolt config files -// and command line options. -type DoltServerConfigReader struct{} +// CheckForUnixSocket evaluates ServerConfig for whether the unix socket is to be used or not. +// If user defined socket flag or host is 'localhost', it returns the unix socket file location +// either user-defined or the default if it was not defined. +func CheckForUnixSocket(config ServerConfig) (string, bool, error) { + if config.Socket() != "" { + if runtime.GOOS == "windows" { + return "", false, fmt.Errorf("cannot define unix socket file on Windows") + } + return config.Socket(), true, nil + } else { + // if host is undefined or defined as "localhost" -> unix + if runtime.GOOS != "windows" && config.Host() == "localhost" { + return DefaultUnixSocketFilePath, true, nil + } + } -var _ ServerConfigReader = DoltServerConfigReader{} - -func (d DoltServerConfigReader) ReadConfigFile(cwdFS filesys.Filesys, file string) (ServerConfig, error) { - return YamlConfigFromFile(cwdFS, file) -} - -func (d DoltServerConfigReader) ReadConfigArgs(args *argparser.ArgParseResults) (ServerConfig, error) { - return NewCommandLineConfig(nil, args) + return "", false, nil } diff --git a/go/cmd/dolt/commands/sqlserver/testdata/chain_cert.pem b/go/libraries/doltcore/servercfg/testdata/chain_cert.pem similarity index 100% rename from go/cmd/dolt/commands/sqlserver/testdata/chain_cert.pem rename to go/libraries/doltcore/servercfg/testdata/chain_cert.pem diff --git a/go/cmd/dolt/commands/sqlserver/testdata/chain_key.pem b/go/libraries/doltcore/servercfg/testdata/chain_key.pem similarity index 100% rename from go/cmd/dolt/commands/sqlserver/testdata/chain_key.pem rename to go/libraries/doltcore/servercfg/testdata/chain_key.pem diff --git a/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt similarity index 75% rename from go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt rename to go/libraries/doltcore/servercfg/testdata/minver_validation.txt index 870ee618c9..0fb115b661 100644 --- a/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt +++ b/go/libraries/doltcore/servercfg/testdata/minver_validation.txt @@ -4,17 +4,17 @@ LogLevelStr *string 0.0.0 log_level,omitempty MaxQueryLenInLogs *int 0.0.0 max_logged_query_len,omitempty EncodeLoggedQuery *bool 0.0.0 encode_logged_query,omitempty -BehaviorConfig sqlserver.BehaviorYAMLConfig 0.0.0 behavior +BehaviorConfig servercfg.BehaviorYAMLConfig 0.0.0 behavior -ReadOnly *bool 0.0.0 read_only -AutoCommit *bool 0.0.0 autocommit -PersistenceBehavior *string 0.0.0 persistence_behavior -DisableClientMultiStatements *bool 0.0.0 disable_client_multi_statements -DoltTransactionCommit *bool 0.0.0 dolt_transaction_commit -EventSchedulerStatus *string 1.17.0 event_scheduler,omitempty -UserConfig sqlserver.UserYAMLConfig 0.0.0 user +UserConfig servercfg.UserYAMLConfig 0.0.0 user -Name *string 0.0.0 name -Password *string 0.0.0 password -ListenerConfig sqlserver.ListenerYAMLConfig 0.0.0 listener +ListenerConfig servercfg.ListenerYAMLConfig 0.0.0 listener -HostStr *string 0.0.0 host -PortNumber *int 0.0.0 port -MaxConnections *uint64 0.0.0 max_connections @@ -25,24 +25,24 @@ ListenerConfig sqlserver.ListenerYAMLConfig 0.0.0 listener -RequireSecureTransport *bool 0.0.0 require_secure_transport -AllowCleartextPasswords *bool 0.0.0 allow_cleartext_passwords -Socket *string 0.0.0 socket,omitempty -PerformanceConfig sqlserver.PerformanceYAMLConfig 0.0.0 performance +PerformanceConfig servercfg.PerformanceYAMLConfig 0.0.0 performance -QueryParallelism *int 0.0.0 query_parallelism DataDirStr *string 0.0.0 data_dir,omitempty CfgDirStr *string 0.0.0 cfg_dir,omitempty -MetricsConfig sqlserver.MetricsYAMLConfig 0.0.0 metrics +MetricsConfig servercfg.MetricsYAMLConfig 0.0.0 metrics -Labels map[string]string 0.0.0 labels -Host *string 0.0.0 host -Port *int 0.0.0 port -RemotesapiConfig sqlserver.RemotesapiYAMLConfig 0.0.0 remotesapi +RemotesapiConfig servercfg.RemotesapiYAMLConfig 0.0.0 remotesapi -Port_ *int 0.0.0 port,omitempty -ReadOnly_ *bool 1.30.5 read_only,omitempty -ClusterCfg *sqlserver.ClusterYAMLConfig 0.0.0 cluster,omitempty --StandbyRemotes_ []sqlserver.StandbyRemoteYAMLConfig 0.0.0 standby_remotes +ClusterCfg *servercfg.ClusterYAMLConfig 0.0.0 cluster,omitempty +-StandbyRemotes_ []servercfg.StandbyRemoteYAMLConfig 0.0.0 standby_remotes --Name_ string 0.0.0 name --RemoteURLTemplate_ string 0.0.0 remote_url_template -BootstrapRole_ string 0.0.0 bootstrap_role -BootstrapEpoch_ int 0.0.0 bootstrap_epoch --RemotesAPI sqlserver.ClusterRemotesAPIYAMLConfig 0.0.0 remotesapi +-RemotesAPI servercfg.ClusterRemotesAPIYAMLConfig 0.0.0 remotesapi --Addr_ string 0.0.0 address --Port_ int 0.0.0 port --TLSKey_ string 0.0.0 tls_key @@ -52,11 +52,11 @@ ClusterCfg *sqlserver.ClusterYAMLConfig 0.0.0 cluster,omitempty --DNSMatches []string 0.0.0 server_name_dns PrivilegeFile *string 0.0.0 privilege_file,omitempty BranchControlFile *string 0.0.0 branch_control_file,omitempty -Vars []sqlserver.UserSessionVars 0.0.0 user_session_vars +Vars []servercfg.UserSessionVars 0.0.0 user_session_vars -Name string 0.0.0 name -Vars map[string]string 0.0.0 vars -SystemVars_ *engine.SystemVariables 1.11.1 system_variables,omitempty -Jwks []engine.JwksConfig 0.0.0 jwks +SystemVars_ map[string]interface{} 1.11.1 system_variables,omitempty +Jwks []servercfg.ServerJwksConfig 0.0.0 jwks -Name string 0.0.0 name -LocationUrl string 0.0.0 location_url -Claims map[string]string 0.0.0 claims diff --git a/go/cmd/dolt/commands/sqlserver/testdata/selfsigned_cert.pem b/go/libraries/doltcore/servercfg/testdata/selfsigned_cert.pem similarity index 100% rename from go/cmd/dolt/commands/sqlserver/testdata/selfsigned_cert.pem rename to go/libraries/doltcore/servercfg/testdata/selfsigned_cert.pem diff --git a/go/cmd/dolt/commands/sqlserver/testdata/selfsigned_key.pem b/go/libraries/doltcore/servercfg/testdata/selfsigned_key.pem similarity index 100% rename from go/cmd/dolt/commands/sqlserver/testdata/selfsigned_key.pem rename to go/libraries/doltcore/servercfg/testdata/selfsigned_key.pem diff --git a/go/cmd/dolt/commands/sqlserver/yaml_config.go b/go/libraries/doltcore/servercfg/yaml_config.go similarity index 85% rename from go/cmd/dolt/commands/sqlserver/yaml_config.go rename to go/libraries/doltcore/servercfg/yaml_config.go index f17d6310a1..4f246d51cb 100644 --- a/go/cmd/dolt/commands/sqlserver/yaml_config.go +++ b/go/libraries/doltcore/servercfg/yaml_config.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqlserver +package servercfg import ( "fmt" @@ -23,15 +23,9 @@ import ( "gopkg.in/yaml.v2" - "github.com/dolthub/dolt/go/cmd/dolt/commands/engine" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/cluster" "github.com/dolthub/dolt/go/libraries/utils/filesys" ) -func strPtr(s string) *string { - return &s -} - func nillableStrPtr(s string) *string { if s == "" { return nil @@ -46,14 +40,6 @@ func nillableBoolPtr(b bool) *bool { return &b } -func boolPtr(b bool) *bool { - return &b -} - -func uint64Ptr(n uint64) *uint64 { - return &n -} - func nillableIntPtr(n int) *int { if n == 0 { return nil @@ -61,10 +47,6 @@ func nillableIntPtr(n int) *int { return &n } -func intPtr(n int) *int { - return &n -} - // BehaviorYAMLConfig contains server configuration regarding how the server should behave type BehaviorYAMLConfig struct { ReadOnly *bool `yaml:"read_only"` @@ -156,14 +138,14 @@ type YAMLConfig struct { PrivilegeFile *string `yaml:"privilege_file,omitempty"` BranchControlFile *string `yaml:"branch_control_file,omitempty"` // TODO: Rename to UserVars_ - Vars []UserSessionVars `yaml:"user_session_vars"` - SystemVars_ *engine.SystemVariables `yaml:"system_variables,omitempty" minver:"1.11.1"` - Jwks []engine.JwksConfig `yaml:"jwks"` - GoldenMysqlConn *string `yaml:"golden_mysql_conn,omitempty"` + Vars []UserSessionVars `yaml:"user_session_vars"` + SystemVars_ map[string]interface{} `yaml:"system_variables,omitempty" minver:"1.11.1"` + Jwks []ServerJwksConfig `yaml:"jwks"` + GoldenMysqlConn *string `yaml:"golden_mysql_conn,omitempty"` } var _ ServerConfig = YAMLConfig{} -var _ validatingServerConfig = YAMLConfig{} +var _ ValidatingServerConfig = YAMLConfig{} var _ WritableServerConfig = &YAMLConfig{} func NewYamlConfig(configFileData []byte) (*YAMLConfig, error) { @@ -192,29 +174,29 @@ func YamlConfigFromFile(fs filesys.Filesys, path string) (ServerConfig, error) { } func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig { - systemVars := cfg.SystemVars() + systemVars := map[string]interface{}(cfg.SystemVars()) return &YAMLConfig{ - LogLevelStr: strPtr(string(cfg.LogLevel())), + LogLevelStr: ptr(string(cfg.LogLevel())), MaxQueryLenInLogs: nillableIntPtr(cfg.MaxLoggedQueryLen()), EncodeLoggedQuery: nillableBoolPtr(cfg.ShouldEncodeLoggedQuery()), BehaviorConfig: BehaviorYAMLConfig{ - boolPtr(cfg.ReadOnly()), - boolPtr(cfg.AutoCommit()), - strPtr(cfg.PersistenceBehavior()), - boolPtr(cfg.DisableClientMultiStatements()), - boolPtr(cfg.DoltTransactionCommit()), - strPtr(cfg.EventSchedulerStatus()), + ptr(cfg.ReadOnly()), + ptr(cfg.AutoCommit()), + ptr(cfg.PersistenceBehavior()), + ptr(cfg.DisableClientMultiStatements()), + ptr(cfg.DoltTransactionCommit()), + ptr(cfg.EventSchedulerStatus()), }, UserConfig: UserYAMLConfig{ - Name: strPtr(cfg.User()), - Password: strPtr(cfg.Password()), + Name: ptr(cfg.User()), + Password: ptr(cfg.Password()), }, ListenerConfig: ListenerYAMLConfig{ - strPtr(cfg.Host()), - intPtr(cfg.Port()), - uint64Ptr(cfg.MaxConnections()), - uint64Ptr(cfg.ReadTimeout()), - uint64Ptr(cfg.WriteTimeout()), + ptr(cfg.Host()), + ptr(cfg.Port()), + ptr(cfg.MaxConnections()), + ptr(cfg.ReadTimeout()), + ptr(cfg.WriteTimeout()), nillableStrPtr(cfg.TLSKey()), nillableStrPtr(cfg.TLSCert()), nillableBoolPtr(cfg.RequireSecureTransport()), @@ -224,27 +206,27 @@ func ServerConfigAsYAMLConfig(cfg ServerConfig) *YAMLConfig { PerformanceConfig: PerformanceYAMLConfig{ QueryParallelism: nillableIntPtr(cfg.QueryParallelism()), }, - DataDirStr: strPtr(cfg.DataDir()), - CfgDirStr: strPtr(cfg.CfgDir()), + DataDirStr: ptr(cfg.DataDir()), + CfgDirStr: ptr(cfg.CfgDir()), MetricsConfig: MetricsYAMLConfig{ Labels: cfg.MetricsLabels(), Host: nillableStrPtr(cfg.MetricsHost()), - Port: intPtr(cfg.MetricsPort()), + Port: ptr(cfg.MetricsPort()), }, RemotesapiConfig: RemotesapiYAMLConfig{ Port_: cfg.RemotesapiPort(), ReadOnly_: cfg.RemotesapiReadOnly(), }, ClusterCfg: clusterConfigAsYAMLConfig(cfg.ClusterConfig()), - PrivilegeFile: strPtr(cfg.PrivilegeFilePath()), - BranchControlFile: strPtr(cfg.BranchControlFilePath()), - SystemVars_: &systemVars, + PrivilegeFile: ptr(cfg.PrivilegeFilePath()), + BranchControlFile: ptr(cfg.BranchControlFilePath()), + SystemVars_: systemVars, Vars: cfg.UserVars(), Jwks: cfg.JwksConfig(), } } -func clusterConfigAsYAMLConfig(config cluster.Config) *ClusterYAMLConfig { +func clusterConfigAsYAMLConfig(config ClusterConfig) *ClusterYAMLConfig { if config == nil { return nil } @@ -300,7 +282,7 @@ func (cfg YAMLConfig) String() string { // Host returns the domain that the server will run on. Accepts an IPv4 or IPv6 address, in addition to localhost. func (cfg YAMLConfig) Host() string { if cfg.ListenerConfig.HostStr == nil { - return defaultHost + return DefaultHost } return *cfg.ListenerConfig.HostStr @@ -309,7 +291,7 @@ func (cfg YAMLConfig) Host() string { // Port returns the port that the server will run on. The valid range is [1024, 65535]. func (cfg YAMLConfig) Port() int { if cfg.ListenerConfig.PortNumber == nil { - return defaultPort + return DefaultPort } return *cfg.ListenerConfig.PortNumber @@ -318,7 +300,7 @@ func (cfg YAMLConfig) Port() int { // ReadTimeout returns the read timeout in milliseconds. func (cfg YAMLConfig) ReadTimeout() uint64 { if cfg.ListenerConfig.ReadTimeoutMillis == nil { - return defaultTimeout + return DefaultTimeout } return *cfg.ListenerConfig.ReadTimeoutMillis @@ -327,7 +309,7 @@ func (cfg YAMLConfig) ReadTimeout() uint64 { // WriteTimeout returns the write timeout in milliseconds. func (cfg YAMLConfig) WriteTimeout() uint64 { if cfg.ListenerConfig.WriteTimeoutMillis == nil { - return defaultTimeout + return DefaultTimeout } return *cfg.ListenerConfig.WriteTimeoutMillis @@ -336,7 +318,7 @@ func (cfg YAMLConfig) WriteTimeout() uint64 { // User returns the username that connecting clients must use. func (cfg YAMLConfig) User() string { if cfg.UserConfig.Name == nil { - return defaultUser + return DefaultUser } return *cfg.UserConfig.Name @@ -353,7 +335,7 @@ func (cfg *YAMLConfig) SetPassword(s string) { // Password returns the password that connecting clients must use. func (cfg YAMLConfig) Password() string { if cfg.UserConfig.Password == nil { - return defaultPass + return DefaultPass } return *cfg.UserConfig.Password @@ -362,7 +344,7 @@ func (cfg YAMLConfig) Password() string { // ReadOnly returns whether the server will only accept read statements or all statements. func (cfg YAMLConfig) ReadOnly() bool { if cfg.BehaviorConfig.ReadOnly == nil { - return defaultReadOnly + return DefaultReadOnly } return *cfg.BehaviorConfig.ReadOnly @@ -371,7 +353,7 @@ func (cfg YAMLConfig) ReadOnly() bool { // AutoCommit defines the value of the @@autocommit session variable used on every connection func (cfg YAMLConfig) AutoCommit() bool { if cfg.BehaviorConfig.AutoCommit == nil { - return defaultAutoCommit + return DefaultAutoCommit } return *cfg.BehaviorConfig.AutoCommit @@ -381,7 +363,7 @@ func (cfg YAMLConfig) AutoCommit() bool { // commits to be automatically created when a SQL transaction is committed. func (cfg YAMLConfig) DoltTransactionCommit() bool { if cfg.BehaviorConfig.DoltTransactionCommit == nil { - return defaultDoltTransactionCommit + return DefaultDoltTransactionCommit } return *cfg.BehaviorConfig.DoltTransactionCommit @@ -390,7 +372,7 @@ func (cfg YAMLConfig) DoltTransactionCommit() bool { // LogLevel returns the level of logging that the server will use. func (cfg YAMLConfig) LogLevel() LogLevel { if cfg.LogLevelStr == nil { - return defaultLogLevel + return DefaultLogLevel } return LogLevel(*cfg.LogLevelStr) @@ -399,7 +381,7 @@ func (cfg YAMLConfig) LogLevel() LogLevel { // MaxConnections returns the maximum number of simultaneous connections the server will allow. The default is 1 func (cfg YAMLConfig) MaxConnections() uint64 { if cfg.ListenerConfig.MaxConnections == nil { - return defaultMaxConnections + return DefaultMaxConnections } return *cfg.ListenerConfig.MaxConnections @@ -423,7 +405,7 @@ func (cfg YAMLConfig) MetricsLabels() map[string]string { func (cfg YAMLConfig) MetricsHost() string { if cfg.MetricsConfig.Host == nil { - return defaultMetricsHost + return DefaultMetricsHost } return *cfg.MetricsConfig.Host @@ -431,7 +413,7 @@ func (cfg YAMLConfig) MetricsHost() string { func (cfg YAMLConfig) MetricsPort() int { if cfg.MetricsConfig.Host == nil { - return defaultMetricsPort + return DefaultMetricsPort } return *cfg.MetricsConfig.Port @@ -451,7 +433,7 @@ func (cfg YAMLConfig) PrivilegeFilePath() string { if cfg.PrivilegeFile != nil { return *cfg.PrivilegeFile } - return filepath.Join(cfg.CfgDir(), defaultPrivilegeFilePath) + return filepath.Join(cfg.CfgDir(), DefaultPrivilegeFilePath) } // BranchControlFilePath returns the path to the file which contains the branch control permissions. @@ -459,7 +441,7 @@ func (cfg YAMLConfig) BranchControlFilePath() string { if cfg.BranchControlFile != nil { return *cfg.BranchControlFile } - return filepath.Join(cfg.CfgDir(), defaultBranchControlFilePath) + return filepath.Join(cfg.CfgDir(), DefaultBranchControlFilePath) } // UserVars is an array containing user specific session variables @@ -471,16 +453,16 @@ func (cfg YAMLConfig) UserVars() []UserSessionVars { return nil } -func (cfg YAMLConfig) SystemVars() engine.SystemVariables { +func (cfg YAMLConfig) SystemVars() map[string]interface{} { if cfg.SystemVars_ == nil { - return engine.SystemVariables{} + return map[string]interface{}{} } - return *cfg.SystemVars_ + return cfg.SystemVars_ } -// JwksConfig is JSON Web Key Set config, and used to validate a user authed with a jwt (JSON Web Token). -func (cfg YAMLConfig) JwksConfig() []engine.JwksConfig { +// wksConfig is JSON Web Key Set config, and used to validate a user authed with a jwt (JSON Web Token). +func (cfg YAMLConfig) JwksConfig() []ServerJwksConfig { if cfg.Jwks != nil { return cfg.Jwks } @@ -489,7 +471,7 @@ func (cfg YAMLConfig) JwksConfig() []engine.JwksConfig { func (cfg YAMLConfig) AllowCleartextPasswords() bool { if cfg.ListenerConfig.AllowCleartextPasswords == nil { - return defaultAllowCleartextPasswords + return DefaultAllowCleartextPasswords } return *cfg.ListenerConfig.AllowCleartextPasswords } @@ -497,7 +479,7 @@ func (cfg YAMLConfig) AllowCleartextPasswords() bool { // QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer func (cfg YAMLConfig) QueryParallelism() int { if cfg.PerformanceConfig.QueryParallelism == nil { - return defaultQueryParallelism + return DefaultQueryParallelism } return *cfg.PerformanceConfig.QueryParallelism @@ -532,7 +514,7 @@ func (cfg YAMLConfig) RequireSecureTransport() bool { // is less than 0 then the queries will be omitted from the logs completely func (cfg YAMLConfig) MaxLoggedQueryLen() int { if cfg.MaxQueryLenInLogs == nil { - return defaultMaxLoggedQueryLen + return DefaultMaxLoggedQueryLen } return *cfg.MaxQueryLenInLogs @@ -540,7 +522,7 @@ func (cfg YAMLConfig) MaxLoggedQueryLen() int { func (cfg YAMLConfig) ShouldEncodeLoggedQuery() bool { if cfg.EncodeLoggedQuery == nil { - return defaultEncodeLoggedQuery + return DefaultEncodeLoggedQuery } return *cfg.EncodeLoggedQuery @@ -549,7 +531,7 @@ func (cfg YAMLConfig) ShouldEncodeLoggedQuery() bool { // PersistenceBehavior is "load" if we include persisted system globals on server init func (cfg YAMLConfig) PersistenceBehavior() string { if cfg.BehaviorConfig.PersistenceBehavior == nil { - return loadPerisistentGlobals + return LoadPerisistentGlobals } return *cfg.BehaviorConfig.PersistenceBehavior } @@ -559,7 +541,7 @@ func (cfg YAMLConfig) DataDir() string { if cfg.DataDirStr != nil { return *cfg.DataDirStr } - return defaultDataDir + return DefaultDataDir } // CfgDir is the path to a directory to use to store the dolt configuration files. @@ -567,7 +549,7 @@ func (cfg YAMLConfig) CfgDir() string { if cfg.CfgDirStr != nil { return *cfg.CfgDirStr } - return filepath.Join(cfg.DataDir(), defaultCfgDir) + return filepath.Join(cfg.DataDir(), DefaultCfgDir) } // Socket is a path to the unix socket file @@ -577,19 +559,19 @@ func (cfg YAMLConfig) Socket() string { } // if defined but empty -> default if *cfg.ListenerConfig.Socket == "" { - return defaultUnixSocketFilePath + return DefaultUnixSocketFilePath } return *cfg.ListenerConfig.Socket } -func (cfg YAMLConfig) goldenMysqlConnectionString() (s string) { +func (cfg YAMLConfig) GoldenMysqlConnectionString() (s string) { if cfg.GoldenMysqlConn != nil { s = *cfg.GoldenMysqlConn } return } -func (cfg YAMLConfig) ClusterConfig() cluster.Config { +func (cfg YAMLConfig) ClusterConfig() ClusterConfig { if cfg.ClusterCfg == nil { return nil } @@ -630,8 +612,8 @@ func (c StandbyRemoteYAMLConfig) RemoteURLTemplate() string { return c.RemoteURLTemplate_ } -func (c *ClusterYAMLConfig) StandbyRemotes() []cluster.StandbyRemoteConfig { - ret := make([]cluster.StandbyRemoteConfig, len(c.StandbyRemotes_)) +func (c *ClusterYAMLConfig) StandbyRemotes() []ClusterStandbyRemoteConfig { + ret := make([]ClusterStandbyRemoteConfig, len(c.StandbyRemotes_)) for i := range c.StandbyRemotes_ { ret[i] = c.StandbyRemotes_[i] } @@ -646,7 +628,7 @@ func (c *ClusterYAMLConfig) BootstrapEpoch() int { return c.BootstrapEpoch_ } -func (c *ClusterYAMLConfig) RemotesAPIConfig() cluster.RemotesAPIConfig { +func (c *ClusterYAMLConfig) RemotesAPIConfig() ClusterRemotesAPIConfig { return c.RemotesAPI } @@ -690,13 +672,13 @@ func (c ClusterRemotesAPIYAMLConfig) ServerNameDNSMatches() []string { func (cfg YAMLConfig) ValueSet(value string) bool { switch value { - case readTimeoutKey: + case ReadTimeoutKey: return cfg.ListenerConfig.ReadTimeoutMillis != nil - case writeTimeoutKey: + case WriteTimeoutKey: return cfg.ListenerConfig.WriteTimeoutMillis != nil - case maxConnectionsKey: + case MaxConnectionsKey: return cfg.ListenerConfig.MaxConnections != nil - case eventSchedulerKey: + case EventSchedulerKey: return cfg.BehaviorConfig.EventSchedulerStatus != nil } return false diff --git a/go/cmd/dolt/commands/sqlserver/yaml_config_test.go b/go/libraries/doltcore/servercfg/yaml_config_test.go similarity index 89% rename from go/cmd/dolt/commands/sqlserver/yaml_config_test.go rename to go/libraries/doltcore/servercfg/yaml_config_test.go index 82035788e5..c2e0e1a630 100644 --- a/go/cmd/dolt/commands/sqlserver/yaml_config_test.go +++ b/go/libraries/doltcore/servercfg/yaml_config_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqlserver +package servercfg import ( "testing" @@ -20,8 +20,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" - - "github.com/dolthub/dolt/go/cmd/dolt/commands/engine" ) var trueValue = true @@ -92,19 +90,19 @@ jwks: expected.BehaviorConfig.DoltTransactionCommit = &trueValue expected.CfgDirStr = nillableStrPtr("") - expected.PrivilegeFile = strPtr("some other nonsense") - expected.BranchControlFile = strPtr("third nonsense") + expected.PrivilegeFile = ptr("some other nonsense") + expected.BranchControlFile = ptr("third nonsense") expected.MetricsConfig = MetricsYAMLConfig{ - Host: strPtr("123.45.67.89"), - Port: intPtr(9091), + Host: ptr("123.45.67.89"), + Port: ptr(9091), Labels: map[string]string{ "label1": "value1", "label2": "2", "label3": "true", }, } - expected.DataDirStr = strPtr("some nonsense") + expected.DataDirStr = ptr("some nonsense") expected.SystemVars_ = nil expected.Vars = []UserSessionVars{ { @@ -124,7 +122,7 @@ jwks: }, }, } - expected.Jwks = []engine.JwksConfig{ + expected.Jwks = []ServerJwksConfig{ { Name: "jwks_name", LocationUrl: "https://website.com", @@ -325,26 +323,26 @@ func TestYAMLConfigDefaults(t *testing.T) { err := yaml.Unmarshal([]byte{}, &cfg) require.NoError(t, err) - assert.Equal(t, defaultHost, cfg.Host()) - assert.Equal(t, defaultPort, cfg.Port()) - assert.Equal(t, defaultUser, cfg.User()) - assert.Equal(t, defaultPass, cfg.Password()) - assert.Equal(t, uint64(defaultTimeout), cfg.WriteTimeout()) - assert.Equal(t, uint64(defaultTimeout), cfg.ReadTimeout()) - assert.Equal(t, defaultReadOnly, cfg.ReadOnly()) - assert.Equal(t, defaultLogLevel, cfg.LogLevel()) - assert.Equal(t, defaultAutoCommit, cfg.AutoCommit()) - assert.Equal(t, defaultDoltTransactionCommit, cfg.DoltTransactionCommit()) - assert.Equal(t, uint64(defaultMaxConnections), cfg.MaxConnections()) + assert.Equal(t, DefaultHost, cfg.Host()) + assert.Equal(t, DefaultPort, cfg.Port()) + assert.Equal(t, DefaultUser, cfg.User()) + assert.Equal(t, DefaultPass, cfg.Password()) + assert.Equal(t, uint64(DefaultTimeout), cfg.WriteTimeout()) + assert.Equal(t, uint64(DefaultTimeout), cfg.ReadTimeout()) + assert.Equal(t, DefaultReadOnly, cfg.ReadOnly()) + assert.Equal(t, DefaultLogLevel, cfg.LogLevel()) + assert.Equal(t, DefaultAutoCommit, cfg.AutoCommit()) + assert.Equal(t, DefaultDoltTransactionCommit, cfg.DoltTransactionCommit()) + assert.Equal(t, uint64(DefaultMaxConnections), cfg.MaxConnections()) assert.Equal(t, "", cfg.TLSKey()) assert.Equal(t, "", cfg.TLSCert()) assert.Equal(t, false, cfg.RequireSecureTransport()) assert.Equal(t, false, cfg.AllowCleartextPasswords()) assert.Equal(t, false, cfg.DisableClientMultiStatements()) - assert.Equal(t, defaultMetricsHost, cfg.MetricsHost()) - assert.Equal(t, defaultMetricsPort, cfg.MetricsPort()) + assert.Equal(t, DefaultMetricsHost, cfg.MetricsHost()) + assert.Equal(t, DefaultMetricsPort, cfg.MetricsPort()) assert.Nil(t, cfg.MetricsConfig.Labels) - assert.Equal(t, defaultAllowCleartextPasswords, cfg.AllowCleartextPasswords()) + assert.Equal(t, DefaultAllowCleartextPasswords, cfg.AllowCleartextPasswords()) assert.Nil(t, cfg.RemotesapiPort()) c, err := LoadTLSConfig(cfg) diff --git a/go/libraries/doltcore/sqle/cluster/config.go b/go/libraries/doltcore/sqle/cluster/config.go deleted file mode 100644 index 8f3c9e358d..0000000000 --- a/go/libraries/doltcore/sqle/cluster/config.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cluster - -type Config interface { - StandbyRemotes() []StandbyRemoteConfig - BootstrapRole() string - BootstrapEpoch() int - RemotesAPIConfig() RemotesAPIConfig -} - -type RemotesAPIConfig interface { - Address() string - Port() int - TLSKey() string - TLSCert() string - TLSCA() string - ServerNameURLMatches() []string - ServerNameDNSMatches() []string -} - -type StandbyRemoteConfig interface { - Name() string - RemoteURLTemplate() string -} diff --git a/go/libraries/doltcore/sqle/cluster/controller.go b/go/libraries/doltcore/sqle/cluster/controller.go index fb28f9bc4f..47433a0d11 100644 --- a/go/libraries/doltcore/sqle/cluster/controller.go +++ b/go/libraries/doltcore/sqle/cluster/controller.go @@ -22,6 +22,7 @@ import ( "crypto/x509" "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "net/http" "net/url" "os" @@ -74,7 +75,7 @@ type databaseDropReplication struct { } type Controller struct { - cfg Config + cfg servercfg.ClusterConfig persistentCfg config.ReadWriteConfig role Role epoch int @@ -130,7 +131,7 @@ const ( DoltClusterRemoteApiAudience = "dolt-cluster-remote-api.dolthub.com" ) -func NewController(lgr *logrus.Logger, cfg Config, pCfg config.ReadWriteConfig) (*Controller, error) { +func NewController(lgr *logrus.Logger, cfg servercfg.ClusterConfig, pCfg config.ReadWriteConfig) (*Controller, error) { if cfg == nil { return nil, nil } @@ -485,7 +486,7 @@ func (c *Controller) persistVariables() error { return c.persistentCfg.SetStrings(toset) } -func applyBootstrapClusterConfig(lgr *logrus.Logger, cfg Config, pCfg config.ReadWriteConfig) (Role, int, error) { +func applyBootstrapClusterConfig(lgr *logrus.Logger, cfg servercfg.ClusterConfig, pCfg config.ReadWriteConfig) (Role, int, error) { toset := make(map[string]string) persistentRole := pCfg.GetStringOrDefault(dsess.DoltClusterRoleVariable, "") var roleFromPersistentConfig bool diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go index 0a7981c33f..8719619253 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_server_test.go @@ -17,6 +17,7 @@ package enginetest import ( "context" gosql "database/sql" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "math/rand" "runtime" "strings" @@ -554,7 +555,7 @@ func testMultiSessionScriptTests(t *testing.T, tests []queries.ScriptTest) { // the block. func testSerialSessionScriptTests(t *testing.T, tests []queries.ScriptTest) { dEnv := dtestutils.CreateTestEnv() - serverConfig := sqlserver.DefaultServerConfig() + serverConfig := sqlserver.DefaultCommandLineServerConfig() rand.Seed(time.Now().UnixNano()) port := 15403 + rand.Intn(25) serverConfig = serverConfig.WithPort(port) @@ -660,9 +661,9 @@ func assertResultsEqual(t *testing.T, expected []sql.Row, rows *gosql.Rows) { } // startServer will start sql-server with given host, unix socket file path and whether to use specific port, which is defined randomly. -func startServer(t *testing.T, withPort bool, host string, unixSocketPath string) (*env.DoltEnv, *svcs.Controller, sqlserver.ServerConfig) { +func startServer(t *testing.T, withPort bool, host string, unixSocketPath string) (*env.DoltEnv, *svcs.Controller, servercfg.ServerConfig) { dEnv := dtestutils.CreateTestEnv() - serverConfig := sqlserver.DefaultServerConfig() + serverConfig := sqlserver.DefaultCommandLineServerConfig() if withPort { rand.Seed(time.Now().UnixNano()) port := 15403 + rand.Intn(25) @@ -679,7 +680,7 @@ func startServer(t *testing.T, withPort bool, host string, unixSocketPath string return dEnv, onEnv, config } -func startServerOnEnv(t *testing.T, serverConfig sqlserver.ServerConfig, dEnv *env.DoltEnv) (*svcs.Controller, sqlserver.ServerConfig) { +func startServerOnEnv(t *testing.T, serverConfig servercfg.ServerConfig, dEnv *env.DoltEnv) (*svcs.Controller, servercfg.ServerConfig) { sc := svcs.NewController() go func() { _, _ = sqlserver.Serve(context.Background(), "0.0.0", serverConfig, sc, dEnv) @@ -691,9 +692,9 @@ func startServerOnEnv(t *testing.T, serverConfig sqlserver.ServerConfig, dEnv *e } // newConnection takes sqlserver.serverConfig and opens a connection, and will return that connection with a new session -func newConnection(t *testing.T, serverConfig sqlserver.ServerConfig) (*dbr.Connection, *dbr.Session) { +func newConnection(t *testing.T, serverConfig servercfg.ServerConfig) (*dbr.Connection, *dbr.Session) { const dbName = "dolt" - conn, err := dbr.Open("mysql", sqlserver.ConnectionString(serverConfig, dbName), nil) + conn, err := dbr.Open("mysql", servercfg.ConnectionString(serverConfig, dbName), nil) require.NoError(t, err) sess := conn.NewSession(nil) return conn, sess @@ -709,7 +710,7 @@ func TestDoltServerRunningUnixSocket(t *testing.T) { dEnv, sc, serverConfig := startServer(t, false, "", defaultUnixSocketPath) sc.WaitForStart() defer dEnv.DoltDB.Close() - require.True(t, strings.Contains(sqlserver.ConnectionString(serverConfig, "dolt"), "unix")) + require.True(t, strings.Contains(servercfg.ConnectionString(serverConfig, "dolt"), "unix")) // default unix socket connection works localConn, localSess := newConnection(t, serverConfig) @@ -719,21 +720,21 @@ func TestDoltServerRunningUnixSocket(t *testing.T) { t.Run("connecting to local server with tcp connections", func(t *testing.T) { // connect with port defined - serverConfigWithPortOnly := sqlserver.DefaultServerConfig().WithPort(3306) + serverConfigWithPortOnly := sqlserver.DefaultCommandLineServerConfig().WithPort(3306) conn1, sess1 := newConnection(t, serverConfigWithPortOnly) rows1, err := sess1.Query("select 1") require.NoError(t, err) assertResultsEqual(t, []sql.Row{{1}}, rows1) // connect with host defined - serverConfigWithPortandHost := sqlserver.DefaultServerConfig().WithHost("127.0.0.1") + serverConfigWithPortandHost := sqlserver.DefaultCommandLineServerConfig().WithHost("127.0.0.1") conn2, sess2 := newConnection(t, serverConfigWithPortandHost) rows2, err := sess2.Query("select 1") require.NoError(t, err) assertResultsEqual(t, []sql.Row{{1}}, rows2) // connect with port and host defined - serverConfigWithPortandHost1 := sqlserver.DefaultServerConfig().WithPort(3306).WithHost("0.0.0.0") + serverConfigWithPortandHost1 := sqlserver.DefaultCommandLineServerConfig().WithPort(3306).WithHost("0.0.0.0") conn3, sess3 := newConnection(t, serverConfigWithPortandHost1) rows3, err := sess3.Query("select 1") require.NoError(t, err) @@ -757,11 +758,11 @@ func TestDoltServerRunningUnixSocket(t *testing.T) { dEnv, tcpSc, tcpServerConfig := startServer(t, true, "0.0.0.0", "") tcpSc.WaitForStart() defer dEnv.DoltDB.Close() - require.False(t, strings.Contains(sqlserver.ConnectionString(tcpServerConfig, "dolt"), "unix")) + require.False(t, strings.Contains(servercfg.ConnectionString(tcpServerConfig, "dolt"), "unix")) t.Run("host and port specified, there should not be unix socket created", func(t *testing.T) { // unix socket connection should fail - localServerConfig := sqlserver.DefaultServerConfig().WithSocket(defaultUnixSocketPath) + localServerConfig := sqlserver.DefaultCommandLineServerConfig().WithSocket(defaultUnixSocketPath) conn, sess := newConnection(t, localServerConfig) _, err := sess.Query("select 1") require.Error(t, err) diff --git a/go/libraries/utils/minver/minvertesting.go b/go/libraries/utils/minver/minvertesting.go index e86cf2b0af..138db09755 100644 --- a/go/libraries/utils/minver/minvertesting.go +++ b/go/libraries/utils/minver/minvertesting.go @@ -55,7 +55,7 @@ func FieldInfoFromLine(l string) (FieldInfo, error) { func FieldInfoFromStructField(field reflect.StructField, depth int) FieldInfo { info := FieldInfo{ Name: strings.Repeat("-", depth) + field.Name, - TypeStr: field.Type.String(), + TypeStr: strings.Replace(field.Type.String(), " ", "", -1), MinVer: field.Tag.Get("minver"), YamlTag: field.Tag.Get("yaml"), } diff --git a/go/performance/replicationbench/async_replica_test.go b/go/performance/replicationbench/async_replica_test.go index 0d290afd4e..3ae424605a 100644 --- a/go/performance/replicationbench/async_replica_test.go +++ b/go/performance/replicationbench/async_replica_test.go @@ -17,11 +17,11 @@ package serverbench import ( "context" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "runtime/pprof" "strings" "testing" - srv "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" "github.com/dolthub/dolt/go/libraries/doltcore/dtestutils/testcommands" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" @@ -59,7 +59,7 @@ func BenchmarkAsyncPushOnWrite(b *testing.B) { func benchmarkAsyncPush(b *testing.B, test serverTest) { var dEnv *env.DoltEnv - var cfg srv.ServerConfig + var cfg servercfg.ServerConfig ctx := context.Background() // setup @@ -86,7 +86,7 @@ func benchmarkAsyncPush(b *testing.B, test serverTest) { }) } -func getAsyncEnvAndConfig(ctx context.Context, b *testing.B) (dEnv *env.DoltEnv, cfg srv.ServerConfig) { +func getAsyncEnvAndConfig(ctx context.Context, b *testing.B) (dEnv *env.DoltEnv, cfg servercfg.ServerConfig) { multiSetup := testcommands.NewMultiRepoTestSetup(b.Fatal) multiSetup.NewDB("dolt_bench") @@ -122,7 +122,7 @@ listener: write_timeout_millis: 28800000 `, writerName, multiSetup.DbPaths[writerName], port)) - cfg, err := srv.NewYamlConfig(yaml) + cfg, err := servercfg.NewYamlConfig(yaml) if err != nil { b.Fatal(err) } diff --git a/go/performance/replicationbench/replica_test.go b/go/performance/replicationbench/replica_test.go index 38f755babc..8fd3b6191a 100644 --- a/go/performance/replicationbench/replica_test.go +++ b/go/performance/replicationbench/replica_test.go @@ -17,6 +17,7 @@ package serverbench import ( "context" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "os" "path" "runtime" @@ -75,7 +76,7 @@ func BenchmarkPushOnWrite(b *testing.B) { func benchmarkServer(b *testing.B, test serverTest) { var dEnv *env.DoltEnv - var cfg srv.ServerConfig + var cfg servercfg.ServerConfig ctx := context.Background() // setup @@ -109,7 +110,7 @@ const ( email = "name@fake.horse" ) -func getEnvAndConfig(ctx context.Context, b *testing.B) (dEnv *env.DoltEnv, cfg srv.ServerConfig) { +func getEnvAndConfig(ctx context.Context, b *testing.B) (dEnv *env.DoltEnv, cfg servercfg.ServerConfig) { multiSetup := testcommands.NewMultiRepoTestSetup(b.Fatal) multiSetup.NewDB("dolt_bench") @@ -145,7 +146,7 @@ listener: write_timeout_millis: 28800000 `, writerName, multiSetup.DbPaths[writerName], port)) - cfg, err := srv.NewYamlConfig(yaml) + cfg, err := servercfg.NewYamlConfig(yaml) if err != nil { b.Fatal(err) } @@ -163,7 +164,7 @@ func getProfFile(b *testing.B) *os.File { return f } -func executeServerQueries(ctx context.Context, b *testing.B, dEnv *env.DoltEnv, cfg srv.ServerConfig, queries []query) { +func executeServerQueries(ctx context.Context, b *testing.B, dEnv *env.DoltEnv, cfg servercfg.ServerConfig, queries []query) { sc := svcs.NewController() eg, ctx := errgroup.WithContext(ctx) @@ -198,8 +199,8 @@ func executeServerQueries(ctx context.Context, b *testing.B, dEnv *env.DoltEnv, } } -func executeQuery(cfg srv.ServerConfig, q query) error { - cs := srv.ConnectionString(cfg, database) +func executeQuery(cfg servercfg.ServerConfig, q query) error { + cs := servercfg.ConnectionString(cfg, database) conn, err := dbr.Open("mysql", cs, nil) if err != nil { return err diff --git a/go/performance/serverbench/bench_test.go b/go/performance/serverbench/bench_test.go index d415bd5247..4d6105bbc2 100644 --- a/go/performance/serverbench/bench_test.go +++ b/go/performance/serverbench/bench_test.go @@ -17,6 +17,7 @@ package serverbench import ( "context" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "os" "path" "runtime" @@ -76,7 +77,7 @@ func BenchmarkServerExample(b *testing.B) { func benchmarkServer(b *testing.B, test serverTest) { var dEnv *env.DoltEnv - var cfg srv.ServerConfig + var cfg servercfg.ServerConfig ctx := context.Background() // setup @@ -110,7 +111,7 @@ const ( email = "name@fake.horse" ) -func getEnvAndConfig(ctx context.Context, b *testing.B) (dEnv *env.DoltEnv, cfg srv.ServerConfig) { +func getEnvAndConfig(ctx context.Context, b *testing.B) (dEnv *env.DoltEnv, cfg servercfg.ServerConfig) { tmp := b.TempDir() //b.Logf("db directory: %s", tmp) dbDir := path.Join(tmp, database) @@ -157,7 +158,7 @@ listener: write_timeout_millis: 28800000 `, database, dbDir, port)) - cfg, err = srv.NewYamlConfig(yaml) + cfg, err = servercfg.NewYamlConfig(yaml) if err != nil { b.Fatal(err) } @@ -175,7 +176,7 @@ func getProfFile(b *testing.B) *os.File { return f } -func executeServerQueries(ctx context.Context, b *testing.B, dEnv *env.DoltEnv, cfg srv.ServerConfig, queries []query) { +func executeServerQueries(ctx context.Context, b *testing.B, dEnv *env.DoltEnv, cfg servercfg.ServerConfig, queries []query) { sc := svcs.NewController() eg, ctx := errgroup.WithContext(ctx) @@ -210,8 +211,8 @@ func executeServerQueries(ctx context.Context, b *testing.B, dEnv *env.DoltEnv, } } -func executeQuery(cfg srv.ServerConfig, q query) error { - cs := srv.ConnectionString(cfg, database) +func executeQuery(cfg servercfg.ServerConfig, q query) error { + cs := servercfg.ConnectionString(cfg, database) conn, err := dbr.Open("mysql", cs, nil) if err != nil { return err diff --git a/go/utils/genminver_validation/main.go b/go/utils/genminver_validation/main.go index 8c788ed73c..a6e57a26e9 100644 --- a/go/utils/genminver_validation/main.go +++ b/go/utils/genminver_validation/main.go @@ -19,7 +19,7 @@ import ( "log" "os" - "github.com/dolthub/dolt/go/cmd/dolt/commands/sqlserver" + "github.com/dolthub/dolt/go/libraries/doltcore/servercfg" "github.com/dolthub/dolt/go/libraries/utils/minver" ) @@ -30,7 +30,7 @@ func main() { outFile := os.Args[1] - err := minver.GenValidationFile(&sqlserver.YAMLConfig{}, outFile) + err := minver.GenValidationFile(&servercfg.YAMLConfig{}, outFile) if err != nil { log.Fatal(err) }