go/libraries/doltcore/sqle/cluster: Add support for configured tls_{cert,key,ca} on the cluster.remotesapi.

For now, this is server-side TLS, not mTLS.

If a tls_ca is configured, the certificates in that file are PEM encoded. They
are the only trusted roots for server certificate verification on both gRPC and
HTTP connections. If tls_ca is set, no server name verification is done for
now.

go/cmd/dolt/commands/credcmds/check.go

@@ -136,7 +136,7 @@ func loadCred(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) (creds.DoltCred
 }
 
 func checkCredAndPrintSuccess(ctx context.Context, dEnv *env.DoltEnv, dc creds.DoltCreds, endpoint string) errhand.VerboseError {
-	endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
+	endpoint, opts, _, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
 		Endpoint: endpoint,
 		Creds:    dc,
 	})

go/cmd/dolt/commands/credcmds/import.go

@@ -161,7 +161,7 @@ func updateProfileWithCredentials(ctx context.Context, dEnv *env.DoltEnv, c cred
 	host := dEnv.Config.GetStringOrDefault(env.RemotesApiHostKey, env.DefaultRemotesApiHost)
 	port := dEnv.Config.GetStringOrDefault(env.RemotesApiHostPortKey, env.DefaultRemotesApiPort)
 	hostAndPort := fmt.Sprintf("%s:%s", host, port)
-	endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
+	endpoint, opts, _, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
 		Endpoint: hostAndPort,
 		Creds:    c,
 	})

go/cmd/dolt/commands/login.go

@@ -238,7 +238,7 @@ func openBrowserForCredsAdd(dc creds.DoltCreds, loginUrl string) {
 }
 
 func getCredentialsClient(dEnv *env.DoltEnv, dc creds.DoltCreds, authEndpoint string, insecure bool) (remotesapi.CredentialsServiceClient, errhand.VerboseError) {
-	endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
+	endpoint, opts, _, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
 		Endpoint: authEndpoint,
 		Creds:    dc,
 		Insecure: insecure,

go/cmd/dolt/commands/send_metrics.go

@@ -151,7 +151,7 @@ func getGRPCEmitter(dEnv *env.DoltEnv) *events.GrpcEmitter {
 	}
 
 	hostAndPort := fmt.Sprintf("%s:%d", host, port)
-	endpoint, opts, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
+	endpoint, opts, _, err := dEnv.GetGRPCDialParams(grpcendpoint.Config{
 		Endpoint: hostAndPort,
 		Insecure: insecure,
 	})

go/cmd/dolt/commands/sqlserver/server.go

@@ -16,6 +16,7 @@ package sqlserver
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
@@ -261,12 +262,22 @@ func Serve(
 			args := clusterController.RemoteSrvServerArgs(remoteSrvSqlCtx, remotesrv.ServerArgs{
 				Logger: logrus.NewEntry(lgr),
 			})
+
+			clusterRemoteSrvTLSConfig, err := LoadClusterTLSConfig(serverConfig.ClusterConfig())
+			if err != nil {
+				lgr.Errorf("error starting remotesapi server for cluster config, could not load tls config: %v", err)
+				startError = err
+				return
+			}
+			args.TLSConfig = clusterRemoteSrvTLSConfig
+
 			clusterRemoteSrv, err = remotesrv.NewServer(args)
 			if err != nil {
 				lgr.Errorf("error creating remotesapi server on port %d: %v", *serverConfig.RemotesapiPort(), err)
 				startError = err
 				return
 			}
+
 			listeners, err := clusterRemoteSrv.Listeners()
 			if err != nil {
 				lgr.Errorf("error starting remotesapi server listeners for cluster config on port %d: %v", clusterController.RemoteSrvPort(), err)
@@ -325,6 +336,22 @@ func Serve(
 	return
 }
 
+func LoadClusterTLSConfig(cfg cluster.Config) (*tls.Config, error) {
+	rcfg := cfg.RemotesAPIConfig()
+	if rcfg.TLSKey() == "" && rcfg.TLSCert() == "" {
+		return nil, nil
+	}
+	c, err := tls.LoadX509KeyPair(rcfg.TLSCert(), rcfg.TLSKey())
+	if err != nil {
+		return nil, err
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{
+			c,
+		},
+	}, nil
+}
+
 func portInUse(hostPort string) bool {
 	timeout := time.Second
 	conn, _ := net.DialTimeout("tcp", hostPort, timeout)

go/cmd/dolt/commands/sqlserver/serverconfig.go

@@ -523,6 +523,12 @@ func ValidateClusterConfig(config cluster.Config) error {
 	if config.RemotesAPIConfig().Port() < 0 || config.RemotesAPIConfig().Port() > 65535 {
 		return fmt.Errorf("cluster: remotesapi: port: is not in range 0-65535: %d", config.RemotesAPIConfig().Port())
 	}
+	if config.RemotesAPIConfig().TLSKey() == "" && config.RemotesAPIConfig().TLSCert() != "" {
+		return fmt.Errorf("cluster: remotesapi: tls_key: must supply a tls_key if you supply a tls_cert")
+	}
+	if config.RemotesAPIConfig().TLSKey() != "" && config.RemotesAPIConfig().TLSCert() == "" {
+		return fmt.Errorf("cluster: remotesapi: tls_cert: must supply a tls_cert if you supply a tls_key")
+	}
 	return nil
 }
 

go/cmd/dolt/commands/sqlserver/yaml_config.go

@@ -517,9 +517,24 @@ func (c *ClusterYAMLConfig) RemotesAPIConfig() cluster.RemotesAPIConfig {
 }
 
 type clusterRemotesAPIYAMLConfig struct {
-	P int `yaml:"port"`
+	Port_    int    `yaml:"port"`
+	TLSKey_  string `yaml:"tls_key"`
+	TLSCert_ string `yaml:"tls_cert"`
+	TLSCA_   string `yaml:"tls_ca"`
 }
 
 func (c clusterRemotesAPIYAMLConfig) Port() int {
-	return c.P
+	return c.Port_
+}
+
+func (c clusterRemotesAPIYAMLConfig) TLSKey() string {
+	return c.TLSKey_
+}
+
+func (c clusterRemotesAPIYAMLConfig) TLSCert() string {
+	return c.TLSCert_
+}
+
+func (c clusterRemotesAPIYAMLConfig) TLSCA() string {
+	return c.TLSCA_
 }

go/libraries/doltcore/dbfactory/grpc.go

@@ -36,7 +36,7 @@ var GRPCDialProviderParam = "__DOLT__grpc_dial_provider"
 
 // GRPCDialProvider is an interface for getting a *grpc.ClientConn.
 type GRPCDialProvider interface {
-	GetGRPCDialParams(grpcendpoint.Config) (string, []grpc.DialOption, error)
+	GetGRPCDialParams(grpcendpoint.Config) (string, []grpc.DialOption, grpcendpoint.HTTPFetcher, error)
 }
 
 // DoldRemoteFactory is a DBFactory implementation for creating databases backed by a remote server that implements the
@@ -84,7 +84,7 @@ func (fact DoltRemoteFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFo
 var NoCachingParameter = "__dolt__NO_CACHING"
 
 func (fact DoltRemoteFactory) newChunkStore(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}, dp GRPCDialProvider) (chunks.ChunkStore, error) {
-	endpoint, opts, err := dp.GetGRPCDialParams(grpcendpoint.Config{
+	endpoint, opts, httpfetcher, err := dp.GetGRPCDialParams(grpcendpoint.Config{
 		Endpoint:     urlObj.Host,
 		Insecure:     fact.insecure,
 		WithEnvCreds: true,
@@ -106,6 +106,7 @@ func (fact DoltRemoteFactory) newChunkStore(ctx context.Context, nbf *types.Noms
 	if err != nil {
 		return nil, fmt.Errorf("could not access dolt url '%s': %w", urlObj.String(), err)
 	}
+	cs = cs.WithHTTPFetcher(httpfetcher)
 
 	if _, ok := params[NoCachingParameter]; ok {
 		cs = cs.WithNoopChunkCache()

go/libraries/doltcore/env/environment.go

@@ -831,7 +831,7 @@ func (dEnv *DoltEnv) UserRPCCreds() (creds.DoltCreds, bool, error) {
 }
 
 // GetGRPCDialParams implements dbfactory.GRPCDialProvider
-func (dEnv *DoltEnv) GetGRPCDialParams(config grpcendpoint.Config) (string, []grpc.DialOption, error) {
+func (dEnv *DoltEnv) GetGRPCDialParams(config grpcendpoint.Config) (string, []grpc.DialOption, grpcendpoint.HTTPFetcher, error) {
 	return NewGRPCDialProviderFromDoltEnv(dEnv).GetGRPCDialParams(config)
 }
 

go/libraries/doltcore/env/grpc_dial_provider.go

@@ -16,6 +16,7 @@ package env
 
 import (
 	"crypto/tls"
+	"net/http"
 	"runtime"
 	"strings"
 	"unicode"
@@ -50,7 +51,7 @@ func NewGRPCDialProviderFromDoltEnv(dEnv *DoltEnv) *GRPCDialProvider {
 }
 
 // GetGRPCDialParms implements dbfactory.GRPCDialProvider
-func (p GRPCDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (string, []grpc.DialOption, error) {
+func (p GRPCDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (string, []grpc.DialOption, grpcendpoint.HTTPFetcher, error) {
 	endpoint := config.Endpoint
 	if strings.IndexRune(endpoint, ':') == -1 {
 		if config.Insecure {
@@ -60,8 +61,20 @@ func (p GRPCDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (string,
 		}
 	}
 
+	var httpfetcher grpcendpoint.HTTPFetcher = http.DefaultClient
+
 	var opts []grpc.DialOption
-	if config.Insecure {
+	if config.TLSConfig != nil {
+		tc := credentials.NewTLS(config.TLSConfig)
+		opts = append(opts, grpc.WithTransportCredentials(tc))
+
+		httpfetcher = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig:   config.TLSConfig,
+				ForceAttemptHTTP2: true,
+			},
+		}
+	} else if config.Insecure {
 		opts = append(opts, grpc.WithInsecure())
 	} else {
 		tc := credentials.NewTLS(&tls.Config{})
@@ -76,14 +89,14 @@ func (p GRPCDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (string,
 	} else if config.WithEnvCreds {
 		rpcCreds, err := p.getRPCCreds()
 		if err != nil {
-			return "", nil, err
+			return "", nil, nil, err
 		}
 		if rpcCreds != nil {
 			opts = append(opts, grpc.WithPerRPCCredentials(rpcCreds))
 		}
 	}
 
-	return endpoint, opts, nil
+	return endpoint, opts, httpfetcher, nil
 }
 
 // getRPCCreds returns any RPC credentials available to this dial provider. If a DoltEnv has been configured

go/libraries/doltcore/grpcendpoint/config.go

@@ -15,6 +15,9 @@
 package grpcendpoint
 
 import (
+	"crypto/tls"
+	"net/http"
+
 	"google.golang.org/grpc/credentials"
 )
 
@@ -23,4 +26,12 @@ type Config struct {
 	Insecure     bool
 	Creds        credentials.PerRPCCredentials
 	WithEnvCreds bool
+
+	// If non-nil, this is used for transport level security in the dial
+	// options, instead of a default option based on `Insecure`.
+	TLSConfig *tls.Config
+}
+
+type HTTPFetcher interface {
+	Do(req *http.Request) (*http.Response, error)
 }

go/libraries/doltcore/remotesrv/grpc.go

@@ -45,11 +45,11 @@ type RemoteChunkStore struct {
 	HttpHost   string
 	httpScheme string
 
-	csCache  DBCache
-	bucket   string
-	fs       filesys.Filesys
-	lgr      *logrus.Entry
-	sealer   Sealer
+	csCache DBCache
+	bucket  string
+	fs      filesys.Filesys
+	lgr     *logrus.Entry
+	sealer  Sealer
 	remotesapi.UnimplementedChunkStoreServiceServer
 }
 

go/libraries/doltcore/sqle/cluster/config.go

@@ -23,6 +23,9 @@ type Config interface {
 
 type RemotesAPIConfig interface {
 	Port() int
+	TLSKey() string
+	TLSCert() string
+	TLSCA() string
 }
 
 type StandbyRemoteConfig interface {

go/libraries/doltcore/sqle/cluster/controller.go

@@ -193,7 +193,7 @@ func (c *Controller) applyCommitHooks(ctx context.Context, name string, bt *sql.
 }
 
 func (c *Controller) gRPCDialProvider(denv *env.DoltEnv) dbfactory.GRPCDialProvider {
-	return grpcDialProvider{env.NewGRPCDialProviderFromDoltEnv(denv), &c.cinterceptor}
+	return grpcDialProvider{env.NewGRPCDialProviderFromDoltEnv(denv), &c.cinterceptor, c.cfg.RemotesAPIConfig().TLSCA()}
 }
 
 func (c *Controller) RegisterStoredProcedures(store procedurestore) {

go/libraries/doltcore/sqle/cluster/dialprovider.go

@@ -15,6 +15,10 @@
 package cluster
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"io/ioutil"
 	"time"
 
 	"google.golang.org/grpc"
@@ -30,15 +34,21 @@ import (
 // - client interceptors for transmitting our replication role.
 // - do not use environment credentials. (for now).
 type grpcDialProvider struct {
-	orig dbfactory.GRPCDialProvider
-	ci   *clientinterceptor
+	orig   dbfactory.GRPCDialProvider
+	ci     *clientinterceptor
+	caPath string
 }
 
-func (p grpcDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (string, []grpc.DialOption, error) {
-	config.WithEnvCreds = false
-	endpoint, opts, err := p.orig.GetGRPCDialParams(config)
+func (p grpcDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (string, []grpc.DialOption, grpcendpoint.HTTPFetcher, error) {
+	tlsConfig, err := p.tlsConfig()
 	if err != nil {
-		return "", nil, err
+		return "", nil, nil, err
+	}
+	config.TLSConfig = tlsConfig
+	config.WithEnvCreds = false
+	endpoint, opts, httpfetcher, err := p.orig.GetGRPCDialParams(config)
+	if err != nil {
+		return "", nil, nil, err
 	}
 	opts = append(opts, p.ci.Options()...)
 	opts = append(opts, grpc.WithConnectParams(grpc.ConnectParams{
@@ -50,5 +60,77 @@ func (p grpcDialProvider) GetGRPCDialParams(config grpcendpoint.Config) (string,
 		},
 		MinConnectTimeout: 250 * time.Millisecond,
 	}))
-	return endpoint, opts, nil
+	return endpoint, opts, httpfetcher, nil
+}
+
+// Within a cluster, if remotesapi is configured with a tls_ca, we take the
+// following semantics:
+// * The configured tls_ca file holds a set of PEM encoded x509 certificates,
+// all of which are trusted roots for the outbound connections the
+// remotestorage client establishes.
+// * The certificate chain presented by the server must validate to a root
+// which was present in tls_ca. In particular, every certificate in the chain
+// must be within its validity window, the signatures must be valid, key usage
+// and isCa must be correctly set for the roots and the intermediates, and the
+// leaf must have extended key usage server auth.
+// * On the other hand, no verification is done against the SAN or the Subject
+// of the certificate.
+//
+// We use these TLS semantics for both connections to the gRPC endpoint which
+// is the actual remotesapi, and for connections to any HTTPS endpoints to
+// which the gRPC service returns URLs. For now, this works perfectly for our
+// use case, but it's tightly coupled to `cluster:` deployment topologies and
+// the likes.
+//
+// If tls_ca is not set then default TLS handling is performed. In particular,
+// if the remotesapi endpoints is HTTPS, then the system roots are used and
+// ServerName is verified against the presented URL SANs of the certificates.
+func (p grpcDialProvider) tlsConfig() (*tls.Config, error) {
+	if p.caPath == "" {
+		return nil, nil
+	}
+	pem, err := ioutil.ReadFile(p.caPath)
+	if err != nil {
+		return nil, err
+	}
+	roots := x509.NewCertPool()
+	if ok := roots.AppendCertsFromPEM(pem); !ok {
+		return nil, errors.New("error loading ca roots from " + p.caPath)
+	}
+	verifyFunc := func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+		certs := make([]*x509.Certificate, len(rawCerts))
+		var err error
+		for i, asn1Data := range rawCerts {
+			certs[i], err = x509.ParseCertificate(asn1Data)
+			if err != nil {
+				return err
+			}
+		}
+		keyUsages := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
+		opts := x509.VerifyOptions{
+			Roots:         roots,
+			CurrentTime:   time.Now(),
+			Intermediates: x509.NewCertPool(),
+			KeyUsages:     keyUsages,
+		}
+		for _, cert := range certs[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		_, err = certs[0].Verify(opts)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	return &tls.Config{
+		// We have to InsecureSkipVerify because ServerName is always
+		// set by the grpc dial provider and golang tls.Config does not
+		// have good support for performing certificate validation
+		// without server name validation.
+		InsecureSkipVerify: true,
+
+		VerifyPeerCertificate: verifyFunc,
+
+		NextProtos: []string{"h2"},
+	}, nil
 }

integration-tests/go-sql-server-driver/testdata/chain_cert.pem

@@ -1,27 +1,46 @@
 -----BEGIN CERTIFICATE-----
-MIIErDCCApQCCQCnSokQKR3M/zANBgkqhkiG9w0BAQUFADAYMRYwFAYDVQQKDA1E
-b2x0SHViLCBJbmMuMB4XDTIyMDcyMTIwMDgzMloXDTI2MDcxOTIwMDgzMlowGDEW
-MBQGA1UECgwNRG9sdEh1YiwgSW5jLjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
-AgoCggIBAMPmzHy0CmW5Xc27rbRYpJG/QKMXVAz+k2v+AkTQkUzBWKv0z8WhePB/
-tDNVfVYuYQ2sBiHTaar9nn2Lokon+YkPjyMis2aMETHVuqx0DmJb9YcxniA8M27o
-ZlfDrJtQO5UzIp9q2zhsFWj30Qdm6YUOhZ3rTnvYOMUYG/cIYLWXyQCg1oPqRVRr
-GldzLP2GdigdrS6QQjA9AdK+Zi3dP2m2vssG4gJ+lkAWOHe7wvv2RJl/alsvWXmw
-pur7Q9Z7M+tQmqGDxlyDtkDDecyqvEkxPH7mnKV1jahJjzUFHND1r44JlCN0eTmD
-Q3+RldBNZCZSJWQ42yOIK+mTSp4QUvZL9wnJ1/lMb/v7atDlF/MSLeN6SDyAPod7
-Oci8PR+nGhaOKacngrogM6SFQ1kF4tlY5Scrpg61IAcf6uxF3eSBP0qEaFvfLXZV
-mc136E4g2G1haLt7y2prckCHLXEnxurXU4xlU/SH4cy4jB/zLZJs46tM7J9ZtCjg
-QScZeNBA91kKAvHr36f/+suU3MNPAP2fmMCziH2uxh6SxTP8yzsUoV9PCTeaSnXX
-rTMB077j0TOB2qsYhLF3XsLMz+B2Jo0b7ydT7c7rMS9yYvyKPA9JSE44nUrZWj3B
-7ity1moIfrzwbH3AK3D5I9iUbBV0+JpuIZFPoqTIb15TUXJSusYHAgMBAAEwDQYJ
-KoZIhvcNAQEFBQADggIBABGrQEUFJk5StmyFUGvaw/57H+K1ZT62rusFBq1NacMb
-61dMh9xJyDMgLiUllQ8q5CS3bjYt2J2KajpU/58ugF/Ct9aoxA4vFDtfHECllYaH
-zvoiK0Dkrf901xxNVeCbHDmXbvzJ0N/xTkP80kbT4o+aBOw6fxQVEBGAGg4EEz1D
-k7v3/lEsZ2TkCPua1p9kXHaG8+wwE0hAWsaUYgXHTpzz0gUBJ69bOIlBpLKqO9It
-HStkPD7wtYnN54pmOM68EAyXAxUC7yZ9PqncX0X04hH0VlmQGfdXFJDR89mSS6B4
-P1qsi1XtnKC/hHuJlrY02uMXn7u1cVCf5uWfFm6Xs8rLL+q28gV6Tr2aXqgY0Cjl
-tNtUEIP23/irWN48c5/rKOTiUIHJy2m6UofwMQO91jgKFxIyUmkgPQmos2LLNjtk
-VFaPRigAaArwvombUmvfXJl6KoyH/je4H4+Gs+rRQURXU/PD1cioHgsOYNXSmYAj
-AQJv/xp9QBmpzb1ExJOKeWjnUWGu0Wdv4TCTXJNvfdQqOVkT6k6ty1urgr9fNOxY
-PDbHZTI6rXMtT57G108k2gAkaCE6O2R2Dm+vfW7auauqF3lNiZU9Y8IEGU2ybmE3
-s2j+THPWmhuepbZKO5daQH0zlma31QgoyhGSoZ6QUWKEjufEvfx4HwGqMP6BEmaP
+MIID/jCCAuagAwIBAgIUB7Qx4HU3Ezu1FCmp4EKMLGzQW5wwDQYJKoZIhvcNAQEL
+BQAwWzELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDURvbHRIdWIsIEluYy4xNDAyBgNV
+BAMTK3Rlc3RkYXRhIEludGVybWVkaWF0ZSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkw
+HhcNMjIxMDI2MjA0MTAwWhcNMjQxMDE1MjA0MTAwWjBBMQswCQYDVQQGEwJVUzEW
+MBQGA1UEChMNRG9sdEh1YiwgSW5jLjEaMBgGA1UEAxMRdGVzdGRhdGEgSW5zdGFu
+Y2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC40mH/fY9PcLNkmDhD
+TLW4jRYpZWaQx5GD2rSgodO7HcdEvECnvFH9AzktNnU2V/O18Ns+Q66DqACdBFie
+wvi3HVD1lp16PeDDzd+U1gsv09aJkyMQ9rgsc9xER1YsW+9W0jVgCi+uYAgXKRol
+kh5E1GPcgXC0PBHs4EhCXIvQ6VsHkswKLjwTWn3RSotkwGlxNQwbKX4BSFdoc5k/
+QFjW0gG+OoISPJyN3zkU//fKP4/jncxw6jev9KNe7iR8D81Or2s5WhAfA6iv86a9
+qDTWEwP01YmW7bodiv1iytJqrmqLq/Nan1B0HyU9szDE1Ulftf3pSfWJo7pBb1Vc
+Rh+3AgMBAAGjgdMwgdAwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUF
+BwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBSwrZlshrfvOW37
+q6uxSNSYshC2tDAfBgNVHSMEGDAWgBSfaRPMObTFtIs5n3a3/gUuHw+7cjBRBgNV
+HREESjBIghF0ZXN0ZGF0YS50ZXN0ZGF0YYYzc3BpZmZlOi8vbG9jYWwuYXdzZGV2
+LmxkLWNvcnAuY29tL3NlcnZpY2VzL3Rlc3RkYXRhMA0GCSqGSIb3DQEBCwUAA4IB
+AQAUWUnILP1AtiL9e4M0dWfPiVyXBDKhJI4DjF/phNF0X+ou+rjFUCJunf29A9YD
+QzJOQaJY0Gw3Gy1zyx7QG1nkZAhNwqsrzHx5XP9b/p07/Oh7RXk27LbMJZ6JTdQ2
+zR4V+oWDRJ4Fm81cgLaRlXg77xsg69pblubLGvPp3/YLYItoA9oTJdmSftFXDUUa
+vz/PqfWriwiBU3BD8plERt7ljbsOUbo1LQEEd9zxYoPzBKDKj8NMIfmY9NK2QiOy
+vAzyAvB7jU7EhcJsrq3G9KW0Fji0/rsLNb9h8U0ketwdXrCjEq9aEOfKDcHYwvPj
+TSo+uj5MuTBHveAuwmcXy7hB
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDnTCCAoWgAwIBAgIUfoPtM9PmrcMNEV4V7XhM0NyrE2AwDQYJKoZIhvcNAQEL
+BQAwTjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDURvbHRIdWIsIEluYy4xJzAlBgNV
+BAMTHnRlc3RkYXRhIENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMjEwMjYyMDQw
+MDBaFw0yNDEwMTUyMDQwMDBaMFsxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1Eb2x0
+SHViLCBJbmMuMTQwMgYDVQQDEyt0ZXN0ZGF0YSBJbnRlcm1lZGlhdGUgQ2VydGlm
+aWNhdGUgQXV0aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+wcKXpc7l19CyYhbkl6j9EfRP5o2VMoRDUHndxvYIciRhy44lmAscjI3ZnCGRV/TX
+iP2x8pvvhltqD5h6Rb0pHG91PwdOb/vqLIfSZ91tCQbpSHIKwWvZmkefp7Xt7AQM
+VPZwMJNq2o1S3m167CkXHzSlHBVq+ztAc9rvkgLSe85dDN54OFWUwwJY8QToLANp
+ElIym6RIKAqwRASWe8bLG18lGEUnpYwseR0KWYcfL5R15QD3Lk8Xb93FSPakYmvI
+7kMje0RwjHZv8GEmiFweFgEiJNtCtdsyoc3reSPHf/hfRSLDV4aqW/BtdYnXJHVn
+RGwT/ZrIDinCSSWEQNiY+wIDAQABo2YwZDAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0T
+AQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUn2kTzDm0xbSLOZ92t/4FLh8Pu3IwHwYD
+VR0jBBgwFoAUxMCLKgcsWqPtQd5U7ZtkVYR1vIIwDQYJKoZIhvcNAQELBQADggEB
+AHwJIEc04BIkww0ljW8A1K9JoNVsnJyxL7cjeEB+A+S64bcG3QN8N1+qwvyOI3a4
+WjhWNfV2oJi7PkJ0WPz+anTHugtwbekKqV45Y3W1X/OdPTKMPWBZ5mvkLecTlobl
+jMh9kWg3F3n+d+KaWGlvdKDPSwaOhpmkgwPthAuztcAkpvJuz7/4jP5jrM2cqD4+
+otDRKr+b73m2w7jqICXxdYXEuFQ9qCZ8VvlYCTF9qOuBlCeAwanRaPj5na+cME5m
+0AIZyTeYCpB6eP5HLWCGvEP6lD5Hv8PMAzh8xgfFDyxZc3jAWFRB5xRidAVC/wtF
+Nhs1l1AIQ5vUOZrOmsHaIHI=
 -----END CERTIFICATE-----

integration-tests/go-sql-server-driver/testdata/chain_key.pem

@@ -1,52 +1,27 @@
------BEGIN PRIVATE KEY-----
-MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDD5sx8tApluV3N
-u620WKSRv0CjF1QM/pNr/gJE0JFMwVir9M/FoXjwf7QzVX1WLmENrAYh02mq/Z59
-i6JKJ/mJD48jIrNmjBEx1bqsdA5iW/WHMZ4gPDNu6GZXw6ybUDuVMyKfats4bBVo
-99EHZumFDoWd60572DjFGBv3CGC1l8kAoNaD6kVUaxpXcyz9hnYoHa0ukEIwPQHS
-vmYt3T9ptr7LBuICfpZAFjh3u8L79kSZf2pbL1l5sKbq+0PWezPrUJqhg8Zcg7ZA
-w3nMqrxJMTx+5pyldY2oSY81BRzQ9a+OCZQjdHk5g0N/kZXQTWQmUiVkONsjiCvp
-k0qeEFL2S/cJydf5TG/7+2rQ5RfzEi3jekg8gD6HeznIvD0fpxoWjimnJ4K6IDOk
-hUNZBeLZWOUnK6YOtSAHH+rsRd3kgT9KhGhb3y12VZnNd+hOINhtYWi7e8tqa3JA
-hy1xJ8bq11OMZVP0h+HMuIwf8y2SbOOrTOyfWbQo4EEnGXjQQPdZCgLx69+n//rL
-lNzDTwD9n5jAs4h9rsYeksUz/Ms7FKFfTwk3mkp1160zAdO+49EzgdqrGISxd17C
-zM/gdiaNG+8nU+3O6zEvcmL8ijwPSUhOOJ1K2Vo9we4rctZqCH688Gx9wCtw+SPY
-lGwVdPiabiGRT6KkyG9eU1FyUrrGBwIDAQABAoICABUIJlQNEECzkfqQd6mxCpoL
-KmlYC9IJUtJ5Rs0Uh0TyTQ7JDbVuDInla/dG6lniSNEq8s2W4PVWnTllUFsdx5CL
-dxaSlygfSYlMJOp220R8EvQcw5k6XVs+4B30CAf0qTDveHwdAMQh9np6gJqG1fNP
-B9FYfeiV4iJm4Dm5UIiubwn+OomXETJq/Tz+RIpDcVQFO56QJkr/gb6aamXqJvC2
-ie1KI+GYrZDb0dwo8FoUqnDAWS7I+pYx/PmlWDciqwRMdw14FEfCbEKvudfbTLOe
-8Zu+LnslD7xNiW5ryhg1CE/7f0f/LTSbfxenDap7ZJEoqJMF96Ds8an2AkDOB9nx
-XB5kVz5jMsaZ1f68Rx8S4EqEEcXxYwiRe5WoDEnnVr2+Q6QzOqh/4DaA5VuId462
-IjPDWmYszSqig9QXjS11SkTMKCKxas4AqfCb8uUlcXdri4aSv0Khb7DgbO2su1KC
-+hcXpiAMH9jVX1d4N8c0Q0HLOT09lRnD2mmEX6Lo2kWgb5Hpzo88Ty9WI7oiszsY
-J1r6qPkXIc9Ft1YwpdVBhkBbxB024l9IG8I1UzjrLFnR/A5sRefzosNi4/ZACPW4
-Kykhy7p+ZV9Kf8cjMbY11afCmi9jlXsVqWwJIMk+LxTCjF/lmbMay/G7j+ibGtSQ
-hU+LNPzAOUEwBj1OqoMhAoIBAQDlo3Ecgeu5zxNILnkut6RHHDJUK2N9+5HIDwi4
-frMlkM3b8NLz09/GtmX4HTKkDBur4x9QeEIsxG19tk2QWZQ4EAKs8OcEXaCL4Q9g
-msZbQC5rrFjRzUC4roxCTEz4g/ANEM+huLq/3a6afUhkmUuGZzK6rf6E36dTx3na
-DP4tDAx1s/DqfMtXYYmzrb3V1Nk9NUwQFRselJ8EHeIA7NEcLcv5yREia57RcYm/
-EfuA90j1ER6iHZIxopPfo1Cx7I9N4eoQM4/Tjb5qu+krfGOFOQbL6hCPHeHkZlAw
-0/2ECxCHS2y+Uih3MkMdnme2tfBr8AQpcfAOxSTMXu1wGDs9AoIBAQDaY+fVJ2G/
-/myI3Nly7MZaJ8NT8kcQx55b1s5vqWU+IQo5YC4KGdUp32U5Uxr2war8SuA2pKW0
-Cv42IJYlGQQUgpj2k+DJcDz+Qz9nqE5Ft8vNmyA3Y2gbwgTkd9dtFCTph4BNiAad
-qyjXwdJ6qwB1dbORsprC/Ue8WcEVwWwvF3PGnvbEiM8qLyxv/WIXnN5B/XcvUFHS
-mS3IVkJpdR8Kzp0Ctro5mHd2L6SQa/XM5tU3bye9Hzf1J3rWM/FGzVtYInC//CoO
-w/sA/ebfhK1iHjYYp4MjyETBkbD1kpCl6eNdTKN9ydSkUzhWlHn3xKQQrdZ7KiiH
-YbIhh1rwB+qTAoIBAFIoOnSfis2MZ3Kgpdxv+UczsFHqwAq3sX1o247eTYu4Fd6F
-d4OinuICKdMt5wtIBbJmbLKmg85ubFnYmkF1uxCfscVb3tryAFlrKMxAM408Fh+R
-pqlRDMHGOQoTMEqNMZoLFK3gYHf6gNhm0DqlmZ65Vy3wyCmTttLDgDXiBiHpuJ93
-xE6wXTOjAtgU5eEV6K78XX03f99d/tJDOrNoBpxVSi/Qnt+4rzZxr317moaWcjSz
-bklD2SUG7G7LiDhP0SllFQ+80s02XhTjq9VSCG0GbQcRc+EwKLxFWpVNktrl9oDh
-HEOvMykKA3caUDLPPvfvBB4r1F4EbFjt8Xb0RGUCggEAO0PrcRvr2gd4ere8RwTc
-WzD5P/m6sWIKpo+nnAPTVsXumV1xgQo7n85hEOptodMyzJ6hNBMAaNim3hd/x3d/
-dPVv/1JoKSJNWw7y0PWKsD7NjvFvD7jpUscXPs0K6C4USk+cUO3+JaGCRvLxZJqt
-WDLl1T8r4oiLhCCzVm0UJ79sitUu0Gz0E1WT8JxJl3DZm/zl8DAS1Fz/YKOQCEBh
-eTRSxZ7C8MhgevE47nxtyvpFmHKQzTEApYXePuz/qCAojsVh5afP3gvvPPiqQ7Qk
-vUDHm28yFm7Nwd4AsNPibzQGoJYgtA0mqKVw34YRh1yUzXXvg6MQNpUbmx+5XPQ5
-AwKCAQEA5Iye1s7RVxZP5iJ3vgy67OU+zza5yEIoJEabhF4bOBDsCwNcx2EwsQll
-X/Su5qqiIVnrRmkdYVhTnZv8bigq/8Hu+BBenMLxkAwZ5ep6gKq9wdiPQArjNBlS
-5KkGuj+7LNCsmmldXVXjjg2BNWBDdVv33hhhqsi/Tzau+qAufdNGdBTS4ZTWEH0z
-X5EBtOphJbBPeMUrm1PFOXKUDDwPfqX86rg1NHr1l5iB7uqShZak1s1ovoyFO6s7
-I9d8chi4/qwwYk8cHczB4C9EwBvWEvcAf1xa6I1Mp8y3tDhWPVIpq5P8i9vQFYIJ
-LWLCd/YowgxkNl5j6a5QMFoZvjLi5A==
------END PRIVATE KEY-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAuNJh/32PT3CzZJg4Q0y1uI0WKWVmkMeRg9q0oKHTux3HRLxA
+p7xR/QM5LTZ1NlfztfDbPkOug6gAnQRYnsL4tx1Q9Zadej3gw83flNYLL9PWiZMj
+EPa4LHPcREdWLFvvVtI1YAovrmAIFykaJZIeRNRj3IFwtDwR7OBIQlyL0OlbB5LM
+Ci48E1p90UqLZMBpcTUMGyl+AUhXaHOZP0BY1tIBvjqCEjycjd85FP/3yj+P453M
+cOo3r/SjXu4kfA/NTq9rOVoQHwOor/Omvag01hMD9NWJlu26HYr9YsrSaq5qi6vz
+Wp9QdB8lPbMwxNVJX7X96Un1iaO6QW9VXEYftwIDAQABAoIBAQC4yEaIPQ2yG/iP
+g40E5EXvDMfyfVntOEopLNlYnFLnCl+3PgvaZ/ME5lsc9Ax+V7Lm3bclal+pa6ep
+VLYRjNdDpMDTuVEa7ZCx0zxNPy8SE1a0V3JAvJrofrHjZfsnAIerIyGQMr73NTYB
+ieuFUrCGml55EMUQvdoiHR7BkmuLYn+3TNJ4Lr2WsNGmChG/W+IwkdmW4RFLZo67
+qHjb7yAYEXFBppgm1YpaHwEnCmOsErmBlAwFZwvPLRezjumics+2yrHt/Tm7uicR
+GQI2ROCp2rst2UOiPtYd+vWCabYB2TtMq9/CLAgQsIkC/iABWDsB/Yasu/xfo1k+
+C0GhlfgJAoGBAMpo1qqjoSDDL+tJrVFg/dEpU7CWodXJ/6vm3Qb1WB7Nm93j5tsf
+7v390uQGsPSl/KD9MqTnDWs4xVgvO6eu/LCB9749ctbBkHvpN1eLjlApWG+eLGHf
+gfqHMiokQ228J0CUhgvrfb0SxIsRnmHqxfbHo8oHBIW+WnlNwMOG0WvTAoGBAOnB
+dlsWedSQJGngsQg7zc9NOJbJz4SxAv/Vp6KjVMiFQOcTJa/PUyaoTDUGoxSysMTl
++5RF70gxPtcZJrqC2OWuULkI1Lm0A69SHU/P2tCJ+Wt/AcK7yH/vTEQd44Pwjkct
+uoCM3Euf/S66GOIPkM3RG5CxuU+SBp+wGaOYhAINAoGAQ+WnHNaG1lajXGn6mbHP
+crpKOJJO90grW561xf/G745JGsW4SwkLQmhCtfsIoQiNFfPZaTeYaL9Cc7JkcHti
+iFMQp+A1BZUowmgZCGTn+DvmTorgmHRBRajUSw6fD9Bt2lv4G0eDhkklZQEj//Sh
+M4cEimCQQ8z2zHooj25KEcECgYAV9792ufsDFfTGGn6opm4mEDzENv0QnE4K2vph
+F3ZtTdCWpr8A8bv/wws+ZHxJAq4IIxDsk1H0d+RO9KcmGgvmMeaWLRVIynkaLd5h
+VMhclsrg5lO1CE7Ebym8sQ5jpOTKHasMT7CYTtXNYWHbRNk37nHnvDwNFU0YDsWq
+ETg+tQKBgBSeRiTSx9Up58kNqRj+Z+EAKY3FoIgN6b83pL71gCXLdP0SQmZ9QOiV
+5RGF0cbE1n3dx+HMKzNKnKhj783hnLosttq5OlO2cxqLTGLjv5aLPDji3HtK1HaA
+IBIg58byFOMoKp95W0QSXVIAymXyvKRyj7dT1kDOidjQo354Y+5R
+-----END RSA PRIVATE KEY-----

integration-tests/go-sql-server-driver/testdata/chain_root.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDbDCCAlSgAwIBAgIUSixGHtBJBKsVdj56puZy3LHvtkgwDQYJKoZIhvcNAQEL
+BQAwTjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDURvbHRIdWIsIEluYy4xJzAlBgNV
+BAMTHnRlc3RkYXRhIENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMjEwMjUyMjU1
+MDBaFw0yNzEwMjQyMjU1MDBaME4xCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1Eb2x0
+SHViLCBJbmMuMScwJQYDVQQDEx50ZXN0ZGF0YSBDZXJ0aWZpY2F0ZSBBdXRob3Jp
+dHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDArzGWFksWPijYVrsq
+Hj1YakHQiUcK7KLsXZHm0Tx3ryUvJZAX86UM+/QXO/TVYoWPoIXVaFFCDMlwKzXU
+FgEHJHQU7NaKcDUN8xaM36Y79VouHJzkUl6UvMGZrQXqdsPsQ56t/GcJCbbBLgki
+9uQPGOB2KhLTkPV4L6CmubIOakCmNI99Ivoo2YGc3m5RYSv5f8/RnyDYBKWAenwI
+omWPFs9te5AriIaXXq4nQhJQ40TCK/P9AZTlJOO5jaZ4Gnt/XWSHoSwxd15bkEos
+19wdqK4oHHnO8luIA4lL7PxyOB5Wz/P+n9epY7aM/AHy7gVoekLhFk1CVuzA30Uv
+ZEfbAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G
+A1UdDgQWBBTEwIsqByxao+1B3lTtm2RVhHW8gjANBgkqhkiG9w0BAQsFAAOCAQEA
+kSXCYNgsLdyWVru8rhshoW7sZjKCttgmUzacv79JPpYuZOTS/YthlYAh6NHqnCuW
+cMRNxjr88LQu/U8MwJ+8qeHZBm2k6RMvvm8/w8WfyP1E/2PgFtF9nqlehj1o3BaR
+UohLNl7YfJORiW8L/z0FAsz24+xsCtvQnvaGxFZcYHKYyg9xS4dbspN5fg2daxP7
+jzzI0xcmpOFJfDpRywhx9iI8J+tJLtVJLZutuah9cK2Y5PGqsTikly//WAv99Rw8
+naMb8DOC0p1RiXakbF7LMSyIbaLdmItlx3Ea/b/Ul/kFw/cueHTCyGZ4swvkq6pR
+1lEUD4MQPt0u9IterfN4yg==
+-----END CERTIFICATE-----

integration-tests/go-sql-server-driver/testdata/invalid_root.pem

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDbDCCAlSgAwIBAgIUdWEanf/1+cmS33nZDPY+gkQwS+gwDQYJKoZIhvcNAQEL
+BQAwTjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDURvbHRIdWIsIEluYy4xJzAlBgNV
+BAMTHnRlc3RkYXRhIENlcnRpZmljYXRlIEF1dGhvcml0eTAeFw0yMjEwMjYyMDM1
+MDBaFw0yNzEwMjUyMDM1MDBaME4xCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1Eb2x0
+SHViLCBJbmMuMScwJQYDVQQDEx50ZXN0ZGF0YSBDZXJ0aWZpY2F0ZSBBdXRob3Jp
+dHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDLAQ88jtxKIH0Uc0Yp
+oUmM0Bx3/fBqgbYAGJ1cxtkXahhGp94ICe0gmASnbPuAY22X0zf55C94semPNNgb
+xV/FHftvyi720z3wwOk8twa8I4vjb1mnxlPZzS2Xd1pb4KnUtjOemGfZOn6OWbXF
+ukf5uNDKUZcFPPjaiAnQ+kK6vjYWZjY6Hn4KVAjBRylQj86hzgF0cc7B4WOX3L6L
+ahY56urFElKnFh8vCydSfyZqtz56ng3Gc83PBIEkTTgQVwFJkx+Azh73NaTGwXcv
+3Wj4D+TzF2T0JsHe6s1CWyoHxvccwoUdAv8HGzzHVcm+81KMdy9r9e7R3kyu9HSK
+D3sBAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G
+A1UdDgQWBBRzOWBY5hQAM5obC3y+nbHKnvQtmzANBgkqhkiG9w0BAQsFAAOCAQEA
+yKsw7CLYQQ2i9jzislIUF0pMW03rLTBPSyv78mhUrfaL2TncdJAPTMdR5KaFTKSy
+2AzuYkIN9gU0blk73sxbtdNyZlpP0MQHRuRkgpuXii0tWQ0f6uhLaZRJvLm4Hjsj
+Sma8ydO3/7FvdTby6Uv1Rivd53BGfVAcw8W1oC+8KfrDhUsWzqcDH6Aiszz0utKr
+XAqiOdNUSy2riyxc3s9RH2j20BNj6vWkz8ZoRdBa2pf/oRtYF2ZJjCZq7eH5hlSj
+/Am5Yw9Cc0/48Tm58e4V2SDHys9ld8EBKOMlo8djk3q0LxGtZ41O1hr4iaHTkWyl
+2wYWEa395xncUBUqvCpKyA==
+-----END CERTIFICATE-----

integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml

@@ -924,3 +924,177 @@ tests:
       result:
         columns: ["count(*)"]
         rows: [["15"]]
+- name: tls, bad root, failover to standby fails
+  multi_repos:
+  - name: server1
+    with_files:
+    - name: server.yaml
+      contents: |
+        log_level: trace
+        listener:
+          host: 0.0.0.0
+          port: 3309
+        cluster:
+          standby_remotes:
+          - name: standby
+            remote_url_template: https://localhost:50052/{database}
+          bootstrap_role: primary
+          bootstrap_epoch: 1
+          remotesapi:
+            port: 50051
+            tls_key: key.pem
+            tls_cert: cert.pem
+            tls_ca: root.pem
+    - name: key.pem
+      source_path: testdata/chain_key.pem
+    - name: cert.pem
+      source_path: testdata/chain_cert.pem
+    - name: root.pem
+      source_path: testdata/invalid_root.pem
+    server:
+      args: ["--config", "server.yaml"]
+      port: 3309
+  - name: server2
+    with_files:
+    - name: server.yaml
+      contents: |
+        log_level: trace
+        listener:
+          host: 0.0.0.0
+          port: 3310
+        cluster:
+          standby_remotes:
+          - name: standby
+            remote_url_template: https://localhost:50051/{database}
+          bootstrap_role: standby
+          bootstrap_epoch: 1
+          remotesapi:
+            port: 50052
+            tls_key: key.pem
+            tls_cert: cert.pem
+            tls_ca: root.pem
+    - name: key.pem
+      source_path: testdata/chain_key.pem
+    - name: cert.pem
+      source_path: testdata/chain_cert.pem
+    - name: root.pem
+      source_path: testdata/invalid_root.pem
+    server:
+      args: ["--config", "server.yaml"]
+      port: 3310
+  connections:
+  - on: server1
+    queries:
+    - exec: 'create database repo1'
+    - exec: "use repo1"
+    - query: "call dolt_assume_cluster_role('standby', '11')"
+      error_match: failed to transition from primary to standby gracefully
+    - exec: "create table vals (i int primary key)"
+    - exec: "insert into vals values (0)"
+- name: tls, good root, create new database, primary replicates to standby, fails over, new primary replicates to standby, fails over, new primary has all writes
+  multi_repos:
+  - name: server1
+    with_files:
+    - name: server.yaml
+      contents: |
+        log_level: trace
+        listener:
+          host: 0.0.0.0
+          port: 3309
+        cluster:
+          standby_remotes:
+          - name: standby
+            remote_url_template: https://localhost:50052/{database}
+          bootstrap_role: primary
+          bootstrap_epoch: 1
+          remotesapi:
+            port: 50051
+            tls_key: key.pem
+            tls_cert: cert.pem
+            tls_ca: root.pem
+    - name: key.pem
+      source_path: testdata/chain_key.pem
+    - name: cert.pem
+      source_path: testdata/chain_cert.pem
+    - name: root.pem
+      source_path: testdata/chain_root.pem
+    server:
+      args: ["--config", "server.yaml"]
+      port: 3309
+  - name: server2
+    with_files:
+    - name: server.yaml
+      contents: |
+        log_level: trace
+        listener:
+          host: 0.0.0.0
+          port: 3310
+        cluster:
+          standby_remotes:
+          - name: standby
+            remote_url_template: https://localhost:50051/{database}
+          bootstrap_role: standby
+          bootstrap_epoch: 1
+          remotesapi:
+            port: 50052
+            tls_key: key.pem
+            tls_cert: cert.pem
+            tls_ca: root.pem
+    - name: key.pem
+      source_path: testdata/chain_key.pem
+    - name: cert.pem
+      source_path: testdata/chain_cert.pem
+    - name: root.pem
+      source_path: testdata/chain_root.pem
+    server:
+      args: ["--config", "server.yaml"]
+      port: 3310
+  connections:
+  - on: server1
+    queries:
+    - exec: 'create database repo1'
+    - exec: 'use repo1'
+    - exec: 'create table vals (i int primary key)'
+    - exec: 'insert into vals values (0),(1),(2),(3),(4)'
+    - query: "call dolt_assume_cluster_role('standby', 2)"
+      result:
+        columns: ["status"]
+        rows: [["0"]]
+  - on: server2
+    queries:
+    - exec: 'use repo1'
+    - query: "select count(*) from vals"
+      result:
+        columns: ["count(*)"]
+        rows: [["5"]]
+    - query: "call dolt_assume_cluster_role('primary', 2)"
+      result:
+        columns: ["status"]
+        rows: [["0"]]
+  - on: server2
+    queries:
+    - exec: 'use repo1'
+    - exec: 'insert into vals values (5),(6),(7),(8),(9)'
+    - query: "call dolt_assume_cluster_role('standby', 3)"
+      result:
+        columns: ["status"]
+        rows: [["0"]]
+  - on: server1
+    queries:
+    - exec: 'use repo1'
+    - query: "select count(*) from vals"
+      result:
+        columns: ["count(*)"]
+        rows: [["10"]]
+    - query: "call dolt_assume_cluster_role('primary', 3)"
+      result:
+        columns: ["status"]
+        rows: [["0"]]
+  - on: server1
+    queries:
+    - exec: 'use repo1'
+    - exec: 'insert into vals values (10),(11),(12),(13),(14)'
+    - query: "select count(*) from vals"
+      result:
+        columns: ["count(*)"]
+        rows: [["15"]]