diff --git a/integration-tests/bats/.sql-server.bats.swp b/integration-tests/bats/.sql-server.bats.swp deleted file mode 100644 index 966909e17b..0000000000 Binary files a/integration-tests/bats/.sql-server.bats.swp and /dev/null differ diff --git a/integration-tests/bats/sql-server-cluster.bats b/integration-tests/bats/sql-server-cluster.bats index b0fab89046..80107f6ff1 100644 --- a/integration-tests/bats/sql-server-cluster.bats +++ b/integration-tests/bats/sql-server-cluster.bats @@ -29,552 +29,6 @@ teardown() { teardown_common } -@test "sql-server-cluster: persisted role and epoch take precedence over bootstrap values" { - cd serverone - - echo " -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - (cd repo1 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo1) - (cd repo2 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo2) - dolt sql-server --config server.yaml & - SERVER_PID=$! - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" dolt_cluster 1 dolt "" "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch;select "'`database`'", standby_remote, role, epoch from dolt_cluster_status order by "'`database`'" asc" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nstandby,10;database,standby_remote,role,epoch\nrepo1,standby,standby,10\nrepo2,standby,standby,10" - - kill $SERVER_PID - wait $SERVER_PID - SERVER_PID= - - echo " -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 0 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - dolt sql-server --config server.yaml & - SERVER_PID=$! - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nstandby,10" -} - -@test "sql-server-cluster: dolt_assume_cluster_role" { - cd serverone - - echo " -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - (cd repo1 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo1) - (cd repo2 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo2) - dolt sql-server --config server.yaml & - SERVER_PID=$! - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - # stale epoch - run server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('standby', '9');" "" 1 - [[ "$output" =~ "error assuming role" ]] || false - # wrong role at current epoch - run server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('primary', '10');" "" 1 - [[ "$output" =~ "error assuming role" ]] || false - # wrong role name - run server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('backup', '11');" "" 1 - [[ "$output" =~ "error assuming role" ]] || false - - # successes - - # same role, same epoch - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('standby', '10');" "status\n0" - # same role, new epoch - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('standby', '12'); select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch;" "status\n0;@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nstandby,12" - # new role, new epoch - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('primary', '13');" "status\n0" - # we assert on a new connection, since the server leaves the previous connection in an permanent error state. - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch;" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nprimary,13" - - # Server comes back up with latest assumed role. - kill $SERVER_PID - wait $SERVER_PID - SERVER_PID= - dolt sql-server --config server.yaml & - SERVER_PID=$! - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch;" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nprimary,13" -} - -@test "sql-server-cluster: create database makes a new remote" { - cd serverone - - echo " -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - (cd repo1 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo1) - (cd repo2 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo2) - dolt sql-server --config server.yaml & - SERVER_PID=$! - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "create database a_new_database;use a_new_database;select name, url from dolt_remotes" ";;name,url\nstandby,http://localhost:50052/a_new_database" -} - -@test "sql-server-cluster: sql-server fails to start if a configured remote is missing" { - cd serverone - - echo " -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - run dolt sql-server --config server.yaml - [ "$status" -ne 0 ] - [[ "$output" =~ "destination remote standby does not exist" ]] || false -} - -@test "sql-server-cluster: primary comes up and replicates to standby" { - cd serverone - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - serverone_pid=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - cd ../servertwo - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERTWO_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERONE_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERTWO_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - servertwo_pid=$! - - wait_for_connection "${SERVERTWO_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "create table vals (i int primary key);insert into vals values (1),(2),(3),(4),(5)" - - server_query_with_port "${SERVERTWO_MYSQL_PORT}" dolt_cluster 1 dolt "" "select "'`database`'", standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster_status order by "'`database`'" asc" "database,standby_remote,role,epoch,replication_lag_millis,current_error\nrepo1,standby,primary,10,0,None\nrepo2,standby,primary,10,0,None" - - kill $servertwo_pid - wait $servertwo_pid - - kill $serverone_pid - wait $serverone_pid - - cd ../serverone - - run env DOLT_ROOT_PATH=`pwd` dolt sql -q 'select count(*) from vals' - [ "$status" -eq 0 ] - [[ "$output" =~ "| 5 " ]] || false -} - -@test "sql-server-cluster: booted standby server is read only" { - cd serverone - - cd repo1 - dolt sql -q 'create table vals (i int primary key)' - dolt sql -q 'insert into vals values (1), (2), (3), (4), (5)' - cd .. - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - SERVER_PID=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals" "count(*)\n5" - run server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "insert into vals values (6),(7),(8),(9),(10)" "" 1 - [[ "$output" =~ "Database repo1 is read-only" ]] || false - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals" "count(*)\n5" -} - -@test "sql-server-cluster: booted primary server is read write" { - cd serverone - - cd repo1 - dolt sql -q 'create table vals (i int primary key)' - dolt sql -q 'insert into vals values (1), (2), (3), (4), (5)' - cd .. - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - SERVER_PID=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals" "count(*)\n5" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "insert into vals values (6),(7),(8),(9),(10)" "" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals" "count(*)\n10" -} - -@test "sql-server-cluster: standby transitioned to primary becomes writable" { - cd serverone - - cd repo1 - dolt sql -q 'create table vals (i int primary key)' - dolt sql -q 'insert into vals values (1), (2), (3), (4), (5)' - cd .. - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - SERVER_PID=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - run server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "insert into vals values (6),(7),(8),(9),(10)" "" 1 - [[ "$output" =~ "Database repo1 is read-only" ]] || false - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('primary', 11)" "status\n0" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals" "count(*)\n5" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "insert into vals values (6),(7),(8),(9),(10)" "" 0 - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals" "count(*)\n10" -} - -@test "sql-server-cluster: primary transitioned to standby becomes read only" { - # In order to gracefully transition to standby, we will need the standby running. - cd serverone - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - serverone_pid=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - cd ../servertwo - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERTWO_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERONE_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERTWO_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - servertwo_pid=$! - - wait_for_connection "${SERVERTWO_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "create table vals (i int primary key);insert into vals values (1),(2),(3),(4),(5)" - run server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('standby', 11)" "" 1 - run server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "insert into vals values (6),(7),(8),(9),(10)" "" 1 - [[ "$output" =~ "Database repo1 is read-only" ]] || false - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals" "count(*)\n5" - - kill $servertwo_pid - wait $servertwo_pid - - kill $serverone_pid - wait $serverone_pid -} - -@test "sql-server-cluster: misconfigured cluster with primaries at same epoch, both transition to detected_broken_config" { - cd serverone - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - serverone_pid=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - cd ../servertwo - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERTWO_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERONE_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERTWO_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - servertwo_pid=$! - - wait_for_connection "${SERVERTWO_MYSQL_PORT}" 5000 - - # Run a query to make sure everyone sees everyone... - run server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "CREATE TABLE vals (i int primary key)" "" 1 - - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\ndetected_broken_config,10" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\ndetected_broken_config,10" - - kill $servertwo_pid - wait $servertwo_pid - - kill $serverone_pid - wait $serverone_pid -} - @test "sql-server-cluster: an older primary comes up, becomes a standby and does not overwrite newer primary" { cd serverone @@ -782,39 +236,6 @@ cluster: server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "create table vals (i int primary key);insert into vals values (0);select i from vals" ";;i\n0" } -@test "sql-server-cluster: standby -> primary leave connection used to transition in an error state" { - cd serverone - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - (cd repo1 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo1) - (cd repo2 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo2) - dolt sql-server --config server.yaml & - SERVER_PID=$! - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - # when we do this, we become read only for a little bit... - run server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('primary', '11');select 2 from dual" "" 1 - [[ "$output" =~ "this connection can no longer be used" ]] || false -} - @test "sql-server-cluster: primary replicates to standby, fails over, new primary replicates to standby, fails over, new primary has all writes" { cd serverone diff --git a/integration-tests/go-sql-server-driver/testdef.go b/integration-tests/go-sql-server-driver/testdef.go index 518c1f89ec..785aad3baa 100644 --- a/integration-tests/go-sql-server-driver/testdef.go +++ b/integration-tests/go-sql-server-driver/testdef.go @@ -55,6 +55,13 @@ type Connection struct { On string `yaml:"on"` Queries []Query `yaml:"queries"` RestartServer *RestartArgs `yaml:"restart_server"` + + // Rarely needed, allows the entire connection assertion to be retried + // on an assertion failure. Use this is only for idempotent connection + // interactions and only if the sql-server is prone to tear down the + // connection based on things that are happening, such as cluster role + // transitions. + RetryAttempts int `yaml:"retry_attempts"` } // |RestartArgs| are possible arguments, to change the arguments which are @@ -296,19 +303,35 @@ func (test Test) Run(t *testing.T) { for i, c := range test.Conns { server := servers[c.On] require.NotNilf(t, server, "error in test spec: could not find server %s for connection %d", c.On, i) - func() { - db, err := server.DB() - require.NoError(t, err) - defer db.Close() + if c.RetryAttempts > 1 { + RetryTestRun(t, c.RetryAttempts, func(t require.TestingT) { + db, err := server.DB() + require.NoError(t, err) + defer db.Close() - conn, err := db.Conn(context.Background()) - require.NoError(t, err) - defer conn.Close() + conn, err := db.Conn(context.Background()) + require.NoError(t, err) + defer conn.Close() - for _, q := range c.Queries { - RunQuery(t, conn, q) - } - }() + for _, q := range c.Queries { + RunQueryAttempt(t, conn, q) + } + }) + } else { + func() { + db, err := server.DB() + require.NoError(t, err) + defer db.Close() + + conn, err := db.Conn(context.Background()) + require.NoError(t, err) + defer conn.Close() + + for _, q := range c.Queries { + RunQuery(t, conn, q) + } + }() + } if c.RestartServer != nil { err := server.Restart(c.RestartServer.Args) require.NoError(t, err) @@ -343,17 +366,14 @@ func (r *retryTestingT) FailNow() { panic(r) } -func RetryTestRun(t *testing.T, attempts int, test func(require.TestingT)) { - if attempts == 0 { - attempts = 1 - } - var rtt *retryTestingT +func (r *retryTestingT) try(attempts int, test func(require.TestingT)) { for i := 0; i < attempts; i++ { + r.errorfStrings = nil + r.errorfArgs = nil + r.failNow = false if i != 0 { time.Sleep(RetrySleepDuration) } - rtt = new(retryTestingT) - rtt.T = t func() { defer func() { if r := recover(); r != nil { @@ -363,20 +383,28 @@ func RetryTestRun(t *testing.T, attempts int, test func(require.TestingT)) { } } }() - test(rtt) + test(r) }() - if !rtt.failNow && len(rtt.errorfStrings) == 0 { + if !r.failNow && len(r.errorfStrings) == 0 { return } } - for i := range rtt.errorfStrings { - t.Errorf(rtt.errorfStrings[i], rtt.errorfArgs[i]...) + for i := range r.errorfStrings { + r.T.Errorf(r.errorfStrings[i], r.errorfArgs[i]...) } - if rtt.failNow { - t.FailNow() + if r.failNow { + r.T.FailNow() } } +func RetryTestRun(t *testing.T, attempts int, test func(require.TestingT)) { + if attempts == 0 { + attempts = 1 + } + rtt := &retryTestingT{T: t} + rtt.try(attempts, test) +} + func RunQuery(t *testing.T, conn *sql.Conn, q Query) { RetryTestRun(t, q.RetryAttempts, func(t require.TestingT) { RunQueryAttempt(t, conn, q) diff --git a/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml b/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml index dbfb192789..0049beec65 100644 --- a/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml +++ b/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml @@ -119,10 +119,6 @@ tests: result: columns: ["status"] rows: [["0"]] - - query: "select 2 from dual" - result: - columns: ["2"] - rows: [["2"]] - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role","@@GLOBAL.dolt_cluster_role_epoch"] @@ -133,7 +129,7 @@ tests: rows: [["0"]] # Connection should be broken now. - query: "select 2 from dual" - error_match: "no longer be used" + error_match: "this connection no longer be used" - on: server1 queries: - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" @@ -300,3 +296,277 @@ tests: result: columns: ["count(*)"] rows: [["5"]] +- name: booted standby server is read only + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: standby_server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: standby + bootstrap_epoch: 10 + remotesapi: + port: 50051 + server: + args: ["--config", "standby_server.yaml"] + port: 3309 + connections: + - on: server1 + queries: + - exec: "use repo1" + - exec: "create table vals (i int primary key)" + error_match: "repo1 is read-only" +- name: booted primary server is read write + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 50051 + server: + args: ["--config", "server.yaml"] + port: 3309 + connections: + - on: server1 + queries: + - exec: "use repo1" + - exec: "create table vals (i int primary key)" +- name: standby transitioned to primary becomes read write + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: standby_server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: standby + bootstrap_epoch: 10 + remotesapi: + port: 50051 + server: + args: ["--config", "standby_server.yaml"] + port: 3309 + connections: + - on: server1 + queries: + - exec: "use repo1" + - exec: "create table vals (i int primary key)" + error_match: "repo1 is read-only" + - query: "call dolt_assume_cluster_role('primary', 11)" + result: + columns: ["status"] + rows: [["0"]] + - on: server1 + queries: + - exec: "use repo1" + - exec: "create table vals (i int primary key)" + - query: "select count(*) from vals" + result: + columns: ["count(*)"] + rows: [["0"]] +- name: primary transitioned to standby becomes read only + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 50051 + server: + args: ["--config", "server.yaml"] + port: 3309 + - name: server2 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50051/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50051/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50051/{database} + bootstrap_role: standby + bootstrap_epoch: 10 + remotesapi: + port: 50052 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server1 + queries: + - exec: "use repo1" + - exec: "create table vals (i int primary key)" + - exec: "insert into vals values (1),(2),(3),(4),(5)" + - query: "call dolt_assume_cluster_role('standby', 11)" + result: + columns: ["status"] + rows: [["0"]] + - on: server1 + queries: + - exec: "use repo1" + - exec: "insert into vals values (6),(7),(8),(9),(10)" + error_match: "repo1 is read-only" + - query: "select count(*) from vals" + result: + columns: ["count(*)"] + rows: [["5"]] +- name: misconfigured cluster with primaries at same epoch, both transition to detected_broken_config + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 50051 + server: + args: ["--config", "server.yaml"] + port: 3309 + - name: server2 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50051/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50051/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50051/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 50052 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server1 + retry_attempts: 10 + queries: + - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" + result: + columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] + rows: [["detected_broken_config", "10"]] + - exec: "use repo1" + - exec: "create table more_vals (i int primary key)" + error_match: "repo1 is read-only" + - on: server2 + queries: + - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" + result: + columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] + rows: [["detected_broken_config", "10"]] + - exec: "use repo1" + - exec: "create table more_vals (i int primary key)" + error_match: "repo1 is read-only"