From c56b0630a96ce64bd72a9907795184eb1fcfd089 Mon Sep 17 00:00:00 2001 From: Aaron Son Date: Tue, 4 Oct 2022 10:22:38 -0700 Subject: [PATCH] integration-tests/go-sql-server-driver: Finish transitioning the rest of the sql-server-cluster tests. --- .../bats/sql-server-cluster.bats | 313 --------------- integration-tests/go-sql-server-driver/cmd.go | 2 +- .../tests/sql-server-cluster.yaml | 373 +++++++++++++++++- 3 files changed, 373 insertions(+), 315 deletions(-) delete mode 100644 integration-tests/bats/sql-server-cluster.bats diff --git a/integration-tests/bats/sql-server-cluster.bats b/integration-tests/bats/sql-server-cluster.bats deleted file mode 100644 index 80107f6ff1..0000000000 --- a/integration-tests/bats/sql-server-cluster.bats +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env bats -load $BATS_TEST_DIRNAME/helper/common.bash -load $BATS_TEST_DIRNAME/helper/query-server-common.bash - -make_repo() { - mkdir -p "$1" - (cd "$1"; dolt init) -} - -SERVERONE_MYSQL_PORT=3309 -SERVERONE_GRPC_PORT=50051 - -SERVERTWO_MYSQL_PORT=3310 -SERVERTWO_GRPC_PORT=50052 - -setup() { - skiponwindows "tests are flaky on Windows" - setup_no_dolt_init - - make_repo serverone/repo1 - make_repo serverone/repo2 - - make_repo servertwo/repo1 - make_repo servertwo/repo2 -} - -teardown() { - stop_sql_server "1" - teardown_common -} - -@test "sql-server-cluster: an older primary comes up, becomes a standby and does not overwrite newer primary" { - cd serverone - - cd repo1 - dolt sql -q 'create table vals (i int primary key)' - dolt sql -q 'insert into vals values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)' - cd ../ - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 15 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - serverone_pid=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - cd ../servertwo - - cd repo1 - dolt sql -q 'create table vals (i int primary key)' - dolt sql -q 'insert into vals values (1),(2),(3),(4),(5)' - cd ../ - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERTWO_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERONE_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERTWO_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - servertwo_pid=$! - - wait_for_connection "${SERVERTWO_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "SELECT count(*) FROM vals" "count(*)\n10" - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nstandby,15" - - kill $servertwo_pid - wait $servertwo_pid - - kill $serverone_pid - wait $serverone_pid -} - -@test "sql-server-cluster: a new primary comes up, old primary becomes a standby and has its state overwritten" { - cd serverone - - cd repo1 - dolt sql -q 'create table vals (i int primary key)' - dolt sql -q 'insert into vals values (1),(2),(3),(4),(5)' - cd ../ - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - serverone_pid=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - cd ../servertwo - - cd repo1 - dolt sql -q 'create table vals (i int primary key)' - dolt sql -q 'insert into vals values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)' - cd ../ - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERTWO_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERONE_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 15 - remotesapi: - port: ${SERVERTWO_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - servertwo_pid=$! - - wait_for_connection "${SERVERTWO_MYSQL_PORT}" 5000 - - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nstandby,15" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "SELECT count(*) FROM vals" "count(*)\n10" - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" "@@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch\nprimary,15" - - kill $servertwo_pid - wait $servertwo_pid - - kill $serverone_pid - wait $serverone_pid -} - -@test "sql-server-cluster: primary -> standby without the standby up fails" { - cd serverone - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 10 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - (cd repo1 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo1) - (cd repo2 && dolt remote add standby http://localhost:"${SERVERTWO_GRPC_PORT}"/repo2) - dolt sql-server --config server.yaml & - SERVER_PID=$! - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - # when we do this, we become read only for a little bit... - run server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "call dolt_assume_cluster_role('standby', '11');" "" 1 - [[ "$output" =~ "failed to transition from primary to standby gracefully" ]] || false - - # after that fails, we should still be primary and we should accept writes. - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "create table vals (i int primary key);insert into vals values (0);select i from vals" ";;i\n0" -} - -@test "sql-server-cluster: primary replicates to standby, fails over, new primary replicates to standby, fails over, new primary has all writes" { - cd serverone - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERONE_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERTWO_GRPC_PORT}/{database} - bootstrap_role: standby - bootstrap_epoch: 1 - remotesapi: - port: ${SERVERONE_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERTWO_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - serverone_pid=$! - - wait_for_connection "${SERVERONE_MYSQL_PORT}" 5000 - - cd ../servertwo - - echo " -log_level: trace -user: - name: dolt -listener: - host: 0.0.0.0 - port: ${SERVERTWO_MYSQL_PORT} -behavior: - read_only: false - autocommit: true -cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:${SERVERONE_GRPC_PORT}/{database} - bootstrap_role: primary - bootstrap_epoch: 1 - remotesapi: - port: ${SERVERTWO_GRPC_PORT}" > server.yaml - - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.email bats@email.fake - DOLT_ROOT_PATH=`pwd` dolt config --global --add user.name "Bats Tests" - DOLT_ROOT_PATH=`pwd` dolt config --global --add metrics.disabled true - - (cd repo1 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo1) - (cd repo2 && dolt remote add standby http://localhost:${SERVERONE_GRPC_PORT}/repo2) - DOLT_ROOT_PATH=`pwd` dolt sql-server --config server.yaml & - servertwo_pid=$! - - wait_for_connection "${SERVERTWO_MYSQL_PORT}" 5000 - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "create table vals (i int primary key);insert into vals values (0),(1),(2),(3),(4);call dolt_assume_cluster_role('standby', 2);" ";;status\n0" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals;call dolt_assume_cluster_role('primary', 2)" "count(*)\n5;status\n0" - server_query_with_port "${SERVERONE_MYSQL_PORT}" repo1 1 dolt "" "insert into vals values (5),(6),(7),(8),(9);call dolt_assume_cluster_role('standby', 3)" ";status\n0" - server_query_with_port "${SERVERTWO_MYSQL_PORT}" repo1 1 dolt "" "select count(*) from vals;call dolt_assume_cluster_role('primary', 3)" "count(*)\n10;status\n0" - - kill $servertwo_pid - wait $servertwo_pid - - kill $serverone_pid - wait $serverone_pid -} diff --git a/integration-tests/go-sql-server-driver/cmd.go b/integration-tests/go-sql-server-driver/cmd.go index fd85c256b7..15d164e6d2 100644 --- a/integration-tests/go-sql-server-driver/cmd.go +++ b/integration-tests/go-sql-server-driver/cmd.go @@ -35,7 +35,7 @@ const TestUserName = "Bats Tests" const TestEmailAddress = "bats@email.fake" const ConnectAttempts = 50 -const RetrySleepDuration = 10 * time.Millisecond +const RetrySleepDuration = 50 * time.Millisecond func init() { var err error diff --git a/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml b/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml index 0049beec65..152af24c86 100644 --- a/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml +++ b/integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml @@ -129,7 +129,7 @@ tests: rows: [["0"]] # Connection should be broken now. - query: "select 2 from dual" - error_match: "this connection no longer be used" + error_match: "this connection can no longer be used" - on: server1 queries: - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" @@ -570,3 +570,374 @@ tests: - exec: "use repo1" - exec: "create table more_vals (i int primary key)" error_match: "repo1 is read-only" +- name: an older primary comes up, becomes a standby and does not overwrite newer primary + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: preserver.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: primary + bootstrap_epoch: 15 + remotesapi: + port: 50051 + server: + args: ["--config", "preserver.yaml"] + port: 3309 + - name: server2 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50051/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50051/repo2 + with_files: + - name: preserver.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50051/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 50052 + server: + args: ["--config", "preserver.yaml"] + port: 3310 + connections: + - on: server1 + queries: + - exec: 'use repo1' + - exec: 'create table vals (i int primary key)' + - exec: 'insert into vals values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)' + - on: server2 + queries: + - exec: 'use repo1' + - exec: 'create table vals (i int primary key)' + - exec: 'insert into vals values (1),(2),(3),(4),(5)' + - on: server1 + restart_server: + args: ["--config", "server.yaml"] + - on: server1 + - on: server2 + restart_server: + args: ["--config", "server.yaml"] + - on: server1 + queries: + - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster.dolt_cluster_status order by `database` asc" + result: + columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] + rows: + - ["repo1","standby","primary","15","0","NULL"] + - ["repo2","standby","primary","15","0","NULL"] + retry_attempts: 100 + - query: "SELECT count(*) FROM repo1.vals" + result: + columns: ["count(*)"] + rows: [["10"]] + - on: server2 + queries: + - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" + result: + columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] + rows: [["standby", "15"]] + - query: "SELECT count(*) FROM repo1.vals" + result: + columns: ["count(*)"] + rows: [["10"]] +- name: a newer primary comes up, old primary becomes a standby has its state overwritten + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: preserver.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 50051 + server: + args: ["--config", "preserver.yaml"] + port: 3309 + - name: server2 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50051/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50051/repo2 + with_files: + - name: preserver.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50051/{database} + bootstrap_role: primary + bootstrap_epoch: 15 + remotesapi: + port: 50052 + server: + args: ["--config", "preserver.yaml"] + port: 3310 + connections: + - on: server1 + queries: + - exec: 'use repo1' + - exec: 'create table vals (i int primary key)' + - exec: 'insert into vals values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)' + - on: server2 + queries: + - exec: 'use repo1' + - exec: 'create table vals (i int primary key)' + - exec: 'insert into vals values (1),(2),(3),(4),(5)' + - on: server1 + restart_server: + args: ["--config", "server.yaml"] + - on: server1 + queries: + - query: "SELECT count(*) FROM repo1.vals" + result: + columns: ["count(*)"] + rows: [["10"]] + - on: server2 + restart_server: + args: ["--config", "server.yaml"] + - on: server2 + queries: + - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster.dolt_cluster_status order by `database` asc" + result: + columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] + rows: + - ["repo1","standby","primary","15","0","NULL"] + - ["repo2","standby","primary","15","0","NULL"] + retry_attempts: 100 + - query: "SELECT count(*) FROM repo1.vals" + result: + columns: ["count(*)"] + rows: [["5"]] + - on: server1 + queries: + - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" + result: + columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] + rows: [["standby", "15"]] + - query: "SELECT count(*) FROM repo1.vals" + result: + columns: ["count(*)"] + rows: [["5"]] +- name: graceful primary to standby transition without the standby up fails + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: primary + bootstrap_epoch: 10 + remotesapi: + port: 50051 + server: + args: ["--config", "server.yaml"] + port: 3309 + connections: + - on: server1 + queries: + - exec: "use repo1" + - query: "call dolt_assume_cluster_role('standby', '11')" + error_match: failed to transition from primary to standby gracefully + - exec: "create table vals (i int primary key)" + - exec: "insert into vals values (0)" +- name: primary replicates to standby, fails over, new primary replicates to standby, fails over, new primary has all writes + multi_repos: + - name: server1 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50052/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50052/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50052/{database} + bootstrap_role: primary + bootstrap_epoch: 1 + remotesapi: + port: 50051 + server: + args: ["--config", "server.yaml"] + port: 3309 + - name: server2 + repos: + - name: repo1 + with_remotes: + - name: standby + url: http://localhost:50051/repo1 + - name: repo2 + with_remotes: + - name: standby + url: http://localhost:50051/repo2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:50051/{database} + bootstrap_role: standby + bootstrap_epoch: 1 + remotesapi: + port: 50052 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server1 + queries: + - exec: 'use repo1' + - exec: 'create table vals (i int primary key)' + - exec: 'insert into vals values (0),(1),(2),(3),(4)' + - query: "call dolt_assume_cluster_role('standby', 2)" + result: + columns: ["status"] + rows: [["0"]] + - on: server2 + queries: + - exec: 'use repo1' + - query: "select count(*) from vals" + result: + columns: ["count(*)"] + rows: [["5"]] + - query: "call dolt_assume_cluster_role('primary', 2)" + result: + columns: ["status"] + rows: [["0"]] + - on: server2 + queries: + - exec: 'use repo1' + - exec: 'insert into vals values (5),(6),(7),(8),(9)' + - query: "call dolt_assume_cluster_role('standby', 3)" + result: + columns: ["status"] + rows: [["0"]] + - on: server1 + queries: + - exec: 'use repo1' + - query: "select count(*) from vals" + result: + columns: ["count(*)"] + rows: [["10"]] + - query: "call dolt_assume_cluster_role('primary', 3)" + result: + columns: ["status"] + rows: [["0"]] + - on: server1 + queries: + - exec: 'use repo1' + - exec: 'insert into vals values (10),(11),(12),(13),(14)' + - query: "select count(*) from vals" + result: + columns: ["count(*)"] + rows: [["15"]]