tests: - name: persisted role and epoch take precedence over bootstrap values multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: standby_server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: standby bootstrap_epoch: 10 remotesapi: port: 3851 - name: primary_server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 15 remotesapi: port: 3851 server: args: ["--config", "standby_server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "use dolt_cluster" - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role","@@GLOBAL.dolt_cluster_role_epoch"] rows: [["standby","10"]] - query: | select `database`, standby_remote, role, epoch from dolt_cluster_status order by `database` asc result: columns: ["database","standby_remote","role","epoch"] rows: - ["repo1","standby","standby","10"] - ["repo2","standby","standby","10"] restart_server: args: ["--config", "primary_server.yaml"] - on: server1 queries: - exec: "use dolt_cluster" - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role","@@GLOBAL.dolt_cluster_role_epoch"] rows: [["standby","10"]] - name: dolt_assume_cluster_role multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: standby bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "use repo1" - query: "call dolt_assume_cluster_role('standby', '9')" error_match: "error assuming role" - query: "call dolt_assume_cluster_role('primary', '10')" error_match: "error assuming role" - query: "call dolt_assume_cluster_role('backup', '11')" error_match: "error assuming role" - query: "call dolt_assume_cluster_role('standby', '10')" result: columns: ["status"] rows: [["0"]] - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role","@@GLOBAL.dolt_cluster_role_epoch"] rows: [["standby","10"]] - query: "call dolt_assume_cluster_role('standby', '12')" result: columns: ["status"] rows: [["0"]] - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role","@@GLOBAL.dolt_cluster_role_epoch"] rows: [["standby","12"]] - query: "call dolt_assume_cluster_role('primary', '13')" result: columns: ["status"] rows: [["0"]] # Connection should be broken now. - query: "select 2 from dual" error_match: "this connection can no longer be used" - on: server1 queries: - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role","@@GLOBAL.dolt_cluster_role_epoch"] rows: [["primary","13"]] restart_server: {} # Assert that it comes back up with newly assumed role. - on: server1 queries: - query: "select @@GLOBAL.dolt_cluster_role, @@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role","@@GLOBAL.dolt_cluster_role_epoch"] rows: [["primary","13"]] - name: create database makes a new remote multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "create database a_new_database" - exec: "use a_new_database" - query: "select name, url from dolt_remotes" result: columns: ["name","url"] rows: [["standby","http://localhost:3852/a_new_database"]] - name: primary comes up and replicates to standby multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--port", "3309"] port: 3309 - name: server2 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3851/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3851/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 10 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" - exec: "insert into vals values (1),(2),(3),(4),(5)" restart_server: args: ["--config", "server.yaml"] - on: server1 queries: - exec: "use dolt_cluster" - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster_status order by `database` asc" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1","standby","primary","10","0","NULL"] - ["repo2","standby","primary","10","0","NULL"] retry_attempts: 100 - on: server2 queries: - exec: "use repo1" - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["5"]] - name: booted standby server is read only multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: standby_server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: standby bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "standby_server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" error_match: "repo1 is read-only" - name: booted primary server is read write multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" - name: standby server takes primary epoch multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--port", "3309"] port: 3309 - name: server2 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3851/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3851/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" - exec: "insert into vals values (1),(2),(3),(4),(5)" restart_server: args: ["--config", "server.yaml"] - on: server1 queries: - exec: "use dolt_cluster" - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster_status order by `database` asc" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1","standby","primary","10","0","NULL"] - ["repo2","standby","primary","10","0","NULL"] retry_attempts: 100 - on: server2 queries: - exec: "use repo1" - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["5"]] - query: "select @@global.dolt_cluster_role_epoch" result: columns: ["@@global.dolt_cluster_role_epoch"] rows: [["10"]] - on: server1 queries: - exec: "use repo1" - exec: "call dolt_assume_cluster_role('primary', 11)" - exec: "insert into vals values (6),(7),(8),(9),(10)" - exec: "use dolt_cluster" - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster_status order by `database` asc" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1","standby","primary","11","0","NULL"] - ["repo2","standby","primary","11","0","NULL"] retry_attempts: 100 - on: server2 queries: - exec: "use repo1" - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["10"]] - query: "select @@global.dolt_cluster_role_epoch" result: columns: ["@@global.dolt_cluster_role_epoch"] rows: [["11"]] - name: standby transitioned to primary becomes read write multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: standby_server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: standby bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "standby_server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" error_match: "repo1 is read-only" - query: "call dolt_assume_cluster_role('primary', 11)" result: columns: ["status"] rows: [["0"]] - on: server1 queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["0"]] - name: primary transitioned to standby becomes read only multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3851/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3851/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 10 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: "use repo1" - exec: "create table vals (i int primary key)" - exec: "insert into vals values (1),(2),(3),(4),(5)" - query: "call dolt_assume_cluster_role('standby', 11)" result: columns: ["status"] rows: [["0"]] - on: server1 queries: - exec: "use repo1" - exec: "insert into vals values (6),(7),(8),(9),(10)" error_match: "repo1 is read-only" - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["5"]] - name: misconfigured cluster with primaries at same epoch, both transition to detected_broken_config multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3851/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3851/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 retry_attempts: 100 queries: - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] rows: [["detected_broken_config", "10"]] - exec: "use repo1" - exec: "create table more_vals (i int primary key)" error_match: "repo1 is read-only" - on: server2 retry_attempts: 100 queries: - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] rows: [["detected_broken_config", "10"]] - exec: "use repo1" - exec: "create table more_vals (i int primary key)" error_match: "repo1 is read-only" - name: an older primary comes up, becomes a standby and does not overwrite newer primary multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: preserver.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 15 remotesapi: port: 3851 server: args: ["--config", "preserver.yaml"] port: 3309 - name: server2 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3851/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3851/repo2 with_files: - name: preserver.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3852 server: args: ["--config", "preserver.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'use repo1' - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)' - on: server2 queries: - exec: 'use repo1' - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (1),(2),(3),(4),(5)' - on: server1 restart_server: args: ["--config", "server.yaml"] - on: server1 - on: server2 restart_server: args: ["--config", "server.yaml"] - on: server1 queries: - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster.dolt_cluster_status order by `database` asc" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1","standby","primary","15","0","NULL"] - ["repo2","standby","primary","15","0","NULL"] retry_attempts: 100 - query: "SELECT count(*) FROM repo1.vals" result: columns: ["count(*)"] rows: [["10"]] - on: server2 queries: - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] rows: [["standby", "15"]] - query: "SELECT count(*) FROM repo1.vals" result: columns: ["count(*)"] rows: [["10"]] - name: a newer primary comes up, old primary becomes a standby has its state overwritten multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: preserver.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "preserver.yaml"] port: 3309 - name: server2 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3851/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3851/repo2 with_files: - name: preserver.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: primary bootstrap_epoch: 15 remotesapi: port: 3852 server: args: ["--config", "preserver.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'use repo1' - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)' - on: server2 queries: - exec: 'use repo1' - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (1),(2),(3),(4),(5)' - on: server1 restart_server: args: ["--config", "server.yaml"] - on: server1 queries: - query: "SELECT count(*) FROM repo1.vals" result: columns: ["count(*)"] rows: [["10"]] - on: server2 restart_server: args: ["--config", "server.yaml"] - on: server2 queries: - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster.dolt_cluster_status order by `database` asc" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1","standby","primary","15","0","NULL"] - ["repo2","standby","primary","15","0","NULL"] retry_attempts: 100 - query: "SELECT count(*) FROM repo1.vals" result: columns: ["count(*)"] rows: [["5"]] - on: server1 queries: - query: "SELECT @@GLOBAL.dolt_cluster_role,@@GLOBAL.dolt_cluster_role_epoch" result: columns: ["@@GLOBAL.dolt_cluster_role", "@@GLOBAL.dolt_cluster_role_epoch"] rows: [["standby", "15"]] - query: "SELECT count(*) FROM repo1.vals" result: columns: ["count(*)"] rows: [["5"]] - name: graceful primary to standby transition without the standby up fails multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 10 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "use repo1" - query: "call dolt_assume_cluster_role('standby', '11')" error_match: failed to transition from primary to standby gracefully - exec: "create table vals (i int primary key)" - exec: "insert into vals values (0)" - name: dolt_cluster_transition_to_standby multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'create database repo1' - exec: "use repo1" - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (0),(1),(2),(3),(4)' - query: "call dolt_cluster_transition_to_standby('2', '1')" result: columns: ["caught_up", "database", "remote", "remote_url"] rows: - ["1", "repo1", "standby", "http://localhost:3852/repo1"] - ["1", "mysql", "standby", "http://localhost:3852"] - ["1", "dolt_branch_control", "standby", "http://localhost:3852"] - name: dolt_cluster_transition_to_standby too many standbys provided multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'create database repo1' - exec: "use repo1" - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (0),(1),(2),(3),(4)' - query: "call dolt_cluster_transition_to_standby('2', '2')" error_match: failed to transition from primary to standby gracefully; could not ensure 2 replicas were caught up on all 3 databases. Only caught up 1 standbys fully. - name: create new database, primary replicates to standby, fails over, new primary replicates to standby, fails over, new primary has all writes multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (0),(1),(2),(3),(4)' - query: "call dolt_assume_cluster_role('standby', 2)" result: columns: ["status"] rows: [["0"]] - on: server2 queries: - exec: 'use repo1' - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["5"]] - query: "call dolt_assume_cluster_role('primary', 2)" result: columns: ["status"] rows: [["0"]] - on: server2 queries: - exec: 'use repo1' - exec: 'insert into vals values (5),(6),(7),(8),(9)' - query: "call dolt_assume_cluster_role('standby', 3)" result: columns: ["status"] rows: [["0"]] - on: server1 queries: - exec: 'use repo1' - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["10"]] - query: "call dolt_assume_cluster_role('primary', 3)" result: columns: ["status"] rows: [["0"]] - on: server1 queries: - exec: 'use repo1' - exec: 'insert into vals values (10),(11),(12),(13),(14)' - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["15"]] - name: last_updated heartbeats multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'create database repo1' - query: "call dolt_assume_cluster_role('standby', 2)" result: columns: ["status"] rows: [["0"]] - on: server1 queries: - query: "call dolt_assume_cluster_role('primary', 3)" result: columns: ["status"] rows: [["0"]] - on: server2 queries: - query: "SELECT TIMESTAMPDIFF(SECOND, CONVERT_TZ(last_update, 'GMT', @@GLOBAL.time_zone), NOW()) < 5 AS within_threshold FROM dolt_cluster.dolt_cluster_status;" result: columns: ["within_threshold"] rows: [["1"]] - query: "SELECT SLEEP(5)" result: columns: ["SLEEP(5)"] rows: [["0"]] - query: "SELECT TIMESTAMPDIFF(SECOND, CONVERT_TZ(last_update, 'GMT', @@GLOBAL.time_zone), NOW()) < 5 AS within_threshold FROM dolt_cluster.dolt_cluster_status;" result: columns: ["within_threshold"] rows: [["1"]] - name: create new database, clone a database, primary replicates to standby, standby has both databases multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (i int primary key)' - exec: 'insert into vals values (0),(1),(2),(3),(4)' - exec: 'call dolt_commit("-Am", "create a commit")' - exec: 'call dolt_remote("add", "origin", "file://my_remote/")' - exec: 'call dolt_push("origin", "main:main")' - exec: 'call dolt_clone("file://repo1/my_remote", "repo_cloned")' - query: "call dolt_assume_cluster_role('standby', 2)" result: columns: ["status"] rows: [["0"]] - on: server2 queries: - exec: 'use repo1' - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["5"]] - exec: 'use repo_cloned' - query: "select count(*) from vals" result: columns: ["count(*)"] rows: [["5"]] - name: dolt_cluster_ack_writes_timeout_secs behavior multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 # This test writes new commits on the primary and quickly checks for them on # the secondary. If dolt_cluster_ack_writes_timeout_secs is working as # intended, the new writes will always be present. connections: - on: server1 queries: - exec: 'SET @@PERSIST.dolt_cluster_ack_writes_timeout_secs = 10' - exec: 'CREATE DATABASE repo1' - exec: 'USE repo1' - exec: 'CREATE TABLE vals (i INT PRIMARY KEY)' - on: server2 queries: - exec: 'USE repo1' - query: "SHOW TABLES" result: columns: ["Tables_in_repo1"] rows: [["vals"]] - on: server1 queries: - exec: 'USE repo1' - exec: 'INSERT INTO vals VALUES (0),(1),(2),(3),(4)' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM vals' result: columns: ["COUNT(*)"] rows: [["5"]] - on: server1 queries: - exec: 'USE repo1' - exec: 'INSERT INTO vals VALUES (5),(6),(7),(8),(9)' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM vals' result: columns: ["COUNT(*)"] rows: [["10"]] # Restart both servers to test the behavior of the persisted variable. - on: server1 restart_server: {} - on: server2 restart_server: {} - on: server1 queries: - exec: 'USE repo1' - exec: 'INSERT INTO vals VALUES (10),(11),(12),(13),(14)' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM vals' result: columns: ["COUNT(*)"] rows: [["15"]] - on: server1 queries: - exec: 'USE repo1' - exec: 'INSERT INTO vals VALUES (15),(16),(17),(18),(19)' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM vals' result: columns: ["COUNT(*)"] rows: [["20"]] # Assert that branch creation and deletion also blocks on replication. - on: server1 queries: - exec: 'USE repo1' - exec: 'CALL DOLT_BRANCH("new_branch")' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM dolt_branches' result: columns: ["COUNT(*)"] rows: [["2"]] - on: server1 queries: - exec: 'USE repo1' - exec: 'CALL DOLT_BRANCH("-d", "new_branch")' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM dolt_branches' result: columns: ["COUNT(*)"] rows: [["1"]] - on: server1 queries: - exec: 'USE repo1' - exec: 'CALL DOLT_BRANCH("new_branch")' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM dolt_branches' result: columns: ["COUNT(*)"] rows: [["2"]] - on: server1 queries: - exec: 'USE repo1' - exec: 'CALL DOLT_BRANCH("-d", "new_branch")' - on: server2 queries: - exec: 'USE repo1' - query: 'SELECT COUNT(*) FROM dolt_branches' result: columns: ["COUNT(*)"] rows: [["1"]] - name: call dolt checkout multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 connections: - on: server1 queries: - exec: 'create database repo1' - exec: 'use repo1' - exec: 'call dolt_branch("-c", "main", "new_branch_name")' - on: server1 queries: - exec: 'use repo1' - exec: 'call dolt_checkout("new_branch_name")' - name: call dolt gc multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (id int primary key, val int)' - exec: 'insert into vals values (1,1)' - exec: 'insert into vals values (2,2)' - exec: 'insert into vals values (3,3)' - exec: 'insert into vals values (4,4)' - exec: 'call dolt_gc()' - exec: 'select * from vals' error_match: "this connection can no longer be used" - on: server1 queries: - query: "select `database`, standby_remote, role, epoch, replication_lag_millis, current_error from dolt_cluster.dolt_cluster_status order by `database` asc" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1","standby","primary","1","0","NULL"] retry_attempts: 100 - on: server2 queries: - exec: 'use repo1' - query: "select * from vals order by id asc" result: columns: ["id","val"] rows: - [1,1] - [2,2] - [3,3] - [4,4] - exec: 'call dolt_gc()' error_match: "must be the primary" - exec: 'call dolt_gc("--shallow")' - name: dropped database is no longer present on replica multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'SET @@GLOBAL.dolt_cluster_ack_writes_timeout_secs = 10' - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (i int primary key)' - exec: 'create database repo2' - exec: 'use repo2' - exec: 'create table vals (i int primary key)' - query: "select `database`, standby_remote, role, epoch, replication_lag_millis is not null as `replication_lag_millis`, current_error from dolt_cluster.dolt_cluster_status" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1", "standby", "primary", "1", "1", "NULL"] - ["repo2", "standby", "primary", "1", "1", "NULL"] - exec: 'use repo1' - exec: 'drop database repo2' - exec: 'insert into vals values (0),(1),(2),(3),(4)' - query: 'show databases' result: columns: ["Database"] rows: - ["dolt_cluster"] - ["information_schema"] - ["mysql"] - ["repo1"] - query: "select `database`, standby_remote, role, epoch, replication_lag_millis is not null as `replication_lag_millis`, current_error from dolt_cluster.dolt_cluster_status" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: - ["repo1", "standby", "primary", "1", "1", "NULL"] - on: server2 queries: - query: 'show databases' result: columns: ["Database"] rows: - ["dolt_cluster"] - ["information_schema"] - ["mysql"] - ["repo1"] retry_attempts: 100 - query: "select `database`, standby_remote, role, epoch from dolt_cluster.dolt_cluster_status" result: columns: ["database","standby_remote","role","epoch",] rows: - ["repo1", "standby", "standby", "1"] - name: undropped database on primary is replicated to standby multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'SET @@GLOBAL.dolt_cluster_ack_writes_timeout_secs = 10' - exec: 'create database repo1' - exec: 'create database droppedrepo' - exec: 'use droppedrepo' - exec: 'create table vals (i int primary key)' - exec: 'use repo1' - exec: 'drop database droppedrepo' - exec: 'create table vals (i int primary key)' - on: server2 queries: - exec: 'use repo1' - exec: 'call dolt_undrop("droppedrepo")' error_match: "database server is set to read only mode" - on: server1 queries: - exec: 'use repo1' - exec: 'call dolt_undrop("droppedrepo")' - exec: 'use droppedrepo' - exec: 'insert into vals values (0),(1),(2)' - on: server2 queries: - exec: 'use droppedrepo' - query: 'select count(*) as cnt from vals' result: columns: ["cnt"] rows: [["3"]] - name: dropped database recreated quiesces to expected state multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - exec: 'SET @@GLOBAL.dolt_cluster_ack_writes_timeout_secs = 10' - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (a int)' - exec: 'use mysql' - exec: 'drop database repo1' - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (b int)' - exec: 'use mysql' - exec: 'drop database repo1' - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (c int)' - exec: 'use mysql' - exec: 'drop database repo1' - query: "select `database`, standby_remote, role, epoch, replication_lag_millis is not null as `replication_lag_millis`, current_error from dolt_cluster.dolt_cluster_status" result: columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"] rows: [] - on: server2 queries: - query: 'show databases' result: columns: ["Database"] rows: - ["dolt_cluster"] - ["information_schema"] - ["mysql"] retry_attempts: 100 - query: "select `database`, standby_remote, role, epoch from dolt_cluster.dolt_cluster_status" result: columns: ["database","standby_remote","role","epoch",] rows: [] - on: server1 queries: - exec: 'create database repo1' - exec: 'use repo1' - exec: 'create table vals (d int)' - on: server2 queries: - query: 'show databases' result: columns: ["Database"] rows: - ["dolt_cluster"] - ["information_schema"] - ["mysql"] - ["repo1"] - exec: 'use repo1' - query: "select d from vals" result: columns: ["d"] rows: [] # Assert that we can gracefully transition to standby even when there are no # dolt databases which we are replicating. - name: dolt_cluster_transition_to_standby no dolt databases exist multi_repos: - name: server1 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 - name: server2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "server.yaml"] port: 3310 connections: - on: server1 queries: - query: "show databases" result: columns: ["Database"] rows: - ["dolt_cluster"] - ["information_schema"] - ["mysql"] - exec: "call dolt_cluster_transition_to_standby('2', '1')" - on: server2 queries: - exec: "call dolt_assume_cluster_role('primary', '2')" - on: server2 queries: - exec: "call dolt_cluster_transition_to_standby('3', '1')" - on: server1 queries: - exec: "call dolt_assume_cluster_role('primary', '3')" # Assert that we can add a new standby to an existing server and bring the server up. - name: adding new remotes to the config creates those remotes on startup multi_repos: - name: server1 with_files: - name: onestandby.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby_one remote_url_template: http://localhost:3852/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 - name: twostandbys.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby_one remote_url_template: http://localhost:3852/{database} - name: standby_two remote_url_template: http://localhost:3853/{database} bootstrap_role: primary bootstrap_epoch: 1 remotesapi: port: 3851 server: args: ["--config", "onestandby.yaml"] port: 3309 - name: standby_one with_files: - name: config.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3310 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3852 server: args: ["--config", "config.yaml"] port: 3310 - name: standby_two with_files: - name: config.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3311 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3851/{database} bootstrap_role: standby bootstrap_epoch: 1 remotesapi: port: 3853 server: args: ["--config", "config.yaml"] port: 3311 connections: - on: server1 queries: - exec: 'SET @@GLOBAL.dolt_cluster_ack_writes_timeout_secs = 10' - exec: "create database testdb" - exec: "use testdb" - exec: "create table vals (id int primary key)" - on: server1 restart_server: args: ["--config", "twostandbys.yaml"] - on: server1 queries: - exec: "use testdb" - query: "select count(*) from dolt_remotes" result: columns: ["count(*)"] rows: - ["2"] - exec: 'SET @@GLOBAL.dolt_cluster_ack_writes_timeout_secs = 10' - exec: "insert into vals values (0),(1),(2)" - query: "show warnings" result: columns: ["Level", "Code", "Message"] rows: [] - on: standby_two queries: - query: "select count(*) from `testdb`.`vals`" result: columns: ["count(*)"] rows: - ["3"] - name: clusterdb doesn't panic for nil write session multi_repos: - name: server1 repos: - name: repo1 with_remotes: - name: standby url: http://localhost:3852/repo1 - name: repo2 with_remotes: - name: standby url: http://localhost:3852/repo2 with_files: - name: server.yaml contents: | log_level: trace listener: host: 0.0.0.0 port: 3309 cluster: standby_remotes: - name: standby remote_url_template: http://localhost:3852/{database} bootstrap_role: primary remotesapi: port: 3851 server: args: ["--config", "server.yaml"] port: 3309 connections: - on: server1 queries: - exec: "set foreign_key_checks=0"