From f1c12765b03dffdcbd3e4f596bb5136a46b977d1 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 8 May 2024 10:27:55 -0700 Subject: [PATCH] Fixing issue where newly cloned databases weren't being configured for remote-based replication pulling --- .../doltcore/sqle/database_provider.go | 13 ++++- go/libraries/doltcore/sqle/replication.go | 2 +- .../bats/replication-multidb.bats | 55 +++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 2be0c31488..b67816168f 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -461,7 +461,7 @@ func (p *DoltDatabaseProvider) CreateCollatedDatabase(ctx *sql.Context, name str type InitDatabaseHook func(ctx *sql.Context, pro *DoltDatabaseProvider, name string, env *env.DoltEnv, db dsess.SqlDatabase) error type DropDatabaseHook func(ctx *sql.Context, name string) -// ConfigureReplicationDatabaseHook sets up replication for a newly created database as necessary +// ConfigureReplicationDatabaseHook sets up the hooks to push to a remote to replicate a newly created database. // TODO: consider the replication heads / all heads setting func ConfigureReplicationDatabaseHook(ctx *sql.Context, p *DoltDatabaseProvider, name string, newEnv *env.DoltEnv, _ dsess.SqlDatabase) error { _, replicationRemoteName, _ := sql.SystemVariables.GetGlobal(dsess.ReplicateToRemote) @@ -723,8 +723,17 @@ func (p *DoltDatabaseProvider) registerNewDatabase(ctx *sql.Context, name string } } + mrEnv, err := env.MultiEnvForSingleEnv(ctx, newEnv) + if err != nil { + return err + } + dbs, err := ApplyReplicationConfig(ctx, sql.NewBackgroundThreads(), mrEnv, cli.CliErr, db) + if err != nil { + return err + } + formattedName := formatDbMapKeyName(db.Name()) - p.databases[formattedName] = db + p.databases[formattedName] = dbs[0] p.dbLocations[formattedName] = newEnv.FS return nil } diff --git a/go/libraries/doltcore/sqle/replication.go b/go/libraries/doltcore/sqle/replication.go index 017ac64b25..6c9bb3d6a5 100644 --- a/go/libraries/doltcore/sqle/replication.go +++ b/go/libraries/doltcore/sqle/replication.go @@ -120,7 +120,7 @@ func ApplyReplicationConfig(ctx context.Context, bThreads *sql.BackgroundThreads for i, db := range dbs { dEnv := mrEnv.GetEnv(db.Name()) if dEnv == nil { - outputDbs = append(outputDbs, db) + outputDbs[i] = db continue } postCommitHooks, err := GetCommitHooks(ctx, bThreads, dEnv, logger) diff --git a/integration-tests/bats/replication-multidb.bats b/integration-tests/bats/replication-multidb.bats index 535ddf2e8c..81297a88c2 100644 --- a/integration-tests/bats/replication-multidb.bats +++ b/integration-tests/bats/replication-multidb.bats @@ -328,6 +328,61 @@ SQL [[ ! "$output" =~ "t2" ]] || false } +# Test that a Dolt sql-server automatically pulls new commits from a remote for new databases +# that have been automatically cloned. +@test "replication-multidb: sql-server pull on read for new databases" { + # Create test data in repo1 and push it to remote1 + cd "${TMPDIRS}/dbs1/repo1" + dolt sql -q "create table t1 (pk int primary key); insert into t1 values (42);" + dolt commit -Am "creating table t1" + dolt push remote1 main:main + + # Create test data in repo2 and push it to remote1 + cd "${TMPDIRS}/dbs1/repo2" + dolt sql -q "create table t2 (pk int primary key); insert into t2 values (123);" + dolt commit -Am "creating table t2" + dolt push remote1 main:main + + # Start a read replica server that replicates from the remotes on disk + mkdir "${TMPDIRS}/readReplicaServer" + cd "${TMPDIRS}/readReplicaServer" + dolt config --global --add sqlserver.global.dolt_read_replica_remote remote1 + dolt config --global --add sqlserver.global.dolt_replicate_heads main + dolt sql -q "set @@persist.dolt_replication_remote_url_template = 'file://$TMPDIRS/rem1/{database}'" + start_multi_db_server information_schema + + # Assert that no databases are synced to the read replica server yet + run dolt -u dolt --port $PORT --host 0.0.0.0 --no-tls sql -q "show databases" + [ $status -eq 0 ] + [[ "$output" =~ "information_schema" ]] || false + [[ ! "$output" =~ "repo1" ]] || false + [[ ! "$output" =~ "repo2" ]] || false + + # Sync repo1 to the read replica server by use'ing it + run dolt -u dolt --port $PORT --host 0.0.0.0 --no-tls sql -q "use repo1; show databases;" + [ $status -eq 0 ] + [[ "$output" =~ "information_schema" ]] || false + [[ "$output" =~ "repo1" ]] || false + [[ ! "$output" =~ "repo2" ]] || false + + # Sync repo1 by using it + run dolt --use-db repo1 -u dolt --port $PORT --host 0.0.0.0 --no-tls sql -q "select * from t1;" + [ $status -eq 0 ] + [[ "$output" =~ "42" ]] || false + + # Push a new change to repo1 + cd "${TMPDIRS}/dbs1/repo1" + dolt sql -q "insert into t1 values (43);" + dolt commit -Am "Adding row 43" + dolt push remote1 main:main + + # Verify that the changes from repo1 have been pulled + run dolt --use-db repo1 -u dolt --port $PORT --host 0.0.0.0 --no-tls sql -q "select * from t1;" + [ $status -eq 0 ] + [[ "$output" =~ "42" ]] || false + [[ "$output" =~ "43" ]] || false +} + @test "replication-multidb: sql-server pull on read" { push_helper $TMPDIRS dolt --data-dir=dbs1 sql -b -q "use repo1; create table t1 (a int primary key)"