Merge remote-tracking branch 'origin/main' into taylor/two-dot-log-cli

integration-tests/bats/helper/query-server-common.bash

@@ -113,7 +113,7 @@ wait_for_connection(port=int(port_str), timeout_ms=int(timeout_ms), database=dat
 
 start_sql_server() {
     DEFAULT_DB="$1"
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server --host 0.0.0.0 --port=$PORT --user dolt &
     SERVER_PID=$!
     wait_for_connection $PORT 5000
@@ -124,7 +124,7 @@ start_sql_server() {
 # this func)
 start_sql_server_with_args() {
     DEFAULT_DB=""
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server "$@" --port=$PORT &
     SERVER_PID=$!
     wait_for_connection $PORT 5000
@@ -132,7 +132,7 @@ start_sql_server_with_args() {
 
 start_sql_server_with_config() {
     DEFAULT_DB="$1"
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     echo "
 log_level: debug
 
@@ -155,7 +155,7 @@ behavior:
 
 start_sql_multi_user_server() {
     DEFAULT_DB="$1"
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     echo "
 log_level: debug
 
@@ -177,7 +177,7 @@ behavior:
 
 start_multi_db_server() {
     DEFAULT_DB="$1"
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server --host 0.0.0.0 --port=$PORT --user dolt --data-dir ./ &
     SERVER_PID=$!
     wait_for_connection $PORT 5000
@@ -186,15 +186,19 @@ start_multi_db_server() {
 # stop_sql_server stops the SQL server. For cases where it's important
 # to wait for the process to exit after the kill signal (e.g. waiting
 # for an async replication push), pass 1.
+# kill the process if it's still running
 stop_sql_server() {
     wait=$1
     if [ ! -z "$SERVER_PID" ]; then
-      kill $SERVER_PID
-      if [ $wait ]; then
-          while ps -p $SERVER_PID > /dev/null; do
-              sleep .1;
-          done
-      fi;
+      serverpidinuse=$(lsof -i -P -n | grep LISTEN | grep $SERVER_PID | wc -l)
+      if [ $serverpidinuse -gt 0 ]; then
+        kill $SERVER_PID
+        if [ $wait ]; then
+            while ps -p $SERVER_PID > /dev/null; do
+                sleep .1;
+            done
+        fi;
+      fi
     fi
     SERVER_PID=
 }
@@ -233,7 +237,7 @@ stop_sql_server() {
 #  * param7: Expected exception value of 1. Mutually exclusive with param6.
 #
 server_query() {
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     server_query_with_port "$PORT" "$@"
 }
 
@@ -246,3 +250,16 @@ server_query_with_port() {
     echo Executing server_query
     python3 -u -c "$PYTHON_QUERY_SCRIPT" -- "$PYTEST_DIR" "$1" "$PORT" "$2" "$3" "$4" "$5" "$6" "$7"
 }
+
+definePORT() {
+  getPORT=""
+  for i in {0..9}
+  do
+    let getPORT="($$ + $i) % (65536-1024) + 1024"
+    portinuse=$(lsof -i -P -n | grep LISTEN | grep $attemptedPORT | wc -l)
+    if [ $portinuse -eq 0 ]; then
+      echo "$getPORT"
+      break
+    fi
+  done
+}

integration-tests/bats/replication-multidb.bats

@@ -119,8 +119,8 @@ SQL
     dolt clone "file://./rem1/newdb" "dbs2/newdb"
 
     # this is a hack: we have to change our persisted global server
-    # vars for the sql command to work on the replica TODO: fix this
-    # mess
+    # vars for the sql command to work on the replica
+    # TODO: fix this mess
     dolt config --global --unset sqlserver.global.dolt_replicate_to_remote
     
     run dolt sql --data-dir=dbs2 -q "use newdb; show tables" -r csv
@@ -129,6 +129,53 @@ SQL
     [[ "$output" =~ "new_table" ]] || false
 }
 
+@test "replication-multidb: push newly cloned database" {
+    pushd .
+
+    mkdir -p "${TMPDIRS}/rem2"
+    # push all the dbs to remote1
+    for i in {1..3}; do
+        cd "${TMPDIRS}/dbs1/repo${i}"
+        dolt push remote1 main
+        # also create a new remote2 for each DB but don't push to it
+        dolt remote add remote2 "file://../../rem2/repo${i}"
+    done
+
+    popd
+
+    dolt config --global --add sqlserver.global.dolt_replicate_to_remote remote2
+    dolt sql -q "set @@persist.dolt_replication_remote_url_template = 'file://$TMPDIRS/rem2/{database}'"
+
+    mkdir -p "${TMPDIRS}/dbs2"
+    dolt sql --data-dir=dbs2 <<SQL
+call dolt_clone('file://${TMPDIRS}/rem1/repo1', 'repo1');
+use repo1;
+create table new_table (b int primary key);
+call dolt_commit('-Am', 'new table');
+call dolt_clone('file://${TMPDIRS}/rem1/repo2', 'repo2');
+SQL
+
+    mkdir -p "${TMPDIRS}/dbs3"
+    cd $TMPDIRS
+    dolt clone "file://./rem2/repo1" "dbs3/repo1"
+    dolt clone "file://./rem2/repo2" "dbs3/repo2"
+
+    # this is a hack: we have to change our persisted global server
+    # vars for the sql command to work on the replica
+    # TODO: fix this mess
+    dolt config --global --unset sqlserver.global.dolt_replicate_to_remote
+    
+    run dolt sql --data-dir=dbs3 -q "use repo1; show tables" -r csv
+    [ $status -eq 0 ]
+    [ "${#lines[@]}" -eq 3 ]
+    [[ "$output" =~ "new_table" ]] || false
+
+    run dolt sql --data-dir=dbs3 -q "use repo2; show tables" -r csv
+    [ $status -eq 0 ]
+    [ "${#lines[@]}" -eq 2 ]
+    [[ ! "$output" =~ "new_table" ]] || false
+}
+
 @test "replication-multidb: push newly created database with no commits" {
     dolt config --global --add sqlserver.global.dolt_replicate_to_remote remote1
     dolt sql -q "set @@persist.dolt_replication_remote_url_template = 'file://$TMPDIRS/rem1/{database}'"

integration-tests/bats/sql-jwt-auth.bats

@@ -29,7 +29,7 @@ teardown() {
     cp "$BATS_TEST_DIRNAME"/../../go/cmd/dolt/commands/sqlserver/testdata/chain_key.pem .
     cp "$BATS_TEST_DIRNAME"/../../go/cmd/dolt/commands/sqlserver/testdata/chain_cert.pem .
     cp "$BATS_TEST_DIRNAME"/../../go/libraries/utils/jwtauth/gen_keys/test_jwks.json .
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     TOKEN="`cat $BATS_TEST_DIRNAME/../../go/libraries/utils/jwtauth/gen_keys/token.jwt`"
 
     cat >config.yml <<EOF

integration-tests/bats/sql-privs.bats

@@ -55,7 +55,7 @@ teardown() {
 
 @test "sql-privs: default user is root. create new user destroys default user." {
     make_test_repo
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server --host 0.0.0.0 --port=$PORT &
     SERVER_PID=$! # will get killed by teardown_common
     sleep 5 # not using python wait so this works on windows
@@ -68,7 +68,7 @@ teardown() {
     rm -f .dolt/sql-server.lock
 
     # restarting server
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server --host 0.0.0.0 --port=$PORT &
     SERVER_PID=$! # will get killed by teardown_common
     sleep 5 # not using python wait so this works on windows
@@ -96,7 +96,7 @@ teardown() {
 @test "sql-privs: yaml with no user is replaced with command line user" {
     make_test_repo
     touch server.yaml
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
 
     echo "log_level: debug
 
@@ -118,7 +118,7 @@ behavior:
 @test "sql-privs: yaml with user is also replaced with command line user" {
     make_test_repo
     touch server.yaml
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
 
     echo "log_level: debug
 user:

integration-tests/bats/sql-server.bats

@@ -53,7 +53,7 @@ teardown() {
     cd repo1
     dolt sql -q "create user dolt@'%' identified by '123'"
 
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server --port=$PORT --user dolt > log.txt 2>&1 &
     SERVER_PID=$!
     sleep 5
@@ -684,7 +684,7 @@ SQL
     skiponwindows "Missing dependencies"
     cd repo1
     dolt sql -q 'create table test (id int primary key)'
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     cat >config.yml <<EOF
 log_level: debug
 behavior:
@@ -731,7 +731,7 @@ END""")
     skiponwindows "Missing dependencies"
     cd repo1
     dolt sql -q 'create table test (id int primary key)'
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     cat >config.yml <<EOF
 log_level: debug
 user:
@@ -1156,7 +1156,7 @@ databases:
 @test "sql-server: sql-server locks database" {
     cd repo1
     start_sql_server
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     run dolt sql-server -P $PORT
     [ "$status" -eq 1 ]
 }
@@ -1164,7 +1164,7 @@ databases:
 @test "sql-server: multi dir sql-server locks out childen" {
     start_sql_server
     cd repo2
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     run dolt sql-server -P $PORT
     [ "$status" -eq 1 ]
 }
@@ -1173,7 +1173,7 @@ databases:
     cd repo2
     start_sql_server
     cd ..
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     run dolt sql-server -P $PORT
     [ "$status" -eq 1 ]
 }
@@ -1183,7 +1183,7 @@ databases:
     start_sql_server
     server_query repo1 1 dolt "" "create database newdb" ""
     cd newdb
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     run dolt sql-server -P $PORT
     [ "$status" -eq 1 ]
 }
@@ -1204,7 +1204,7 @@ databases:
     skiponwindows "unix socket is not available on Windows"
     cd repo2
     DEFAULT_DB="repo2"
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
 
     dolt sql-server --port $PORT --user dolt >> log.txt 2>&1 &
     SERVER_PID=$!
@@ -1228,7 +1228,7 @@ databases:
     skiponwindows "unix socket is not available on Windows"
     cd repo2
     DEFAULT_DB="repo2"
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
 
     dolt sql-server --port $PORT --user dolt --socket > log.txt 2>&1 &
     SERVER_PID=$!
@@ -1249,7 +1249,7 @@ databases:
     run pwd
     REPO_NAME=$output
 
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server --port=$PORT --socket="$REPO_NAME/mysql.sock" --user dolt > log.txt 2>&1 &
     SERVER_PID=$!
     run wait_for_connection $PORT 5000
@@ -1264,7 +1264,7 @@ databases:
     skiponwindows "unix socket is not available on Windows"
     cd repo2
     DEFAULT_DB="repo2"
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
 
     echo "
 log_level: debug
@@ -1382,7 +1382,7 @@ s.close()
     run dolt init --new-format
     [ $status -eq 0 ]
 
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     dolt sql-server --host 0.0.0.0 --port=$PORT --user dolt &
     SERVER_PID=$! # will get killed by teardown_common
     sleep 5 # not using python wait so this works on windows
@@ -1420,6 +1420,11 @@ s.close()
 
     run grep "failed to access 'mydb2' database: can no longer find .dolt dir on disk" server_log.txt
     [ "${#lines[@]}" -eq 1 ]
+
+    # this tests fails sometimes as the server is stopped from the above error
+    # but stop_sql_server in teardown tries to kill process that is not running anymore,
+    # so start the server again, and it will be stopped in teardown
+    start_sql_server
 }
 
 @test "sql-server: dropping database that the server is running in should drop only the db itself not its nested dbs" {

integration-tests/mysql-client-tests/mysql-client-tests.bats

@@ -19,7 +19,7 @@ setup() {
     dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext)"
     dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');"
 
-    let PORT="$$ % (65536-1024) + 1024"
+    PORT=$( definePORT )
     USER="dolt"
     dolt sql-server --host 0.0.0.0 --port=$PORT --user=$USER --loglevel=trace &
     SERVER_PID=$!
@@ -167,3 +167,17 @@ EOF" -m "postgres"
 @test "R RMariaDB client" {
     Rscript $BATS_TEST_DIRNAME/r/rmariadb-test.r $USER $PORT $REPO_NAME
 }
+
+definePORT() {
+  getPORT=""
+  for i in {0..9}
+  do
+    let getPORT="($$ + $i) % (65536-1024) + 1024"
+    portinuse=$(lsof -i -P -n | grep LISTEN | grep $attemptedPORT | wc -l)
+      if [ $portinuse -eq 0 ]
+      then
+        echo "$getPORT"
+        break
+      fi
+  done
+}

integration-tests/orm-tests/orm-tests.bats

@@ -7,7 +7,7 @@ setup() {
 
   dolt init
 
-  let PORT="$$ % (65536-1024) + 1024"
+  PORT=$( definePORT )
   USER="dolt"
   dolt sql-server --host 0.0.0.0 --port=$PORT --user=$USER --loglevel=trace &
   SERVER_PID=$!