Merge branch 'main' into fulghum/kill-python

.github/workflows/ci-bats-unix-remote.yaml

@@ -0,0 +1,118 @@
+name: Test Bats Unix Remote
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'go/**'
+      - 'integration-tests/**'
+
+concurrency:
+  group: ci-bats-unix-remote${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Bats tests
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ ubuntu-22.04, macos-latest ]
+    env:
+      use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }}
+    steps:
+      - name: Conditionally Set ENV VARS for AWS tests
+        run: |
+          if [[ $use_credentials == true ]]; then
+            echo "AWS_SDK_LOAD_CONFIG=1" >> $GITHUB_ENV
+            echo "AWS_REGION=us-west-2" >> $GITHUB_ENV
+            echo "DOLT_BATS_AWS_TABLE=dolt-ci-bats-manifests-us-west-2" >> $GITHUB_ENV
+            echo "DOLT_BATS_AWS_BUCKET=dolt-ci-bats-chunks-us-west-2" >> $GITHUB_ENV
+            echo "DOLT_BATS_AWS_EXISTING_REPO=aws_remote_bats_tests__dolt__" >> $GITHUB_ENV
+          fi
+      - name: Configure AWS Credentials
+        if: ${{ env.use_credentials == 'true' }}
+        uses: aws-actions/configure-aws-credentials@567d4149d67f15f52b09796bea6573fc32952783
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-duration-seconds: 10800 # 3 hours D:
+      - name: Setup Go 1.x
+        uses: actions/setup-go@v3
+        with:
+          go-version: ^1.19
+        id: go
+      - name: Setup Python 3.x
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: ^16
+      - name: Create CI Bin
+        run: |
+          mkdir -p ./.ci_bin
+          echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
+      - name: Install Bats
+        run: |
+          npm i bats
+          echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH
+        working-directory: ./.ci_bin
+      - name: Install Python Deps
+        run: |
+          pip install mysql-connector-python
+          pip install pandas
+          pip install pyarrow
+      - name: Install Dolt
+        working-directory: ./go
+        run: |
+          go build -mod=readonly -o ../.ci_bin/dolt ./cmd/dolt/.
+
+          go build -mod=readonly -o ../.ci_bin/remotesrv ./utils/remotesrv/.
+          go build -mod=readonly -o ../.ci_bin/noms ./store/cmd/noms/.
+      - name: Setup Dolt Config
+        run: |
+          dolt config --global --add user.name 'Dolthub Actions'
+          dolt config --global --add user.email 'actions@dolthub.com'
+      - name: Install expect
+        if: matrix.os == 'ubuntu-22.04'
+        run: sudo apt-get install -y expect
+      - name: Install Maven
+        working-directory: ./.ci_bin
+        run: |
+          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
+          tar -xf apache-maven-3.9.2-bin.tar.gz
+          echo "$(pwd)/apache-maven-3.9.2/bin" >> $GITHUB_PATH
+      - name: Install Hadoop
+        working-directory: ./.ci_bin
+        run: |
+          curl -LO https://downloads.apache.org/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz
+          tar xvf hadoop-3.3.4.tar.gz
+          echo "$(pwd)/hadoop-3.3.4/bin" >> $GITHUB_PATH
+      - name: Install parquet-cli
+        id: parquet_cli
+        working-directory: ./.ci_bin
+        run: |
+          curl -OL https://github.com/apache/parquet-mr/archive/refs/tags/apache-parquet-1.12.3.tar.gz
+          tar zxvf apache-parquet-1.12.3.tar.gz
+          cd parquet-mr-apache-parquet-1.12.3/parquet-cli
+          mvn clean install -DskipTests
+          runtime_jar="$(pwd)"/target/parquet-cli-1.12.3-runtime.jar
+          echo "runtime_jar=$runtime_jar" >> $GITHUB_OUTPUT
+      - name: Check expect
+        run: expect -v
+      - name: Test all Unix
+        env:
+          SQL_ENGINE: "remote-engine"
+          PARQUET_RUNTIME_JAR: ${{ steps.parquet_cli.outputs.runtime_jar }}
+          BATS_TEST_RETRIES: "3"
+        run: |
+          bats --tap .
+        working-directory: ./integration-tests/bats

.github/workflows/ci-bats-unix.yaml

@@ -22,7 +22,6 @@ jobs:
       fail-fast: true
       matrix:
         os: [ ubuntu-22.04, macos-latest ]
-        sql_engine: [ remote-engine, local-engine ]
     env:
       use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }}
     steps:
@@ -88,9 +87,9 @@ jobs:
       - name: Install Maven
         working-directory: ./.ci_bin
         run: |
-          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.1/binaries/apache-maven-3.9.1-bin.tar.gz
-          tar -xf apache-maven-3.9.1-bin.tar.gz
-          echo "$(pwd)/apache-maven-3.9.1/bin" >> $GITHUB_PATH
+          curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
+          tar -xf apache-maven-3.9.2-bin.tar.gz
+          echo "$(pwd)/apache-maven-3.9.2/bin" >> $GITHUB_PATH
       - name: Install Hadoop
         working-directory: ./.ci_bin
         run: |
@@ -111,7 +110,7 @@ jobs:
         run: expect -v
       - name: Test all Unix
         env:
-          SQL_ENGINE: ${{ matrix.sql_engine }}
+          SQL_ENGINE: "local-engine"
           PARQUET_RUNTIME_JAR: ${{ steps.parquet_cli.outputs.runtime_jar }}
           BATS_TEST_RETRIES: "3"
         run: |

README.md

@@ -173,6 +173,12 @@ Clone this repository and cd into the `go` directory. Then run:
 go install ./cmd/dolt
 ```
 
+The output will be in `$GOPATH/bin`, which defaults to `~/go/bin`. To test your build, try:
+
+```
+~/go/bin/dolt version
+```
+
 # Configuration
 
 Verify that your installation has succeeded by running `dolt` in your

go/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/dolthub/fslock v0.0.3
 	github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488
 	github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
-	github.com/dolthub/vitess v0.0.0-20230606164345-6cb0d80700b2
+	github.com/dolthub/vitess v0.0.0-20230608223125-8cd7c28985bf
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fatih/color v1.13.0
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@@ -59,7 +59,7 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.15.1-0.20230606174340-20dde39da840
+	github.com/dolthub/go-mysql-server v0.15.1-0.20230609162957-fc7e3c87e05a
 	github.com/dolthub/swiss v0.1.0
 	github.com/goccy/go-json v0.10.2
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510

go/go.sum

@@ -168,8 +168,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
-github.com/dolthub/go-mysql-server v0.15.1-0.20230606174340-20dde39da840 h1:LNrH1zxCioDcpTMkTV/cr3LI4AlyxrmILI17OUbijUU=
-github.com/dolthub/go-mysql-server v0.15.1-0.20230606174340-20dde39da840/go.mod h1:TP8QrAsULBEK7nP0BHRSfZ9l8oiAeaRzIREHVk2wnz0=
+github.com/dolthub/go-mysql-server v0.15.1-0.20230609162957-fc7e3c87e05a h1:caUBEa4yJZ5R7asrtRZkLCXlEJ5fCK3hXTP61faf1Ow=
+github.com/dolthub/go-mysql-server v0.15.1-0.20230609162957-fc7e3c87e05a/go.mod h1:XtLmc6ifDvaDySVCxu6Hhj6klAZq4um3oyTGzjaQqLc=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto=
 github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ=
@@ -180,8 +180,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X
 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
 github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
 github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
-github.com/dolthub/vitess v0.0.0-20230606164345-6cb0d80700b2 h1:KGlVqYOuu6iG6DJAK06mZQPeXB/ElYlOH5UsuqUNf0A=
-github.com/dolthub/vitess v0.0.0-20230606164345-6cb0d80700b2/go.mod h1:IyoysiiOzrIs7QsEHC+yVF0yRQ6W70GXyCXqtI2vVTs=
+github.com/dolthub/vitess v0.0.0-20230608223125-8cd7c28985bf h1:xUGTpOlvHw6TdwXZWXktEnaL5rSOj6NQwgWGeqMYyJY=
+github.com/dolthub/vitess v0.0.0-20230608223125-8cd7c28985bf/go.mod h1:IyoysiiOzrIs7QsEHC+yVF0yRQ6W70GXyCXqtI2vVTs=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=

go/libraries/doltcore/sqle/binlogreplication/binlog_replica_controller.go

@@ -443,7 +443,7 @@ func getOptionValueAsTableNames(option binlogreplication.ReplicationOption) ([]s
 
 func verifyAllTablesAreQualified(urts []sql.UnresolvedTable) error {
 	for _, urt := range urts {
-		if urt.Database() == "" {
+		if urt.Database().Name() == "" {
 			return fmt.Errorf("no database specified for table '%s'; "+
 				"all filter table names must be qualified with a database name", urt.Name())
 		}

go/libraries/doltcore/sqle/binlogreplication/binlog_replica_filtering.go

@@ -59,7 +59,7 @@ func (fc *filterConfiguration) setDoTables(urts []sql.UnresolvedTable) error {
 
 	for _, urt := range urts {
 		table := strings.ToLower(urt.Name())
-		db := strings.ToLower(urt.Database())
+		db := strings.ToLower(urt.Database().Name())
 		if fc.doTables[db] == nil {
 			fc.doTables[db] = make(map[string]struct{})
 		}
@@ -86,7 +86,7 @@ func (fc *filterConfiguration) setIgnoreTables(urts []sql.UnresolvedTable) error
 
 	for _, urt := range urts {
 		table := strings.ToLower(urt.Name())
-		db := strings.ToLower(urt.Database())
+		db := strings.ToLower(urt.Database().Name())
 		if fc.ignoreTables[db] == nil {
 			fc.ignoreTables[db] = make(map[string]struct{})
 		}

go/libraries/doltcore/sqle/binlogreplication/binlog_replication_test.go

@@ -452,24 +452,6 @@ func TestCharsetsAndCollations(t *testing.T) {
 	require.Equal(t, "one", row["c1"])
 	require.Equal(t, "\x00o\x00n\x00e", row["c2"])
 	require.NoError(t, rows.Close())
-
-	// Test that we get an error for unsupported charsets/collations
-	primaryDatabase.MustExec("CREATE TABLE t2 (pk int primary key, c1 varchar(255) COLLATE utf8mb4_bg_0900_as_cs);")
-	waitForReplicaToCatchUp(t)
-	replicaDatabase.MustExec("use db01;")
-	rows, err = replicaDatabase.Queryx("SHOW TABLES WHERE Tables_in_db01 like 't2';")
-	require.NoError(t, err)
-	require.False(t, rows.Next())
-	require.NoError(t, rows.Close())
-
-	rows, err = replicaDatabase.Queryx("SHOW REPLICA STATUS;")
-	require.NoError(t, err)
-	row = convertByteArraysToStrings(readNextRow(t, rows))
-	require.Equal(t, "1105", row["Last_SQL_Errno"])
-	require.NotEmpty(t, row["Last_SQL_Error_Timestamp"])
-	require.Contains(t, row["Last_SQL_Error"], "The collation `utf8mb4_bg_0900_as_cs` has not yet been implemented")
-	require.False(t, rows.Next())
-	require.NoError(t, rows.Close())
 }
 
 //

integration-tests/bats/remotes-sql-server.bats

@@ -502,7 +502,7 @@ teardown() {
 
     run dolt sql-client --use-db repo2/feature -P $PORT -u dolt -q "select active_branch()"
     [ $status -eq 1 ]
-    [[ "$output" =~ "database not found: repo2/feature" ]] || false
+    [[ "$output" =~ "'feature' matched multiple remote tracking branches" ]] || false
 
     run grep "'feature' matched multiple remote tracking branches" server_log.txt
     [ "${#lines[@]}" -ne 0 ]

integration-tests/bats/sql-shell.bats

@@ -57,7 +57,7 @@ teardown() {
 
     run dolt --user=new_user sql <<< "select user from mysql.user"
     [ "$status" -eq 1 ]
-    [[ "$output" =~ "Access denied for user" ]] || false
+    [[ "$output" =~ "command denied to user " ]] || false
 
     rm -rf .doltcfg
 }

integration-tests/bats/sql.bats

@@ -75,7 +75,7 @@ teardown() {
 
     run dolt --user=new_user sql -q "select user from mysql.user"
     [ "$status" -eq 1 ]
-    [[ "$output" =~ "Access denied for user" ]]
+    [[ "$output" =~ "command denied to user" ]] || false
 
     rm -rf .doltcfg
 }