Merge pull request #7501 from dolthub/db/benchmark-tests

[no-release-notes] /.github/workflows/ci-benchmark-runner-tests.yaml: test benchmark runner in ci
2026-04-19 19:21:44 -05:00 · 2024-02-16 09:31:30 -08:00
parent 12e4a6646e 69724df93f
commit 36efb2f1fa
5 changed files with 111 additions and 104 deletions
--- a/.github/workflows/ci-benchmark-runner-tests.yaml
+++ b/.github/workflows/ci-benchmark-runner-tests.yaml
@@ -1,24 +1,24 @@
-#name: Test Sysbench Runner Utility Works
-#
-#on:
-#  pull_request:
-#    branches: [ main ]
-#    paths:
-#      - 'go/**'
-#      - 'integration-tests/**'
-#
-#concurrency:
-#  group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }}
-#  cancel-in-progress: true
-#
-#jobs:
-#  mysql_client_integrations_job:
-#    runs-on: ubuntu-22.04
-#    name: Test Sysbench Runner
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v3
-#      - name: Copy Dockerfile
-#        run: cp -r ./go/performance/continuous_integration/. .
-#      - name: Test sysbench runner
-#        uses: ./.github/actions/sysbench-runner-tests
+name: Test Benchmark Runner Utility Works
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'go/**'
+      - 'integration-tests/**'
+
+concurrency:
+  group: ci-benchmark-runner-tests-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark_runner:
+    runs-on: ubuntu-22.04
+    name: Test Benchmark Runner
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Copy Dockerfile
+        run: cp -r ./go/performance/continuous_integration/. .
+      - name: Test runner
+        uses: ./.github/actions/benchmark-runner-tests
--- a/go/performance/continuous_integration/SysbenchDockerfile
+++ b/go/performance/continuous_integration/SysbenchDockerfile
@@ -1,47 +1,33 @@
-FROM --platform=linux/amd64 golang:1.21.2-bullseye
+FROM --platform=linux/amd64 golang:1.21-alpine as gobin
+FROM --platform=linux/amd64 ubuntu:22.04

+COPY --from=gobin /usr/local/go/ /go/
+
+ENV GOPATH=$HOME/go
+ENV PATH="/go/bin:${PATH}"
 ENV DEBIAN_FRONTEND=noninteractive

-# Get sysbench installed
 RUN apt update
-RUN apt install -y curl
-
-# Install sysbench deps
-RUN apt update -y && apt install -y \
-  automake \
-  libtool \
-  pkg-config \
-  libaio-dev \
-  default-libmysqlclient-dev \
-  libssl-dev
-
-# Build sysbench from source
-RUN git clone https://github.com/akopytov/sysbench.git && \
-  cd sysbench && \
-  git checkout -b 1.0.20 tags/1.0.20 && \
-  ./autogen.sh && \
-  ./configure && \
-  make -j && \
-  make install
+RUN apt install -y curl wget git

 # Install sqlite3 from source
 RUN \
  apt-get install -y \
  build-essential \
  tcl \
-  lsb-release \
-  && wget \
+  lsb-release
+
+RUN wget \
    -O sqlite.tar.gz \
    https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release \
  && tar xvfz sqlite.tar.gz \
-  # Configure and make SQLite3 binary
+   # Configure and make SQLite3 binary
  && ./sqlite/configure --prefix=/usr \
  && make \
  && make install \
  # Smoke test
  && sqlite3 --version

-
 WORKDIR /
 COPY ./sysbench-runner-tests-entrypoint.sh /entrypoint.sh
 RUN git clone https://github.com/dolthub/sysbench-lua-scripts.git
@@ -55,18 +41,14 @@ ENV PATH="/doltgresql/utils/doltgres_builder/cmd/doltgresBin/$DOLTGRESQL_VERSION
 RUN doltgres version

 WORKDIR /mysql
-RUN curl -L -O https://dev.mysql.com/get/mysql-apt-config_0.8.22-1_all.deb
-RUN dpkg -i mysql-apt-config_0.8.22-1_all.deb
-# On 2023-12-14, the GPG key this repository uses to publish these packages expired. We make it insecure and install it anyway for now.
-# See https://bugs.mysql.com/bug.php?id=113427
-# Hopefully we can remove this soon.
-RUN sed -i.bak \
-  -e 's|^deb |deb [allow-insecure=true allow-weak=true allow-downgrade-to-insecure=true] |' \
-  -e 's|^deb-src |deb-src [allow-insecure=true allow-weak=true allow-downgrade-to-insecure=true] |' \
-  /etc/apt/sources.list.d/mysql.list 
-RUN apt-get update && apt-get install -y --allow-unauthenticated mysql-server
+RUN apt install -y mysql-server
 RUN mysql --version

+# Install sysbench
+RUN apt update
+RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash
+RUN apt -y install sysbench
+
 # Install Postgres 15
 WORKDIR /postgres
 RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
@@ -88,15 +70,34 @@ COPY ./mysqld.cnf /etc/mysql/mysql.conf.d/mysqld.cnf

 # install dolt
 COPY ./go /home/tester/dolt/go
+ENV DOLT_ROOT_PATH=/home/tester
 WORKDIR /home/tester/dolt/go/cmd/dolt
 RUN go build -o /usr/local/bin/dolt .
 RUN go mod tidy
 RUN chown -R tester:tester /go
 USER $UNAME

+RUN mkdir -p /home/tester/.cache
+ENV GOCACHE=/home/tester/.cache/
+
 # create directories owned by new user for mysql as well
 RUN mkdir -p /home/tester/.mysql/log
 WORKDIR /home/tester

+# supply env vars for tests
+ENV BENCHMARK_RUNNER_DOLT_VERSION="HEAD"
+ENV BENCHMARK_RUNNER_DOLTGRES_VERSION="$DOLTGRESQL_VERSION"
+ENV BENCHMARK_RUNNER_MYSQL_VERSION="8.0.35"
+ENV BENCHMARK_RUNNER_POSTGRES_VERSION="15.5"
+ENV BENCHMARK_RUNNER_DOLT_EXEC="/usr/local/bin/dolt"
+ENV BENCHMARK_RUNNER_MYSQL_EXEC="/usr/sbin/mysqld"
+ENV BENCHMARK_RUNNER_MYSQL_PROTOCOL="unix"
+ENV BENCHMARK_RUNNER_MYSQL_SOCKET="/home/tester/.mysql/mysqld.sock"
+ENV BENCHMARK_RUNNER_DOLTGRES_EXEC="/doltgresql/utils/doltgres_builder/cmd/doltgresBin/$DOLTGRESQL_VERSION/doltgres"
+ENV BENCHMARK_RUNNER_POSTGRES_EXEC="/usr/lib/postgresql/15/bin/postgres"
+ENV BENCHMARK_RUNNER_POSTGRES_INIT_EXEC="/usr/lib/postgresql/15/bin/initdb"
+ENV BENCHMARK_RUNNER_SYSBENCH_LUA_SCRIPTS="/sysbench-lua-scripts"
+ENV BENCHMARK_RUNNER_TPCC_LUA_SCRIPTS="/sysbench-tpcc"
+
 WORKDIR /home/tester/dolt/go/performance/utils/benchmark_runner
 ENTRYPOINT ["/entrypoint.sh"]
--- a/go/performance/continuous_integration/sysbench-runner-tests-entrypoint.sh
+++ b/go/performance/continuous_integration/sysbench-runner-tests-entrypoint.sh
@@ -6,4 +6,4 @@ set -o pipefail
 dolt version
 # Github Actions ignores the WORKDIR?
 cd ./go/performance/utils/benchmark_runner
-DEBUG=true RUN_BENCHMARK_RUNNER_TESTS=true go test ./...
+DEBUG=true RUN_BENCHMARK_RUNNER_TESTS=true go test -timeout 15m ./...
--- a/go/performance/utils/benchmark_runner/run_test.go
+++ b/go/performance/utils/benchmark_runner/run_test.go
@@ -25,6 +25,19 @@ import (
 )

 var runTests = os.Getenv("RUN_BENCHMARK_RUNNER_TESTS")
+var doltExec = os.Getenv("BENCHMARK_RUNNER_DOLT_EXEC")
+var doltVersion = os.Getenv("BENCHMARK_RUNNER_DOLT_VERSION")
+var mysqlExec = os.Getenv("BENCHMARK_RUNNER_MYSQL_EXEC")
+var mysqlProtocol = os.Getenv("BENCHMARK_RUNNER_MYSQL_PROTOCOL")
+var mysqlSocket = os.Getenv("BENCHMARK_RUNNER_MYSQL_SOCKET")
+var mysqlVersion = os.Getenv("BENCHMARK_RUNNER_MYSQL_VERSION")
+var doltgresExec = os.Getenv("BENCHMARK_RUNNER_DOLTGRES_EXEC")
+var doltgresVersion = os.Getenv("BENCHMARK_RUNNER_DOLTGRES_VERSION")
+var postgresExec = os.Getenv("BENCHMARK_RUNNER_POSTGRES_EXEC")
+var postgresInitExec = os.Getenv("BENCHMARK_RUNNER_POSTGRES_INIT_EXEC")
+var postgresVersion = os.Getenv("BENCHMARK_RUNNER_POSTGRES_VERSION")
+var sysbenchLuaScripts = os.Getenv("BENCHMARK_RUNNER_SYSBENCH_LUA_SCRIPTS")
+var tpccLuaScripts = os.Getenv("BENCHMARK_RUNNER_TPCC_LUA_SCRIPTS")

 func TestRunner(t *testing.T) {
 	if runTests == "" {
@@ -39,20 +52,20 @@ func TestRunner(t *testing.T) {

 	conf := &sysbenchRunnerConfigImpl{
 		Tests: []TestConfig{
-			NewTestConfig("oltp_read_write", nil, false),
-			NewTestConfig("oltp_update_index", nil, false),
-			NewTestConfig("oltp_delete_insert", nil, true),
+			NewTestConfig("oltp_read_only", nil, false),
+			//NewTestConfig("oltp_update_index", nil, false),
+			//NewTestConfig("oltp_delete_insert", nil, true),
 		},
 		//Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"),
 		Servers: []ServerConfig{
 			&doltServerConfigImpl{
 				Id:            "test",
-				Version:       "HEAD",
+				Version:       doltVersion,
 				ResultsFormat: CsvFormat,
-				ServerExec:    "/usr/local/bin/dolt",
+				ServerExec:    doltExec,
 			},
 		},
-		ScriptDir: "/sysbench-lua-scripts",
+		ScriptDir: sysbenchLuaScripts,
 		TestOptions: []string{
 			"--rand-seed=1",
 			"--table-size=30",
@@ -90,29 +103,30 @@ func TestDoltMysqlSysbenchRunner(t *testing.T) {

 	conf := &sysbenchRunnerConfigImpl{
 		Tests: []TestConfig{
-			NewTestConfig("oltp_read_write", nil, false),
-			NewTestConfig("oltp_update_index", nil, false),
-			NewTestConfig("oltp_delete_insert", nil, true),
+			NewTestConfig("oltp_read_only", nil, false),
+			//NewTestConfig("oltp_update_index", nil, false),
+			//NewTestConfig("oltp_delete_insert", nil, true),
 		},
 		Servers: []ServerConfig{
 			&doltServerConfigImpl{
 				Id:            "test-dolt",
-				Version:       "HEAD",
+				Version:       doltVersion,
 				ResultsFormat: CsvFormat,
-				ServerExec:    "/usr/local/bin/dolt",
+				ServerExec:    doltExec,
 			},
 			&mysqlServerConfigImpl{
 				Id:                 "test-mysql",
 				Port:               3606,
-				Version:            "8.0.35",
+				Version:            mysqlVersion,
 				ResultsFormat:      CsvFormat,
-				ServerExec:         "/usr/sbin/mysqld",
+				ServerExec:         mysqlExec,
 				ServerUser:         "root",
 				SkipLogBin:         true,
-				ConnectionProtocol: "unix",
+				ConnectionProtocol: mysqlProtocol,
+				Socket:             mysqlSocket,
 			},
 		},
-		ScriptDir: "/sysbench-lua-scripts",
+		ScriptDir: sysbenchLuaScripts,
 		TestOptions: []string{
 			"--rand-seed=1",
 			"--table-size=30",
@@ -142,29 +156,28 @@ func TestDoltgresPostgresSysbenchRunner(t *testing.T) {

 	conf := &sysbenchRunnerConfigImpl{
 		Tests: []TestConfig{
-			NewTestConfig("oltp_read_write", nil, false),
-			NewTestConfig("oltp_update_index", nil, false),
+			NewTestConfig("oltp_read_only", nil, false),
+			//NewTestConfig("oltp_update_index", nil, false),
 		},
 		Servers: []ServerConfig{
 			&postgresServerConfigImpl{
 				Id:            "test-postgres",
 				Host:          "127.0.0.1",
-				Version:       "15.5",
+				Version:       postgresVersion,
 				ResultsFormat: CsvFormat,
-				ServerExec:    "/usr/lib/postgresql/15/bin/postgres",
-				InitExec:      "/usr/lib/postgresql/15/bin/initdb",
+				ServerExec:    postgresExec,
+				InitExec:      postgresInitExec,
 				ServerUser:    "root",
 			},
 			&doltgresServerConfigImpl{
 				Id:            "test-doltgres",
 				Port:          4433,
 				Host:          "127.0.0.1",
-				Version:       "b139dfb",
+				Version:       doltgresVersion,
 				ResultsFormat: CsvFormat,
-				ServerExec:    "doltgres",
+				ServerExec:    doltgresExec,
 			},
 		},
-		ScriptDir: "/sysbench-lua-scripts",
 		TestOptions: []string{
 			"--rand-seed=1",
 			"--table-size=30",
@@ -172,7 +185,6 @@ func TestDoltgresPostgresSysbenchRunner(t *testing.T) {
 			"--time=30",
 			"--percentile=50",
 		},
-		InitBigRepo: true,
 	}

 	err = Run(context.Background(), conf)
@@ -195,14 +207,14 @@ func TestDoltProfiler(t *testing.T) {
 	id := "test-dolt-profile"
 	conf := &sysbenchRunnerConfigImpl{
 		Tests: []TestConfig{
-			NewTestConfig("oltp_read_write", nil, false),
+			NewTestConfig("oltp_read_only", nil, false),
 		},
 		Servers: []ServerConfig{
 			&doltServerConfigImpl{
 				Id:            id,
-				Version:       "HEAD",
+				Version:       doltVersion,
 				ResultsFormat: CsvFormat,
-				ServerExec:    "/usr/local/bin/dolt",
+				ServerExec:    doltExec,
 				ServerProfile: CpuServerProfile,
 				ProfilePath:   dir,
 			},
@@ -228,9 +240,7 @@ func TestDoltProfiler(t *testing.T) {
 }

 func TestDoltMysqlTpccRunner(t *testing.T) {
-	if runTests == "" {
-		t.Skip()
-	}
+	t.Skip() // skip for now since this is kinda slow for pr ci
 	dir := t.TempDir()
 	log.Println(dir)
 	err := os.Chdir(dir)
@@ -242,22 +252,23 @@ func TestDoltMysqlTpccRunner(t *testing.T) {
 		Servers: []ServerConfig{
 			&doltServerConfigImpl{
 				Id:            "test-dolt-tpcc",
-				Version:       "HEAD",
+				Version:       doltVersion,
 				ResultsFormat: CsvFormat,
-				ServerExec:    "/usr/local/bin/dolt",
+				ServerExec:    doltExec,
 			},
 			&mysqlServerConfigImpl{
 				Id:                 "test-mysql-tpcc",
 				Port:               3606,
-				Version:            "8.0.35",
+				Version:            mysqlVersion,
 				ResultsFormat:      CsvFormat,
-				ServerExec:         "/usr/sbin/mysqld",
+				ServerExec:         mysqlExec,
 				ServerUser:         "root",
 				SkipLogBin:         true,
-				ConnectionProtocol: "unix",
+				ConnectionProtocol: mysqlProtocol,
+				Socket:             mysqlSocket,
 			},
 		},
-		ScriptDir: "/sysbench-tpcc",
+		ScriptDir: tpccLuaScripts,
 	}

 	err = RunTpcc(context.Background(), conf)
--- a/go/performance/utils/benchmark_runner/sysbench.go
+++ b/go/performance/utils/benchmark_runner/sysbench.go
@@ -91,18 +91,13 @@ func (t *sysbenchTesterImpl) outputToResult(output []byte) (*Result, error) {
 }

 func (t *sysbenchTesterImpl) prepare(ctx context.Context) error {
-	cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetPrepareArgs(t.serverConfig)...)
+	cmd := ExecCommand(ctx, sysbenchCommand, t.test.GetPrepareArgs(t.serverConfig)...)
 	if t.test.GetFromScript() {
 		lp := filepath.Join(t.config.GetScriptDir(), luaPath)
 		cmd.Env = os.Environ()
 		cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp))
 	}
-	out, err := cmd.Output()
-	if err != nil {
-		fmt.Println(string(out))
-		return err
-	}
-	return nil
+	return cmd.Run()
 }

 func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) {
@@ -134,7 +129,7 @@ func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) {
 }

 func (t *sysbenchTesterImpl) cleanup(ctx context.Context) error {
-	cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetCleanupArgs(t.serverConfig)...)
+	cmd := ExecCommand(ctx, sysbenchCommand, t.test.GetCleanupArgs(t.serverConfig)...)
 	if t.test.GetFromScript() {
 		lp := filepath.Join(t.config.GetScriptDir(), luaPath)
 		cmd.Env = os.Environ()