From 3d0c9df51589799470748c9ba5fd601fda29fa8a Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Thu, 15 Feb 2024 16:32:59 -0800 Subject: [PATCH] /{.github,go}: fix benchmark_runner tests so that they test all use cases --- .../workflows/ci-benchmark-runner-tests.yaml | 43 +++++------ .../continuous_integration/SysbenchDockerfile | 71 ++++++++++--------- .../utils/benchmark_runner/run_test.go | 61 +++++++++------- .../utils/benchmark_runner/sysbench.go | 11 +-- 4 files changed, 98 insertions(+), 88 deletions(-) diff --git a/.github/workflows/ci-benchmark-runner-tests.yaml b/.github/workflows/ci-benchmark-runner-tests.yaml index 0b2535f864..f94eb3b37f 100644 --- a/.github/workflows/ci-benchmark-runner-tests.yaml +++ b/.github/workflows/ci-benchmark-runner-tests.yaml @@ -1,24 +1,25 @@ -#name: Test Sysbench Runner Utility Works -# -#on: -# pull_request: -# branches: [ main ] +# name: Test Benchmark Runner Utility Works + +on: + pull_request: + branches: [ main ] # paths: # - 'go/**' # - 'integration-tests/**' -# -#concurrency: -# group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }} -# cancel-in-progress: true -# -#jobs: -# mysql_client_integrations_job: -# runs-on: ubuntu-22.04 -# name: Test Sysbench Runner -# steps: -# - name: Checkout -# uses: actions/checkout@v3 -# - name: Copy Dockerfile -# run: cp -r ./go/performance/continuous_integration/. . -# - name: Test sysbench runner -# uses: ./.github/actions/sysbench-runner-tests + +concurrency: + group: ci-benchmark-runner-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + benchmark_runner: + if: ${{ github.actor == 'coffeegoddd' }} + runs-on: ubuntu-22.04 + name: Test Benchmark Runner + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Copy Dockerfile + run: cp -r ./go/performance/continuous_integration/. . + - name: Test runner + uses: ./.github/actions/benchmark-runner-tests diff --git a/go/performance/continuous_integration/SysbenchDockerfile b/go/performance/continuous_integration/SysbenchDockerfile index 195114eb35..424fdd66ae 100644 --- a/go/performance/continuous_integration/SysbenchDockerfile +++ b/go/performance/continuous_integration/SysbenchDockerfile @@ -1,47 +1,33 @@ -FROM --platform=linux/amd64 golang:1.21.2-bullseye +FROM --platform=linux/amd64 golang:1.21-alpine as gobin +FROM --platform=linux/amd64 ubuntu:22.04 +COPY --from=gobin /usr/local/go/ /go/ + +ENV GOPATH=$HOME/go +ENV PATH="/go/bin:${PATH}" ENV DEBIAN_FRONTEND=noninteractive -# Get sysbench installed RUN apt update -RUN apt install -y curl - -# Install sysbench deps -RUN apt update -y && apt install -y \ - automake \ - libtool \ - pkg-config \ - libaio-dev \ - default-libmysqlclient-dev \ - libssl-dev - -# Build sysbench from source -RUN git clone https://github.com/akopytov/sysbench.git && \ - cd sysbench && \ - git checkout -b 1.0.20 tags/1.0.20 && \ - ./autogen.sh && \ - ./configure && \ - make -j && \ - make install +RUN apt install -y curl wget git # Install sqlite3 from source RUN \ apt-get install -y \ build-essential \ tcl \ - lsb-release \ - && wget \ + lsb-release + +RUN wget \ -O sqlite.tar.gz \ https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release \ && tar xvfz sqlite.tar.gz \ - # Configure and make SQLite3 binary + # Configure and make SQLite3 binary && ./sqlite/configure --prefix=/usr \ && make \ && make install \ # Smoke test && sqlite3 --version - WORKDIR / COPY ./sysbench-runner-tests-entrypoint.sh /entrypoint.sh RUN git clone https://github.com/dolthub/sysbench-lua-scripts.git @@ -55,18 +41,14 @@ ENV PATH="/doltgresql/utils/doltgres_builder/cmd/doltgresBin/$DOLTGRESQL_VERSION RUN doltgres version WORKDIR /mysql -RUN curl -L -O https://dev.mysql.com/get/mysql-apt-config_0.8.22-1_all.deb -RUN dpkg -i mysql-apt-config_0.8.22-1_all.deb -# On 2023-12-14, the GPG key this repository uses to publish these packages expired. We make it insecure and install it anyway for now. -# See https://bugs.mysql.com/bug.php?id=113427 -# Hopefully we can remove this soon. -RUN sed -i.bak \ - -e 's|^deb |deb [allow-insecure=true allow-weak=true allow-downgrade-to-insecure=true] |' \ - -e 's|^deb-src |deb-src [allow-insecure=true allow-weak=true allow-downgrade-to-insecure=true] |' \ - /etc/apt/sources.list.d/mysql.list -RUN apt-get update && apt-get install -y --allow-unauthenticated mysql-server +RUN apt install -y mysql-server RUN mysql --version +# Install sysbench +RUN apt update +RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash +RUN apt -y install sysbench + # Install Postgres 15 WORKDIR /postgres RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' @@ -88,15 +70,34 @@ COPY ./mysqld.cnf /etc/mysql/mysql.conf.d/mysqld.cnf # install dolt COPY ./go /home/tester/dolt/go +ENV DOLT_ROOT_PATH=/home/tester WORKDIR /home/tester/dolt/go/cmd/dolt RUN go build -o /usr/local/bin/dolt . RUN go mod tidy RUN chown -R tester:tester /go USER $UNAME +RUN mkdir -p /home/tester/.cache +ENV GOCACHE=/home/tester/.cache/ + # create directories owned by new user for mysql as well RUN mkdir -p /home/tester/.mysql/log WORKDIR /home/tester +# supply env vars for tests +ENV BENCHMARK_RUNNER_DOLT_VERSION="HEAD" +ENV BENCHMARK_RUNNER_DOLTGRES_VERSION="$DOLTGRESQL_VERSION" +ENV BENCHMARK_RUNNER_MYSQL_VERSION="8.0.35" +ENV BENCHMARK_RUNNER_POSTGRES_VERSION="15.5" +ENV BENCHMARK_RUNNER_DOLT_EXEC="/usr/local/bin/dolt" +ENV BENCHMARK_RUNNER_MYSQL_EXEC="/usr/sbin/mysqld" +ENV BENCHMARK_RUNNER_MYSQL_PROTOCOL="unix" +ENV BENCHMARK_RUNNER_MYSQL_SOCKET="/home/tester/.mysql/mysqld.sock" +ENV BENCHMARK_RUNNER_DOLTGRES_EXEC="/doltgresql/utils/doltgres_builder/cmd/doltgresBin/$DOLTGRESQL_VERSION/doltgres" +ENV BENCHMARK_RUNNER_POSTGRES_EXEC="/usr/lib/postgresql/15/bin/postgres" +ENV BENCHMARK_RUNNER_POSTGRES_INIT_EXEC="/usr/lib/postgresql/15/bin/initdb" +ENV BENCHMARK_RUNNER_SYSBENCH_LUA_SCRIPTS="/sysbench-lua-scripts" +ENV BENCHMARK_RUNNER_TPCC_LUA_SCRIPTS="/sysbench-tpcc" + WORKDIR /home/tester/dolt/go/performance/utils/benchmark_runner ENTRYPOINT ["/entrypoint.sh"] diff --git a/go/performance/utils/benchmark_runner/run_test.go b/go/performance/utils/benchmark_runner/run_test.go index 7202cb57ac..562793af1a 100644 --- a/go/performance/utils/benchmark_runner/run_test.go +++ b/go/performance/utils/benchmark_runner/run_test.go @@ -25,6 +25,19 @@ import ( ) var runTests = os.Getenv("RUN_BENCHMARK_RUNNER_TESTS") +var doltExec = os.Getenv("BENCHMARK_RUNNER_DOLT_EXEC") +var doltVersion = os.Getenv("BENCHMARK_RUNNER_DOLT_VERSION") +var mysqlExec = os.Getenv("BENCHMARK_RUNNER_MYSQL_EXEC") +var mysqlProtocol = os.Getenv("BENCHMARK_RUNNER_MYSQL_PROTOCOL") +var mysqlSocket = os.Getenv("BENCHMARK_RUNNER_MYSQL_SOCKET") +var mysqlVersion = os.Getenv("BENCHMARK_RUNNER_MYSQL_VERSION") +var doltgresExec = os.Getenv("BENCHMARK_RUNNER_DOLTGRES_EXEC") +var doltgresVersion = os.Getenv("BENCHMARK_RUNNER_DOLTGRES_VERSION") +var postgresExec = os.Getenv("BENCHMARK_RUNNER_POSTGRES_EXEC") +var postgresInitExec = os.Getenv("BENCHMARK_RUNNER_POSTGRES_INIT_EXEC") +var postgresVersion = os.Getenv("BENCHMARK_RUNNER_POSTGRES_VERSION") +var sysbenchLuaScripts = os.Getenv("BENCHMARK_RUNNER_SYSBENCH_LUA_SCRIPTS") +var tpccLuaScripts = os.Getenv("BENCHMARK_RUNNER_TPCC_LUA_SCRIPTS") func TestRunner(t *testing.T) { if runTests == "" { @@ -47,12 +60,12 @@ func TestRunner(t *testing.T) { Servers: []ServerConfig{ &doltServerConfigImpl{ Id: "test", - Version: "HEAD", + Version: doltVersion, ResultsFormat: CsvFormat, - ServerExec: "/usr/local/bin/dolt", + ServerExec: doltExec, }, }, - ScriptDir: "/sysbench-lua-scripts", + ScriptDir: sysbenchLuaScripts, TestOptions: []string{ "--rand-seed=1", "--table-size=30", @@ -97,22 +110,23 @@ func TestDoltMysqlSysbenchRunner(t *testing.T) { Servers: []ServerConfig{ &doltServerConfigImpl{ Id: "test-dolt", - Version: "HEAD", + Version: doltVersion, ResultsFormat: CsvFormat, - ServerExec: "/usr/local/bin/dolt", + ServerExec: doltExec, }, &mysqlServerConfigImpl{ Id: "test-mysql", Port: 3606, - Version: "8.0.35", + Version: mysqlVersion, ResultsFormat: CsvFormat, - ServerExec: "/usr/sbin/mysqld", + ServerExec: mysqlExec, ServerUser: "root", SkipLogBin: true, - ConnectionProtocol: "unix", + ConnectionProtocol: mysqlProtocol, + Socket: mysqlSocket, }, }, - ScriptDir: "/sysbench-lua-scripts", + ScriptDir: sysbenchLuaScripts, TestOptions: []string{ "--rand-seed=1", "--table-size=30", @@ -149,22 +163,21 @@ func TestDoltgresPostgresSysbenchRunner(t *testing.T) { &postgresServerConfigImpl{ Id: "test-postgres", Host: "127.0.0.1", - Version: "15.5", + Version: postgresVersion, ResultsFormat: CsvFormat, - ServerExec: "/usr/lib/postgresql/15/bin/postgres", - InitExec: "/usr/lib/postgresql/15/bin/initdb", + ServerExec: postgresExec, + InitExec: postgresInitExec, ServerUser: "root", }, &doltgresServerConfigImpl{ Id: "test-doltgres", Port: 4433, Host: "127.0.0.1", - Version: "b139dfb", + Version: doltgresVersion, ResultsFormat: CsvFormat, - ServerExec: "doltgres", + ServerExec: doltgresExec, }, }, - ScriptDir: "/sysbench-lua-scripts", TestOptions: []string{ "--rand-seed=1", "--table-size=30", @@ -172,7 +185,6 @@ func TestDoltgresPostgresSysbenchRunner(t *testing.T) { "--time=30", "--percentile=50", }, - InitBigRepo: true, } err = Run(context.Background(), conf) @@ -200,9 +212,9 @@ func TestDoltProfiler(t *testing.T) { Servers: []ServerConfig{ &doltServerConfigImpl{ Id: id, - Version: "HEAD", + Version: doltVersion, ResultsFormat: CsvFormat, - ServerExec: "/usr/local/bin/dolt", + ServerExec: doltExec, ServerProfile: CpuServerProfile, ProfilePath: dir, }, @@ -242,22 +254,23 @@ func TestDoltMysqlTpccRunner(t *testing.T) { Servers: []ServerConfig{ &doltServerConfigImpl{ Id: "test-dolt-tpcc", - Version: "HEAD", + Version: doltVersion, ResultsFormat: CsvFormat, - ServerExec: "/usr/local/bin/dolt", + ServerExec: doltExec, }, &mysqlServerConfigImpl{ Id: "test-mysql-tpcc", Port: 3606, - Version: "8.0.35", + Version: mysqlVersion, ResultsFormat: CsvFormat, - ServerExec: "/usr/sbin/mysqld", + ServerExec: mysqlExec, ServerUser: "root", SkipLogBin: true, - ConnectionProtocol: "unix", + ConnectionProtocol: mysqlProtocol, + Socket: mysqlSocket, }, }, - ScriptDir: "/sysbench-tpcc", + ScriptDir: tpccLuaScripts, } err = RunTpcc(context.Background(), conf) diff --git a/go/performance/utils/benchmark_runner/sysbench.go b/go/performance/utils/benchmark_runner/sysbench.go index 2b8636dffc..bd2822928d 100644 --- a/go/performance/utils/benchmark_runner/sysbench.go +++ b/go/performance/utils/benchmark_runner/sysbench.go @@ -91,18 +91,13 @@ func (t *sysbenchTesterImpl) outputToResult(output []byte) (*Result, error) { } func (t *sysbenchTesterImpl) prepare(ctx context.Context) error { - cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetPrepareArgs(t.serverConfig)...) + cmd := ExecCommand(ctx, sysbenchCommand, t.test.GetPrepareArgs(t.serverConfig)...) if t.test.GetFromScript() { lp := filepath.Join(t.config.GetScriptDir(), luaPath) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp)) } - out, err := cmd.Output() - if err != nil { - fmt.Println(string(out)) - return err - } - return nil + return cmd.Run() } func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { @@ -134,7 +129,7 @@ func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) { } func (t *sysbenchTesterImpl) cleanup(ctx context.Context) error { - cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetCleanupArgs(t.serverConfig)...) + cmd := ExecCommand(ctx, sysbenchCommand, t.test.GetCleanupArgs(t.serverConfig)...) if t.test.GetFromScript() { lp := filepath.Join(t.config.GetScriptDir(), luaPath) cmd.Env = os.Environ()