Use docker-compose for orchestrating benchmarking

.gitignore

@@ -6,5 +6,6 @@ go.sum
 go.mod
 .DS_Store
 .sqlhistory
-benchmark-tools/working
+benchmark-tools/dolt-builds/dolt
+benchmark-tools/dolt-builds/working
 benchmark-tools/output

benchmark-tools/Dockerfile

@@ -1,67 +0,0 @@
-FROM python:3.7-slim-stretch
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV TERM linux
-
-RUN set -ex \
-    && buildDeps=' \
-        freetds-dev \
-        libkrb5-dev \
-        libsasl2-dev \
-        libssl-dev \
-        libffi-dev \
-        git \
-    ' \
-    && apt-get update -yqq \
-    && apt-get upgrade -yqq \
-    && apt-get install -yqq --no-install-recommends \
-        $buildDeps \
-        apt-utils \
-        build-essential \
-        ca-certificates \
-        cpanminus \
-        curl \
-        dumb-init \
-        fonts-liberation \
-        freetds-bin \
-        gconf-service \
-        libappindicator1 \
-        libasound2 \
-        libatk-bridge2.0-0 \
-        libatk1.0-0 \
-        libc6 \
-        libcairo2 \
-        libcups2 \
-        libdbus-1-3 \
-        libexpat1 \
-        libfontconfig1 \
-        libgcc1 \
-        libgconf-2-4 \
-        libgdk-pixbuf2.0-0 \
-        libglib2.0-0 \
-        libgtk-3-0 \
-        libnspr4 \
-        libnss3 \
-        libpango-1.0-0 \
-        libpangocairo-1.0-0 \
-        libpq-dev \
-        libstdc++6 \
-        libx11-6 \
-        libx11-xcb1 \
-        libxcb1 \
-        libxcomposite1 \
-        libxcursor1 \
-        libxdamage1 \
-        libxext6 \
-        libxfixes3 \
-        libxi6 \
-        libxrandr2 \
-        libxrender1 \
-        libxss1 \
-        libxtst6 \
-        locales \
-        lsb-release \
-        netcat
-RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash
-RUN apt -y install sysbench
-RUN pip install doltpy

benchmark-tools/dolt-builds/Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:18.04
+
+COPY ./dolt /usr/local/bin/dolt
+COPY ./start_dolt_sql_server.sh /start_dolt_sql_server.sh
+
+ENTRYPOINT [ "/start_dolt_sql_server.sh"]

benchmark-tools/dolt-builds/start_dolt_sql_server.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+echo "Creating data directory and configuring Dolt"
+mkdir /test
+cd /test || return
+dolt config --global --add user.name benchmark
+dolt config --global --add user.email benchmark@dolthub.com
+dolt init
+dolt sql-server
+

benchmark-tools/dolt/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+services:
+  db:
+    build: ../dolt-builds
+    ports:
+      - "3306:3306"
+  sysbench:
+    build: ../sysbench
+    environment:
+      - DOLT_COMMITTISH
+      - SYSBENCH_TESTS
+      - TEST_USERNAME
+      - DB_HOST=db
+    volumes:
+      - ../python:/python
+      - ../output:/output
+    depends_on:
+      - db

benchmark-tools/mysql/docker-compose.yml

@@ -0,0 +1,21 @@
+version: "3.8"
+services:
+  mysql:
+    image: mysql:latest
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD="no"
+      - MYSQL_DATABASE=test
+      - MYSQL_USER=root
+    ports:
+      - "3306:3306"
+  sysbench:
+    build: ../sysbench
+    environment:
+      - SYSBENCH_TESTS
+      - TEST_USERNAME
+      - DB_HOST=db
+    volumes:
+      - ../python:/python
+      - ../output:/output
+    depends_on:
+      - mysql

benchmark-tools/python/sysbench_wrapper.py

@@ -3,11 +3,9 @@ import getpass
 import logging
 import os
 import platform
-import tempfile
 from datetime import datetime
 from subprocess import Popen, PIPE
-from typing import List, Tuple
-from doltpy.core import Dolt
+from typing import List, Optional
 import csv
 
 
@@ -50,30 +48,41 @@ class SysbenchFailureException(Exception):
         return '{} failed to {} with message:\n'.format(self.test, self.stage, self.message)
 
 
-def setup() -> Dolt:
-    # Setup a test repository and start the server
-    logger.info('Setting up test repo for benchmarking')
-    test_repo = init_empty_test_repo()
-    logger.info('Test repo directory {}, starting Dolt SQL server'.format(test_repo.repo_dir()))
-    test_repo.sql_server()
-    return test_repo
-
-
-def init_empty_test_repo() -> Dolt:
-    temp_dir = tempfile.mkdtemp()
-    repo_path, repo_data_dir = get_repo_path_tmp_path(temp_dir)
-    assert not os.path.exists(repo_data_dir)
-    return Dolt.init(repo_path)
-
-
-def get_repo_path_tmp_path(path: str, subpath: str = None) -> Tuple[str, str]:
-    if subpath:
-        return os.path.join(path, subpath), os.path.join(path, subpath, '.dolt')
+def main():
+    args = get_args()
+    test_list = args.tests.split(',')
+    assert all(test in SUPPORTED_BENCHMARKS for test in test_list), 'Must provide list of supported tests'
+    if args.committish:
+        logger.info('Committish provided, benchmarking Dolt')
+        run_dolt_benchmarks(args.db_host, args.committish, args.username, test_list)
     else:
-        return path, os.path.join(path, '.dolt')
+        logger.info('No committish provided, benchmarking MySQL')
+        run_mysql_benchmarks(args.db_host, args.username, test_list)
 
 
-def test_loop(test_list: List[str], test_repo) -> List[dict]:
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--db-host', help='The host for the database we will connect to')
+    parser.add_argument('--committish', help='Commit used to build Dolt bianry being tested')
+    parser.add_argument('--tests', help='List of benchmarks', type=str, required=True)
+    parser.add_argument('--username', type=str, required=False, default=getpass.getuser())
+    parser.add_argument('--note', type=str, required=False, default=None)
+    return parser.parse_args()
+
+
+def run_dolt_benchmarks(test_db_host: str, committish: str, username: str, test_list: List[str]):
+    logger.info('Executing the following tests in sysbench against Dolt: {}'.format(test_list))
+    results = test_loop(test_db_host, test_list, 'test')
+    write_output_file('dolt', committish, username, results)
+
+
+def run_mysql_benchmarks(test_db_host: str, username: str, test_list: List[str]):
+    logger.info('Executing the following tests in sysbench against MySQL: {}'.format(test_list))
+    results = test_loop(test_db_host, test_list, 'test')
+    write_output_file('mysql', None, username, results)
+
+
+def test_loop(test_db_host: str, test_list: List[str], test_db: str) -> List[dict]:
     """
     This is the main loop for running the tests and collecting the output
     :param test_list:
@@ -82,7 +91,7 @@ def test_loop(test_list: List[str], test_repo) -> List[dict]:
     result = []
     for test in test_list:
         try:
-            test_output = run_test(test_repo, test)
+            test_output = run_test(test_db_host, test_db, test)
             cur_test_res = parse_output(test_output)
             cur_test_res['test_name'] = test
             result.append(cur_test_res)
@@ -96,19 +105,15 @@ def test_loop(test_list: List[str], test_repo) -> List[dict]:
     return result
 
 
-def run_test(test_repo: Dolt, test: str) -> str:
-    # ensure table is removed
-    if TEST_TABLE in [t.name for t in test_repo.ls()]:
-        test_repo.table_rm(TEST_TABLE)
-
+def run_test(test_db_host: str, test_db: str, test: str) -> str:
     sysbench_args = [
         'sysbench',
         test,
         '--table-size=1000000',
         '--db-driver=mysql',
-        '--mysql-db={}'.format(test_repo.repo_name),
+        '--mysql-db={}'.format(test_db),
         '--mysql-user=root',
-        '--mysql-host=127.0.0.1',
+        '--mysql-host={}'.format(test_db_host),
     ]
 
     # Prepare the test
@@ -161,23 +166,14 @@ def get_os_detail():
     return '{}-{}-{}'.format(os.name, platform.system(), platform.release())
 
 
-def get_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--committish', help='Commit used to build Dolt bianry being tested', required=True)
-    parser.add_argument('--tests', help='List of benchmarks', type=str, required=True)
-    parser.add_argument('--username', type=str, required=False, default=getpass.getuser())
-    parser.add_argument('--note', type=str, required=False, default=None)
-    return parser.parse_args()
-
-
-def write_output_file(committish: str, username: str, output: List[dict]):
-    if not os.path.exists('output'):
-        os.mkdir('output')
-    output_file = 'output/{}.csv'.format(committish)
+def write_output_file(database_name: str, committish: Optional[str], username: str, output: List[dict]):
+    if not os.path.exists('/output'):
+        os.mkdir('/output')
+    output_file = '/output/{}.csv'.format(committish if committish else database_name)
     logger.info('Writing output file to {}'.format(output_file))
     with open(output_file, 'w', newline='') as csvfile:
         metadata = {
-            'database': 'dolt',
+            'database': database_name,
             'username': username,
             'committish': committish,
             'timestamp': datetime.now(),
@@ -191,15 +187,5 @@ def write_output_file(committish: str, username: str, output: List[dict]):
             writer.writerow(to_write)
 
 
-def main():
-    args = get_args()
-    test_list = args.tests.split(',')
-    assert all(test in SUPPORTED_BENCHMARKS for test in test_list), 'Must provide list of supported tests'
-    test_db = setup()
-    logger.info('Executing the following tests in sysbench: {}'.format(test_list))
-    results = test_loop(test_list, test_db)
-    write_output_file(args.committish, args.username, results)
-
-
 if __name__ == '__main__':
     main()

benchmark-tools/run_benchmarks.sh

@@ -2,9 +2,9 @@
 set -e
 set -o pipefail
 
-[ ! -z "$1" ] || (echo "Please supply a comma separated list of tests to be run"; exit 1)
+[ -n "$1" ] || (echo "Please supply a comma separated list of tests to be run"; exit 1)
 tests=$1
-[ ! -z "$1" ] || (echo "Please supply a username to associate with the benchmark"; exit 1)
+[ -n "$1" ] || (echo "Please supply a username to associate with the benchmark"; exit 1)
 username=$2
 committish_one=${3:-current}
 committish_two=${4:-current}
@@ -59,12 +59,16 @@ function build_binary_at_committish() {
     GOOS="$linux" GOARCH="$amd64" go build -o "$o/bin/$obin" "./cmd/dolt/"
   '
   echo "Moving binary to temp out/bin/dolt to $script_dir/working/$commit-dolt"
-  mv "out/bin/dolt" "$absolute_script_dir/working/$commit-dolt"
+  mv "out/bin/dolt" "$absolute_script_dir/dolt-builds/working/$commit-dolt"
 }
 
+# Set environment variables to be picked up by docker-compose
+SYSBENCH_TEST=$tests
+TEST_USERNAME=$username
 
 echo "Building binaries and benchmarking for $committish_list"
 for committish in $committish_list; do
+  DOLT_COMMITTISH=committish
   build_binary_at_committish "$committish"
   cd "$absolute_script_dir"
   if [ "$committish" != "current" ]; then
@@ -79,22 +83,14 @@ for committish in $committish_list; do
       committish="$cur_commit"
     fi
   fi
-  echo "Built binary $bin_committish, executing benchmarks"
-  docker run --rm -v `pwd`:/tools oscarbatori/dolt-sysbench /bin/bash -c '
-    set -e
-    set -o pipefail
+  echo "Built binary $bin_committish, moving to  dolt-buidls/dolt"
+  mv "dolt-builds/working/$bin_committish" "dolt-builds/dolt"
 
-    ln -s /tools/working/'"$bin_committish"' /usr/bin/dolt
-    cd /tools
+  cd dolt
+  docker-compose up -e DOLT_COMMITTISH,SYSBENCH_TEST,TEST_USERNAME
 
-    dolt config --add --global user.name benchmark
-    dolt config --add --global user.email benchmark
-
-    python3 \
-      sysbench_wrapper.py \
-      --committish='"$committish"' \
-      --tests='"$tests"' \
-      --username='"$username"'
-  '
 done
 
+echo "Running benchmarks for MySQL for comparison"
+cd mysql
+docker-compose run -e SYSBENCH_TEST,TEST_USERNAME

benchmark-tools/sysbench/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8.6-slim-buster
+
+# Get sysbench installed
+RUN apt-get install -y curl
+RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash
+RUN apt update
+RUN apt -y install sysbench
+
+COPY ./benchmark.sh /benchmark.sh
+
+ENTRYPOINT ["/benchmark.sh"]

benchmark-tools/sysbench/benchmark.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+set -o pipefail
+
+if [ -n "$DOLT_COMMITTISH" ]; then
+echo "Running sysbench tests $SYSBENCH_TESTS for test user $TEST_USERNAME"
+python /python/sysbench_wrapper.py \
+  --db-host="$DB_HOST" \
+  --committish="$DOLT_COMMITTISH" \
+  --tests="$SYSBENCH_TESTS" \
+  --username="$TEST_USERNAME"
+else
+  echo "Running sysbench tests $SYSBENCH_TESTS for test user $TEST_USERNAME"
+  python /python/sysbench_wrapper.py \
+    --db-host="$DB_HOST" \
+    --tests="$SYSBENCH_TESTS" \
+    --username="$TEST_USERNAME"
+fi

benchmark-tools/sysbench_scripts/lua/bulk_insert.lua

@@ -0,0 +1,57 @@
+#!/usr/bin/env sysbench
+-- -------------------------------------------------------------------------- --
+-- Bulk insert benchmark: do multi-row INSERTs concurrently in --threads
+-- threads with each thread inserting into its own table. The number of INSERTs
+-- executed by each thread is controlled by either --time or --events.
+-- -------------------------------------------------------------------------- --
+sysbench.hooks.report_intermediate = sysbench.report_json
+sysbench.hooks.report_cumulative = sysbench.report_json
+cursize=0
+
+function thread_init()
+   drv = sysbench.sql.driver()
+   con = drv:connect()
+end
+
+function prepare()
+   local i
+
+   local drv = sysbench.sql.driver()
+   local con = drv:connect()
+
+   for i = 1, sysbench.opt.threads do
+      print("Creating table 'sbtest" .. i .. "'...")
+      con:query(string.format([[
+        CREATE TABLE IF NOT EXISTS sbtest%d (
+          id INTEGER NOT NULL,
+          k INTEGER DEFAULT '0' NOT NULL,
+          PRIMARY KEY (id))]], i))
+   end
+end
+
+function event()
+   if (cursize == 0) then
+      con:bulk_insert_init("INSERT INTO sbtest" .. sysbench.tid+1 .. " VALUES")
+   end
+
+   cursize = cursize + 1
+
+   con:bulk_insert_next("(" .. cursize .. "," .. cursize .. ")")
+end
+
+function thread_done()
+   con:bulk_insert_done()
+   con:disconnect()
+end
+
+function cleanup()
+   local i
+
+   local drv = sysbench.sql.driver()
+   local con = drv:connect()
+
+   for i = 1, sysbench.opt.threads do
+      print("Dropping table 'sbtest" .. i .. "'...")
+      con:query("DROP TABLE IF EXISTS sbtest" .. i )
+   end
+end