diff --git a/.gitignore b/.gitignore index 8df0aeac45..11436edb59 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ go.sum go.mod .DS_Store .sqlhistory -benchmark-tools/working +benchmark-tools/dolt-builds/dolt +benchmark-tools/dolt-builds/working benchmark-tools/output diff --git a/benchmark-tools/Dockerfile b/benchmark-tools/Dockerfile deleted file mode 100644 index 61e2d6bae9..0000000000 --- a/benchmark-tools/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -FROM python:3.7-slim-stretch - -ENV DEBIAN_FRONTEND noninteractive -ENV TERM linux - -RUN set -ex \ - && buildDeps=' \ - freetds-dev \ - libkrb5-dev \ - libsasl2-dev \ - libssl-dev \ - libffi-dev \ - git \ - ' \ - && apt-get update -yqq \ - && apt-get upgrade -yqq \ - && apt-get install -yqq --no-install-recommends \ - $buildDeps \ - apt-utils \ - build-essential \ - ca-certificates \ - cpanminus \ - curl \ - dumb-init \ - fonts-liberation \ - freetds-bin \ - gconf-service \ - libappindicator1 \ - libasound2 \ - libatk-bridge2.0-0 \ - libatk1.0-0 \ - libc6 \ - libcairo2 \ - libcups2 \ - libdbus-1-3 \ - libexpat1 \ - libfontconfig1 \ - libgcc1 \ - libgconf-2-4 \ - libgdk-pixbuf2.0-0 \ - libglib2.0-0 \ - libgtk-3-0 \ - libnspr4 \ - libnss3 \ - libpango-1.0-0 \ - libpangocairo-1.0-0 \ - libpq-dev \ - libstdc++6 \ - libx11-6 \ - libx11-xcb1 \ - libxcb1 \ - libxcomposite1 \ - libxcursor1 \ - libxdamage1 \ - libxext6 \ - libxfixes3 \ - libxi6 \ - libxrandr2 \ - libxrender1 \ - libxss1 \ - libxtst6 \ - locales \ - lsb-release \ - netcat -RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash -RUN apt -y install sysbench -RUN pip install doltpy diff --git a/benchmark-tools/dolt-builds/Dockerfile b/benchmark-tools/dolt-builds/Dockerfile new file mode 100644 index 0000000000..468d25132d --- /dev/null +++ b/benchmark-tools/dolt-builds/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:18.04 + +COPY ./dolt /usr/local/bin/dolt +COPY ./start_dolt_sql_server.sh /start_dolt_sql_server.sh + +ENTRYPOINT [ "/start_dolt_sql_server.sh"] diff --git a/benchmark-tools/dolt-builds/start_dolt_sql_server.sh b/benchmark-tools/dolt-builds/start_dolt_sql_server.sh new file mode 100755 index 0000000000..2bf52724dd --- /dev/null +++ b/benchmark-tools/dolt-builds/start_dolt_sql_server.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Creating data directory and configuring Dolt" +mkdir /test +cd /test || return +dolt config --global --add user.name benchmark +dolt config --global --add user.email benchmark@dolthub.com +dolt init +dolt sql-server + diff --git a/benchmark-tools/dolt/docker-compose.yml b/benchmark-tools/dolt/docker-compose.yml new file mode 100644 index 0000000000..98d877da6e --- /dev/null +++ b/benchmark-tools/dolt/docker-compose.yml @@ -0,0 +1,18 @@ +version: "3.8" +services: + db: + build: ../dolt-builds + ports: + - "3306:3306" + sysbench: + build: ../sysbench + environment: + - DOLT_COMMITTISH + - SYSBENCH_TESTS + - TEST_USERNAME + - DB_HOST=db + volumes: + - ../python:/python + - ../output:/output + depends_on: + - db diff --git a/benchmark-tools/mysql/docker-compose.yml b/benchmark-tools/mysql/docker-compose.yml new file mode 100644 index 0000000000..1eae07f9ab --- /dev/null +++ b/benchmark-tools/mysql/docker-compose.yml @@ -0,0 +1,21 @@ +version: "3.8" +services: + mysql: + image: mysql:latest + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD="no" + - MYSQL_DATABASE=test + - MYSQL_USER=root + ports: + - "3306:3306" + sysbench: + build: ../sysbench + environment: + - SYSBENCH_TESTS + - TEST_USERNAME + - DB_HOST=db + volumes: + - ../python:/python + - ../output:/output + depends_on: + - mysql diff --git a/benchmark-tools/push_output_to_dolthub.py b/benchmark-tools/python/push_output_to_dolthub.py similarity index 100% rename from benchmark-tools/push_output_to_dolthub.py rename to benchmark-tools/python/push_output_to_dolthub.py diff --git a/benchmark-tools/sysbench_wrapper.py b/benchmark-tools/python/sysbench_wrapper.py similarity index 74% rename from benchmark-tools/sysbench_wrapper.py rename to benchmark-tools/python/sysbench_wrapper.py index 2c02a20dab..0f632c3758 100755 --- a/benchmark-tools/sysbench_wrapper.py +++ b/benchmark-tools/python/sysbench_wrapper.py @@ -3,11 +3,9 @@ import getpass import logging import os import platform -import tempfile from datetime import datetime from subprocess import Popen, PIPE -from typing import List, Tuple -from doltpy.core import Dolt +from typing import List, Optional import csv @@ -50,30 +48,41 @@ class SysbenchFailureException(Exception): return '{} failed to {} with message:\n'.format(self.test, self.stage, self.message) -def setup() -> Dolt: - # Setup a test repository and start the server - logger.info('Setting up test repo for benchmarking') - test_repo = init_empty_test_repo() - logger.info('Test repo directory {}, starting Dolt SQL server'.format(test_repo.repo_dir())) - test_repo.sql_server() - return test_repo - - -def init_empty_test_repo() -> Dolt: - temp_dir = tempfile.mkdtemp() - repo_path, repo_data_dir = get_repo_path_tmp_path(temp_dir) - assert not os.path.exists(repo_data_dir) - return Dolt.init(repo_path) - - -def get_repo_path_tmp_path(path: str, subpath: str = None) -> Tuple[str, str]: - if subpath: - return os.path.join(path, subpath), os.path.join(path, subpath, '.dolt') +def main(): + args = get_args() + test_list = args.tests.split(',') + assert all(test in SUPPORTED_BENCHMARKS for test in test_list), 'Must provide list of supported tests' + if args.committish: + logger.info('Committish provided, benchmarking Dolt') + run_dolt_benchmarks(args.db_host, args.committish, args.username, test_list) else: - return path, os.path.join(path, '.dolt') + logger.info('No committish provided, benchmarking MySQL') + run_mysql_benchmarks(args.db_host, args.username, test_list) -def test_loop(test_list: List[str], test_repo) -> List[dict]: +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--db-host', help='The host for the database we will connect to') + parser.add_argument('--committish', help='Commit used to build Dolt bianry being tested') + parser.add_argument('--tests', help='List of benchmarks', type=str, required=True) + parser.add_argument('--username', type=str, required=False, default=getpass.getuser()) + parser.add_argument('--note', type=str, required=False, default=None) + return parser.parse_args() + + +def run_dolt_benchmarks(test_db_host: str, committish: str, username: str, test_list: List[str]): + logger.info('Executing the following tests in sysbench against Dolt: {}'.format(test_list)) + results = test_loop(test_db_host, test_list, 'test') + write_output_file('dolt', committish, username, results) + + +def run_mysql_benchmarks(test_db_host: str, username: str, test_list: List[str]): + logger.info('Executing the following tests in sysbench against MySQL: {}'.format(test_list)) + results = test_loop(test_db_host, test_list, 'test') + write_output_file('mysql', None, username, results) + + +def test_loop(test_db_host: str, test_list: List[str], test_db: str) -> List[dict]: """ This is the main loop for running the tests and collecting the output :param test_list: @@ -82,7 +91,7 @@ def test_loop(test_list: List[str], test_repo) -> List[dict]: result = [] for test in test_list: try: - test_output = run_test(test_repo, test) + test_output = run_test(test_db_host, test_db, test) cur_test_res = parse_output(test_output) cur_test_res['test_name'] = test result.append(cur_test_res) @@ -96,19 +105,15 @@ def test_loop(test_list: List[str], test_repo) -> List[dict]: return result -def run_test(test_repo: Dolt, test: str) -> str: - # ensure table is removed - if TEST_TABLE in [t.name for t in test_repo.ls()]: - test_repo.table_rm(TEST_TABLE) - +def run_test(test_db_host: str, test_db: str, test: str) -> str: sysbench_args = [ 'sysbench', test, '--table-size=1000000', '--db-driver=mysql', - '--mysql-db={}'.format(test_repo.repo_name), + '--mysql-db={}'.format(test_db), '--mysql-user=root', - '--mysql-host=127.0.0.1', + '--mysql-host={}'.format(test_db_host), ] # Prepare the test @@ -161,23 +166,14 @@ def get_os_detail(): return '{}-{}-{}'.format(os.name, platform.system(), platform.release()) -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--committish', help='Commit used to build Dolt bianry being tested', required=True) - parser.add_argument('--tests', help='List of benchmarks', type=str, required=True) - parser.add_argument('--username', type=str, required=False, default=getpass.getuser()) - parser.add_argument('--note', type=str, required=False, default=None) - return parser.parse_args() - - -def write_output_file(committish: str, username: str, output: List[dict]): - if not os.path.exists('output'): - os.mkdir('output') - output_file = 'output/{}.csv'.format(committish) +def write_output_file(database_name: str, committish: Optional[str], username: str, output: List[dict]): + if not os.path.exists('/output'): + os.mkdir('/output') + output_file = '/output/{}.csv'.format(committish if committish else database_name) logger.info('Writing output file to {}'.format(output_file)) with open(output_file, 'w', newline='') as csvfile: metadata = { - 'database': 'dolt', + 'database': database_name, 'username': username, 'committish': committish, 'timestamp': datetime.now(), @@ -191,15 +187,5 @@ def write_output_file(committish: str, username: str, output: List[dict]): writer.writerow(to_write) -def main(): - args = get_args() - test_list = args.tests.split(',') - assert all(test in SUPPORTED_BENCHMARKS for test in test_list), 'Must provide list of supported tests' - test_db = setup() - logger.info('Executing the following tests in sysbench: {}'.format(test_list)) - results = test_loop(test_list, test_db) - write_output_file(args.committish, args.username, results) - - if __name__ == '__main__': main() diff --git a/benchmark-tools/run_benchmarks.sh b/benchmark-tools/run_benchmarks.sh index 06dc00fba5..082f23b791 100755 --- a/benchmark-tools/run_benchmarks.sh +++ b/benchmark-tools/run_benchmarks.sh @@ -2,9 +2,9 @@ set -e set -o pipefail -[ ! -z "$1" ] || (echo "Please supply a comma separated list of tests to be run"; exit 1) +[ -n "$1" ] || (echo "Please supply a comma separated list of tests to be run"; exit 1) tests=$1 -[ ! -z "$1" ] || (echo "Please supply a username to associate with the benchmark"; exit 1) +[ -n "$1" ] || (echo "Please supply a username to associate with the benchmark"; exit 1) username=$2 committish_one=${3:-current} committish_two=${4:-current} @@ -59,12 +59,16 @@ function build_binary_at_committish() { GOOS="$linux" GOARCH="$amd64" go build -o "$o/bin/$obin" "./cmd/dolt/" ' echo "Moving binary to temp out/bin/dolt to $script_dir/working/$commit-dolt" - mv "out/bin/dolt" "$absolute_script_dir/working/$commit-dolt" + mv "out/bin/dolt" "$absolute_script_dir/dolt-builds/working/$commit-dolt" } +# Set environment variables to be picked up by docker-compose +SYSBENCH_TEST=$tests +TEST_USERNAME=$username echo "Building binaries and benchmarking for $committish_list" for committish in $committish_list; do + DOLT_COMMITTISH=committish build_binary_at_committish "$committish" cd "$absolute_script_dir" if [ "$committish" != "current" ]; then @@ -79,22 +83,14 @@ for committish in $committish_list; do committish="$cur_commit" fi fi - echo "Built binary $bin_committish, executing benchmarks" - docker run --rm -v `pwd`:/tools oscarbatori/dolt-sysbench /bin/bash -c ' - set -e - set -o pipefail + echo "Built binary $bin_committish, moving to dolt-buidls/dolt" + mv "dolt-builds/working/$bin_committish" "dolt-builds/dolt" - ln -s /tools/working/'"$bin_committish"' /usr/bin/dolt - cd /tools + cd dolt + docker-compose up -e DOLT_COMMITTISH,SYSBENCH_TEST,TEST_USERNAME - dolt config --add --global user.name benchmark - dolt config --add --global user.email benchmark - - python3 \ - sysbench_wrapper.py \ - --committish='"$committish"' \ - --tests='"$tests"' \ - --username='"$username"' - ' done +echo "Running benchmarks for MySQL for comparison" +cd mysql +docker-compose run -e SYSBENCH_TEST,TEST_USERNAME \ No newline at end of file diff --git a/benchmark-tools/sysbench/Dockerfile b/benchmark-tools/sysbench/Dockerfile new file mode 100644 index 0000000000..209e1a6144 --- /dev/null +++ b/benchmark-tools/sysbench/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.8.6-slim-buster + +# Get sysbench installed +RUN apt-get install -y curl +RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash +RUN apt update +RUN apt -y install sysbench + +COPY ./benchmark.sh /benchmark.sh + +ENTRYPOINT ["/benchmark.sh"] \ No newline at end of file diff --git a/benchmark-tools/sysbench/benchmark.sh b/benchmark-tools/sysbench/benchmark.sh new file mode 100644 index 0000000000..e6f29daa03 --- /dev/null +++ b/benchmark-tools/sysbench/benchmark.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e +set -o pipefail + +if [ -n "$DOLT_COMMITTISH" ]; then +echo "Running sysbench tests $SYSBENCH_TESTS for test user $TEST_USERNAME" +python /python/sysbench_wrapper.py \ + --db-host="$DB_HOST" \ + --committish="$DOLT_COMMITTISH" \ + --tests="$SYSBENCH_TESTS" \ + --username="$TEST_USERNAME" +else + echo "Running sysbench tests $SYSBENCH_TESTS for test user $TEST_USERNAME" + python /python/sysbench_wrapper.py \ + --db-host="$DB_HOST" \ + --tests="$SYSBENCH_TESTS" \ + --username="$TEST_USERNAME" +fi diff --git a/benchmark-tools/sysbench_scripts/lua/bulk_insert.lua b/benchmark-tools/sysbench_scripts/lua/bulk_insert.lua new file mode 100755 index 0000000000..e4227fb993 --- /dev/null +++ b/benchmark-tools/sysbench_scripts/lua/bulk_insert.lua @@ -0,0 +1,57 @@ +#!/usr/bin/env sysbench +-- -------------------------------------------------------------------------- -- +-- Bulk insert benchmark: do multi-row INSERTs concurrently in --threads +-- threads with each thread inserting into its own table. The number of INSERTs +-- executed by each thread is controlled by either --time or --events. +-- -------------------------------------------------------------------------- -- +sysbench.hooks.report_intermediate = sysbench.report_json +sysbench.hooks.report_cumulative = sysbench.report_json +cursize=0 + +function thread_init() + drv = sysbench.sql.driver() + con = drv:connect() +end + +function prepare() + local i + + local drv = sysbench.sql.driver() + local con = drv:connect() + + for i = 1, sysbench.opt.threads do + print("Creating table 'sbtest" .. i .. "'...") + con:query(string.format([[ + CREATE TABLE IF NOT EXISTS sbtest%d ( + id INTEGER NOT NULL, + k INTEGER DEFAULT '0' NOT NULL, + PRIMARY KEY (id))]], i)) + end +end + +function event() + if (cursize == 0) then + con:bulk_insert_init("INSERT INTO sbtest" .. sysbench.tid+1 .. " VALUES") + end + + cursize = cursize + 1 + + con:bulk_insert_next("(" .. cursize .. "," .. cursize .. ")") +end + +function thread_done() + con:bulk_insert_done() + con:disconnect() +end + +function cleanup() + local i + + local drv = sysbench.sql.driver() + local con = drv:connect() + + for i = 1, sysbench.opt.threads do + print("Dropping table 'sbtest" .. i .. "'...") + con:query("DROP TABLE IF EXISTS sbtest" .. i ) + end +end diff --git a/benchmark-tools/sysbench_scripts/lua/table_scan.lua b/benchmark-tools/sysbench_scripts/lua/table_scan.lua new file mode 100644 index 0000000000..e69de29bb2