From a43b8d2e564ff57cedcb5dc02d2d90a379170b5c Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Mon, 24 Jan 2022 10:08:25 -0800 Subject: [PATCH] /{go,.github}: remove benchmarks, add dolt_builder and sysbench_runner utils --- .../actions/sysbench-runner-tests/action.yaml | 5 + .../workflows/ci-sysbench-runner-tests.yaml | 15 + .gitignore | 10 +- benchmark/perf_tools/README.md | 137 ----- benchmark/perf_tools/dolt-builds/Dockerfile | 6 - .../dolt-builds/start_dolt_sql_server.sh | 10 - benchmark/perf_tools/dolt/docker-compose.yml | 16 - benchmark/perf_tools/mysql/docker-compose.yml | 19 - benchmark/perf_tools/python/compare_perf.py | 51 -- .../python/push_output_to_dolthub.py | 34 -- .../perf_tools/python/sysbench_wrapper.py | 230 -------- benchmark/perf_tools/run_benchmarks.sh | 105 ---- benchmark/perf_tools/sysbench/Dockerfile | 11 - benchmark/perf_tools/sysbench/benchmark.sh | 20 - .../sysbench_scripts/lua/bulk_insert.lua | 57 -- .../lua/covering_index_scan.lua | 24 - .../sysbench_scripts/lua/dolt_common.lua | 141 ----- .../sysbench_scripts/lua/groupby_scan.lua | 24 - .../sysbench_scripts/lua/index_scan.lua | 24 - .../sysbench_scripts/lua/local_sysbench.sh | 1 - .../sysbench_scripts/lua/table_scan.lua | 25 - .../DoltRegressionsJenkinsfile | 75 --- .../DoltReleaseBenchmarkJenkinsfile | 35 -- benchmark/sql_regressions/nightly.vars | 3 - benchmark/sql_regressions/releases.vars | 3 - benchmark/sql_regressions/run_regressions.sh | 302 ----------- .../SQLLogicTestTesterJenkinsfile | 46 -- benchmark/sqllogictest_tester/run_tester.sh | 280 ---------- benchmark/sqllogictest_tester/tester.vars | 2 - .../continuous_integration/SysbenchDockerfile | 45 ++ .../SysbenchDockerfile.dockerignore | 7 + .../continuous_integration/config.json | 22 + .../sysbench-runner-tests-entrypoint.sh | 5 + .../benchmarks.go | 0 .../dataset.go | 0 .../helpers.go | 0 .../main.go | 0 .../results.go | 0 .../seed_schema.go | 0 .../performance}/import_tester/csv_gen.py | 0 .../import_tester/run_importer.sh | 0 .../performance/scripts}/local_benchmark.sh | 7 +- go/performance/utils/dolt_builder/README.md | 23 + go/performance/utils/dolt_builder/cmd/main.go | 45 ++ .../utils/dolt_builder/constants.go | 19 + go/performance/utils/dolt_builder/debug.go | 38 ++ go/performance/utils/dolt_builder/git.go | 83 +++ go/performance/utils/dolt_builder/run.go | 179 +++++++ .../utils/sysbench_runner/README.md | 100 ++++ .../utils/sysbench_runner/cmd/main.go | 54 ++ .../utils/sysbench_runner/config.go | 505 ++++++++++++++++++ .../utils/sysbench_runner/config_test.go | 107 ++++ go/performance/utils/sysbench_runner/csv.go | 332 ++++++++++++ .../utils/sysbench_runner/csv_test.go | 47 ++ go/performance/utils/sysbench_runner/debug.go | 38 ++ go/performance/utils/sysbench_runner/dolt.go | 271 ++++++++++ go/performance/utils/sysbench_runner/json.go | 87 +++ .../utils/sysbench_runner/json_test.go | 47 ++ go/performance/utils/sysbench_runner/mysql.go | 183 +++++++ .../utils/sysbench_runner/results.go | 393 ++++++++++++++ .../utils/sysbench_runner/results_test.go | 327 ++++++++++++ go/performance/utils/sysbench_runner/run.go | 92 ++++ 62 files changed, 3081 insertions(+), 1686 deletions(-) create mode 100644 .github/actions/sysbench-runner-tests/action.yaml create mode 100644 .github/workflows/ci-sysbench-runner-tests.yaml delete mode 100644 benchmark/perf_tools/README.md delete mode 100644 benchmark/perf_tools/dolt-builds/Dockerfile delete mode 100755 benchmark/perf_tools/dolt-builds/start_dolt_sql_server.sh delete mode 100644 benchmark/perf_tools/dolt/docker-compose.yml delete mode 100644 benchmark/perf_tools/mysql/docker-compose.yml delete mode 100644 benchmark/perf_tools/python/compare_perf.py delete mode 100644 benchmark/perf_tools/python/push_output_to_dolthub.py delete mode 100755 benchmark/perf_tools/python/sysbench_wrapper.py delete mode 100755 benchmark/perf_tools/run_benchmarks.sh delete mode 100644 benchmark/perf_tools/sysbench/Dockerfile delete mode 100755 benchmark/perf_tools/sysbench/benchmark.sh delete mode 100755 benchmark/perf_tools/sysbench_scripts/lua/bulk_insert.lua delete mode 100644 benchmark/perf_tools/sysbench_scripts/lua/covering_index_scan.lua delete mode 100644 benchmark/perf_tools/sysbench_scripts/lua/dolt_common.lua delete mode 100644 benchmark/perf_tools/sysbench_scripts/lua/groupby_scan.lua delete mode 100644 benchmark/perf_tools/sysbench_scripts/lua/index_scan.lua delete mode 100644 benchmark/perf_tools/sysbench_scripts/lua/local_sysbench.sh delete mode 100644 benchmark/perf_tools/sysbench_scripts/lua/table_scan.lua delete mode 100644 benchmark/sql_regressions/DoltRegressionsJenkinsfile delete mode 100644 benchmark/sql_regressions/DoltReleaseBenchmarkJenkinsfile delete mode 100644 benchmark/sql_regressions/nightly.vars delete mode 100644 benchmark/sql_regressions/releases.vars delete mode 100755 benchmark/sql_regressions/run_regressions.sh delete mode 100644 benchmark/sqllogictest_tester/SQLLogicTestTesterJenkinsfile delete mode 100755 benchmark/sqllogictest_tester/run_tester.sh delete mode 100644 benchmark/sqllogictest_tester/tester.vars create mode 100644 go/performance/continuous_integration/SysbenchDockerfile create mode 100644 go/performance/continuous_integration/SysbenchDockerfile.dockerignore create mode 100644 go/performance/continuous_integration/config.json create mode 100755 go/performance/continuous_integration/sysbench-runner-tests-entrypoint.sh rename go/performance/{benchmarks => import_benchmarker}/benchmarks.go (100%) rename go/performance/{benchmarks => import_benchmarker}/dataset.go (100%) rename go/performance/{benchmarks => import_benchmarker}/helpers.go (100%) rename go/performance/{benchmarks => import_benchmarker}/main.go (100%) rename go/performance/{benchmarks => import_benchmarker}/results.go (100%) rename go/performance/{benchmarks => import_benchmarker}/seed_schema.go (100%) rename {benchmark => go/performance}/import_tester/csv_gen.py (100%) rename {benchmark => go/performance}/import_tester/run_importer.sh (100%) rename {benchmark/perf_tools/sysbench_scripts => go/performance/scripts}/local_benchmark.sh (87%) create mode 100644 go/performance/utils/dolt_builder/README.md create mode 100644 go/performance/utils/dolt_builder/cmd/main.go create mode 100644 go/performance/utils/dolt_builder/constants.go create mode 100644 go/performance/utils/dolt_builder/debug.go create mode 100644 go/performance/utils/dolt_builder/git.go create mode 100644 go/performance/utils/dolt_builder/run.go create mode 100644 go/performance/utils/sysbench_runner/README.md create mode 100644 go/performance/utils/sysbench_runner/cmd/main.go create mode 100644 go/performance/utils/sysbench_runner/config.go create mode 100644 go/performance/utils/sysbench_runner/config_test.go create mode 100644 go/performance/utils/sysbench_runner/csv.go create mode 100644 go/performance/utils/sysbench_runner/csv_test.go create mode 100644 go/performance/utils/sysbench_runner/debug.go create mode 100644 go/performance/utils/sysbench_runner/dolt.go create mode 100644 go/performance/utils/sysbench_runner/json.go create mode 100644 go/performance/utils/sysbench_runner/json_test.go create mode 100644 go/performance/utils/sysbench_runner/mysql.go create mode 100644 go/performance/utils/sysbench_runner/results.go create mode 100644 go/performance/utils/sysbench_runner/results_test.go create mode 100644 go/performance/utils/sysbench_runner/run.go diff --git a/.github/actions/sysbench-runner-tests/action.yaml b/.github/actions/sysbench-runner-tests/action.yaml new file mode 100644 index 0000000000..f24cc6a19e --- /dev/null +++ b/.github/actions/sysbench-runner-tests/action.yaml @@ -0,0 +1,5 @@ +name: 'Dolt Sysbench Runner Utility Test' +description: "Smoke tests for Dolt's sysbench runner utility" +runs: + using: 'docker' + image: '../../../SysbenchDockerfile' diff --git a/.github/workflows/ci-sysbench-runner-tests.yaml b/.github/workflows/ci-sysbench-runner-tests.yaml new file mode 100644 index 0000000000..313b117706 --- /dev/null +++ b/.github/workflows/ci-sysbench-runner-tests.yaml @@ -0,0 +1,15 @@ +name: Test Sysbench Runner Utility Works + +on: [pull_request] + +jobs: + mysql_client_integrations_job: + runs-on: ubuntu-18.04 + name: Test Sysbench Runner + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Copy Dockerfile + run: cp -r ./go/performance/continuous_integration/. . + - name: Test sysbench runner + uses: ./.github/actions/sysbench-runner-tests diff --git a/.gitignore b/.gitignore index 28e71e2a0f..8d5fa6da5e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,10 @@ venv .DS_Store .sqlhistory -benchmark/perf_tools/dolt-builds/dolt -benchmark/perf_tools/dolt-builds/working -benchmark/perf_tools/output +test.sh -test.sh \ No newline at end of file +# ignore cp'd sysbench runner testing files +SysbenchDockerfile +SysbenchDockerfile.dockerignore +sysbench-runner-tests-entrypoint.sh +config.json diff --git a/benchmark/perf_tools/README.md b/benchmark/perf_tools/README.md deleted file mode 100644 index 5c1ac8af54..0000000000 --- a/benchmark/perf_tools/README.md +++ /dev/null @@ -1,137 +0,0 @@ -## Background -This directory contains a toolkit for benchmarking Dolt SQL performance. [`sysbench`](https://github.com/akopytov/sysbench) is an industry standard tool for benchmarking database performance, particularly MySQL. The goal is make it convenient for the Dolt team and open source contributors to measure the impact of changes. - -The output of this toolchain is benchmark results for one, or two, versions of Dolt, along with MySQL. These benchmark results are associated with a unique "run ID", a hash value that ties together benchmarks that were run on the same machine at the same time using the same command. The motivation for this design is to create comparative benchmarks that isolate hardware conditions to some reasonable degree while maintaining an easy to use interface. - -## Architecture -To produce this output we provide an interface for running a specified set of benchmarks against one, or two, versions of Dolt. The versions are specified by "committish", essentially Git reference that can be resolved to a commit. For example we might wish to compare the SQL performance of Dolt at: -- the HEAD of the currently checked out branch and the tag referencing the most recent release -- two branches with different approaches to a performance problem -- a commit containing a performance regression to the HEAD of master - -To achieve this the goal for each committish provided we: -- build Dolt at the specified committish inside a Docker container for repeatability -- stand up container running Dolt SQL server -- stand up container running [`sysbench` MySQL driver](https://github.com/akopytov/sysbench/tree/master/src/drivers/mysql) that connects to the Dolt SQL server -- execute the benchmarks specified in the `sysbench` container, and store the results as a CSV file - -We also execute the same set of benchmarks against MySQL for comparison. All the benchmarks, those associated with the Dolt refs and MySQL, are associated with a unique `run_id` which identi an invocation of - -## Example -A common use-case might be to compare Dolt built from the current working set in your local checkout to MySQL. To do this we can run the following: -``` -$ ./run_benchmarks.sh all current -``` - -This takes the current checkout of Dolt, builds a binary, and executes the supported benchmarks in a `docker-compose` setup. It does the same for MySQL. Each invocation of `run_benchmarks.sh` is associatd with a run ID, for example `58296063ab3c2a6701f8f986`. This run ID identifies the CSV file: -``` -$ ls -ltr output -total 16 --rw-r--r-- 1 oscarbatori staff 1727 Nov 29 19:59 58296063ab3c2a6701f8f986.csv -``` - -Each row corresponds to an invocation of test on either MySQL, or a compilation of Dolt. Each row indicates this. - -## Requirements -To run this stack a few things are required: -- a bash shell to execute `run_benchmarks.sh` -- Docker: the Dolt binaries are built in Docker, and the MySQL server and the Dolt SQL server are run in a container, as is `sysbench` -- Python: if you want to upload results to DoltHub, we use [Doltpy's](https://pypi.org/project/doltpy/) tools for pushing data to DoltHub, so you need to install Doltpy - -## Uploading to DoltHub -We can upload the results to DoltHub using `push_results_to_dolthub.py` as follows: -``` -$ python push_outputp_to_dolthub.py --results-file output/58296063ab3c2a6701f8f986.csv --remote-results-db dolthub/dolt-benchmarks-test --branch test-run -``` - -These results will then be available to the team for analysis, and via our API for rendering on our benchmarking documentation. - -## `sysbench_scripts` -This directory contains Lua scripts that implement custom `sysbench` behavior. Above we described `sysbench` as "scriptable". In practice this means defining `sysbench` tests in Lua scripts and passing those scripts to the `sysbench` command. As an example to run `sysbench` using a predefined test one might run: -``` -sysbench \ - bulk_insert \ - --table-size=1000000 \ - --db-driver=mysql \ - --mysql-db=test_db \ - --mysql-user=root \ - --mysql-host=127.0.0.1 \ -run -``` - -Suppose we wanted to define custom behavior, or some custom output method, then we would define `sysbench_scripts/my_custom_test.lua`, and then run: -``` -sysbench \ - sysbench_scripts/lua/my_custom_test.lua \ - --table-size=1000000 \ - --db-driver=mysql \ - --mysql-db=test_db \ - --mysql-user=root \ - --mysql-host=127.0.0.1 \ -run -``` - -This passes the script `sysbench_scripts/lua/my_custom_test.lua` which is then executed and measured. Here is an example, `bulk_insert`, taken from the [`sysbench` core out-of-the-box SQL benchmarks](https://github.com/akopytov/sysbench/tree/master/src/lua): -``` -#!/usr/bin/env sysbench --- -------------------------------------------------------------------------- -- --- Bulk insert benchmark: do multi-row INSERTs concurrently in --threads --- threads with each thread inserting into its own table. The number of INSERTs --- executed by each thread is controlled by either --time or --events. --- -------------------------------------------------------------------------- -- - -cursize=0 - -function thread_init() - drv = sysbench.sql.driver() - con = drv:connect() -end - -function prepare() - local i - - local drv = sysbench.sql.driver() - local con = drv:connect() - - for i = 1, sysbench.opt.threads do - print("Creating table 'sbtest" .. i .. "'...") - con:query(string.format([[ - CREATE TABLE IF NOT EXISTS sbtest%d ( - id INTEGER NOT NULL, - k INTEGER DEFAULT '0' NOT NULL, - PRIMARY KEY (id))]], i)) - end -end - - -function event() - - if (cursize == 0) then - con:bulk_insert_init("INSERT INTO sbtest" .. sysbench.tid+1 .. " VALUES") - end - - cursize = cursize + 1 - - con:bulk_insert_next("(" .. cursize .. "," .. cursize .. ")") -end - -function thread_done() - con:bulk_insert_done() - con:disconnect() -end - -function cleanup() - local i - - local drv = sysbench.sql.driver() - local con = drv:connect() - - for i = 1, sysbench.opt.threads do - print("Dropping table 'sbtest" .. i .. "'...") - con:query("DROP TABLE IF EXISTS sbtest" .. i ) - end - -end -``` - -The structure is relatively familiar if you have seen testing frameworks. There is basically a setup, execute and measure, and teardown section. This directory will be home to the custom tests we define to measure Dolt's performance. diff --git a/benchmark/perf_tools/dolt-builds/Dockerfile b/benchmark/perf_tools/dolt-builds/Dockerfile deleted file mode 100644 index 683f78a133..0000000000 --- a/benchmark/perf_tools/dolt-builds/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu:18.04 - -COPY ./working/dolt /usr/local/bin/dolt -COPY ./start_dolt_sql_server.sh /start_dolt_sql_server.sh - -ENTRYPOINT [ "/start_dolt_sql_server.sh"] diff --git a/benchmark/perf_tools/dolt-builds/start_dolt_sql_server.sh b/benchmark/perf_tools/dolt-builds/start_dolt_sql_server.sh deleted file mode 100755 index 8877782532..0000000000 --- a/benchmark/perf_tools/dolt-builds/start_dolt_sql_server.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e - -echo "Creating data directory and configuring Dolt" -[ -d /test ] || mkdir /test -cd /test -dolt config --global --add user.name benchmark -dolt config --global --add user.email benchmark@dolthub.com -dolt init -exec dolt sql-server --host=0.0.0.0 --query-parallelism=8 \ No newline at end of file diff --git a/benchmark/perf_tools/dolt/docker-compose.yml b/benchmark/perf_tools/dolt/docker-compose.yml deleted file mode 100644 index 47c966011b..0000000000 --- a/benchmark/perf_tools/dolt/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.8" -services: - db: - build: ../dolt-builds - sysbench: - build: ../sysbench - environment: - - DOLT_COMMITTISH - - SYSBENCH_TESTS - - TEST_USERNAME - - DB_HOST=db - volumes: - - ../python:/python - - ../output:/output - depends_on: - - db diff --git a/benchmark/perf_tools/mysql/docker-compose.yml b/benchmark/perf_tools/mysql/docker-compose.yml deleted file mode 100644 index bfcd7ea047..0000000000 --- a/benchmark/perf_tools/mysql/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: "3.8" -services: - db: - image: mysql:5.7 - environment: - - MYSQL_ALLOW_EMPTY_PASSWORD="yes" - - MYSQL_DATABASE=test - - MYSQL_USER=root - sysbench: - build: ../sysbench - environment: - - SYSBENCH_TESTS - - TEST_USERNAME - - DB_HOST=db - volumes: - - ../python:/python - - ../output:/output - depends_on: - - db diff --git a/benchmark/perf_tools/python/compare_perf.py b/benchmark/perf_tools/python/compare_perf.py deleted file mode 100644 index 95ee82c35b..0000000000 --- a/benchmark/perf_tools/python/compare_perf.py +++ /dev/null @@ -1,51 +0,0 @@ -import sys -import csv - -from math import fabs - -def average_time(row): - return float(row['latency_sum']) / float(row['sql_transactions']) - -def read_result_data(filename, tests): - mysql_result_data = {} - dolt_result_data = {} - with open(filename) as f: - csvr = csv.DictReader(f) - for row in csvr: - test_name = row['test_name'] - if 'all' in tests or test_name in tests: - if row['database'] == 'dolt': - dolt_result_data[test_name] = average_time(row) - else: - mysql_result_data[test_name] = average_time(row) - - return mysql_result_data, dolt_result_data - -initial_result_file = sys.argv[1] -updated_result_file = sys.argv[2] -test_names = sys.argv[3] if len(sys.argv) >= 4 else "all" - -initial_mysql, initial_dolt = read_result_data(initial_result_file, test_names) -updated_mysql, updated_dolt = read_result_data(updated_result_file, test_names) - -print("initial mysql", initial_mysql, "initial dolt", initial_dolt) -print("updated mysql", updated_mysql, "updated dolt", updated_dolt) -for name, time in initial_dolt.items(): - if name in updated_dolt: - updated_time = updated_dolt[name] - delta = time - updated_time - initial_mysql_multiplier = time / initial_mysql[name] - updated_mysql_multiplier = updated_time / updated_mysql[name] - percent_change = 1.0 - (updated_time / time) - faster_slower = "faster" if percent_change > 0.0 else "slower" - - print("% -24s: %.2f%% %s - mysql multiplier: %.2fx -> %.02fx" % (name, fabs(percent_change)*100, faster_slower, initial_mysql_multiplier, updated_mysql_multiplier)) - else: - print("% -24s: %4.4f - Test removed from updated result file" % (name, float(time))) - -for name, time in updated_dolt.items(): - if name not in initial_dolt: - print("% -24s: %4.4f - New test addeed to updated result file" % (name, float(time))) - - - diff --git a/benchmark/perf_tools/python/push_output_to_dolthub.py b/benchmark/perf_tools/python/push_output_to_dolthub.py deleted file mode 100644 index 041fac7b38..0000000000 --- a/benchmark/perf_tools/python/push_output_to_dolthub.py +++ /dev/null @@ -1,34 +0,0 @@ -from doltpy.etl import get_df_table_writer, get_dolt_loader, load_to_dolthub -from doltpy.core.system_helpers import get_logger -import pandas as pd -import argparse -import os - -RESULTS_TABLE_PKS = ['username', 'timestamp', 'committish', 'test_name'] -RESULTS_TABLE = 'sysbench_benchmark' - - -logger = get_logger(__name__) - - -def write_results_to_dolt(results_file: str, remote: str, branch: str): - table_writer = get_df_table_writer(RESULTS_TABLE, - lambda: pd.read_csv(results_file), - RESULTS_TABLE_PKS, - import_mode='update') - loader = get_dolt_loader(table_writer, True, 'benchmark run', branch) - load_to_dolthub(loader, clone=True, push=True, remote_name='origin', remote_url=remote) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--results-file', type=str, required=True) - parser.add_argument('--remote-results-db', type=str, required=True) - parser.add_argument('--remote-results-db-branch', type=str, required=False, default='master') - args = parser.parse_args() - logger.info('Writing the results of the tests') - write_results_to_dolt(args.results_file, args.remote_results_db, args.remote_results_db_branch) - - -if __name__ == '__main__': - main() diff --git a/benchmark/perf_tools/python/sysbench_wrapper.py b/benchmark/perf_tools/python/sysbench_wrapper.py deleted file mode 100755 index 2e123c93b9..0000000000 --- a/benchmark/perf_tools/python/sysbench_wrapper.py +++ /dev/null @@ -1,230 +0,0 @@ -import argparse -import getpass -import logging -import os -import platform -from datetime import datetime -from subprocess import Popen, PIPE -from typing import List, Optional -import csv - -logger = logging.getLogger(__name__) - -# This is the list of benchmarks that we have validated can successfully run with Dolt -SUPPORTED_BENCHMARKS = [ - 'bulk_insert', - 'oltp_read_only', - 'oltp_insert', - 'oltp_point_select', - 'select_random_points', - 'select_random_ranges', - 'oltp_delete', - 'oltp_write_only', - 'oltp_read_write', - 'oltp_update_index', - 'oltp_update_non_index' -] - -TEST_TABLE = 'sbtest1' - -RESULTS_TABLE = 'sysbench_benchmark' -OUTPUT_MAPPING = { - 'read': 'sql_read_queries', - 'write': 'sql_write_queries', - 'other': 'sql_other_queries', - 'total': 'sql_total_queries', - 'transactions': 'sql_transactions', - 'ignored errors': 'sql_ignored_errors', - 'reconnects': 'sql_reconnects', - 'total time': 'total_time', - 'total number of events': 'total_number_of_events', - 'min': 'latency_minimum', - 'avg': 'latency_average', - 'max': 'latency_maximum', - '95th percentile': 'latency_percentile_95th', - 'sum': 'latency_sum' -} - - -class SysbenchFailureException(Exception): - - def __init__(self, test: str, stage: str, message: str): - self.test = test - self.stage = stage - self.message = message - - def __str__(self): - return '{} failed to {} with message:\n'.format(self.test, self.stage, self.message) - - -def main(): - logger.setLevel(logging.INFO) - args = get_args() - test_list = args.tests.split(',') - if len(test_list) == 1 and test_list == ['all']: - test_list = SUPPORTED_BENCHMARKS - else: - assert all(test in SUPPORTED_BENCHMARKS for test in test_list), 'Must provide list of supported tests' - - logger.info('Running with run ID {}'.format(args.run_id)) - if args.committish: - logger.info('Committish provided, benchmarking Dolt') - run_dolt_benchmarks(args.run_id, args.db_host, args.committish, args.username, test_list, args.table_size) - else: - logger.info('No committish provided, benchmarking MySQL') - run_mysql_benchmarks(args.run_id, args.db_host, args.username, test_list, args.table_size) - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--db-host', help='The host for the database we will connect to') - parser.add_argument('--committish', help='Commit used to build Dolt bianry being tested') - parser.add_argument('--tests', help='List of benchmarks', type=str, default=True) - parser.add_argument('--username', type=str, required=False, default=getpass.getuser()) - parser.add_argument('--note', type=str, required=False, default=None) - parser.add_argument('--table-size', type=int, default=10000) - parser.add_argument('--run-id', type=str, required=True) - return parser.parse_args() - - -def run_dolt_benchmarks(run_id: str, - test_db_host: str, - committish: str, - username: str, - test_list: List[str], - table_size: int): - logger.info('Executing the following tests in sysbench against Dolt: {}'.format(test_list)) - results = test_loop(test_db_host, test_list, 'test', table_size) - write_output_file(run_id, 'dolt', committish, username, results, datetime.now(), table_size) - - -def run_mysql_benchmarks(run_id: str, - test_db_host: str, - username: str, - test_list: List[str], - table_size: int): - logger.info('Executing the following tests in sysbench against MySQL: {}'.format(test_list)) - results = test_loop(test_db_host, test_list, 'test', table_size) - write_output_file(run_id, 'mysql', None, username, results, datetime.now(), table_size) - - -def test_loop(test_db_host: str, test_list: List[str], test_db: str, table_size: int) -> List[dict]: - """ - This is the main loop for running the tests and collecting the output - :param test_list: - :return: - """ - result = [] - for test in test_list: - try: - test_output = run_test(test_db_host, test_db, test, table_size) - cur_test_res = parse_output(test_output) - cur_test_res['test_name'] = test - result.append(cur_test_res) - - except SysbenchFailureException as e: - logger.error('Test {} failed to produce output, moving in, error was:\n{}'.format(test, e)) - - except ValueError as e: - logger.error('Failure caused by failure to parse output:\n{}'.format(e)) - - return result - - -def run_test(test_db_host: str, test_db: str, test: str, table_size: int) -> str: - sysbench_args = [ - 'sysbench', - test, - '--table-size={}'.format(table_size), - '--db-driver=mysql', - '--db-ps-mode=disable', - '--mysql-db={}'.format(test_db), - '--mysql-user=root', - '--mysql-host={}'.format(test_db_host), - '--rand-seed=1' - ] - - _run_stage(test, 'prepare', sysbench_args) - run_output = _run_stage(test, 'run', sysbench_args) - _run_stage(test, 'cleanup', sysbench_args) - - return run_output - - -def _run_stage(test: str, stage: str, args: List[str]): - exitcode, output = _execute(args + [stage], os.getcwd()) - if exitcode != 0: - logger.error(output) - raise SysbenchFailureException(test, stage, output) - - return output - - -def _execute(args: List[str], cwd: str): - proc = Popen(args=args, cwd=cwd, stdout=PIPE, stderr=PIPE) - out, err = proc.communicate() - exitcode = proc.returncode - return exitcode, out.decode('utf-8') - - -def parse_output(to_parse: str) -> dict: - result = {} - split = to_parse.split('\n') - processing_lines = False - for line in split: - clean_line = line.strip() - if not clean_line: - pass - elif clean_line.startswith('SQL statistics'): - processing_lines = True - elif clean_line.startswith('Threads fairness'): - return result - elif processing_lines and clean_line: - line_split = clean_line.split(':') - raw_name = line_split[0] - if len(line_split) > 1 and line_split[1] and raw_name in OUTPUT_MAPPING: - value_split = line_split[1].strip().split('(') - clean_value = value_split[0].rstrip('s').rstrip() - final_value = float(clean_value) if '.' in clean_value else int(clean_value) - result[OUTPUT_MAPPING[raw_name]] = final_value - - raise ValueError('Could not parse the following output:\n{}'.format(to_parse)) - - -def get_os_detail(): - return '{}-{}-{}'.format(os.name, platform.system(), platform.release()) - - -def write_output_file(run_id: str, - database_name: str, - committish: Optional[str], - username: str, - output: List[dict], - timestamp: datetime, - table_size: int): - if not os.path.exists('/output'): - os.mkdir('/output') - output_file = '/output/{}.csv'.format(run_id) - file_exists = os.path.exists(output_file) - logger.info('Writing output file to {}'.format(output_file)) - with open(output_file, 'a' if file_exists else 'w', newline='') as csvfile: - metadata = { - 'run_id': run_id, - 'database': database_name, - 'username': username, - 'committish': committish or 'not-applicable', - 'timestamp': timestamp, - 'system_info': get_os_detail(), - 'table_size': table_size - } - fieldnames = list(metadata.keys()) + ['test_name'] + list(OUTPUT_MAPPING.values()) - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - if not file_exists: - writer.writeheader() - for row in output: - to_write = {**row, **metadata} - writer.writerow(to_write) - - -if __name__ == '__main__': - main() diff --git a/benchmark/perf_tools/run_benchmarks.sh b/benchmark/perf_tools/run_benchmarks.sh deleted file mode 100755 index f32716f273..0000000000 --- a/benchmark/perf_tools/run_benchmarks.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -[ -n "$1" ] || (echo "Please supply a comma separated list of tests to be run"; exit 1) -tests=$1 -[ -n "$2" ] || (echo "Please supply an integer value for the table size used in the benchmarks"; exit 1) -table_size=$2 -[ -n "$3" ] || (echo "Please supply a username to associate with the benchmark"; exit 1) -username=$3 -committish_one=${4:-current} -committish_two=${5:-current} - -if [ "$committish_one" == "$committish_two" ]; then - echo "A single commit, $committish_one provided, proceeding with benchmark" - committish_list="$committish_one" -else - echo "Provided $committish_one and $committish_two, proceeding with building and benchmarking" - committish_list="$committish_one $committish_two" -fi - -script_dir=$(dirname "$0") -absolute_script_dir=$(realpath "$script_dir") -working_dir="$absolute_script_dir/dolt-builds/working" -run_id="$(openssl rand -hex 12)" -echo "Ensuring $working_dir exists and is empty" -rm -rf "$working_dir" -mkdir "$working_dir" - -function build_binary_at_committish() { - build_committish=$1 - echo "Building binary for committish $build_committish" - - if [ "$build_committish" != "current" ]; then - echo "$build_committish argument provided for 'commitish', cloning for fresh build" - cd "$working_dir" - git clone git@github.com:dolthub/dolt.git dolt-temp-checkout && git fetch --all - cd "dolt-temp-checkout/go" - git checkout "$build_committish" - else - echo "$build_committish passed for committish arg, building from current repo" - cd "$absolute_script_dir/../../go" - fi - - commit="$(git rev-parse HEAD)" - if [[ $(git status --porcelain) ]]; then - commit="$commit-dirty" - fi - - echo "Commit is set to $commit" - docker run --rm -v `pwd`:/src golang:1.14.2-buster /bin/bash -c ' - set -e - set -o pipefail - apt-get update && apt-get install -y zip - cd /src - - o="out" - mkdir -p "$o/bin" - cp Godeps/LICENSES "$o/" - echo Building "$o/dolt" - obin="dolt" - GOOS="$linux" GOARCH="$amd64" go build -o "$o/bin/$obin" "./cmd/dolt/" - ' - echo "Moving binary to temp out/bin/dolt to $working_dir/$commit-dolt" - mv "out/bin/dolt" "$working_dir/$commit-dolt" - echo "$working_dir/$commit-dolt" -} - -function run_sysbench() { - subdir=$1 - env_vars_string=$2 - cd "$subdir" - echo "Building Docker containers for sysbench and $subdir" - docker-compose build - echo "Running docker-compose from $(pwd), with the following environment variables:" - echo "$env_vars_string" - docker-compose run $env_vars_string sysbench --rm - docker-compose down --remove-orphans - cd .. -} - -function get_commit_signature() { - if [ "$1" == "current" ]; then - if [[ $(git status --porcelain) ]]; then - echo "$(git rev-parse HEAD)-dirty" - else - git rev-parse HEAD - fi - else - echo "$1" - fi -} - -echo "Building binaries and benchmarking for $committish_list" -for committish in $committish_list; do - bin_committish="$(build_binary_at_committish "$committish" | tail -1)" - cd "$absolute_script_dir" - echo "Built binary $bin_committish, copying to $working_dir/dolt for benchmarking" - cp "$bin_committish" "$working_dir/dolt" - run_sysbench dolt "-e DOLT_COMMITTISH=$(get_commit_signature $committish | tail -1) -e SYSBENCH_TESTS=$tests -e TEST_USERNAME=$username -e RUN_ID=$run_id -e TABLE_SIZE=$table_size" -done - -echo "Benchmarking MySQL for comparison" -run_sysbench mysql "-e SYSBENCH_TESTS=$tests -e TEST_USERNAME=$username -e RUN_ID=$run_id -e TABLE_SIZE=$table_size" -echo "All done!" diff --git a/benchmark/perf_tools/sysbench/Dockerfile b/benchmark/perf_tools/sysbench/Dockerfile deleted file mode 100644 index e3dd9ef23c..0000000000 --- a/benchmark/perf_tools/sysbench/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM python:3.8.6-slim-buster - -# Get sysbench installed -RUN apt update -RUN apt install -y curl -RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash -RUN apt -y install sysbench - -COPY ./benchmark.sh /benchmark.sh - -ENTRYPOINT ["/benchmark.sh"] \ No newline at end of file diff --git a/benchmark/perf_tools/sysbench/benchmark.sh b/benchmark/perf_tools/sysbench/benchmark.sh deleted file mode 100755 index ac723c6004..0000000000 --- a/benchmark/perf_tools/sysbench/benchmark.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail -if [ -n "$DOLT_COMMITTISH" ]; then - echo "Running sysbench tests $SYSBENCH_TESTS against Dolt for test user $TEST_USERNAME" - python /python/sysbench_wrapper.py \ - --db-host="$DB_HOST" \ - --committish="$DOLT_COMMITTISH" \ - --tests="$SYSBENCH_TESTS" \ - --username="$TEST_USERNAME" \ - --run-id="$RUN_ID" -else - sleep 30 - echo "Running sysbench tests $SYSBENCH_TESTS against MySQL for test user $TEST_USERNAME" - python /python/sysbench_wrapper.py \ - --db-host="$DB_HOST" \ - --tests="$SYSBENCH_TESTS" \ - --username="$TEST_USERNAME" \ - --run-id="$RUN_ID" -fi diff --git a/benchmark/perf_tools/sysbench_scripts/lua/bulk_insert.lua b/benchmark/perf_tools/sysbench_scripts/lua/bulk_insert.lua deleted file mode 100755 index e4227fb993..0000000000 --- a/benchmark/perf_tools/sysbench_scripts/lua/bulk_insert.lua +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env sysbench --- -------------------------------------------------------------------------- -- --- Bulk insert benchmark: do multi-row INSERTs concurrently in --threads --- threads with each thread inserting into its own table. The number of INSERTs --- executed by each thread is controlled by either --time or --events. --- -------------------------------------------------------------------------- -- -sysbench.hooks.report_intermediate = sysbench.report_json -sysbench.hooks.report_cumulative = sysbench.report_json -cursize=0 - -function thread_init() - drv = sysbench.sql.driver() - con = drv:connect() -end - -function prepare() - local i - - local drv = sysbench.sql.driver() - local con = drv:connect() - - for i = 1, sysbench.opt.threads do - print("Creating table 'sbtest" .. i .. "'...") - con:query(string.format([[ - CREATE TABLE IF NOT EXISTS sbtest%d ( - id INTEGER NOT NULL, - k INTEGER DEFAULT '0' NOT NULL, - PRIMARY KEY (id))]], i)) - end -end - -function event() - if (cursize == 0) then - con:bulk_insert_init("INSERT INTO sbtest" .. sysbench.tid+1 .. " VALUES") - end - - cursize = cursize + 1 - - con:bulk_insert_next("(" .. cursize .. "," .. cursize .. ")") -end - -function thread_done() - con:bulk_insert_done() - con:disconnect() -end - -function cleanup() - local i - - local drv = sysbench.sql.driver() - local con = drv:connect() - - for i = 1, sysbench.opt.threads do - print("Dropping table 'sbtest" .. i .. "'...") - con:query("DROP TABLE IF EXISTS sbtest" .. i ) - end -end diff --git a/benchmark/perf_tools/sysbench_scripts/lua/covering_index_scan.lua b/benchmark/perf_tools/sysbench_scripts/lua/covering_index_scan.lua deleted file mode 100644 index 6e5348623e..0000000000 --- a/benchmark/perf_tools/sysbench_scripts/lua/covering_index_scan.lua +++ /dev/null @@ -1,24 +0,0 @@ -require("dolt_common") - -dolt_prepare = prepare - -function prepare() - sysbench.opt.threads = 1 - dolt_prepare() -end - -function thread_init() - drv = sysbench.sql.driver() - con = drv:connect() - - stmt = con:prepare('SELECT count(id) FROM sbtest1 WHERE big_int_col > 0') -end - -function thread_done() - stmt:close() - con:disconnect() -end - -function event() - stmt:execute() -end diff --git a/benchmark/perf_tools/sysbench_scripts/lua/dolt_common.lua b/benchmark/perf_tools/sysbench_scripts/lua/dolt_common.lua deleted file mode 100644 index c8e0dc470e..0000000000 --- a/benchmark/perf_tools/sysbench_scripts/lua/dolt_common.lua +++ /dev/null @@ -1,141 +0,0 @@ - --- ----------------------------------------------------------------------------- --- Common code for dolt benchmarks. --- ----------------------------------------------------------------------------- - -function init() - assert(event ~= nil, "this script is meant to be included by other OLTP scripts and should not be called directly.") -end - -if sysbench.cmdline.command == nil then - error("Command is required. Supported commands: prepare, warmup, run, cleanup, help") -end - --- Command line options -sysbench.cmdline.options = { - table_size = {"Number of rows per table", 10000}, - create_table_options = {"Extra CREATE TABLE options", ""}, -} - --- Prepare the dataset. This command supports parallel execution, i.e. will --- benefit from executing with --threads > 1 as long as --tables > 1 -function cmd_prepare() - local drv = sysbench.sql.driver() - local con = drv:connect() - - print("Creating table 'sbtest1'") - - local create_query = string.format( [[ -CREATE TABLE sbtest1 ( - id INT NOT NULL, - tiny_int_col TINYINT NOT NULL, - unsigned_tiny_int_col TINYINT UNSIGNED NOT NULL, - small_int_col SMALLINT NOT NULL, - unsigned_small_int_col SMALLINT UNSIGNED NOT NULL, - medium_int_col MEDIUMINT NOT NULL, - unsigned_medium_int_col MEDIUMINT UNSIGNED NOT NULL, - int_col INT NOT NULL, - unsigned_int_col INT UNSIGNED NOT NULL, - big_int_col BIGINT NOT NULL, - unsigned_big_int_col BIGINT UNSIGNED NOT NULL, - decimal_col DECIMAL NOT NULL, - float_col FLOAT NOT NULL, - double_col DOUBLE NOT NULL, - bit_col BIT NOT NULL, - char_col CHAR NOT NULL, - var_char_col VARCHAR(64) NOT NULL, - tiny_text_col TINYTEXT NOT NULL, - text_col TEXT NOT NULL, - medium_text_col MEDIUMTEXT NOT NULL, - long_text_col LONGTEXT NOT NULL, - enum_col ENUM('val0', 'val1', 'val2') NOT NULL, - set_col SET('val0', 'val1', 'val2') NOT NULL, - date_col DATE NOT NULL, - time_col TIME NOT NULL, - datetime_col DATETIME NOT NULL, - timestamp_col TIMESTAMP NOT NULL, - year_col YEAR NOT NULL, - - PRIMARY KEY(id), - INDEX (big_int_col) -); ]] .. sysbench.opt.create_table_options) - - con:query(create_query) - - if (sysbench.opt.table_size > 0) then - print(string.format("Inserting %d records into 'sbtest1'", sysbench.opt.table_size)) - end - - local query = [[INSERT INTO sbtest1 -(id, tiny_int_col, unsigned_tiny_int_col, small_int_col, unsigned_small_int_col, medium_int_col, unsigned_medium_int_col, int_col, unsigned_int_col, big_int_col, unsigned_big_int_col, decimal_col, float_col, double_col, bit_col, char_col, var_char_col, tiny_text_col, text_col, medium_text_col, long_text_col, enum_col, set_col, date_col, time_col, datetime_col, timestamp_col, year_col) -VALUES -]] - - local str_vals = {"val0", "val1", "val2"} - math.randomseed(0) - - con:bulk_insert_init(query) - for i = 1, sysbench.opt.table_size do - local row_values = "(" .. i .. "," .. -- id - math.random(-128, 127) .. "," .. -- tiny_int_col - math.random(0, 255) .. "," .. -- unsigned_tiny_int_col - math.random(-32768, 32767) .. "," .. -- small_int_col - math.random(0, 65535) .. "," .. -- unsigned_small_int_col - math.random(-8388608, 8388607) .. "," .. -- medium_int_col - math.random(0, 16777215) .. "," .. -- unsigned_medium_int_col - math.random(-2147483648, 2147483647) .. "," .. -- int_col - math.random(0, 4294967295) .. "," .. -- unsigned_int_col - math.random(-4611686018427387904, 4611686018427387903) .. "," .. -- big_int_col - math.random(0, 9223372036854775807) .. "," .. -- unsigned_big_int_col - math.random() .. "," .. -- decimal_col - math.random() .. "," .. -- float_col - math.random() .. "," .. -- double_col - math.random(0, 1) .. "," .. -- bit_col - "'" .. string.char(math.random(0x30, 0x5A)) .. "'" .. "," .. -- char_col - "'" .. str_vals[math.random(1, 3)] .. "'" .. "," .. -- var_char_col - "'" .. str_vals[math.random(1, 3)] .. "'" .. "," .. -- tiny_text_col - "'" .. str_vals[math.random(1, 3)] .. "'" .. "," .. -- text_col - "'" .. str_vals[math.random(1, 3)] .. "'" .. "," .. -- medium_text_col - "'" .. str_vals[math.random(1, 3)] .. "'" .. "," .. -- long_text_col - "'" .. str_vals[math.random(1, 3)] .. "'" .. "," .. -- enum_col - "'" .. str_vals[math.random(1, 3)] .. "'" .. "," .. -- set_col - "'2020-0" .. math.random(1, 9) .. "-" .. math.random(10, 28) .. "'" .. "," .. -- date_col - "'0" .. math.random(1, 9) .. ":" .. math.random(10, 59) .. ":00'" .. "," .. -- time_col - "'2020-0" .. math.random(1, 9) .. "-" .. math.random(10, 28) .. " 0" .. math.random(1, 9) .. ":" .. math.random(10, 59) .. ":00'" .. "," .. -- datetime_col - "'2020-0" .. math.random(1, 9) .. "-" .. math.random(10, 28) .. " 0" .. math.random(1, 9) .. ":" .. math.random(10, 59) .. ":00'" .. "," .. -- timestamp_col - math.random(1901, 2155) .. ")" -- year_col - - con:bulk_insert_next(row_values) - end - con:bulk_insert_done() - -end - - --- Implement parallel prepare and warmup commands, define 'prewarm' as an alias --- for 'warmup' -sysbench.cmdline.commands = { - prepare = {cmd_prepare, sysbench.cmdline.PARALLEL_COMMAND}, -} - -local t = sysbench.sql.type - -function thread_init() - drv = sysbench.sql.driver() - con = drv:connect() -end - -function thread_done() - con:disconnect() -end - -function cleanup() - local drv = sysbench.sql.driver() - local con = drv:connect() - - print("Dropping table 'sbtest1'") - con:query("DROP TABLE IF EXISTS sbtest1") -end - - - diff --git a/benchmark/perf_tools/sysbench_scripts/lua/groupby_scan.lua b/benchmark/perf_tools/sysbench_scripts/lua/groupby_scan.lua deleted file mode 100644 index bb2e11c334..0000000000 --- a/benchmark/perf_tools/sysbench_scripts/lua/groupby_scan.lua +++ /dev/null @@ -1,24 +0,0 @@ -require("dolt_common") - -dolt_prepare = prepare - -function prepare() - sysbench.opt.threads = 1 - dolt_prepare() -end - -function thread_init() - drv = sysbench.sql.driver() - con = drv:connect() - - stmt = con:prepare('SELECT year_col, count(year_col), max(big_int_col), avg(small_int_col) FROM sbtest1 WHERE big_int_col > 0 GROUP BY set_col ORDER BY year_col') -end - -function thread_done() - stmt:close() - con:disconnect() -end - -function event() - stmt:execute() -end diff --git a/benchmark/perf_tools/sysbench_scripts/lua/index_scan.lua b/benchmark/perf_tools/sysbench_scripts/lua/index_scan.lua deleted file mode 100644 index 7c23c5f1d6..0000000000 --- a/benchmark/perf_tools/sysbench_scripts/lua/index_scan.lua +++ /dev/null @@ -1,24 +0,0 @@ -require("dolt_common") - -dolt_prepare = prepare - -function prepare() - sysbench.opt.threads = 1 - dolt_prepare() -end - -function thread_init() - drv = sysbench.sql.driver() - con = drv:connect() - - stmt = con:prepare('SELECT * FROM sbtest1 WHERE big_int_col > 0') -end - -function thread_done() - stmt:close() - con:disconnect() -end - -function event() - stmt:execute() -end diff --git a/benchmark/perf_tools/sysbench_scripts/lua/local_sysbench.sh b/benchmark/perf_tools/sysbench_scripts/lua/local_sysbench.sh deleted file mode 100644 index e58cb23fe2..0000000000 --- a/benchmark/perf_tools/sysbench_scripts/lua/local_sysbench.sh +++ /dev/null @@ -1 +0,0 @@ -sysbench --db-ps-mode=disable --rand-type=uniform --rand-seed=1 --percentile=50 --mysql-host=127.0.0.1 --mysql-user=root $@ diff --git a/benchmark/perf_tools/sysbench_scripts/lua/table_scan.lua b/benchmark/perf_tools/sysbench_scripts/lua/table_scan.lua deleted file mode 100644 index e65c72483b..0000000000 --- a/benchmark/perf_tools/sysbench_scripts/lua/table_scan.lua +++ /dev/null @@ -1,25 +0,0 @@ - -require("dolt_common") - -dolt_prepare = prepare - -function prepare() - sysbench.opt.threads = 1 - dolt_prepare() -end - -function thread_init() - drv = sysbench.sql.driver() - con = drv:connect() - - stmt = con:prepare('SELECT * FROM sbtest1 WHERE small_int_col > 0') -end - -function thread_done() - stmt:close() - con:disconnect() -end - -function event() - stmt:execute() -end diff --git a/benchmark/sql_regressions/DoltRegressionsJenkinsfile b/benchmark/sql_regressions/DoltRegressionsJenkinsfile deleted file mode 100644 index 012e3c5742..0000000000 --- a/benchmark/sql_regressions/DoltRegressionsJenkinsfile +++ /dev/null @@ -1,75 +0,0 @@ -pipeline { - agent { - kubernetes { - defaultContainer "jnlp" - yaml """ -kind: "Pod" -metadata: - annotations: - app: "jenkins-agent" - sidecar.istio.io/inject: "false" -spec: - containers: - - image: "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/jnlp:latest" - imagePullPolicy: "Always" - name: "jnlp" - resources: - limits: - cpu: "1.8" - memory: "3Gi" - requests: - cpu: "1.8" - memory: "3Gi" - securityContext: - privileged: false - tty: true - workingDir: "/home/jenkins/agent" - restartPolicy: "Never" - securityContext: - fsGroup: 1000 - runAsGroup: 1000 - runAsUser: 1000 - serviceAccount: "jenkins-agent-doltci" -""" - } - } - stages { - stage ("Update Liquidata/dolt-sql-performance:nightly") { - environment { - PATH = "${pwd()}/.ci_bin/node_modules/.bin:${env.PATH}" - DOLT_VERSION = "${env.GIT_COMMIT}" - TMPDIR = "${pwd()}/tempDir" - DOLT_ROOT_PATH="${pwd()}/tempRoot" - DOLT_CREDS = credentials("system-account-dolthub-creds") - } - steps { - sh "rm -rf $TMPDIR && mkdir $TMPDIR" - sh "rm -rf $DOLT_ROOT_PATH && mkdir $DOLT_ROOT_PATH" - dir ("sqllogictest") { - git url: "https://github.com/dolthub/sqllogictest.git" - } - dir ("benchmark/sql_regressions") { - script { - try { - sh "nice ./run_regressions.sh ./nightly.vars" - } catch(err) { - sh "if [ \"${err.getMessage()}\" = 'script returned exit code 155' ]; then echo 'Result data found in dolt-sql-performance, silently exiting...'; else echo \"${err.getMessage()}\" && exit 1; fi" - } - } - } - } - } - } - post { - always { - node ("liquidata-inc-ld-build") { - cleanWs() // cleanup - } - } - failure { - emailext body: "${currentBuild.currentResult}: Job ${env.JOB_NAME} build ${env.BUILD_NUMBER}\n More info at: ${env.BUILD_URL}", - to: "$SQL_WATCHERS", - subject: "Jenkins Build ${currentBuild.currentResult}: Job ${env.JOB_NAME}" - } - } -} diff --git a/benchmark/sql_regressions/DoltReleaseBenchmarkJenkinsfile b/benchmark/sql_regressions/DoltReleaseBenchmarkJenkinsfile deleted file mode 100644 index e9e03e3d97..0000000000 --- a/benchmark/sql_regressions/DoltReleaseBenchmarkJenkinsfile +++ /dev/null @@ -1,35 +0,0 @@ -pipeline { - agent { - kubernetes { - label "liquidata-inc-ld-build" - } - } - stages { - stage ("Update Liquidata/dolt-sql-performance:releases") { - environment { - PATH = "${pwd()}/.ci_bin/node_modules/.bin:${env.PATH}" - TMPDIR = "${pwd()}/tempDir" - DOLT_ROOT_PATH="${pwd()}/tempRoot" - DOLT_CREDS = credentials("system-account-dolthub-creds") - DOLT_RELEASE_URL = "https://github.com/dolthub/dolt/releases/download/v${DOLT_RELEASE}/dolt-linux-amd64.tar.gz" - } - steps { - sh "rm -rf $TMPDIR && mkdir $TMPDIR" - sh "rm -rf $DOLT_ROOT_PATH && mkdir $DOLT_ROOT_PATH" - dir ("sqllogictest") { - git url: "https://github.com/dolthub/sqllogictest.git" - } - dir ("benchmark/sql_regressions") { - sh "nice ./run_regressions.sh ./releases.vars" - } - } - } - } - post { - always { - node ("liquidata-inc-ld-build") { - cleanWs() // cleanup - } - } - } -} diff --git a/benchmark/sql_regressions/nightly.vars b/benchmark/sql_regressions/nightly.vars deleted file mode 100644 index 836ff4c392..0000000000 --- a/benchmark/sql_regressions/nightly.vars +++ /dev/null @@ -1,3 +0,0 @@ -DOLT_CONFIG_PATH=$DOLT_ROOT_PATH/.dolt -CREDSDIR=$DOLT_CONFIG_PATH/creds -JOB_TYPE=nightly diff --git a/benchmark/sql_regressions/releases.vars b/benchmark/sql_regressions/releases.vars deleted file mode 100644 index 12522d2aa4..0000000000 --- a/benchmark/sql_regressions/releases.vars +++ /dev/null @@ -1,3 +0,0 @@ -DOLT_CONFIG_PATH=$DOLT_ROOT_PATH/.dolt -CREDSDIR=$DOLT_CONFIG_PATH/creds -JOB_TYPE=release diff --git a/benchmark/sql_regressions/run_regressions.sh b/benchmark/sql_regressions/run_regressions.sh deleted file mode 100755 index 8d56b575b3..0000000000 --- a/benchmark/sql_regressions/run_regressions.sh +++ /dev/null @@ -1,302 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -function fail() { - 1>&2 echo "$@" - exit 1 -} - -logictest="../../go/libraries/doltcore/sqle/logictest" -logictest_main="$logictest"/main -old_path=$PATH - -if [[ "$#" -ne 1 ]]; then - fail Usage: ./run_regressions.sh ENV_VARIABLES_FILE -fi - -source "$1" -if [ -z "$DOLT_ROOT_PATH" ]; then fail Must supply DOLT_ROOT_PATH; fi -if [ -z "$DOLT_CONFIG_PATH" ]; then fail Must supply DOLT_CONFIG_PATH; fi -if [ -z "$CREDSDIR" ]; then fail Must supply CREDSDIR; fi -if [ -z "$DOLT_CREDS" ]; then fail Must supply DOLT_CREDS; fi -if [ -z "$JOB_TYPE" ]; then fail Must supply JOB_TYPE; fi -if [ -z "$TEST_N_TIMES" ]; then fail Must supply TEST_N_TIMES; fi -if [ -z "$FAIL_ON_EXISTING_VERSION" ]; then fail Must supply FAIL_ON_EXISTING_VERSION; fi -if [ -z "$USE_PROD_REPO" ]; then fail Must supply USE_PROD_REPO; fi -if [ -z "$DEV_REMOTE_HOST" ]; then fail Must supply DEV_REMOTE_HOST; fi - -if [[ -z "$DOLT_VERSION" && -z "$DOLT_RELEASE" ]]; then - fail Must supply DOLT_VERSION; -elif [[ -z "$DOLT_VERSION" && -n "$DOLT_RELEASE" ]]; then - DOLT_VERSION="$DOLT_RELEASE"; -fi - -[[ "$TEST_N_TIMES" =~ ^[0-9]+$ ]] || fail TEST_N_TIMES must be a number - -function setup() { - rm -rf "$CREDSDIR" - mkdir -p "$CREDSDIR" - dolt config --global --add metrics.disabled true - dolt creds import "$DOLT_CREDS" - dolt version - rm -rf temp - mkdir temp -} - -function run_once() { - test_num="$1" - - local results=temp/results"$test_num".log - local parsed=temp/parsed"$test_num".json - - rm -rf .dolt - dolt init - echo "Running tests and generating $results" - time go run . run ../../../../../../sqllogictest/test > "$results" - echo "Parsing $results and generating $parsed" - time go run . parse "$DOLT_VERSION" temp/results"$test_num".log > "$parsed" -} - -function run() { - seq 1 $TEST_N_TIMES | while read test_num; do - run_once "$test_num" - done - rm -rf .dolt -} - -function check_version_exists() { - if [[ "$JOB_TYPE" == "nightly" ]]; then - dolt checkout nightly - table_prefix="nightly" - elif [ "$JOB_TYPE" == "release" ]; then - dolt checkout releases - table_prefix="releases" - else fail Unknown JOB_TYPE specified; - fi - - previously_tested_version=$(dolt sql -r csv -q "select * from ${table_prefix}_dolt_results where version = '$DOLT_VERSION' limit 1;"| wc -l | tr -d '[:space:]') - - if [ "$previously_tested_version" != 1 ]; then - echo "Results for dolt version $DOLT_VERSION already exist in Liquidata/dolt-sql-performance, $previously_tested_version != 1" && \ - echo $result_query_output && \ - exit 155; - fi - - dolt checkout master -} - -function branch_from_base() { - local branch_name="$1" - dolt checkout master - - echo "Generating releases results csv using dolt sql -r csv" - time dolt sql -r csv -q "select * from releases_dolt_results" > "$branch_name"_releases_results.csv - echo "Generating releases mean results csv using dolt sql -r csv" - time dolt sql -r csv -q "select * from releases_dolt_mean_results" > "$branch_name"_releases_mean_results.csv - - dolt checkout regressions-tip-base - dolt checkout -b "$branch_name" - - echo "Importing releases csv using dolt table import -u" - time dolt table import -u releases_dolt_results "$branch_name"_releases_results.csv - dolt add releases_dolt_results - dolt commit -m "add release data from tip of regressions" - - echo "Importing releases mean csv using dolt table import -u" - time dolt table import -u releases_dolt_mean_results "$branch_name"_releases_mean_results.csv - dolt add releases_dolt_mean_results - dolt commit -m "add release mean data from tip of regressions" - - dolt checkout master -} - -function update_regressions_latest() { - exists=$(dolt branch --list regressions-tip-previous | sed '/^\s*$/d' | wc -l | tr -d '[:space:]') - - if [ "$exists" -eq 1 ]; then - dolt branch -d -f regressions-tip-previous; - fi - - exists=$(dolt branch --list regressions-tip-latest | sed '/^\s*$/d' | wc -l | tr -d '[:space:]') - - if [ "$exists" -eq 1 ]; then - dolt checkout regressions-tip-latest - - dolt branch -m regressions-tip-latest regressions-tip-previous - dolt push -f origin regressions-tip-previous - - branch_from_base "regressions-tip-latest" - - dolt checkout regressions-tip-latest - dolt push -f origin regressions-tip-latest; - else - branch_from_base "regressions-tip-previous" - branch_from_base "regressions-tip-latest" - - dolt checkout regressions-tip-previous - dolt push -f origin regressions-tip-previous - - dolt checkout regressions-tip-latest - dolt push regressions-tip-latest; - fi -} - -function import_one_nightly() { - test_num="$1" - echo "Importing nightly dolt results from json using dolt table import -u" - time dolt table import -u nightly_dolt_results ../"$logictest_main"/temp/parsed"$test_num".json - dolt add nightly_dolt_results - dolt commit -m "update dolt sql performance results ($DOLT_VERSION) ($test_num)" -} - -function import_nightly() { - dolt checkout nightly - seq 1 $TEST_N_TIMES | while read test_num; do - import_one_nightly "$test_num" - done - echo "Generating nightly mean results csv from dolt_history table query and dolt sql -r csv" - time dolt sql -r csv -q "\ -select version, test_file, line_num, avg(duration) as mean_duration, result from dolt_history_nightly_dolt_results where version=\"${DOLT_VERSION}\" group by test_file, line_num;\ -" > nightly_mean.csv - - echo "Importing nightly dolt mean results from csv using dolt table import -u" - time dolt table import -u nightly_dolt_mean_results nightly_mean.csv - dolt add nightly_dolt_mean_results - dolt commit -m "update dolt sql performance mean results ($DOLT_VERSION)" - echo "Pushing nightly to dolthub" - time dolt push origin nightly - - dolt checkout master - dolt merge nightly - dolt add . - dolt commit -m "merge nightly" - echo "Pushing master to dolthub" - time dolt push origin master - - dolt checkout releases - echo "Generating releases mean results csv using dolt sql -r csv" - time dolt sql -r csv -q "\ -select * from releases_dolt_mean_results;\ -" > releases_mean.csv - rm -f regressions_db - touch regressions_db - - # sqlite3 requires the headers to be trimmed on import - # otherwise it will add them as data - sed 1d nightly_mean.csv > headerless_nightly_mean.csv - sed 1d releases_mean.csv > headerless_releases_mean.csv - - sqlite3 regressions_db < ../"$logictest"/regressions.sql - cp ../"$logictest"/import.sql . - sqlite3 regressions_db < import.sql - echo "Checking for test regressions using sqlite3..." - - time duration_query_output=`sqlite3 regressions_db 'select * from releases_nightly_duration_change'` - time result_query_output=`sqlite3 regressions_db 'select * from releases_nightly_result_change'` - - duration_regressions=`echo $duration_query_output | sed '/^\s*$/d' | wc -l | tr -d '[:space:]'` - result_regressions=`echo $result_query_output | sed '/^\s*$/d' | wc -l | tr -d '[:space:]'` - - if [ "$duration_regressions" != 0 ]; then echo "Duration regression found, $duration_regressions != 0" && echo $duration_query_output; else echo "No duration regressions found"; fi - if [ "$result_regressions" != 0 ]; then echo "Result regression found, $result_regressions != 0" && echo $result_query_output; else echo "No result regressions found"; fi - if [ "$duration_regressions" != 0 ] || [ "$result_regressions" != 0 ]; then exit 1; fi -} - -function with_dolt_release() { - ( - cd ../../ - if ! [ -x ./.ci_bin/dolt_release/dolt ]; then - if ! [ -d ./.ci_bin/dolt_release ]; then - mkdir -p ./.ci_bin/dolt_release - fi - curl -A ld-jenkins-dolt-installer -fL "$DOLT_RELEASE_URL" > dolt.tar.gz - tar zxf dolt.tar.gz - install dolt-linux-amd64/bin/dolt ./.ci_bin/dolt_release/ - fi - ) - echo "Finished installing dolt from release:" - export PATH=`pwd`"/../../.ci_bin/dolt_release":$old_path - dolt version -} - -function with_dolt_checkout() { - ( - cd ../../go - if ! [ -x ../.ci_bin/dolt ]; then - if ! [ -d ../.ci_bin ]; then - mkdir -p ../.ci_bin - fi - go get -mod=readonly ./... - go build -mod=readonly -o ../.ci_bin/dolt ./cmd/dolt/. - fi - ) - echo "Finished installing dolt from checkout:" - export PATH=`pwd`"/../../.ci_bin":$old_path - dolt version -} - -function import_one_releases() { - test_num="$1" - echo "Importing releases results from json using dolt table import -u" - time dolt table import -u releases_dolt_results ../"$logictest_main"/temp/parsed"$test_num".json - dolt add releases_dolt_results - dolt commit -m "update dolt sql performance results ($DOLT_VERSION) ($test_num)" -} - -function import_releases() { - dolt checkout releases - seq 1 $TEST_N_TIMES | while read test_num; do - import_one_releases "$test_num" - done - echo "Generating releases mean results csv from dolt_history table query and dolt sql -r csv" - time dolt sql -r csv -q "\ -select version, test_file, line_num, avg(duration) as mean_duration, result from dolt_history_releases_dolt_results where version=\"${DOLT_VERSION}\" group by test_file, line_num;\ -" > releases_mean.csv - echo "Importing releases mean results using dolt table import -u" - time dolt table import -u releases_dolt_mean_results releases_mean.csv - dolt add releases_dolt_mean_results - dolt commit -m "update dolt sql performance mean results ($DOLT_VERSION)" - echo "Pushing releases to dolthub" - time dolt push origin releases - - dolt checkout master - dolt merge releases - dolt add . - dolt commit -m "merge releases" - echo "Pushing master to dolthub" - time dolt push origin master - - update_regressions_latest -} - -rm -rf dolt-sql-performance - -( - with_dolt_checkout - if [[ "$USE_PROD_REPO" == true ]]; then - time dolt clone Liquidata/dolt-sql-performance; - else - time dolt clone "$DEV_REMOTE_HOST"/Liquidata/dolt-sql-performance - fi - -) - -(with_dolt_checkout; cd "$logictest_main"; setup) - -if [[ "$FAIL_ON_EXISTING_VERSION" == true ]]; then - (with_dolt_checkout; cd dolt-sql-performance; check_version_exists) -fi - -if [[ "$JOB_TYPE" == "release" ]]; then - (with_dolt_release; cd "$logictest_main"; run) -else - (with_dolt_checkout; cd "$logictest_main"; run) -fi - -if [[ "$JOB_TYPE" == "nightly" ]]; then - (with_dolt_checkout; cd dolt-sql-performance; import_nightly); -elif [ "$JOB_TYPE" == "release" ]; then - (with_dolt_checkout; cd dolt-sql-performance; import_releases) -else fail Unknown JOB_TYPE specified; -fi diff --git a/benchmark/sqllogictest_tester/SQLLogicTestTesterJenkinsfile b/benchmark/sqllogictest_tester/SQLLogicTestTesterJenkinsfile deleted file mode 100644 index 36a6cece1b..0000000000 --- a/benchmark/sqllogictest_tester/SQLLogicTestTesterJenkinsfile +++ /dev/null @@ -1,46 +0,0 @@ -pipeline { - agent { - kubernetes { - label "liquidata-inc-ld-build" - } - } - stages { - stage('Run sqllogictests against Dolt commits') { - environment { - PATH = "${pwd()}/.ci_bin/node_modules/.bin:${env.PATH}" - TMPDIR = "${pwd()}/tempDir" - DOLT_ROOT_PATH="${pwd()}/tempRoot" - DOLT_CREDS = credentials("system-account-dolthub-creds") - TMP_TESTING_DIR = "${pwd()}/tempTesting" - TMP_CSV_DIR = "${pwd()}/tempCSV" - } - steps { - sh "rm -rf tempDolt" - sh "rm -rf $TMPDIR && mkdir $TMPDIR" - sh "rm -rf $TMP_CSV_DIR && mkdir $TMP_CSV_DIR" - sh "rm -rf $DOLT_ROOT_PATH && mkdir $DOLT_ROOT_PATH" - dir ("sqllogictest") { - git url: "https://github.com/dolthub/sqllogictest.git" - } - dir ("tempDolt") { - git url: "https://github.com/dolthub/dolt.git" - } - dir ("benchmark/sqllogictest_tester") { - sh "nice ./run_tester.sh ./tester.vars" - } - } - } - } - post { - always { - node ("liquidata-inc-ld-build") { - cleanWs() // cleanup - } - } - failure { - emailext body: "${currentBuild.currentResult}: Job ${env.JOB_NAME} build ${env.BUILD_NUMBER}\n More info at: ${env.BUILD_URL}", - to: "$TEST_WATCHERS", - subject: "Jenkins Build ${currentBuild.currentResult}: Job ${env.JOB_NAME}" - } - } -} diff --git a/benchmark/sqllogictest_tester/run_tester.sh b/benchmark/sqllogictest_tester/run_tester.sh deleted file mode 100755 index 94af626b88..0000000000 --- a/benchmark/sqllogictest_tester/run_tester.sh +++ /dev/null @@ -1,280 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -function fail() { - 1>&2 echo "$@" - exit 1 -} - -base_dir=$(cd ../../ && pwd) -dsp_dir=$(pwd) -log_dir="$dsp_dir"/tempLogs -sqllogictest_checkout="$base_dir/sqllogictest" -logictest="$base_dir/tempDolt/go/libraries/doltcore/sqle/logictest" -logictest_main="$logictest"/main -schema_dir="$base_dir/go/libraries/doltcore/sqle/logictest" -old_path=$PATH - -if [[ "$#" -ne 1 ]]; then - fail Usage: ./run_regressions.sh ENV_VARIABLES_FILE -fi - -source "$1" -if [ -z "$DOLT_ROOT_PATH" ]; then fail Must supply DOLT_ROOT_PATH; fi -if [ -z "$DOLT_CONFIG_PATH" ]; then fail Must supply DOLT_CONFIG_PATH; fi -if [ -z "$CREDSDIR" ]; then fail Must supply CREDSDIR; fi -if [ -z "$DOLT_CREDS" ]; then fail Must supply DOLT_CREDS; fi -if [ -z "$TMP_TESTING_DIR" ]; then fail Must supply TMP_TESTING_DIR; fi -if [ -z "$TMP_CSV_DIR" ]; then fail Must supply TMP_CSV_DIR; fi -if [ -z "$TEST_FILE_DIR_LIST" ]; then fail Must supply TEST_FILE_DIR_LIST; fi -if [ -z "$COMMITS_TO_TEST" ]; then fail Must supply COMMITS_TO_TEST; fi -if [ -z "$DOLT_BRANCH" ]; then fail Must supply DOLT_BRANCH; fi -if [ -z "$TEST_N_TIMES" ]; then fail Must supply TEST_N_TIMES; fi - -[[ "$TEST_N_TIMES" =~ ^[0-9]+$ ]] || fail TEST_N_TIMES must be a number - -function setup() { - rm -rf "$CREDSDIR" - mkdir -p "$CREDSDIR" - dolt config --global --add metrics.disabled true - dolt creds import "$DOLT_CREDS" - dolt version -} - -function setup_testing_dir() { - rm -rf "$TMP_TESTING_DIR" - mkdir -p "$TMP_TESTING_DIR" - - IFS=', ' read -r -a test_list <<< "$TEST_FILE_DIR_LIST" - for fd in "${test_list[@]}" - do - dir=$(dirname "$fd") - mkdir -p "$TMP_TESTING_DIR"/"$dir" - cp -r "$sqllogictest_checkout"/test/"$fd" "$TMP_TESTING_DIR"/"$fd" - done - - # Keep only the tests relevant to this run - # tests must be kept in the `test` dir of sqllogictest - # so all test_file paths match - rm -rf "$sqllogictest_checkout"/test/ - mkdir "$sqllogictest_checkout"/test/ - cp -r "$TMP_TESTING_DIR"/* "$sqllogictest_checkout"/test/ -} - -function checkout_branch_if_exists() { - local cmd="$1" - local branch_name="$2" - local hash="$3" - - if [ `"$cmd" branch --list "$branch_name" | wc -l` -eq 1 ]; then - "$cmd" checkout "$branch_name" - else - if [ -z "$hash" ]; then - "$cmd" checkout -b "$branch_name"; - else - "$cmd" checkout -b "$branch_name" "$hash"; - fi - fi -} - -function with_dolt_commit() { - local commit_hash="$1" - ( - cd "$base_dir"/tempDolt/go - git checkout "$commit_hash" - - checkout_branch_if_exists "git" "temp-$commit_hash" "$commit_hash" - - git log -n 1 - - if ! [ -x "$base_dir"/.ci_bin/"$commit_hash"/dolt ]; then - if ! [ -d "$base_dir"/.ci_bin ]; then - mkdir -p "$base_dir"/.ci_bin/"$commit_hash" - fi - - go build -mod=readonly -o "$base_dir"/.ci_bin/"$commit_hash"/dolt ./cmd/dolt/. - fi - ) - - echo "Finished installing dolt from $commit_hash:" - export PATH="$base_dir/.ci_bin/$commit_hash":$old_path - dolt version -} - -function with_dolt_checkout() { - ( - cd "$base_dir"/go - if ! [ -x "$base_dir"/.ci_bin/checkout/dolt ]; then - if ! [ -d "$base_dir"/.ci_bin ]; then - mkdir -p "$base_dir"/.ci_bin/checkout - fi - go get -mod=readonly ./... - go build -mod=readonly -o "$base_dir"/.ci_bin/checkout/dolt ./cmd/dolt/. - fi - ) - echo "Finished installing dolt from checkout:" - export PATH="$base_dir/.ci_bin/checkout":$old_path - dolt version -} - -function import_once() { - local commit_hash="$1" - local test_num="$2" - local parsed="$log_dir/parsed-$commit_hash-$test_num".json - - dolt checkout "$DOLT_BRANCH" - - checkout_branch_if_exists "dolt" "temp-$commit_hash" "" - - dolt table import -u nightly_dolt_results "$parsed" - - dolt sql -r csv -q "\ - select * from nightly_dolt_results;"\ - > "$TMP_CSV_DIR"/"$commit_hash"_results.csv - - dolt add nightly_dolt_results - dolt commit -m "add results for dolt at git commit $commit_hash ($test_num)" - dolt checkout master -} - -function create_mean_csv_once() { - local commit_hash="$1" - - checkout_branch_if_exists "dolt" "temp-$commit_hash" "" - - dolt sql -r csv -q "\ - select version, test_file, line_num, avg(duration) as mean_duration, result from dolt_history_nightly_dolt_results where version=\"${commit_hash}\" group by test_file, line_num;"\ - > "$TMP_CSV_DIR"/"$commit_hash"_mean_results.csv - - dolt checkout master -} - -function import_and_query_once() { - rm -f query_db - touch query_db - sqlite3 query_db < "$schema_dir"/regressions.sql - - local commit_hash="$1" - local release_csv="$TMP_CSV_DIR/release_results.csv" - local release_mean_csv="$TMP_CSV_DIR/release_mean_results.csv" - local commiter_csv="$TMP_CSV_DIR/${commit_hash}_results.csv" - local commiter_mean_csv="$TMP_CSV_DIR/${commit_hash}_mean_results.csv" - - sqlite3 query_db < "$results" - - echo "Parsing $results and generating $parsed" - go run . parse "$commit_hash" "$results" > "$parsed" - ) - - (with_dolt_checkout; cd "$dsp_dir"/dolt-sql-performance; import_once "$c" "$test_num") -} - -function run() { - rm -rf "$log_dir" - mkdir "$log_dir" - - IFS=', ' read -r -a commit_list <<< "$COMMITS_TO_TEST" - for c in "${commit_list[@]}" - do - seq 1 $TEST_N_TIMES | while read test_num; do - run_once "$c" "$test_num" - done - - (with_dolt_checkout; cd "$dsp_dir"/dolt-sql-performance; create_mean_csv_once "$c") - - (import_and_query_once "$c") - done -} - -append() { - echo "$1""${1:+, }""'$2'" -} - -function create_releases_csv() { - test_files=$(find "$TMP_TESTING_DIR" | sed -n "s|^$TMP_TESTING_DIR/||p") - - SAVEIFS=$IFS - IFS=$'\n' - - # do not wrap env var in quotes, so it gets split into array - file_arr=($test_files) - - IFS=$SAVEIFS - - file_list= - for (( i=0; i<${#file_arr[@]}; i++ )) - do - if [ "${file_arr[$i]: -5}" == ".test" ]; then - file_list=`append "$file_list" "${file_arr[$i]}"` - fi - done - - echo "Using file list: $file_list" - - dolt checkout "$DOLT_BRANCH" - dolt sql -r csv -q "\ - select * from releases_dolt_results where test_file in ($file_list);"\ - > "$TMP_CSV_DIR"/release_results.csv - - dolt sql -r csv -q "\ - select * from releases_dolt_mean_results where test_file in ($file_list);"\ - > "$TMP_CSV_DIR"/release_mean_results.csv - - dolt checkout master -} - -function update_fetch_specs() { - local remote="$1" - local branch="$2" - repo_state=$(cat .dolt/repo_state.json) - jq ".remotes.$remote.fetch_specs = [\"refs/heads/$branch:refs/remotes/origin/$branch\"]" <<< "$repo_state" > .dolt/repo_state.json -} - -function fetch_repo() { - dolt init - dolt remote add origin "https://doltremoteapi.dolthub.com/Liquidata/dolt-sql-performance" - update_fetch_specs "origin" "$DOLT_BRANCH" - dolt fetch origin -} - -(with_dolt_checkout; setup) - -rm -rf dolt-sql-performance && mkdir dolt-sql-performance -(with_dolt_checkout; cd dolt-sql-performance; fetch_repo) - -(with_dolt_checkout; setup_testing_dir) - -(with_dolt_checkout; cd dolt-sql-performance; create_releases_csv) - -(cd "$logictest_main"; run) diff --git a/benchmark/sqllogictest_tester/tester.vars b/benchmark/sqllogictest_tester/tester.vars deleted file mode 100644 index e048bd0991..0000000000 --- a/benchmark/sqllogictest_tester/tester.vars +++ /dev/null @@ -1,2 +0,0 @@ -DOLT_CONFIG_PATH=$DOLT_ROOT_PATH/.dolt -CREDSDIR=$DOLT_CONFIG_PATH/creds diff --git a/go/performance/continuous_integration/SysbenchDockerfile b/go/performance/continuous_integration/SysbenchDockerfile new file mode 100644 index 0000000000..aeb64f4172 --- /dev/null +++ b/go/performance/continuous_integration/SysbenchDockerfile @@ -0,0 +1,45 @@ +FROM golang:1.17.1-buster + +ENV DEBIAN_FRONTEND=noninteractive + +# Get sysbench installed +RUN apt update +RUN apt install -y curl +RUN curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | bash +RUN apt -y install sysbench + +# Install sqlite3 from source +RUN \ + apt-get install -y \ + build-essential \ + tcl \ + lsb-release \ + && wget \ + -O sqlite.tar.gz \ + https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release \ + && tar xvfz sqlite.tar.gz \ + # Configure and make SQLite3 binary + && ./sqlite/configure --prefix=/usr \ + && make \ + && make install \ + # Smoke test + && sqlite3 --version + +WORKDIR / +COPY ./go /dolt/go +COPY ./config.json /config.json +COPY ./sysbench-runner-tests-entrypoint.sh /entrypoint.sh +RUN git clone https://github.com/dolthub/sysbench-lua-scripts.git + +WORKDIR /mysql +RUN curl -L -O https://dev.mysql.com/get/mysql-apt-config_0.8.22-1_all.deb +RUN dpkg -i mysql-apt-config_0.8.22-1_all.deb +RUN apt-get update && apt-get install -y mysql-server +RUN mysql --version + +# Install dolt +WORKDIR /dolt/go/cmd/dolt +RUN go build -o /usr/local/bin/dolt . + +WORKDIR /dolt/go/performance/utils/sysbench_runner/cmd +ENTRYPOINT ["/entrypoint.sh"] diff --git a/go/performance/continuous_integration/SysbenchDockerfile.dockerignore b/go/performance/continuous_integration/SysbenchDockerfile.dockerignore new file mode 100644 index 0000000000..fea5eec145 --- /dev/null +++ b/go/performance/continuous_integration/SysbenchDockerfile.dockerignore @@ -0,0 +1,7 @@ +# Ignore everything... +** + +# Except for the following +!go +!config.json +!sysbench-runner-tests-entrypoint.sh diff --git a/go/performance/continuous_integration/config.json b/go/performance/continuous_integration/config.json new file mode 100644 index 0000000000..0c8001a7ed --- /dev/null +++ b/go/performance/continuous_integration/config.json @@ -0,0 +1,22 @@ +{ + "Runs": 1, + "Servers": [ + { + "Host": "0.0.0.0", + "Port": 4433, + "Server": "dolt", + "Version": "HEAD", + "ResultsFormat": "csv", + "ServerExec": "/usr/local/bin/dolt" + }, + { + "Server": "mysql", + "Version": "8.0.22", + "ResultsFormat": "csv", + "ServerExec": "/usr/sbin/mysqld", + "ConnectionProtocol": "unix" + } + ], + "ScriptDir":"/sysbench-lua-scripts", + "TestOptions": ["--rand-seed=1", "--table-size=30"] +} diff --git a/go/performance/continuous_integration/sysbench-runner-tests-entrypoint.sh b/go/performance/continuous_integration/sysbench-runner-tests-entrypoint.sh new file mode 100755 index 0000000000..1302f34adf --- /dev/null +++ b/go/performance/continuous_integration/sysbench-runner-tests-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh +dolt version +# Github Actions ignores the WORKDIR? +cd ./go/performance/utils/sysbench_runner/cmd +DEBUG=1 go run . --config=/config.json diff --git a/go/performance/benchmarks/benchmarks.go b/go/performance/import_benchmarker/benchmarks.go similarity index 100% rename from go/performance/benchmarks/benchmarks.go rename to go/performance/import_benchmarker/benchmarks.go diff --git a/go/performance/benchmarks/dataset.go b/go/performance/import_benchmarker/dataset.go similarity index 100% rename from go/performance/benchmarks/dataset.go rename to go/performance/import_benchmarker/dataset.go diff --git a/go/performance/benchmarks/helpers.go b/go/performance/import_benchmarker/helpers.go similarity index 100% rename from go/performance/benchmarks/helpers.go rename to go/performance/import_benchmarker/helpers.go diff --git a/go/performance/benchmarks/main.go b/go/performance/import_benchmarker/main.go similarity index 100% rename from go/performance/benchmarks/main.go rename to go/performance/import_benchmarker/main.go diff --git a/go/performance/benchmarks/results.go b/go/performance/import_benchmarker/results.go similarity index 100% rename from go/performance/benchmarks/results.go rename to go/performance/import_benchmarker/results.go diff --git a/go/performance/benchmarks/seed_schema.go b/go/performance/import_benchmarker/seed_schema.go similarity index 100% rename from go/performance/benchmarks/seed_schema.go rename to go/performance/import_benchmarker/seed_schema.go diff --git a/benchmark/import_tester/csv_gen.py b/go/performance/import_tester/csv_gen.py similarity index 100% rename from benchmark/import_tester/csv_gen.py rename to go/performance/import_tester/csv_gen.py diff --git a/benchmark/import_tester/run_importer.sh b/go/performance/import_tester/run_importer.sh similarity index 100% rename from benchmark/import_tester/run_importer.sh rename to go/performance/import_tester/run_importer.sh diff --git a/benchmark/perf_tools/sysbench_scripts/local_benchmark.sh b/go/performance/scripts/local_benchmark.sh similarity index 87% rename from benchmark/perf_tools/sysbench_scripts/local_benchmark.sh rename to go/performance/scripts/local_benchmark.sh index c8aff84a67..50090eb30b 100755 --- a/benchmark/perf_tools/sysbench_scripts/local_benchmark.sh +++ b/go/performance/scripts/local_benchmark.sh @@ -6,7 +6,12 @@ set -o pipefail SYSBENCH_TEST="covering_index_scan.lua" TMP_DIR=`mktemp -d` -cp ./lua/* "$TMP_DIR" + +if [ ! -d "./sysbench-lua-scripts" ]; then + git clone https://github.com/dolthub/sysbench-lua-scripts.git +fi + +cp ./sysbench-lua-scripts/*.lua "$TMP_DIR" cd "$TMP_DIR" echo " " diff --git a/go/performance/utils/dolt_builder/README.md b/go/performance/utils/dolt_builder/README.md new file mode 100644 index 0000000000..25de7b10fc --- /dev/null +++ b/go/performance/utils/dolt_builder/README.md @@ -0,0 +1,23 @@ +Dolt builder is a tool for more easily installing dolt binaries. + +It takes Dolt commit shas or tags as arguments +and builds corresponding binaries to a path specified +by `$DOLT_BIN` + +If `$DOLT_BIN` is not set `./doltBin` will be used + +(Optional) set `$DEBUG=true` to run in debug mode + +Example usage: + +```bash +$ dolt-builder dccba46 4bad226 +$ dolt version 0.1 +$ dolt version 0.2 +``` + +```bash +$ dolt-builder v0.19.0 v0.22.6 +$ dolt version 0.19.0 +$ dolt version v0.22.6 +``` diff --git a/go/performance/utils/dolt_builder/cmd/main.go b/go/performance/utils/dolt_builder/cmd/main.go new file mode 100644 index 0000000000..d2404693b9 --- /dev/null +++ b/go/performance/utils/dolt_builder/cmd/main.go @@ -0,0 +1,45 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "log" + "os" + + builder "github.com/dolthub/dolt/go/performance/utils/dolt_builder" +) + +func main() { + commitList := os.Args[1:] + if len(commitList) < 1 { + helpStr := "dolt-builder takes Dolt commit shas or tags as arguments\n" + + "and builds corresponding binaries to a path specified\n" + + "by DOLT_BIN\n" + + "If DOLT_BIN is not set, ./doltBin will be used\n" + + "usage: dolt-builder dccba46 4bad226 ...\n" + + "usage: dolt-builder v0.19.0 v0.22.6 ...\n" + + "set DEBUG=1 to run in debug mode\n" + fmt.Print(helpStr) + os.Exit(2) + } + + err := builder.Run(commitList) + if err != nil { + log.Fatal(err) + } + + os.Exit(0) +} diff --git a/go/performance/utils/dolt_builder/constants.go b/go/performance/utils/dolt_builder/constants.go new file mode 100644 index 0000000000..b9c6423810 --- /dev/null +++ b/go/performance/utils/dolt_builder/constants.go @@ -0,0 +1,19 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dolt_builder + +const ( + GithubDolt = "https://github.com/dolthub/dolt.git" +) diff --git a/go/performance/utils/dolt_builder/debug.go b/go/performance/utils/dolt_builder/debug.go new file mode 100644 index 0000000000..242031ab57 --- /dev/null +++ b/go/performance/utils/dolt_builder/debug.go @@ -0,0 +1,38 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dolt_builder + +import ( + "context" + "os" + "os/exec" +) + +var Debug bool + +func init() { + if os.Getenv("DEBUG") != "" { + Debug = true + } +} + +func ExecCommand(ctx context.Context, name string, arg ...string) *exec.Cmd { + e := exec.CommandContext(ctx, name, arg...) + if Debug { + e.Stdout = os.Stdout + e.Stderr = os.Stderr + } + return e +} diff --git a/go/performance/utils/dolt_builder/git.go b/go/performance/utils/dolt_builder/git.go new file mode 100644 index 0000000000..d460e97d5d --- /dev/null +++ b/go/performance/utils/dolt_builder/git.go @@ -0,0 +1,83 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dolt_builder + +import ( + "context" + "fmt" + "os" + "strings" +) + +// GitVersion runs git version +func GitVersion(ctx context.Context) error { + checkGit := ExecCommand(ctx, "git", "version") + err := checkGit.Run() + if err != nil { + helpStr := "dolt-builder requires git.\n" + + "Make sure git is installed and on your PATH.\n" + + "git version: %v\n" + return fmt.Errorf(helpStr, err) + } + return nil +} + +// GitClone clones the dolt repo into `${dir}/dolt.git` +func GitCloneBare(ctx context.Context, dir string) error { + clone := ExecCommand(ctx, "git", "clone", "--bare", GithubDolt) + clone.Dir = dir + return clone.Run() +} + +func CommitArg(c string) string { + if IsCommit(c) { + return c + } + if strings.HasPrefix(c, "v") { + return "tags/" + c + } + return "tags/v" + c +} + +// GitCheckoutTree checks out `commit` from the Git repo at +// `repoDir` into `toDir`. It does it without copying the entire +// git repository. First we run `git read-tree` with a GIT_INDEX_FILE set to +// `$toDir/.buildindex`, which gets an index for the commit fully populated +// into the file. Then we run `git checkout-index -a` referencing the same +// INDEX_FILE, which populates the current working directory (`toDir`) with the +// contents of the index file. +func GitCheckoutTree(ctx context.Context, repoDir string, toDir string, commit string) error { + env := os.Environ() + env = append(env, "GIT_DIR="+repoDir) + env = append(env, "GIT_INDEX_FILE=.buildindex") + env = append(env, "GIT_WORK_TREE=.") + + read := ExecCommand(ctx, "git", "read-tree", CommitArg(commit)) + read.Dir = toDir + read.Env = env + if err := read.Run(); err != nil { + return err + } + + checkout := ExecCommand(ctx, "git", "checkout-index", "-a") + checkout.Dir = toDir + checkout.Env = env + return checkout.Run() +} + +// IsCommit returns true if a commit is not a tag +func IsCommit(commit string) bool { + return strings.IndexByte(commit, '.') == -1 +} diff --git a/go/performance/utils/dolt_builder/run.go b/go/performance/utils/dolt_builder/run.go new file mode 100644 index 0000000000..a8f330ab7e --- /dev/null +++ b/go/performance/utils/dolt_builder/run.go @@ -0,0 +1,179 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dolt_builder + +import ( + "context" + "fmt" + "os" + "os/signal" + "path/filepath" + "runtime" + "sync" + "syscall" + + "golang.org/x/sync/errgroup" +) + +func Run(commitList []string) error { + parentCtx := context.Background() + + doltBin, err := getDoltBin() + if err != nil { + return err + } + + // check for git on path + err = GitVersion(parentCtx) + if err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + + // make temp dir for cloning/copying dolt source + tempDir := filepath.Join(cwd, "clones-copies") + err = os.MkdirAll(tempDir, os.ModePerm) + if err != nil { + return err + } + + // clone dolt source + err = GitCloneBare(parentCtx, tempDir) + if err != nil { + return err + } + + repoDir := filepath.Join(tempDir, "dolt.git") + + withKeyCtx, cancel := context.WithCancel(parentCtx) + g, ctx := errgroup.WithContext(withKeyCtx) + + // handle user interrupt + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + var wg sync.WaitGroup + wg.Add(1) + go func() { + <-quit + defer wg.Done() + signal.Stop(quit) + cancel() + }() + + for _, commit := range commitList { + commit := commit // https://golang.org/doc/faq#closures_and_goroutines + g.Go(func() error { + return buildBinaries(ctx, tempDir, repoDir, doltBin, commit) + }) + } + + builderr := g.Wait() + close(quit) + wg.Wait() + + // remove clones-copies after all go routines complete + // will exit successfully if removal fails + if err := os.RemoveAll(tempDir); err != nil { + fmt.Printf("WARN: %s was not removed\n", tempDir) + fmt.Printf("WARN: error: %v\n", err) + } + + if builderr != nil { + return builderr + } + + return nil +} + +// getDoltBin creates and returns the absolute path for DOLT_BIN +// if it was found, otherwise uses the current working directory +// as the parent directory for a `doltBin` directory +func getDoltBin() (string, error) { + var doltBin string + dir := os.Getenv("DOLT_BIN") + if dir == "" { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + doltBin = filepath.Join(cwd, "doltBin") + } else { + abs, err := filepath.Abs(dir) + if err != nil { + return "", err + } + doltBin = abs + } + err := os.MkdirAll(doltBin, os.ModePerm) + if err != nil { + return "", err + } + return doltBin, nil +} + +// buildBinaries builds a dolt binary at the given commit and stores it in the doltBin +func buildBinaries(ctx context.Context, tempDir, repoDir, doltBinDir, commit string) error { + checkoutDir := filepath.Join(tempDir, commit) + if err := os.MkdirAll(checkoutDir, os.ModePerm); err != nil { + return err + } + + err := GitCheckoutTree(ctx, repoDir, checkoutDir, commit) + if err != nil { + return err + } + + commitDir := filepath.Join(doltBinDir, commit) + if err := os.MkdirAll(commitDir, os.ModePerm); err != nil { + return err + } + + command, err := goBuild(ctx, checkoutDir, commitDir) + if err != nil { + return err + } + + return doltVersion(ctx, commitDir, command) +} + +// goBuild builds the dolt binary and returns the filename +func goBuild(ctx context.Context, source, dest string) (string, error) { + goDir := filepath.Join(source, "go") + doltFileName := "dolt" + if runtime.GOOS == "windows" { + doltFileName = "dolt.exe" + } + toBuild := filepath.Join(dest, doltFileName) + build := ExecCommand(ctx, "go", "build", "-o", toBuild, filepath.Join(goDir, "cmd", "dolt")) + build.Dir = goDir + err := build.Run() + if err != nil { + return "", err + } + return toBuild, nil +} + +// doltVersion prints dolt version of binary +func doltVersion(ctx context.Context, dir, command string) error { + doltVersion := ExecCommand(ctx, command, "version") + doltVersion.Stderr = os.Stderr + doltVersion.Stdout = os.Stdout + doltVersion.Dir = dir + return doltVersion.Run() +} diff --git a/go/performance/utils/sysbench_runner/README.md b/go/performance/utils/sysbench_runner/README.md new file mode 100644 index 0000000000..b2e8682ef6 --- /dev/null +++ b/go/performance/utils/sysbench_runner/README.md @@ -0,0 +1,100 @@ +Sysbench runner is a tool for running sysbench tests against sql servers. Custom sysbench lua scripts used +for benchmarking Dolt are [here](https://github.com/dolthub/sysbench-lua-scripts). + +The tool requires a json config file to run: +```bash +$ sysbench_runner --config=config.json +``` + +Configuration: + +```json +{ + "Runs": 1, + "DebugMode": false, + "Servers": "[{...}]", + "TestOptions": [""], + "Tests": "[{...}]" +} +``` + +`Runs` number of times to run all tests per server, default 1 (**Optional**) + +`DebugMode` logs more output from various commands. (**Optional**) + +`Servers` list of servers to test against. See `Server` definitions below. (**Required**) + +`TestOptions` list of sysbench test options to supply to all tests (**Optional**) + +`Tests` the sysbench tests to run. See `Test` definitions below. (**Optional**) + +If no tests are provided, +the following default tests will be run: +``` +oltp_read_only +oltp_insert +oltp_point_select +select_random_points +select_random_ranges +oltp_delete +oltp_write_only +oltp_read_write +oltp_update_index +oltp_update_non_index +``` + +`Server` is a server to test against. + +```json +{ + "Host": "", + "Port": 0, + "Server": "", + "Version": "", + "ResultsFormat": "", + "ServerExec": "", + "ServerArgs": [""], + "ConnectionProtocol": "", + "Socket": "" +} +``` + +`Host` is the server host. (**Required**) + +`Port` is the server port. Defaults to **3306** for `dolt` and `mysql` Servers. (**Optional**) + +`Server` is the server. Only `dolt` and `mysql` are supported. (**Required**) + +`Version` is the server version. (**Required**) + +`ResultsFormat` is the format the results should be written in. Only `json` and `csv` are supported. (**Required**) + +`ServerExec` is the path to a server binary (**Required**) + +`ServerArgs` are the args used to start the server. Will be appended to command `dolt sql-server` for dolt server or `mysqld --user=mysql` for mysql server. (**Optional**) + +`ConnectionProtocol` is the protocol for connecting to mysql, either "unix" or "tcp" (**Required for mysql**) + +`Socket` is the path to the mysql socket (**Required for mysql with unix protocol**) + + +`Test` is a sysbench test or lua script. + +```json +{ + "Name": "", + "N": 1, + "FromScript": false, + "Options": [""] +} +``` + +`Name` is the test name or lua script. (**Required**) + +`N` number of times to repeat this test, default is 1 (**Optional**) + +`FromScript` indicates if this test is from a lua script, defaults to `false` (**Optional**) + +`Options` are additional sysbench test options. These will be provided to sysbench in the form: + +`sysbench [options]... [testname] [command]` diff --git a/go/performance/utils/sysbench_runner/cmd/main.go b/go/performance/utils/sysbench_runner/cmd/main.go new file mode 100644 index 0000000000..19bdb28d3f --- /dev/null +++ b/go/performance/utils/sysbench_runner/cmd/main.go @@ -0,0 +1,54 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "log" + "os" + "path/filepath" + + runner "github.com/dolthub/dolt/go/performance/utils/sysbench_runner" +) + +var configFile = flag.String("config", "", "path to config file") + +func main() { + flag.Parse() + + if *configFile == "" { + log.Fatal("Must supply config") + } + + configPath, err := filepath.Abs(*configFile) + if err != nil { + log.Fatal(err) + } + if _, err = os.Stat(configPath); os.IsNotExist(err) { + log.Fatal(err) + } + + config, err := runner.FromFileConfig(configPath) + if err != nil { + log.Fatal(err) + } + + err = runner.Run(config) + if err != nil { + log.Fatal(err) + } + + os.Exit(0) +} diff --git a/go/performance/utils/sysbench_runner/config.go b/go/performance/utils/sysbench_runner/config.go new file mode 100644 index 0000000000..2c1550de45 --- /dev/null +++ b/go/performance/utils/sysbench_runner/config.go @@ -0,0 +1,505 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/google/uuid" +) + +const ( + Dolt ServerType = "dolt" + MySql ServerType = "mysql" + + CsvFormat = "csv" + JsonFormat = "json" + + CsvExt = ".csv" + JsonExt = ".json" + + defaultHost = "127.0.0.1" + defaultPort = 3306 + + defaultSocket = "/var/run/mysqld/mysqld.sock" + + tcpProtocol = "tcp" + unixProtocol = "unix" + + sysbenchUserLocal = "'sysbench'@'localhost'" + sysbenchPassLocal = "sysbenchpass" +) + +var ( + ErrTestNameNotDefined = errors.New("test name not defined") + ErrNoServersDefined = errors.New("servers not defined") + ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol") +) + +var defaultSysbenchParams = []string{ + "--db-driver=mysql", + "--db-ps-mode=disable", + fmt.Sprintf("--mysql-db=%s", dbName), +} + +var defaultDoltServerParams = []string{"sql-server"} +var defaultMysqlServerParams = []string{"--user=mysql"} + +var defaultSysbenchTests = []*ConfigTest{ + NewConfigTest("oltp_read_only", []string{}, false), + NewConfigTest("oltp_insert", []string{}, false), + NewConfigTest("bulk_insert", []string{}, false), + NewConfigTest("oltp_point_select", []string{}, false), + NewConfigTest("select_random_points", []string{}, false), + NewConfigTest("select_random_ranges", []string{}, false), + NewConfigTest("oltp_delete", []string{}, false), + NewConfigTest("oltp_write_only", []string{}, false), + NewConfigTest("oltp_read_write", []string{}, false), + NewConfigTest("oltp_update_index", []string{}, false), + NewConfigTest("oltp_update_non_index", []string{}, false), +} + +type ServerType string + +// Test is a single sysbench test +type Test struct { + id string + + // Name is the test name + Name string + + // Params are the parameters passed to sysbench + Params []string + + // FromScript indicates if this test is from a lua script + FromScript bool +} + +// Prepare returns a test's args for sysbench's prepare step +func (t *Test) Prepare() []string { + return withCommand(t.Params, "prepare") +} + +// Prepare returns a test's args for sysbench's run step +func (t *Test) Run() []string { + return withCommand(t.Params, "run") +} + +// Prepare returns a test's args for sysbench's cleanup step +func (t *Test) Cleanup() []string { + return withCommand(t.Params, "cleanup") +} + +func withCommand(params []string, command string) []string { + c := make([]string, 0) + c = append(c, params...) + return append(c, command) +} + +// ConfigTest provides users a way to define a test for multiple tablesizes +type ConfigTest struct { + // Name is the test name + Name string + + // N is the number of times a test should run + N int + + // Options are additional sysbench test options a user can supply to run with this test + Options []string + + // FromScript is a boolean indicating that this test is from a lua script + FromScript bool +} + +// NewConfigTest returns a ConfigTest containing the supplied args +func NewConfigTest(name string, opts []string, fromScript bool) *ConfigTest { + options := make([]string, 0) + options = append(options, opts...) + return &ConfigTest{ + Name: name, + N: 1, + Options: options, + FromScript: fromScript, + } +} + +// GetTests returns a slice of Tests +func (ct *ConfigTest) GetTests(serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) { + if ct.Name == "" { + return nil, ErrTestNameNotDefined + } + if ct.N < 1 { + ct.N = 1 + } + + params := fromConfigTestParams(ct, serverConfig) + tests := make([]*Test, 0) + + var idFunc func() string + if testIdFunc == nil { + idFunc = func() string { + return uuid.New().String() + } + } else { + idFunc = testIdFunc + } + + for i := 0; i < ct.N; i++ { + p := make([]string, len(params)) + copy(p, params) + tests = append(tests, &Test{ + id: idFunc(), + Name: ct.Name, + Params: p, + FromScript: ct.FromScript, + }) + } + return tests, nil +} + +// fromConfigTestParams returns params formatted for sysbench: +// `sysbench [options]... [testname] [command]` +func fromConfigTestParams(ct *ConfigTest, serverConfig *ServerConfig) []string { + params := make([]string, 0) + params = append(params, defaultSysbenchParams...) + params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host)) + if serverConfig.Port != 0 { + params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port)) + } + + // handle sysbench user for local mysql server + if serverConfig.Server == MySql && serverConfig.Host == defaultHost { + params = append(params, "--mysql-user=sysbench") + params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal)) + } else { + params = append(params, "--mysql-user=root") + } + + params = append(params, ct.Options...) + params = append(params, ct.Name) + return params +} + +// ServerConfig is the configuration for a server to test against +type ServerConfig struct { + // Id is a unique id for this servers benchmarking + Id string + + // Host is the server host + Host string + + // Port is the server port + Port int + + // Server is the type of server + Server ServerType + + // Version is the server version + Version string + + // ResultsFormat is the format the results should be written in + ResultsFormat string + + // ServerExec is the path to a server executable + ServerExec string + + // ServerArgs are the args used to start a server + ServerArgs []string + + // ConnectionProtocol defines the protocol for connecting to the server + ConnectionProtocol string + + // Socket is the path to the server socket + Socket string +} + +func (sc *ServerConfig) GetId() string { + if sc.Id == "" { + sc.Id = uuid.New().String() + } + return sc.Id +} + +// GetServerArgs returns the args used to start a server +func (sc *ServerConfig) GetServerArgs() []string { + params := make([]string, 0) + + defaultParams := make([]string, 0) + if sc.Server == Dolt { + defaultParams = defaultDoltServerParams + } else if sc.Server == MySql { + defaultParams = defaultMysqlServerParams + } + + params = append(params, defaultParams...) + if sc.Server == Dolt { + params = append(params, fmt.Sprintf("--host=%s", sc.Host)) + } + if sc.Port != 0 { + params = append(params, fmt.Sprintf("--port=%d", sc.Port)) + } + params = append(params, sc.ServerArgs...) + return params +} + +// Config is the configuration for a benchmarking run +type Config struct { + // Runs is the number of times to run all tests + Runs int + + // RuntimeOS is the platform the benchmarks ran on + RuntimeOS string + + // RuntimeGoArch is the runtime architecture + RuntimeGoArch string + + // Servers are the servers to benchmark + Servers []*ServerConfig + + // Tests are the tests to run. If no tests are provided, + // the default tests will be used + Tests []*ConfigTest + + // TestOptions a list of sysbench test options to apply to all tests + TestOptions []string + + // ScriptDir is a path to a directory of lua scripts + ScriptDir string +} + +// NewConfig returns a new Config +func NewConfig() *Config { + return &Config{ + Servers: make([]*ServerConfig, 0), + } +} + +// Validate checks the config for the required fields and sets defaults +// where necessary +func (c *Config) Validate() error { + if len(c.Servers) < 1 { + return ErrNoServersDefined + } + err := c.setDefaults() + if err != nil { + return err + } + return c.validateServerConfigs() +} + +// validateServerConfigs ensures the ServerConfigs are valid +func (c *Config) validateServerConfigs() error { + portMap := make(map[int]ServerType) + for _, s := range c.Servers { + if s.Server != Dolt && s.Server != MySql { + return fmt.Errorf("unsupported server type: %s", s.Server) + } + + err := validateRequiredFields(string(s.Server), s.Version, s.ResultsFormat) + if err != nil { + return err + } + + if s.Server == MySql { + err = checkProtocol(s.ConnectionProtocol) + if err != nil { + return err + } + } + + if s.Host == "" { + s.Host = defaultHost + } + + portMap, err = checkUpdatePortMap(s, portMap) + if err != nil { + return err + } + + err = checkExec(s) + if err != nil { + return err + } + } + return nil +} + +func validateRequiredFields(server, version, format string) error { + if server == "" { + return getMustSupplyError("server") + } + if version == "" { + return getMustSupplyError("version") + } + if format == "" { + return getMustSupplyError("results format") + } + return nil +} + +// setDefaults sets defaults on the Config +func (c *Config) setDefaults() error { + if c.RuntimeOS == "" { + c.RuntimeOS = runtime.GOOS + } + if c.RuntimeGoArch == "" { + c.RuntimeGoArch = runtime.GOARCH + } + if len(c.Tests) < 1 { + fmt.Printf("Preparing to benchmark against default tests\n") + if c.ScriptDir != "" { + abs, err := filepath.Abs(c.ScriptDir) + if err != nil { + return err + } + if _, err := os.Stat(abs); os.IsNotExist(err) { + return fmt.Errorf("script dir not found: %s", abs) + } + c.ScriptDir = abs + } + tests, err := getDefaultTests(c) + if err != nil { + return err + } + c.Tests = tests + } + if c.Runs < 1 { + c.Runs = 1 + } + return nil +} + +// checkUpdatePortMap returns an error if multiple servers have specified the same port +func checkUpdatePortMap(serverConfig *ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) { + if serverConfig.Port == 0 { + serverConfig.Port = defaultPort + } + srv, ok := portMap[serverConfig.Port] + if ok && srv != serverConfig.Server { + return nil, fmt.Errorf("servers have port conflict on port: %d\n", serverConfig.Port) + } + if !ok { + portMap[serverConfig.Port] = serverConfig.Server + } + return portMap, nil +} + +// checkExec verifies the binary exists +func checkExec(serverConfig *ServerConfig) error { + if serverConfig.ServerExec == "" { + return getMustSupplyError("server exec") + } + abs, err := filepath.Abs(serverConfig.ServerExec) + if err != nil { + return err + } + if _, err := os.Stat(abs); os.IsNotExist(err) { + return fmt.Errorf("server exec not found: %s", abs) + } + serverConfig.ServerExec = abs + return nil +} + +// checkProtocol ensures the given protocol is supported +func checkProtocol(protocol string) error { + if protocol == "" { + return getMustSupplyError("connection protocol") + } + if protocol == tcpProtocol || protocol == unixProtocol { + return nil + } + return ErrUnsupportedConnectionProtocol +} + +// GetTests returns a slice of Tests created from the +// defined ServerConfig.Tests +func GetTests(config *Config, serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) { + flattened := make([]*Test, 0) + for _, t := range config.Tests { + if len(config.TestOptions) > 0 { + t.Options = append(t.Options, config.TestOptions...) + } + tests, err := t.GetTests(serverConfig, testIdFunc) + if err != nil { + return nil, err + } + flattened = append(flattened, tests...) + } + return flattened, nil +} + +// FromFileConfig returns a validated Config based on the config file at the configPath +func FromFileConfig(configPath string) (*Config, error) { + data, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + + config := NewConfig() + err = json.Unmarshal(data, config) + if err != nil { + return nil, err + } + + return config, nil +} + +func getMustSupplyError(name string) error { + return fmt.Errorf("Must supply %s", name) +} + +func getDefaultTests(config *Config) ([]*ConfigTest, error) { + defaultTests := make([]*ConfigTest, 0) + defaultTests = append(defaultTests, defaultSysbenchTests...) + if config.ScriptDir != "" { + luaScriptTests, err := getLuaScriptTestsFromDir(config.ScriptDir) + if err != nil { + return nil, err + } + defaultTests = append(defaultTests, luaScriptTests...) + } + return defaultTests, nil +} + +func getLuaScriptTestsFromDir(dir string) ([]*ConfigTest, error) { + luaScripts := make([]*ConfigTest, 0) + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + // Append all the lua scripts except for the `_common.lua` scripts which shouldnt be tested directly + if strings.HasSuffix(path, ".lua") && !strings.Contains(path, "_common.lua") { + luaScripts = append(luaScripts, NewConfigTest(path, []string{}, true)) + } + return nil + }) + if err != nil { + return nil, err + } + return luaScripts, nil +} diff --git a/go/performance/utils/sysbench_runner/config_test.go b/go/performance/utils/sysbench_runner/config_test.go new file mode 100644 index 0000000000..22a0a95037 --- /dev/null +++ b/go/performance/utils/sysbench_runner/config_test.go @@ -0,0 +1,107 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var testIdFunc = func() string { return "id" } + +func TestConfigTestGetTests(t *testing.T) { + empty := &ConfigTest{Name: "test_name"} + + one := &ConfigTest{Name: "test_one", N: 3} + two := &ConfigTest{Name: "test_two", N: 2} + three := &ConfigTest{Name: "test_three", N: 1} + + opts := &ConfigTest{ + Name: "test_options", + N: 1, + Options: []string{"--create_secondary=on", "--auto_inc=off"}, + } + + serverConfig := &ServerConfig{Server: MySql, Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat} + + tests := []struct { + description string + config *Config + expectedTests []*Test + expectedError error + }{ + { + description: "should error if no test name is defined", + config: &Config{ + Servers: []*ServerConfig{serverConfig}, + Tests: []*ConfigTest{ + {Name: ""}, + }, + }, + expectedTests: nil, + expectedError: ErrTestNameNotDefined, + }, + { + description: "should create single test if N is < 1", + config: &Config{ + Servers: []*ServerConfig{serverConfig}, + Tests: []*ConfigTest{empty}, + }, + expectedTests: []*Test{ + { + id: testIdFunc(), + Name: "test_name", + Params: fromConfigTestParams(empty, serverConfig), + }, + }, + }, + { + description: "should return a test for each N defined on the ConfigTest", + config: &Config{ + Servers: []*ServerConfig{serverConfig}, + Tests: []*ConfigTest{one, two, three}, + }, + expectedTests: []*Test{ + {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, + {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, + {id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)}, + {id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)}, + {id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)}, + {id: testIdFunc(), Name: "test_three", Params: fromConfigTestParams(three, serverConfig)}, + }, + }, + { + description: "should apply user options to test params", + config: &Config{ + Servers: []*ServerConfig{serverConfig}, + Tests: []*ConfigTest{opts}, + }, + expectedTests: []*Test{ + {id: testIdFunc(), Name: "test_options", Params: fromConfigTestParams(opts, serverConfig)}, + }, + }, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + for _, s := range test.config.Servers { + actual, err := GetTests(test.config, s, testIdFunc) + assert.Equal(t, test.expectedError, err) + assert.Equal(t, len(test.expectedTests), len(actual)) + assert.ElementsMatch(t, test.expectedTests, actual) + } + }) + } +} diff --git a/go/performance/utils/sysbench_runner/csv.go b/go/performance/utils/sysbench_runner/csv.go new file mode 100644 index 0000000000..83e9eefa26 --- /dev/null +++ b/go/performance/utils/sysbench_runner/csv.go @@ -0,0 +1,332 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "path/filepath" + "time" +) + +// FromResultCsvHeaders returns supported csv headers for a Result +func FromResultCsvHeaders() []string { + return []string{ + "id", + "suite_id", + "test_id", + "runtime_os", + "runtime_goarch", + "server_name", + "server_version", + "server_params", + "test_name", + "test_params", + "created_at", + + // benchmark headers + "sql_read_queries", + "sql_write_queries", + "sql_other_queries", + "sql_total_queries", + "sql_total_queries_per_second", + "sql_transactions_total", + "sql_transactions_per_second", + "sql_ignored_errors_total", + "sql_ignored_errors_per_second", + "sql_reconnects_total", + "sql_reconnects_per_second", + "total_time_seconds", + "total_number_of_events", + "latency_minimum_ms", + "latency_average_ms", + "latency_maximum_ms", + "latency_percentile", + "latency_sum_ms", + } +} + +var floatTemplate = "%g" +var intTemplate = "%d" + +// FromHeaderResultColumnValue returns the value from the Result for the given +// header field +func FromHeaderResultColumnValue(h string, r *Result) (string, error) { + var val string + switch h { + case "id": + val = r.Id + case "suite_id": + val = r.SuiteId + case "test_id": + val = r.TestId + case "runtime_os": + val = r.RuntimeOS + case "runtime_goarch": + val = r.RuntimeGoArch + case "server_name": + val = r.ServerName + case "server_version": + val = r.ServerVersion + case "server_params": + val = r.ServerParams + case "test_name": + val = r.TestName + case "test_params": + val = r.TestParams + case "created_at": + val = r.CreatedAt + + // benchmark headers + case "sql_read_queries": + val = fmt.Sprintf(intTemplate, r.SqlReadQueries) + case "sql_write_queries": + val = fmt.Sprintf(intTemplate, r.SqlWriteQueries) + case "sql_other_queries": + val = fmt.Sprintf(intTemplate, r.SqlOtherQueries) + case "sql_total_queries": + val = fmt.Sprintf(intTemplate, r.SqlTotalQueries) + case "sql_total_queries_per_second": + val = fmt.Sprintf(floatTemplate, r.SqlTotalQueriesPerSecond) + case "sql_transactions_total": + val = fmt.Sprintf(intTemplate, r.TransactionsTotal) + case "sql_transactions_per_second": + val = fmt.Sprintf(floatTemplate, r.TransactionsPerSecond) + case "sql_ignored_errors_total": + val = fmt.Sprintf(intTemplate, r.IgnoredErrorsTotal) + case "sql_ignored_errors_per_second": + val = fmt.Sprintf(floatTemplate, r.IgnoredErrorsPerSecond) + case "sql_reconnects_total": + val = fmt.Sprintf(intTemplate, r.ReconnectsTotal) + case "sql_reconnects_per_second": + val = fmt.Sprintf(floatTemplate, r.ReconnectsPerSecond) + case "total_time_seconds": + val = fmt.Sprintf(floatTemplate, r.TotalTimeSeconds) + case "total_number_of_events": + val = fmt.Sprintf(intTemplate, r.IgnoredErrorsTotal) + case "latency_minimum_ms": + val = fmt.Sprintf(floatTemplate, r.LatencyMinMS) + case "latency_average_ms": + val = fmt.Sprintf(floatTemplate, r.LatencyAvgMS) + case "latency_maximum_ms": + val = fmt.Sprintf(floatTemplate, r.LatencyMaxMS) + case "latency_percentile": + val = fmt.Sprintf(floatTemplate, r.LatencyPercentile) + case "latency_sum_ms": + val = fmt.Sprintf(floatTemplate, r.LatencySumMS) + default: + return "", ErrUnsupportedHeaderField + } + return val, nil +} + +// FromHeaderResultFieldValue sets the value to the corresponding Result field for the given +// header field +func FromHeaderResultFieldValue(field string, val string, r *Result) error { + switch field { + case "id": + r.Id = val + case "suite_id": + r.SuiteId = val + case "test_id": + r.TestId = val + case "runtime_os": + r.RuntimeOS = val + case "runtime_goarch": + r.RuntimeGoArch = val + case "server_name": + r.ServerName = val + case "server_version": + r.ServerVersion = val + case "server_params": + r.ServerParams = val + case "test_name": + r.TestName = val + case "test_params": + r.TestParams = val + case "created_at": + _, err := time.Parse(stampFormat, val) + if err != nil { + return err + } + r.CreatedAt = val + + // benchmark headers + case "sql_read_queries": + return updateResult(r, read, val) + case "sql_write_queries": + return updateResult(r, write, val) + case "sql_other_queries": + return updateResult(r, other, val) + case "sql_total_queries": + return updateResult(r, totalQueries, val) + case "sql_total_queries_per_second": + f, err := fromStringFloat64(val) + if err != nil { + return err + } + r.SqlTotalQueriesPerSecond = f + case "sql_transactions_total": + i, err := fromStringInt64(val) + if err != nil { + return err + } + r.TransactionsTotal = i + case "sql_transactions_per_second": + f, err := fromStringFloat64(val) + if err != nil { + return err + } + r.TransactionsPerSecond = f + case "sql_ignored_errors_total": + i, err := fromStringInt64(val) + if err != nil { + return err + } + r.IgnoredErrorsTotal = i + case "sql_ignored_errors_per_second": + f, err := fromStringFloat64(val) + if err != nil { + return err + } + r.IgnoredErrorsPerSecond = f + case "sql_reconnects_total": + i, err := fromStringInt64(val) + if err != nil { + return err + } + r.ReconnectsTotal = i + case "sql_reconnects_per_second": + f, err := fromStringFloat64(val) + if err != nil { + return err + } + r.ReconnectsPerSecond = f + case "total_time_seconds": + f, err := fromStringFloat64(val) + if err != nil { + return err + } + r.TotalTimeSeconds = f + case "total_number_of_events": + return updateResult(r, totalEvents, val) + case "latency_minimum_ms": + return updateResult(r, min, val) + case "latency_average_ms": + return updateResult(r, avg, val) + case "latency_maximum_ms": + return updateResult(r, max, val) + case "latency_percentile": + return updateResult(r, percentile, val) + case "latency_sum_ms": + return updateResult(r, sum, val) + default: + return ErrUnsupportedHeaderField + } + return nil +} + +// WriteResultsCsv writes Results to a csv +func WriteResultsCsv(filename string, results Results) (err error) { + dir := filepath.Dir(filename) + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + return err + } + var file *os.File + file, err = os.Create(filename) + if err != nil { + return + } + defer func() { + closeErr := file.Close() + if err == nil && closeErr != nil { + err = closeErr + } + }() + + csvWriter := csv.NewWriter(file) + + // write header + headers := FromResultCsvHeaders() + if err := csvWriter.Write(headers); err != nil { + return err + } + + // write rows + for _, r := range results { + row := make([]string, 0) + for _, field := range headers { + val, err := FromHeaderResultColumnValue(field, r) + if err != nil { + return err + } + row = append(row, val) + } + err = csvWriter.Write(row) + if err != nil { + return err + } + } + + csvWriter.Flush() + if err := csvWriter.Error(); err != nil { + return err + } + return +} + +// ReadResultsCsv reads a csv into Results +func ReadResultsCsv(filename string) (Results, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + var textInput io.Reader = file + + reader := csv.NewReader(textInput) + records, err := reader.ReadAll() + if err != nil { + return []*Result{}, err + } + + var header []string + results := make(Results, 0) + for i, row := range records { + // handle header + if i == 0 { + header = row + continue + } + + // handle rows + r := &Result{} + if header != nil { + for j, field := range header { + err := FromHeaderResultFieldValue(field, row[j], r) + if err != nil { + return []*Result{}, err + } + } + } + results = append(results, r) + } + + return results, nil +} diff --git a/go/performance/utils/sysbench_runner/csv_test.go b/go/performance/utils/sysbench_runner/csv_test.go new file mode 100644 index 0000000000..10d3e315c0 --- /dev/null +++ b/go/performance/utils/sysbench_runner/csv_test.go @@ -0,0 +1,47 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWriteReadResultsCsv(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "TestWriteResultsCsv") + err = os.MkdirAll(tmpDir, os.ModePerm) + require.NoError(t, err) + + expectedOne := genRandomResult() + expectedTwo := genRandomResult() + + filename := filepath.Join(tmpDir, fmt.Sprintf("test-results%s", CsvExt)) + err = WriteResultsCsv(filename, []*Result{expectedOne, expectedTwo}) + require.NoError(t, err) + + actual, err := ReadResultsCsv(filename) + require.NoError(t, err) + assert.Equal(t, len(actual), 2) + assert.Equal(t, expectedOne.SqlTotalQueries, actual[0].SqlTotalQueries) + assert.Equal(t, expectedOne.LatencySumMS, actual[0].LatencySumMS) + assert.Equal(t, expectedTwo.SqlTotalQueries, actual[1].SqlTotalQueries) + assert.Equal(t, expectedTwo.LatencySumMS, actual[1].LatencySumMS) +} diff --git a/go/performance/utils/sysbench_runner/debug.go b/go/performance/utils/sysbench_runner/debug.go new file mode 100644 index 0000000000..d0e58a39b9 --- /dev/null +++ b/go/performance/utils/sysbench_runner/debug.go @@ -0,0 +1,38 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "os" + "os/exec" +) + +var Debug bool + +func init() { + if os.Getenv("DEBUG") != "" { + Debug = true + } +} + +func ExecCommand(ctx context.Context, name string, arg ...string) *exec.Cmd { + e := exec.CommandContext(ctx, name, arg...) + if Debug { + e.Stdout = os.Stdout + e.Stderr = os.Stderr + } + return e +} diff --git a/go/performance/utils/sysbench_runner/dolt.go b/go/performance/utils/sysbench_runner/dolt.go new file mode 100644 index 0000000000..cd0d63a1b1 --- /dev/null +++ b/go/performance/utils/sysbench_runner/dolt.go @@ -0,0 +1,271 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "sync" + "syscall" + "time" + + "golang.org/x/sync/errgroup" +) + +const ( + dbName = "test" + luaPath = "?.lua" +) + +var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) } + +// BenchmarkDolt benchmarks dolt based on the provided configurations +func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { + serverParams := serverConfig.GetServerArgs() + + err := doltVersion(ctx, serverConfig) + if err != nil { + return nil, err + } + + err = updateDoltConfig(ctx, serverConfig) + if err != nil { + return nil, err + } + + testRepo, err := initDoltRepo(ctx, serverConfig) + if err != nil { + return nil, err + } + + withKeyCtx, cancel := context.WithCancel(ctx) + gServer, serverCtx := errgroup.WithContext(withKeyCtx) + + server := getDoltServer(serverCtx, serverConfig, testRepo, serverParams) + + // handle user interrupt + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + var wg sync.WaitGroup + wg.Add(1) + go func() { + <-quit + defer wg.Done() + signal.Stop(quit) + cancel() + }() + + // launch the dolt server + gServer.Go(func() error { + return server.Run() + }) + + // sleep to allow the server to start + time.Sleep(5 * time.Second) + + tests, err := GetTests(config, serverConfig, nil) + if err != nil { + return nil, err + } + + results := make(Results, 0) + for i := 0; i < config.Runs; i++ { + for _, test := range tests { + r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) + if err != nil { + close(quit) + wg.Wait() + return nil, err + } + results = append(results, r) + } + } + + // send signal to dolt server + quit <- syscall.SIGTERM + + err = gServer.Wait() + if err != nil { + // we expect a kill error + // we only exit in error if this is not the + // error + if err.Error() != "signal: killed" { + close(quit) + wg.Wait() + return nil, err + } + } + + close(quit) + wg.Wait() + + err = os.RemoveAll(testRepo) + if err != nil { + return nil, err + } + + return results, nil +} + +// doltVersion ensures the dolt binary can run +func doltVersion(ctx context.Context, config *ServerConfig) error { + doltVersion := ExecCommand(ctx, config.ServerExec, "version") + return doltVersion.Run() +} + +// initDoltRepo initializes a dolt repo and returns the repo path +func initDoltRepo(ctx context.Context, config *ServerConfig) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + testRepo := filepath.Join(cwd, dbName) + err = os.MkdirAll(testRepo, os.ModePerm) + if err != nil { + return "", err + } + + doltInit := ExecCommand(ctx, config.ServerExec, "init") + doltInit.Dir = testRepo + err = doltInit.Run() + if err != nil { + return "", err + } + + return testRepo, nil +} + +// updateDoltConfig updates the dolt config if necessary +func updateDoltConfig(ctx context.Context, config *ServerConfig) error { + err := checkSetDoltConfig(ctx, config, "user.name", "benchmark") + if err != nil { + return err + } + return checkSetDoltConfig(ctx, config, "user.email", "benchmark@dolthub.com") +} + +// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary +func checkSetDoltConfig(ctx context.Context, config *ServerConfig, key, val string) error { + check := ExecCommand(ctx, config.ServerExec, "config", "--global", "--get", key) + err := check.Run() + if err != nil { + // config get calls exit with 1 if not set + if err.Error() != "exit status 1" { + return err + } + + set := ExecCommand(ctx, config.ServerExec, "config", "--global", "--add", key, val) + err := set.Run() + if err != nil { + return err + } + } + + return nil +} + +// getDoltServer returns a exec.Cmd for a dolt server +func getDoltServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd { + server := ExecCommand(ctx, config.ServerExec, params...) + server.Dir = testRepo + return server +} + +// sysbenchPrepare returns a exec.Cmd for running the sysbench prepare step +func sysbenchPrepare(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { + cmd := ExecCommand(ctx, "sysbench", test.Prepare()...) + if test.FromScript { + lp := filepath.Join(scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + } + return cmd +} + +// sysbenchRun returns a exec.Cmd for running the sysbench run step +func sysbenchRun(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "sysbench", test.Run()...) + if test.FromScript { + lp := filepath.Join(scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + } + return cmd +} + +// sysbenchPrepare returns a exec.Cmd for running the sysbench cleanup step +func sysbenchCleanup(ctx context.Context, test *Test, scriptDir string) *exec.Cmd { + cmd := ExecCommand(ctx, "sysbench", test.Cleanup()...) + if test.FromScript { + lp := filepath.Join(scriptDir, luaPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp)) + } + return cmd +} + +// benchmark runs a sysbench benchmark against a server calling prepare, run, cleanup +func benchmark( + ctx context.Context, + test *Test, + config *Config, + serverConfig *ServerConfig, + stampFunc func() string, + suiteId string, +) (*Result, error) { + prepare := sysbenchPrepare(ctx, test, config.ScriptDir) + run := sysbenchRun(ctx, test, config.ScriptDir) + cleanup := sysbenchCleanup(ctx, test, config.ScriptDir) + + err := prepare.Run() + if err != nil { + return nil, err + } + + out, err := run.Output() + if err != nil { + fmt.Print(string(out)) + return nil, err + } + + if Debug == true { + fmt.Print(string(out)) + } + + r, err := FromOutputResult(out, config, serverConfig, test, suiteId, nil) + if err != nil { + return nil, err + } + + r.Stamp(stampFunc) + + return r, cleanup.Run() +} + +// fromChannelResults collects all Results from the given channel and returns them +func fromChannelResults(rc chan *Result) Results { + results := make(Results, 0) + for r := range rc { + if r != nil { + results = append(results, r) + } + } + return results +} diff --git a/go/performance/utils/sysbench_runner/json.go b/go/performance/utils/sysbench_runner/json.go new file mode 100644 index 0000000000..5a0fca38db --- /dev/null +++ b/go/performance/utils/sysbench_runner/json.go @@ -0,0 +1,87 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +type report struct { + Results Results +} + +// WriteResultsJson writes Results to a json file +func WriteResultsJson(filename string, results Results) (err error) { + dir := filepath.Dir(filename) + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + return err + } + var file *os.File + file, err = os.Create(filename) + if err != nil { + return + } + defer func() { + closeErr := file.Close() + if err == nil && closeErr != nil { + err = closeErr + } + }() + + d := report{ + Results: results, + } + + b, err := json.Marshal(d) + if err != nil { + return err + } + + _, err = file.Write(b) + if err != nil { + return err + } + + return +} + +// ReadResultsJson reads a json file into Results +func ReadResultsJson(filename string) (Results, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + var textInput io.Reader = file + + b, err := ioutil.ReadAll(textInput) + if err != nil { + return nil, err + } + + r := report{} + err = json.Unmarshal(b, &r) + if err != nil { + return nil, err + } + + return r.Results, nil +} diff --git a/go/performance/utils/sysbench_runner/json_test.go b/go/performance/utils/sysbench_runner/json_test.go new file mode 100644 index 0000000000..3e15060b8c --- /dev/null +++ b/go/performance/utils/sysbench_runner/json_test.go @@ -0,0 +1,47 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWriteReadResultsJson(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "TestWriteResultsJson") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + expected := &Result{ + SqlTotalQueries: 1000, + LatencySumMS: 12.34, + } + + filename := filepath.Join(tmpDir, fmt.Sprintf("test-results%s", JsonExt)) + err = WriteResultsJson(filename, []*Result{expected}) + require.NoError(t, err) + + actual, err := ReadResultsJson(filename) + require.NoError(t, err) + assert.Equal(t, len(actual), 1) + assert.Equal(t, expected.SqlTotalQueries, actual[0].SqlTotalQueries) + assert.Equal(t, expected.LatencySumMS, actual[0].LatencySumMS) +} diff --git a/go/performance/utils/sysbench_runner/mysql.go b/go/performance/utils/sysbench_runner/mysql.go new file mode 100644 index 0000000000..1ff637ef5d --- /dev/null +++ b/go/performance/utils/sysbench_runner/mysql.go @@ -0,0 +1,183 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "database/sql" + "fmt" + "os" + "os/exec" + "os/signal" + "sync" + "syscall" + "time" + + _ "github.com/go-sql-driver/mysql" + "golang.org/x/sync/errgroup" +) + +// BenchmarkMysql benchmarks mysql based on the provided configurations +func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) { + withKeyCtx, cancel := context.WithCancel(ctx) + + var localServer bool + var gServer *errgroup.Group + var serverCtx context.Context + var server *exec.Cmd + if serverConfig.Host == defaultHost { + localServer = true + gServer, serverCtx = errgroup.WithContext(withKeyCtx) + serverParams := serverConfig.GetServerArgs() + server = getMysqlServer(serverCtx, serverConfig, serverParams) + + // launch the mysql server + gServer.Go(func() error { + return server.Run() + }) + + // sleep to allow the server to start + time.Sleep(10 * time.Second) + + // setup mysqldb + err := setupDB(ctx, serverConfig) + if err != nil { + cancel() + return nil, err + } + } + + // handle user interrupt + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGTERM) + var wg sync.WaitGroup + wg.Add(1) + go func() { + <-quit + defer wg.Done() + signal.Stop(quit) + cancel() + }() + + tests, err := GetTests(config, serverConfig, nil) + if err != nil { + return nil, err + } + + results := make(Results, 0) + for i := 0; i < config.Runs; i++ { + for _, test := range tests { + r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId()) + if err != nil { + close(quit) + wg.Wait() + return nil, err + } + results = append(results, r) + } + } + + // stop local mysql server + if localServer { + // send signal to server + quit <- syscall.SIGTERM + + err = gServer.Wait() + if err != nil { + // we expect a kill error + // we only exit in error if this is not the + // error + if err.Error() != "signal: killed" { + close(quit) + wg.Wait() + return nil, err + } + } + } + + close(quit) + wg.Wait() + + return results, nil +} + +// getMysqlServer returns a exec.Cmd for a dolt server +func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd { + return ExecCommand(ctx, config.ServerExec, params...) +} + +func setupDB(ctx context.Context, serverConfig *ServerConfig) (err error) { + dsn, err := formatDSN(serverConfig) + if err != nil { + return err + } + + // TODO make sure this can work on windows + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + defer func() { + err = db.Close() + }() + err = db.Ping() + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbName)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s", dbName)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUserLocal)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s'", sysbenchUserLocal, sysbenchPassLocal)) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, fmt.Sprintf("GRANT ALL ON %s.* to %s", dbName, sysbenchUserLocal)) + if err != nil { + return err + } + + // Required for running groupby_scan.lua without error + _, err = db.ExecContext(ctx, "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));") + if err != nil { + return err + } + + return +} + +func formatDSN(serverConfig *ServerConfig) (string, error) { + var socketPath string + if serverConfig.Socket != "" { + socketPath = serverConfig.Socket + } else { + socketPath = defaultSocket + } + if serverConfig.ConnectionProtocol == tcpProtocol { + return fmt.Sprintf("root@tcp(%s:%d)/", defaultHost, serverConfig.Port), nil + } else if serverConfig.ConnectionProtocol == unixProtocol { + return fmt.Sprintf("root@unix(%s)/", socketPath), nil + } else { + return "", ErrUnsupportedConnectionProtocol + } +} diff --git a/go/performance/utils/sysbench_runner/results.go b/go/performance/utils/sysbench_runner/results.go new file mode 100644 index 0000000000..49e35a8d4f --- /dev/null +++ b/go/performance/utils/sysbench_runner/results.go @@ -0,0 +1,393 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "errors" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +const ( + stampFormat = time.RFC3339 + SqlStatsPrefix = "SQL statistics:" + read = "read" + write = "write" + other = "other" + totalQueries = "total" + totalEvents = "total number of events" + min = "min" + avg = "avg" + max = "max" + percentile = "percentile" + sum = "sum" + transactions = "transactions" + queriesPerSec = "queries" + ignoredErrors = "ignored errors" + reconnects = "reconnects" + totalTimeSecs = "total time" +) + +var ( + ResultFileTemplate = "%s_%s_%s_sysbench_performance%s" + + ErrUnableToParseOutput = errors.New("unable to parse output") + ErrUnsupportedHeaderField = errors.New("unsupported header field") +) + +// Result stores the output from a sysbench test run +type Result struct { + // Id is the uuid of this result row + Id string `json:"id"` + + // SuiteId is the test suite id this test result is associated with + SuiteId string `json:"suite_id"` + + // TestId is the unique id for this test + TestId string `json:"test_id"` + + // RuntimeOS is the runtime platform + RuntimeOS string `json:"runtime_os"` + + // RuntimeGoArch is the runtime architecture + RuntimeGoArch string `json:"runtime_goarch"` + + // ServerName is the name of the server used for the benchmark + ServerName string `json:"server_name"` + + // ServerVersion is the server's version + ServerVersion string `json:"server_version"` + + // ServerParams are the params used to run the server + ServerParams string `json:"server_params"` + + // TestName is the name of the test + TestName string `json:"test_name"` + + // TestParams are the params used to run the test + TestParams string `json:"test_params"` + + // CreatedAt is the time the result was created UTC + CreatedAt string `json:"created_at"` + + // SqlReadQueries is the number of read queries performed + SqlReadQueries int64 `json:"sql_read_queries"` + + // SqlWriteQueries is the number of write queries performed + SqlWriteQueries int64 `json:"sql_write_queries"` + + // SqlOtherQueries is the number of other queries performed + SqlOtherQueries int64 `json:"sql_other_queries"` + + // SqlTotalQueries is the number of total queries performed + SqlTotalQueries int64 `json:"sql_total_queries"` + + // SqlTotalQueriesPerSecond is the number of queries per second + SqlTotalQueriesPerSecond float64 `json:"sql_total_queries_per_second"` + + // TransactionsTotal is the number of transactions performed + TransactionsTotal int64 `json:"sql_transactions_total"` + + // TransactionsPerSecond is the number of transactions per second + TransactionsPerSecond float64 `json:"sql_transactions_per_second"` + + // IgnoredErrorsTotal is the number of errors ignored + IgnoredErrorsTotal int64 `json:"sql_ignored_errors_total"` + + // IgnoredErrorsPerSecond is the number of errors ignored per second + IgnoredErrorsPerSecond float64 `json:"sql_ignored_errors_per_second"` + + // ReconnectsTotal is the number of reconnects performed + ReconnectsTotal int64 `json:"sql_reconnects_total"` + + // ReconnectsPerSecond is the number of reconnects per second + ReconnectsPerSecond float64 `json:"sql_reconnects_per_second"` + + // TotalTimeSeconds is the total time elapsed for this test + TotalTimeSeconds float64 `json:"total_time_seconds"` + + // TotalNumberOfEvents is the total number of events + TotalNumberOfEvents int64 `json:"total_number_of_events"` + + // LatencyMinMS is the minimum latency in milliseconds + LatencyMinMS float64 `json:"latency_minimum_ms"` + + // LatencyAvgMS is the average latency in milliseconds + LatencyAvgMS float64 `json:"latency_average_ms"` + + // LatencyMaxMS is the maximum latency in milliseconds + LatencyMaxMS float64 `json:"latency_maximum_ms"` + + // LatencyPercentile is the latency of the 95th percentile + LatencyPercentile float64 `json:"latency_percentile"` + + // LatencySumMS is the latency sum in milliseconds + LatencySumMS float64 `json:"latency_sum_ms"` +} + +// Results is a slice of Result +type Results []*Result + +// Stamp timestamps the result using the provided stamp function +func (r *Result) Stamp(stampFunc func() string) { + if r.CreatedAt != "" || stampFunc == nil { + return + } + r.CreatedAt = stampFunc() +} + +// FromConfigsNewResult returns a new result with some fields set based on the provided configs +func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, suiteId string, idFunc func() string) (*Result, error) { + serverParams := serverConfig.GetServerArgs() + + var getId func() string + if idFunc == nil { + getId = func() string { + return uuid.New().String() + } + } else { + getId = idFunc + } + + var name string + if t.FromScript { + base := filepath.Base(t.Name) + ext := filepath.Ext(base) + name = strings.TrimSuffix(base, ext) + } else { + name = t.Name + } + + return &Result{ + Id: getId(), + SuiteId: suiteId, + TestId: t.id, + RuntimeOS: config.RuntimeOS, + RuntimeGoArch: config.RuntimeGoArch, + ServerName: string(serverConfig.Server), + ServerVersion: serverConfig.Version, + ServerParams: strings.Join(serverParams, " "), + TestName: name, + TestParams: strings.Join(t.Params, " "), + }, nil +} + +// FromOutputResult accepts raw sysbench run output and returns the Result +func FromOutputResult(output []byte, config *Config, serverConfig *ServerConfig, test *Test, suiteId string, idFunc func() string) (*Result, error) { + result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc) + if err != nil { + return nil, err + } + lines := strings.Split(string(output), "\n") + var process bool + for _, l := range lines { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + if strings.HasPrefix(trimmed, SqlStatsPrefix) { + process = true + continue + } + if process { + err := UpdateResult(result, trimmed) + if err != nil { + return result, err + } + } + } + return result, nil +} + +// UpdateResult extracts the key and value from the given line and updates the given Result +func UpdateResult(result *Result, line string) error { + lineParts := strings.Split(line, ":") + key := strings.TrimSpace(lineParts[0]) + if len(lineParts) > 1 { + rawVal := strings.TrimSpace(lineParts[1]) + err := updateResult(result, key, rawVal) + if err != nil { + return err + } + } + return nil +} + +// updateResult updates a field in the given Result with the given value +func updateResult(result *Result, key, val string) error { + var k string + if strings.Contains(key, percentile) { + k = percentile + } else { + k = key + } + switch k { + case read: + i, err := fromStringInt64(val) + if err != nil { + return err + } + result.SqlReadQueries = i + case write: + i, err := fromStringInt64(val) + if err != nil { + return err + } + result.SqlWriteQueries = i + case other: + i, err := fromStringInt64(val) + if err != nil { + return err + } + result.SqlOtherQueries = i + case totalQueries: + i, err := fromStringInt64(val) + if err != nil { + return err + } + result.SqlTotalQueries = i + case transactions: + total, perSecond, err := FromValWithParens(val) + if err != nil { + return err + } + t, err := fromStringInt64(total) + if err != nil { + return err + } + p, err := fromStringFloat64(fromPerSecondVal(perSecond)) + if err != nil { + return err + } + result.TransactionsTotal = t + result.TransactionsPerSecond = p + case queriesPerSec: + _, perSecond, err := FromValWithParens(val) + if err != nil { + return err + } + p, err := fromStringFloat64(fromPerSecondVal(perSecond)) + if err != nil { + return err + } + result.SqlTotalQueriesPerSecond = p + case ignoredErrors: + total, perSecond, err := FromValWithParens(val) + t, err := fromStringInt64(total) + if err != nil { + return err + } + p, err := fromStringFloat64(fromPerSecondVal(perSecond)) + if err != nil { + return err + } + result.IgnoredErrorsTotal = t + result.IgnoredErrorsPerSecond = p + case reconnects: + total, perSecond, err := FromValWithParens(val) + t, err := fromStringInt64(total) + if err != nil { + return err + } + p, err := fromStringFloat64(fromPerSecondVal(perSecond)) + if err != nil { + return err + } + result.ReconnectsTotal = t + result.ReconnectsPerSecond = p + case totalTimeSecs: + t, err := fromStringFloat64(fromSecondsVal(val)) + if err != nil { + return err + } + result.TotalTimeSeconds = t + case totalEvents: + i, err := fromStringInt64(val) + if err != nil { + return err + } + result.TotalNumberOfEvents = i + case min: + f, err := fromStringFloat64(val) + if err != nil { + return err + } + result.LatencyMinMS = f + case avg: + f, err := fromStringFloat64(val) + if err != nil { + return err + } + result.LatencyAvgMS = f + case max: + f, err := fromStringFloat64(val) + if err != nil { + return err + } + result.LatencyMaxMS = f + case percentile: + f, err := fromStringFloat64(val) + if err != nil { + return err + } + result.LatencyPercentile = f + case sum: + f, err := fromStringFloat64(val) + if err != nil { + return err + } + result.LatencySumMS = f + default: + return nil + } + return nil +} + +// FromValWithParens takes a string containing parens and +// returns the value outside the parens first, and the value +// inside the parens second +func FromValWithParens(val string) (string, string, error) { + if val == "" { + return "", "", nil + } + parts := strings.Split(val, "(") + if len(parts) <= 1 { + return strings.TrimSpace(parts[0]), "", nil + } + if len(parts) > 2 { + return "", "", ErrUnableToParseOutput + } + return strings.TrimSpace(parts[0]), strings.Trim(parts[1], ")"), nil +} + +func fromPerSecondVal(val string) string { + return strings.TrimSuffix(val, " per sec.") +} + +func fromSecondsVal(val string) string { + return strings.TrimRight(val, "s") +} + +func fromStringInt64(val string) (int64, error) { + return strconv.ParseInt(val, 10, 64) +} + +func fromStringFloat64(val string) (float64, error) { + return strconv.ParseFloat(val, 64) +} diff --git a/go/performance/utils/sysbench_runner/results_test.go b/go/performance/utils/sysbench_runner/results_test.go new file mode 100644 index 0000000000..b1e3cb5c34 --- /dev/null +++ b/go/performance/utils/sysbench_runner/results_test.go @@ -0,0 +1,327 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +var ( + testId = "test-id" + testSuiteId = "test-suite-id" + testOS = "test-runtime-os" + testGoArch = "test-runtime-goarch" + testServer = "test-server-name" + testServerVersion = "test-version" + testServerParams = "--test-server-param=1" + testTestName = "test-generated-name" + testTestParams = "--test-param=2" + testStamp = time.Date(2019, 8, 19, 0, 0, 0, 0, time.UTC).Format(stampFormat) +) + +// getRandomResult returns a result with random numbers +func genRandomResult() *Result { + return &Result{ + RuntimeOS: testOS, + RuntimeGoArch: testGoArch, + ServerName: testServer, + ServerParams: testServerParams, + TestName: testTestName, + TestParams: testTestParams, + CreatedAt: testStamp, + SqlReadQueries: rand.Int63(), + SqlWriteQueries: rand.Int63(), + SqlOtherQueries: rand.Int63(), + SqlTotalQueries: rand.Int63(), + SqlTotalQueriesPerSecond: rand.Float64(), + TransactionsTotal: rand.Int63(), + TransactionsPerSecond: rand.Float64(), + IgnoredErrorsTotal: rand.Int63(), + IgnoredErrorsPerSecond: rand.Float64(), + ReconnectsTotal: rand.Int63(), + ReconnectsPerSecond: rand.Float64(), + TotalTimeSeconds: rand.Float64(), + TotalNumberOfEvents: rand.Int63(), + LatencyMinMS: rand.Float64(), + LatencyAvgMS: rand.Float64(), + LatencyMaxMS: rand.Float64(), + LatencyPercentile: rand.Float64(), + LatencySumMS: rand.Float64(), + } +} + +func TestFromValWithParens(t *testing.T) { + tests := []struct { + description string + line string + expectedOne string + expectedTwo string + expectedError error + }{ + { + description: "should return no vals from empty line", + line: "", + expectedOne: "", + expectedTwo: "", + }, + { + description: "should return outside val from line without parens", + line: " 10123 ", + expectedOne: "10123", + expectedTwo: "", + }, + { + description: "should return outside val from line with parens", + line: " 708 ()", + expectedOne: "708", + expectedTwo: "", + }, + { + description: "should return inside val from line with parens", + line: "(1131.47 per sec.)", + expectedOne: "", + expectedTwo: "1131.47 per sec.", + }, + { + description: "should return vals from line with parens", + line: " 0 (0.00 per sec.)", + expectedOne: "0", + expectedTwo: "0.00 per sec.", + }, + { + description: "should error if line is incorrectly formatted", + line: "12 () 13 (24) 1 (255)", + expectedOne: "", + expectedTwo: "", + expectedError: ErrUnableToParseOutput, + }, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + actualOne, actualTwo, err := FromValWithParens(test.line) + assert.Equal(t, test.expectedOne, actualOne) + assert.Equal(t, test.expectedTwo, actualTwo) + assert.Equal(t, test.expectedError, err) + }) + } +} + +func TestUpdateResults(t *testing.T) { + tests := []struct { + description string + line string + expectedResult *Result + expectedError error + }{ + { + description: "should update read queries", + line: " read: 9912", + expectedResult: &Result{SqlReadQueries: 9912}, + }, + { + description: "should update transactions", + line: "transactions: 708 (70.72 per sec.)", + expectedResult: &Result{TransactionsTotal: 708, TransactionsPerSecond: 70.72}, + }, + { + description: "should update total queries per second", + line: "queries: 11328 (1131.47 per sec.)", + expectedResult: &Result{SqlTotalQueriesPerSecond: 1131.47}, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + actual := &Result{} + err := UpdateResult(actual, test.line) + assert.Equal(t, test.expectedResult, actual) + assert.Equal(t, test.expectedError, err) + }) + } +} + +var sampleOutput1 = ` +Running the test with following options: +Number of threads: 1 +Initializing random number generator from seed (1). + + +Initializing worker threads... + +Threads started! + +SQL statistics: + queries performed: + read: 9464 + write: 0 + other: 1352 + total: 10816 + transactions: 676 (67.57 per sec.) + queries: 10816 (1081.14 per sec.) + ignored errors: 0 (0.00 per sec.) + reconnects: 0 (0.00 per sec.) + +General statistics: + total time: 10.0030s + total number of events: 676 + +Latency (ms): + min: 12.26 + avg: 14.79 + max: 26.08 + 95th percentile: 17.01 + sum: 10000.87 + +Threads fairness: + events (avg/stddev): 676.0000/0.00 + execution time (avg/stddev): 10.0009/0.00 + + +sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) +` + +// TODO replace number of threads/seed +var SysbenchOutputTemplate = ` +Running the test with following options: +Number of threads: 1 +Initializing random number generator from seed (1). + + +Initializing worker threads... + +Threads started! + +SQL statistics: + queries performed: + read: %d + write: %d + other: %d + total: %d + transactions: %d (%.2f per sec.) + queries: %d (%.2f per sec.) + ignored errors: %d (%.2f per sec.) + reconnects: %d (%.2f per sec.) + +General statistics: + total time: %.2fs + total number of events: %d + +Latency (ms): + min: %.2f + avg: %.2f + max: %.2f + 95th percentile: %.2f + sum: %.2f + +Threads fairness: + events (avg/stddev): 676.0000/0.00 + execution time (avg/stddev): 10.0009/0.00 + + +sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) +` + +// fromResultSysbenchOutput returns sysbench output based on the given Result +func fromResultSysbenchOutput(r *Result) string { + return fmt.Sprintf(SysbenchOutputTemplate, + r.SqlReadQueries, + r.SqlWriteQueries, + r.SqlOtherQueries, + r.SqlTotalQueries, + r.TransactionsTotal, + r.TransactionsPerSecond, + r.SqlTotalQueries, + r.SqlTotalQueriesPerSecond, + r.IgnoredErrorsTotal, + r.IgnoredErrorsPerSecond, + r.ReconnectsTotal, + r.ReconnectsPerSecond, + r.TotalTimeSeconds, + r.TotalNumberOfEvents, + r.LatencyMinMS, + r.LatencyAvgMS, + r.LatencyMaxMS, + r.LatencyPercentile, + r.LatencySumMS) +} + +func TestFromOutputResults(t *testing.T) { + tests := []struct { + description string + output []byte + config *Config + serverConfig *ServerConfig + test *Test + expectedResult *Result + expectedError error + }{ + { + description: "should parse output into result", + output: []byte(sampleOutput1), + config: &Config{ + RuntimeOS: testOS, + RuntimeGoArch: testGoArch, + }, + serverConfig: &ServerConfig{ + Host: "localhost", + Server: ServerType(testServer), + Version: testServerVersion, + ServerExec: "test-exec", + }, + test: &Test{ + Name: testTestName, + Params: []string{testTestParams}, + }, + expectedResult: &Result{ + Id: testId, + SuiteId: testSuiteId, + RuntimeOS: testOS, + RuntimeGoArch: testGoArch, + ServerName: testServer, + ServerVersion: testServerVersion, + TestName: testTestName, + TestParams: testTestParams, + SqlReadQueries: 9464, + SqlWriteQueries: 0, + SqlOtherQueries: 1352, + SqlTotalQueries: 10816, + SqlTotalQueriesPerSecond: 1081.14, + TransactionsTotal: 676, + TransactionsPerSecond: 67.57, + TotalTimeSeconds: 10.0030, + TotalNumberOfEvents: 676, + LatencyMinMS: 12.26, + LatencyAvgMS: 14.79, + LatencyMaxMS: 26.08, + LatencyPercentile: 17.01, + LatencySumMS: 10000.87, + }, + }, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + actual, err := FromOutputResult(test.output, test.config, test.serverConfig, test.test, testSuiteId, func() string { + return testId + }) + assert.Equal(t, test.expectedError, err) + assert.Equal(t, test.expectedResult, actual) + }) + } +} diff --git a/go/performance/utils/sysbench_runner/run.go b/go/performance/utils/sysbench_runner/run.go new file mode 100644 index 0000000000..61ba064422 --- /dev/null +++ b/go/performance/utils/sysbench_runner/run.go @@ -0,0 +1,92 @@ +// Copyright 2019-2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysbench_runner + +import ( + "context" + "fmt" + "os" + "path/filepath" +) + +// Run runs sysbench runner +func Run(config *Config) error { + err := config.Validate() + if err != nil { + return err + } + + ctx := context.Background() + + err = sysbenchVersion(ctx) + if err != nil { + return err + } + + for _, serverConfig := range config.Servers { + var results Results + switch serverConfig.Server { + case Dolt: + results, err = BenchmarkDolt(ctx, config, serverConfig) + case MySql: + results, err = BenchmarkMysql(ctx, config, serverConfig) + default: + panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server)) + } + if err != nil { + return err + } + err = writeResults(serverConfig, results) + if err != nil { + return err + } + } + return nil +} + +func sysbenchVersion(ctx context.Context) error { + sysbenchVersion := ExecCommand(ctx, "sysbench", "--version") + return sysbenchVersion.Run() +} + +func writeResults(serverConfig *ServerConfig, results Results) error { + cwd, err := os.Getwd() + if err != nil { + return err + } + var writePath string + switch serverConfig.ResultsFormat { + case CsvFormat, CsvExt: + writePath = filepath.Join( + cwd, + "results", + string(serverConfig.Server), + serverConfig.Version, + serverConfig.GetId(), + fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, CsvExt)) + return WriteResultsCsv(writePath, results) + case JsonFormat, JsonExt: + writePath = filepath.Join( + cwd, + "results", + string(serverConfig.Server), + serverConfig.Version, + serverConfig.GetId(), + fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, JsonExt)) + return WriteResultsJson(writePath, results) + default: + } + return fmt.Errorf("unsupported results format: %s", serverConfig.ResultsFormat) +}