Merge pull request #7493 from dolthub/db/refactor

[no-release-notes] refactor sysbench_runner in preparation for pgo dolt releases

.github/workflows/ci-sysbench-runner-tests.yaml

@@ -1,24 +1,24 @@
-name: Test Sysbench Runner Utility Works
-
-on:
-  pull_request:
-    branches: [ main ]
-    paths:
-      - 'go/**'
-      - 'integration-tests/**'
-
-concurrency:
-  group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  mysql_client_integrations_job:
-    runs-on: ubuntu-22.04
-    name: Test Sysbench Runner
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Copy Dockerfile
-        run: cp -r ./go/performance/continuous_integration/. .
-      - name: Test sysbench runner
-        uses: ./.github/actions/sysbench-runner-tests
+#name: Test Sysbench Runner Utility Works
+#
+#on:
+#  pull_request:
+#    branches: [ main ]
+#    paths:
+#      - 'go/**'
+#      - 'integration-tests/**'
+#
+#concurrency:
+#  group: ci-sysbench-runner-tests-${{ github.event.pull_request.number || github.ref }}
+#  cancel-in-progress: true
+#
+#jobs:
+#  mysql_client_integrations_job:
+#    runs-on: ubuntu-22.04
+#    name: Test Sysbench Runner
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v3
+#      - name: Copy Dockerfile
+#        run: cp -r ./go/performance/continuous_integration/. .
+#      - name: Test sysbench runner
+#        uses: ./.github/actions/sysbench-runner-tests

go/performance/utils/benchmark_runner/README.md

@@ -1,11 +1,6 @@
 Sysbench runner is a tool for running sysbench tests against sql servers. Custom sysbench lua scripts used
 for benchmarking Dolt are [here](https://github.com/dolthub/sysbench-lua-scripts).
 
-The tool requires a json config file to run:
-```bash
-$ sysbench_runner --config=config.json
-```
-
 Configuration:
 
 ```json
@@ -62,8 +57,6 @@ oltp_update_non_index
 
 `Port` is the server port. Defaults to **3306** for `dolt` and `mysql` Servers. (**Optional**)
 
-`Server` is the server. Only `dolt` and `mysql` are supported. (**Required**)
-
 `Version` is the server version. (**Required**)
 
 `ResultsFormat` is the format the results should be written in. Only `json` and `csv` are supported. (**Required**)
@@ -99,3 +92,33 @@ oltp_update_non_index
 `sysbench [options]... [testname] [command]`
 
 Note: Be sure that all mysql processes are off when running this locally.
+
+# TPCC
+
+TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the
+Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc).
+
+Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked
+to the path of the cloned repo.
+
+Configuration:
+
+```json
+{
+  "Servers": "[...]",
+  "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc",
+  "ScaleFactors": [1]
+}
+```
+
+`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn.
+
+`ScriptDir`: The directory of the TPCC testing scripts
+
+`ScaleFactors`: The number of warehouse to be generated in the test case.
+
+`NomsBinFormat`: The NomsBinFormat to use for this benchmark.
+
+Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we
+want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the
+variables TPCC varies.

go/performance/utils/benchmark_runner/benchmark.go

@@ -0,0 +1,21 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "context"
+
+type Benchmarker interface {
+	Benchmark(ctx context.Context) (Results, error)
+}

go/performance/utils/benchmark_runner/config.go

@@ -0,0 +1,39 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "context"
+
+type Config interface {
+	GetRuns() int
+	GetScriptDir() string
+	GetNomsBinFormat() string
+	GetRuntimeOs() string
+	GetRuntimeGoArch() string
+	GetServerConfigs() []ServerConfig
+	Validate(ctx context.Context) error
+	ContainsServerOfType(server ServerType) bool
+}
+
+type SysbenchConfig interface {
+	Config
+	GetTestOptions() []string
+	GetTestConfigs() []TestConfig
+}
+
+type TpccConfig interface {
+	Config
+	GetScaleFactors() []int
+}

go/performance/utils/benchmark_runner/constants.go

@@ -0,0 +1,209 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "time"
+
+const (
+	Dolt     ServerType = "dolt"
+	Doltgres ServerType = "doltgres"
+	Postgres ServerType = "postgres"
+	MySql    ServerType = "mysql"
+
+	CsvFormat  = "csv"
+	JsonFormat = "json"
+
+	CsvExt  = ".csv"
+	JsonExt = ".json"
+
+	CpuServerProfile ServerProfile = "cpu"
+
+	defaultHost         = "127.0.0.1"
+	defaultDoltPort     = 3306
+	defaultMysqlPort    = defaultDoltPort
+	defaultDoltgresPort = 5432
+	defaultPostgresPort = defaultDoltgresPort
+
+	defaultMysqlSocket = "/var/run/mysqld/mysqld.sock"
+
+	tcpProtocol  = "tcp"
+	unixProtocol = "unix"
+
+	sysbenchUsername          = "sysbench"
+	sysbenchUserLocal         = "'sysbench'@'localhost'"
+	sysbenchPassLocal         = "sysbenchpass"
+	sysbenchDbPsModeFlag      = "--db-ps-mode"
+	sysbenchDbPsModeDisable   = "disable"
+	sysbenchRandTypeFlag      = "--rand-type"
+	sysbenchRandTypeUniform   = "uniform"
+	sysbenchMysqlDbFlag       = "--mysql-db"
+	sysbenchDbDriverFlag      = "--db-driver"
+	sysbenchMysqlHostFlag     = "--mysql-host"
+	sysbenchMysqlPortFlag     = "--mysql-port"
+	sysbenchMysqlUserFlag     = "--mysql-user"
+	sysbenchMysqlPasswordFlag = "--mysql-password"
+	sysbenchPostgresDbDriver  = "pgsql"
+	sysbenchPostgresDbFlag    = "--pgsql-db"
+	sysbenchPostgresHostFlag  = "--pgsql-host"
+	sysbenchPostgresPortFlag  = "--pgsql-port"
+	sysbenchPostgresUserFlag  = "--pgsql-user"
+
+	doltSqlServerCommand = "sql-server"
+
+	userFlag                    = "--user"
+	hostFlag                    = "--host"
+	portFlag                    = "--port"
+	skipBinLogFlag              = "--skip-log-bin"
+	profileFlag                 = "--prof"
+	profilePathFlag             = "--prof-path"
+	cpuProfile                  = "cpu"
+	doltgresDataDirFlag         = "--data-dir"
+	MysqlDataDirFlag            = "--datadir"
+	MysqlInitializeInsecureFlag = "--initialize-insecure"
+	cpuProfileFilename          = "cpu.pprof"
+
+	sysbenchOltpReadOnlyTestName       = "oltp_read_only"
+	sysbenchOltpInsertTestName         = "oltp_insert"
+	sysbenchBulkInsertTestName         = "bulk_insert"
+	sysbenchOltpPointSelectTestName    = "oltp_point_select"
+	sysbenchSelectRandomPointsTestName = "select_random_points"
+	sysbenchSelectRandomRangesTestName = "select_random_ranges"
+	sysbenchOltpWriteOnlyTestName      = "oltp_write_only"
+	sysbenchOltpReadWriteTestName      = "oltp_read_write"
+	sysbenchOltpUpdateIndexTestName    = "oltp_update_index"
+	sysbenchOltpUpdateNonIndexTestName = "oltp_update_non_index"
+
+	sysbenchCoveringIndexScanLuaTestName = "covering_index_scan.lua"
+	sysbenchGroupByScanLuaTestName       = "groupby_scan.lua"
+	sysbenchIndexJoinLuaTestName         = "index_join.lua"
+	sysbenchIndexJoinScanLuaTestName     = "index_join_scan.lua"
+	sysbenchIndexScanLuaTestName         = "index_scan.lua"
+	sysbenchOltpDeleteInsertLuaTestName  = "oltp_delete_insert.lua"
+	sysbenchTableScanLuaTestName         = "table_scan.lua"
+	sysbenchTypesDeleteInsertLuaTestName = "types_delete_insert.lua"
+	sysbenchTypesTableScanLuaTestName    = "types_table_scan.lua"
+
+	sysbenchCoveringIndexScanPostgresLuaTestName = "covering_index_scan_postgres.lua"
+	sysbenchGroupByScanPostgresLuaTestName       = "groupby_scan_postgres.lua"
+	sysbenchIndexJoinPostgresLuaTestName         = "index_join_postgres.lua"
+	sysbenchIndexJoinScanPostgresLuaTestName     = "index_join_scan_postgres.lua"
+	sysbenchIndexScanPostgresLuaTestName         = "index_scan_postgres.lua"
+	sysbenchOltpDeleteInsertPostgresLuaTestName  = "oltp_delete_insert_postgres.lua"
+	sysbenchTableScanPostgresLuaTestName         = "table_scan_postgres.lua"
+	sysbenchTypesDeleteInsertPostgresLuaTestName = "types_delete_insert_postgres.lua"
+	sysbenchTypesTableScanPostgresLuaTestName    = "types_table_scan_postgres.lua"
+
+	doltConfigUsernameKey = "user.name"
+	doltConfigEmailKey    = "user.email"
+	doltBenchmarkUser     = "benchmark"
+	doltBenchmarkEmail    = "benchmark@dolthub.com"
+	doltConfigCommand     = "config"
+	doltConfigGlobalFlag  = "--global"
+	doltConfigGetFlag     = "--get"
+	doltConfigAddFlag     = "--add"
+	doltCloneCommand      = "clone"
+	doltVersionCommand    = "version"
+	doltInitCommand       = "init"
+	dbName                = "test"
+	bigEmptyRepo          = "max-hoffman/big-empty"
+	nbfEnvVar             = "DOLT_DEFAULT_BIN_FORMAT"
+
+	postgresDriver         = "postgres"
+	doltgresUser           = "doltgres"
+	doltDataDir            = ".dolt"
+	createDatabaseTemplate = "create database %s;"
+	psqlDsnTemplate        = "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable"
+
+	expectedServerKilledErrorMessage     = "signal: killed"
+	expectedServerTerminatedErrorMessage = "signal: terminated"
+
+	sysbenchCommand        = "sysbench"
+	sysbenchVersionFlag    = "--version"
+	sysbenchPrepareCommand = "prepare"
+	sysbenchRunCommand     = "run"
+	sysbenchCleanupCommand = "cleanup"
+	luaPathEnvVarTemplate  = "LUA_PATH=%s"
+	luaPath                = "?.lua"
+
+	defaultMysqlUser = "root"
+
+	// Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables
+	// for your local MySQL setup.
+	tpccUserLocal = "'sysbench'@'localhost'"
+	tpccPassLocal = "sysbenchpass"
+
+	tpccDbName              = "sbt"
+	tpccScaleFactorTemplate = "tpcc-scale-factor-%d"
+
+	tpccDbDriverFlag         = "--db-driver"
+	tpccMysqlUsername        = "sysbench"
+	tpccMysqlDbFlag          = "--mysql-db"
+	tpccMysqlHostFlag        = "--mysql-host"
+	tpccMysqlUserFlag        = "--mysql-user"
+	tpccMysqlPasswordFlag    = "--mysql-password"
+	tpccMysqlPortFlag        = "--mysql-port"
+	tpccTimeFlag             = "--time"
+	tpccThreadsFlag          = "--threads"
+	tpccReportIntervalFlag   = "--report_interval"
+	tpccTablesFlag           = "--tables"
+	tpccScaleFlag            = "--scale"
+	tpccTransactionLevelFlag = "--trx_level"
+	tpccReportCsv            = "reportCsv"
+	tpccTransactionLevelRr   = "RR"
+	tpccLuaFilename          = "tpcc.lua"
+
+	mysqlDriverName                  = "mysql"
+	mysqlRootTCPDsnTemplate          = "root@tcp(%s:%d)/"
+	mysqlRootUnixDsnTemplate         = "root@unix(%s)/"
+	mysqlDropDatabaseSqlTemplate     = "DROP DATABASE IF EXISTS %s;"
+	mysqlCreateDatabaseSqlTemplate   = "CREATE DATABASE %s;"
+	mysqlDropUserSqlTemplate         = "DROP USER IF EXISTS %s;"
+	mysqlCreateUserSqlTemplate       = "CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s';"
+	mysqlGrantPermissionsSqlTemplate = "GRANT ALL ON %s.* to %s;"
+	mysqlSetGlobalLocalInfileSql     = "SET GLOBAL local_infile = 'ON';"
+	mysqlSetGlobalSqlModeSql         = "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));"
+
+	postgresInitDbDataDirFlag         = "--pgdata"
+	postgresUsernameFlag              = "--username"
+	postgresUsername                  = "postgres"
+	postgresDataDirFlag               = "-D"
+	postgresDropDatabaseSqlTemplate   = "DROP DATABASE IF EXISTS %s;"
+	postgresDropUserSqlTemplate       = "DROP USER IF EXISTS %s;"
+	postgresCreateUserSqlTemplate     = "CREATE USER %s WITH PASSWORD '%s';"
+	postgresCreateDatabaseSqlTemplate = "CREATE DATABASE %s WITH OWNER %s;"
+	postgresLcAllEnvVarKey            = "LC_ALL"
+	postgresLcAllEnvVarValue          = "C"
+
+	resultsDirname = "results"
+	stampFormat    = time.RFC3339
+	SqlStatsPrefix = "SQL statistics:"
+	read           = "read"
+	write          = "write"
+	other          = "other"
+	totalQueries   = "total"
+	totalEvents    = "total number of events"
+	min            = "min"
+	avg            = "avg"
+	max            = "max"
+	percentile     = "percentile"
+	sum            = "sum"
+	transactions   = "transactions"
+	queriesPerSec  = "queries"
+	ignoredErrors  = "ignored errors"
+	reconnects     = "reconnects"
+	totalTimeSecs  = "total time"
+
+	ResultFileTemplate = "%s_%s_%s_sysbench_performance%s"
+)

go/performance/utils/benchmark_runner/csv.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"encoding/csv"
@@ -26,7 +26,7 @@ import (
 // FromResultCsvHeaders returns supported csv headers for a Result
 func FromResultCsvHeaders() []string {
 	return []string{
-		"id",
+		"id", // todo: replace with constants
 		"suite_id",
 		"test_id",
 		"runtime_os",

go/performance/utils/benchmark_runner/csv_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"fmt"

go/performance/utils/benchmark_runner/debug.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"context"

go/performance/utils/benchmark_runner/dolt.go

@@ -0,0 +1,172 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/dolthub/dolt/go/store/types"
+)
+
+var ErrNotSysbenchTest = errors.New("sysbench test is required")
+
+var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) }
+
+type doltBenchmarkerImpl struct {
+	dir          string // cwd
+	config       SysbenchConfig
+	serverConfig ServerConfig
+}
+
+var _ Benchmarker = &doltBenchmarkerImpl{}
+
+func NewDoltBenchmarker(dir string, config SysbenchConfig, serverConfig ServerConfig) *doltBenchmarkerImpl {
+	return &doltBenchmarkerImpl{
+		dir:          dir,
+		config:       config,
+		serverConfig: serverConfig,
+	}
+}
+
+func (b *doltBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error {
+	err := CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser)
+	if err != nil {
+		return err
+	}
+	return CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail)
+}
+
+func (b *doltBenchmarkerImpl) checkInstallation(ctx context.Context) error {
+	version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand)
+	return version.Run()
+}
+
+func (b *doltBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) {
+	return InitDoltRepo(ctx, b.dir, b.serverConfig.GetServerExec(), b.config.GetNomsBinFormat(), dbName)
+}
+
+func (b *doltBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) {
+	err := b.checkInstallation(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	err = b.updateGlobalConfig(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	testRepo, err := b.initDoltRepo(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(testRepo)
+
+	serverParams, err := b.serverConfig.GetServerArgs()
+	if err != nil {
+		return nil, err
+	}
+
+	server := NewServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams)
+	err = server.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	tests, err := GetTests(b.config, b.serverConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	results := make(Results, 0)
+	runs := b.config.GetRuns()
+	for i := 0; i < runs; i++ {
+		for _, test := range tests {
+			t, ok := test.(SysbenchTest)
+			if !ok {
+				return nil, ErrNotSysbenchTest
+			}
+			tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc)
+			r, err := tester.Test(ctx)
+			if err != nil {
+				server.Stop()
+				return nil, err
+			}
+			results = append(results, r)
+		}
+	}
+
+	err = server.Stop()
+	if err != nil {
+		return nil, err
+	}
+
+	return results, nil
+}
+
+// InitDoltRepo initializes a dolt database and returns its path
+func InitDoltRepo(ctx context.Context, dir, serverExec, nomsBinFormat, dbName string) (string, error) {
+	testRepo := filepath.Join(dir, dbName)
+	if nomsBinFormat == types.Format_LD_1.VersionString() {
+		err := ExecCommand(ctx, serverExec, doltCloneCommand, bigEmptyRepo, dbName).Run()
+		if err != nil {
+			return "", err
+		}
+		return testRepo, nil
+	}
+
+	err := os.MkdirAll(testRepo, os.ModePerm)
+	if err != nil {
+		return "", err
+	}
+
+	if nomsBinFormat != "" {
+		if err = os.Setenv(nbfEnvVar, nomsBinFormat); err != nil {
+			return "", err
+		}
+	}
+
+	doltInit := ExecCommand(ctx, serverExec, doltInitCommand)
+	doltInit.Dir = testRepo
+	err = doltInit.Run()
+	if err != nil {
+		return "", err
+	}
+
+	return testRepo, nil
+}
+
+// CheckSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary
+func CheckSetDoltConfig(ctx context.Context, serverExec, key, val string) error {
+	check := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigGetFlag, key)
+	err := check.Run()
+	if err != nil {
+		// config get calls exit with 1 if not set
+		if err.Error() != "exit status 1" {
+			return err
+		}
+		set := ExecCommand(ctx, serverExec, doltConfigCommand, doltConfigGlobalFlag, doltConfigAddFlag, key, val)
+		err := set.Run()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

go/performance/utils/benchmark_runner/dolt_server_config.go

@@ -0,0 +1,175 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/google/uuid"
+)
+
+type ServerProfile string
+
+type doltServerConfigImpl struct {
+	// Id is a unique id for this servers benchmarking
+	Id string
+
+	// Host is the server host
+	Host string
+
+	// Port is the server port
+	Port int
+
+	// Version is the server version
+	Version string
+
+	// ResultsFormat is the format the results should be written in
+	ResultsFormat string
+
+	// ServerExec is the path to a server executable
+	ServerExec string
+
+	// ServerUser is the user account that should start the server
+	ServerUser string
+
+	// ServerArgs are the args used to start a server
+	ServerArgs []string
+
+	// ServerProfile specifies the golang profile to take of a Dolt server
+	ServerProfile ServerProfile
+
+	// ProfilePath path to directory where server profile will be written
+	ProfilePath string
+}
+
+var _ ProfilingServerConfig = &doltServerConfigImpl{}
+
+func NewDoltServerConfig(version, serverExec, serverUser, host, resultsFormat, profilePath string, serverProfile ServerProfile, port int, serverArgs []string) *doltServerConfigImpl {
+	return &doltServerConfigImpl{
+		Id:            uuid.New().String(),
+		Host:          host,
+		Port:          port,
+		Version:       version,
+		ResultsFormat: resultsFormat,
+		ServerExec:    serverExec,
+		ServerUser:    serverUser,
+		ServerArgs:    serverArgs,
+		ServerProfile: serverProfile,
+		ProfilePath:   profilePath,
+	}
+}
+
+func (sc *doltServerConfigImpl) GetId() string {
+	return sc.Id
+}
+
+func (sc *doltServerConfigImpl) GetHost() string {
+	return sc.Host
+}
+
+func (sc *doltServerConfigImpl) GetPort() int {
+	return sc.Port
+}
+
+func (sc *doltServerConfigImpl) GetVersion() string {
+	return sc.Version
+}
+
+func (sc *doltServerConfigImpl) GetProfilePath() string {
+	return sc.ProfilePath
+}
+
+func (sc *doltServerConfigImpl) GetServerProfile() ServerProfile {
+	return sc.ServerProfile
+}
+
+func (sc *doltServerConfigImpl) GetServerType() ServerType {
+	return Dolt
+}
+
+func (sc *doltServerConfigImpl) GetResultsFormat() string {
+	return sc.ResultsFormat
+}
+
+func (sc *doltServerConfigImpl) GetServerExec() string {
+	return sc.ServerExec
+}
+
+// GetServerArgs returns the args used to start a server
+func (sc *doltServerConfigImpl) GetServerArgs() ([]string, error) {
+	params := make([]string, 0)
+	params = append(params, defaultDoltServerParams...)
+	if sc.Host != "" {
+		params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host))
+	}
+	if sc.Port != 0 {
+		params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port))
+	}
+	params = append(params, sc.ServerArgs...)
+	return params, nil
+}
+
+func (sc *doltServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams {
+	params := NewSysbenchTestParams()
+	params.Append(defaultSysbenchParams...)
+	params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlDbFlag, dbName))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, mysqlDriverName))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlHostFlag, sc.Host))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlUserFlag, defaultMysqlUser))
+	if sc.Port != 0 {
+		params.Append(fmt.Sprintf("%s=%d", sysbenchMysqlPortFlag, sc.Port))
+	}
+	params.Append(testConfig.GetOptions()...)
+	params.Append(testConfig.GetName())
+	return params
+}
+
+func (sc *doltServerConfigImpl) Validate() error {
+	if sc.Version == "" {
+		return getMustSupplyError("version")
+	}
+	if sc.ResultsFormat == "" {
+		return getMustSupplyError("results format")
+	}
+	if sc.ServerExec == "" {
+		return getMustSupplyError("server exec")
+	}
+	if sc.ServerProfile != "" {
+		if sc.ServerProfile != CpuServerProfile {
+			return fmt.Errorf("unsupported server profile: %s", sc.ServerProfile)
+		}
+	}
+	return CheckExec(sc.ServerExec, "server exec")
+}
+
+func (sc *doltServerConfigImpl) SetDefaults() error {
+	if sc.Host == "" {
+		sc.Host = defaultHost
+	}
+	if sc.Port < 1 {
+		sc.Port = defaultDoltPort
+	}
+	if sc.ServerProfile != "" {
+		if sc.ProfilePath == "" {
+			cwd, err := os.Getwd()
+			if err != nil {
+				return err
+			}
+			sc.ProfilePath = cwd
+		}
+	}
+	return nil
+}

go/performance/utils/benchmark_runner/dolt_tpcc.go

@@ -0,0 +1,116 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"syscall"
+)
+
+type doltTpccBenchmarkerImpl struct {
+	dir          string // cwd
+	config       TpccConfig
+	serverConfig ServerConfig
+}
+
+var _ Benchmarker = &doltTpccBenchmarkerImpl{}
+
+func NewDoltTpccBenchmarker(dir string, config TpccConfig, serverConfig ServerConfig) *doltTpccBenchmarkerImpl {
+	return &doltTpccBenchmarkerImpl{
+		dir:          dir,
+		config:       config,
+		serverConfig: serverConfig,
+	}
+}
+
+func (b *doltTpccBenchmarkerImpl) updateGlobalConfig(ctx context.Context) error {
+	err := CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser)
+	if err != nil {
+		return err
+	}
+	return CheckSetDoltConfig(ctx, b.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail)
+}
+
+func (b *doltTpccBenchmarkerImpl) checkInstallation(ctx context.Context) error {
+	version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand)
+	return version.Run()
+}
+
+func (b *doltTpccBenchmarkerImpl) initDoltRepo(ctx context.Context) (string, error) {
+	return InitDoltRepo(ctx, b.dir, b.serverConfig.GetServerExec(), b.config.GetNomsBinFormat(), tpccDbName)
+}
+
+func (b *doltTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) {
+	err := b.checkInstallation(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	err = b.updateGlobalConfig(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	testRepo, err := b.initDoltRepo(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(testRepo)
+
+	serverParams, err := b.serverConfig.GetServerArgs()
+	if err != nil {
+		return nil, err
+	}
+
+	server := NewServer(ctx, testRepo, b.serverConfig, syscall.SIGTERM, serverParams)
+	err = server.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	tests := GetTpccTests(b.config)
+
+	results := make(Results, 0)
+	for _, test := range tests {
+		tester := NewTpccTester(b.config, b.serverConfig, test, serverParams, stampFunc)
+		r, err := tester.Test(ctx)
+		if err != nil {
+			server.Stop()
+			return nil, err
+		}
+		results = append(results, r)
+	}
+
+	err = server.Stop()
+	if err != nil {
+		return nil, err
+	}
+
+	return results, nil
+}
+
+// GetTpccTests creates a set of tests that the server needs to be executed on.
+func GetTpccTests(config TpccConfig) []Test {
+	tests := make([]Test, 0)
+	for _, sf := range config.GetScaleFactors() {
+		params := NewDefaultTpccParams()
+		params.ScaleFactor = sf
+		test := NewTpccTest(fmt.Sprintf(tpccScaleFactorTemplate, sf), params)
+		tests = append(tests, test)
+	}
+	return tests
+}

go/performance/utils/benchmark_runner/doltgres.go

@@ -0,0 +1,172 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+type doltgresBenchmarkerImpl struct {
+	dir          string // cwd
+	config       SysbenchConfig
+	serverConfig ServerConfig
+}
+
+var _ Benchmarker = &doltgresBenchmarkerImpl{}
+
+func NewDoltgresBenchmarker(dir string, config SysbenchConfig, serverConfig ServerConfig) *doltgresBenchmarkerImpl {
+	return &doltgresBenchmarkerImpl{
+		dir:          dir,
+		config:       config,
+		serverConfig: serverConfig,
+	}
+}
+
+func (b *doltgresBenchmarkerImpl) checkInstallation(ctx context.Context) error {
+	version := ExecCommand(ctx, b.serverConfig.GetServerExec(), doltVersionCommand)
+	return version.Run()
+}
+
+func (b *doltgresBenchmarkerImpl) createServerDir() (string, error) {
+	return CreateServerDir(dbName)
+}
+
+func (b *doltgresBenchmarkerImpl) cleanupServerDir(dir string) error {
+	dataDir := filepath.Join(dir, doltDataDir)
+	defaultDir := filepath.Join(dir, doltgresUser)
+	testDir := filepath.Join(dir, dbName)
+	for _, d := range []string{dataDir, defaultDir, testDir} {
+		if _, err := os.Stat(d); !os.IsNotExist(err) {
+			err = os.RemoveAll(d)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (b *doltgresBenchmarkerImpl) createTestingDb(ctx context.Context) error {
+	psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.GetHost(), b.serverConfig.GetPort(), doltgresUser, "", dbName)
+
+	// open database
+	db, err := sql.Open(postgresDriver, psqlconn)
+	if err != nil {
+		return err
+	}
+
+	// close database
+	defer db.Close()
+
+	// check db
+	err = db.PingContext(ctx)
+	if err != nil {
+		return err
+	}
+
+	_, err = db.ExecContext(ctx, fmt.Sprintf(createDatabaseTemplate, dbName))
+	return err
+}
+
+func (b *doltgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Results, err error) {
+	err = b.checkInstallation(ctx)
+	if err != nil {
+		return
+	}
+
+	var serverDir string
+	serverDir, err = CreateServerDir(dbName)
+	if err != nil {
+		return
+	}
+	defer func() {
+		rerr := b.cleanupServerDir(serverDir)
+		if err == nil {
+			err = rerr
+		}
+	}()
+
+	var serverParams []string
+	serverParams, err = b.serverConfig.GetServerArgs()
+	if err != nil {
+		return
+	}
+
+	serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir))
+
+	server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams)
+	err = server.Start()
+	if err != nil {
+		return
+	}
+
+	err = b.createTestingDb(ctx)
+	if err != nil {
+		return
+	}
+
+	var tests []Test
+	tests, err = GetTests(b.config, b.serverConfig)
+	if err != nil {
+		return
+	}
+
+	results = make(Results, 0)
+	runs := b.config.GetRuns()
+	for i := 0; i < runs; i++ {
+		for _, test := range tests {
+			t, ok := test.(SysbenchTest)
+			if !ok {
+				return nil, ErrNotSysbenchTest
+			}
+			tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc)
+			var r *Result
+			r, err = tester.Test(ctx)
+			if err != nil {
+				server.Stop()
+				return
+			}
+			results = append(results, r)
+		}
+	}
+
+	err = server.Stop()
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// CreateServerDir creates a server directory
+func CreateServerDir(dbName string) (string, error) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	serverDir := filepath.Join(cwd, dbName)
+	err = os.MkdirAll(serverDir, os.ModePerm)
+	if err != nil {
+		return "", err
+	}
+
+	return serverDir, nil
+}

go/performance/utils/benchmark_runner/doltgres_server_config.go

@@ -0,0 +1,140 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"fmt"
+
+	"github.com/google/uuid"
+)
+
+type doltgresServerConfigImpl struct {
+	// Id is a unique id for this servers benchmarking
+	Id string
+
+	// Host is the server host
+	Host string
+
+	// Port is the server port
+	Port int
+
+	// Version is the server version
+	Version string
+
+	// ResultsFormat is the format the results should be written in
+	ResultsFormat string
+
+	// ServerExec is the path to a server executable
+	ServerExec string
+
+	// ServerUser is the user account that should start the server
+	ServerUser string
+
+	// ServerArgs are the args used to start a server
+	ServerArgs []string
+}
+
+var _ ServerConfig = &doltgresServerConfigImpl{}
+
+func NewDoltgresServerConfig(version, serverExec, serverUser, host, resultsFormat string, port int, serverArgs []string) *doltgresServerConfigImpl {
+	return &doltgresServerConfigImpl{
+		Id:            uuid.New().String(),
+		Host:          host,
+		Port:          port,
+		Version:       version,
+		ResultsFormat: resultsFormat,
+		ServerExec:    serverExec,
+		ServerUser:    serverUser,
+		ServerArgs:    serverArgs,
+	}
+}
+
+func (sc *doltgresServerConfigImpl) GetServerType() ServerType {
+	return Doltgres
+}
+
+func (sc *doltgresServerConfigImpl) GetServerExec() string {
+	return sc.ServerExec
+}
+
+func (sc *doltgresServerConfigImpl) GetResultsFormat() string {
+	return sc.ResultsFormat
+}
+
+func (sc *doltgresServerConfigImpl) GetServerArgs() ([]string, error) {
+	params := make([]string, 0)
+	if sc.Host != "" {
+		params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host))
+	}
+	if sc.Port != 0 {
+		params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port))
+	}
+	params = append(params, sc.ServerArgs...)
+	return params, nil
+}
+
+func (sc *doltgresServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams {
+	params := NewSysbenchTestParams()
+	params.Append(defaultSysbenchParams...)
+	params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, sysbenchPostgresDbDriver))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresDbFlag, dbName))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresHostFlag, sc.Host))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresUserFlag, doltgresUser))
+	if sc.Port != 0 {
+		params.Append(fmt.Sprintf("%s=%d", sysbenchPostgresPortFlag, sc.Port))
+	}
+	params.Append(testConfig.GetOptions()...)
+	params.Append(testConfig.GetName())
+	return params
+}
+
+func (sc *doltgresServerConfigImpl) Validate() error {
+	if sc.Version == "" {
+		return getMustSupplyError("version")
+	}
+	if sc.ResultsFormat == "" {
+		return getMustSupplyError("results format")
+	}
+	if sc.ServerExec == "" {
+		return getMustSupplyError("server exec")
+	}
+	return CheckExec(sc.ServerExec, "server exec")
+}
+
+func (sc *doltgresServerConfigImpl) SetDefaults() error {
+	if sc.Host == "" {
+		sc.Host = defaultHost
+	}
+	if sc.Port < 1 {
+		sc.Port = defaultDoltgresPort
+	}
+	return nil
+}
+
+func (sc *doltgresServerConfigImpl) GetId() string {
+	return sc.Id
+}
+
+func (sc *doltgresServerConfigImpl) GetHost() string {
+	return sc.Host
+}
+
+func (sc *doltgresServerConfigImpl) GetPort() int {
+	return sc.Port
+}
+
+func (sc *doltgresServerConfigImpl) GetVersion() string {
+	return sc.Version
+}

go/performance/utils/benchmark_runner/json.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"encoding/json"

go/performance/utils/benchmark_runner/json_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"fmt"

go/performance/utils/benchmark_runner/mysql.go

@@ -0,0 +1,177 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os"
+	"syscall"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type mysqlBenchmarkerImpl struct {
+	dir          string // cwd
+	config       SysbenchConfig
+	serverConfig ProtocolServerConfig
+}
+
+var _ Benchmarker = &mysqlBenchmarkerImpl{}
+
+func NewMysqlBenchmarker(dir string, config SysbenchConfig, serverConfig ProtocolServerConfig) *mysqlBenchmarkerImpl {
+	return &mysqlBenchmarkerImpl{
+		dir:          dir,
+		config:       config,
+		serverConfig: serverConfig,
+	}
+}
+
+func (b *mysqlBenchmarkerImpl) getDsn() (string, error) {
+	return GetMysqlDsn(b.serverConfig.GetHost(), b.serverConfig.GetSocket(), b.serverConfig.GetConnectionProtocol(), b.serverConfig.GetPort())
+}
+
+func (b *mysqlBenchmarkerImpl) createTestingDb(ctx context.Context) error {
+	dsn, err := b.getDsn()
+	if err != nil {
+		return err
+	}
+	return CreateMysqlTestingDb(ctx, dsn, dbName)
+}
+
+func (b *mysqlBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) {
+	serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.GetServerExec(), dbName)
+	if err != nil {
+		return nil, err
+	}
+
+	serverParams, err := b.serverConfig.GetServerArgs()
+	if err != nil {
+		return nil, err
+	}
+	serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir))
+
+	server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams)
+	err = server.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	err = b.createTestingDb(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	tests, err := GetTests(b.config, b.serverConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	results := make(Results, 0)
+	runs := b.config.GetRuns()
+	for i := 0; i < runs; i++ {
+		for _, test := range tests {
+			t, ok := test.(SysbenchTest)
+			if !ok {
+				return nil, ErrNotSysbenchTest
+			}
+			tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc)
+			r, err := tester.Test(ctx)
+			if err != nil {
+				server.Stop()
+				return nil, err
+			}
+			results = append(results, r)
+		}
+	}
+
+	err = server.Stop()
+	if err != nil {
+		return nil, err
+	}
+
+	return results, os.RemoveAll(serverDir)
+}
+
+func InitMysqlDataDir(ctx context.Context, serverExec, dbName string) (string, error) {
+	serverDir, err := CreateServerDir(dbName)
+	if err != nil {
+		return "", err
+	}
+
+	msInit := ExecCommand(ctx, serverExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir))
+	err = msInit.Run()
+	if err != nil {
+		return "", err
+	}
+
+	return serverDir, nil
+}
+
+func CreateMysqlTestingDb(ctx context.Context, dsn, dbName string) (err error) {
+	var db *sql.DB
+	db, err = sql.Open(mysqlDriverName, dsn)
+	if err != nil {
+		return
+	}
+	defer func() {
+		rerr := db.Close()
+		if err == nil {
+			err = rerr
+		}
+	}()
+
+	err = db.Ping()
+	if err != nil {
+		return
+	}
+
+	stmts := []string{
+		fmt.Sprintf(mysqlDropDatabaseSqlTemplate, dbName),
+		fmt.Sprintf(mysqlCreateDatabaseSqlTemplate, dbName),
+		fmt.Sprintf(mysqlDropUserSqlTemplate, sysbenchUserLocal),
+		fmt.Sprintf(mysqlCreateUserSqlTemplate, sysbenchUserLocal, sysbenchPassLocal),
+		fmt.Sprintf(mysqlGrantPermissionsSqlTemplate, dbName, sysbenchUserLocal),
+		mysqlSetGlobalLocalInfileSql,
+		mysqlSetGlobalSqlModeSql, // Required for running groupby_scan.lua without error
+	}
+
+	for _, s := range stmts {
+		_, err = db.ExecContext(ctx, s)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func GetMysqlDsn(host, socket, protocol string, port int) (string, error) {
+	var socketPath string
+	if socket != "" {
+		socketPath = socket
+	} else {
+		socketPath = defaultMysqlSocket
+	}
+
+	if protocol == tcpProtocol {
+		return fmt.Sprintf(mysqlRootTCPDsnTemplate, host, port), nil
+	} else if protocol == unixProtocol {
+		return fmt.Sprintf(mysqlRootUnixDsnTemplate, socketPath), nil
+	} else {
+		return "", ErrUnsupportedConnectionProtocol
+	}
+}

go/performance/utils/benchmark_runner/mysql_server_config.go

@@ -0,0 +1,168 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"fmt"
+
+	"github.com/google/uuid"
+)
+
+type mysqlServerConfigImpl struct {
+	// Id is a unique id for this servers benchmarking
+	Id string
+
+	// Host is the server host
+	Host string
+
+	// Port is the server port
+	Port int
+
+	// Version is the server version
+	Version string
+
+	// ResultsFormat is the format the results should be written in
+	ResultsFormat string
+
+	// ServerExec is the path to a server executable
+	ServerExec string
+
+	// ServerUser is the user account that should start the server
+	ServerUser string
+
+	// SkipLogBin will skip bin logging
+	SkipLogBin bool
+
+	// ServerArgs are the args used to start a server
+	ServerArgs []string
+
+	// ConnectionProtocol defines the protocol for connecting to the server
+	ConnectionProtocol string
+
+	// Socket is the path to the server socket
+	Socket string
+}
+
+var _ ProtocolServerConfig = &mysqlServerConfigImpl{}
+
+func NewMysqlServerConfig(version, serverExec, serverUser, host, resultsFormat, protocol, socket string, port int, serverArgs []string, skipBinLog bool) *mysqlServerConfigImpl {
+	return &mysqlServerConfigImpl{
+		Id:                 uuid.New().String(),
+		Host:               host,
+		Port:               port,
+		Version:            version,
+		ResultsFormat:      resultsFormat,
+		ServerExec:         serverExec,
+		ServerUser:         serverUser,
+		SkipLogBin:         skipBinLog,
+		ServerArgs:         serverArgs,
+		ConnectionProtocol: protocol,
+		Socket:             socket,
+	}
+}
+
+func (sc *mysqlServerConfigImpl) GetServerExec() string {
+	return sc.ServerExec
+}
+
+func (sc *mysqlServerConfigImpl) GetId() string {
+	return sc.Id
+}
+
+func (sc *mysqlServerConfigImpl) GetHost() string {
+	return sc.Host
+}
+
+func (sc *mysqlServerConfigImpl) GetPort() int {
+	return sc.Port
+}
+
+func (sc *mysqlServerConfigImpl) GetVersion() string {
+	return sc.Version
+}
+
+func (sc *mysqlServerConfigImpl) GetServerType() ServerType {
+	return MySql
+}
+
+func (sc *mysqlServerConfigImpl) GetResultsFormat() string {
+	return sc.ResultsFormat
+}
+
+func (sc *mysqlServerConfigImpl) GetConnectionProtocol() string {
+	return sc.ConnectionProtocol
+}
+
+func (sc *mysqlServerConfigImpl) GetSocket() string {
+	return sc.Socket
+}
+
+func (sc *mysqlServerConfigImpl) GetServerArgs() ([]string, error) {
+	params := make([]string, 0)
+	if sc.ServerUser != "" {
+		params = append(params, fmt.Sprintf("%s=%s", userFlag, sc.ServerUser))
+	}
+	if sc.SkipLogBin {
+		params = append(params, skipBinLogFlag)
+	}
+	if sc.Port != 0 {
+		params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port))
+	}
+	params = append(params, sc.ServerArgs...)
+	return params, nil
+}
+
+func (sc *mysqlServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams {
+	params := NewSysbenchTestParams()
+	params.Append(defaultSysbenchParams...)
+	params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlDbFlag, dbName))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, mysqlDriverName))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlHostFlag, sc.Host))
+	if sc.Port != 0 {
+		params.Append(fmt.Sprintf("%s=%d", sysbenchMysqlPortFlag, sc.Port))
+	}
+	params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlUserFlag, sysbenchCommand))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchMysqlPasswordFlag, sysbenchPassLocal))
+	params.Append(testConfig.GetOptions()...)
+	params.Append(testConfig.GetName())
+	return params
+}
+
+func (sc *mysqlServerConfigImpl) Validate() error {
+	if sc.Version == "" {
+		return getMustSupplyError("version")
+	}
+	if sc.ResultsFormat == "" {
+		return getMustSupplyError("results format")
+	}
+	if sc.ServerExec == "" {
+		return getMustSupplyError("server exec")
+	}
+	err := CheckProtocol(sc.ConnectionProtocol)
+	if err != nil {
+		return err
+	}
+	return CheckExec(sc.ServerExec, "server exec")
+}
+
+func (sc *mysqlServerConfigImpl) SetDefaults() error {
+	if sc.Host == "" {
+		sc.Host = defaultHost
+	}
+	if sc.Port < 1 {
+		sc.Port = defaultMysqlPort
+	}
+	return nil
+}

go/performance/utils/benchmark_runner/mysql_tpcc.go

@@ -0,0 +1,94 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"syscall"
+)
+
+type mysqlTpccBenchmarkerImpl struct {
+	dir          string // cwd
+	config       TpccConfig
+	serverConfig ProtocolServerConfig
+}
+
+var _ Benchmarker = &mysqlTpccBenchmarkerImpl{}
+
+func NewMysqlTpccBenchmarker(dir string, config TpccConfig, serverConfig ProtocolServerConfig) *mysqlTpccBenchmarkerImpl {
+	return &mysqlTpccBenchmarkerImpl{
+		dir:          dir,
+		config:       config,
+		serverConfig: serverConfig,
+	}
+}
+
+func (b *mysqlTpccBenchmarkerImpl) getDsn() (string, error) {
+	return GetMysqlDsn(b.serverConfig.GetHost(), b.serverConfig.GetSocket(), b.serverConfig.GetConnectionProtocol(), b.serverConfig.GetPort())
+}
+
+func (b *mysqlTpccBenchmarkerImpl) createTestingDb(ctx context.Context) error {
+	dsn, err := b.getDsn()
+	if err != nil {
+		return err
+	}
+	return CreateMysqlTestingDb(ctx, dsn, tpccDbName)
+}
+
+func (b *mysqlTpccBenchmarkerImpl) Benchmark(ctx context.Context) (Results, error) {
+	serverDir, err := InitMysqlDataDir(ctx, b.serverConfig.GetServerExec(), tpccDbName)
+	if err != nil {
+		return nil, err
+	}
+
+	serverParams, err := b.serverConfig.GetServerArgs()
+	if err != nil {
+		return nil, err
+	}
+	serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir))
+
+	server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams)
+	err = server.Start()
+	if err != nil {
+		return nil, err
+	}
+
+	err = b.createTestingDb(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	tests := GetTpccTests(b.config)
+
+	results := make(Results, 0)
+	for _, test := range tests {
+		tester := NewTpccTester(b.config, b.serverConfig, test, serverParams, stampFunc)
+		r, err := tester.Test(ctx)
+		if err != nil {
+			server.Stop()
+			return nil, err
+		}
+		results = append(results, r)
+	}
+
+	err = server.Stop()
+	if err != nil {
+		return nil, err
+	}
+
+	return results, os.RemoveAll(serverDir)
+}

go/performance/utils/benchmark_runner/postgres.go

@@ -0,0 +1,159 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os"
+	"syscall"
+
+	_ "github.com/lib/pq"
+)
+
+type postgresBenchmarkerImpl struct {
+	dir          string // cwd
+	config       SysbenchConfig
+	serverConfig InitServerConfig
+}
+
+var _ Benchmarker = &postgresBenchmarkerImpl{}
+
+func NewPostgresBenchmarker(dir string, config SysbenchConfig, serverConfig InitServerConfig) *postgresBenchmarkerImpl {
+	return &postgresBenchmarkerImpl{
+		dir:          dir,
+		config:       config,
+		serverConfig: serverConfig,
+	}
+}
+
+func (b *postgresBenchmarkerImpl) initDataDir(ctx context.Context) (string, error) {
+	serverDir, err := CreateServerDir(dbName)
+	if err != nil {
+		return "", err
+	}
+
+	pgInit := ExecCommand(ctx, b.serverConfig.GetInitDbExec(), fmt.Sprintf("%s=%s", postgresInitDbDataDirFlag, serverDir), fmt.Sprintf("%s=%s", postgresUsernameFlag, postgresUsername))
+	err = pgInit.Run()
+	if err != nil {
+		return "", err
+	}
+
+	return serverDir, nil
+}
+
+func (b *postgresBenchmarkerImpl) createTestingDb(ctx context.Context) (err error) {
+	psqlconn := fmt.Sprintf(psqlDsnTemplate, b.serverConfig.GetHost(), b.serverConfig.GetPort(), postgresUsername, "", dbName)
+
+	var db *sql.DB
+	db, err = sql.Open(postgresDriver, psqlconn)
+	if err != nil {
+		return
+	}
+	defer func() {
+		rerr := db.Close()
+		if err == nil {
+			err = rerr
+		}
+	}()
+	err = db.PingContext(ctx)
+	if err != nil {
+		return
+	}
+
+	stmts := []string{
+		fmt.Sprintf(postgresDropDatabaseSqlTemplate, dbName),
+		fmt.Sprintf(postgresDropUserSqlTemplate, sysbenchUsername),
+		fmt.Sprintf(postgresCreateUserSqlTemplate, sysbenchUsername, sysbenchPassLocal),
+		fmt.Sprintf(postgresCreateDatabaseSqlTemplate, dbName, sysbenchUsername),
+	}
+
+	for _, s := range stmts {
+		_, err = db.ExecContext(ctx, s)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func (b *postgresBenchmarkerImpl) Benchmark(ctx context.Context) (results Results, err error) {
+	var serverDir string
+	serverDir, err = b.initDataDir(ctx)
+	if err != nil {
+		return
+	}
+	defer func() {
+		rerr := os.RemoveAll(serverDir)
+		if err == nil {
+			err = rerr
+		}
+	}()
+
+	var serverParams []string
+	serverParams, err = b.serverConfig.GetServerArgs()
+	if err != nil {
+		return
+	}
+
+	serverParams = append(serverParams, postgresDataDirFlag, serverDir)
+
+	server := NewServer(ctx, serverDir, b.serverConfig, syscall.SIGTERM, serverParams)
+	server.WithEnv(postgresLcAllEnvVarKey, postgresLcAllEnvVarValue)
+
+	err = server.Start()
+	if err != nil {
+		return
+	}
+
+	err = b.createTestingDb(ctx)
+	if err != nil {
+		return
+	}
+
+	var tests []Test
+	tests, err = GetTests(b.config, b.serverConfig)
+	if err != nil {
+		return
+	}
+
+	results = make(Results, 0)
+	runs := b.config.GetRuns()
+	for i := 0; i < runs; i++ {
+		for _, test := range tests {
+			t, ok := test.(SysbenchTest)
+			if !ok {
+				return nil, ErrNotSysbenchTest
+			}
+			tester := NewSysbenchTester(b.config, b.serverConfig, t, serverParams, stampFunc)
+			var r *Result
+			r, err = tester.Test(ctx)
+			if err != nil {
+				server.Stop()
+				return
+			}
+			results = append(results, r)
+		}
+	}
+
+	err = server.Stop()
+	if err != nil {
+		return
+	}
+
+	return
+}

go/performance/utils/benchmark_runner/postgres_server_config.go

@@ -0,0 +1,149 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"fmt"
+
+	"github.com/google/uuid"
+)
+
+type postgresServerConfigImpl struct {
+	// Id is a unique id for this servers benchmarking
+	Id string
+
+	// Host is the server host
+	Host string
+
+	// Port is the server port
+	Port int
+
+	// Version is the server version
+	Version string
+
+	// ResultsFormat is the format the results should be written in
+	ResultsFormat string
+
+	// ServerExec is the path to a server executable
+	ServerExec string
+
+	// InitExec is the path to the server init db executable
+	InitExec string
+
+	// ServerUser is the user account that should start the server
+	ServerUser string
+
+	// ServerArgs are the args used to start a server
+	ServerArgs []string
+}
+
+var _ InitServerConfig = &postgresServerConfigImpl{}
+
+func NewPostgresServerConfig(version, serverExec, initDbExec, serverUser, host, resultsFormat string, port int, serverArgs []string) *postgresServerConfigImpl {
+	return &postgresServerConfigImpl{
+		Id:            uuid.New().String(),
+		Host:          host,
+		Port:          port,
+		Version:       version,
+		ResultsFormat: resultsFormat,
+		ServerExec:    serverExec,
+		InitExec:      initDbExec,
+		ServerUser:    serverUser,
+		ServerArgs:    serverArgs,
+	}
+}
+
+func (sc *postgresServerConfigImpl) GetServerExec() string {
+	return sc.ServerExec
+}
+
+func (sc *postgresServerConfigImpl) GetInitDbExec() string {
+	return sc.InitExec
+}
+
+func (sc *postgresServerConfigImpl) GetId() string {
+	return sc.Id
+}
+
+func (sc *postgresServerConfigImpl) GetHost() string {
+	return sc.Host
+}
+
+func (sc *postgresServerConfigImpl) GetPort() int {
+	return sc.Port
+}
+
+func (sc *postgresServerConfigImpl) GetVersion() string {
+	return sc.Version
+}
+
+func (sc *postgresServerConfigImpl) GetResultsFormat() string {
+	return sc.ResultsFormat
+}
+
+func (sc *postgresServerConfigImpl) GetServerType() ServerType {
+	return Postgres
+}
+
+func (sc *postgresServerConfigImpl) GetServerArgs() ([]string, error) {
+	params := make([]string, 0)
+	if sc.Port != 0 {
+		params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port))
+	}
+	params = append(params, sc.ServerArgs...)
+	return params, nil
+}
+
+func (sc *postgresServerConfigImpl) GetTestingParams(testConfig TestConfig) TestParams {
+	params := NewSysbenchTestParams()
+	params.Append(defaultSysbenchParams...)
+	params.Append(fmt.Sprintf("%s=%s", sysbenchDbDriverFlag, sysbenchPostgresDbDriver))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresDbFlag, dbName))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresHostFlag, sc.Host))
+	params.Append(fmt.Sprintf("%s=%s", sysbenchPostgresUserFlag, postgresUsername))
+	if sc.Port != 0 {
+		params.Append(fmt.Sprintf("%s=%d", sysbenchPostgresPortFlag, sc.Port))
+	}
+	params.Append(testConfig.GetOptions()...)
+	params.Append(testConfig.GetName())
+	return params
+}
+
+func (sc *postgresServerConfigImpl) Validate() error {
+	if sc.Version == "" {
+		return getMustSupplyError("version")
+	}
+	if sc.ResultsFormat == "" {
+		return getMustSupplyError("results format")
+	}
+	if sc.ServerExec == "" {
+		return getMustSupplyError("server exec")
+	}
+	err := CheckExec(sc.ServerExec, "server exec")
+	if err != nil {
+		return err
+	}
+	return CheckExec(sc.InitExec, "initdb exec")
+}
+
+func (sc *postgresServerConfigImpl) SetDefaults() error {
+	if sc.Host == "" {
+		sc.Host = defaultHost
+	}
+	if sc.Port < 1 {
+		sc.Port = defaultPostgresPort
+	}
+	return nil
+}

go/performance/utils/benchmark_runner/profile.go

@@ -0,0 +1,140 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+type Profiler interface {
+	Profile(ctx context.Context) error
+}
+
+type doltProfilerImpl struct {
+	dir          string // cwd
+	config       SysbenchConfig
+	serverConfig ProfilingServerConfig
+}
+
+var _ Profiler = &doltProfilerImpl{}
+
+func NewDoltProfiler(dir string, config SysbenchConfig, serverConfig ProfilingServerConfig) *doltProfilerImpl {
+	return &doltProfilerImpl{
+		dir:          dir,
+		config:       config,
+		serverConfig: serverConfig,
+	}
+}
+
+func (p *doltProfilerImpl) updateGlobalConfig(ctx context.Context) error {
+	err := CheckSetDoltConfig(ctx, p.serverConfig.GetServerExec(), doltConfigUsernameKey, doltBenchmarkUser)
+	if err != nil {
+		return err
+	}
+	return CheckSetDoltConfig(ctx, p.serverConfig.GetServerExec(), doltConfigEmailKey, doltBenchmarkEmail)
+}
+
+func (p *doltProfilerImpl) checkInstallation(ctx context.Context) error {
+	version := ExecCommand(ctx, p.serverConfig.GetServerExec(), doltVersionCommand)
+	return version.Run()
+}
+
+func (p *doltProfilerImpl) initDoltRepo(ctx context.Context) (string, error) {
+	return InitDoltRepo(ctx, p.dir, p.serverConfig.GetServerExec(), p.config.GetNomsBinFormat(), dbName)
+}
+
+func (p *doltProfilerImpl) Profile(ctx context.Context) error {
+	err := p.checkInstallation(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = p.updateGlobalConfig(ctx)
+	if err != nil {
+		return err
+	}
+
+	testRepo, err := p.initDoltRepo(ctx)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(testRepo)
+
+	serverParams, err := p.serverConfig.GetServerArgs()
+	if err != nil {
+		return err
+	}
+
+	profilePath, err := os.MkdirTemp("", "dolt_profile_path_*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(profilePath)
+
+	tempProfile := filepath.Join(profilePath, cpuProfileFilename)
+	profileParams := make([]string, 0)
+	profileParams = append(profileParams, profileFlag, cpuProfile, profilePathFlag, profilePath)
+	profileParams = append(profileParams, serverParams...)
+
+	server := NewServer(ctx, testRepo, p.serverConfig, syscall.SIGTERM, profileParams)
+	err = server.Start()
+	if err != nil {
+		return err
+	}
+
+	tests, err := GetTests(p.config, p.serverConfig)
+	if err != nil {
+		return err
+	}
+
+	results := make(Results, 0)
+	runs := p.config.GetRuns()
+	for i := 0; i < runs; i++ {
+		for _, test := range tests {
+			t, ok := test.(SysbenchTest)
+			if !ok {
+				return ErrNotSysbenchTest
+			}
+			tester := NewSysbenchTester(p.config, p.serverConfig, t, profileParams, stampFunc)
+			r, err := tester.Test(ctx)
+			if err != nil {
+				server.Stop()
+				return err
+			}
+			results = append(results, r)
+		}
+	}
+
+	err = server.Stop()
+	if err != nil {
+		return err
+	}
+
+	info, err := os.Stat(tempProfile)
+	if err != nil {
+		return err
+	}
+
+	if info.Size() < 1 {
+		return fmt.Errorf("failed to create profile: file was empty")
+	}
+
+	finalProfile := filepath.Join(p.serverConfig.GetProfilePath(), fmt.Sprintf("%s_%s", p.serverConfig.GetId(), cpuProfileFilename))
+	return os.Rename(tempProfile, finalProfile)
+}

go/performance/utils/benchmark_runner/results.go

@@ -12,41 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"errors"
 	"path/filepath"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/google/uuid"
 )
 
-const (
-	stampFormat    = time.RFC3339
-	SqlStatsPrefix = "SQL statistics:"
-	read           = "read"
-	write          = "write"
-	other          = "other"
-	totalQueries   = "total"
-	totalEvents    = "total number of events"
-	min            = "min"
-	avg            = "avg"
-	max            = "max"
-	percentile     = "percentile"
-	sum            = "sum"
-	transactions   = "transactions"
-	queriesPerSec  = "queries"
-	ignoredErrors  = "ignored errors"
-	reconnects     = "reconnects"
-	totalTimeSecs  = "total time"
-)
-
 var (
-	ResultFileTemplate = "%s_%s_%s_sysbench_performance%s"
-
 	ErrUnableToParseOutput    = errors.New("unable to parse output")
 	ErrUnsupportedHeaderField = errors.New("unsupported header field")
 )
@@ -152,13 +129,7 @@ func (r *Result) Stamp(stampFunc func() string) {
 	r.CreatedAt = stampFunc()
 }
 
-// FromConfigsNewResult returns a new result with some fields set based on the provided configs
-func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, suiteId string, idFunc func() string) (*Result, error) {
-	serverParams, err := serverConfig.GetServerArgs()
-	if err != nil {
-		return nil, err
-	}
-
+func NewResult(server ServerType, version, testName, testId, suiteId, runtimeOs, runtimeGoArch string, serverParams, testParams []string, idFunc func() string, fromScript bool) *Result {
 	var getId func() string
 	if idFunc == nil {
 		getId = func() string {
@@ -169,53 +140,26 @@ func FromConfigsNewResult(config *Config, serverConfig *ServerConfig, t *Test, s
 	}
 
 	var name string
-	if t.FromScript {
-		base := filepath.Base(t.Name)
+	if fromScript {
+		base := filepath.Base(testName)
 		ext := filepath.Ext(base)
 		name = strings.TrimSuffix(base, ext)
 	} else {
-		name = t.Name
+		name = testName
 	}
 
 	return &Result{
 		Id:            getId(),
 		SuiteId:       suiteId,
-		TestId:        t.id,
-		RuntimeOS:     config.RuntimeOS,
-		RuntimeGoArch: config.RuntimeGoArch,
-		ServerName:    string(serverConfig.Server),
-		ServerVersion: serverConfig.Version,
+		TestId:        testId,
+		RuntimeOS:     runtimeOs,
+		RuntimeGoArch: runtimeGoArch,
+		ServerName:    string(server),
+		ServerVersion: version,
 		ServerParams:  strings.Join(serverParams, " "),
 		TestName:      name,
-		TestParams:    strings.Join(t.Params, " "),
-	}, nil
-}
-
-// FromOutputResult accepts raw sysbench run output and returns the Result
-func FromOutputResult(output []byte, config *Config, serverConfig *ServerConfig, test *Test, suiteId string, idFunc func() string) (*Result, error) {
-	result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc)
-	if err != nil {
-		return nil, err
+		TestParams:    strings.Join(testParams, " "),
 	}
-	lines := strings.Split(string(output), "\n")
-	var process bool
-	for _, l := range lines {
-		trimmed := strings.TrimSpace(l)
-		if trimmed == "" {
-			continue
-		}
-		if strings.HasPrefix(trimmed, SqlStatsPrefix) {
-			process = true
-			continue
-		}
-		if process {
-			err := UpdateResult(result, trimmed)
-			if err != nil {
-				return result, err
-			}
-		}
-	}
-	return result, nil
 }
 
 // UpdateResult extracts the key and value from the given line and updates the given Result
@@ -368,6 +312,31 @@ func updateResult(result *Result, key, val string) error {
 	return nil
 }
 
+func OutputToResult(output []byte, server ServerType, version, testName, testId, suiteId, runtimeOs, runtimeGoArch string, serverParams, testParams []string, idFunc func() string, fromScript bool) (*Result, error) {
+	result := NewResult(server, version, testName, testId, suiteId, runtimeOs, runtimeGoArch, serverParams, testParams, idFunc, fromScript)
+
+	lines := strings.Split(string(output), "\n")
+	var process bool
+	for _, l := range lines {
+		trimmed := strings.TrimSpace(l)
+		if trimmed == "" {
+			continue
+		}
+		if strings.HasPrefix(trimmed, SqlStatsPrefix) {
+			process = true
+			continue
+		}
+		if process {
+			err := UpdateResult(result, trimmed)
+			if err != nil {
+				return result, err
+			}
+		}
+	}
+
+	return result, nil
+}
+
 // FromValWithParens takes a string containing parens and
 // returns the value outside the parens first, and the value
 // inside the parens second

go/performance/utils/benchmark_runner/results_test.go

@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sysbench_runner
+package benchmark_runner
 
 import (
 	"fmt"
 	"math/rand"
+	"strings"
 	"testing"
 	"time"
 
@@ -263,38 +264,43 @@ func fromResultSysbenchOutput(r *Result) string {
 }
 
 func TestFromOutputResults(t *testing.T) {
+	params := NewSysbenchTestParams()
+	params.Append(testTestParams)
+
+	serverParams := defaultDoltServerParams
+	serverParams = append(serverParams, testServerParams)
+
 	tests := []struct {
 		description    string
 		output         []byte
-		config         *Config
-		serverConfig   *ServerConfig
-		test           *Test
+		config         Config
+		serverConfig   ServerConfig
+		test           SysbenchTest
 		expectedResult *Result
 		expectedError  error
 	}{
 		{
 			description: "should parse output into result",
 			output:      []byte(sampleOutput1),
-			config: &Config{
+			config: &sysbenchRunnerConfigImpl{
 				RuntimeOS:     testOS,
 				RuntimeGoArch: testGoArch,
 			},
-			serverConfig: &ServerConfig{
-				Host:       "localhost",
-				Server:     ServerType(testServer),
+			serverConfig: &doltServerConfigImpl{
 				Version:    testServerVersion,
 				ServerExec: "test-exec",
 			},
-			test: &Test{
+			test: &sysbenchTestImpl{
 				Name:   testTestName,
-				Params: []string{testTestParams},
+				Params: params,
 			},
 			expectedResult: &Result{
 				Id:                       testId,
 				SuiteId:                  testSuiteId,
 				RuntimeOS:                testOS,
 				RuntimeGoArch:            testGoArch,
-				ServerName:               testServer,
+				ServerName:               string(Dolt),
+				ServerParams:             strings.Join(defaultDoltServerParams, " "),
 				ServerVersion:            testServerVersion,
 				TestName:                 testTestName,
 				TestParams:               testTestParams,
@@ -315,11 +321,12 @@ func TestFromOutputResults(t *testing.T) {
 			},
 		},
 	}
+
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
-			actual, err := FromOutputResult(test.output, test.config, test.serverConfig, test.test, testSuiteId, func() string {
-				return testId
-			})
+			serverParams, err := test.serverConfig.GetServerArgs()
+			assert.NoError(t, err)
+			actual, err := OutputToResult(test.output, test.serverConfig.GetServerType(), test.serverConfig.GetVersion(), test.test.GetName(), test.test.GetId(), testSuiteId, test.config.GetRuntimeOs(), test.config.GetRuntimeGoArch(), serverParams, test.test.GetParamsToSlice(), func() string { return testId }, test.test.GetFromScript())
 			assert.Equal(t, test.expectedError, err)
 			assert.Equal(t, test.expectedResult, actual)
 		})

go/performance/utils/benchmark_runner/run.go

@@ -0,0 +1,141 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+var ErrNotProtocolServerConfig = errors.New("protocol server config required")
+var ErrNotInitDbServerConfig = errors.New("init db server config required")
+
+// Run runs sysbench runner
+func Run(ctx context.Context, config SysbenchConfig) error {
+	err := config.Validate(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = sysbenchVersion(ctx)
+	if err != nil {
+		return err
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	svs := config.GetServerConfigs()
+	for _, serverConfig := range svs {
+		var results Results
+		var b Benchmarker
+		st := serverConfig.GetServerType()
+		switch st {
+		case Dolt:
+			// handle a profiling run
+			sc, ok := serverConfig.(ProfilingServerConfig)
+			if ok {
+				if string(sc.GetServerProfile()) != "" {
+					fmt.Println("Profiling dolt while running sysbench tests")
+					p := NewDoltProfiler(cwd, config, sc)
+					return p.Profile(ctx)
+				}
+			}
+
+			fmt.Println("Running dolt sysbench tests")
+			b = NewDoltBenchmarker(cwd, config, serverConfig)
+		case Doltgres:
+			fmt.Println("Running doltgres sysbench tests")
+			b = NewDoltgresBenchmarker(cwd, config, serverConfig)
+		case MySql:
+			sc, ok := serverConfig.(ProtocolServerConfig)
+			if !ok {
+				return ErrNotProtocolServerConfig
+			}
+			fmt.Println("Running mysql sysbench tests")
+			b = NewMysqlBenchmarker(cwd, config, sc)
+		case Postgres:
+			sc, ok := serverConfig.(InitServerConfig)
+			if !ok {
+				return ErrNotInitDbServerConfig
+			}
+			fmt.Println("Running postgres sysbench tests")
+			b = NewPostgresBenchmarker(cwd, config, sc)
+		default:
+			panic(fmt.Sprintf("unexpected server type: %s", st))
+		}
+
+		results, err = b.Benchmark(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Successfuly finished %s\n", st)
+
+		err = WriteResults(serverConfig, results)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Successfuly wrote results for %s\n", st)
+	}
+	return nil
+}
+
+func sysbenchVersion(ctx context.Context) error {
+	version := ExecCommand(ctx, sysbenchCommand, sysbenchVersionFlag)
+	return version.Run()
+}
+
+func WriteResults(serverConfig ServerConfig, results Results) error {
+	st := serverConfig.GetServerType()
+	version := serverConfig.GetVersion()
+	id := serverConfig.GetId()
+	format := serverConfig.GetResultsFormat()
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	//var writePath string
+	switch format {
+	case CsvFormat, CsvExt:
+		writePath := filepath.Join(
+			cwd,
+			resultsDirname,
+			string(st),
+			version,
+			id,
+			fmt.Sprintf(ResultFileTemplate, id, st, version, CsvExt))
+		return WriteResultsCsv(writePath, results)
+	case JsonFormat, JsonExt:
+		writePath := filepath.Join(
+			cwd,
+			resultsDirname,
+			string(st),
+			version,
+			id,
+			fmt.Sprintf(ResultFileTemplate, id, st, version, JsonExt))
+		return WriteResultsJson(writePath, results)
+	default:
+		return fmt.Errorf("unsupported results format: %s", format)
+	}
+}

go/performance/utils/benchmark_runner/run_test.go

@@ -0,0 +1,265 @@
+// Copyright 2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var runTests = os.Getenv("RUN_BENCHMARK_RUNNER_TESTS")
+
+func TestRunner(t *testing.T) {
+	if runTests == "" {
+		t.Skip()
+	}
+	dir := t.TempDir()
+	log.Println(dir)
+	err := os.Chdir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	conf := &sysbenchRunnerConfigImpl{
+		Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_delete_insert"),
+		//Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"),
+		Servers: []ServerConfig{
+			&doltServerConfigImpl{
+				Id:            "test",
+				Version:       "0.39.2",
+				ResultsFormat: CsvFormat,
+				ServerExec:    "/Users/max-hoffman/go/bin/dolt",
+			},
+		},
+		ScriptDir: "/Users/max-hoffman/Documents/dolthub/sysbench-lua-scripts",
+		TestOptions: []string{
+			"--rand-seed=1",
+			"--table-size=10000",
+			"--rand-type=uniform",
+			"--time=120",
+			"--percentile=50",
+		},
+		InitBigRepo: true,
+	}
+
+	err = Run(context.Background(), conf)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func selectTests(names ...string) []TestConfig {
+	tests := make([]TestConfig, len(names))
+	for i := range names {
+		tests[i] = &testConfigImpl{Name: names[i], FromScript: false}
+	}
+	return tests
+}
+
+func TestDoltMysqlSysbenchRunner(t *testing.T) {
+	if runTests == "" {
+		t.Skip()
+	}
+	dir := t.TempDir()
+	log.Println(dir)
+	err := os.Chdir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	conf := &sysbenchRunnerConfigImpl{
+		Tests: []TestConfig{
+			NewTestConfig("oltp_read_write", nil, false),
+			NewTestConfig("oltp_update_index", nil, false),
+			NewTestConfig("oltp_delete_insert", nil, true),
+		},
+		Servers: []ServerConfig{
+			&doltServerConfigImpl{
+				Id:            "test-dolt",
+				Version:       "1.33.0",
+				ResultsFormat: CsvFormat,
+				ServerExec:    "/Users/dustin/go/bin/dolt",
+			},
+			&mysqlServerConfigImpl{
+				Id:                 "test-mysql",
+				Host:               "127.0.0.1",
+				Port:               3606,
+				Version:            "8.0.35",
+				ResultsFormat:      CsvFormat,
+				ServerExec:         "/opt/homebrew/bin/mysqld",
+				ServerUser:         "root",
+				SkipLogBin:         true,
+				ConnectionProtocol: "tcp",
+			},
+		},
+		ScriptDir: "/Users/dustin/src/sysbench-lua-scripts",
+		TestOptions: []string{
+			"--rand-seed=1",
+			"--table-size=10000",
+			"--rand-type=uniform",
+			"--time=120",
+			"--percentile=50",
+		},
+		InitBigRepo: true,
+	}
+
+	err = Run(context.Background(), conf)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func TestDoltgresPostgresSysbenchRunner(t *testing.T) {
+	if runTests == "" {
+		t.Skip()
+	}
+	dir := t.TempDir()
+	log.Println(dir)
+	err := os.Chdir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	conf := &sysbenchRunnerConfigImpl{
+		Tests: []TestConfig{
+			NewTestConfig("oltp_read_write", nil, false),
+			NewTestConfig("oltp_update_index", nil, false),
+		},
+		Servers: []ServerConfig{
+			&postgresServerConfigImpl{
+				Id:            "test-postgres",
+				Host:          "127.0.0.1",
+				Version:       "15.5",
+				ResultsFormat: CsvFormat,
+				ServerExec:    "/opt/homebrew/opt/postgresql@15/bin/postgres",
+				InitExec:      "/opt/homebrew/opt/postgresql@15/bin/initdb",
+				ServerUser:    "root",
+			},
+			&doltgresServerConfigImpl{
+				Id:            "test-doltgres",
+				Port:          4433,
+				Host:          "127.0.0.1",
+				Version:       "b139dfb",
+				ResultsFormat: CsvFormat,
+				ServerExec:    "/Users/dustin/go/bin/doltgres",
+			},
+		},
+		ScriptDir: "/Users/dustin/src/sysbench-lua-scripts",
+		TestOptions: []string{
+			"--rand-seed=1",
+			"--table-size=10000",
+			"--rand-type=uniform",
+			"--time=120",
+			"--percentile=50",
+		},
+		InitBigRepo: true,
+	}
+
+	err = Run(context.Background(), conf)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func TestDoltProfiler(t *testing.T) {
+	if runTests == "" {
+		t.Skip()
+	}
+	dir := t.TempDir()
+	log.Println(dir)
+	err := os.Chdir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	id := "test-dolt-profile"
+	conf := &sysbenchRunnerConfigImpl{
+		Tests: []TestConfig{
+			NewTestConfig("oltp_read_write", nil, false),
+		},
+		Servers: []ServerConfig{
+			&doltServerConfigImpl{
+				Id:            id,
+				Version:       "1.33.0",
+				ResultsFormat: CsvFormat,
+				ServerExec:    "/Users/dustin/go/bin/dolt",
+				ServerProfile: CpuServerProfile,
+				ProfilePath:   dir,
+			},
+		},
+		TestOptions: []string{
+			"--rand-seed=1",
+			"--table-size=10000",
+			"--rand-type=uniform",
+			"--time=30",
+			"--percentile=50",
+		},
+	}
+
+	err = Run(context.Background(), conf)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	expectedProfile := filepath.Join(dir, fmt.Sprintf("%s_%s", id, cpuProfileFilename))
+	if _, err := os.Stat(expectedProfile); errors.Is(err, os.ErrNotExist) {
+		log.Fatal("failed to create dolt cpu profile")
+	}
+}
+
+func TestDoltMysqlTpccRunner(t *testing.T) {
+	if runTests == "" {
+		t.Skip()
+	}
+	dir := t.TempDir()
+	log.Println(dir)
+	err := os.Chdir(dir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	conf := &tpccConfigImpl{
+		Servers: []ServerConfig{
+			&doltServerConfigImpl{
+				Id:            "test-dolt-tpcc",
+				Version:       "1.33.0",
+				ResultsFormat: CsvFormat,
+				ServerExec:    "/Users/dustin/go/bin/dolt",
+			},
+			&mysqlServerConfigImpl{
+				Id:                 "test-mysql-tpcc",
+				Host:               "127.0.0.1",
+				Port:               3606,
+				Version:            "8.0.35",
+				ResultsFormat:      CsvFormat,
+				ServerExec:         "/opt/homebrew/bin/mysqld",
+				ServerUser:         "root",
+				SkipLogBin:         true,
+				ConnectionProtocol: "tcp",
+			},
+		},
+		ScriptDir: "/Users/dustin/src/sysbench-tpcc",
+	}
+
+	err = RunTpcc(context.Background(), conf)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

go/performance/utils/benchmark_runner/run_tpcc.go

@@ -0,0 +1,69 @@
+// Copyright 2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"fmt"
+	"os"
+)
+
+func RunTpcc(ctx context.Context, config TpccConfig) error {
+	err := config.Validate(ctx)
+	if err != nil {
+		return err
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	svs := config.GetServerConfigs()
+	for _, serverConfig := range svs {
+		var b Benchmarker
+		var results Results
+		st := serverConfig.GetServerType()
+		switch st {
+		case Dolt:
+			fmt.Println("Running dolt tpcc benchmarks")
+			b = NewDoltTpccBenchmarker(cwd, config, serverConfig)
+		case MySql:
+			sc, ok := serverConfig.(ProtocolServerConfig)
+			if !ok {
+				return ErrNotProtocolServerConfig
+			}
+
+			fmt.Println("Running mysql tpcc benchmarks")
+			b = NewMysqlTpccBenchmarker(cwd, config, sc)
+		default:
+			panic(fmt.Sprintf("unexpected server type: %s", st))
+		}
+
+		results, err = b.Benchmark(ctx)
+		if err != nil {
+			return err
+		}
+
+		err = WriteResults(serverConfig, results)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Successfuly wrote results for %s\n", st)
+	}
+
+	return nil
+}

go/performance/utils/benchmark_runner/server.go

@@ -0,0 +1,126 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+var ErrServerClosed = errors.New("server was previously closed")
+
+type Server interface {
+	Start() error
+	Stop() error
+	WithEnv(key, value string)
+}
+
+type doltServerImpl struct {
+	dir                 string
+	serverConfig        ServerConfig
+	serverCtx           context.Context
+	serverCtxCancelFunc context.CancelFunc
+	server              *exec.Cmd
+	serverEg            *errgroup.Group
+	quit                chan os.Signal
+	killSignal          os.Signal
+}
+
+var _ Server = &doltServerImpl{}
+
+func NewServer(ctx context.Context, dir string, serverConfig ServerConfig, killSignal os.Signal, serverParams []string) *doltServerImpl {
+	withKeyCtx, cancel := context.WithCancel(ctx)
+	gServer, serverCtx := errgroup.WithContext(withKeyCtx)
+
+	server := ExecCommand(serverCtx, serverConfig.GetServerExec(), serverParams...)
+	server.Dir = dir
+
+	quit := make(chan os.Signal, 1)
+	return &doltServerImpl{
+		dir:                 dir,
+		serverConfig:        serverConfig,
+		serverCtx:           serverCtx,
+		server:              server,
+		serverCtxCancelFunc: cancel,
+		serverEg:            gServer,
+		quit:                quit,
+		killSignal:          killSignal,
+	}
+}
+
+func (s *doltServerImpl) WithEnv(key, val string) {
+	if s.server != nil {
+		s.server.Env = append(s.server.Env, fmt.Sprintf("%s=%s", key, val))
+	}
+}
+
+func (s *doltServerImpl) Start() error {
+	if s.serverEg == nil || s.serverCtx == nil || s.quit == nil {
+		return ErrServerClosed
+	}
+
+	s.serverEg.Go(func() error {
+		<-s.quit
+		return s.server.Process.Signal(s.killSignal)
+	})
+
+	s.serverEg.Go(func() error {
+		if Debug {
+			s.server.Stdout = os.Stdout
+			s.server.Stderr = os.Stderr
+		}
+		return s.server.Run()
+	})
+
+	// sleep to allow the server to start
+	time.Sleep(10 * time.Second)
+	fmt.Println("Successfully started database server")
+	return nil
+}
+
+func (s *doltServerImpl) Stop() error {
+	defer s.serverCtxCancelFunc()
+	if s.serverEg != nil && s.serverCtx != nil && s.quit != nil {
+		// send signal to dolt server
+		s.quit <- s.killSignal
+		err := s.serverEg.Wait()
+		if err != nil {
+			// we expect a kill error
+			// we only exit in error if this is not the
+			// error
+			if err.Error() != expectedServerKilledErrorMessage && err.Error() != expectedServerTerminatedErrorMessage {
+				fmt.Println(err)
+				close(s.quit)
+				return err
+			}
+		}
+
+		fmt.Println("Successfully killed database server")
+		close(s.quit)
+
+		s.quit = nil
+		s.serverCtx = nil
+		s.serverEg = nil
+		s.serverCtxCancelFunc = func() {}
+	}
+
+	return nil
+}

go/performance/utils/benchmark_runner/server_config.go

@@ -0,0 +1,48 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+type ServerType string
+
+type ServerConfig interface {
+	GetId() string
+	GetHost() string
+	GetPort() int
+	GetVersion() string
+	GetServerExec() string
+	GetResultsFormat() string
+	GetServerType() ServerType
+	GetServerArgs() ([]string, error)
+	GetTestingParams(testConfig TestConfig) TestParams
+	Validate() error
+	SetDefaults() error
+}
+
+type InitServerConfig interface {
+	ServerConfig
+	GetInitDbExec() string
+}
+
+type ProtocolServerConfig interface {
+	ServerConfig
+	GetConnectionProtocol() string
+	GetSocket() string
+}
+
+type ProfilingServerConfig interface {
+	ServerConfig
+	GetServerProfile() ServerProfile
+	GetProfilePath() string
+}

go/performance/utils/benchmark_runner/sysbench.go

@@ -0,0 +1,160 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+)
+
+type sysbenchTesterImpl struct {
+	test         SysbenchTest
+	config       Config
+	serverConfig ServerConfig
+	serverParams []string
+	stampFunc    func() string
+	idFunc       func() string
+	suiteId      string
+}
+
+var _ Tester = &sysbenchTesterImpl{}
+
+func NewSysbenchTester(config Config, serverConfig ServerConfig, test SysbenchTest, serverParams []string, stampFunc func() string) *sysbenchTesterImpl {
+	return &sysbenchTesterImpl{
+		config:       config,
+		serverParams: serverParams,
+		serverConfig: serverConfig,
+		test:         test,
+		suiteId:      serverConfig.GetId(),
+		stampFunc:    stampFunc,
+	}
+}
+
+func (t *sysbenchTesterImpl) newResult() (*Result, error) {
+	serverParams, err := t.serverConfig.GetServerArgs()
+	if err != nil {
+		return nil, err
+	}
+
+	var getId func() string
+	if t.idFunc == nil {
+		getId = func() string {
+			return uuid.New().String()
+		}
+	} else {
+		getId = t.idFunc
+	}
+
+	var name string
+	if t.test.GetFromScript() {
+		base := filepath.Base(t.test.GetName())
+		ext := filepath.Ext(base)
+		name = strings.TrimSuffix(base, ext)
+	} else {
+		name = t.test.GetName()
+	}
+
+	return &Result{
+		Id:            getId(),
+		SuiteId:       t.suiteId,
+		TestId:        t.test.GetId(),
+		RuntimeOS:     t.config.GetRuntimeOs(),
+		RuntimeGoArch: t.config.GetRuntimeGoArch(),
+		ServerName:    string(t.serverConfig.GetServerType()),
+		ServerVersion: t.serverConfig.GetVersion(),
+		ServerParams:  strings.Join(serverParams, " "),
+		TestName:      name,
+		TestParams:    strings.Join(t.test.GetParamsToSlice(), " "),
+	}, nil
+}
+
+func (t *sysbenchTesterImpl) outputToResult(output []byte) (*Result, error) {
+	return OutputToResult(output, t.serverConfig.GetServerType(), t.serverConfig.GetVersion(), t.test.GetName(), t.test.GetId(), t.suiteId, t.config.GetRuntimeOs(), t.config.GetRuntimeGoArch(), t.serverParams, t.test.GetParamsToSlice(), nil, t.test.GetFromScript())
+}
+
+func (t *sysbenchTesterImpl) prepare(ctx context.Context) error {
+	cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetPrepareArgs(t.serverConfig)...)
+	if t.test.GetFromScript() {
+		lp := filepath.Join(t.config.GetScriptDir(), luaPath)
+		cmd.Env = os.Environ()
+		cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp))
+	}
+	out, err := cmd.Output()
+	if err != nil {
+		fmt.Println(string(out))
+		return err
+	}
+	return nil
+}
+
+func (t *sysbenchTesterImpl) run(ctx context.Context) (*Result, error) {
+	cmd := exec.CommandContext(ctx, sysbenchCommand, t.test.GetRunArgs(t.serverConfig)...)
+	if t.test.GetFromScript() {
+		lp := filepath.Join(t.config.GetScriptDir(), luaPath)
+		cmd.Env = os.Environ()
+		cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp))
+	}
+
+	out, err := cmd.Output()
+	if err != nil {
+		fmt.Print(string(out))
+		return nil, err
+	}
+
+	if Debug == true {
+		fmt.Print(string(out))
+	}
+
+	rs, err := t.outputToResult(out)
+	if err != nil {
+		return nil, err
+	}
+
+	rs.Stamp(t.stampFunc)
+
+	return rs, nil
+}
+
+func (t *sysbenchTesterImpl) cleanup(ctx context.Context) error {
+	cmd := ExecCommand(ctx, sysbenchCommand, t.test.GetCleanupArgs(t.serverConfig)...)
+	if t.test.GetFromScript() {
+		lp := filepath.Join(t.config.GetScriptDir(), luaPath)
+		cmd.Env = os.Environ()
+		cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp))
+	}
+	return cmd.Run()
+}
+
+func (t *sysbenchTesterImpl) Test(ctx context.Context) (*Result, error) {
+	err := t.prepare(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("Running test", t.test.GetName())
+
+	rs, err := t.run(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return rs, t.cleanup(ctx)
+}

go/performance/utils/benchmark_runner/sysbench_config.go

@@ -0,0 +1,330 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+var (
+	ErrTestNameNotDefined            = errors.New("test name not defined")
+	ErrNoServersDefined              = errors.New("servers not defined")
+	ErrTooManyServersDefined         = errors.New("too many servers defined, two max")
+	ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol")
+)
+
+var defaultSysbenchParams = []string{
+	fmt.Sprintf("%s=%s", sysbenchDbPsModeFlag, sysbenchDbPsModeDisable),
+	fmt.Sprintf("%s=%s", sysbenchRandTypeFlag, sysbenchRandTypeUniform),
+}
+
+var defaultDoltServerParams = []string{doltSqlServerCommand}
+
+var defaultSysbenchTests = []TestConfig{
+	NewTestConfig(sysbenchOltpReadOnlyTestName, []string{}, false),
+	NewTestConfig(sysbenchOltpInsertTestName, []string{}, false),
+	NewTestConfig(sysbenchBulkInsertTestName, []string{}, false),
+	NewTestConfig(sysbenchOltpPointSelectTestName, []string{}, false),
+	NewTestConfig(sysbenchSelectRandomPointsTestName, []string{}, false),
+	NewTestConfig(sysbenchSelectRandomRangesTestName, []string{}, false),
+	NewTestConfig(sysbenchOltpWriteOnlyTestName, []string{}, false),
+	NewTestConfig(sysbenchOltpReadWriteTestName, []string{}, false),
+	NewTestConfig(sysbenchOltpUpdateIndexTestName, []string{}, false),
+	NewTestConfig(sysbenchOltpUpdateNonIndexTestName, []string{}, false),
+}
+
+var defaultDoltLuaScripts = map[string]string{
+	sysbenchCoveringIndexScanLuaTestName: sysbenchCoveringIndexScanLuaTestName,
+	sysbenchGroupByScanLuaTestName:       sysbenchGroupByScanLuaTestName,
+	sysbenchIndexJoinLuaTestName:         sysbenchIndexJoinLuaTestName,
+	sysbenchIndexJoinScanLuaTestName:     sysbenchIndexJoinScanLuaTestName,
+	sysbenchIndexScanLuaTestName:         sysbenchIndexScanLuaTestName,
+	sysbenchOltpDeleteInsertLuaTestName:  sysbenchOltpDeleteInsertLuaTestName,
+	sysbenchTableScanLuaTestName:         sysbenchTableScanLuaTestName,
+	sysbenchTypesDeleteInsertLuaTestName: sysbenchTypesDeleteInsertLuaTestName,
+	sysbenchTypesTableScanLuaTestName:    sysbenchTypesTableScanLuaTestName,
+}
+
+// todo: check expressions need to be supported in doltgres for these
+// todo: postgres does not have geometry types also
+var defaultDoltgresLuaScripts = map[string]string{
+	//sysbenchCoveringIndexScanPostgresLuaTestName: sysbenchCoveringIndexScanPostgresLuaTestName,
+	//sysbenchGroupByScanPostgresLuaTestName:       sysbenchGroupByScanPostgresLuaTestName,
+	//sysbenchIndexJoinPostgresLuaTestName:         sysbenchIndexJoinPostgresLuaTestName,
+	//sysbenchIndexJoinScanPostgresLuaTestName:     sysbenchIndexJoinScanPostgresLuaTestName,
+	//sysbenchIndexScanPostgresLuaTestName:         sysbenchIndexScanPostgresLuaTestName,
+	//sysbenchOltpDeleteInsertPostgresLuaTestName:  sysbenchOltpDeleteInsertPostgresLuaTestName,
+	//sysbenchTableScanPostgresLuaTestName:         sysbenchTableScanPostgresLuaTestName,
+	//sysbenchTypesDeleteInsertPostgresLuaTestName: sysbenchTypesDeleteInsertPostgresLuaTestName,
+	//sysbenchTypesTableScanPostgresLuaTestName:    sysbenchTypesTableScanPostgresLuaTestName,
+}
+
+// sysbenchRunnerConfigImpl is the configuration for a benchmarking run
+type sysbenchRunnerConfigImpl struct {
+	// Runs is the number of times to run all tests
+	Runs int
+	// RuntimeOS is the platform the benchmarks ran on
+	RuntimeOS string
+	// RuntimeGoArch is the runtime architecture
+	RuntimeGoArch string
+	// Servers are the servers to benchmark
+	Servers []ServerConfig
+	// Tests are the tests to run. If no tests are provided,
+	// the default tests will be used
+	Tests []TestConfig
+	// TestOptions a list of sysbench test options to apply to all tests
+	TestOptions []string
+	// ScriptDir is a path to a directory of lua scripts
+	ScriptDir string
+	// InitBigRepo downloads a database with existing chunks and commits
+	InitBigRepo bool
+	// NomsBinFormat specifies the NomsBinFormat
+	NomsBinFormat string
+}
+
+var _ SysbenchConfig = &sysbenchRunnerConfigImpl{}
+
+// NewRunnerConfig returns a new sysbenchRunnerConfigImpl
+func NewRunnerConfig() *sysbenchRunnerConfigImpl {
+	return &sysbenchRunnerConfigImpl{
+		Servers: make([]ServerConfig, 0),
+	}
+}
+
+func (c *sysbenchRunnerConfigImpl) GetRuns() int {
+	return c.Runs
+}
+
+func (c *sysbenchRunnerConfigImpl) GetScriptDir() string {
+	return c.ScriptDir
+}
+
+func (c *sysbenchRunnerConfigImpl) GetNomsBinFormat() string {
+	return c.NomsBinFormat
+}
+
+func (c *sysbenchRunnerConfigImpl) GetRuntimeOs() string {
+	return c.RuntimeOS
+}
+
+func (c *sysbenchRunnerConfigImpl) GetRuntimeGoArch() string {
+	return c.RuntimeGoArch
+}
+
+func (c *sysbenchRunnerConfigImpl) GetTestConfigs() []TestConfig {
+	return c.Tests
+}
+
+func (c *sysbenchRunnerConfigImpl) GetTestOptions() []string {
+	return c.TestOptions
+}
+
+func (c *sysbenchRunnerConfigImpl) GetServerConfigs() []ServerConfig {
+	return c.Servers
+}
+
+// Validate checks the config for the required fields and sets defaults
+// where necessary
+func (c *sysbenchRunnerConfigImpl) Validate(ctx context.Context) error {
+	if len(c.Servers) < 1 {
+		return ErrNoServersDefined
+	}
+	if len(c.Servers) > 2 {
+		return ErrTooManyServersDefined
+	}
+	err := c.setDefaults()
+	if err != nil {
+		return err
+	}
+	return c.validateServerConfigs()
+}
+
+// validateServerConfigs ensures the ServerConfigs are valid
+func (c *sysbenchRunnerConfigImpl) validateServerConfigs() error {
+	portMap := make(map[int]ServerType)
+	for _, s := range c.Servers {
+		st := s.GetServerType()
+		if st != Dolt && st != MySql && st != Doltgres && st != Postgres {
+			return fmt.Errorf("unsupported server type: %s", st)
+		}
+
+		err := s.Validate()
+		if err != nil {
+			return err
+		}
+
+		err = s.SetDefaults()
+		if err != nil {
+			return err
+		}
+
+		portMap, err = CheckUpdatePortMap(s, portMap)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *sysbenchRunnerConfigImpl) ContainsServerOfType(st ServerType) bool {
+	for _, s := range c.Servers {
+		if s.GetServerType() == st {
+			return true
+		}
+	}
+	return false
+}
+
+// setDefaults sets defaults on the sysbenchRunnerConfigImpl
+func (c *sysbenchRunnerConfigImpl) setDefaults() error {
+	if c.RuntimeOS == "" {
+		c.RuntimeOS = runtime.GOOS
+	}
+	if c.RuntimeGoArch == "" {
+		c.RuntimeGoArch = runtime.GOARCH
+	}
+	if len(c.Tests) < 1 {
+		fmt.Printf("Preparing to benchmark against default tests\n")
+		if c.ScriptDir != "" {
+			abs, err := filepath.Abs(c.ScriptDir)
+			if err != nil {
+				return err
+			}
+			if _, err := os.Stat(abs); os.IsNotExist(err) {
+				return fmt.Errorf("script dir not found: %s", abs)
+			}
+			c.ScriptDir = abs
+		}
+		tests, err := c.getDefaultTests()
+		if err != nil {
+			return err
+		}
+		c.Tests = tests
+	}
+	if c.Runs < 1 {
+		c.Runs = 1
+	}
+	return nil
+}
+
+func (c *sysbenchRunnerConfigImpl) getLuaScriptTestsFromDir(toInclude map[string]string) ([]TestConfig, error) {
+	luaScripts := make([]TestConfig, 0)
+	abs, err := filepath.Abs(c.ScriptDir)
+	if err != nil {
+		return nil, err
+	}
+	err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		file := filepath.Base(path)
+		if _, ok := toInclude[file]; ok {
+			luaScripts = append(luaScripts, NewTestConfig(path, []string{}, true))
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return luaScripts, nil
+}
+
+func (c *sysbenchRunnerConfigImpl) getDefaultTests() ([]TestConfig, error) {
+	defaultTests := make([]TestConfig, 0)
+	defaultTests = append(defaultTests, defaultSysbenchTests...)
+	if c.ScriptDir != "" {
+		var luaScriptTests []TestConfig
+		var err error
+		if !c.ContainsServerOfType(Doltgres) && !c.ContainsServerOfType(Postgres) {
+			luaScriptTests, err = c.getLuaScriptTestsFromDir(defaultDoltLuaScripts)
+		} else {
+			luaScriptTests, err = c.getLuaScriptTestsFromDir(defaultDoltgresLuaScripts)
+		}
+		if err != nil {
+			return nil, err
+		}
+		defaultTests = append(defaultTests, luaScriptTests...)
+	}
+	return defaultTests, nil
+}
+
+// CheckUpdatePortMap returns an error if multiple servers have specified the same port
+func CheckUpdatePortMap(serverConfig ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) {
+	port := serverConfig.GetPort()
+	st := serverConfig.GetServerType()
+	srv, ok := portMap[port]
+	if ok && srv != st {
+		return nil, fmt.Errorf("servers have port conflict on port: %d\n", port)
+	}
+	if !ok {
+		portMap[port] = st
+	}
+	return portMap, nil
+}
+
+// CheckExec verifies the binary exists
+func CheckExec(path, messageIfMissing string) error {
+	if path == "" {
+		return getMustSupplyError(messageIfMissing)
+	}
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return err
+	}
+	if _, err := os.Stat(abs); os.IsNotExist(err) {
+		return fmt.Errorf("exec not found: %s", abs)
+	}
+	return nil
+}
+
+// CheckProtocol ensures the given protocol is supported
+func CheckProtocol(protocol string) error {
+	if protocol == "" {
+		return getMustSupplyError("connection protocol")
+	}
+	if protocol == tcpProtocol || protocol == unixProtocol {
+		return nil
+	}
+	return ErrUnsupportedConnectionProtocol
+}
+
+// GetTests returns a slice of Tests
+func GetTests(config SysbenchConfig, serverConfig ServerConfig) ([]Test, error) {
+	flattened := make([]Test, 0)
+	for _, t := range config.GetTestConfigs() {
+		opts := config.GetTestOptions()
+		for _, o := range opts {
+			t.AppendOption(o)
+		}
+		tests, err := t.GetTests(serverConfig)
+		if err != nil {
+			return nil, err
+		}
+		flattened = append(flattened, tests...)
+	}
+	return flattened, nil
+}
+
+func getMustSupplyError(name string) error {
+	return fmt.Errorf("Must supply %s", name)
+}

go/performance/utils/benchmark_runner/sysbench_config_test.go

@@ -0,0 +1,133 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigTestGetTests(t *testing.T) {
+	empty := &testConfigImpl{Name: "test_name"}
+
+	one := &testConfigImpl{Name: "test_one", N: 3}
+	two := &testConfigImpl{Name: "test_two", N: 2}
+	three := &testConfigImpl{Name: "test_three", N: 1}
+
+	opts := &testConfigImpl{
+		Name:    "test_options",
+		N:       1,
+		Options: []string{"--create_secondary=on", "--auto_inc=off"},
+	}
+
+	serverConfig := &doltServerConfigImpl{Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat}
+
+	tests := []struct {
+		description   string
+		config        SysbenchConfig
+		expectedTests []testTest
+		expectedError error
+	}{
+		{
+			description: "should error if no test name is defined",
+			config: &sysbenchRunnerConfigImpl{
+				Servers: []ServerConfig{serverConfig},
+				Tests: []TestConfig{
+					&testConfigImpl{Name: ""},
+				},
+			},
+			expectedTests: nil,
+			expectedError: ErrTestNameNotDefined,
+		},
+		{
+			description: "should create single test if N is < 1",
+			config: &sysbenchRunnerConfigImpl{
+				Servers: []ServerConfig{serverConfig},
+				Tests:   []TestConfig{empty},
+			},
+			expectedTests: []testTest{
+				&testTestImpl{
+					&sysbenchTestImpl{
+						Name:   "test_name",
+						Params: serverConfig.GetTestingParams(empty),
+					},
+				},
+			},
+		},
+		{
+			description: "should return a test for each N defined on the TestConfigImpl",
+			config: &sysbenchRunnerConfigImpl{
+				Servers: []ServerConfig{serverConfig},
+				Tests:   []TestConfig{one, two, three},
+			},
+			expectedTests: []testTest{
+				&testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}},
+				&testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}},
+				&testTestImpl{&sysbenchTestImpl{Name: "test_one", Params: serverConfig.GetTestingParams(one)}},
+				&testTestImpl{&sysbenchTestImpl{Name: "test_two", Params: serverConfig.GetTestingParams(two)}},
+				&testTestImpl{&sysbenchTestImpl{Name: "test_two", Params: serverConfig.GetTestingParams(two)}},
+				&testTestImpl{&sysbenchTestImpl{Name: "test_three", Params: serverConfig.GetTestingParams(three)}},
+			},
+		},
+		{
+			description: "should apply user options to test params",
+			config: &sysbenchRunnerConfigImpl{
+				Servers: []ServerConfig{serverConfig},
+				Tests:   []TestConfig{opts},
+			},
+			expectedTests: []testTest{
+				&testTestImpl{&sysbenchTestImpl{Name: "test_options", Params: serverConfig.GetTestingParams(opts)}},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			svs := test.config.GetServerConfigs()
+			for _, s := range svs {
+				actual, err := GetTests(test.config, s)
+				assert.Equal(t, test.expectedError, err)
+				assert.Equal(t, len(test.expectedTests), len(actual))
+				updatedExpected := make([]SysbenchTest, len(actual))
+				for idx, a := range actual {
+					e := test.expectedTests[idx]
+					e.SetId(a.GetId())
+					updatedExpected[idx] = e.GetSysbenchTest()
+				}
+				assert.ElementsMatch(t, updatedExpected, actual)
+			}
+		})
+	}
+}
+
+type testTest interface {
+	SetId(id string)
+	GetSysbenchTest() SysbenchTest
+	SysbenchTest
+}
+
+type testTestImpl struct {
+	*sysbenchTestImpl
+}
+
+func (t *testTestImpl) GetSysbenchTest() SysbenchTest {
+	return t.sysbenchTestImpl
+}
+
+func (t *testTestImpl) SetId(id string) {
+	t.id = id
+}
+
+var _ testTest = &testTestImpl{}

go/performance/utils/benchmark_runner/sysbench_tests.go

@@ -0,0 +1,95 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+type sysbenchTestParamsImpl struct {
+	params []string
+}
+
+var _ SysbenchTestParams = &sysbenchTestParamsImpl{}
+
+func (s *sysbenchTestParamsImpl) ToSlice() []string {
+	return s.params
+}
+
+func (s *sysbenchTestParamsImpl) Append(params ...string) {
+	s.params = append(s.params, params...)
+}
+
+func NewSysbenchTestParams() *sysbenchTestParamsImpl {
+	return &sysbenchTestParamsImpl{params: make([]string, 0)}
+}
+
+// sysbenchTestImpl is a single sysbench test
+type sysbenchTestImpl struct {
+	id string
+
+	// Name is the test name
+	Name string
+
+	// Params are the parameters passed to sysbench
+	Params TestParams
+
+	// FromScript indicates if this test is from a lua script
+	FromScript bool
+}
+
+var _ SysbenchTest = &sysbenchTestImpl{}
+
+func NewSysbenchTest(id, name string, params TestParams, fromScript bool) *sysbenchTestImpl {
+	return &sysbenchTestImpl{
+		id:         id,
+		Name:       name,
+		Params:     params,
+		FromScript: fromScript,
+	}
+}
+
+func (t *sysbenchTestImpl) GetId() string {
+	return t.id
+}
+
+func (t *sysbenchTestImpl) GetName() string {
+	return t.Name
+}
+
+func (t *sysbenchTestImpl) GetParamsToSlice() []string {
+	return t.Params.ToSlice()
+}
+
+func (t *sysbenchTestImpl) GetFromScript() bool {
+	return t.FromScript
+}
+
+// PrepareArgs returns a test's args for sysbench's prepare step
+func (t *sysbenchTestImpl) GetPrepareArgs(serverConfig ServerConfig) []string {
+	return withCommand(t.Params, sysbenchPrepareCommand)
+}
+
+// Run returns a test's args for sysbench's run step
+func (t *sysbenchTestImpl) GetRunArgs(serverConfig ServerConfig) []string {
+	return withCommand(t.Params, sysbenchRunCommand)
+}
+
+// Cleanup returns a test's args for sysbench's cleanup step
+func (t *sysbenchTestImpl) GetCleanupArgs(serverConfig ServerConfig) []string {
+	return withCommand(t.Params, sysbenchCleanupCommand)
+}
+
+func withCommand(params TestParams, command string) []string {
+	c := make([]string, 0)
+	c = append(c, params.ToSlice()...)
+	return append(c, command)
+}

go/performance/utils/benchmark_runner/test_config.go

@@ -0,0 +1,88 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "github.com/google/uuid"
+
+type TestConfig interface {
+	GetName() string
+	GetOptions() []string
+	AppendOption(opt string)
+	GetTests(serverConfig ServerConfig) ([]Test, error)
+	NewId() string
+}
+
+type testConfigImpl struct {
+	// Name is the test name
+	Name string
+
+	// N is the number of times a test should run
+	N int
+
+	// Options are additional sysbench test options a user can supply to run with this test
+	Options []string
+
+	// FromScript is a boolean indicating that this test is from a lua script
+	FromScript bool
+}
+
+var _ TestConfig = &testConfigImpl{}
+
+func NewTestConfig(name string, opts []string, fromScript bool) *testConfigImpl {
+	options := make([]string, 0)
+	options = append(options, opts...)
+	return &testConfigImpl{
+		Name:       name,
+		N:          1,
+		Options:    options,
+		FromScript: fromScript,
+	}
+}
+
+func (ct *testConfigImpl) NewId() string {
+	return uuid.New().String()
+}
+
+func (ct *testConfigImpl) GetName() string {
+	return ct.Name
+}
+
+func (ct *testConfigImpl) GetOptions() []string {
+	return ct.Options
+}
+
+func (ct *testConfigImpl) AppendOption(opt string) {
+	ct.Options = append(ct.Options, opt)
+}
+
+func (ct *testConfigImpl) GetTests(serverConfig ServerConfig) ([]Test, error) {
+	if ct.Name == "" {
+		return nil, ErrTestNameNotDefined
+	}
+	if ct.N < 1 {
+		ct.N = 1
+	}
+
+	params := serverConfig.GetTestingParams(ct)
+	tests := make([]Test, 0)
+
+	for i := 0; i < ct.N; i++ {
+		//p := make([]string, params.Len())
+		//copy(p, params)
+		tests = append(tests, NewSysbenchTest(ct.NewId(), ct.Name, params, ct.FromScript))
+	}
+
+	return tests, nil
+}

go/performance/utils/benchmark_runner/tester.go

@@ -0,0 +1,55 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import "context"
+
+type Tester interface {
+	Test(ctx context.Context) (*Result, error)
+}
+
+type Test interface {
+	GetId() string
+	GetName() string
+	GetParamsToSlice() []string
+	GetPrepareArgs(serverConfig ServerConfig) []string
+	GetRunArgs(serverConfig ServerConfig) []string
+	GetCleanupArgs(serverConfig ServerConfig) []string
+}
+
+type SysbenchTest interface {
+	Test
+	GetFromScript() bool
+}
+
+type TestParams interface {
+	ToSlice() []string
+}
+
+type SysbenchTestParams interface {
+	TestParams
+	Append(params ...string)
+}
+
+type TpccTestParams interface {
+	TestParams
+	GetNumThreads() int
+	GetScaleFactor() int
+	GetTables() int
+	GetTrxLevel() string
+	GetReportCSV() bool
+	GetReportInterval() int
+	GetTime() int
+}

go/performance/utils/benchmark_runner/tpcc.go

@@ -0,0 +1,123 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+type tpccTesterImpl struct {
+	test         Test
+	config       Config
+	serverConfig ServerConfig
+	tpccCommand  string
+	serverParams []string
+	stampFunc    func() string
+	idFunc       func() string
+	suiteId      string
+}
+
+var _ Tester = &tpccTesterImpl{}
+
+func NewTpccTester(config TpccConfig, serverConfig ServerConfig, test Test, serverParams []string, stampFunc func() string) *tpccTesterImpl {
+	return &tpccTesterImpl{
+		tpccCommand:  filepath.Join(config.GetScriptDir(), tpccLuaFilename),
+		config:       config,
+		serverParams: serverParams,
+		serverConfig: serverConfig,
+		test:         test,
+		suiteId:      serverConfig.GetId(),
+		stampFunc:    stampFunc,
+	}
+}
+
+func (t *tpccTesterImpl) outputToResult(output []byte) (*Result, error) {
+	return OutputToResult(output, t.serverConfig.GetServerType(), t.serverConfig.GetVersion(), t.test.GetName(), t.test.GetId(), t.suiteId, t.config.GetRuntimeOs(), t.config.GetRuntimeGoArch(), t.serverParams, t.test.GetParamsToSlice(), nil, false)
+}
+
+func (t *tpccTesterImpl) prepare(ctx context.Context) error {
+	args := t.test.GetPrepareArgs(t.serverConfig)
+	cmd := ExecCommand(ctx, t.tpccCommand, args...)
+	cmd = t.updateCmdEnv(cmd)
+	out, err := cmd.Output()
+	if err != nil {
+		fmt.Println(string(out))
+		return err
+	}
+	return nil
+}
+
+func (t *tpccTesterImpl) run(ctx context.Context) (*Result, error) {
+	args := t.test.GetRunArgs(t.serverConfig)
+	cmd := ExecCommand(ctx, t.tpccCommand, args...)
+	cmd = t.updateCmdEnv(cmd)
+
+	out, err := cmd.Output()
+	if err != nil {
+		fmt.Print(string(out))
+		return nil, err
+	}
+
+	if Debug == true {
+		fmt.Print(string(out))
+	}
+
+	rs, err := t.outputToResult(out)
+	if err != nil {
+		return nil, err
+	}
+
+	rs.Stamp(t.stampFunc)
+
+	return rs, nil
+}
+
+func (t *tpccTesterImpl) cleanup(ctx context.Context) error {
+	args := t.test.GetCleanupArgs(t.serverConfig)
+	cmd := ExecCommand(ctx, t.tpccCommand, args...)
+	cmd = t.updateCmdEnv(cmd)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *tpccTesterImpl) Test(ctx context.Context) (*Result, error) {
+	err := t.prepare(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("Running test", t.test.GetName())
+
+	rs, err := t.run(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return rs, t.cleanup(ctx)
+}
+
+func (t *tpccTesterImpl) updateCmdEnv(cmd *exec.Cmd) *exec.Cmd {
+	lp := filepath.Join(t.config.GetScriptDir(), luaPath)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, fmt.Sprintf(luaPathEnvVarTemplate, lp))
+	return cmd
+}

go/performance/utils/benchmark_runner/tpcc_config.go

@@ -0,0 +1,145 @@
+// Copyright 2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+)
+
+var defaultTpccParams = []string{
+	fmt.Sprintf("%s=%s", tpccMysqlDbFlag, tpccDbName),
+	fmt.Sprintf("%s=%s", tpccDbDriverFlag, mysqlDriverName),
+}
+
+// tpccConfigImpl represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests
+// against different ServerConfigurations.
+type tpccConfigImpl struct {
+	// RuntimeOS is the platform the benchmarks ran on
+	RuntimeOS string
+
+	// RuntimeGoArch is the runtime architecture
+	RuntimeGoArch string
+
+	// ScriptDir represents the location of the TPCC tests
+	ScriptDir string
+
+	// Servers are the servers to benchmark.
+	Servers []ServerConfig
+
+	// ScaleFactors represent the scale at which to run each TpccBenchmark at.
+	ScaleFactors []int
+
+	// NomsBinFormat specifies the NomsBinFormat
+	NomsBinFormat string
+}
+
+var _ TpccConfig = &tpccConfigImpl{}
+
+func NewTpccRunnerConfig() *tpccConfigImpl {
+	return &tpccConfigImpl{
+		Servers:      make([]ServerConfig, 0),
+		ScaleFactors: make([]int, 0),
+	}
+}
+
+func (c *tpccConfigImpl) GetRuns() int {
+	return 1
+}
+
+func (c *tpccConfigImpl) GetScriptDir() string {
+	return c.ScriptDir
+}
+
+func (c *tpccConfigImpl) GetNomsBinFormat() string {
+	return c.NomsBinFormat
+}
+
+func (c *tpccConfigImpl) GetRuntimeOs() string {
+	return c.RuntimeOS
+}
+
+func (c *tpccConfigImpl) GetRuntimeGoArch() string {
+	return c.RuntimeGoArch
+}
+
+func (c *tpccConfigImpl) ContainsServerOfType(st ServerType) bool {
+	for _, s := range c.Servers {
+		if s.GetServerType() == st {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *tpccConfigImpl) GetScaleFactors() []int {
+	return c.ScaleFactors
+}
+
+func (c *tpccConfigImpl) GetServerConfigs() []ServerConfig {
+	return c.Servers
+}
+
+func (c *tpccConfigImpl) setDefaults() {
+	// TODO: Eventually we need to support scale factors all the way to 10
+	if len(c.ScaleFactors) == 0 {
+		c.ScaleFactors = append(c.ScaleFactors, 1)
+	}
+	if c.RuntimeOS == "" {
+		c.RuntimeOS = runtime.GOOS
+	}
+	if c.RuntimeGoArch == "" {
+		c.RuntimeGoArch = runtime.GOARCH
+	}
+}
+
+// validateServerConfigs ensures the ServerConfigs are valid
+func (c *tpccConfigImpl) validateServerConfigs() error {
+	portMap := make(map[int]ServerType)
+	for _, s := range c.Servers {
+		st := s.GetServerType()
+		if st != Dolt && st != MySql && st != Doltgres && st != Postgres {
+			return fmt.Errorf("unsupported server type: %s", st)
+		}
+
+		err := s.Validate()
+		if err != nil {
+			return err
+		}
+
+		err = s.SetDefaults()
+		if err != nil {
+			return err
+		}
+
+		portMap, err = CheckUpdatePortMap(s, portMap)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *tpccConfigImpl) Validate(ctx context.Context) error {
+	if len(c.Servers) < 1 {
+		return ErrNoServersDefined
+	}
+	if len(c.Servers) > 2 {
+		return ErrTooManyServersDefined
+	}
+	c.setDefaults()
+	return c.validateServerConfigs()
+}

go/performance/utils/benchmark_runner/tpcc_tests.go

@@ -0,0 +1,242 @@
+// Copyright 2019-2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package benchmark_runner
+
+import (
+	"fmt"
+
+	"github.com/google/uuid"
+)
+
+type tpccTestParamsImpl struct {
+	// NumThreads represents the number of threads running queries concurrently.
+	NumThreads int
+
+	// ScaleFactor represents the number of warehouse to test this at scale.
+	ScaleFactor int
+
+	// Tables represents the number of tables created per warehouse.
+	Tables int
+
+	// TrxLevel represents what transaction level to use
+	TrxLevel string
+
+	// ReportCSV determines whether to report output as a csv.
+	ReportCSV bool
+
+	// ReportInterval defines how often the tpcc benchmark outputs performance stats.
+	ReportInterval int
+
+	// Time represents how long
+	Time int
+}
+
+var _ TestParams = &tpccTestParamsImpl{}
+
+// NewDefaultTpccParams returns default TpccTestParams.
+func NewDefaultTpccParams() *tpccTestParamsImpl {
+	return &tpccTestParamsImpl{
+		NumThreads:     2, // TODO: When ready, expose as command line argument.
+		ScaleFactor:    1,
+		Tables:         1,
+		TrxLevel:       tpccTransactionLevelRr,
+		ReportCSV:      true,
+		ReportInterval: 1,
+		Time:           30,
+	}
+}
+
+func (t *tpccTestParamsImpl) GetNumThreads() int {
+	return t.NumThreads
+}
+
+func (t *tpccTestParamsImpl) GetScaleFactor() int {
+	return t.ScaleFactor
+}
+
+func (t *tpccTestParamsImpl) GetTables() int {
+	return t.Tables
+}
+
+func (t *tpccTestParamsImpl) GetTrxLevel() string {
+	return t.TrxLevel
+}
+
+func (t *tpccTestParamsImpl) GetReportCSV() bool {
+	return t.ReportCSV
+}
+
+func (t *tpccTestParamsImpl) GetReportInterval() int {
+	return t.ReportInterval
+}
+
+func (t *tpccTestParamsImpl) GetTime() int {
+	return t.Time
+}
+
+func (t *tpccTestParamsImpl) ToSlice() []string {
+	params := make([]string, 0)
+	params = append(params, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.NumThreads))
+	params = append(params, fmt.Sprintf("%s=%d", tpccScaleFlag, t.ScaleFactor))
+	params = append(params, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Tables))
+	params = append(params, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.TrxLevel))
+	params = append(params, fmt.Sprintf("%s=%t", tpccReportCsv, t.ReportCSV))
+	params = append(params, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.ReportInterval))
+	params = append(params, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Time))
+	return params
+}
+
+// tpccTestImpl encapsulates an End to End prepare, run, cleanup test case.
+type tpccTestImpl struct {
+	// Id represents a unique test id
+	Id string
+
+	// Name represents the name of the test case
+	Name string
+
+	// Params are associated parameters this test runs with
+	Params TpccTestParams
+}
+
+var _ Test = &tpccTestImpl{}
+
+// NewTpccTest instantiates and returns a TPCC test.
+func NewTpccTest(name string, params TpccTestParams) *tpccTestImpl {
+	return &tpccTestImpl{
+		Id:     uuid.New().String(),
+		Name:   name,
+		Params: params,
+	}
+}
+
+func (t *tpccTestImpl) doltArgs(serverConfig ServerConfig) []string {
+	args := make([]string, 0)
+	args = append(args, defaultTpccParams...)
+	args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, serverConfig.GetHost()))
+	port := serverConfig.GetPort()
+	if port > 0 {
+		args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort()))
+	}
+	args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort()))
+	args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser))
+	args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor()))
+	args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel()))
+	return args
+}
+
+func (t *tpccTestImpl) mysqlArgs(serverConfig ServerConfig) []string {
+	args := make([]string, 0)
+	args = append(args, defaultTpccParams...)
+	host := serverConfig.GetHost()
+	port := serverConfig.GetPort()
+	args = append(args, fmt.Sprintf("%s=%s", tpccMysqlHostFlag, host))
+	if host == defaultHost {
+		args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, tpccMysqlUsername))
+		args = append(args, fmt.Sprintf("%s=%s", tpccMysqlPasswordFlag, tpccPassLocal))
+	} else {
+		args = append(args, fmt.Sprintf("%s=%s", tpccMysqlUserFlag, defaultMysqlUser))
+	}
+	if port > 0 {
+		args = append(args, fmt.Sprintf("%s=%d", tpccMysqlPortFlag, serverConfig.GetPort()))
+	}
+	args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor()))
+	args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel()))
+	return args
+}
+
+func (t *tpccTestImpl) doltgresArgs(serverConfig ServerConfig) []string {
+	args := make([]string, 0)
+	args = append(args, defaultTpccParams...)
+	args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor()))
+	args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel()))
+	return args
+}
+
+func (t *tpccTestImpl) postgresArgs(serverConfig ServerConfig) []string {
+	args := make([]string, 0)
+	args = append(args, defaultTpccParams...)
+	args = append(args, fmt.Sprintf("%s=%d", tpccTimeFlag, t.Params.GetTime()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccThreadsFlag, t.Params.GetNumThreads()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccReportIntervalFlag, t.Params.GetReportInterval()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccTablesFlag, t.Params.GetTables()))
+	args = append(args, fmt.Sprintf("%s=%d", tpccScaleFlag, t.Params.GetScaleFactor()))
+	args = append(args, fmt.Sprintf("%s=%s", tpccTransactionLevelFlag, t.Params.GetTrxLevel()))
+	return args
+}
+
+// getArgs returns a test's args for all TPCC steps
+func (t *tpccTestImpl) getArgs(serverConfig ServerConfig) []string {
+	st := serverConfig.GetServerType()
+	switch st {
+	case Dolt:
+		return t.doltArgs(serverConfig)
+	case Doltgres:
+		return t.doltgresArgs(serverConfig)
+	case Postgres:
+		return t.postgresArgs(serverConfig)
+	case MySql:
+		return t.mysqlArgs(serverConfig)
+	default:
+		panic(fmt.Sprintf("unexpected server type: %s", st))
+	}
+}
+
+func (t *tpccTestImpl) GetId() string {
+	return t.Id
+}
+
+func (t *tpccTestImpl) GetName() string {
+	return t.Name
+}
+
+func (t *tpccTestImpl) GetParamsToSlice() []string {
+	return t.Params.ToSlice()
+}
+
+func (t *tpccTestImpl) GetPrepareArgs(serverConfg ServerConfig) []string {
+	args := make([]string, 0)
+	serverArgs := t.getArgs(serverConfg)
+	args = append(args, serverArgs...)
+	args = append(args, sysbenchPrepareCommand)
+	return args
+}
+
+func (t *tpccTestImpl) GetRunArgs(serverConfg ServerConfig) []string {
+	args := make([]string, 0)
+	serverArgs := t.getArgs(serverConfg)
+	args = append(args, serverArgs...)
+	args = append(args, sysbenchRunCommand)
+	return args
+}
+
+func (t *tpccTestImpl) GetCleanupArgs(serverConfg ServerConfig) []string {
+	args := make([]string, 0)
+	serverArgs := t.getArgs(serverConfg)
+	args = append(args, serverArgs...)
+	args = append(args, sysbenchCleanupCommand)
+	return args
+}

go/performance/utils/sysbench_runner/cmd/main.go

@@ -1,54 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-	"path/filepath"
-
-	runner "github.com/dolthub/dolt/go/performance/utils/sysbench_runner"
-)
-
-var configFile = flag.String("config", "", "path to config file")
-
-func main() {
-	flag.Parse()
-
-	if *configFile == "" {
-		log.Fatal("Must supply config")
-	}
-
-	configPath, err := filepath.Abs(*configFile)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if _, err = os.Stat(configPath); os.IsNotExist(err) {
-		log.Fatal(err)
-	}
-
-	config, err := runner.FromFileConfig(configPath)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = runner.Run(config)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	os.Exit(0)
-}

go/performance/utils/sysbench_runner/config.go

@@ -1,616 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"runtime"
-
-	"github.com/google/uuid"
-)
-
-const (
-	Dolt     ServerType = "dolt"
-	Doltgres ServerType = "doltgres"
-	Postgres ServerType = "postgres"
-	MySql    ServerType = "mysql"
-
-	CsvFormat  = "csv"
-	JsonFormat = "json"
-
-	CsvExt  = ".csv"
-	JsonExt = ".json"
-
-	defaultHost = "127.0.0.1"
-	defaultPort = 3306
-
-	defaultMysqlSocket = "/var/run/mysqld/mysqld.sock"
-
-	tcpProtocol  = "tcp"
-	unixProtocol = "unix"
-
-	sysbenchUsername  = "sysbench"
-	sysbenchUserLocal = "'sysbench'@'localhost'"
-	sysbenchPassLocal = "sysbenchpass"
-
-	userFlag                    = "--user"
-	hostFlag                    = "--host"
-	portFlag                    = "--port"
-	skipBinLogFlag              = "--skip-log-bin"
-	profileFlag                 = "--prof"
-	profilePathFlag             = "--prof-path"
-	cpuProfile                  = "cpu"
-	doltgresDataDirFlag         = "--data-dir"
-	MysqlDataDirFlag            = "--datadir"
-	MysqlInitializeInsecureFlag = "--initialize-insecure"
-	cpuProfileFilename          = "cpu.pprof"
-)
-
-var (
-	ErrTestNameNotDefined            = errors.New("test name not defined")
-	ErrNoServersDefined              = errors.New("servers not defined")
-	ErrTooManyServersDefined         = errors.New("too many servers defined, two max")
-	ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol")
-)
-
-var defaultSysbenchParams = []string{
-	"--db-ps-mode=disable",
-	"--rand-type=uniform",
-}
-
-var defaultDoltServerParams = []string{"sql-server"}
-
-var defaultSysbenchTests = []*ConfigTest{
-	NewConfigTest("oltp_read_only", []string{}, false),
-	NewConfigTest("oltp_insert", []string{}, false),
-	NewConfigTest("bulk_insert", []string{}, false),
-	NewConfigTest("oltp_point_select", []string{}, false),
-	NewConfigTest("select_random_points", []string{}, false),
-	NewConfigTest("select_random_ranges", []string{}, false),
-	NewConfigTest("oltp_write_only", []string{}, false),
-	NewConfigTest("oltp_read_write", []string{}, false),
-	NewConfigTest("oltp_update_index", []string{}, false),
-	NewConfigTest("oltp_update_non_index", []string{}, false),
-}
-
-var defaultDoltLuaScripts = map[string]string{
-	"covering_index_scan.lua": "covering_index_scan.lua",
-	"groupby_scan.lua":        "groupby_scan.lua",
-	"index_join.lua":          "index_join.lua",
-	"index_join_scan.lua":     "index_join_scan.lua",
-	"index_scan.lua":          "index_scan.lua",
-	"oltp_delete_insert.lua":  "oltp_delete_insert.lua",
-	"table_scan.lua":          "table_scan.lua",
-	"types_delete_insert.lua": "types_delete_insert.lua",
-	"types_table_scan.lua":    "types_table_scan.lua",
-}
-
-// todo: check expressions need to be supported in doltgres for these
-// todo: postgres does not have geometry types also
-var defaultDoltgresLuaScripts = map[string]string{
-	//"covering_index_scan_postgres.lua": "covering_index_scan_postgres.lua",
-	//"groupby_scan_postgres.lua":        "groupby_scan_postgres.lua",
-	//"index_join_postgres.lua":          "index_join_postgres.lua",
-	//"index_join_scan_postgres.lua":     "index_join_scan_postgres.lua",
-	//"index_scan_postgres.lua":          "index_scan_postgres.lua",
-	//"oltp_delete_insert_postgres.lua":  "oltp_delete_insert_postgres.lua",
-	//"table_scan_postgres.lua":          "table_scan_postgres.lua",
-	//"types_delete_insert_postgres.lua": "types_delete_insert_postgres.lua",
-	//"types_table_scan_postgres.lua":    "types_table_scan_postgres.lua",
-}
-
-type ServerType string
-
-// Test is a single sysbench test
-type Test struct {
-	id string
-
-	// Name is the test name
-	Name string
-
-	// Params are the parameters passed to sysbench
-	Params []string
-
-	// FromScript indicates if this test is from a lua script
-	FromScript bool
-}
-
-// Prepare returns a test's args for sysbench's prepare step
-func (t *Test) Prepare() []string {
-	return withCommand(t.Params, "prepare")
-}
-
-// Run returns a test's args for sysbench's run step
-func (t *Test) Run() []string {
-	return withCommand(t.Params, "run")
-}
-
-// Cleanup returns a test's args for sysbench's cleanup step
-func (t *Test) Cleanup() []string {
-	return withCommand(t.Params, "cleanup")
-}
-
-func withCommand(params []string, command string) []string {
-	c := make([]string, 0)
-	c = append(c, params...)
-	return append(c, command)
-}
-
-// ConfigTest provides users a way to define a test for multiple tablesizes
-type ConfigTest struct {
-	// Name is the test name
-	Name string
-
-	// N is the number of times a test should run
-	N int
-
-	// Options are additional sysbench test options a user can supply to run with this test
-	Options []string
-
-	// FromScript is a boolean indicating that this test is from a lua script
-	FromScript bool
-}
-
-// NewConfigTest returns a ConfigTest containing the supplied args
-func NewConfigTest(name string, opts []string, fromScript bool) *ConfigTest {
-	options := make([]string, 0)
-	options = append(options, opts...)
-	return &ConfigTest{
-		Name:       name,
-		N:          1,
-		Options:    options,
-		FromScript: fromScript,
-	}
-}
-
-// GetTests returns a slice of Tests
-func (ct *ConfigTest) GetTests(serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) {
-	if ct.Name == "" {
-		return nil, ErrTestNameNotDefined
-	}
-	if ct.N < 1 {
-		ct.N = 1
-	}
-
-	params := fromConfigTestParams(ct, serverConfig)
-	tests := make([]*Test, 0)
-
-	var idFunc func() string
-	if testIdFunc == nil {
-		idFunc = func() string {
-			return uuid.New().String()
-		}
-	} else {
-		idFunc = testIdFunc
-	}
-
-	for i := 0; i < ct.N; i++ {
-		p := make([]string, len(params))
-		copy(p, params)
-		tests = append(tests, &Test{
-			id:         idFunc(),
-			Name:       ct.Name,
-			Params:     p,
-			FromScript: ct.FromScript,
-		})
-	}
-	return tests, nil
-}
-
-// fromConfigTestParams returns params formatted for sysbench:
-// `sysbench [options]... [testname] [command]`
-func fromConfigTestParams(ct *ConfigTest, serverConfig *ServerConfig) []string {
-	params := make([]string, 0)
-	params = append(params, defaultSysbenchParams...)
-	if serverConfig.Server == MySql || serverConfig.Server == Dolt {
-		params = append(params, fmt.Sprintf("--mysql-db=%s", dbName))
-		params = append(params, "--db-driver=mysql")
-		params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host))
-		if serverConfig.Port != 0 {
-			params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port))
-		}
-	} else if serverConfig.Server == Doltgres || serverConfig.Server == Postgres {
-		params = append(params, "--db-driver=pgsql")
-		params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName))
-		params = append(params, fmt.Sprintf("--pgsql-host=%s", serverConfig.Host))
-		if serverConfig.Port != 0 {
-			params = append(params, fmt.Sprintf("--pgsql-port=%d", serverConfig.Port))
-		}
-	}
-
-	// handle sysbench user for local mysql server
-	if serverConfig.Server == MySql && serverConfig.Host == defaultHost {
-		params = append(params, "--mysql-user=sysbench")
-		params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal))
-	} else if serverConfig.Server == Dolt {
-		params = append(params, "--mysql-user=root")
-	} else if serverConfig.Server == Doltgres {
-		params = append(params, "--pgsql-user=doltgres")
-	} else if serverConfig.Server == Postgres {
-		params = append(params, "--pgsql-user=postgres")
-	}
-
-	params = append(params, ct.Options...)
-	params = append(params, ct.Name)
-	return params
-}
-
-// ServerConfig is the configuration for a server to test against
-type ServerConfig struct {
-	// Id is a unique id for this servers benchmarking
-	Id string
-
-	// Host is the server host
-	Host string
-
-	// Port is the server port
-	Port int
-
-	// Server is the type of server
-	Server ServerType
-
-	// Version is the server version
-	Version string
-
-	// ResultsFormat is the format the results should be written in
-	ResultsFormat string
-
-	// ServerExec is the path to a server executable
-	ServerExec string
-
-	// InitExec is the path to the server init db executable
-	InitExec string
-
-	// ServerUser is the user account that should start the server
-	ServerUser string
-
-	// SkipLogBin will skip bin logging
-	SkipLogBin bool
-
-	// ServerArgs are the args used to start a server
-	ServerArgs []string
-
-	// ConnectionProtocol defines the protocol for connecting to the server
-	ConnectionProtocol string
-
-	// Socket is the path to the server socket
-	Socket string
-
-	// ServerProfile specifies the golang profile to take of a Dolt server
-	ServerProfile string
-
-	// ProfilePath path to directory where server profile will be written
-	ProfilePath string
-}
-
-func (sc *ServerConfig) GetId() string {
-	if sc.Id == "" {
-		sc.Id = uuid.New().String()
-	}
-	return sc.Id
-}
-
-// GetServerArgs returns the args used to start a server
-func (sc *ServerConfig) GetServerArgs() ([]string, error) {
-	params := make([]string, 0)
-
-	if sc.Server == Dolt {
-		params = append(params, defaultDoltServerParams...)
-	} else if sc.Server == MySql {
-		if sc.ServerUser != "" {
-			params = append(params, fmt.Sprintf("%s=%s", userFlag, sc.ServerUser))
-		}
-		if sc.SkipLogBin {
-			params = append(params, skipBinLogFlag)
-		}
-	}
-
-	if sc.Server == Dolt || sc.Server == Doltgres {
-		params = append(params, fmt.Sprintf("%s=%s", hostFlag, sc.Host))
-	}
-	if sc.Port != 0 {
-		params = append(params, fmt.Sprintf("%s=%d", portFlag, sc.Port))
-	}
-
-	params = append(params, sc.ServerArgs...)
-	return params, nil
-}
-
-// Config is the configuration for a benchmarking run
-type Config struct {
-	// Runs is the number of times to run all tests
-	Runs int
-	// RuntimeOS is the platform the benchmarks ran on
-	RuntimeOS string
-	// RuntimeGoArch is the runtime architecture
-	RuntimeGoArch string
-	// Servers are the servers to benchmark
-	Servers []*ServerConfig
-	// Tests are the tests to run. If no tests are provided,
-	// the default tests will be used
-	Tests []*ConfigTest
-	// TestOptions a list of sysbench test options to apply to all tests
-	TestOptions []string
-	// ScriptDir is a path to a directory of lua scripts
-	ScriptDir string
-	// InitBigRepo downloads a database with existing chunks and commits
-	InitBigRepo bool
-	// NomsBinFormat specifies the NomsBinFormat
-	NomsBinFormat string
-}
-
-// NewConfig returns a new Config
-func NewConfig() *Config {
-	return &Config{
-		Servers: make([]*ServerConfig, 0),
-	}
-}
-
-// Validate checks the config for the required fields and sets defaults
-// where necessary
-func (c *Config) Validate() error {
-	if len(c.Servers) < 1 {
-		return ErrNoServersDefined
-	}
-	if len(c.Servers) > 2 {
-		return ErrTooManyServersDefined
-	}
-	err := c.setDefaults()
-	if err != nil {
-		return err
-	}
-	return c.validateServerConfigs()
-}
-
-// validateServerConfigs ensures the ServerConfigs are valid
-func (c *Config) validateServerConfigs() error {
-	portMap := make(map[int]ServerType)
-	for _, s := range c.Servers {
-		if s.Server != Dolt && s.Server != MySql && s.Server != Doltgres && s.Server != Postgres {
-			return fmt.Errorf("unsupported server type: %s", s.Server)
-		}
-
-		err := ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat)
-		if err != nil {
-			return err
-		}
-
-		if s.Server == MySql {
-			err = CheckProtocol(s.ConnectionProtocol)
-			if err != nil {
-				return err
-			}
-		}
-
-		if s.Host == "" {
-			s.Host = defaultHost
-		}
-
-		portMap, err = CheckUpdatePortMap(s, portMap)
-		if err != nil {
-			return err
-		}
-
-		err = CheckExec(s.ServerExec, "server exec")
-		if err != nil {
-			return err
-		}
-
-		if s.Server == Postgres {
-			err = CheckExec(s.InitExec, "initdb exec")
-			if err != nil {
-				return err
-			}
-		}
-
-		if s.Server != Dolt && s.ServerProfile != "" {
-			return fmt.Errorf("profiling can only be done against a dolt server")
-		}
-
-		if s.Server == Dolt && s.ServerProfile != "" {
-			if s.ServerProfile != cpuProfile {
-				return fmt.Errorf("unsupported server profile: %s", s.ServerProfile)
-			}
-			if s.ProfilePath == "" {
-				cwd, err := os.Getwd()
-				if err != nil {
-					return err
-				}
-				s.ProfilePath = cwd
-			}
-		}
-	}
-
-	return nil
-}
-
-func (c *Config) Contains(st ServerType) bool {
-	for _, s := range c.Servers {
-		if s.Server == st {
-			return true
-		}
-	}
-	return false
-}
-
-func ValidateRequiredFields(server, version, format string) error {
-	if server == "" {
-		return getMustSupplyError("server")
-	}
-	if version == "" {
-		return getMustSupplyError("version")
-	}
-	if format == "" {
-		return getMustSupplyError("results format")
-	}
-	return nil
-}
-
-// setDefaults sets defaults on the Config
-func (c *Config) setDefaults() error {
-	if c.RuntimeOS == "" {
-		c.RuntimeOS = runtime.GOOS
-	}
-	if c.RuntimeGoArch == "" {
-		c.RuntimeGoArch = runtime.GOARCH
-	}
-	if len(c.Tests) < 1 {
-		fmt.Printf("Preparing to benchmark against default tests\n")
-		if c.ScriptDir != "" {
-			abs, err := filepath.Abs(c.ScriptDir)
-			if err != nil {
-				return err
-			}
-			if _, err := os.Stat(abs); os.IsNotExist(err) {
-				return fmt.Errorf("script dir not found: %s", abs)
-			}
-			c.ScriptDir = abs
-		}
-		tests, err := getDefaultTests(c)
-		if err != nil {
-			return err
-		}
-		c.Tests = tests
-	}
-	if c.Runs < 1 {
-		c.Runs = 1
-	}
-	return nil
-}
-
-// CheckUpdatePortMap returns an error if multiple servers have specified the same port
-func CheckUpdatePortMap(serverConfig *ServerConfig, portMap map[int]ServerType) (map[int]ServerType, error) {
-	if serverConfig.Port == 0 {
-		serverConfig.Port = defaultPort
-	}
-	srv, ok := portMap[serverConfig.Port]
-	if ok && srv != serverConfig.Server {
-		return nil, fmt.Errorf("servers have port conflict on port: %d\n", serverConfig.Port)
-	}
-	if !ok {
-		portMap[serverConfig.Port] = serverConfig.Server
-	}
-	return portMap, nil
-}
-
-// CheckExec verifies the binary exists
-func CheckExec(path, messageIfMissing string) error {
-	if path == "" {
-		return getMustSupplyError(messageIfMissing)
-	}
-	abs, err := filepath.Abs(path)
-	if err != nil {
-		return err
-	}
-	if _, err := os.Stat(abs); os.IsNotExist(err) {
-		return fmt.Errorf("exec not found: %s", abs)
-	}
-	return nil
-}
-
-// CheckProtocol ensures the given protocol is supported
-func CheckProtocol(protocol string) error {
-	if protocol == "" {
-		return getMustSupplyError("connection protocol")
-	}
-	if protocol == tcpProtocol || protocol == unixProtocol {
-		return nil
-	}
-	return ErrUnsupportedConnectionProtocol
-}
-
-// GetTests returns a slice of Tests created from the
-// defined ServerConfig.Tests
-func GetTests(config *Config, serverConfig *ServerConfig, testIdFunc func() string) ([]*Test, error) {
-	flattened := make([]*Test, 0)
-	for _, t := range config.Tests {
-		if len(config.TestOptions) > 0 {
-			t.Options = append(t.Options, config.TestOptions...)
-		}
-		tests, err := t.GetTests(serverConfig, testIdFunc)
-		if err != nil {
-			return nil, err
-		}
-		flattened = append(flattened, tests...)
-	}
-	return flattened, nil
-}
-
-// FromFileConfig returns a validated Config based on the config file at the configPath
-func FromFileConfig(configPath string) (*Config, error) {
-	data, err := os.ReadFile(configPath)
-	if err != nil {
-		return nil, err
-	}
-
-	config := NewConfig()
-	err = json.Unmarshal(data, config)
-	if err != nil {
-		return nil, err
-	}
-
-	return config, nil
-}
-
-func getMustSupplyError(name string) error {
-	return fmt.Errorf("Must supply %s", name)
-}
-
-func getDefaultTests(config *Config) ([]*ConfigTest, error) {
-	defaultTests := make([]*ConfigTest, 0)
-	defaultTests = append(defaultTests, defaultSysbenchTests...)
-	if config.ScriptDir != "" {
-		var luaScriptTests []*ConfigTest
-		var err error
-		if !config.Contains(Doltgres) && !config.Contains(Postgres) {
-			luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltLuaScripts)
-		} else {
-			luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltgresLuaScripts)
-		}
-		if err != nil {
-			return nil, err
-		}
-		defaultTests = append(defaultTests, luaScriptTests...)
-	}
-	return defaultTests, nil
-}
-
-func getLuaScriptTestsFromDir(dir string, toInclude map[string]string) ([]*ConfigTest, error) {
-	luaScripts := make([]*ConfigTest, 0)
-	abs, err := filepath.Abs(dir)
-	if err != nil {
-		return nil, err
-	}
-	err = filepath.Walk(abs, func(path string, info fs.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		file := filepath.Base(path)
-		if _, ok := toInclude[file]; ok {
-			luaScripts = append(luaScripts, NewConfigTest(path, []string{}, true))
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return luaScripts, nil
-}

go/performance/utils/sysbench_runner/config_test.go

@@ -1,107 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-var testIdFunc = func() string { return "id" }
-
-func TestConfigTestGetTests(t *testing.T) {
-	empty := &ConfigTest{Name: "test_name"}
-
-	one := &ConfigTest{Name: "test_one", N: 3}
-	two := &ConfigTest{Name: "test_two", N: 2}
-	three := &ConfigTest{Name: "test_three", N: 1}
-
-	opts := &ConfigTest{
-		Name:    "test_options",
-		N:       1,
-		Options: []string{"--create_secondary=on", "--auto_inc=off"},
-	}
-
-	serverConfig := &ServerConfig{Server: MySql, Version: "test-version", Host: "localhost", ResultsFormat: CsvFormat}
-
-	tests := []struct {
-		description   string
-		config        *Config
-		expectedTests []*Test
-		expectedError error
-	}{
-		{
-			description: "should error if no test name is defined",
-			config: &Config{
-				Servers: []*ServerConfig{serverConfig},
-				Tests: []*ConfigTest{
-					{Name: ""},
-				},
-			},
-			expectedTests: nil,
-			expectedError: ErrTestNameNotDefined,
-		},
-		{
-			description: "should create single test if N is < 1",
-			config: &Config{
-				Servers: []*ServerConfig{serverConfig},
-				Tests:   []*ConfigTest{empty},
-			},
-			expectedTests: []*Test{
-				{
-					id:     testIdFunc(),
-					Name:   "test_name",
-					Params: fromConfigTestParams(empty, serverConfig),
-				},
-			},
-		},
-		{
-			description: "should return a test for each N defined on the ConfigTest",
-			config: &Config{
-				Servers: []*ServerConfig{serverConfig},
-				Tests:   []*ConfigTest{one, two, three},
-			},
-			expectedTests: []*Test{
-				{id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)},
-				{id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)},
-				{id: testIdFunc(), Name: "test_one", Params: fromConfigTestParams(one, serverConfig)},
-				{id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)},
-				{id: testIdFunc(), Name: "test_two", Params: fromConfigTestParams(two, serverConfig)},
-				{id: testIdFunc(), Name: "test_three", Params: fromConfigTestParams(three, serverConfig)},
-			},
-		},
-		{
-			description: "should apply user options to test params",
-			config: &Config{
-				Servers: []*ServerConfig{serverConfig},
-				Tests:   []*ConfigTest{opts},
-			},
-			expectedTests: []*Test{
-				{id: testIdFunc(), Name: "test_options", Params: fromConfigTestParams(opts, serverConfig)},
-			},
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.description, func(t *testing.T) {
-			for _, s := range test.config.Servers {
-				actual, err := GetTests(test.config, s, testIdFunc)
-				assert.Equal(t, test.expectedError, err)
-				assert.Equal(t, len(test.expectedTests), len(actual))
-				assert.ElementsMatch(t, test.expectedTests, actual)
-			}
-		})
-	}
-}

go/performance/utils/sysbench_runner/dolt.go

@@ -1,296 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/exec"
-	"os/signal"
-	"path/filepath"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/dolthub/dolt/go/store/types"
-
-	"golang.org/x/sync/errgroup"
-)
-
-const (
-	dbName       = "test"
-	luaPath      = "?.lua"
-	bigEmptyRepo = "max-hoffman/big-empty"
-	nbfEnvVar    = "DOLT_DEFAULT_BIN_FORMAT"
-)
-
-var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) }
-
-// BenchmarkDolt benchmarks dolt based on the provided configurations
-func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) {
-	serverParams, err := serverConfig.GetServerArgs()
-	if err != nil {
-		return nil, err
-	}
-
-	err = DoltVersion(ctx, serverConfig.ServerExec)
-	if err != nil {
-		return nil, err
-	}
-
-	err = UpdateDoltConfig(ctx, serverConfig.ServerExec)
-	if err != nil {
-		return nil, err
-	}
-
-	testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat)
-	if err != nil {
-		return nil, err
-	}
-
-	withKeyCtx, cancel := context.WithCancel(ctx)
-	gServer, serverCtx := errgroup.WithContext(withKeyCtx)
-
-	server := getServer(serverCtx, serverConfig, testRepo, serverParams)
-
-	// handle user interrupt
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		<-quit
-		defer wg.Done()
-		signal.Stop(quit)
-		cancel()
-	}()
-
-	// launch the dolt server
-	gServer.Go(func() error {
-		return server.Run()
-	})
-
-	// sleep to allow the server to start
-	time.Sleep(5 * time.Second)
-
-	tests, err := GetTests(config, serverConfig, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	results := make(Results, 0)
-	for i := 0; i < config.Runs; i++ {
-		for _, test := range tests {
-			r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId())
-			if err != nil {
-				close(quit)
-				wg.Wait()
-				return nil, err
-			}
-			results = append(results, r)
-		}
-	}
-
-	// send signal to dolt server
-	quit <- syscall.SIGTERM
-
-	err = gServer.Wait()
-	if err != nil {
-		// we expect a kill error
-		// we only exit in error if this is not the
-		// error
-		if err.Error() != "signal: killed" {
-			fmt.Println(err)
-			close(quit)
-			wg.Wait()
-			return nil, err
-		}
-	}
-
-	fmt.Println("Successfully killed server")
-	close(quit)
-	wg.Wait()
-
-	err = os.RemoveAll(testRepo)
-	if err != nil {
-		return nil, err
-	}
-
-	return results, nil
-}
-
-// DoltVersion ensures the dolt binary can run
-func DoltVersion(ctx context.Context, serverExec string) error {
-	doltVersion := ExecCommand(ctx, serverExec, "version")
-	return doltVersion.Run()
-}
-
-// initDoltRepo initializes a dolt repo and returns the repo path
-func initDoltRepo(ctx context.Context, config *ServerConfig, nbf string) (string, error) {
-	cwd, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-
-	testRepo := filepath.Join(cwd, dbName)
-	if nbf == types.Format_LD_1.VersionString() {
-		err := ExecCommand(ctx, config.ServerExec, "clone", bigEmptyRepo, dbName).Run()
-		if err != nil {
-			return "", err
-		}
-		return testRepo, nil
-	}
-	err = os.MkdirAll(testRepo, os.ModePerm)
-	if err != nil {
-		return "", err
-	}
-
-	if nbf != "" {
-		if err = os.Setenv(nbfEnvVar, nbf); err != nil {
-			return "", err
-		}
-	}
-
-	doltInit := ExecCommand(ctx, config.ServerExec, "init")
-	doltInit.Dir = testRepo
-	err = doltInit.Run()
-	if err != nil {
-		return "", err
-	}
-
-	return testRepo, nil
-}
-
-// UpdateDoltConfig updates the dolt config if necessary
-func UpdateDoltConfig(ctx context.Context, serverExec string) error {
-	err := checkSetDoltConfig(ctx, serverExec, "user.name", "benchmark")
-	if err != nil {
-		return err
-	}
-	return checkSetDoltConfig(ctx, serverExec, "user.email", "benchmark@dolthub.com")
-}
-
-// checkSetDoltConfig checks the output of `dolt config --global --get` and sets the key, val if necessary
-func checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error {
-	check := ExecCommand(ctx, serverExec, "config", "--global", "--get", key)
-	err := check.Run()
-	if err != nil {
-		// config get calls exit with 1 if not set
-		if err.Error() != "exit status 1" {
-			return err
-		}
-
-		set := ExecCommand(ctx, serverExec, "config", "--global", "--add", key, val)
-		err := set.Run()
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// getServer returns a exec.Cmd for a dolt server
-func getServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd {
-	server := ExecCommand(ctx, config.ServerExec, params...)
-	server.Dir = testRepo
-	return server
-}
-
-// sysbenchPrepare returns a exec.Cmd for running the sysbench prepare step
-func sysbenchPrepare(ctx context.Context, test *Test, scriptDir string) *exec.Cmd {
-	cmd := exec.CommandContext(ctx, "sysbench", test.Prepare()...)
-	if test.FromScript {
-		lp := filepath.Join(scriptDir, luaPath)
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp))
-	}
-	return cmd
-}
-
-// sysbenchRun returns a exec.Cmd for running the sysbench run step
-func sysbenchRun(ctx context.Context, test *Test, scriptDir string) *exec.Cmd {
-	cmd := exec.CommandContext(ctx, "sysbench", test.Run()...)
-	if test.FromScript {
-		lp := filepath.Join(scriptDir, luaPath)
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp))
-	}
-	return cmd
-}
-
-// sysbenchPrepare returns a exec.Cmd for running the sysbench cleanup step
-func sysbenchCleanup(ctx context.Context, test *Test, scriptDir string) *exec.Cmd {
-	cmd := ExecCommand(ctx, "sysbench", test.Cleanup()...)
-	if test.FromScript {
-		lp := filepath.Join(scriptDir, luaPath)
-		cmd.Env = os.Environ()
-		cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp))
-	}
-	return cmd
-}
-
-// benchmark runs a sysbench benchmark against a server calling prepare, run, cleanup
-func benchmark(
-	ctx context.Context,
-	test *Test,
-	config *Config,
-	serverConfig *ServerConfig,
-	stampFunc func() string,
-	suiteId string,
-) (*Result, error) {
-	prepare := sysbenchPrepare(ctx, test, config.ScriptDir)
-	run := sysbenchRun(ctx, test, config.ScriptDir)
-	cleanup := sysbenchCleanup(ctx, test, config.ScriptDir)
-
-	fmt.Println("Running test ", test.Name)
-
-	out, err := prepare.Output()
-	if err != nil {
-		fmt.Println(string(out))
-		return nil, err
-	}
-
-	out, err = run.Output()
-	if err != nil {
-		fmt.Print(string(out))
-		return nil, err
-	}
-
-	if Debug == true {
-		fmt.Print(string(out))
-	}
-
-	r, err := FromOutputResult(out, config, serverConfig, test, suiteId, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	r.Stamp(stampFunc)
-
-	return r, cleanup.Run()
-}
-
-// fromChannelResults collects all Results from the given channel and returns them
-func fromChannelResults(rc chan *Result) Results {
-	results := make(Results, 0)
-	for r := range rc {
-		if r != nil {
-			results = append(results, r)
-		}
-	}
-	return results
-}

go/performance/utils/sysbench_runner/doltgres.go

@@ -1,183 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"sync"
-	"syscall"
-	"time"
-
-	"database/sql"
-
-	_ "github.com/lib/pq"
-	"golang.org/x/sync/errgroup"
-)
-
-// BenchmarkDoltgres benchmarks doltgres based on the provided configurations
-func BenchmarkDoltgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) {
-	serverParams, err := serverConfig.GetServerArgs()
-	if err != nil {
-		return nil, err
-	}
-
-	err = DoltVersion(ctx, serverConfig.ServerExec)
-	if err != nil {
-		return nil, err
-	}
-
-	serverDir, err := createServerDir(dbName)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		cleanupDoltgresServerDir(serverDir)
-	}()
-
-	serverParams = append(serverParams, fmt.Sprintf("%s=%s", doltgresDataDirFlag, serverDir))
-
-	withKeyCtx, cancel := context.WithCancel(ctx)
-	gServer, serverCtx := errgroup.WithContext(withKeyCtx)
-
-	server := getServer(serverCtx, serverConfig, serverDir, serverParams)
-
-	// handle user interrupt
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		<-quit
-		defer wg.Done()
-		signal.Stop(quit)
-		cancel()
-	}()
-
-	// launch the dolt server
-	gServer.Go(func() error {
-		return server.Run()
-	})
-
-	// sleep to allow the server to start
-	time.Sleep(5 * time.Second)
-
-	// create the db against the running server
-	err = createDb(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "doltgres", dbName)
-	if err != nil {
-		close(quit)
-		wg.Wait()
-		return nil, err
-	}
-
-	tests, err := GetTests(config, serverConfig, nil)
-	if err != nil {
-		close(quit)
-		wg.Wait()
-		return nil, err
-	}
-
-	results := make(Results, 0)
-	for i := 0; i < config.Runs; i++ {
-		for _, test := range tests {
-			r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId())
-			if err != nil {
-				close(quit)
-				wg.Wait()
-				return nil, err
-			}
-			results = append(results, r)
-		}
-	}
-
-	// send signal to dolt server
-	quit <- syscall.SIGTERM
-
-	err = gServer.Wait()
-	if err != nil {
-		// we expect a kill error
-		// we only exit in error if this is not the
-		// error
-		if err.Error() != "signal: killed" {
-			fmt.Println(err)
-			close(quit)
-			wg.Wait()
-			return nil, err
-		}
-	}
-
-	fmt.Println("Successfully killed server")
-	close(quit)
-	wg.Wait()
-
-	return results, nil
-}
-
-func createDb(ctx context.Context, host, port, user, dbname string) error {
-	psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname)
-
-	// open database
-	db, err := sql.Open("postgres", psqlconn)
-	if err != nil {
-		return err
-	}
-
-	// close database
-	defer db.Close()
-
-	// check db
-	err = db.PingContext(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = db.ExecContext(ctx, fmt.Sprintf("create database %s;", dbname))
-	return err
-}
-
-// createServerDir creates a server directory
-func createServerDir(dbName string) (string, error) {
-	cwd, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-
-	serverDir := filepath.Join(cwd, dbName)
-	err = os.MkdirAll(serverDir, os.ModePerm)
-	if err != nil {
-		return "", err
-	}
-
-	return serverDir, nil
-}
-
-// cleanupDoltgresServerDir cleans up the doltgres assets in the provided dir
-func cleanupDoltgresServerDir(dir string) error {
-	dataDir := filepath.Join(dir, ".dolt")
-	defaultDir := filepath.Join(dir, "doltgres")
-	testDir := filepath.Join(dir, dbName)
-	for _, d := range []string{dataDir, defaultDir, testDir} {
-		if _, err := os.Stat(d); !os.IsNotExist(err) {
-			err = os.RemoveAll(d)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}

go/performance/utils/sysbench_runner/mysql.go

@@ -1,248 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
-
-	_ "github.com/go-sql-driver/mysql"
-	"golang.org/x/sync/errgroup"
-)
-
-type MysqlConfig struct {
-	Socket             string
-	ConnectionProtocol string
-	Port               int
-	Host               string
-}
-
-// BenchmarkMysql benchmarks mysql based on the provided configurations
-func BenchmarkMysql(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) {
-	withKeyCtx, cancel := context.WithCancel(ctx)
-
-	var serverDir string
-	defer func() {
-		if serverDir != "" {
-			os.RemoveAll(serverDir)
-		}
-	}()
-
-	var localServer bool
-	var gServer *errgroup.Group
-	var serverCtx context.Context
-	var server *exec.Cmd
-	var err error
-	if serverConfig.Host == defaultHost {
-		log.Println("Launching the default server")
-		localServer = true
-
-		serverDir, err = InitMysqlDataDir(ctx, serverConfig)
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-
-		gServer, serverCtx = errgroup.WithContext(withKeyCtx)
-		var serverParams []string
-		serverParams, err = serverConfig.GetServerArgs()
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-		serverParams = append(serverParams, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir))
-
-		server = getMysqlServer(serverCtx, serverConfig, serverParams)
-
-		// launch the mysql server
-		gServer.Go(func() error {
-			return server.Run()
-		})
-
-		// sleep to allow the server to start
-		time.Sleep(10 * time.Second)
-
-		// setup mysqldb
-		err := SetupDB(ctx, GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName)
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-		log.Println("Successfully set up the MySQL database")
-	}
-
-	// handle user interrupt
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		<-quit
-		defer wg.Done()
-		signal.Stop(quit)
-		cancel()
-	}()
-
-	tests, err := GetTests(config, serverConfig, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	results := make(Results, 0)
-	for i := 0; i < config.Runs; i++ {
-		for _, test := range tests {
-			r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId())
-			if err != nil {
-				close(quit)
-				wg.Wait()
-				return nil, err
-			}
-			results = append(results, r)
-		}
-	}
-
-	// stop local mysql server
-	if localServer {
-		// send signal to server
-		quit <- syscall.SIGTERM
-
-		err = gServer.Wait()
-		if err != nil {
-			// we expect a kill error
-			// we only exit in error if this is not the
-			// error
-			if err.Error() != "signal: killed" {
-				close(quit)
-				wg.Wait()
-				return nil, err
-			}
-		}
-	}
-
-	fmt.Println("Successfully killed server")
-	close(quit)
-	wg.Wait()
-
-	return results, nil
-}
-
-// getMysqlServer returns a exec.Cmd for a dolt server
-func getMysqlServer(ctx context.Context, config *ServerConfig, params []string) *exec.Cmd {
-	return ExecCommand(ctx, config.ServerExec, params...)
-}
-
-// InitMysqlDataDir initializes a mysql data dir and returns the path
-func InitMysqlDataDir(ctx context.Context, config *ServerConfig) (string, error) {
-	serverDir, err := createServerDir(dbName)
-	if err != nil {
-		return "", err
-	}
-
-	msInit := ExecCommand(ctx, config.ServerExec, MysqlInitializeInsecureFlag, fmt.Sprintf("%s=%s", MysqlDataDirFlag, serverDir))
-	err = msInit.Run()
-	if err != nil {
-		return "", err
-	}
-
-	return serverDir, nil
-}
-
-func SetupDB(ctx context.Context, mConfig MysqlConfig, databaseName string) (err error) {
-	dsn, err := FormatDsn(mConfig)
-	if err != nil {
-		return err
-	}
-
-	// TODO make sure this can work on windows
-	db, err := sql.Open("mysql", dsn)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		rerr := db.Close()
-		if err == nil {
-			err = rerr
-		}
-	}()
-	err = db.Ping()
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", databaseName))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s", databaseName))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUserLocal))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s IDENTIFIED WITH mysql_native_password BY '%s'", sysbenchUserLocal, sysbenchPassLocal))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("GRANT ALL ON %s.* to %s", databaseName, sysbenchUserLocal))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, "SET GLOBAL local_infile = 'ON'")
-	if err != nil {
-		return err
-	}
-	// Required for running groupby_scan.lua without error
-	_, err = db.ExecContext(ctx, "SET GLOBAL sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));")
-	if err != nil {
-		return err
-	}
-
-	return
-}
-
-func FormatDsn(mConfig MysqlConfig) (string, error) {
-	var socketPath string
-	if mConfig.Socket != "" {
-		socketPath = mConfig.Socket
-	} else {
-		socketPath = defaultMysqlSocket
-	}
-
-	if mConfig.ConnectionProtocol == tcpProtocol {
-		return fmt.Sprintf("root@tcp(%s:%d)/", mConfig.Host, mConfig.Port), nil
-	} else if mConfig.ConnectionProtocol == unixProtocol {
-		return fmt.Sprintf("root@unix(%s)/", socketPath), nil
-	} else {
-		return "", ErrUnsupportedConnectionProtocol
-	}
-}
-
-func GetMysqlConnectionConfigFromServerConfig(config *ServerConfig) MysqlConfig {
-	return MysqlConfig{
-		Socket:             config.Socket,
-		ConnectionProtocol: config.ConnectionProtocol,
-		Port:               config.Port,
-		Host:               defaultHost,
-	}
-}

go/performance/utils/sysbench_runner/postgres.go

@@ -1,200 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
-
-	"database/sql"
-
-	_ "github.com/lib/pq"
-	"golang.org/x/sync/errgroup"
-)
-
-type PostgresConfig struct {
-	Socket             string
-	ConnectionProtocol string
-	Port               int
-	Host               string
-}
-
-// BenchmarkPostgres benchmarks postgres based on the provided configurations
-func BenchmarkPostgres(ctx context.Context, config *Config, serverConfig *ServerConfig) (Results, error) {
-	withKeyCtx, cancel := context.WithCancel(ctx)
-
-	var serverDir string
-	defer func() {
-		if serverDir != "" {
-			os.RemoveAll(serverDir)
-		}
-	}()
-
-	var localServer bool
-	var gServer *errgroup.Group
-	var serverCtx context.Context
-	var server *exec.Cmd
-	var err error
-	if serverConfig.Host == defaultHost {
-		log.Println("Launching the default server")
-		localServer = true
-
-		serverDir, err = initPostgresDataDir(ctx, serverConfig)
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-		gServer, serverCtx = errgroup.WithContext(withKeyCtx)
-		var serverParams []string
-		serverParams, err = serverConfig.GetServerArgs()
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-		serverParams = append(serverParams, "-D", serverDir)
-		server = getMysqlServer(serverCtx, serverConfig, serverParams)
-		server.Env = append(server.Env, "LC_ALL=C")
-
-		// launch the postgres server
-		gServer.Go(func() error {
-			return server.Run()
-		})
-
-		// sleep to allow the server to start
-		time.Sleep(10 * time.Second)
-
-		// setup postgres db
-		err := setupPostgresDB(ctx, serverConfig.Host, fmt.Sprintf("%d", serverConfig.Port), "postgres", dbName)
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-
-		log.Println("Successfully set up the Postgres database")
-	}
-
-	// handle user interrupt
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		<-quit
-		defer wg.Done()
-		signal.Stop(quit)
-		cancel()
-	}()
-
-	tests, err := GetTests(config, serverConfig, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	results := make(Results, 0)
-	for i := 0; i < config.Runs; i++ {
-		for _, test := range tests {
-			r, err := benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId())
-			if err != nil {
-				close(quit)
-				wg.Wait()
-				return nil, err
-			}
-			results = append(results, r)
-		}
-	}
-
-	// stop local mysql server
-	if localServer {
-		// send signal to server
-		quit <- syscall.SIGTERM
-
-		err = gServer.Wait()
-		if err != nil {
-			// we expect a kill error
-			// we only exit in error if this is not the
-			// error
-			if err.Error() != "signal: killed" {
-				close(quit)
-				wg.Wait()
-				return nil, err
-			}
-		}
-	}
-
-	fmt.Println("Successfully killed server")
-	close(quit)
-	wg.Wait()
-
-	return results, nil
-}
-
-// initPostgresDataDir initializes a postgres data dir and returns the path
-func initPostgresDataDir(ctx context.Context, config *ServerConfig) (string, error) {
-	serverDir, err := createServerDir(dbName)
-	if err != nil {
-		return "", err
-	}
-
-	pgInit := ExecCommand(ctx, config.InitExec, fmt.Sprintf("--pgdata=%s", serverDir), "--username=postgres")
-	err = pgInit.Run()
-	if err != nil {
-		return "", err
-	}
-
-	return serverDir, nil
-}
-
-func setupPostgresDB(ctx context.Context, host, port, user, dbname string) (err error) {
-	psqlconn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, "", dbname)
-
-	db, err := sql.Open("postgres", psqlconn)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		rerr := db.Close()
-		if err == nil {
-			err = rerr
-		}
-	}()
-	err = db.PingContext(ctx)
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbname))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s", sysbenchUsername))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("CREATE USER %s WITH PASSWORD '%s'", sysbenchUsername, sysbenchPassLocal))
-	if err != nil {
-		return err
-	}
-	_, err = db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE %s WITH OWNER %s", dbname, sysbenchUsername))
-	if err != nil {
-		return err
-	}
-	return
-}

go/performance/utils/sysbench_runner/profile.go

@@ -1,206 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"sync"
-	"syscall"
-	"time"
-
-	"golang.org/x/sync/errgroup"
-)
-
-// ProfileDolt profiles dolt while running the provided tests
-func ProfileDolt(ctx context.Context, config *Config, serverConfig *ServerConfig) error {
-	serverParams, err := serverConfig.GetServerArgs()
-	if err != nil {
-		return err
-	}
-
-	err = DoltVersion(ctx, serverConfig.ServerExec)
-	if err != nil {
-		return err
-	}
-
-	err = UpdateDoltConfig(ctx, serverConfig.ServerExec)
-	if err != nil {
-		return err
-	}
-
-	testRepo, err := initDoltRepo(ctx, serverConfig, config.NomsBinFormat)
-	if err != nil {
-		return err
-	}
-
-	tests, err := GetTests(config, serverConfig, nil)
-	if err != nil {
-		return err
-	}
-
-	tempProfilesDir, err := os.MkdirTemp("", "")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tempProfilesDir)
-
-	for i := 0; i < config.Runs; i++ {
-		for _, test := range tests {
-			_, err = profileTest(ctx, test, config, serverConfig, serverParams, testRepo, tempProfilesDir)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	profile, err := mergeProfiles(ctx, tempProfilesDir, serverConfig.ProfilePath)
-	if err != nil {
-		return err
-	}
-
-	fmt.Println("Profile created at:", profile)
-
-	err = os.RemoveAll(testRepo)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func profileTest(ctx context.Context, test *Test, config *Config, serverConfig *ServerConfig, serverParams []string, testRepo, profileDir string) (string, error) {
-	profilePath, err := os.MkdirTemp("", filepath.Base(test.Name))
-	if err != nil {
-		return "", err
-	}
-	defer os.RemoveAll(profilePath)
-
-	tempProfile := filepath.Join(profilePath, cpuProfileFilename)
-	profileParams := make([]string, 0)
-	profileParams = append(profileParams, profileFlag, cpuProfile, profilePathFlag, profilePath)
-	profileParams = append(profileParams, serverParams...)
-
-	withKeyCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	gServer, serverCtx := errgroup.WithContext(withKeyCtx)
-	server := getServer(serverCtx, serverConfig, testRepo, profileParams)
-
-	// handle user interrupt
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt, syscall.SIGINT)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		s := <-quit
-		defer wg.Done()
-		server.Process.Signal(s)
-		signal.Stop(quit)
-	}()
-
-	// launch the dolt server
-	gServer.Go(func() error {
-		return server.Run()
-	})
-
-	// sleep to allow the server to start
-	time.Sleep(5 * time.Second)
-
-	_, err = benchmark(withKeyCtx, test, config, serverConfig, stampFunc, serverConfig.GetId())
-	if err != nil {
-		close(quit)
-		wg.Wait()
-		return "", err
-	}
-
-	// send signal to dolt server
-	quit <- syscall.SIGINT
-
-	err = gServer.Wait()
-	if err != nil {
-		// we expect a kill error
-		// we only exit in error if this is not the
-		// error
-		if err.Error() != "signal: killed" {
-			fmt.Println(err)
-			close(quit)
-			wg.Wait()
-			return "", err
-		}
-	}
-
-	fmt.Println("Successfully killed server")
-	close(quit)
-	wg.Wait()
-
-	info, err := os.Stat(tempProfile)
-	if err != nil {
-		return "", err
-	}
-	if info.Size() < 1 {
-		return "", fmt.Errorf("failed to create profile: file was empty")
-	}
-
-	finalProfile := filepath.Join(profileDir, fmt.Sprintf("%s_%s_%s", serverConfig.Id, test.Name, cpuProfileFilename))
-	err = moveFile(tempProfile, finalProfile)
-	return finalProfile, err
-}
-
-func mergeProfiles(ctx context.Context, sourceProfilesDir, destProfileDir string) (string, error) {
-	tmp, err := os.MkdirTemp("", "final_cpu_pprof")
-	if err != nil {
-		return "", err
-	}
-	defer os.RemoveAll(tmp)
-	outfile := filepath.Join(tmp, "cpu.pprof")
-
-	merge := ExecCommand(ctx, "/bin/sh", "-c", fmt.Sprintf("go tool pprof -proto *.pprof > %s", outfile))
-	merge.Dir = sourceProfilesDir
-	err = merge.Run()
-	if err != nil {
-		return "", err
-	}
-
-	final := filepath.Join(destProfileDir, "cpu.pprof")
-	err = moveFile(outfile, final)
-	return final, err
-}
-
-func moveFile(sourcePath, destPath string) error {
-	err := copyFile(sourcePath, destPath)
-	if err != nil {
-		return err
-	}
-	return os.Remove(sourcePath)
-}
-
-func copyFile(sourcePath, destPath string) error {
-	inputFile, err := os.Open(sourcePath)
-	if err != nil {
-		return err
-	}
-	defer inputFile.Close()
-	outputFile, err := os.Create(destPath)
-	if err != nil {
-		return err
-	}
-	defer outputFile.Close()
-	_, err = io.Copy(outputFile, inputFile)
-	return err
-}

go/performance/utils/sysbench_runner/run.go

@@ -1,110 +0,0 @@
-// Copyright 2019-2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-)
-
-// Run runs sysbench runner
-func Run(config *Config) error {
-	err := config.Validate()
-	if err != nil {
-		return err
-	}
-
-	ctx := context.Background()
-
-	err = sysbenchVersion(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, serverConfig := range config.Servers {
-		var results Results
-		switch serverConfig.Server {
-		case Dolt:
-			// handle a profiling run
-			if serverConfig.ServerProfile != "" {
-				fmt.Println("Profiling dolt while running sysbench tests")
-				return ProfileDolt(ctx, config, serverConfig)
-			}
-			fmt.Println("Running dolt sysbench test")
-			results, err = BenchmarkDolt(ctx, config, serverConfig)
-		case Doltgres:
-			fmt.Println("Running doltgres sysbench test")
-			results, err = BenchmarkDoltgres(ctx, config, serverConfig)
-		case MySql:
-			fmt.Println("Running mysql sysbench test")
-			results, err = BenchmarkMysql(ctx, config, serverConfig)
-		case Postgres:
-			fmt.Println("Running postgres sysbench test")
-			results, err = BenchmarkPostgres(ctx, config, serverConfig)
-		default:
-			panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server))
-		}
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Successfuly finished %s\n", serverConfig.Server)
-
-		err = WriteResults(serverConfig, results)
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server)
-	}
-	return nil
-}
-
-func sysbenchVersion(ctx context.Context) error {
-	sysbenchVersion := ExecCommand(ctx, "sysbench", "--version")
-	return sysbenchVersion.Run()
-}
-
-func WriteResults(serverConfig *ServerConfig, results Results) error {
-	cwd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-	var writePath string
-	switch serverConfig.ResultsFormat {
-	case CsvFormat, CsvExt:
-		writePath = filepath.Join(
-			cwd,
-			"results",
-			string(serverConfig.Server),
-			serverConfig.Version,
-			serverConfig.GetId(),
-			fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, CsvExt))
-		return WriteResultsCsv(writePath, results)
-	case JsonFormat, JsonExt:
-		writePath = filepath.Join(
-			cwd,
-			"results",
-			string(serverConfig.Server),
-			serverConfig.Version,
-			serverConfig.GetId(),
-			fmt.Sprintf(ResultFileTemplate, serverConfig.GetId(), serverConfig.Server, serverConfig.Version, JsonExt))
-		return WriteResultsJson(writePath, results)
-	default:
-	}
-	return fmt.Errorf("unsupported results format: %s", serverConfig.ResultsFormat)
-}

go/performance/utils/sysbench_runner/run_test.go

@@ -1,67 +0,0 @@
-// Copyright 2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package sysbench_runner
-
-import (
-	"log"
-	"os"
-	"testing"
-)
-
-func TestRunner(t *testing.T) {
-	t.Skip()
-	dir := t.TempDir()
-	log.Println(dir)
-	err := os.Chdir(dir)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	conf := &Config{
-		Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_delete_insert"),
-		//Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"),
-		Servers: []*ServerConfig{
-			{
-				Id:            "test",
-				Server:        Dolt,
-				Version:       "0.39.2",
-				ResultsFormat: CsvFormat,
-				ServerExec:    "/Users/max-hoffman/go/bin/dolt",
-			},
-		},
-		ScriptDir: "/Users/max-hoffman/Documents/dolthub/sysbench-lua-scripts",
-		TestOptions: []string{
-			"--rand-seed=1",
-			"--table-size=10000",
-			"--rand-type=uniform",
-			"--time=120",
-			"--percentile=50",
-		},
-		InitBigRepo: true,
-	}
-
-	err = Run(conf)
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func selectTests(names ...string) []*ConfigTest {
-	tests := make([]*ConfigTest, len(names))
-	for i := range names {
-		tests[i] = &ConfigTest{Name: names[i], FromScript: false}
-	}
-	return tests
-}

go/performance/utils/tpcc_runner/README.md

@@ -1,33 +0,0 @@
-TPCC runner is a tool for running TPCC tests against sql servers. These tests run against the 
-Percona Labs repo [here](https://github.com/Percona-Lab/sysbench-tpcc).
-
-The tool requires a json config file to run.
-
-```bash
-$ go run cmd/main.go --config=sample-tpcc-config.json
-```
-
-Note to this run this locally you need to have the TPCC repo cloned. The `ScriptDir` variable should then be linked
-to the path of the cloned repo.
-
-Configuration:
-
-```json
-{
-  "Servers": "[...]",
-  "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc",
-  "ScaleFactors": [1]
-}
-```
-
-`Servers`: The server defintions to run the benchmark against. Accepts Dolt and MySQL configuratiosn.
-
-`ScriptDir`: The directory of the TPCC testing scripts
-
-`ScaleFactors`: The number of warehouse to be generated in the test case.
-
-`NomsBinFormat`: The NomsBinFormat to use for this benchmark.
-
-Note that this configuration is still incomplete for the amount of the variable TPCC varies. This intentional as we 
-want expose small amounts of independent variables until Dolt gets more robust. See `config.go` to get a breakdown of all the
-variables TPCC varies.

go/performance/utils/tpcc_runner/cmd/main.go

@@ -1,59 +0,0 @@
-// Copyright 2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-
-	tpcc_runner "github.com/dolthub/dolt/go/performance/utils/tpcc_runner"
-)
-
-var configFile = flag.String("config", "", "path to config file q")
-
-func main() {
-	fmt.Println("Running the TPCC benchmark.")
-
-	flag.Parse()
-	if *configFile == "" {
-		log.Fatal("Must supply config")
-	}
-
-	configPath, err := filepath.Abs(*configFile)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if _, err = os.Stat(configPath); os.IsNotExist(err) {
-		log.Fatal(err)
-	}
-
-	tpccBenchmarkConfig, err := tpcc_runner.FromFileConfig(configPath)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// Run the TPCC test
-	err = tpcc_runner.Run(tpccBenchmarkConfig)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	os.Exit(0)
-}

go/performance/utils/tpcc_runner/config.go

@@ -1,255 +0,0 @@
-// Copyright 2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tpcc_runner
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-
-	"github.com/google/uuid"
-
-	"github.com/dolthub/dolt/go/performance/utils/sysbench_runner"
-)
-
-const (
-	defaultHost = "127.0.0.1"
-	defaultUser = "root"
-
-	// Note this is built for the SysbenchDocker file. If you want to run locally you'll need to override these variables
-	// for your local MySQL setup.
-	tpccUserLocal = "'sysbench'@'localhost'"
-	tpccPassLocal = "sysbenchpass"
-)
-
-var defaultTpccParams = []string{
-	fmt.Sprintf("--mysql-db=%s", dbName),
-	"--db-driver=mysql",
-}
-
-// TpccBenchmarkConfig represents a configuration for an execution of the TPCC Benchmark. It executes a series of tests
-// against different ServerConfigurations.
-type TpccBenchmarkConfig struct {
-	// RuntimeOS is the platform the benchmarks ran on
-	RuntimeOS string
-
-	// RuntimeGoArch is the runtime architecture
-	RuntimeGoArch string
-
-	// ScriptDir represents the location of the TPCC tests
-	ScriptDir string
-
-	// Servers are the servers to benchmark.
-	Servers []*sysbench_runner.ServerConfig
-
-	// ScaleFactors represent the scale at which to run each TpccBenchmark at.
-	ScaleFactors []int
-
-	// NomsBinFormat specifies the NomsBinFormat
-	NomsBinFormat string
-}
-
-func NewTpccConfig() *TpccBenchmarkConfig {
-	return &TpccBenchmarkConfig{
-		Servers:      make([]*sysbench_runner.ServerConfig, 0),
-		ScaleFactors: make([]int, 0),
-	}
-}
-
-func (c *TpccBenchmarkConfig) updateDefaults() error {
-	if len(c.Servers) < 1 {
-		return sysbench_runner.ErrNoServersDefined
-	}
-
-	// TODO: Eventually we need to support scale factors all the way to 10
-	if len(c.ScaleFactors) == 0 {
-		c.ScaleFactors = append(c.ScaleFactors, 1)
-	}
-
-	if c.RuntimeOS == "" {
-		c.RuntimeOS = runtime.GOOS
-	}
-	if c.RuntimeGoArch == "" {
-		c.RuntimeGoArch = runtime.GOARCH
-	}
-
-	return c.validateServerConfigs()
-}
-
-// validateServerConfigs ensures the ServerConfigs are valid
-func (c *TpccBenchmarkConfig) validateServerConfigs() error {
-	portMap := make(map[int]sysbench_runner.ServerType)
-	for _, s := range c.Servers {
-		if s.Server != sysbench_runner.Dolt && s.Server != sysbench_runner.MySql {
-			return fmt.Errorf("unsupported server type: %s", s.Server)
-		}
-
-		err := sysbench_runner.ValidateRequiredFields(string(s.Server), s.Version, s.ResultsFormat)
-		if err != nil {
-			return err
-		}
-
-		if s.Server == sysbench_runner.MySql {
-			err = sysbench_runner.CheckProtocol(s.ConnectionProtocol)
-			if err != nil {
-				return err
-			}
-		}
-
-		if s.Host == "" {
-			s.Host = defaultHost
-		}
-
-		portMap, err = sysbench_runner.CheckUpdatePortMap(s, portMap)
-		if err != nil {
-			return err
-		}
-
-		err = sysbench_runner.CheckExec(s.ServerExec, "server exec")
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// FromFileConfig returns a validated Config based on the config file at the configPath
-func FromFileConfig(configPath string) (*TpccBenchmarkConfig, error) {
-	data, err := os.ReadFile(configPath)
-	if err != nil {
-		return nil, err
-	}
-
-	config := NewTpccConfig()
-	err = json.Unmarshal(data, config)
-	if err != nil {
-		return nil, err
-	}
-
-	return config, nil
-}
-
-// TpccTest encapsulates an End to End prepare, run, cleanup test case.
-type TpccTest struct {
-	// Id represents a unique test id
-	Id string
-
-	// Name represents the name of the test case
-	Name string
-
-	// Params are associated parameters this test runs with
-	Params *TpccTestParams
-}
-
-type TpccTestParams struct {
-	// NumThreads represents the number of threads running queries concurrently.
-	NumThreads int
-
-	// ScaleFactor represents the number of warehouse to test this at scale.
-	ScaleFactor int
-
-	// Tables represents the number of tables created per warehouse.
-	Tables int
-
-	// TrxLevel represents what transaction level to use
-	TrxLevel string
-
-	// ReportCSV determines whether to report output as a csv.
-	ReportCSV bool
-
-	// ReportInterval defines how often the tpcc benchmark outputs performance stats.
-	ReportInterval int
-
-	// Time represents how long
-	Time int
-}
-
-// NewDefaultTpccParams returns default TpccTestParams.
-func NewDefaultTpccParams() *TpccTestParams {
-	return &TpccTestParams{
-		NumThreads:     2, // TODO: When ready, expose as command line argument.
-		ScaleFactor:    1,
-		Tables:         1,
-		TrxLevel:       "RR",
-		ReportCSV:      true,
-		ReportInterval: 1,
-		Time:           30,
-	}
-}
-
-// NewTpccTest instantiates and returns a TPCC test.
-func NewTpccTest(name string, params *TpccTestParams) *TpccTest {
-	return &TpccTest{
-		Id:     uuid.New().String(),
-		Name:   name,
-		Params: params,
-	}
-}
-
-// getArgs returns a test's args for all TPCC steps
-func (t *TpccTest) getArgs(serverConfig *sysbench_runner.ServerConfig) []string {
-	params := make([]string, 0)
-	params = append(params, defaultTpccParams...)
-
-	params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host))
-
-	// handle sysbench user for local mysql server
-	if serverConfig.Server == sysbench_runner.MySql && serverConfig.Host == defaultHost {
-		params = append(params, fmt.Sprintf("--mysql-user=%s", "sysbench"))
-		params = append(params, fmt.Sprintf("--mysql-password=%s", tpccPassLocal))
-	} else {
-		params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port))
-		params = append(params, fmt.Sprintf("--mysql-user=%s", defaultUser))
-	}
-
-	params = append(params, fmt.Sprintf("--time=%d", t.Params.Time))
-	params = append(params, fmt.Sprintf("--threads=%d", t.Params.NumThreads))
-	params = append(params, fmt.Sprintf("--report_interval=%d", t.Params.ReportInterval))
-	params = append(params, fmt.Sprintf("--tables=%d", t.Params.Tables))
-	params = append(params, fmt.Sprintf("--scale=%d", t.Params.ScaleFactor))
-	params = append(params, fmt.Sprintf("--trx_level=%s", t.Params.TrxLevel))
-
-	return params
-}
-
-// TpccPrepare prepares the command executable for the Prepare step.
-func (t *TpccTest) TpccPrepare(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd {
-	cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "prepare")...)
-	return addParamsToCmd(cmd, scriptDir)
-}
-
-// TpccRun prepares the command executable for the Run step.
-func (t *TpccTest) TpccRun(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd {
-	cmd := exec.CommandContext(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "run")...)
-	return addParamsToCmd(cmd, scriptDir)
-}
-
-// TpccCleanup prepares the cleanup executable for the Cleanup step.
-func (t *TpccTest) TpccCleanup(ctx context.Context, serverConfig *sysbench_runner.ServerConfig, scriptDir string) *exec.Cmd {
-	cmd := sysbench_runner.ExecCommand(ctx, scriptDir+"/tpcc.lua", append(t.getArgs(serverConfig), "cleanup")...)
-	return addParamsToCmd(cmd, scriptDir)
-}
-
-func addParamsToCmd(cmd *exec.Cmd, scriptDir string) *exec.Cmd {
-	lp := filepath.Join(scriptDir, "?.lua")
-	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, fmt.Sprintf("LUA_PATH=%s", lp))
-
-	return cmd
-}

go/performance/utils/tpcc_runner/dolt.go

@@ -1,204 +0,0 @@
-// Copyright 2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tpcc_runner
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/exec"
-	"os/signal"
-	"path/filepath"
-	"sync"
-	"syscall"
-	"time"
-
-	"golang.org/x/sync/errgroup"
-
-	"github.com/dolthub/dolt/go/performance/utils/sysbench_runner"
-)
-
-const (
-	dbName    = "sbt"
-	nbfEnvVar = "DOLT_DEFAULT_BIN_FORMAT"
-)
-
-// BenchmarkDolt executes a set of tpcc tests against a dolt server.
-func BenchmarkDolt(ctx context.Context, tppcConfig *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) {
-	serverParams, err := serverConfig.GetServerArgs()
-	if err != nil {
-		return nil, err
-	}
-
-	err = sysbench_runner.UpdateDoltConfig(ctx, serverConfig.ServerExec)
-	if err != nil {
-		return nil, err
-	}
-
-	testRepo, err := initDoltRepo(ctx, serverConfig, tppcConfig.NomsBinFormat)
-	if err != nil {
-		return nil, err
-	}
-
-	withKeyCtx, cancel := context.WithCancel(ctx)
-	gServer, serverCtx := errgroup.WithContext(withKeyCtx)
-
-	server := getDoltServer(serverCtx, serverConfig, testRepo, serverParams)
-
-	// handle user interrupt
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		<-quit
-		defer wg.Done()
-		signal.Stop(quit)
-		cancel()
-	}()
-
-	// launch the dolt server
-	gServer.Go(func() error {
-		return server.Run()
-	})
-
-	// sleep to allow the server to start
-	time.Sleep(5 * time.Second)
-
-	// GetTests and Benchmarks
-	tests := getTests(tppcConfig)
-	results := make(sysbench_runner.Results, 0)
-
-	for _, test := range tests {
-		result, err := benchmark(ctx, test, serverConfig, tppcConfig)
-		if err != nil {
-			close(quit)
-			wg.Wait()
-			return nil, err
-		}
-
-		results = append(results, result)
-	}
-
-	// send signal to dolt server
-	quit <- syscall.SIGTERM
-
-	err = gServer.Wait()
-	if err != nil {
-		// we expect a kill error
-		// we only exit in error if this is not the
-		// error
-		if err.Error() != "signal: killed" {
-			close(quit)
-			wg.Wait()
-			return nil, err
-		}
-	}
-
-	close(quit)
-	wg.Wait()
-
-	return results, os.RemoveAll(testRepo)
-}
-
-// initDoltRepo initializes a dolt repo and returns the repo path
-func initDoltRepo(ctx context.Context, config *sysbench_runner.ServerConfig, nbf string) (string, error) {
-	if nbf != "" {
-		if err := os.Setenv(nbfEnvVar, nbf); err != nil {
-			return "", err
-		}
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-
-	testRepo := filepath.Join(cwd, dbName)
-	err = os.MkdirAll(testRepo, os.ModePerm)
-	if err != nil {
-		return "", err
-	}
-
-	err = os.RemoveAll(filepath.Join(testRepo, ".dolt"))
-	if err != nil {
-		return "", err
-	}
-
-	doltInit := sysbench_runner.ExecCommand(ctx, config.ServerExec, "init")
-	doltInit.Dir = testRepo
-	err = doltInit.Run()
-	if err != nil {
-		return "", err
-	}
-
-	return testRepo, nil
-}
-
-// getDoltServer returns a exec.Cmd for a dolt server
-func getDoltServer(ctx context.Context, config *sysbench_runner.ServerConfig, testRepo string, params []string) *exec.Cmd {
-	server := sysbench_runner.ExecCommand(ctx, config.ServerExec, params...)
-	server.Dir = testRepo
-	return server
-}
-
-// getTests creates a set of tests that the server needs to be executed on.
-func getTests(config *TpccBenchmarkConfig) []*TpccTest {
-	tests := make([]*TpccTest, 0)
-	for _, sf := range config.ScaleFactors {
-		params := NewDefaultTpccParams()
-		params.ScaleFactor = sf
-		test := NewTpccTest(fmt.Sprintf("tpcc-scale-factor-%d", sf), params)
-		tests = append(tests, test)
-	}
-
-	return tests
-}
-
-// benchmark runs the relevant tpcc test against a server with a config.
-func benchmark(ctx context.Context, test *TpccTest, serverConfig *sysbench_runner.ServerConfig, config *TpccBenchmarkConfig) (*sysbench_runner.Result, error) {
-	prepare := test.TpccPrepare(ctx, serverConfig, config.ScriptDir)
-	run := test.TpccRun(ctx, serverConfig, config.ScriptDir)
-	cleanup := test.TpccCleanup(ctx, serverConfig, config.ScriptDir)
-
-	err := prepare.Run()
-	if err != nil {
-		return nil, err
-	}
-
-	if run.Stdout != nil {
-		run.Stdout = nil
-	}
-
-	out, err := run.Output()
-	if err != nil {
-		fmt.Print(string(out))
-		return nil, err
-	}
-
-	fmt.Print(string(out))
-
-	result, err := FromOutputResult(out, config, serverConfig, test, "tpcc", nil)
-	if err != nil {
-		return nil, err
-	}
-
-	err = cleanup.Run()
-	if err != nil {
-		return nil, err
-	}
-
-	return result, nil
-}

go/performance/utils/tpcc_runner/mysql.go

@@ -1,135 +0,0 @@
-// Copyright 2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tpcc_runner
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/exec"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
-
-	"golang.org/x/sync/errgroup"
-
-	"github.com/dolthub/dolt/go/performance/utils/sysbench_runner"
-)
-
-// BenchmarkMysql benchmarks a mysql server based on the provided configurations.
-func BenchmarkMysql(ctx context.Context, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig) (sysbench_runner.Results, error) {
-	withKeyCtx, cancel := context.WithCancel(ctx)
-
-	var serverDir string
-	defer func() {
-		if serverDir != "" {
-			os.RemoveAll(serverDir)
-		}
-	}()
-
-	var localServer bool
-	var gServer *errgroup.Group
-	var serverCtx context.Context
-	var server *exec.Cmd
-	var err error
-	if serverConfig.Host == defaultHost {
-		localServer = true
-		serverDir, err = sysbench_runner.InitMysqlDataDir(ctx, serverConfig)
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-
-		gServer, serverCtx = errgroup.WithContext(withKeyCtx)
-		var serverParams []string
-		serverParams, err = serverConfig.GetServerArgs()
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-		serverParams = append(serverParams, fmt.Sprintf("%s=%s", sysbench_runner.MysqlDataDirFlag, serverDir))
-		server = getMysqlServer(serverCtx, serverConfig, serverParams)
-
-		// launch the mysql server
-		gServer.Go(func() error {
-			return server.Run()
-		})
-
-		// sleep to allow the server to start
-		time.Sleep(10 * time.Second)
-
-		// setup mysqldb
-		err := sysbench_runner.SetupDB(ctx, sysbench_runner.GetMysqlConnectionConfigFromServerConfig(serverConfig), dbName)
-		if err != nil {
-			cancel()
-			return nil, err
-		}
-	}
-
-	// handle user interrupt
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		<-quit
-		defer wg.Done()
-		signal.Stop(quit)
-		cancel()
-	}()
-
-	tests := getTests(config)
-
-	results := make(sysbench_runner.Results, 0)
-
-	for _, test := range tests {
-		r, err := benchmark(withKeyCtx, test, serverConfig, config)
-		if err != nil {
-			close(quit)
-			wg.Wait()
-			return nil, err
-		}
-		results = append(results, r)
-	}
-
-	// stop local mysql server
-	if localServer {
-		// send signal to server
-		quit <- syscall.SIGTERM
-
-		err := gServer.Wait()
-		if err != nil {
-			// we expect a kill error
-			// we only exit in error if this is not the
-			// error
-			if err.Error() != "signal: killed" {
-				close(quit)
-				wg.Wait()
-				return nil, err
-			}
-		}
-	}
-
-	close(quit)
-	wg.Wait()
-
-	return results, nil
-}
-
-// getMysqlServer returns a exec.Cmd for a dolt server
-func getMysqlServer(ctx context.Context, config *sysbench_runner.ServerConfig, params []string) *exec.Cmd {
-	return sysbench_runner.ExecCommand(ctx, config.ServerExec, params...)
-}

go/performance/utils/tpcc_runner/results.go

@@ -1,86 +0,0 @@
-// Copyright 2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tpcc_runner
-
-import (
-	"path/filepath"
-	"strings"
-
-	"github.com/google/uuid"
-
-	"github.com/dolthub/dolt/go/performance/utils/sysbench_runner"
-)
-
-// FromConfigsNewResult returns a new result with some fields set based on the provided configs
-func FromConfigsNewResult(config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) {
-	serverParams, err := serverConfig.GetServerArgs()
-	if err != nil {
-		return nil, err
-	}
-
-	var getId func() string
-	if idFunc == nil {
-		getId = func() string {
-			return uuid.New().String()
-		}
-	} else {
-		getId = idFunc
-	}
-
-	var name string
-	base := filepath.Base(test.Name)
-	ext := filepath.Ext(base)
-	name = strings.TrimSuffix(base, ext)
-
-	return &sysbench_runner.Result{
-		Id:            getId(),
-		SuiteId:       suiteId,
-		TestId:        test.Id,
-		RuntimeOS:     config.RuntimeOS,
-		RuntimeGoArch: config.RuntimeGoArch,
-		ServerName:    string(serverConfig.Server),
-		ServerVersion: serverConfig.Version,
-		ServerParams:  strings.Join(serverParams, " "),
-		TestName:      name,
-		TestParams:    strings.Join(test.getArgs(serverConfig), " "),
-	}, nil
-}
-
-// FromOutputResult accepts raw sysbench run output and returns the Result
-func FromOutputResult(output []byte, config *TpccBenchmarkConfig, serverConfig *sysbench_runner.ServerConfig, test *TpccTest, suiteId string, idFunc func() string) (*sysbench_runner.Result, error) {
-	result, err := FromConfigsNewResult(config, serverConfig, test, suiteId, idFunc)
-	if err != nil {
-		return nil, err
-	}
-	lines := strings.Split(string(output), "\n")
-	var process bool
-	for _, l := range lines {
-		trimmed := strings.TrimSpace(l)
-		if trimmed == "" {
-			continue
-		}
-		if strings.HasPrefix(trimmed, sysbench_runner.SqlStatsPrefix) {
-			process = true
-			continue
-		}
-		if process {
-			err := sysbench_runner.UpdateResult(result, trimmed)
-			if err != nil {
-				return result, err
-			}
-		}
-	}
-	return result, nil
-}

go/performance/utils/tpcc_runner/run.go

@@ -1,64 +0,0 @@
-// Copyright 2022 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tpcc_runner
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/dolthub/dolt/go/performance/utils/sysbench_runner"
-)
-
-func Run(config *TpccBenchmarkConfig) error {
-	err := config.updateDefaults()
-	if err != nil {
-		return err
-	}
-
-	ctx := context.Background()
-
-	for _, serverConfig := range config.Servers {
-		var results sysbench_runner.Results
-		var err error
-		switch serverConfig.Server {
-		case sysbench_runner.Dolt:
-			fmt.Println("Running Dolt Benchmark")
-			results, err = BenchmarkDolt(ctx, config, serverConfig)
-			if err != nil {
-				return err
-			}
-		case sysbench_runner.MySql:
-			fmt.Println("Running MySQL benchmark")
-			results, err = BenchmarkMysql(ctx, config, serverConfig)
-			if err != nil {
-				return err
-			}
-		default:
-			panic(fmt.Sprintf("unexpected server type: %s", serverConfig.Server))
-		}
-		if err != nil {
-			return err
-		}
-
-		err = sysbench_runner.WriteResults(serverConfig, results)
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Successfuly wrote results for %s\n", serverConfig.Server)
-	}
-
-	return nil
-}

go/performance/utils/tpcc_runner/sample-tpcc-config.json

@@ -1,22 +0,0 @@
-{
-  "Servers": [
-    {
-      "Host": "127.0.0.1",
-      "Port": 3307,
-      "Server": "dolt",
-      "Version": "HEAD",
-      "ResultsFormat": "csv",
-      "ServerExec": "/Users/vinairachakonda/go/bin/dolt"
-    },
-    {
-      "Server": "mysql",
-      "Version": "8.0.22",
-      "ResultsFormat": "csv",
-      "ServerExec": "/usr/local/bin/mysqld",
-      "ConnectionProtocol": "tcp"
-    }
-  ],
-  "ScriptDir":"/Users/vinairachakonda/go/src/dolthub/sysbench-tpcc",
-  "ScaleFactors": [1],
-  "NomsBinFormat": "__DOLT__"
-}