Option for sysbench perf tests to clone big repo (#3326)

* Option for sysbench perf tests to clone big repo, rather than starting empty

* put remove dir back

* flag

* hanging comma

* sysbench canary update

.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh

@@ -73,7 +73,8 @@ echo '
               "--sysbenchQueries='"$medianLatencyChangeReadsQuery"'",
               "--sysbenchQueries='"$medianLatencyChangeWritesQuery"'",
               "--tpccQueries='"$tpccLatencyQuery"'",
-              "--tpccQueries='"$tpccTpsQuery"'"
+              "--tpccQueries='"$tpccTpsQuery"'",
+              "--init-big-repo"
             ]
           }
         ],

go/performance/continuous_integration/config.json

@@ -18,5 +18,6 @@
     }
   ],
   "ScriptDir":"/sysbench-lua-scripts",
-  "TestOptions": ["--rand-seed=1", "--table-size=30"]
+  "TestOptions": ["--rand-seed=1", "--table-size=30"],
+  "InitBigRepo": true
 }

go/performance/utils/sysbench_runner/config.go

@@ -266,25 +266,21 @@ func (sc *ServerConfig) GetServerArgs() []string {
 type Config struct {
 	// Runs is the number of times to run all tests
 	Runs int
-
 	// RuntimeOS is the platform the benchmarks ran on
 	RuntimeOS string
-
 	// RuntimeGoArch is the runtime architecture
 	RuntimeGoArch string
-
 	// Servers are the servers to benchmark
 	Servers []*ServerConfig
-
 	// Tests are the tests to run. If no tests are provided,
 	// the default tests will be used
 	Tests []*ConfigTest
-
 	// TestOptions a list of sysbench test options to apply to all tests
 	TestOptions []string
-
 	// ScriptDir is a path to a directory of lua scripts
 	ScriptDir string
+	// DirtyClone downloads a database with existing chunks and commits
+	InitBigRepo bool
 }
 
 // NewConfig returns a new Config

go/performance/utils/sysbench_runner/dolt.go

@@ -29,8 +29,9 @@ import (
 )
 
 const (
-	dbName  = "test"
-	luaPath = "?.lua"
+	dbName       = "test"
+	luaPath      = "?.lua"
+	bigEmptyRepo = "max-hoffman/big-empty"
 )
 
 var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) }
@@ -49,7 +50,7 @@ func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConf
 		return nil, err
 	}
 
-	testRepo, err := initDoltRepo(ctx, serverConfig)
+	testRepo, err := initDoltRepo(ctx, serverConfig, config.InitBigRepo)
 	if err != nil {
 		return nil, err
 	}
@@ -132,13 +133,20 @@ func doltVersion(ctx context.Context, config *ServerConfig) error {
 }
 
 // initDoltRepo initializes a dolt repo and returns the repo path
-func initDoltRepo(ctx context.Context, config *ServerConfig) (string, error) {
+func initDoltRepo(ctx context.Context, config *ServerConfig, initBigRepo bool) (string, error) {
 	cwd, err := os.Getwd()
 	if err != nil {
 		return "", err
 	}
 
 	testRepo := filepath.Join(cwd, dbName)
+	if initBigRepo {
+		err := ExecCommand(ctx, config.ServerExec, "clone", bigEmptyRepo, dbName).Run()
+		if err != nil {
+			return "", err
+		}
+		return testRepo, nil
+	}
 	err = os.MkdirAll(testRepo, os.ModePerm)
 	if err != nil {
 		return "", err

go/performance/utils/sysbench_runner/run_test.go

@@ -53,6 +53,7 @@ func TestRunner(t *testing.T) {
 			"--time=120",
 			"--percentile=50",
 		},
+		InitBigRepo: true,
 	}
 
 	err = Run(conf)