Merge pull request #3524 from dolthub/andy/benchmark-setup-optional

[no-release-notes] refactor benchmark repo setup

.github/scripts/performance-benchmarking/get-andy-dolt-dolt-json.sh

@@ -0,0 +1,100 @@
+#!/bin/sh
+
+set -e
+
+if [ "$#" -lt 10 ]; then
+    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat>"
+    exit 1
+fi
+
+jobName="$1"
+fromServer="$2"
+fromVersion="$3"
+toServer="$4"
+toVersion="$5"
+timePrefix="$6"
+actorPrefix="$7"
+format="$8"
+issueNumber="$9"
+initBigRepo="${10}"
+nomsBinFormat="${11}"
+tpccRegex="tpcc%"
+
+readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan')"
+medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"
+
+writeTests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'bulk_insert', 'oltp_write_only', 'oltp_delete')"
+medianLatencyChangeWritesQuery="select f.test_name as write_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name;"
+
+tpccLatencyQuery="select f.test_name as test_name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.25 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.25 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE '$tpccRegex' group by f.test_name;"
+tpccTpsQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.sql_transactions_per_second) as tps, t.test_name as test_name, t.server_name, t.server_version, avg(t.sql_transactions_per_second) as tps, case when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) < -0.5 then 1 when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) > 0.5 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE 'tpcc%' group by f.test_name;"
+
+echo '
+{
+  "apiVersion": "batch/v1",
+  "kind": "Job",
+  "metadata": {
+    "name": "'$jobName'",
+    "namespace": "performance-benchmarking"
+  },
+  "spec": {
+    "backoffLimit": 1,
+    "template": {
+      "spec": {
+        "serviceAccountName": "performance-benchmarking",
+        "containers": [
+          {
+            "name": "performance-benchmarking",
+            "image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/performance-benchmarking:andy",
+            "resources": {
+              "limits": {
+                "cpu": "7000m"
+              }
+            },
+            "env": [
+              { "name": "GOMAXPROCS", "value": "7" },
+              { "name": "ACTOR", "value": "'$ACTOR'" },
+              { "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
+              { "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
+            ],
+            "imagePullPolicy": "Always",
+            "args": [
+              "--schema=/schema.sql",
+              "--useDoltHubLuaScriptsRepo",
+              "--output='$format'",
+              "--from-server='$fromServer'",
+              "--from-version='$fromVersion'",
+              "--to-server='$toServer'",
+              "--to-version='$toVersion'",
+              "--bucket=performance-benchmarking-github-actions-results",
+              "--region=us-west-2",
+              "--issue-number='$issueNumber'",
+              "--results-dir='$timePrefix'",
+              "--results-prefix='$actorPrefix'",
+              "--withTpcc=true",
+              "--sysbenchQueries='"$medianLatencyChangeReadsQuery"'",
+              "--sysbenchQueries='"$medianLatencyChangeWritesQuery"'",
+              "--tpccQueries='"$tpccLatencyQuery"'",
+              "--tpccQueries='"$tpccTpsQuery"'",
+              "--init-big-repo='"$initBigRepo"'"",
+              "--noms-bin-format='"$nomsBinFormat"'""
+            ]
+          }
+        ],
+        "restartPolicy": "Never",
+        "nodeSelector": {
+          "performance-benchmarking-worker": "true"
+        },
+        "tolerations": [
+          {
+              "effect": "NoSchedule",
+              "key": "dedicated",
+              "operator": "Equal",
+              "value": "performance-benchmarking-worker"
+          }
+        ]
+      }
+    }
+  }
+}
+'

.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh

@@ -2,8 +2,8 @@
 
 set -e
 
-if [ "$#" -lt 9 ]; then
-    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <format> <issueNumber>"
+if [ "$#" -lt 10 ]; then
+    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat>"
     exit 1
 fi
 
@@ -16,6 +16,8 @@ timePrefix="$6"
 actorPrefix="$7"
 format="$8"
 issueNumber="$9"
+initBigRepo="${10}"
+nomsBinFormat="${11}"
 tpccRegex="tpcc%"
 
 readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan')"
@@ -74,7 +76,8 @@ echo '
               "--sysbenchQueries='"$medianLatencyChangeWritesQuery"'",
               "--tpccQueries='"$tpccLatencyQuery"'",
               "--tpccQueries='"$tpccTpsQuery"'",
-              "--init-big-repo"
+              "--init-big-repo='"$initBigRepo"'"",
+              "--noms-bin-format='"$nomsBinFormat"'""
             ]
           }
         ],

.github/scripts/performance-benchmarking/run-benchmarks.sh

@@ -36,7 +36,11 @@ short=${TO_VERSION:0:8}
 lowered=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]')
 actorShort="$lowered-$short"
 
-jobname="$actorShort"
+# random sleep
+sleep 0.$[ ( $RANDOM % 10 )  + 1 ]s
+
+timesuffix=`date +%s%N`
+jobname="$actorShort_$timesuffix"
 
 timeprefix=$(date +%Y/%m/%d)
 
@@ -51,7 +55,19 @@ fi
 # or default to -1
 issuenumber=${ISSUE_NUMBER:-"-1"}
 
-source "$TEMPLATE_SCRIPT" "$jobname" "$FROM_SERVER" "$FROM_VERSION" "$TO_SERVER" "$TO_VERSION" "$timeprefix" "$actorprefix" "$format" "$issuenumber" > job.json
+source \
+  "$TEMPLATE_SCRIPT" \
+  "$jobname"         \
+  "$FROM_SERVER"     \
+  "$FROM_VERSION"    \
+  "$TO_SERVER"       \
+  "$TO_VERSION"      \
+  "$timeprefix"      \
+  "$actorprefix"     \
+  "$format"          \
+  "$issuenumber"     \
+  "$INIT_BIG_REPO"   \
+  "$NOMS_BIN_FORMAT" > job.json
 
 out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
 

.github/workflows/ci-performance-benchmarks-andy.yaml

@@ -0,0 +1,109 @@
+name: Benchmark Pull Requests
+
+on:
+  pull_request:
+    types: [ opened ]
+  issue_comment:
+    types: [ created ]
+
+jobs:
+  validate-commentor:
+    runs-on: ubuntu-18.04
+    outputs:
+      valid: ${{ steps.set_valid.outputs.valid }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Validate Commentor
+        id: set_valid
+        run: ./.github/scripts/performance-benchmarking/validate-commentor.sh "$ACTOR"
+        env:
+          ACTOR: ${{ github.actor }}
+
+  check-comments:
+    runs-on: ubuntu-18.04
+    needs: validate-commentor
+    if: ${{ needs.validate-commentor.outputs.valid == 'true' }}
+    outputs:
+      benchmark: ${{ steps.set_benchmark.outputs.benchmark }}
+      comment-body: ${{ steps.set_body.outputs.body }}
+    steps:
+      - name: Check for Deploy Trigger
+        uses: dolthub/pull-request-comment-trigger@master
+        id: check
+        with:
+          trigger: '#benchmark'
+          reaction: rocket
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set Benchmark
+        if: ${{ steps.check.outputs.triggered == 'true' }}
+        id: set_benchmark
+        run: |
+          echo "::set-output name=benchmark::true"
+
+  performance:
+    strategy:
+      matrix:
+        biginit: ["true", "false"]
+        nbf: ["__LD_1__", "__DOLT_1__"]
+        exclude:
+          - biginit: "true"
+            nbf: "__DOLT_1__"
+          - bigint: "false"
+            nbf: "__LD_1__"
+    runs-on: ubuntu-18.04
+    needs: [validate-commentor, check-comments]
+    if: ${{ needs.check-comments.outputs.benchmark == 'true' }}
+    name: Benchmark Performance
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: azure/setup-kubectl@v2.0
+        with:
+          version: 'v1.23.6'
+      - name: Install aws-iam-authenticator
+        run: |
+          curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \
+          chmod +x ./aws-iam-authenticator && \
+          sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
+          aws-iam-authenticator version
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - uses: xt0rted/pull-request-comment-branch@v1
+        id: comment-branch
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create and Auth kubeconfig
+        run: |
+          echo "$CONFIG" > kubeconfig
+          KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1
+          KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking
+          KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context
+        env:
+          CONFIG: ${{ secrets.CORP_KUBECONFIG }}
+      - name: Get pull number
+        uses: actions/github-script@v3
+        id: get_pull_number
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: core.setOutput("pull_number", JSON.stringify(context.issue.number));
+      - name: Run benchmarks
+        id: run-benchmarks
+        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
+        env:
+          FROM_SERVER: 'dolt'
+          FROM_VERSION: ${{ github.sha }}
+          TO_SERVER: 'dolt'
+          TO_VERSION: ${{ steps.comment-branch.outputs.head_sha }}
+          MODE: 'pullRequest'
+          ISSUE_NUMBER: ${{ steps.get_pull_number.outputs.pull_number }}
+          ACTOR: ${{ github.actor }}
+          REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
+          KUBECONFIG: "./kubeconfig"
+          INIT_BIG_REPO: ${{ matrix.biginit }}
+          NOMS_BIN_FORMAT: ${{ matrix.nbf }}
+          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-andy-dolt-dolt-job-json.sh"

.github/workflows/ci-performance-benchmarks.yaml

@@ -42,6 +42,15 @@ jobs:
           echo "::set-output name=benchmark::true"
 
   performance:
+    strategy:
+      matrix:
+        biginit: ["true", "false"]
+        nbf: ["__LD_1__", "__DOLT_1__"]
+        exclude:
+          - biginit: "true"
+            nbf: "__DOLT_1__"
+          - bigint: "false"
+            nbf: "__LD_1__"
     runs-on: ubuntu-18.04
     needs: [validate-commentor, check-comments]
     if: ${{ needs.check-comments.outputs.benchmark == 'true' }}
@@ -95,4 +104,6 @@ jobs:
           ACTOR: ${{ github.actor }}
           REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
           KUBECONFIG: "./kubeconfig"
+          INIT_BIG_REPO: ${{ matrix.biginit }}
+          NOMS_BIN_FORMAT: ${{ matrix.nbf }}
           TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"

go/performance/utils/sysbench_runner/config.go

@@ -279,8 +279,10 @@ type Config struct {
 	TestOptions []string
 	// ScriptDir is a path to a directory of lua scripts
 	ScriptDir string
-	// DirtyClone downloads a database with existing chunks and commits
+	// InitBigRepo downloads a database with existing chunks and commits
 	InitBigRepo bool
+	// NomsBinFormat specifies the NomsBinFormat
+	NomsBinFormat string
 }
 
 // NewConfig returns a new Config

go/performance/utils/sysbench_runner/dolt.go

@@ -32,6 +32,7 @@ const (
 	dbName       = "test"
 	luaPath      = "?.lua"
 	bigEmptyRepo = "max-hoffman/big-empty"
+	nbfEnvVar    = "DOLT_DEFAULT_BIN_FORMAT"
 )
 
 var stampFunc = func() string { return time.Now().UTC().Format(stampFormat) }
@@ -50,7 +51,7 @@ func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConf
 		return nil, err
 	}
 
-	testRepo, err := initDoltRepo(ctx, serverConfig, config.InitBigRepo)
+	testRepo, err := initDoltRepo(ctx, serverConfig, config.InitBigRepo, config.NomsBinFormat)
 	if err != nil {
 		return nil, err
 	}
@@ -133,7 +134,7 @@ func doltVersion(ctx context.Context, config *ServerConfig) error {
 }
 
 // initDoltRepo initializes a dolt repo and returns the repo path
-func initDoltRepo(ctx context.Context, config *ServerConfig, initBigRepo bool) (string, error) {
+func initDoltRepo(ctx context.Context, config *ServerConfig, initBigRepo bool, nbf string) (string, error) {
 	cwd, err := os.Getwd()
 	if err != nil {
 		return "", err
@@ -152,6 +153,12 @@ func initDoltRepo(ctx context.Context, config *ServerConfig, initBigRepo bool) (
 		return "", err
 	}
 
+	if nbf != "" {
+		if err = os.Setenv(nbfEnvVar, nbf); err != nil {
+			return "", err
+		}
+	}
+
 	doltInit := ExecCommand(ctx, config.ServerExec, "init")
 	doltInit.Dir = testRepo
 	err = doltInit.Run()