/.github/{scripts,workflows}: iterating on benchmarking against mysql for release

.github/scripts/performance-benchmarking/get-job-json-updated.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+
+set -e
+
+if [ "$#" -ne 7 ]; then
+    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timeprefix> <actorprefix>"
+    exit 1
+fi
+
+jobname="$1"
+fromServer="$2"
+fromVersion="$3"
+toServer="$4"
+toVersion="$5"
+timeprefix="$6"
+actorprefix="$7"
+
+average_time_change_query="select f.test_name as test_name, ROUND(100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) as average_time_percent_change, case when (100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) < 0 then true else false end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name group by f.test_name;"
+
+echo '
+{
+  "apiVersion": "batch/v1",
+  "kind": "Job",
+  "metadata": {
+    "name": "'$jobname'",
+    "namespace": "performance-benchmarking"
+  },
+  "spec": {
+    "backoffLimit": 1,
+    "template": {
+      "spec": {
+        "serviceAccountName": "performance-benchmarking",
+        "containers": [
+          {
+            "name": "performance-benchmarking",
+            "image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/performance-benchmarking:latest",
+            "args": [
+              "--schema=/schema.sql",
+              "--output=html",
+              "--from-server='$fromServer'",
+              "--from-version='$fromVersion'",
+              "--to-server='$toServer'",
+              "--to-version='$toVersion'",
+              "--bucket=performance-benchmarking-github-actions-results",
+              "--region=us-west-2",
+              "--results-dir='$timeprefix'",
+              "--results-prefix='$actorprefix'",
+              "'"$average_time_change_query"'"
+            ]
+          }
+        ],
+        "restartPolicy": "Never",
+        "nodeSelector": {
+          "performance-benchmarking-worker": "true"
+        },
+        "tolerations": [
+          {
+              "effect": "NoSchedule",
+              "key": "dedicated",
+              "operator": "Equal",
+              "value": "performance-benchmarking-worker"
+          }
+        ]
+      }
+    }
+  }
+}
+'

.github/scripts/performance-benchmarking/run-benchmarks-updated.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "$KUBECONFIG" ]; then
+    echo  "Must set KUBECONFIG"
+    exit 1
+fi
+
+if [ -z "$TEMPLATE_SCRIPT" ]; then
+    echo  "Must set TEMPLATE_SCRIPT"
+    exit 1
+fi
+
+if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then
+    echo  "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION"
+    exit 1
+fi
+
+if [ -z "$ACTOR" ]; then
+    echo  "Must set ACTOR"
+    exit 1
+fi
+
+if [ -z "$MODE" ]; then
+    echo  "Must set MODE"
+    exit 1
+fi
+
+echo "Setting from $FROM_SERVER: $FROM_VERSION"
+echo "Setting to $TO_SERVER: $TO_VERSION"
+
+jobname="performance-benchmarking-$ACTOR"
+
+timeprefix=$(date +%Y/%m/%d)
+
+actorprefix="$MODE/$ACTOR/$TO_VERSION"
+
+source "$TEMPLATE_SCRIPT" "$jobname" "$FROM_SERVER" "$FROM_VERSION" "$TO_SERVER" "$TO_VERSION" "$timeprefix" "$actorprefix" > job.json
+
+KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json
+
+out=$(KUBECONFIG="$KUBECONFIG" kubectl wait job/"$jobname" --for=condition=complete -n performance-benchmarking --timeout=1500s || true)
+
+if [ "$out" != "job.batch/$jobname condition met" ]; then
+  echo "output of kubectl wait: $out"
+  KUBECONFIG="$KUBECONFIG" kubectl logs job/"$jobname" -n performance-benchmarking
+else
+  echo "::set-output name=object-key::$timeprefix/$actorprefix/comparison-results.log"
+fi
+
+KUBECONFIG="$KUBECONFIG" kubectl delete job/"$jobname" -n performance-benchmarking
+
+exit 0

.github/workflows/ci-performance-benchmarks-release.yaml

@@ -4,9 +4,9 @@ on:
       - 'db/add-ses-email-action'
 
 jobs:
-  test-emailer:
+  benchmark-release-mysql:
     runs-on: ubuntu-18.04
-    name: Test Emailer
+    name: Benchmark Dolt Release vs MySQL 8
     steps:
       - uses: actions/checkout@v2
       - name: Configure AWS Credentials
@@ -15,17 +15,43 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2
+      - name: Install aws-iam-authenticator
+        run: |
+          curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \
+          chmod +x ./aws-iam-authenticator && \
+          sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
+          aws-iam-authenticator version
+      - name: Create and Auth kubeconfig
+        run: |
+          echo "$CONFIG" > kubeconfig
+          KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1
+          KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking
+          KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context
+        env:
+          CONFIG: ${{ secrets.CORP_KUBECONFIG }}
+      - name: Run benchmarks
+        id: run-benchmarks
+        run: ./.github/scripts/performance-benchmarking/run-benchmarks-updated.sh
+        env:
+          FROM_SERVER: 'mysql'
+          FROM_VERSION: '8.0.22'
+          TO_SERVER: 'dolt'
+          TO_VERSION: '0.22.11'
+          MODE: 'release'
+          ACTOR: ${{ github.actor }}
+          KUBECONFIG: "./kubeconfig"
+          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-job-json-updated.sh"
       - name: Get benchmark results
         id: get-results
         run: |
           echo "Get benchmark results here: $KEY"
           aws s3api get-object --bucket=performance-benchmarking-github-actions-results --key="$KEY" results.log
         env:
-          KEY: '2021/01/19/coffeegoddd/0.22.11/comparison-results.log'
+          KEY: ${{ steps.run-benchmarks.outputs.object-key }}
       - name: Send Email
         uses: ./.github/actions/ses-email-action
         with:
           region: us-west-2
-          version: 'test-version-0.22.11'
+          version: '0.22.11'
           toAddresses: '["dustin@dolthub.com"]'
           dataFile: ${{ format('{0}/results.log', github.workspace) }}