diff --git a/.github/scripts/performance-benchmarking/get-job-json-updated.sh b/.github/scripts/performance-benchmarking/get-job-json-updated.sh new file mode 100755 index 0000000000..c8e3bbc6f5 --- /dev/null +++ b/.github/scripts/performance-benchmarking/get-job-json-updated.sh @@ -0,0 +1,68 @@ +#!/bin/sh + +set -e + +if [ "$#" -ne 7 ]; then + echo "Usage: ./get-job-json.sh " + exit 1 +fi + +jobname="$1" +fromServer="$2" +fromVersion="$3" +toServer="$4" +toVersion="$5" +timeprefix="$6" +actorprefix="$7" + +average_time_change_query="select f.test_name as test_name, ROUND(100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) as average_time_percent_change, case when (100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) < 0 then true else false end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name group by f.test_name;" + +echo ' +{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "'$jobname'", + "namespace": "performance-benchmarking" + }, + "spec": { + "backoffLimit": 1, + "template": { + "spec": { + "serviceAccountName": "performance-benchmarking", + "containers": [ + { + "name": "performance-benchmarking", + "image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/performance-benchmarking:latest", + "args": [ + "--schema=/schema.sql", + "--output=html", + "--from-server='$fromServer'", + "--from-version='$fromVersion'", + "--to-server='$toServer'", + "--to-version='$toVersion'", + "--bucket=performance-benchmarking-github-actions-results", + "--region=us-west-2", + "--results-dir='$timeprefix'", + "--results-prefix='$actorprefix'", + "'"$average_time_change_query"'" + ] + } + ], + "restartPolicy": "Never", + "nodeSelector": { + "performance-benchmarking-worker": "true" + }, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "dedicated", + "operator": "Equal", + "value": "performance-benchmarking-worker" + } + ] + } + } + } +} +' diff --git a/.github/scripts/performance-benchmarking/run-benchmarks-updated.sh b/.github/scripts/performance-benchmarking/run-benchmarks-updated.sh new file mode 100755 index 0000000000..843da3a998 --- /dev/null +++ b/.github/scripts/performance-benchmarking/run-benchmarks-updated.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -e + +if [ -z "$KUBECONFIG" ]; then + echo "Must set KUBECONFIG" + exit 1 +fi + +if [ -z "$TEMPLATE_SCRIPT" ]; then + echo "Must set TEMPLATE_SCRIPT" + exit 1 +fi + +if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then + echo "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION" + exit 1 +fi + +if [ -z "$ACTOR" ]; then + echo "Must set ACTOR" + exit 1 +fi + +if [ -z "$MODE" ]; then + echo "Must set MODE" + exit 1 +fi + +echo "Setting from $FROM_SERVER: $FROM_VERSION" +echo "Setting to $TO_SERVER: $TO_VERSION" + +jobname="performance-benchmarking-$ACTOR" + +timeprefix=$(date +%Y/%m/%d) + +actorprefix="$MODE/$ACTOR/$TO_VERSION" + +source "$TEMPLATE_SCRIPT" "$jobname" "$FROM_SERVER" "$FROM_VERSION" "$TO_SERVER" "$TO_VERSION" "$timeprefix" "$actorprefix" > job.json + +KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json + +out=$(KUBECONFIG="$KUBECONFIG" kubectl wait job/"$jobname" --for=condition=complete -n performance-benchmarking --timeout=1500s || true) + +if [ "$out" != "job.batch/$jobname condition met" ]; then + echo "output of kubectl wait: $out" + KUBECONFIG="$KUBECONFIG" kubectl logs job/"$jobname" -n performance-benchmarking +else + echo "::set-output name=object-key::$timeprefix/$actorprefix/comparison-results.log" +fi + +KUBECONFIG="$KUBECONFIG" kubectl delete job/"$jobname" -n performance-benchmarking + +exit 0 diff --git a/.github/workflows/ci-performance-benchmarks-release.yaml b/.github/workflows/ci-performance-benchmarks-release.yaml index 54b5d9b1d4..a9b25ae2bd 100644 --- a/.github/workflows/ci-performance-benchmarks-release.yaml +++ b/.github/workflows/ci-performance-benchmarks-release.yaml @@ -4,9 +4,9 @@ on: - 'db/add-ses-email-action' jobs: - test-emailer: + benchmark-release-mysql: runs-on: ubuntu-18.04 - name: Test Emailer + name: Benchmark Dolt Release vs MySQL 8 steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials @@ -15,17 +15,43 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-west-2 + - name: Install aws-iam-authenticator + run: | + curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \ + chmod +x ./aws-iam-authenticator && \ + sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator + aws-iam-authenticator version + - name: Create and Auth kubeconfig + run: | + echo "$CONFIG" > kubeconfig + KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1 + KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking + KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context + env: + CONFIG: ${{ secrets.CORP_KUBECONFIG }} + - name: Run benchmarks + id: run-benchmarks + run: ./.github/scripts/performance-benchmarking/run-benchmarks-updated.sh + env: + FROM_SERVER: 'mysql' + FROM_VERSION: '8.0.22' + TO_SERVER: 'dolt' + TO_VERSION: '0.22.11' + MODE: 'release' + ACTOR: ${{ github.actor }} + KUBECONFIG: "./kubeconfig" + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-job-json-updated.sh" - name: Get benchmark results id: get-results run: | echo "Get benchmark results here: $KEY" aws s3api get-object --bucket=performance-benchmarking-github-actions-results --key="$KEY" results.log env: - KEY: '2021/01/19/coffeegoddd/0.22.11/comparison-results.log' + KEY: ${{ steps.run-benchmarks.outputs.object-key }} - name: Send Email uses: ./.github/actions/ses-email-action with: region: us-west-2 - version: 'test-version-0.22.11' + version: '0.22.11' toAddresses: '["dustin@dolthub.com"]' dataFile: ${{ format('{0}/results.log', github.workspace) }}