diff --git a/.github/scripts/performance-benchmarking/get-andy-dolt-dolt-json.sh b/.github/scripts/performance-benchmarking/get-andy-dolt-dolt-json.sh new file mode 100755 index 0000000000..ade3b73613 --- /dev/null +++ b/.github/scripts/performance-benchmarking/get-andy-dolt-dolt-json.sh @@ -0,0 +1,100 @@ +#!/bin/sh + +set -e + +if [ "$#" -lt 10 ]; then + echo "Usage: ./get-job-json.sh " + exit 1 +fi + +jobName="$1" +fromServer="$2" +fromVersion="$3" +toServer="$4" +toVersion="$5" +timePrefix="$6" +actorPrefix="$7" +format="$8" +issueNumber="$9" +initBigRepo="${10}" +nomsBinFormat="${11}" +tpccRegex="tpcc%" + +readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan')" +medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;" + +writeTests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'bulk_insert', 'oltp_write_only', 'oltp_delete')" +medianLatencyChangeWritesQuery="select f.test_name as write_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name;" + +tpccLatencyQuery="select f.test_name as test_name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.25 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.25 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE '$tpccRegex' group by f.test_name;" +tpccTpsQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.sql_transactions_per_second) as tps, t.test_name as test_name, t.server_name, t.server_version, avg(t.sql_transactions_per_second) as tps, case when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) < -0.5 then 1 when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) > 0.5 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE 'tpcc%' group by f.test_name;" + +echo ' +{ + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "name": "'$jobName'", + "namespace": "performance-benchmarking" + }, + "spec": { + "backoffLimit": 1, + "template": { + "spec": { + "serviceAccountName": "performance-benchmarking", + "containers": [ + { + "name": "performance-benchmarking", + "image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/performance-benchmarking:andy", + "resources": { + "limits": { + "cpu": "7000m" + } + }, + "env": [ + { "name": "GOMAXPROCS", "value": "7" }, + { "name": "ACTOR", "value": "'$ACTOR'" }, + { "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" }, + { "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" } + ], + "imagePullPolicy": "Always", + "args": [ + "--schema=/schema.sql", + "--useDoltHubLuaScriptsRepo", + "--output='$format'", + "--from-server='$fromServer'", + "--from-version='$fromVersion'", + "--to-server='$toServer'", + "--to-version='$toVersion'", + "--bucket=performance-benchmarking-github-actions-results", + "--region=us-west-2", + "--issue-number='$issueNumber'", + "--results-dir='$timePrefix'", + "--results-prefix='$actorPrefix'", + "--withTpcc=true", + "--sysbenchQueries='"$medianLatencyChangeReadsQuery"'", + "--sysbenchQueries='"$medianLatencyChangeWritesQuery"'", + "--tpccQueries='"$tpccLatencyQuery"'", + "--tpccQueries='"$tpccTpsQuery"'", + "--init-big-repo='"$initBigRepo"'"", + "--noms-bin-format='"$nomsBinFormat"'"" + ] + } + ], + "restartPolicy": "Never", + "nodeSelector": { + "performance-benchmarking-worker": "true" + }, + "tolerations": [ + { + "effect": "NoSchedule", + "key": "dedicated", + "operator": "Equal", + "value": "performance-benchmarking-worker" + } + ] + } + } + } +} +' diff --git a/.github/workflows/ci-performance-benchmarks-andy.yaml b/.github/workflows/ci-performance-benchmarks-andy.yaml new file mode 100644 index 0000000000..ed2578154b --- /dev/null +++ b/.github/workflows/ci-performance-benchmarks-andy.yaml @@ -0,0 +1,109 @@ +name: Benchmark Pull Requests + +on: + pull_request: + types: [ opened ] + issue_comment: + types: [ created ] + +jobs: + validate-commentor: + runs-on: ubuntu-18.04 + outputs: + valid: ${{ steps.set_valid.outputs.valid }} + steps: + - uses: actions/checkout@v2 + - name: Validate Commentor + id: set_valid + run: ./.github/scripts/performance-benchmarking/validate-commentor.sh "$ACTOR" + env: + ACTOR: ${{ github.actor }} + + check-comments: + runs-on: ubuntu-18.04 + needs: validate-commentor + if: ${{ needs.validate-commentor.outputs.valid == 'true' }} + outputs: + benchmark: ${{ steps.set_benchmark.outputs.benchmark }} + comment-body: ${{ steps.set_body.outputs.body }} + steps: + - name: Check for Deploy Trigger + uses: dolthub/pull-request-comment-trigger@master + id: check + with: + trigger: '#benchmark' + reaction: rocket + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set Benchmark + if: ${{ steps.check.outputs.triggered == 'true' }} + id: set_benchmark + run: | + echo "::set-output name=benchmark::true" + + performance: + strategy: + matrix: + biginit: ["true", "false"] + nbf: ["__LD_1__", "__DOLT_1__"] + exclude: + - biginit: "true" + nbf: "__DOLT_1__" + - bigint: "false" + nbf: "__LD_1__" + runs-on: ubuntu-18.04 + needs: [validate-commentor, check-comments] + if: ${{ needs.check-comments.outputs.benchmark == 'true' }} + name: Benchmark Performance + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: azure/setup-kubectl@v2.0 + with: + version: 'v1.23.6' + - name: Install aws-iam-authenticator + run: | + curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \ + chmod +x ./aws-iam-authenticator && \ + sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator + aws-iam-authenticator version + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + - name: Create and Auth kubeconfig + run: | + echo "$CONFIG" > kubeconfig + KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1 + KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking + KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context + env: + CONFIG: ${{ secrets.CORP_KUBECONFIG }} + - name: Get pull number + uses: actions/github-script@v3 + id: get_pull_number + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: core.setOutput("pull_number", JSON.stringify(context.issue.number)); + - name: Run benchmarks + id: run-benchmarks + run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + env: + FROM_SERVER: 'dolt' + FROM_VERSION: ${{ github.sha }} + TO_SERVER: 'dolt' + TO_VERSION: ${{ steps.comment-branch.outputs.head_sha }} + MODE: 'pullRequest' + ISSUE_NUMBER: ${{ steps.get_pull_number.outputs.pull_number }} + ACTOR: ${{ github.actor }} + REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + KUBECONFIG: "./kubeconfig" + INIT_BIG_REPO: ${{ matrix.biginit }} + NOMS_BIN_FORMAT: ${{ matrix.nbf }} + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-andy-dolt-dolt-job-json.sh"