From 60f7f2a9dc855ff4fbbe6a91f486134dd4e465df Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Thu, 21 Jul 2022 12:12:01 -0700 Subject: [PATCH] /.github/{scripts,workflows}: run tpcc as separate job --- .../get-dolt-dolt-job-json.sh | 9 +++++++-- .../get-mysql-dolt-job-json.sh | 9 +++++++-- .../run-benchmarks.sh | 3 ++- .github/workflows/ci-nightly.yaml | 18 +++++++++++++++--- .../ci-performance-benchmarks-new-format.yaml | 19 +++++++++++++++++-- .../ci-performance-benchmarks-release.yaml | 18 ++++++++++++++++-- .../workflows/ci-performance-benchmarks.yaml | 19 +++++++++++++++++-- 7 files changed, 81 insertions(+), 14 deletions(-) diff --git a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh index 213a058223..5a1cd72923 100755 --- a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh +++ b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh @@ -3,7 +3,7 @@ set -e if [ "$#" -lt 9 ]; then - echo "Usage: ./get-job-json.sh " + echo "Usage: ./get-job-json.sh " exit 1 fi @@ -18,6 +18,7 @@ format="$8" issueNumber="$9" initBigRepo="${10}" nomsBinFormat="${11}" +withTpcc="${12}" tpccRegex="tpcc%" if [ -n "$initBigRepo" ]; then @@ -28,6 +29,10 @@ if [ -n "$nomsBinFormat" ]; then nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\"," fi +if [ -n "$withTpcc" ]; then + withTpcc="\"--withTpcc=$withTpcc\"," +fi + readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan')" medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;" @@ -79,7 +84,7 @@ echo ' "--issue-number='$issueNumber'", "--results-dir='$timePrefix'", "--results-prefix='$actorPrefix'", - "--withTpcc=true", + '"$withTpcc"' '"$initBigRepo"' '"$nomsBinFormat"' "--sysbenchQueries='"$medianLatencyChangeReadsQuery"'", diff --git a/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh b/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh index 8c3b217c2e..49362d1df2 100755 --- a/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh +++ b/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh @@ -3,7 +3,7 @@ set -e if [ "$#" -lt 9 ]; then - echo "Usage: ./get-job-json.sh " + echo "Usage: ./get-job-json.sh " exit 1 fi @@ -18,6 +18,7 @@ format="$8" issueNumber="$9" initBigRepo="${10}" nomsBinFormat="${11}" +withTpcc="${12}" precision="1" tpccRegex="tpcc%" @@ -29,6 +30,10 @@ if [ -n "$nomsBinFormat" ]; then nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\"," fi +if [ -n "$withTpcc" ]; then + withTpcc="\"--withTpcc=$withTpcc\"," +fi + readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan')" medianLatencyMultiplierReadsQuery="select f.test_name as read_tests, f.server_name, f.server_version, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, t.server_name, t.server_version, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) < 1.0 then 1.0 else ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) end as multiplier from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;" meanMultiplierReadsQuery="select round(avg(multipliers), $precision) as reads_mean_multiplier from (select case when (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) < 1.0 then 1.0 else (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) end as multipliers from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name)" @@ -85,7 +90,7 @@ echo ' "--region=us-west-2", "--results-dir='$timeprefix'", "--results-prefix='$actorprefix'", - "--withTpcc=true", + '"$withTpcc"' '"$initBigRepo"' '"$nomsBinFormat"' "--sysbenchQueries='"$medianLatencyMultiplierReadsQuery"'", diff --git a/.github/scripts/performance-benchmarking/run-benchmarks.sh b/.github/scripts/performance-benchmarking/run-benchmarks.sh index abe922df4d..28a1e54186 100755 --- a/.github/scripts/performance-benchmarking/run-benchmarks.sh +++ b/.github/scripts/performance-benchmarking/run-benchmarks.sh @@ -72,7 +72,8 @@ source \ "$format" \ "$issuenumber" \ "$INIT_BIG_REPO" \ - "$NOMS_BIN_FORMAT" > job.json + "$NOMS_BIN_FORMAT" \ + "$WITH_TPCC" > job.json out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true) diff --git a/.github/workflows/ci-nightly.yaml b/.github/workflows/ci-nightly.yaml index 2be56c985f..2c47468cc0 100644 --- a/.github/workflows/ci-nightly.yaml +++ b/.github/workflows/ci-nightly.yaml @@ -38,8 +38,7 @@ jobs: KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context env: CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Run benchmarks - id: run-benchmarks + - name: Run Sysbench benchmarks run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh env: FROM_SERVER: 'mysql' @@ -52,8 +51,21 @@ jobs: KUBECONFIG: "./kubeconfig" NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh" + - name: Run Tpcc benchmarks + run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + env: + FROM_SERVER: 'mysql' + FROM_VERSION: '8.0.28' + TO_SERVER: 'dolt' + TO_VERSION: ${{ github.sha }} + MODE: 'nightly' + ACTOR: ${{ github.actor }} + REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + KUBECONFIG: "./kubeconfig" + NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} + WITH_TPCC: 'true' + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh" - name: Run correctness - id: run-correctness run: ./.github/scripts/sql-correctness/run-correctness.sh env: TO_VERSION: ${{ github.sha }} diff --git a/.github/workflows/ci-performance-benchmarks-new-format.yaml b/.github/workflows/ci-performance-benchmarks-new-format.yaml index b7666de1aa..b1afa4ed89 100644 --- a/.github/workflows/ci-performance-benchmarks-new-format.yaml +++ b/.github/workflows/ci-performance-benchmarks-new-format.yaml @@ -82,8 +82,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: core.setOutput("pull_number", JSON.stringify(context.issue.number)); - - name: Run benchmarks - id: run-benchmarks + - name: Run Sysbench benchmarks run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh env: FROM_SERVER: 'dolt' @@ -98,3 +97,19 @@ jobs: INIT_BIG_REPO: "false" NOMS_BIN_FORMAT: "__DOLT_1__" TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh" + - name: Run Tpcc benchmarks + run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + env: + FROM_SERVER: 'dolt' + FROM_VERSION: ${{ github.sha }} + TO_SERVER: 'dolt' + TO_VERSION: ${{ steps.comment-branch.outputs.head_sha }} + MODE: 'pullRequest' + ISSUE_NUMBER: ${{ steps.get_pull_number.outputs.pull_number }} + ACTOR: ${{ github.actor }} + REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + KUBECONFIG: "./kubeconfig" + INIT_BIG_REPO: "false" + NOMS_BIN_FORMAT: "__DOLT_1__" + WITH_TPCC: 'true' + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh" diff --git a/.github/workflows/ci-performance-benchmarks-release.yaml b/.github/workflows/ci-performance-benchmarks-release.yaml index 9b02f2eff3..cdf99279c7 100644 --- a/.github/workflows/ci-performance-benchmarks-release.yaml +++ b/.github/workflows/ci-performance-benchmarks-release.yaml @@ -66,8 +66,7 @@ jobs: KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context env: CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Run benchmarks - id: run-benchmarks + - name: Run Sysbench benchmarks run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh env: FROM_SERVER: 'mysql' @@ -81,3 +80,18 @@ jobs: KUBECONFIG: "./kubeconfig" NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh" + - name: Run Tpcc benchmarks + run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + env: + FROM_SERVER: 'mysql' + FROM_VERSION: '8.0.28' + TO_SERVER: 'dolt' + TO_VERSION: ${{ needs.set-version-actor.outputs.version }} + MODE: 'release' + ACTOR: ${{ needs.set-version-actor.outputs.actor }} + ACTOR_EMAIL: ${{ needs.set-version-actor.outputs.actor_email }} + REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + KUBECONFIG: "./kubeconfig" + NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} + WITH_TPCC: 'true' + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh" diff --git a/.github/workflows/ci-performance-benchmarks.yaml b/.github/workflows/ci-performance-benchmarks.yaml index af6da5e0d4..9461cf14ab 100644 --- a/.github/workflows/ci-performance-benchmarks.yaml +++ b/.github/workflows/ci-performance-benchmarks.yaml @@ -85,8 +85,7 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} script: core.setOutput("pull_number", JSON.stringify(context.issue.number)); - - name: Run benchmarks - id: run-benchmarks + - name: Run Sysbench benchmarks run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh env: FROM_SERVER: 'dolt' @@ -101,3 +100,19 @@ jobs: INIT_BIG_REPO: "true" NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh" + - name: Run Tpcc benchmarks + run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + env: + FROM_SERVER: 'dolt' + FROM_VERSION: ${{ github.sha }} + TO_SERVER: 'dolt' + TO_VERSION: ${{ steps.comment-branch.outputs.head_sha }} + MODE: 'pullRequest' + ISSUE_NUMBER: ${{ steps.get_pull_number.outputs.pull_number }} + ACTOR: ${{ github.actor }} + REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + KUBECONFIG: "./kubeconfig" + INIT_BIG_REPO: "true" + NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} + WITH_TPCC: 'true' + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"