/.github/{scripts,workflows}: run tpcc as separate job

2026-01-05 00:50:17 -06:00 · 2022-07-21 12:12:01 -07:00
parent df8a3bd647
commit 60f7f2a9dc
7 changed files with 81 additions and 14 deletions
--- a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh
+++ b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh
@@ -3,7 +3,7 @@
 set -e

 if [ "$#" -lt 9 ]; then
-    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat>"
+    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat> <withTpcc>"
    exit 1
 fi

@@ -18,6 +18,7 @@ format="$8"
 issueNumber="$9"
 initBigRepo="${10}"
 nomsBinFormat="${11}"
+withTpcc="${12}"
 tpccRegex="tpcc%"

 if [ -n "$initBigRepo" ]; then
@@ -28,6 +29,10 @@ if [ -n "$nomsBinFormat" ]; then
  nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
 fi

+if [ -n "$withTpcc" ]; then
+  withTpcc="\"--withTpcc=$withTpcc\","
+fi
+
 readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan')"
 medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"

@@ -79,7 +84,7 @@ echo '
              "--issue-number='$issueNumber'",
              "--results-dir='$timePrefix'",
              "--results-prefix='$actorPrefix'",
-              "--withTpcc=true",
+              '"$withTpcc"'
              '"$initBigRepo"'
              '"$nomsBinFormat"'
              "--sysbenchQueries='"$medianLatencyChangeReadsQuery"'",
--- a/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh
+++ b/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh
@@ -3,7 +3,7 @@
 set -e

 if [ "$#" -lt 9 ]; then
-    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timeprefix> <actorprefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat>"
+    echo  "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timeprefix> <actorprefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat> <withTpcc>"
    exit 1
 fi

@@ -18,6 +18,7 @@ format="$8"
 issueNumber="$9"
 initBigRepo="${10}"
 nomsBinFormat="${11}"
+withTpcc="${12}"
 precision="1"
 tpccRegex="tpcc%"

@@ -29,6 +30,10 @@ if [ -n "$nomsBinFormat" ]; then
  nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
 fi

+if [ -n "$withTpcc" ]; then
+  withTpcc="\"--withTpcc=$withTpcc\","
+fi
+
 readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan')"
 medianLatencyMultiplierReadsQuery="select f.test_name as read_tests, f.server_name, f.server_version, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, t.server_name, t.server_version, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) < 1.0 then 1.0 else ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) end as multiplier from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"
 meanMultiplierReadsQuery="select round(avg(multipliers), $precision) as reads_mean_multiplier from (select case when (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) < 1.0 then 1.0 else (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) end as multipliers from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name)"
@@ -85,7 +90,7 @@ echo '
              "--region=us-west-2",
              "--results-dir='$timeprefix'",
              "--results-prefix='$actorprefix'",
-              "--withTpcc=true",
+              '"$withTpcc"'
              '"$initBigRepo"'
              '"$nomsBinFormat"'
              "--sysbenchQueries='"$medianLatencyMultiplierReadsQuery"'",
--- a/.github/scripts/performance-benchmarking/run-benchmarks.sh
+++ b/.github/scripts/performance-benchmarking/run-benchmarks.sh
@@ -72,7 +72,8 @@ source \
  "$format"          \
  "$issuenumber"     \
  "$INIT_BIG_REPO"   \
-  "$NOMS_BIN_FORMAT" > job.json
+  "$NOMS_BIN_FORMAT" \
+  "$WITH_TPCC" > job.json

 out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)

--- a/.github/workflows/ci-nightly.yaml
+++ b/.github/workflows/ci-nightly.yaml
@@ -38,8 +38,7 @@ jobs:
          KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context
        env:
          CONFIG: ${{ secrets.CORP_KUBECONFIG }}
-      - name: Run benchmarks
-        id: run-benchmarks
+      - name: Run Sysbench benchmarks
        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
        env:
          FROM_SERVER: 'mysql'
@@ -52,8 +51,21 @@ jobs:
          KUBECONFIG: "./kubeconfig"
          NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"
+      - name: Run Tpcc benchmarks
+        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
+        env:
+          FROM_SERVER: 'mysql'
+          FROM_VERSION: '8.0.28'
+          TO_SERVER: 'dolt'
+          TO_VERSION: ${{ github.sha }}
+          MODE: 'nightly'
+          ACTOR: ${{ github.actor }}
+          REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
+          KUBECONFIG: "./kubeconfig"
+          NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
+          WITH_TPCC: 'true'
+          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"
      - name: Run correctness
-        id: run-correctness
        run: ./.github/scripts/sql-correctness/run-correctness.sh
        env:
          TO_VERSION: ${{ github.sha }}
--- a/.github/workflows/ci-performance-benchmarks-new-format.yaml
+++ b/.github/workflows/ci-performance-benchmarks-new-format.yaml
@@ -82,8 +82,7 @@ jobs:
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: core.setOutput("pull_number", JSON.stringify(context.issue.number));
-      - name: Run benchmarks
-        id: run-benchmarks
+      - name: Run Sysbench benchmarks
        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
        env:
          FROM_SERVER: 'dolt'
@@ -98,3 +97,19 @@ jobs:
          INIT_BIG_REPO: "false"
          NOMS_BIN_FORMAT: "__DOLT_1__"
          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"
+      - name: Run Tpcc benchmarks
+        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
+        env:
+          FROM_SERVER: 'dolt'
+          FROM_VERSION: ${{ github.sha }}
+          TO_SERVER: 'dolt'
+          TO_VERSION: ${{ steps.comment-branch.outputs.head_sha }}
+          MODE: 'pullRequest'
+          ISSUE_NUMBER: ${{ steps.get_pull_number.outputs.pull_number }}
+          ACTOR: ${{ github.actor }}
+          REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
+          KUBECONFIG: "./kubeconfig"
+          INIT_BIG_REPO: "false"
+          NOMS_BIN_FORMAT: "__DOLT_1__"
+          WITH_TPCC: 'true'
+          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"
--- a/.github/workflows/ci-performance-benchmarks-release.yaml
+++ b/.github/workflows/ci-performance-benchmarks-release.yaml
@@ -66,8 +66,7 @@ jobs:
          KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context
        env:
          CONFIG: ${{ secrets.CORP_KUBECONFIG }}
-      - name: Run benchmarks
-        id: run-benchmarks
+      - name: Run Sysbench benchmarks
        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
        env:
          FROM_SERVER: 'mysql'
@@ -81,3 +80,18 @@ jobs:
          KUBECONFIG: "./kubeconfig"
          NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"
+      - name: Run Tpcc benchmarks
+        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
+        env:
+          FROM_SERVER: 'mysql'
+          FROM_VERSION: '8.0.28'
+          TO_SERVER: 'dolt'
+          TO_VERSION: ${{ needs.set-version-actor.outputs.version }}
+          MODE: 'release'
+          ACTOR: ${{ needs.set-version-actor.outputs.actor }}
+          ACTOR_EMAIL: ${{ needs.set-version-actor.outputs.actor_email }}
+          REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
+          KUBECONFIG: "./kubeconfig"
+          NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
+          WITH_TPCC: 'true'
+          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"
--- a/.github/workflows/ci-performance-benchmarks.yaml
+++ b/.github/workflows/ci-performance-benchmarks.yaml
@@ -85,8 +85,7 @@ jobs:
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: core.setOutput("pull_number", JSON.stringify(context.issue.number));
-      - name: Run benchmarks
-        id: run-benchmarks
+      - name: Run Sysbench benchmarks
        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
        env:
          FROM_SERVER: 'dolt'
@@ -101,3 +100,19 @@ jobs:
          INIT_BIG_REPO: "true"
          NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"
+      - name: Run Tpcc benchmarks
+        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
+        env:
+          FROM_SERVER: 'dolt'
+          FROM_VERSION: ${{ github.sha }}
+          TO_SERVER: 'dolt'
+          TO_VERSION: ${{ steps.comment-branch.outputs.head_sha }}
+          MODE: 'pullRequest'
+          ISSUE_NUMBER: ${{ steps.get_pull_number.outputs.pull_number }}
+          ACTOR: ${{ github.actor }}
+          REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
+          KUBECONFIG: "./kubeconfig"
+          INIT_BIG_REPO: "true"
+          NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
+          WITH_TPCC: 'true'
+          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"