starter (#4912)

.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh

@@ -19,6 +19,7 @@ issueNumber="$9"
 initBigRepo="${10}"
 nomsBinFormat="${11}"
 withTpcc="${12}"
+withSysbench="${13}"
 tpccRegex="tpcc%"
 
 if [ -n "$initBigRepo" ]; then
@@ -33,11 +34,35 @@ if [ -n "$withTpcc" ]; then
   withTpcc="\"--withTpcc=$withTpcc\","
 fi
 
-readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan', 'types_table_scan', 'index_join')"
-medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"
+query1Tests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan', 'types_table_scan', 'index_join')"
+query2Tests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'bulk_insert', 'oltp_write_only', 'oltp_delete_insert', 'types_delete_insert')"
 
-writeTests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'bulk_insert', 'oltp_write_only', 'oltp_delete_insert', 'types_delete_insert')"
-medianLatencyChangeWritesQuery="select f.test_name as write_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name;"
+query1Name="read_tests"
+query2Name="write_tests"
+
+latencyQuery1="select f.test_name as $query1Name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $query1Tests group by f.test_name;"
+latencyQuery2="select f.test_name as $query2Name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $query2Tests group by f.test_name;"
+
+if [ -n "$withSysbench" ]; then
+    withSystab="\"--withSystab=$withSystab\","
+    query1Name="sys_tab"
+    query2Name="sys_tab_dummy"
+    #query1Tests="('dolt_diff_table_commit_filter', 'dolt_diffs_commit_filter', 'dolt_history_commit_filter', 'dolt_log_commit_filter', 'dolt_commits_commit_filter', 'dolt_commit_ancestors_commit_filter', 'dolt_diff_log_join_on_commit')"
+    #query2Tests="('dolt_diff_table_commit_filter_dummy', 'dolt_diffs_commit_filter_dummy', 'dolt_history_commit_filter_dummy', 'dolt_log_commit_filter_dummy', 'dolt_commits_commit_filter_dummy', 'dolt_commit_ancestors_commit_filter_dummy', 'dolt_diff_log_join_on_commit_dummy')"
+    query1Tests="('gen/dolt_diff_table_commit_filter.gen.lua')"
+    query2Tests="('gen/dolt_diff_table_commit_filter_dummy.gen.lua')"
+    latencyQuery1="
+Select name, dolt_multiple from (
+  Select
+    trim(TRAILING '.gen.lua' FROM trim(LEADING 'gen/' FROM test_name)) as name,
+    round(latency_percentile / lead(latency_percentile) over w, 2) as dolt_multiple,
+    row_number() over w as rn
+  From from_results
+  Having mod(rn,2) = 1
+  Window w as (order by test_name ROWS between 1 preceding and 1 following)
+)a;"
+    latencyQuery2=
+fi
 
 tpccLatencyQuery="select f.test_name as test_name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.25 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.25 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE '$tpccRegex' group by f.test_name;"
 tpccTpsQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.sql_transactions_per_second) as tps, t.test_name as test_name, t.server_name, t.server_version, avg(t.sql_transactions_per_second) as tps, case when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) < -0.5 then 1 when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) > 0.5 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE 'tpcc%' group by f.test_name;"
@@ -95,8 +120,8 @@ echo '
               '"$withTpcc"'
               '"$initBigRepo"'
               '"$nomsBinFormat"'
-              "--sysbenchQueries='"$medianLatencyChangeReadsQuery"'",
-              "--sysbenchQueries='"$medianLatencyChangeWritesQuery"'",
+              "--sysbenchQueries='"$latencyQuery1"'",
+              "--sysbenchQueries='"$latencyQuery2"'",
               "--tpccQueries='"$tpccLatencyQuery"'",
               "--tpccQueries='"$tpccTpsQuery"'"
             ]

.github/scripts/performance-benchmarking/run-benchmarks.sh

@@ -56,6 +56,8 @@ timesuffix=`date +%s%N`
 jobname="$actorShort"
 if [ -n "$WITH_TPCC" ]; then
   jobname="$jobname-tpcc"
+elif [ -n "$WITH_SYSTAB" ]; then
+  jobname="$jobname-systab"
 fi
 jobname="$jobname-$timesuffix"
 
@@ -85,7 +87,8 @@ source \
   "$issuenumber"     \
   "$INIT_BIG_REPO"   \
   "$NOMS_BIN_FORMAT" \
-  "$WITH_TPCC" > job.json
+  "$WITH_TPCC" \
+  "$WITH_SYSTAB" > job.json
 
 out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
 

.github/workflows/k8s-benchmark-latency.yaml

@@ -10,7 +10,7 @@ jobs:
     name: Benchmark Performance
     strategy:
       matrix:
-        dolt_fmt: [ "__LD_1__", "__DOLT__" ]
+        dolt_fmt: [ "__DOLT__" ]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -37,11 +37,27 @@ jobs:
           KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context
         env:
           CONFIG: ${{ secrets.CORP_KUBECONFIG }}
-      - name: Create Sysbench Performance Benchmarking K8s Job
+      #- name: Create Sysbench Performance Benchmarking K8s Job
+        #run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
+        #env:
+          #FROM_SERVER: ${{ github.event.client_payload.from_server }}
+          #FROM_VERSION: ${{ github.event.client_payload.from_version }}
+          #TO_SERVER: ${{ github.event.client_payload.to_server }}
+          #TO_VERSION: ${{ github.event.client_payload.to_version }}
+          #MODE: ${{ github.event.client_payload.mode }}
+          #ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
+          #ACTOR: ${{ github.event.client_payload.actor }}
+          #ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
+          #REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
+          #KUBECONFIG: "./kubeconfig"
+          #INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
+          #NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
+          #TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
+      - name: System Tables
         run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
         env:
-          FROM_SERVER: ${{ github.event.client_payload.from_server }}
-          FROM_VERSION: ${{ github.event.client_payload.from_version }}
+          FROM_SERVER: ${{ github.event.client_payload.to_server }}
+          FROM_VERSION: ${{ github.event.client_payload.to_version }}
           TO_SERVER: ${{ github.event.client_payload.to_server }}
           TO_VERSION: ${{ github.event.client_payload.to_version }}
           MODE: ${{ github.event.client_payload.mode }}
@@ -50,23 +66,24 @@ jobs:
           ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
           REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
           KUBECONFIG: "./kubeconfig"
-          INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
+          INIT_BIG_REPO: "false"
           NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
-          TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
-      - name: Create TPCC Performance Benchmarking K8s Job
-        run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
-        env:
-          FROM_SERVER: ${{ github.event.client_payload.from_server }}
-          FROM_VERSION: ${{ github.event.client_payload.from_version }}
-          TO_SERVER: ${{ github.event.client_payload.to_server }}
-          TO_VERSION: ${{ github.event.client_payload.to_version }}
-          MODE: ${{ github.event.client_payload.mode }}
-          ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
-          ACTOR: ${{ github.event.client_payload.actor }}
-          ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
-          REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
-          KUBECONFIG: "./kubeconfig"
-          INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
-          NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
-          WITH_TPCC: "true"
-          TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
+          TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"
+          WITH_SYSTAB: "true"
+      #- name: Create TPCC Performance Benchmarking K8s Job
+        #run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
+        #env:
+          #FROM_SERVER: ${{ github.event.client_payload.from_server }}
+          #FROM_VERSION: ${{ github.event.client_payload.from_version }}
+          #TO_SERVER: ${{ github.event.client_payload.to_server }}
+          #TO_VERSION: ${{ github.event.client_payload.to_version }}
+          #MODE: ${{ github.event.client_payload.mode }}
+          #ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
+          #ACTOR: ${{ github.event.client_payload.actor }}
+          #ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
+          #REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
+          #KUBECONFIG: "./kubeconfig"
+          #INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
+          #NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }}
+          #WITH_TPCC: "true"
+          #TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}

.github/workflows/nightly-performance-benchmarks-email-report.yaml

@@ -14,12 +14,27 @@ jobs:
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}
           event-type: benchmark-latency
-          client-payload: '{"from_server": "mysql", "from_version": "8.0.28", "to_server": "dolt", "to_version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"}'
+          client-payload: |
+            {
+              "from_server": "mysql",
+              "from_version": "8.0.28",
+              "to_server": "dolt",
+              "to_version": "${{ github.sha }}",
+              "mode": "nightly",
+              "actor": "${{ github.actor }}",
+              "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"
+            }
       - uses: peter-evans/repository-dispatch@v2.0.0
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}
           event-type: sql-correctness
-          client-payload: '{"to_version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}'
+          client-payload: |
+            {
+              "to_version": "${{ github.sha }}",
+              "mode": "nightly",
+              "actor": "${{ github.actor }}",
+              "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"
+            }
       - uses: peter-evans/repository-dispatch@v2.0.0
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}

.github/workflows/performance-benchmarks-email-report.yaml

@@ -43,7 +43,17 @@ jobs:
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}
           event-type: benchmark-latency
-          client-payload: '{"from_server": "mysql", "from_version": "8.0.28", "to_server": "dolt", "to_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"}'
+          client-payload: |
+            {
+              "from_server": "mysql",
+              "from_version": "8.0.28",
+              "to_server": "dolt",
+              "to_version": "${{ needs.set-version-actor.outputs.version }}",
+              "mode": "release",
+              "actor": "${{ needs.set-version-actor.outputs.actor }}",
+              "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}",
+              "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"
+            }
       - uses: peter-evans/repository-dispatch@v2.0.0
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}

.github/workflows/performance-benchmarks-pull-report.yaml

@@ -61,4 +61,15 @@ jobs:
         with:
           token: ${{ secrets.REPO_ACCESS_TOKEN }}
           event-type: benchmark-latency
-          client-payload: '{"from_server": "dolt", "from_version": "${{ github.sha }}", "to_server": "dolt", "to_version": "${{ steps.comment-branch.outputs.head_sha }}", "mode": "pullRequest", "issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "init_big_repo": "true", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"}'
+          client-payload: |
+          {
+            "from_server": "dolt",
+            "from_version": "${{ github.sha }}",
+            "to_server": "dolt",
+            "to_version": "${{ steps.comment-branch.outputs.head_sha }}",
+            "mode": "pullRequest",
+            "issue_number": "${{ steps.get_pull_number.outputs.pull_number }}",
+            "init_big_repo": "true",
+            "actor": "${{ github.actor }}",
+            "template_script": "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"
+          }