diff --git a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh index a90a2d3cbb..087825c28c 100755 --- a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh +++ b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh @@ -19,6 +19,7 @@ issueNumber="$9" initBigRepo="${10}" nomsBinFormat="${11}" withTpcc="${12}" +withSysbench="${13}" tpccRegex="tpcc%" if [ -n "$initBigRepo" ]; then @@ -33,11 +34,35 @@ if [ -n "$withTpcc" ]; then withTpcc="\"--withTpcc=$withTpcc\"," fi -readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan', 'types_table_scan', 'index_join')" -medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;" +query1Tests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan', 'types_table_scan', 'index_join')" +query2Tests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'bulk_insert', 'oltp_write_only', 'oltp_delete_insert', 'types_delete_insert')" -writeTests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'bulk_insert', 'oltp_write_only', 'oltp_delete_insert', 'types_delete_insert')" -medianLatencyChangeWritesQuery="select f.test_name as write_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name;" +query1Name="read_tests" +query2Name="write_tests" + +latencyQuery1="select f.test_name as $query1Name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $query1Tests group by f.test_name;" +latencyQuery2="select f.test_name as $query2Name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $query2Tests group by f.test_name;" + +if [ -n "$withSysbench" ]; then + withSystab="\"--withSystab=$withSystab\"," + query1Name="sys_tab" + query2Name="sys_tab_dummy" + #query1Tests="('dolt_diff_table_commit_filter', 'dolt_diffs_commit_filter', 'dolt_history_commit_filter', 'dolt_log_commit_filter', 'dolt_commits_commit_filter', 'dolt_commit_ancestors_commit_filter', 'dolt_diff_log_join_on_commit')" + #query2Tests="('dolt_diff_table_commit_filter_dummy', 'dolt_diffs_commit_filter_dummy', 'dolt_history_commit_filter_dummy', 'dolt_log_commit_filter_dummy', 'dolt_commits_commit_filter_dummy', 'dolt_commit_ancestors_commit_filter_dummy', 'dolt_diff_log_join_on_commit_dummy')" + query1Tests="('gen/dolt_diff_table_commit_filter.gen.lua')" + query2Tests="('gen/dolt_diff_table_commit_filter_dummy.gen.lua')" + latencyQuery1=" +Select name, dolt_multiple from ( + Select + trim(TRAILING '.gen.lua' FROM trim(LEADING 'gen/' FROM test_name)) as name, + round(latency_percentile / lead(latency_percentile) over w, 2) as dolt_multiple, + row_number() over w as rn + From from_results + Having mod(rn,2) = 1 + Window w as (order by test_name ROWS between 1 preceding and 1 following) +)a;" + latencyQuery2= +fi tpccLatencyQuery="select f.test_name as test_name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.25 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.25 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE '$tpccRegex' group by f.test_name;" tpccTpsQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.sql_transactions_per_second) as tps, t.test_name as test_name, t.server_name, t.server_version, avg(t.sql_transactions_per_second) as tps, case when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) < -0.5 then 1 when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) > 0.5 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE 'tpcc%' group by f.test_name;" @@ -95,8 +120,8 @@ echo ' '"$withTpcc"' '"$initBigRepo"' '"$nomsBinFormat"' - "--sysbenchQueries='"$medianLatencyChangeReadsQuery"'", - "--sysbenchQueries='"$medianLatencyChangeWritesQuery"'", + "--sysbenchQueries='"$latencyQuery1"'", + "--sysbenchQueries='"$latencyQuery2"'", "--tpccQueries='"$tpccLatencyQuery"'", "--tpccQueries='"$tpccTpsQuery"'" ] diff --git a/.github/scripts/performance-benchmarking/run-benchmarks.sh b/.github/scripts/performance-benchmarking/run-benchmarks.sh index b679c5df35..59dfd49e6e 100755 --- a/.github/scripts/performance-benchmarking/run-benchmarks.sh +++ b/.github/scripts/performance-benchmarking/run-benchmarks.sh @@ -56,6 +56,8 @@ timesuffix=`date +%s%N` jobname="$actorShort" if [ -n "$WITH_TPCC" ]; then jobname="$jobname-tpcc" +elif [ -n "$WITH_SYSTAB" ]; then + jobname="$jobname-systab" fi jobname="$jobname-$timesuffix" @@ -85,7 +87,8 @@ source \ "$issuenumber" \ "$INIT_BIG_REPO" \ "$NOMS_BIN_FORMAT" \ - "$WITH_TPCC" > job.json + "$WITH_TPCC" \ + "$WITH_SYSTAB" > job.json out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true) diff --git a/.github/workflows/k8s-benchmark-latency.yaml b/.github/workflows/k8s-benchmark-latency.yaml index 3b3ccdd38c..51cacae457 100644 --- a/.github/workflows/k8s-benchmark-latency.yaml +++ b/.github/workflows/k8s-benchmark-latency.yaml @@ -10,7 +10,7 @@ jobs: name: Benchmark Performance strategy: matrix: - dolt_fmt: [ "__LD_1__", "__DOLT__" ] + dolt_fmt: [ "__DOLT__" ] steps: - name: Checkout uses: actions/checkout@v3 @@ -37,11 +37,27 @@ jobs: KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context env: CONFIG: ${{ secrets.CORP_KUBECONFIG }} - - name: Create Sysbench Performance Benchmarking K8s Job + #- name: Create Sysbench Performance Benchmarking K8s Job + #run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + #env: + #FROM_SERVER: ${{ github.event.client_payload.from_server }} + #FROM_VERSION: ${{ github.event.client_payload.from_version }} + #TO_SERVER: ${{ github.event.client_payload.to_server }} + #TO_VERSION: ${{ github.event.client_payload.to_version }} + #MODE: ${{ github.event.client_payload.mode }} + #ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} + #ACTOR: ${{ github.event.client_payload.actor }} + #ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }} + #REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + #KUBECONFIG: "./kubeconfig" + #INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }} + #NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} + #TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }} + - name: System Tables run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh env: - FROM_SERVER: ${{ github.event.client_payload.from_server }} - FROM_VERSION: ${{ github.event.client_payload.from_version }} + FROM_SERVER: ${{ github.event.client_payload.to_server }} + FROM_VERSION: ${{ github.event.client_payload.to_version }} TO_SERVER: ${{ github.event.client_payload.to_server }} TO_VERSION: ${{ github.event.client_payload.to_version }} MODE: ${{ github.event.client_payload.mode }} @@ -50,23 +66,24 @@ jobs: ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }} REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} KUBECONFIG: "./kubeconfig" - INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }} + INIT_BIG_REPO: "false" NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} - TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }} - - name: Create TPCC Performance Benchmarking K8s Job - run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh - env: - FROM_SERVER: ${{ github.event.client_payload.from_server }} - FROM_VERSION: ${{ github.event.client_payload.from_version }} - TO_SERVER: ${{ github.event.client_payload.to_server }} - TO_VERSION: ${{ github.event.client_payload.to_version }} - MODE: ${{ github.event.client_payload.mode }} - ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} - ACTOR: ${{ github.event.client_payload.actor }} - ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }} - REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} - KUBECONFIG: "./kubeconfig" - INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }} - NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} - WITH_TPCC: "true" - TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }} + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh" + WITH_SYSTAB: "true" + #- name: Create TPCC Performance Benchmarking K8s Job + #run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + #env: + #FROM_SERVER: ${{ github.event.client_payload.from_server }} + #FROM_VERSION: ${{ github.event.client_payload.from_version }} + #TO_SERVER: ${{ github.event.client_payload.to_server }} + #TO_VERSION: ${{ github.event.client_payload.to_version }} + #MODE: ${{ github.event.client_payload.mode }} + #ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }} + #ACTOR: ${{ github.event.client_payload.actor }} + #ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }} + #REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + #KUBECONFIG: "./kubeconfig" + #INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }} + #NOMS_BIN_FORMAT: ${{ matrix.dolt_fmt }} + #WITH_TPCC: "true" + #TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }} diff --git a/.github/workflows/nightly-performance-benchmarks-email-report.yaml b/.github/workflows/nightly-performance-benchmarks-email-report.yaml index 36a32db910..62f6cfddaf 100644 --- a/.github/workflows/nightly-performance-benchmarks-email-report.yaml +++ b/.github/workflows/nightly-performance-benchmarks-email-report.yaml @@ -14,12 +14,27 @@ jobs: with: token: ${{ secrets.REPO_ACCESS_TOKEN }} event-type: benchmark-latency - client-payload: '{"from_server": "mysql", "from_version": "8.0.28", "to_server": "dolt", "to_version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"}' + client-payload: | + { + "from_server": "mysql", + "from_version": "8.0.28", + "to_server": "dolt", + "to_version": "${{ github.sha }}", + "mode": "nightly", + "actor": "${{ github.actor }}", + "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh" + } - uses: peter-evans/repository-dispatch@v2.0.0 with: token: ${{ secrets.REPO_ACCESS_TOKEN }} event-type: sql-correctness - client-payload: '{"to_version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh"}' + client-payload: | + { + "to_version": "${{ github.sha }}", + "mode": "nightly", + "actor": "${{ github.actor }}", + "template_script": "./.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh" + } - uses: peter-evans/repository-dispatch@v2.0.0 with: token: ${{ secrets.REPO_ACCESS_TOKEN }} diff --git a/.github/workflows/performance-benchmarks-email-report.yaml b/.github/workflows/performance-benchmarks-email-report.yaml index 40298c91d3..d1cd410cb8 100644 --- a/.github/workflows/performance-benchmarks-email-report.yaml +++ b/.github/workflows/performance-benchmarks-email-report.yaml @@ -43,7 +43,17 @@ jobs: with: token: ${{ secrets.REPO_ACCESS_TOKEN }} event-type: benchmark-latency - client-payload: '{"from_server": "mysql", "from_version": "8.0.28", "to_server": "dolt", "to_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"}' + client-payload: | + { + "from_server": "mysql", + "from_version": "8.0.28", + "to_server": "dolt", + "to_version": "${{ needs.set-version-actor.outputs.version }}", + "mode": "release", + "actor": "${{ needs.set-version-actor.outputs.actor }}", + "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", + "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh" + } - uses: peter-evans/repository-dispatch@v2.0.0 with: token: ${{ secrets.REPO_ACCESS_TOKEN }} diff --git a/.github/workflows/performance-benchmarks-pull-report.yaml b/.github/workflows/performance-benchmarks-pull-report.yaml index e3f3d7b9e5..69b136fd81 100644 --- a/.github/workflows/performance-benchmarks-pull-report.yaml +++ b/.github/workflows/performance-benchmarks-pull-report.yaml @@ -61,4 +61,15 @@ jobs: with: token: ${{ secrets.REPO_ACCESS_TOKEN }} event-type: benchmark-latency - client-payload: '{"from_server": "dolt", "from_version": "${{ github.sha }}", "to_server": "dolt", "to_version": "${{ steps.comment-branch.outputs.head_sha }}", "mode": "pullRequest", "issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "init_big_repo": "true", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"}' + client-payload: | + { + "from_server": "dolt", + "from_version": "${{ github.sha }}", + "to_server": "dolt", + "to_version": "${{ steps.comment-branch.outputs.head_sha }}", + "mode": "pullRequest", + "issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", + "init_big_repo": "true", + "actor": "${{ github.actor }}", + "template_script": "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh" + }