From 870852f1a6941434a3cd654de8654b28f61e4b23 Mon Sep 17 00:00:00 2001 From: James Cor Date: Tue, 8 Oct 2024 01:18:45 -0700 Subject: [PATCH] [no-release-notes] add percent_change instead of is_faster for benchmarks (#8419) --- .../get-dolt-dolt-job-json.sh | 131 +++++++++++++++++- 1 file changed, 125 insertions(+), 6 deletions(-) diff --git a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh index 0a3d717fbc..146f517585 100755 --- a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh +++ b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh @@ -33,14 +33,133 @@ if [ -n "$withTpcc" ]; then withTpcc="\"--withTpcc=$withTpcc\"," fi -readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan', 'types_table_scan', 'index_join')" -medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;" +readTests="( +'oltp_read_only', +'oltp_point_select', +'select_random_points', +'select_random_ranges', +'covering_index_scan', +'index_scan', +'table_scan', +'groupby_scan', +'index_join_scan', +'types_table_scan', +'index_join' +)" -writeTests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'oltp_write_only', 'oltp_delete_insert', 'types_delete_insert')" -medianLatencyChangeWritesQuery="select f.test_name as write_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name;" +medianLatencyChangeReadsQuery=" +with result(test_name, from_latency, to_latency) as ( + select + f.test_name, + avg(f.latency_percentile), + avg(t.latency_percentile) + from + from_results as f + join + to_results as t + on + f.test_name = t.test_name + where + f.test_name in $readTests + group by + f.test_name +) +select + test_name as read_tests, + from_latency, + to_latency, + round(100 * ((from_latency - to_latency) / from_latency), 2) as percent_change +from result;" -tpccLatencyQuery="select f.test_name as test_name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_p95, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_p95, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.25 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.25 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE '$tpccRegex' group by f.test_name;" -tpccTpsQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.sql_transactions_per_second) as tps, t.test_name as test_name, t.server_name, t.server_version, avg(t.sql_transactions_per_second) as tps, case when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) < -0.5 then 1 when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) > 0.5 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE 'tpcc%' group by f.test_name;" +writeTests="( +'oltp_read_write', +'oltp_update_index', +'oltp_update_non_index', +'oltp_insert', +'oltp_write_only', +'oltp_delete_insert', +'types_delete_insert' +)" + +medianLatencyChangeWritesQuery=" +with result(test_name, from_latency, to_latency) as ( + select + f.test_name, + avg(f.latency_percentile), + avg(t.latency_percentile) + from + from_results as f + join + to_results as t + on + f.test_name = t.test_name + where + f.test_name in $writeTests + group by + f.test_name +) +select + test_name as write_tests, + from_latency, + to_latency, + round(100 * ((from_latency - to_latency) / from_latency), 2) as percent_change +from result;" + +tpccLatencyQuery=" +with result(test_name, from_latency, to_latency) as ( + select + f.test_name, + avg(f.latency_percentile) + 0.001, + avg(t.latency_percentile) + 0.001 + from + from_results as f + join + to_results as t + on + f.test_name = t.test_name + where + f.test_name LIKE '$tpccRegex' + group by + f.test_name +) +select + test_name, + from_latency as from_latency_p95, + to_latency as to_latency_p95, + round(100 * ((from_latency - to_latency) / from_latency), 2) as percentage_change +from result;" + +tpccTpsQuery=" +with result(test_name, from_server_name, from_server_version, from_tps, to_server_name, to_server_version, to_tps) as ( + select + f.test_name, + f.server_name, + f.server_version, + avg(f.sql_transactions_per_second), + t.server_name, + t.server_version, + avg(t.sql_transactions_per_second) + from + from_results as f + join + to_results as t + on + f.test_name = t.test_name + where + f.test_name LIKE 'tpcc%' + group by + f.test_name +) +select + test_name, + from_server_name, + from_server_version, + from_tps, + to_server_name, + to_server_version, + to_tps, + round(100 * ((from_tps - to_tps) / from_tps), 2) as percentage_change +from result;" echo ' {