From 6d7f04e58f0973f7f42f8d4cf7166310100d67d6 Mon Sep 17 00:00:00 2001
From: Dustin Brown <dustin@dolthub.com>
Date: Thu, 21 Jan 2021 18:35:10 -0800
Subject: [PATCH] /.github/scripts/performance-benchmarking: update queries to
 display stable latency median, increase timeout to 90mins

---
 .../performance-benchmarking/get-dolt-dolt-job-json.sh        | 4 ++--
 .../performance-benchmarking/get-mysql-dolt-job-json.sh       | 4 ++--
 .github/scripts/performance-benchmarking/run-benchmarks.sh    | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh
index 42b18c3c60..7fc0a3a9e4 100755
--- a/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh
+++ b/.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh
@@ -16,7 +16,7 @@ timePrefix="$6"
 actorPrefix="$7"
 format="$8"
 
-averageTimeChangeQuery="select f.test_name as test_name, ROUND(100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) as average_time_percent_change, case when (100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) < 0 then true else false end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name group by f.test_name;"
+medianLatencyChangeQuery="select f.test_name as test_name, avg(f.latency_percentile) as from_latency_median, avg(t.latency_percentile) as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -10.0 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 10.0 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name group by f.test_name;"
 
 echo '
 {
@@ -46,7 +46,7 @@ echo '
               "--region=us-west-2",
               "--results-dir='$timePrefix'",
               "--results-prefix='$actorPrefix'",
-              "'"$averageTimeChangeQuery"'"
+              "'"$medianLatencyChangeQuery"'"
             ]
           }
         ],
diff --git a/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh b/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh
index c37400bb72..be38c03b21 100755
--- a/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh
+++ b/.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh
@@ -16,7 +16,7 @@ timeprefix="$6"
 actorprefix="$7"
 format="$8"
 
-average_time_change_query="select f.test_name as test_name, ROUND(100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) as average_time_percent_change, case when (100 * (1.0 - ((AVG(t.latency_sum_ms) / (AVG(cast(t.sql_transactions_total as decimal)) + .000001)) / (AVG(f.latency_sum_ms) / (AVG(cast(f.sql_transactions_total as decimal)) + .000001))))) < 0 then true else false end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name group by f.test_name;"
+medianLatencyMultiplierQuery="select f.test_name as test_name, avg(f.latency_percentile) as from_latency_median, avg(t.latency_percentile) as to_latency_median, ROUND(avg(f.latency_percentile) / (avg(t.latency_percentile) + .000001)) as multiplier from from_results as f join to_results as t on f.test_name = t.test_name group by f.test_name;"
 
 echo '
 {
@@ -48,7 +48,7 @@ echo '
               "--region=us-west-2",
               "--results-dir='$timeprefix'",
               "--results-prefix='$actorprefix'",
-              "'"$average_time_change_query"'"
+              "'"$medianLatencyMultiplierQuery"'"
             ]
           }
         ],
diff --git a/.github/scripts/performance-benchmarking/run-benchmarks.sh b/.github/scripts/performance-benchmarking/run-benchmarks.sh
index 07df3e5a76..b5f7f55402 100755
--- a/.github/scripts/performance-benchmarking/run-benchmarks.sh
+++ b/.github/scripts/performance-benchmarking/run-benchmarks.sh
@@ -45,7 +45,7 @@ source "$TEMPLATE_SCRIPT" "$jobname" "$FROM_SERVER" "$FROM_VERSION" "$TO_SERVER"
 
 KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json
 
-out=$(KUBECONFIG="$KUBECONFIG" kubectl wait job/"$jobname" --for=condition=complete -n performance-benchmarking --timeout=1500s || true)
+out=$(KUBECONFIG="$KUBECONFIG" kubectl wait job/"$jobname" --for=condition=complete -n performance-benchmarking --timeout=5400s || true)
 
 if [ "$out" != "job.batch/$jobname condition met" ]; then
   echo "output of kubectl wait: $out"