/.github/{scripts,workflows}: add test regressions workflow, add regressions job config script

2026-04-23 13:48:42 -05:00 · 2024-01-10 11:43:44 -08:00
parent 231d8bcb9a
commit d1c491f707
2 changed files with 149 additions and 0 deletions
@@ -0,0 +1,101 @@
+#!/bin/sh
+
+set -e
+
+if [ "$#" -lt 6 ]; then
+    echo  "Usage: ./get-dolt-correctness-job-json.sh <jobname> <fromVersion> <toVersion> <timeprefix> <actorprefix> <format> <nomsBinFormat>"
+    exit 1
+fi
+
+jobname="$1"
+fromVersion="$2"
+toVersion="$3"
+timeprefix="$4"
+actorprefix="$5"
+format="$6"
+nomsBinFormat="$7"
+
+precision="6"
+
+if [ -n "$nomsBinFormat" ]; then
+  nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
+fi
+
+regressionsCountQuery="select count(*) as from_to_regressions from from_results as f join to_results t on f.test_file = t.test_file and f.line_num = t.line_num and f.result = 'ok' and t.result != 'ok';"
+improvementsCountQuery="select count(*) as from_to_improvements from from_results as f join to_results t on f.test_file = t.test_file and f.line_num = t.line_num and f.result != 'ok' and t.result = 'ok';"
+fromResultsCountQuery="select result as from_version_result, count(*) as from_version_total from from_results where result != 'skipped' group by result;"
+toResultsCountQuery="select result as to_version_result, count(*) as to_version_total from to_results where result != 'skipped' group by result;"
+testCountQuery="select count(*) as total_tests from from_results where result != 'skipped';"
+
+echo '
+{
+  "apiVersion": "batch/v1",
+  "kind": "Job",
+  "metadata": {
+    "name": "'$jobname'",
+    "namespace": "sql-correctness"
+  },
+  "spec": {
+    "backoffLimit": 3,
+    "template": {
+      "metadata": {
+        "annotations": {
+          "alert_recipients": "'$ACTOR_EMAIL'"
+        },
+        "labels": {
+          "k8s-liquidata-inc-monitored-job": "created-by-static-config"
+        }
+      },
+      "spec": {
+        "serviceAccountName": "sql-correctness",
+        "containers": [
+          {
+            "name": "sql-correctness",
+            "image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/sql-correctness:latest",
+            "resources": {
+              "limits": {
+                "cpu": "7000m"
+              }
+            },
+            "env": [
+              { "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'"},
+              { "name": "ACTOR", "value": "'$ACTOR'"},
+              { "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'"},
+              { "name": "DOLT_DEFAULT_BIN_FORMAT", "value": "'$NOMS_BIN_FORMAT'"}
+            ],
+            "args": [
+              "--schema=/regressions.sql",
+              "--concurrent",
+              "--output='$format'",
+              "--to-version='$toVersion'",
+              "--from-version='$fromVersion'",
+              '"$nomsBinFormat"'
+              "--bucket=sql-correctness-github-actions-results",
+              "--region=us-west-2",
+              "--results-dir='$timeprefix'",
+              "--results-prefix='$actorprefix'",
+              "'"$regressionsCountQuery"'",
+              "'"$improvementsCountQuery"'",
+              "'"$fromResultsCountQuery"'",
+              "'"$toResultsCountQuery"'",
+              "'"$testCountQuery"'"
+            ]
+          }
+        ],
+        "restartPolicy": "Never",
+        "nodeSelector": {
+          "sql-correctness-worker": "true"
+        },
+        "tolerations": [
+          {
+              "effect": "NoSchedule",
+              "key": "dedicated",
+              "operator": "Equal",
+              "value": "sql-correctness-worker"
+          }
+        ]
+      }
+    }
+  }
+}
+'
@@ -0,0 +1,48 @@
+name: Benchmark SQL Regressions
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  set-version-actor:
+    name: Set Version and Actor
+    runs-on: ubuntu-22.04
+    outputs:
+      to_version: ${{ steps.set-vars.outputs.to_version }}
+      from_version: ${{ steps.set-vars.outputs.from_version }}
+      actor: ${{ steps.set-vars.outputs.actor }}
+    steps:
+      - name: Set variables
+        id: set-vars
+        run: |
+          echo "from_version=$FROM_VERSION" >> $GITHUB_OUTPUT
+          echo "to_version=$TO_VERSION" >> $GITHUB_OUTPUT
+          echo "actor=$ACTOR" >> $GITHUB_OUTPUT
+        env:
+          FROM_VERSION: ${{ github.head_ref }}
+          TO_VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }}
+          ACTOR: ${{ github.event.client_payload.actor || github.actor }}
+  test:
+    runs-on: ubuntu-22.04
+    needs: set-version-actor
+    name: Test values
+    steps:
+      - name: Echo vals
+        run: |
+          echo "FROM_VERSION: $FROM_VERSION"
+          echo "TO_VERSION: $TO_VERSION"
+        env:
+          FROM_VERSION: ${{ needs.set-version-actor.outputs.from_version }}
+          TO_VERSION: ${{ needs.set-version-actor.outputs.to_version }}
+
+#  regressions:
+#    runs-on: ubuntu-22.04
+#    needs: set-version-actor
+#    name: Trigger SQL Regressions K8s Workflow
+#    steps:
+#      - uses: peter-evans/repository-dispatch@v2.0.0
+#        with:
+#          token: ${{ secrets.REPO_ACCESS_TOKEN }}
+#          event-type: sql-correctness
+#          client-payload: '{"from_version": "${{ needs.set-version-actor.outputs.from_version }}", "to_version": "${{ needs.set-version-actor.outputs.to_version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-regressions-job-json.sh"}'