mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-08 02:36:27 -05:00
Code Issues Packages Projects Releases 163 Wiki Activity

Merge pull request #7289 from dolthub/db/correctness-regression

Browse Source 
[no-release-notes] /.github/{scripts,workflows}: add test regressions workflow, add regressions job config script
This commit is contained in:
Dustin Brown
2024-01-10 13:17:10 -08:00
committed by GitHub
parent 231d8bcb9a 3fe6ae2e1e
commit 461e25ab3e
5 changed files with 200 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/sql-correctness/get-dolt-correctness-job-json.sh
+2 -1
View File
@@ -3,7 +3,7 @@
set -e
if [ "$#" -lt 6 ]; then
echo "Usage: ./get-dolt-correctness-job-json.sh <jobname> <fromVersion> <toVersion> <timeprefix> <actorprefix> <format> <nomsBinFormat>"
echo "Usage: ./get-dolt-correctness-job-json.sh <jobname> <fromVersion> <toVersion> <timeprefix> <actorprefix> <format> <nomsBinFormat> <issueNumber>"
exit 1
fi
@@ -14,6 +14,7 @@ timeprefix="$4"
actorprefix="$5"
format="$6"
nomsBinFormat="$7"
issueNumber="$8"
precision="6"
.github/scripts/sql-correctness/get-dolt-regressions-job-json.sh
Executable
+102
View File
@@ -0,0 +1,102 @@
#!/bin/sh
set -e
if [ "$#" -lt 6 ]; then
echo "Usage: ./get-dolt-correctness-job-json.sh <jobname> <fromVersion> <toVersion> <timeprefix> <actorprefix> <format> <nomsBinFormat> <issueNumber>"
exit 1
fi
jobname="$1"
fromVersion="$2"
toVersion="$3"
timeprefix="$4"
actorprefix="$5"
format="$6"
nomsBinFormat="$7"
issueNumber="$8"
precision="6"
if [ -n "$nomsBinFormat" ]; then
nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
fi
regressionsCountQuery="select count(*) as from_to_regressions from from_results as f join to_results t on f.test_file = t.test_file and f.line_num = t.line_num and f.result = 'ok' and t.result != 'ok';"
improvementsCountQuery="select count(*) as from_to_improvements from from_results as f join to_results t on f.test_file = t.test_file and f.line_num = t.line_num and f.result != 'ok' and t.result = 'ok';"
fromResultsCountQuery="select result as from_version_result, count(*) as from_version_total from from_results where result != 'skipped' group by result;"
toResultsCountQuery="select result as to_version_result, count(*) as to_version_total from to_results where result != 'skipped' group by result;"
testCountQuery="select count(*) as total_tests from from_results where result != 'skipped';"
echo '
{
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": "'$jobname'",
"namespace": "sql-correctness"
},
"spec": {
"backoffLimit": 3,
"template": {
"metadata": {
"annotations": {
"alert_recipients": "'$ACTOR_EMAIL'"
},
"labels": {
"k8s-liquidata-inc-monitored-job": "created-by-static-config"
}
},
"spec": {
"serviceAccountName": "sql-correctness",
"containers": [
{
"name": "sql-correctness",
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/sql-correctness:latest",
"resources": {
"limits": {
"cpu": "7000m"
}
},
"env": [
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'"},
{ "name": "ACTOR", "value": "'$ACTOR'"},
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'"},
{ "name": "DOLT_DEFAULT_BIN_FORMAT", "value": "'$NOMS_BIN_FORMAT'"}
],
"args": [
"--schema=/regressions.sql",
"--concurrent",
"--output='$format'",
"--to-version='$toVersion'",
"--from-version='$fromVersion'",
'"$nomsBinFormat"'
"--bucket=sql-correctness-github-actions-results",
"--region=us-west-2",
"--results-dir='$timeprefix'",
"--results-prefix='$actorprefix'",
"'"$regressionsCountQuery"'",
"'"$improvementsCountQuery"'",
"'"$fromResultsCountQuery"'",
"'"$toResultsCountQuery"'",
"'"$testCountQuery"'"
]
}
],
"restartPolicy": "Never",
"nodeSelector": {
"sql-correctness-worker": "true"
},
"tolerations": [
{
"effect": "NoSchedule",
"key": "dedicated",
"operator": "Equal",
"value": "sql-correctness-worker"
}
]
}
}
}
}
'
.github/scripts/sql-correctness/run-correctness.sh
+29 -2
View File
@@ -28,6 +28,17 @@ if [ ! -z "$FROM_VERSION" ] && [ -z "$TO_VERSION" ]; then
TO_VERSION="$FROM_VERSION"
fi
is_regressions=""
if [ -n "$FROM_VERSION" ] && [ -n "$TO_VERSION" ]; then
if [ -z "$PR_NUMBER" ]; then
echo "Must set PR_NUMBER for regressions run"
exit 1
fi
echo "Starting regressions run from: $FROM_VERSION to: $TO_VERSION"
is_regressions="true"
fi
if [ -z "$ACTOR" ]; then
echo "Must set ACTOR"
exit 1
@@ -49,7 +60,12 @@ sleep 0.$[ ( $RANDOM % 10 ) + 1 ]s
timesuffix=`date +%s%N`
jobname="$actorShort-$timesuffix"
jobname=""
if [ -z "$is_regressions" ]; then
jobname="$lowered-$PR_NUMBER"
else
jobname="$actorShort-$timesuffix"
fi
timeprefix=$(date +%Y/%m/%d)
@@ -60,6 +76,10 @@ if [[ "$MODE" = "release" || "$MODE" = "nightly" ]]; then
format="html"
fi
# set value to PR_NUMBER environment variable
# or default to -1
issuenumber=${PR_NUMBER:-"-1"}
source \
"$TEMPLATE_SCRIPT" \
"$jobname" \
@@ -68,7 +88,14 @@ source \
"$timeprefix" \
"$actorprefix" \
"$format" \
"$NOMS_BIN_FORMAT" > job.json
"$NOMS_BIN_FORMAT" \
"$issuenumber" > job.json
# delete existing job with same name if this is a regressions job
if [ -z "$is_regressions" ]; then
out=$(KUBECONFIG="$KUBECONFIG" kubectl delete job/"$jobname" -n sql-correctness || true)
echo "Delete regressions job if exists: $out"
fi
out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
.github/workflows/k8s-sql-correctness.yaml
+2
View File
@@ -39,6 +39,8 @@ jobs:
- name: Create SQL Correctness K8s Job
run: ./.github/scripts/sql-correctness/run-correctness.sh
env:
PR_NUMBER: $${{ github.event.client_payload.issue_number }}
FROM_VERSION: ${{ github.event.client_payload.from_version }}
TO_VERSION: ${{ github.event.client_payload.to_version }}
MODE: ${{ github.event.client_payload.mode }}
ACTOR: ${{ github.event.client_payload.actor }}
.github/workflows/sql-regressions.yaml
+65
View File
@@ -0,0 +1,65 @@
name: Benchmark SQL Regressions
on:
pull_request:
branches: [ main ]
jobs:
set-version-actor:
name: Set Version and Actor
runs-on: ubuntu-22.04
outputs:
to_version: ${{ steps.set-vars.outputs.to_version }}
from_version: ${{ steps.set-from.outputs.from_version }}
actor: ${{ steps.set-vars.outputs.actor }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.base_ref }}
- name: Set From Version
id: set-from
run: |
sha=$(git rev-parse --short HEAD)
echo "from_version=$sha" >> $GITHUB_OUTPUT
- name: Set variables
id: set-vars
run: |
short_to=${TO_VERSION:0:8}
echo "to_version=$short_to" >> $GITHUB_OUTPUT
echo "actor=$ACTOR" >> $GITHUB_OUTPUT
env:
TO_VERSION: ${{ github.sha }}
ACTOR: ${{ github.event.client_payload.actor || github.actor }}
test:
runs-on: ubuntu-22.04
needs: set-version-actor
name: Test values
steps:
- name: Echo vals
run: |
echo "FROM_VERSION: $FROM_VERSION"
echo "TO_VERSION: $TO_VERSION"
echo "ACTOR: $ACTOR"
env:
FROM_VERSION: ${{ needs.set-version-actor.outputs.from_version }}
TO_VERSION: ${{ needs.set-version-actor.outputs.to_version }}
ACTOR: ${{ needs.set-version-actor.outputs.actor }}
regressions:
runs-on: ubuntu-22.04
needs: set-version-actor
name: Trigger SQL Regressions K8s Workflow
steps:
- name: Get pull number
uses: actions/github-script@v6
id: get_pull_number
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: core.setOutput("pull_number", JSON.stringify(context.issue.number));
- uses: peter-evans/repository-dispatch@v2.0.0
if: ${{ needs.set-version-actor.outputs.actor == 'coffeegoddd' }}
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: sql-correctness
client-payload: '{"issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "from_version": "${{ needs.set-version-actor.outputs.from_version }}", "to_version": "${{ needs.set-version-actor.outputs.to_version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-dolt-regressions-job-json.sh"}'