mirror of
https://github.com/dolthub/dolt.git
synced 2026-05-06 11:20:30 -05:00
Revamp the Import Benchmarker (#3744)
This commit is contained in:
Vinai Rachakonda
@@ -0,0 +1,80 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
if [ "$#" -lt 8 ]; then
|
||||
echo "Usage: ./get-dolt-dolt-job-json.sh <jobName> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <issueNumber>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jobName="$1"
|
||||
fromServer="$2"
|
||||
fromVersion="$3"
|
||||
toServer="$4"
|
||||
toVersion="$5"
|
||||
timePrefix="$6"
|
||||
actorPrefix="$7"
|
||||
issueNumber="$8" # TODO: Use this to paste the results onto the github issue
|
||||
|
||||
echo '
|
||||
{
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"metadata": {
|
||||
"name": "'$jobName'",
|
||||
"namespace": "performance-benchmarking"
|
||||
},
|
||||
"spec": {
|
||||
"backoffLimit": 1,
|
||||
"template": {
|
||||
"spec": {
|
||||
"serviceAccountName": "import-benchmarking",
|
||||
"containers": [
|
||||
{
|
||||
"name": "import-benchmarking",
|
||||
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/import-benchmarking:latest",
|
||||
"resources": {
|
||||
"limits": {
|
||||
"cpu": "7000m"
|
||||
}
|
||||
},
|
||||
"env": [
|
||||
{ "name": "GOMAXPROCS", "value": "7" },
|
||||
{ "name": "ACTOR", "value": "'$ACTOR'" },
|
||||
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
|
||||
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
|
||||
],
|
||||
"imagePullPolicy": "Always",
|
||||
"args": [
|
||||
"--from-server='$fromServer'",
|
||||
"--from-version='$fromVersion'",
|
||||
"--to-server='$toServer'",
|
||||
"--to-version='$toVersion'",
|
||||
"--bucket=import-benchmarking-github-actions-results",
|
||||
"--region=us-west-2",
|
||||
"--results-dir='$timePrefix'",
|
||||
"--results-prefix='$actorPrefix'",
|
||||
"--fileNames=100k-sorted.csv",
|
||||
"--fileNames=100k-random.csv",
|
||||
"--fileNames=1m-sorted.csv",
|
||||
"--fileNames=1m-random.csv"
|
||||
]
|
||||
}
|
||||
],
|
||||
"restartPolicy": "Never",
|
||||
"nodeSelector": {
|
||||
"performance-benchmarking-worker": "true"
|
||||
},
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoSchedule",
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "performance-benchmarking-worker"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
if [ "$#" -lt 8 ]; then
|
||||
echo "Usage: ./get-mysql-dolt-job-json.sh <jobName> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <issueNumber>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jobName="$1"
|
||||
fromServer="$2"
|
||||
fromVersion="$3"
|
||||
toServer="$4"
|
||||
toVersion="$5"
|
||||
timePrefix="$6"
|
||||
actorPrefix="$7"
|
||||
issueNumber="$8" # TODO: Use this to paste the results onto the github issue
|
||||
|
||||
echo '
|
||||
{
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"metadata": {
|
||||
"name": "'$jobName'",
|
||||
"namespace": "performance-benchmarking"
|
||||
},
|
||||
"spec": {
|
||||
"backoffLimit": 1,
|
||||
"template": {
|
||||
"spec": {
|
||||
"serviceAccountName": "import-benchmarking",
|
||||
"containers": [
|
||||
{
|
||||
"name": "import-benchmarking",
|
||||
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/import-benchmarking:latest",
|
||||
"resources": {
|
||||
"limits": {
|
||||
"cpu": "7000m"
|
||||
}
|
||||
},
|
||||
"env": [
|
||||
{ "name": "GOMAXPROCS", "value": "7" },
|
||||
{ "name": "ACTOR", "value": "'$ACTOR'" },
|
||||
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
|
||||
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
|
||||
],
|
||||
"imagePullPolicy": "Always",
|
||||
"args": [
|
||||
"--from-server='$fromServer'",
|
||||
"--from-version='$fromVersion'",
|
||||
"--to-server='$toServer'",
|
||||
"--to-version='$toVersion'",
|
||||
"--bucket=import-benchmarking-github-actions-results",
|
||||
"--region=us-west-2",
|
||||
"--results-dir='$timePrefix'",
|
||||
"--results-prefix='$actorPrefix'",
|
||||
"--mysql-exec=/usr/sbin/mysqld",
|
||||
"--mysql-schema-file=schema.sql",
|
||||
"--fileNames=100k-sorted.csv",
|
||||
"--fileNames=100k-random.csv",
|
||||
"--fileNames=1m-sorted.csv",
|
||||
"--fileNames=1m-random.csv"
|
||||
]
|
||||
}
|
||||
],
|
||||
"restartPolicy": "Never",
|
||||
"nodeSelector": {
|
||||
"performance-benchmarking-worker": "true"
|
||||
},
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoSchedule",
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "performance-benchmarking-worker"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [ -z "$KUBECONFIG" ]; then
|
||||
echo "Must set KUBECONFIG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$TEMPLATE_SCRIPT" ]; then
|
||||
echo "Must set TEMPLATE_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then
|
||||
echo "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$ACTOR" ]; then
|
||||
echo "Must set ACTOR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$MODE" ]; then
|
||||
echo "Must set MODE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting from $FROM_SERVER: $FROM_VERSION"
|
||||
|
||||
# use first 8 characters of TO_VERSION to differentiate
|
||||
# jobs
|
||||
short=${TO_VERSION:0:8}
|
||||
lowered=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]')
|
||||
actorShort="$lowered-$short"
|
||||
|
||||
# random sleep
|
||||
sleep 0.$[ ( $RANDOM % 10 ) + 1 ]s
|
||||
|
||||
timesuffix=`date +%s%N`
|
||||
jobname="$actorShort-$timesuffix"
|
||||
|
||||
timeprefix=$(date +%Y/%m/%d)
|
||||
|
||||
actorprefix="$MODE/$ACTOR/$actorShort"
|
||||
|
||||
# set value to ISSUE_NUMBER environment variable
|
||||
# or default to -1
|
||||
issuenumber=${ISSUE_NUMBER:-"-1"}
|
||||
|
||||
source \
|
||||
"$TEMPLATE_SCRIPT" \
|
||||
"$jobname" \
|
||||
"$FROM_VERSION" \
|
||||
"$TO_VERSION" \
|
||||
"$timeprefix" \
|
||||
"$actorprefix" \
|
||||
"$issuenumber" > job.json
|
||||
|
||||
out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
|
||||
|
||||
if [ "$out" != "job.batch/$jobname created" ]; then
|
||||
echo "something went wrong creating job... this job likely already exists in the cluster"
|
||||
echo "$out"
|
||||
exit 1
|
||||
else
|
||||
echo "$out"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user