mirror of
https://github.com/dolthub/dolt.git
synced 2026-01-07 00:39:44 -06:00
delete old files (#4757)
This commit is contained in:
Maximilian Hoffman
committed by
GitHub
GitHub
parent
061e44e89e
commit
2240cd2da7
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "Usage: csv_to_html.sh <csv_file> <html_file>"
|
||||
fi
|
||||
|
||||
gw="$GITHUB_WORKSPACE"
|
||||
in="$1"
|
||||
out="$2"
|
||||
|
||||
echo "<table>" > "$out"
|
||||
print_header=true
|
||||
while read line; do
|
||||
if "$print_header"; then
|
||||
#echo " <tr><th>$line" | sed -e 's/:[^,]*\(,\|$\)/<\/th><th>/g' >> "$out"
|
||||
echo " <tr><th>${line//,/</th><th>}</th></tr>" >> "$out"
|
||||
print_header=false
|
||||
continue
|
||||
fi
|
||||
echo " <tr><td>${line//,/</td><td>}</td></tr>" >> "$out"
|
||||
done < "$in"
|
||||
echo "</table>" >> "$out"
|
||||
|
||||
cat "$out"
|
||||
|
||||
echo "::set-output name=html::$(echo $out)"
|
||||
@@ -1,98 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
if [ "$#" -lt 9 ]; then
|
||||
echo "Usage: ./get-dolt-dolt-job-json.sh <jobName> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <nomsBinFormat> <format> <issueNumber>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jobName="$1"
|
||||
fromServer="$2"
|
||||
fromVersion="$3"
|
||||
toServer="$4"
|
||||
toVersion="$5"
|
||||
timePrefix="$6"
|
||||
actorPrefix="$7"
|
||||
nomsBinFormat="$8"
|
||||
format="$9"
|
||||
issueNumber="${10}"
|
||||
|
||||
echo '
|
||||
{
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"metadata": {
|
||||
"name": "'$jobName'",
|
||||
"namespace": "performance-benchmarking"
|
||||
},
|
||||
"spec": {
|
||||
"backoffLimit": 1,
|
||||
"template": {
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"alert_recipients": "'$ACTOR_EMAIL'"
|
||||
},
|
||||
"labels": {
|
||||
"k8s-liquidata-inc-monitored-job": "created-by-static-config"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"serviceAccountName": "import-benchmarking",
|
||||
"containers": [
|
||||
{
|
||||
"name": "import-benchmarking",
|
||||
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/import-benchmarking:latest",
|
||||
"resources": {
|
||||
"limits": {
|
||||
"cpu": "7000m"
|
||||
}
|
||||
},
|
||||
"env": [
|
||||
{ "name": "GOMAXPROCS", "value": "7" },
|
||||
{ "name": "ACTOR", "value": "'$ACTOR'" },
|
||||
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
|
||||
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
|
||||
],
|
||||
"imagePullPolicy": "Always",
|
||||
"args": [
|
||||
"--from-server='$fromServer'",
|
||||
"--from-version='$fromVersion'",
|
||||
"--to-server='$toServer'",
|
||||
"--to-version='$toVersion'",
|
||||
"--bucket=import-benchmarking-github-actions-results",
|
||||
"--region=us-west-2",
|
||||
"--results-dir='$timePrefix'",
|
||||
"--results-prefix='$actorPrefix'",
|
||||
"--mysql-schema-file=schema.sql",
|
||||
"--nbf='$nomsBinFormat'",
|
||||
"--email-template=ImportBenchmarkingReleaseTemplate",
|
||||
"--results-schema=/results-schema.sql",
|
||||
"--issue-number='$issueNumber'",
|
||||
"--output='$format'",
|
||||
"--fileNames=100k-sorted.csv",
|
||||
"--fileNames=100k-random.csv",
|
||||
"--fileNames=1m-sorted.csv",
|
||||
"--fileNames=1m-random.csv",
|
||||
"--fileNames=10m-sorted.csv",
|
||||
"--fileNames=10m-random.csv"
|
||||
]
|
||||
}
|
||||
],
|
||||
"restartPolicy": "Never",
|
||||
"nodeSelector": {
|
||||
"performance-benchmarking-worker": "true"
|
||||
},
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoSchedule",
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "performance-benchmarking-worker"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
@@ -1,99 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
if [ "$#" -lt 9 ]; then
|
||||
echo "Usage: ./get-mysql-dolt-job-json.sh <jobName> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <nomsBinFormat> <format> <issueNumber>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jobName="$1"
|
||||
fromServer="$2"
|
||||
fromVersion="$3"
|
||||
toServer="$4" # make this mysql
|
||||
toVersion="$5"
|
||||
timePrefix="$6"
|
||||
actorPrefix="$7"
|
||||
nomsBinFormat="$8"
|
||||
format="$9"
|
||||
issueNumber="${10}"
|
||||
|
||||
echo '
|
||||
{
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"metadata": {
|
||||
"name": "'$jobName'",
|
||||
"namespace": "performance-benchmarking"
|
||||
},
|
||||
"spec": {
|
||||
"backoffLimit": 1,
|
||||
"template": {
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"alert_recipients": "'$ACTOR_EMAIL'"
|
||||
},
|
||||
"labels": {
|
||||
"k8s-liquidata-inc-monitored-job": "created-by-static-config"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"serviceAccountName": "import-benchmarking",
|
||||
"containers": [
|
||||
{
|
||||
"name": "import-benchmarking",
|
||||
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/import-benchmarking:latest",
|
||||
"resources": {
|
||||
"limits": {
|
||||
"cpu": "7000m"
|
||||
}
|
||||
},
|
||||
"env": [
|
||||
{ "name": "GOMAXPROCS", "value": "7" },
|
||||
{ "name": "ACTOR", "value": "'$ACTOR'" },
|
||||
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
|
||||
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
|
||||
],
|
||||
"imagePullPolicy": "Always",
|
||||
"args": [
|
||||
"--from-server='$fromServer'",
|
||||
"--from-version='$fromVersion'",
|
||||
"--to-server='$toServer'",
|
||||
"--to-version='$toVersion'",
|
||||
"--bucket=import-benchmarking-github-actions-results",
|
||||
"--region=us-west-2",
|
||||
"--results-dir='$timePrefix'",
|
||||
"--results-prefix='$actorPrefix'",
|
||||
"--mysql-exec=/usr/sbin/mysqld",
|
||||
"--mysql-schema-file=schema.sql",
|
||||
"--nbf='$nomsBinFormat'",
|
||||
"--email-template=ImportBenchmarkingReleaseTemplate",
|
||||
"--results-schema=/results-schema.sql",
|
||||
"--issue-number='$issueNumber'",
|
||||
"--output='$format'",
|
||||
"--fileNames=100k-sorted.csv",
|
||||
"--fileNames=100k-random.csv",
|
||||
"--fileNames=1m-sorted.csv",
|
||||
"--fileNames=1m-random.csv",
|
||||
"--fileNames=10m-sorted.csv",
|
||||
"--fileNames=10m-random.csv"
|
||||
]
|
||||
}
|
||||
],
|
||||
"restartPolicy": "Never",
|
||||
"nodeSelector": {
|
||||
"performance-benchmarking-worker": "true"
|
||||
},
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoSchedule",
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "performance-benchmarking-worker"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [ -z "$KUBECONFIG" ]; then
|
||||
echo "Must set KUBECONFIG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$TEMPLATE_SCRIPT" ]; then
|
||||
echo "Must set TEMPLATE_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then
|
||||
echo "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$ACTOR" ]; then
|
||||
echo "Must set ACTOR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$MODE" ]; then
|
||||
echo "Must set MODE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
nomsFormat="ldnbf"
|
||||
if [ "$NOMS_BIN_FORMAT" == "__DOLT__" ]; then
|
||||
nomsFormat="doltnbf"
|
||||
fi
|
||||
|
||||
# use first 8 characters of TO_VERSION to differentiate
|
||||
# jobs
|
||||
short=${TO_VERSION:0:8}
|
||||
lowered=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]')
|
||||
actorShort="$lowered-$nomsFormat-$short"
|
||||
|
||||
# random sleep
|
||||
sleep 0.$[ ( $RANDOM % 10 ) + 1 ]s
|
||||
|
||||
timesuffix=`date +%s`
|
||||
jobname="$actorShort-$timesuffix"
|
||||
|
||||
timeprefix=$(date +%Y/%m/%d)
|
||||
|
||||
actorprefix="$MODE/$ACTOR/$actorShort"
|
||||
|
||||
format="markdown"
|
||||
if [[ "$MODE" = "release" || "$MODE" = "nightly" ]]; then
|
||||
format="html"
|
||||
fi
|
||||
|
||||
# set value to ISSUE_NUMBER environment variable
|
||||
# or default to -1
|
||||
issuenumber=${ISSUE_NUMBER:-"-1"}
|
||||
|
||||
source \
|
||||
"$TEMPLATE_SCRIPT" \
|
||||
"$jobname" \
|
||||
"$FROM_SERVER" \
|
||||
"$FROM_VERSION" \
|
||||
"$TO_SERVER" \
|
||||
"$TO_VERSION" \
|
||||
"$timeprefix" \
|
||||
"$actorprefix" \
|
||||
"$NOMS_BIN_FORMAT" \
|
||||
"$format" \
|
||||
"$issuenumber" > job.json
|
||||
|
||||
out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
|
||||
|
||||
if [ "$out" != "job.batch/$jobname created" ]; then
|
||||
echo "something went wrong creating job... this job likely already exists in the cluster"
|
||||
echo "$out"
|
||||
exit 1
|
||||
else
|
||||
echo "$out"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"Jobs": [
|
||||
{
|
||||
"Name": "Medium MySQL import with Sorted Rows",
|
||||
"NumRows": 1000000,
|
||||
"Sorted": true,
|
||||
"Format": "csv",
|
||||
"FilePath": "/Users/vinairachakonda/misc/import-benchmark-files/100k-sorted.csv",
|
||||
"Program": "mysql",
|
||||
"Version": "8.0.22",
|
||||
"ExecPath": "/usr/local/bin/mysqld",
|
||||
"SchemaPath": "/Users/vinairachakonda/go/src/dolthub/dolt/go/performance/import_benchmarker/cmd/schema.sql"
|
||||
},
|
||||
{
|
||||
"Name": "Medium Dolt import with Sorted Rows",
|
||||
"NumRows": 1000000,
|
||||
"Sorted": true,
|
||||
"Format": "csv",
|
||||
"FilePath": "/Users/vinairachakonda/misc/import-benchmark-files/100k-sorted.csv",
|
||||
"Program": "dolt",
|
||||
"Version": "HEAD",
|
||||
"ExecPath": "/Users/vinairachakonda/go/bin/dolt",
|
||||
"SchemaPath": "/Users/vinairachakonda/go/src/dolthub/dolt/go/performance/import_benchmarker/cmd/schema.sql"
|
||||
}
|
||||
],
|
||||
"MysqlConnectionProtocol": "tcp"
|
||||
}
|
||||
Reference in New Issue
Block a user