From 2240cd2da770feedb72d8bb378f6e77e0d00bc7f Mon Sep 17 00:00:00 2001 From: Maximilian Hoffman Date: Fri, 11 Nov 2022 08:14:44 -0800 Subject: [PATCH] delete old files (#4757) --- .../import-benchmarking/csv_to_html.sh | 26 ----- .../get-dolt-dolt-job-json.sh | 98 ------------------ .../get-mysql-dolt-job-json.sh | 99 ------------------- .../import-benchmarking/run-benchmarks.sh | 83 ---------------- .../import_benchmarker/cmd/sample-config.json | 27 ----- 5 files changed, 333 deletions(-) delete mode 100755 .github/scripts/import-benchmarking/csv_to_html.sh delete mode 100755 .github/scripts/import-benchmarking/get-dolt-dolt-job-json.sh delete mode 100755 .github/scripts/import-benchmarking/get-mysql-dolt-job-json.sh delete mode 100755 .github/scripts/import-benchmarking/run-benchmarks.sh delete mode 100644 go/performance/import_benchmarker/cmd/sample-config.json diff --git a/.github/scripts/import-benchmarking/csv_to_html.sh b/.github/scripts/import-benchmarking/csv_to_html.sh deleted file mode 100755 index ca4f1666b8..0000000000 --- a/.github/scripts/import-benchmarking/csv_to_html.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -if [ "$#" -ne 2 ]; then - echo "Usage: csv_to_html.sh " -fi - -gw="$GITHUB_WORKSPACE" -in="$1" -out="$2" - -echo "" > "$out" -print_header=true -while read line; do - if "$print_header"; then - #echo " " >> "$out" - print_header=false - continue - fi - echo " " >> "$out" -done < "$in" -echo "
$line" | sed -e 's/:[^,]*\(,\|$\)/<\/th>/g' >> "$out" - echo "
${line//,/}
${line//,/}
" >> "$out" - -cat "$out" - -echo "::set-output name=html::$(echo $out)" diff --git a/.github/scripts/import-benchmarking/get-dolt-dolt-job-json.sh b/.github/scripts/import-benchmarking/get-dolt-dolt-job-json.sh deleted file mode 100755 index a32043a052..0000000000 --- a/.github/scripts/import-benchmarking/get-dolt-dolt-job-json.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$#" -lt 9 ]; then - echo "Usage: ./get-dolt-dolt-job-json.sh " - exit 1 -fi - -jobName="$1" -fromServer="$2" -fromVersion="$3" -toServer="$4" -toVersion="$5" -timePrefix="$6" -actorPrefix="$7" -nomsBinFormat="$8" -format="$9" -issueNumber="${10}" - -echo ' -{ - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "name": "'$jobName'", - "namespace": "performance-benchmarking" - }, - "spec": { - "backoffLimit": 1, - "template": { - "metadata": { - "annotations": { - "alert_recipients": "'$ACTOR_EMAIL'" - }, - "labels": { - "k8s-liquidata-inc-monitored-job": "created-by-static-config" - } - }, - "spec": { - "serviceAccountName": "import-benchmarking", - "containers": [ - { - "name": "import-benchmarking", - "image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/import-benchmarking:latest", - "resources": { - "limits": { - "cpu": "7000m" - } - }, - "env": [ - { "name": "GOMAXPROCS", "value": "7" }, - { "name": "ACTOR", "value": "'$ACTOR'" }, - { "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" }, - { "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" } - ], - "imagePullPolicy": "Always", - "args": [ - "--from-server='$fromServer'", - "--from-version='$fromVersion'", - "--to-server='$toServer'", - "--to-version='$toVersion'", - "--bucket=import-benchmarking-github-actions-results", - "--region=us-west-2", - "--results-dir='$timePrefix'", - "--results-prefix='$actorPrefix'", - "--mysql-schema-file=schema.sql", - "--nbf='$nomsBinFormat'", - "--email-template=ImportBenchmarkingReleaseTemplate", - "--results-schema=/results-schema.sql", - "--issue-number='$issueNumber'", - "--output='$format'", - "--fileNames=100k-sorted.csv", - "--fileNames=100k-random.csv", - "--fileNames=1m-sorted.csv", - "--fileNames=1m-random.csv", - "--fileNames=10m-sorted.csv", - "--fileNames=10m-random.csv" - ] - } - ], - "restartPolicy": "Never", - "nodeSelector": { - "performance-benchmarking-worker": "true" - }, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "dedicated", - "operator": "Equal", - "value": "performance-benchmarking-worker" - } - ] - } - } - } -} -' diff --git a/.github/scripts/import-benchmarking/get-mysql-dolt-job-json.sh b/.github/scripts/import-benchmarking/get-mysql-dolt-job-json.sh deleted file mode 100755 index c43c0b854c..0000000000 --- a/.github/scripts/import-benchmarking/get-mysql-dolt-job-json.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/sh - -set -e - -if [ "$#" -lt 9 ]; then - echo "Usage: ./get-mysql-dolt-job-json.sh " - exit 1 -fi - -jobName="$1" -fromServer="$2" -fromVersion="$3" -toServer="$4" # make this mysql -toVersion="$5" -timePrefix="$6" -actorPrefix="$7" -nomsBinFormat="$8" -format="$9" -issueNumber="${10}" - -echo ' -{ - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "name": "'$jobName'", - "namespace": "performance-benchmarking" - }, - "spec": { - "backoffLimit": 1, - "template": { - "metadata": { - "annotations": { - "alert_recipients": "'$ACTOR_EMAIL'" - }, - "labels": { - "k8s-liquidata-inc-monitored-job": "created-by-static-config" - } - }, - "spec": { - "serviceAccountName": "import-benchmarking", - "containers": [ - { - "name": "import-benchmarking", - "image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/import-benchmarking:latest", - "resources": { - "limits": { - "cpu": "7000m" - } - }, - "env": [ - { "name": "GOMAXPROCS", "value": "7" }, - { "name": "ACTOR", "value": "'$ACTOR'" }, - { "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" }, - { "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" } - ], - "imagePullPolicy": "Always", - "args": [ - "--from-server='$fromServer'", - "--from-version='$fromVersion'", - "--to-server='$toServer'", - "--to-version='$toVersion'", - "--bucket=import-benchmarking-github-actions-results", - "--region=us-west-2", - "--results-dir='$timePrefix'", - "--results-prefix='$actorPrefix'", - "--mysql-exec=/usr/sbin/mysqld", - "--mysql-schema-file=schema.sql", - "--nbf='$nomsBinFormat'", - "--email-template=ImportBenchmarkingReleaseTemplate", - "--results-schema=/results-schema.sql", - "--issue-number='$issueNumber'", - "--output='$format'", - "--fileNames=100k-sorted.csv", - "--fileNames=100k-random.csv", - "--fileNames=1m-sorted.csv", - "--fileNames=1m-random.csv", - "--fileNames=10m-sorted.csv", - "--fileNames=10m-random.csv" - ] - } - ], - "restartPolicy": "Never", - "nodeSelector": { - "performance-benchmarking-worker": "true" - }, - "tolerations": [ - { - "effect": "NoSchedule", - "key": "dedicated", - "operator": "Equal", - "value": "performance-benchmarking-worker" - } - ] - } - } - } -} -' diff --git a/.github/scripts/import-benchmarking/run-benchmarks.sh b/.github/scripts/import-benchmarking/run-benchmarks.sh deleted file mode 100755 index 34521bb52d..0000000000 --- a/.github/scripts/import-benchmarking/run-benchmarks.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -set -e - -if [ -z "$KUBECONFIG" ]; then - echo "Must set KUBECONFIG" - exit 1 -fi - -if [ -z "$TEMPLATE_SCRIPT" ]; then - echo "Must set TEMPLATE_SCRIPT" - exit 1 -fi - -if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then - echo "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION" - exit 1 -fi - -if [ -z "$ACTOR" ]; then - echo "Must set ACTOR" - exit 1 -fi - -if [ -z "$MODE" ]; then - echo "Must set MODE" - exit 1 -fi - -nomsFormat="ldnbf" -if [ "$NOMS_BIN_FORMAT" == "__DOLT__" ]; then - nomsFormat="doltnbf" -fi - -# use first 8 characters of TO_VERSION to differentiate -# jobs -short=${TO_VERSION:0:8} -lowered=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]') -actorShort="$lowered-$nomsFormat-$short" - -# random sleep -sleep 0.$[ ( $RANDOM % 10 ) + 1 ]s - -timesuffix=`date +%s` -jobname="$actorShort-$timesuffix" - -timeprefix=$(date +%Y/%m/%d) - -actorprefix="$MODE/$ACTOR/$actorShort" - -format="markdown" -if [[ "$MODE" = "release" || "$MODE" = "nightly" ]]; then - format="html" -fi - -# set value to ISSUE_NUMBER environment variable -# or default to -1 -issuenumber=${ISSUE_NUMBER:-"-1"} - -source \ - "$TEMPLATE_SCRIPT" \ - "$jobname" \ - "$FROM_SERVER" \ - "$FROM_VERSION" \ - "$TO_SERVER" \ - "$TO_VERSION" \ - "$timeprefix" \ - "$actorprefix" \ - "$NOMS_BIN_FORMAT" \ - "$format" \ - "$issuenumber" > job.json - -out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true) - -if [ "$out" != "job.batch/$jobname created" ]; then - echo "something went wrong creating job... this job likely already exists in the cluster" - echo "$out" - exit 1 -else - echo "$out" -fi - -exit 0 diff --git a/go/performance/import_benchmarker/cmd/sample-config.json b/go/performance/import_benchmarker/cmd/sample-config.json deleted file mode 100644 index c15d6e96b1..0000000000 --- a/go/performance/import_benchmarker/cmd/sample-config.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "Jobs": [ - { - "Name": "Medium MySQL import with Sorted Rows", - "NumRows": 1000000, - "Sorted": true, - "Format": "csv", - "FilePath": "/Users/vinairachakonda/misc/import-benchmark-files/100k-sorted.csv", - "Program": "mysql", - "Version": "8.0.22", - "ExecPath": "/usr/local/bin/mysqld", - "SchemaPath": "/Users/vinairachakonda/go/src/dolthub/dolt/go/performance/import_benchmarker/cmd/schema.sql" - }, - { - "Name": "Medium Dolt import with Sorted Rows", - "NumRows": 1000000, - "Sorted": true, - "Format": "csv", - "FilePath": "/Users/vinairachakonda/misc/import-benchmark-files/100k-sorted.csv", - "Program": "dolt", - "Version": "HEAD", - "ExecPath": "/Users/vinairachakonda/go/bin/dolt", - "SchemaPath": "/Users/vinairachakonda/go/src/dolthub/dolt/go/performance/import_benchmarker/cmd/schema.sql" - } - ], - "MysqlConnectionProtocol": "tcp" -}