mirror of
https://github.com/dolthub/dolt.git
synced 2026-01-06 00:39:40 -06:00
/{go,.github}: add scripts for building pgo dolt binaries
This commit is contained in:
coffeegoddd
93
.github/scripts/performance-benchmarking/get-dolt-profile-job-json.sh
vendored
Executable file
93
.github/scripts/performance-benchmarking/get-dolt-profile-job-json.sh
vendored
Executable file
@@ -0,0 +1,93 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
if [ "$#" -lt 5 ]; then
|
||||
echo "Usage: ./get-job-json.sh <jobname> <version> <timePrefix> <actorPrefix> <format> <initBigRepo> <nomsBinFormat>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
jobName="$1"
|
||||
version="$2"
|
||||
timePrefix="$3"
|
||||
actorPrefix="$4"
|
||||
format="$5"
|
||||
initBigRepo="$6"
|
||||
nomsBinFormat="$7"
|
||||
|
||||
if [ -n "$initBigRepo" ]; then
|
||||
initBigRepo="\"--init-big-repo=$initBigRepo\","
|
||||
fi
|
||||
|
||||
if [ -n "$nomsBinFormat" ]; then
|
||||
nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
|
||||
fi
|
||||
|
||||
echo '
|
||||
{
|
||||
"apiVersion": "batch/v1",
|
||||
"kind": "Job",
|
||||
"metadata": {
|
||||
"name": "'$jobName'",
|
||||
"namespace": "performance-benchmarking"
|
||||
},
|
||||
"spec": {
|
||||
"backoffLimit": 1,
|
||||
"template": {
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"alert_recipients": "'$ACTOR_EMAIL'"
|
||||
},
|
||||
"labels": {
|
||||
"k8s-liquidata-inc-monitored-job": "created-by-static-config"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"serviceAccountName": "performance-benchmarking",
|
||||
"containers": [
|
||||
{
|
||||
"name": "performance-benchmarking",
|
||||
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/performance-benchmarking:latest",
|
||||
"resources": {
|
||||
"limits": {
|
||||
"cpu": "7000m"
|
||||
}
|
||||
},
|
||||
"env": [
|
||||
{ "name": "GOMAXPROCS", "value": "7" },
|
||||
{ "name": "ACTOR", "value": "'$ACTOR'" },
|
||||
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
|
||||
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
|
||||
],
|
||||
"imagePullPolicy": "Always",
|
||||
"args": [
|
||||
"--schema=/schema.sql",
|
||||
"--useDoltHubLuaScriptsRepo",
|
||||
"--profile-version='$version'",
|
||||
"--bucket=performance-benchmarking-github-actions-results",
|
||||
"--region=us-west-2",
|
||||
"--results-dir='$timePrefix'",
|
||||
"--results-prefix='$actorPrefix'",
|
||||
'"$initBigRepo"'
|
||||
'"$nomsBinFormat"'
|
||||
"--output='$format'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"restartPolicy": "Never",
|
||||
"nodeSelector": {
|
||||
"performance-benchmarking-worker": "true"
|
||||
},
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoSchedule",
|
||||
"key": "dedicated",
|
||||
"operator": "Equal",
|
||||
"value": "performance-benchmarking-worker"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'
|
||||
@@ -17,9 +17,21 @@ if [ -z "$NOMS_BIN_FORMAT" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then
|
||||
echo "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION"
|
||||
exit 1
|
||||
if [ -n "$PROFILE" ]; then
|
||||
if [ -z "$FROM_VERSION" ]; then
|
||||
echo "Must set FROM_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting profile version to $FROM_VERSION"
|
||||
else
|
||||
if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then
|
||||
echo "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting from $FROM_SERVER: $FROM_VERSION"
|
||||
echo "Setting to $TO_SERVER: $TO_VERSION"
|
||||
fi
|
||||
|
||||
if [ -z "$ACTOR" ]; then
|
||||
@@ -36,9 +48,6 @@ if [ "$NOMS_BIN_FORMAT" = "__DOLT__" ]; then
|
||||
INIT_BIG_REPO="false"
|
||||
fi
|
||||
|
||||
echo "Setting from $FROM_SERVER: $FROM_VERSION"
|
||||
echo "Setting to $TO_SERVER: $TO_VERSION"
|
||||
|
||||
# use first 8 characters of TO_VERSION to differentiate
|
||||
# jobs
|
||||
short=${TO_VERSION:0:8}
|
||||
@@ -69,20 +78,32 @@ fi
|
||||
# or default to -1
|
||||
issuenumber=${ISSUE_NUMBER:-"-1"}
|
||||
|
||||
if [ -n "$PROFILE" ]; then
|
||||
source \
|
||||
"$TEMPLATE_SCRIPT" \
|
||||
"$jobname" \
|
||||
"$FROM_SERVER" \
|
||||
"$FROM_VERSION" \
|
||||
"$TO_SERVER" \
|
||||
"$TO_VERSION" \
|
||||
"$timeprefix" \
|
||||
"$actorprefix" \
|
||||
"$format" \
|
||||
"$issuenumber" \
|
||||
"$INIT_BIG_REPO" \
|
||||
"$NOMS_BIN_FORMAT" \
|
||||
"$WITH_TPCC" > job.json
|
||||
"$TEMPLATE_SCRIPT" \
|
||||
"$jobname" \
|
||||
"$FROM_VERSION" \
|
||||
"$timeprefix" \
|
||||
"$actorprefix" \
|
||||
"$format" \
|
||||
"$INIT_BIG_REPO" \
|
||||
"$NOMS_BIN_FORMAT" > job.json
|
||||
else
|
||||
source \
|
||||
"$TEMPLATE_SCRIPT" \
|
||||
"$jobname" \
|
||||
"$FROM_SERVER" \
|
||||
"$FROM_VERSION" \
|
||||
"$TO_SERVER" \
|
||||
"$TO_VERSION" \
|
||||
"$timeprefix" \
|
||||
"$actorprefix" \
|
||||
"$format" \
|
||||
"$issuenumber" \
|
||||
"$INIT_BIG_REPO" \
|
||||
"$NOMS_BIN_FORMAT" \
|
||||
"$WITH_TPCC" > job.json
|
||||
fi
|
||||
|
||||
out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
|
||||
|
||||
|
||||
49
.github/workflows/k8s-dolt-profile.yaml
vendored
Normal file
49
.github/workflows/k8s-dolt-profile.yaml
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
name: Profile Dolt while Benchmarking
|
||||
|
||||
on:
|
||||
repository_dispatch:
|
||||
types: [ profile-dolt ]
|
||||
|
||||
jobs:
|
||||
performance:
|
||||
runs-on: ubuntu-22.04
|
||||
name: Profile Dolt while Benchmarking
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
- uses: azure/setup-kubectl@v3.0
|
||||
with:
|
||||
version: 'v1.23.6'
|
||||
- name: Install aws-iam-authenticator
|
||||
run: |
|
||||
curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \
|
||||
chmod +x ./aws-iam-authenticator && \
|
||||
sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
|
||||
aws-iam-authenticator version
|
||||
- name: Configure AWS Credentials
|
||||
uses: aws-actions/configure-aws-credentials@v2.2.0
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-west-2
|
||||
- name: Create and Auth kubeconfig
|
||||
run: |
|
||||
echo "$CONFIG" > kubeconfig
|
||||
KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1
|
||||
KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking
|
||||
KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context
|
||||
env:
|
||||
CONFIG: ${{ secrets.CORP_KUBECONFIG }}
|
||||
- name: Create Profile Benchmarking K8s Job
|
||||
run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
|
||||
env:
|
||||
PROFILE: "true"
|
||||
FROM_VERSION: ${{ github.event.client_payload.from_version }}
|
||||
MODE: ${{ github.event.client_payload.mode }}
|
||||
ACTOR: ${{ github.event.client_payload.actor }}
|
||||
ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
|
||||
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
|
||||
KUBECONFIG: "./kubeconfig"
|
||||
INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
|
||||
NOMS_BIN_FORMAT: "__DOLT__"
|
||||
TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
|
||||
58
.github/workflows/test-build-pgo-dolt.yaml
vendored
Normal file
58
.github/workflows/test-build-pgo-dolt.yaml
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
on:
|
||||
repository_dispatch:
|
||||
types: [ pgo-release ]
|
||||
|
||||
jobs:
|
||||
email-team:
|
||||
runs-on: ubuntu-22.04
|
||||
name: Test Build PGO Dolt
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: ^1.21
|
||||
- name: Configure AWS Credentials
|
||||
uses: aws-actions/configure-aws-credentials@v2.2.0
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-west-2
|
||||
- name: Get Results
|
||||
id: get-results
|
||||
run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" dolt-cpu-profile.pprof
|
||||
env:
|
||||
KEY: ${{ github.event.client_payload.profile_key }}
|
||||
BUCKET: ${{ github.event.client_payload.bucket }}
|
||||
|
||||
# - name: Update dolt version command
|
||||
# run: sed -i -e 's/ Version = ".*"/ Version = "'"$NEW_VERSION"'"/' "$FILE"
|
||||
# env:
|
||||
# FILE: ${{ format('{0}/go/cmd/dolt/dolt.go', github.workspace) }}
|
||||
# NEW_VERSION: ${{ needs.format-version.outputs.version }}
|
||||
# - name: Set minver TBD to version
|
||||
# run: sed -i -e 's/minver:"TBD"/minver:"'"$NEW_VERSION"'"/' "$FILE"
|
||||
# env:
|
||||
# FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go', github.workspace) }}
|
||||
# NEW_VERSION: ${{ needs.format-version.outputs.version }}
|
||||
# - name: update minver_validation.txt
|
||||
# working-directory: ./go
|
||||
# run: go run -mod=readonly ./utils/genminver_validation/ $FILE
|
||||
# env:
|
||||
# FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt', github.workspace) }}
|
||||
# - uses: EndBug/add-and-commit@v9.1.1
|
||||
# with:
|
||||
# message: ${{ format('[ga-bump-release] Update Dolt version to {0} and release v{0}', needs.format-version.outputs.version) }}
|
||||
# add: ${{ format('["{0}/go/cmd/dolt/dolt.go", "{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go", "{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt"]', github.workspace) }}
|
||||
# cwd: "."
|
||||
# pull: "--ff"
|
||||
|
||||
- name: Build Binaries
|
||||
id: build_binaries
|
||||
run: |
|
||||
latest=$(git rev-parse HEAD)
|
||||
echo "commitish=$latest" >> $GITHUB_OUTPUT
|
||||
GO_BUILD_VERSION=1.21 go/utils/publishrelease/buildpgobinaries.sh
|
||||
env:
|
||||
GO_BUILD_VERSION: "1.21"
|
||||
PROFILE: ${{ format('{0}/dolt-cpu-profile.pprof', github.workspace) }}
|
||||
40
.github/workflows/test-performance-profile-job.yaml
vendored
Normal file
40
.github/workflows/test-performance-profile-job.yaml
vendored
Normal file
@@ -0,0 +1,40 @@
|
||||
on:
|
||||
# repository_dispatch:
|
||||
# types: [ dolt-prerelease ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'SemVer format release tag, i.e. 0.24.5'
|
||||
required: true
|
||||
default: ''
|
||||
|
||||
jobs:
|
||||
set-version-actor:
|
||||
name: Set Version and Actor
|
||||
runs-on: ubuntu-22.04
|
||||
outputs:
|
||||
version: ${{ steps.set-vars.outputs.version }}
|
||||
actor: ${{ steps.set-vars.outputs.actor }}
|
||||
actor_email: ${{ steps.set-vars.outputs.actor_email }}
|
||||
steps:
|
||||
- name: Set variables
|
||||
id: set-vars
|
||||
run: |
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
echo "actor=$ACTOR" >> $GITHUB_OUTPUT
|
||||
echo "actor_email=$ACTOR_EMAIL" >> $GITHUB_OUTPUT
|
||||
env:
|
||||
VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }}
|
||||
ACTOR: ${{ github.event.client_payload.actor || github.actor }}
|
||||
ACTOR_EMAIL: ${{ github.event.inputs.email }}
|
||||
|
||||
profile-benchmark-dolt:
|
||||
runs-on: ubuntu-22.04
|
||||
needs: set-version-actor
|
||||
name: Trigger Benchmark Profile K8s Workflows
|
||||
steps:
|
||||
- uses: peter-evans/repository-dispatch@v2.0.0
|
||||
with:
|
||||
token: ${{ secrets.REPO_ACCESS_TOKEN }}
|
||||
event-type: profile-dolt
|
||||
client-payload: '{"from_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/performance-benchmarking/get-dolt-profile-job-json.sh"}'
|
||||
58
go/utils/publishrelease/buildpgobinaries.sh
Executable file
58
go/utils/publishrelease/buildpgobinaries.sh
Executable file
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
script_dir=$(dirname "$0")
|
||||
cd $script_dir/../..
|
||||
|
||||
[ ! -z "$GO_BUILD_VERSION" ] || (echo "Must supply GO_BUILD_VERSION"; exit 1)
|
||||
|
||||
cmd='docker run --rm -v '`pwd`':/src -v golang:'"$GO_BUILD_VERSION"'-bookworm /bin/bash -c'
|
||||
build_cmd='go build'
|
||||
|
||||
if [ -n "$PROFILE" ]; then
|
||||
echo "Building PGO binaries"
|
||||
cmd='docker run --rm -v '`pwd`':/src -v '"$PROFILE"':/cpu.pprof golang:'"$GO_BUILD_VERSION"'-bookworm /bin/bash -c'
|
||||
build_cmd='go build -pgo=/cpu.pprof'
|
||||
fi
|
||||
|
||||
eval "$cmd" '
|
||||
set -e
|
||||
set -o pipefail
|
||||
apt-get update && apt-get install -y p7zip-full pigz
|
||||
cd /src
|
||||
|
||||
BINS="dolt"
|
||||
OS_ARCH_TUPLES="windows-amd64 linux-amd64 linux-arm64 darwin-amd64 darwin-arm64"
|
||||
|
||||
for tuple in $OS_ARCH_TUPLES; do
|
||||
os=`echo $tuple | sed 's/-.*//'`
|
||||
arch=`echo $tuple | sed 's/.*-//'`
|
||||
o="out/dolt-$os-$arch"
|
||||
mkdir -p "$o/bin"
|
||||
cp Godeps/LICENSES "$o/"
|
||||
for bin in $BINS; do
|
||||
echo Building "$o/$bin"
|
||||
obin="$bin"
|
||||
if [ "$os" = windows ]; then
|
||||
obin="$bin.exe"
|
||||
fi
|
||||
CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" '"$build_cmd"' -trimpath -ldflags="-s -w" -o "$o/bin/$obin" "./cmd/$bin/"
|
||||
done
|
||||
if [ "$os" = windows ]; then
|
||||
(cd out && 7z a "dolt-$os-$arch.zip" "dolt-$os-$arch" && 7z a "dolt-$os-$arch.7z" "dolt-$os-$arch")
|
||||
else
|
||||
tar cf - -C out "dolt-$os-$arch" | pigz -9 > "out/dolt-$os-$arch.tar.gz"
|
||||
fi
|
||||
done
|
||||
|
||||
render_install_sh() {
|
||||
local parsed=(`grep "Version = " ./cmd/dolt/dolt.go`)
|
||||
local DOLT_VERSION=`eval echo ${parsed[2]}`
|
||||
sed '\''s|__DOLT_VERSION__|'\''"$DOLT_VERSION"'\''|'\'' utils/publishrelease/install.sh
|
||||
}
|
||||
|
||||
render_install_sh > out/install.sh
|
||||
chmod 755 out/install.sh
|
||||
'
|
||||
Reference in New Issue
Block a user