mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-09 11:10:15 -05:00
Code Issues Packages Projects Releases 164 Wiki Activity

/.github/{scripts, workflows}: update release workflows for pgo dolt builds

Browse Source
This commit is contained in:
coffeegoddd
2024-02-15 17:27:47 -08:00
parent 36efb2f1fa
commit d42dd6d7ec
8 changed files with 216 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh
+6
View File
@@ -21,6 +21,7 @@ nomsBinFormat="${11}"
withTpcc="${12}"
precision="1"
tpccRegex="tpcc%"
toProfileKey=""
if [ -n "$initBigRepo" ]; then
initBigRepo="\"--init-big-repo=$initBigRepo\","
@@ -34,6 +35,10 @@ if [ -n "$withTpcc" ]; then
withTpcc="\"--withTpcc=$withTpcc\","
fi
if [ -n "$TO_PROFILE_KEY" ]; then
toProfileKey="\"--to-profile-key=$TO_PROFILE_KEY\","
fi
readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan', 'index_join_scan', 'types_table_scan', 'index_join')"
medianLatencyMultiplierReadsQuery="select f.test_name as read_tests, f.server_name, f.server_version, avg(f.latency_percentile) as from_latency_median, t.server_name, t.server_version, avg(t.latency_percentile) as to_latency_median, ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) as multiplier from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"
meanMultiplierReadsQuery="select round(avg(multipliers), $precision) as reads_mean_multiplier from (select (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) as multipliers from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name)"
@@ -96,6 +101,7 @@ echo '
"--from-version='$fromVersion'",
"--to-server='$toServer'",
"--to-version='$toVersion'",
'"$toProfileKey"'
"--bucket=performance-benchmarking-github-actions-results",
"--region=us-west-2",
"--results-dir='$timeprefix'",
.github/scripts/performance-benchmarking/run-benchmarks.sh
+2
View File
@@ -81,6 +81,8 @@ if [[ "$MODE" = "release" || "$MODE" = "nightly" ]]; then
format="html"
fi
export TO_PROFILE_KEY
# set value to ISSUE_NUMBER environment variable
# or default to -1
issuenumber=${ISSUE_NUMBER:-"-1"}
.github/workflows/cd-release-pgo.yaml
+194
View File
@@ -0,0 +1,194 @@
on:
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.24.5'
required: true
profile_bucket:
description: 's3 bucket of dolt profile used to build pgo'
required: true
profile_key:
description: 's3 key of dolt profile used to build pgo'
required: true
repository_dispatch:
types: [ pgo-release ]
jobs:
create-pgo-release:
runs-on: ubuntu-22.04
name: Release PGO Dolt
steps:
- uses: actions/checkout@v3
- name: Set up Go 1.x
uses: actions/setup-go@v3
with:
go-version: ^1.21
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2.2.0
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Get Results
id: get-results
run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" dolt-cpu-profile.pprof
env:
KEY: ${{ github.event.client_payload.profile_key }}
BUCKET: ${{ github.event.client_payload.bucket }}
- name: Update dolt version command
run: sed -i -e 's/ Version = ".*"/ Version = "'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/go/cmd/dolt/dolt.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: Set minver TBD to version
run: sed -i -e 's/minver:"TBD"/minver:"'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: update minver_validation.txt
working-directory: ./go
run: go run -mod=readonly ./utils/genminver_validation/ $FILE
env:
FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt', github.workspace) }}
- uses: EndBug/add-and-commit@v9.1.1
with:
message: ${{ format('[ga-bump-release] Update Dolt version to {0} and release v{0}', needs.format-version.outputs.version) }}
add: ${{ format('["{0}/go/cmd/dolt/dolt.go", "{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go", "{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt"]', github.workspace) }}
cwd: "."
pull: "--ff"
- name: Build PGO Binaries
id: build_binaries
run: |
latest=$(git rev-parse HEAD)
echo "commitish=$latest" >> $GITHUB_OUTPUT
GO_BUILD_VERSION=1.21 go/utils/publishrelease/buildpgobinaries.sh
env:
GO_BUILD_VERSION: "1.21"
PROFILE: ${{ format('{0}/dolt-cpu-profile.pprof', github.workspace) }}
- name: Create Release
id: create_release
uses: dolthub/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v${{ needs.format-version.outputs.version }}
release_name: ${{ needs.format-version.outputs.version }}
draft: false
prerelease: false
commitish: ${{ steps.build_binaries.outputs.commitish }}
- name: Upload Linux AMD64 Distro
id: upload-linux-amd64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-linux-amd64.tar.gz
asset_name: dolt-linux-amd64.tar.gz
asset_content_type: application/zip
- name: Upload Linux ARM64 Distro
id: upload-linux-arm64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-linux-arm64.tar.gz
asset_name: dolt-linux-arm64.tar.gz
asset_content_type: application/zip
- name: Upload OSX AMD64 Distro
id: upload-osx-amd64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-darwin-amd64.tar.gz
asset_name: dolt-darwin-amd64.tar.gz
asset_content_type: application/zip
- name: Upload OSX ARM64 Distro
id: upload-osx-arm64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-darwin-arm64.tar.gz
asset_name: dolt-darwin-arm64.tar.gz
asset_content_type: application/zip
- name: Upload Windows Distro
id: upload-windows-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-windows-amd64.zip
asset_name: dolt-windows-amd64.zip
asset_content_type: application/zip
- name: Upload Windows Distro 7z
id: upload-windows-distro-7z
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-windows-amd64.7z
asset_name: dolt-windows-amd64.7z
asset_content_type: application/x-7z-compressed
- name: Upload Install Script
id: upload-install-script
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/install.sh
asset_name: install.sh
asset_content_type: text/plain
create-windows-msi:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Upload MSI
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: upload-msi
repository: dolthub/windows-msi-creator
client-payload: '{"tag": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}", "actor": "${{ github.actor }}", "bucket": "${{ github.event.client_payload.bucket }}", "profile_key": "${{ github.event.client_payload.profile_key }}"}'
create-release-notes:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Release Notes
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: release-notes
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}"}'
trigger-performance-benchmark-email:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Performance Benchmarks
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: release-dolt
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "actor": "${{ github.actor }}", "profile_key": "${{ github.event.client_payload.profile_key }}"}'
docker-image-push:
needs: [ format-version, create-release ]
runs-on: ubuntu-22.04
steps:
- name: Trigger Push Docker Image
uses: peter-evans/repository-dispatch@v1
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: push-docker-image
client-payload: '{"version": "${{ needs.format-version.outputs.version }}"}'
.github/workflows/cd-release.yaml
+6 -162
View File
@@ -23,169 +23,13 @@ jobs:
fi
echo "version=$version" >> $GITHUB_OUTPUT
create-release:
profile-benchmark-dolt:
runs-on: ubuntu-22.04
needs: format-version
name: Create release
runs-on: ubuntu-22.04
outputs:
release_id: ${{ steps.create_release.outputs.id }}
name: Trigger Benchmark Profile K8s Workflows
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Go 1.x
uses: actions/setup-go@v3
with:
go-version: ^1.21
- name: Update dolt version command
run: sed -i -e 's/ Version = ".*"/ Version = "'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/go/cmd/dolt/dolt.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: Set minver TBD to version
run: sed -i -e 's/minver:"TBD"/minver:"'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: update minver_validation.txt
working-directory: ./go
run: go run -mod=readonly ./utils/genminver_validation/ $FILE
env:
FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt', github.workspace) }}
- uses: EndBug/add-and-commit@v9.1.1
with:
message: ${{ format('[ga-bump-release] Update Dolt version to {0} and release v{0}', needs.format-version.outputs.version) }}
add: ${{ format('["{0}/go/cmd/dolt/dolt.go", "{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go", "{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt"]', github.workspace) }}
cwd: "."
pull: "--ff"
- name: Build Binaries
id: build_binaries
run: |
latest=$(git rev-parse HEAD)
echo "commitish=$latest" >> $GITHUB_OUTPUT
GO_BUILD_VERSION=1.21 go/utils/publishrelease/buildbinaries.sh
- name: Create Release
id: create_release
uses: dolthub/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v${{ needs.format-version.outputs.version }}
release_name: ${{ needs.format-version.outputs.version }}
draft: false
prerelease: false
commitish: ${{ steps.build_binaries.outputs.commitish }}
- name: Upload Linux AMD64 Distro
id: upload-linux-amd64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-linux-amd64.tar.gz
asset_name: dolt-linux-amd64.tar.gz
asset_content_type: application/zip
- name: Upload Linux ARM64 Distro
id: upload-linux-arm64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-linux-arm64.tar.gz
asset_name: dolt-linux-arm64.tar.gz
asset_content_type: application/zip
- name: Upload OSX AMD64 Distro
id: upload-osx-amd64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-darwin-amd64.tar.gz
asset_name: dolt-darwin-amd64.tar.gz
asset_content_type: application/zip
- name: Upload OSX ARM64 Distro
id: upload-osx-arm64-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-darwin-arm64.tar.gz
asset_name: dolt-darwin-arm64.tar.gz
asset_content_type: application/zip
- name: Upload Windows Distro
id: upload-windows-distro
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-windows-amd64.zip
asset_name: dolt-windows-amd64.zip
asset_content_type: application/zip
- name: Upload Windows Distro 7z
id: upload-windows-distro-7z
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/dolt-windows-amd64.7z
asset_name: dolt-windows-amd64.7z
asset_content_type: application/x-7z-compressed
- name: Upload Install Script
id: upload-install-script
uses: dolthub/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: go/out/install.sh
asset_name: install.sh
asset_content_type: text/plain
create-windows-msi:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Upload MSI
uses: peter-evans/repository-dispatch@v2.0.0
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: upload-msi
repository: dolthub/windows-msi-creator
client-payload: '{"tag": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}", "actor": "${{ github.actor }}"}'
create-release-notes:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Release Notes
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: release-notes
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}"}'
trigger-performance-benchmark-email:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Performance Benchmarks
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: release-dolt
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "actor": "${{ github.actor }}"}'
docker-image-push:
needs: [ format-version, create-release ]
runs-on: ubuntu-22.04
steps:
- name: Trigger Push Docker Image
uses: peter-evans/repository-dispatch@v1
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: push-docker-image
client-payload: '{"version": "${{ needs.format-version.outputs.version }}"}'
event-type: profile-dolt
client-payload: '{"from_version": "${{ needs.format-version.outputs.version }}", "mode": "release", "actor": "${{ github.actor }}", "actor_email": "dustin@dolthub.com", "template_script": "./.github/scripts/performance-benchmarking/get-dolt-profile-job-json.sh"}'
.github/workflows/k8s-benchmark-latency.yaml
+1
View File
@@ -44,6 +44,7 @@ jobs:
FROM_VERSION: ${{ github.event.client_payload.from_version }}
TO_SERVER: ${{ github.event.client_payload.to_server }}
TO_VERSION: ${{ github.event.client_payload.to_version }}
TO_PROFILE_KEY: ${{ github.event.client_payload.to_profile_key }}
MODE: ${{ github.event.client_payload.mode }}
ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
ACTOR: ${{ github.event.client_payload.actor }}
.github/workflows/performance-benchmarks-email-report.yaml
+7
View File
@@ -40,10 +40,17 @@ jobs:
name: Trigger Benchmark Latency and Benchmark Import K8s Workflows
steps:
- uses: peter-evans/repository-dispatch@v2.0.0
if: ${{ github.event.client_payload.profile_key == '' }}
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "mysql", "from_version": "8.0.35", "to_server": "dolt", "to_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"}'
- uses: peter-evans/repository-dispatch@v2.0.0
if: ${{ github.event.client_payload.profile_key != '' }}
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "mysql", "from_version": "8.0.35", "to_server": "dolt", "to_version": "${{ needs.set-version-actor.outputs.version }}", "profile_key": "${{ github.event.client_payload.profile_key }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh"}'
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
.github/workflows/test-build-pgo-dolt.yaml
-58
View File
@@ -1,58 +0,0 @@
on:
repository_dispatch:
types: [ pgo-release ]
jobs:
email-team:
runs-on: ubuntu-22.04
name: Test Build PGO Dolt
steps:
- uses: actions/checkout@v3
- name: Set up Go 1.x
uses: actions/setup-go@v3
with:
go-version: ^1.21
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v2.2.0
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Get Results
id: get-results
run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" dolt-cpu-profile.pprof
env:
KEY: ${{ github.event.client_payload.profile_key }}
BUCKET: ${{ github.event.client_payload.bucket }}
# - name: Update dolt version command
# run: sed -i -e 's/ Version = ".*"/ Version = "'"$NEW_VERSION"'"/' "$FILE"
# env:
# FILE: ${{ format('{0}/go/cmd/dolt/dolt.go', github.workspace) }}
# NEW_VERSION: ${{ needs.format-version.outputs.version }}
# - name: Set minver TBD to version
# run: sed -i -e 's/minver:"TBD"/minver:"'"$NEW_VERSION"'"/' "$FILE"
# env:
# FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go', github.workspace) }}
# NEW_VERSION: ${{ needs.format-version.outputs.version }}
# - name: update minver_validation.txt
# working-directory: ./go
# run: go run -mod=readonly ./utils/genminver_validation/ $FILE
# env:
# FILE: ${{ format('{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt', github.workspace) }}
# - uses: EndBug/add-and-commit@v9.1.1
# with:
# message: ${{ format('[ga-bump-release] Update Dolt version to {0} and release v{0}', needs.format-version.outputs.version) }}
# add: ${{ format('["{0}/go/cmd/dolt/dolt.go", "{0}/go/cmd/dolt/commands/sqlserver/yaml_config.go", "{0}/go/cmd/dolt/commands/sqlserver/testdata/minver_validation.txt"]', github.workspace) }}
# cwd: "."
# pull: "--ff"
- name: Build Binaries
id: build_binaries
run: |
latest=$(git rev-parse HEAD)
echo "commitish=$latest" >> $GITHUB_OUTPUT
GO_BUILD_VERSION=1.21 go/utils/publishrelease/buildpgobinaries.sh
env:
GO_BUILD_VERSION: "1.21"
PROFILE: ${{ format('{0}/dolt-cpu-profile.pprof', github.workspace) }}
.github/workflows/test-pgo-worflow.yaml → .github/workflows/test-pgo-workflow.yaml
View File