From 6a3dafc75fa2a57a4c813331c74cff98f7232155 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 3 Jun 2022 14:57:04 -0700 Subject: [PATCH] hardcode benchmark params --- .../ci-performance-benchmarks-new-format.yaml | 100 ++++++++++++++++++ .../ci-performance-benchmarks-release.yaml | 5 +- .../workflows/ci-performance-benchmarks.yaml | 13 +-- 3 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/ci-performance-benchmarks-new-format.yaml diff --git a/.github/workflows/ci-performance-benchmarks-new-format.yaml b/.github/workflows/ci-performance-benchmarks-new-format.yaml new file mode 100644 index 0000000000..0222ff0f5f --- /dev/null +++ b/.github/workflows/ci-performance-benchmarks-new-format.yaml @@ -0,0 +1,100 @@ +name: Benchmark Pull Requests + +on: + pull_request: + types: [ opened ] + issue_comment: + types: [ created ] + +jobs: + validate-commentor: + runs-on: ubuntu-18.04 + outputs: + valid: ${{ steps.set_valid.outputs.valid }} + steps: + - uses: actions/checkout@v2 + - name: Validate Commentor + id: set_valid + run: ./.github/scripts/performance-benchmarking/validate-commentor.sh "$ACTOR" + env: + ACTOR: ${{ github.actor }} + + check-comments: + runs-on: ubuntu-18.04 + needs: validate-commentor + if: ${{ needs.validate-commentor.outputs.valid == 'true' }} + outputs: + benchmark: ${{ steps.set_benchmark.outputs.benchmark }} + comment-body: ${{ steps.set_body.outputs.body }} + steps: + - name: Check for Deploy Trigger + uses: dolthub/pull-request-comment-trigger@master + id: check + with: + trigger: '#newbenchmark' + reaction: rocket + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set Benchmark + if: ${{ steps.check.outputs.triggered == 'true' }} + id: set_benchmark + run: | + echo "::set-output name=benchmark::true" + + performance: + runs-on: ubuntu-18.04 + needs: [validate-commentor, check-comments] + if: ${{ needs.check-comments.outputs.benchmark == 'true' }} + name: Benchmark Performance + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: azure/setup-kubectl@v2.0 + with: + version: 'v1.23.6' + - name: Install aws-iam-authenticator + run: | + curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \ + chmod +x ./aws-iam-authenticator && \ + sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator + aws-iam-authenticator version + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + - name: Create and Auth kubeconfig + run: | + echo "$CONFIG" > kubeconfig + KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-dolt --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1 + KUBECONFIG=kubeconfig kubectl config set-context github-actions-dolt-context --cluster=eks-cluster-1 --user=github-actions-dolt --namespace=performance-benchmarking + KUBECONFIG=kubeconfig kubectl config use-context github-actions-dolt-context + env: + CONFIG: ${{ secrets.CORP_KUBECONFIG }} + - name: Get pull number + uses: actions/github-script@v3 + id: get_pull_number + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: core.setOutput("pull_number", JSON.stringify(context.issue.number)); + - name: Run benchmarks + id: run-benchmarks + run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh + env: + FROM_SERVER: 'dolt' + FROM_VERSION: ${{ github.sha }} + TO_SERVER: 'dolt' + TO_VERSION: ${{ steps.comment-branch.outputs.head_sha }} + MODE: 'pullRequest' + ISSUE_NUMBER: ${{ steps.get_pull_number.outputs.pull_number }} + ACTOR: ${{ github.actor }} + REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} + KUBECONFIG: "./kubeconfig" + INIT_BIG_REPO: "false" + NOMS_BIN_FORMAT: "__DOLT_1__" + TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh" diff --git a/.github/workflows/ci-performance-benchmarks-release.yaml b/.github/workflows/ci-performance-benchmarks-release.yaml index cca94493a0..fb2c6c00eb 100644 --- a/.github/workflows/ci-performance-benchmarks-release.yaml +++ b/.github/workflows/ci-performance-benchmarks-release.yaml @@ -35,9 +35,12 @@ jobs: ACTOR_EMAIL: ${{ github.event.inputs.email }} benchmark-release-mysql: +<<<<<<< HEAD strategy: matrix: nbf: [ "__LD_1__", "__DOLT_1__" ] +======= +>>>>>>> hardcode benchmark params runs-on: ubuntu-18.04 needs: set-version-actor name: Benchmark Dolt Release vs MySQL 8 @@ -79,5 +82,5 @@ jobs: ACTOR_EMAIL: ${{ needs.set-version-actor.outputs.actor_email }} REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} KUBECONFIG: "./kubeconfig" - NOMS_BIN_FORMAT: ${{ matrix.nbf }} + NOMS_BIN_FORMAT: "__LD_1__" TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh" diff --git a/.github/workflows/ci-performance-benchmarks.yaml b/.github/workflows/ci-performance-benchmarks.yaml index 515576f4f1..e65aa614a3 100644 --- a/.github/workflows/ci-performance-benchmarks.yaml +++ b/.github/workflows/ci-performance-benchmarks.yaml @@ -42,15 +42,6 @@ jobs: echo "::set-output name=benchmark::true" performance: - strategy: - matrix: - biginit: ["true", "false"] - nbf: ["__LD_1__", "__DOLT_1__"] - exclude: - - biginit: "true" - nbf: "__DOLT_1__" - - biginit: "false" - nbf: "__LD_1__" runs-on: ubuntu-18.04 needs: [validate-commentor, check-comments] if: ${{ needs.check-comments.outputs.benchmark == 'true' }} @@ -104,6 +95,6 @@ jobs: ACTOR: ${{ github.actor }} REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} KUBECONFIG: "./kubeconfig" - INIT_BIG_REPO: ${{ matrix.biginit }} - NOMS_BIN_FORMAT: ${{ matrix.nbf }} + INIT_BIG_REPO: "true" + NOMS_BIN_FORMAT: "__LD_1__" TEMPLATE_SCRIPT: "./.github/scripts/performance-benchmarking/get-dolt-dolt-job-json.sh"