diff --git a/.github/workflows/ci-bats-unix-remote.yaml b/.github/workflows/ci-bats-unix-remote.yaml new file mode 100644 index 0000000000..e03b733aa5 --- /dev/null +++ b/.github/workflows/ci-bats-unix-remote.yaml @@ -0,0 +1,118 @@ +name: Test Bats Unix + +on: + pull_request: + branches: [ main ] + paths: + - 'go/**' + - 'integration-tests/**' + +concurrency: + group: ci-bats-unix-remote${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Bats tests + defaults: + run: + shell: bash + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ ubuntu-22.04, macos-latest ] + env: + use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }} + steps: + - name: Conditionally Set ENV VARS for AWS tests + run: | + if [[ $use_credentials == true ]]; then + echo "AWS_SDK_LOAD_CONFIG=1" >> $GITHUB_ENV + echo "AWS_REGION=us-west-2" >> $GITHUB_ENV + echo "DOLT_BATS_AWS_TABLE=dolt-ci-bats-manifests-us-west-2" >> $GITHUB_ENV + echo "DOLT_BATS_AWS_BUCKET=dolt-ci-bats-chunks-us-west-2" >> $GITHUB_ENV + echo "DOLT_BATS_AWS_EXISTING_REPO=aws_remote_bats_tests__dolt__" >> $GITHUB_ENV + fi + - name: Configure AWS Credentials + if: ${{ env.use_credentials == 'true' }} + uses: aws-actions/configure-aws-credentials@567d4149d67f15f52b09796bea6573fc32952783 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-duration-seconds: 10800 # 3 hours D: + - name: Setup Go 1.x + uses: actions/setup-go@v3 + with: + go-version: ^1.19 + id: go + - name: Setup Python 3.x + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ^16 + - name: Create CI Bin + run: | + mkdir -p ./.ci_bin + echo "$(pwd)/.ci_bin" >> $GITHUB_PATH + - name: Install Bats + run: | + npm i bats + echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH + working-directory: ./.ci_bin + - name: Install Python Deps + run: | + pip install mysql-connector-python + pip install pandas + pip install pyarrow + - name: Install Dolt + working-directory: ./go + run: | + go build -mod=readonly -o ../.ci_bin/dolt ./cmd/dolt/. + + go build -mod=readonly -o ../.ci_bin/remotesrv ./utils/remotesrv/. + go build -mod=readonly -o ../.ci_bin/noms ./store/cmd/noms/. + - name: Setup Dolt Config + run: | + dolt config --global --add user.name 'Dolthub Actions' + dolt config --global --add user.email 'actions@dolthub.com' + - name: Install expect + if: matrix.os == 'ubuntu-22.04' + run: sudo apt-get install -y expect + - name: Install Maven + working-directory: ./.ci_bin + run: | + curl -LO https://dlcdn.apache.org/maven/maven-3/3.9.1/binaries/apache-maven-3.9.1-bin.tar.gz + tar -xf apache-maven-3.9.1-bin.tar.gz + echo "$(pwd)/apache-maven-3.9.1/bin" >> $GITHUB_PATH + - name: Install Hadoop + working-directory: ./.ci_bin + run: | + curl -LO https://downloads.apache.org/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz + tar xvf hadoop-3.3.4.tar.gz + echo "$(pwd)/hadoop-3.3.4/bin" >> $GITHUB_PATH + - name: Install parquet-cli + id: parquet_cli + working-directory: ./.ci_bin + run: | + curl -OL https://github.com/apache/parquet-mr/archive/refs/tags/apache-parquet-1.12.3.tar.gz + tar zxvf apache-parquet-1.12.3.tar.gz + cd parquet-mr-apache-parquet-1.12.3/parquet-cli + mvn clean install -DskipTests + runtime_jar="$(pwd)"/target/parquet-cli-1.12.3-runtime.jar + echo "runtime_jar=$runtime_jar" >> $GITHUB_OUTPUT + - name: Check expect + run: expect -v + - name: Test all Unix + env: + SQL_ENGINE: ${{ remote-engine }} + PARQUET_RUNTIME_JAR: ${{ steps.parquet_cli.outputs.runtime_jar }} + BATS_TEST_RETRIES: "3" + run: | + bats --tap . + working-directory: ./integration-tests/bats diff --git a/.github/workflows/ci-bats-unix.yaml b/.github/workflows/ci-bats-unix.yaml index 0c9dd655d2..8fe4dbc17b 100644 --- a/.github/workflows/ci-bats-unix.yaml +++ b/.github/workflows/ci-bats-unix.yaml @@ -22,7 +22,6 @@ jobs: fail-fast: true matrix: os: [ ubuntu-22.04, macos-latest ] - sql_engine: [ remote-engine, local-engine ] env: use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }} steps: @@ -111,7 +110,7 @@ jobs: run: expect -v - name: Test all Unix env: - SQL_ENGINE: ${{ matrix.sql_engine }} + SQL_ENGINE: ${{ local-engine }} PARQUET_RUNTIME_JAR: ${{ steps.parquet_cli.outputs.runtime_jar }} BATS_TEST_RETRIES: "3" run: |