From d35b0d23114fb7c7c85f7044ad7eb77facc8b9f2 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 27 Aug 2020 13:30:14 -0700 Subject: [PATCH 1/7] /{mysql-client-tests, .github/}: Iterating on github actions integration --- .../workflows/ci-mysql-integration-tests.yaml | 68 +++++++++++++++++++ .../MySQLDockerfile => MySQLDockerfile | 5 +- 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci-mysql-integration-tests.yaml rename mysql-client-tests/MySQLDockerfile => MySQLDockerfile (94%) diff --git a/.github/workflows/ci-mysql-integration-tests.yaml b/.github/workflows/ci-mysql-integration-tests.yaml new file mode 100644 index 0000000000..9b8e1a6b32 --- /dev/null +++ b/.github/workflows/ci-mysql-integration-tests.yaml @@ -0,0 +1,68 @@ +name: Test MySQL Integrations + +on: + pull_request: + branches: [ master ] + +jobs: + test: + name: Bats tests for MySQL integrations + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ ubuntu-latest ] + steps: + - name: Setup Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + id: go + - name: Setup Python 3.x + uses: actions/setup-python@v2 + with: + python-version: ^3.6 + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: ^12 + - name: Create CI Bin + run: | + mkdir -p ./.ci_bin + echo "::add-path::$(pwd)/.ci_bin" + - name: Install Bats + run: | + npm i bats + echo "::add-path::$(pwd)/node_modules/.bin" + working-directory: ./.ci_bin + - name: Install Dolt + working-directory: ./go + run: | + go build -mod=readonly -o ../.ci_bin/dolt ./cmd/dolt/. + go build -mod=readonly -o ../.ci_bin/git-dolt ./cmd/git-dolt/. + go build -mod=readonly -o ../.ci_bin/git-dolt-smudge ./cmd/git-dolt-smudge/. + go build -mod=readonly -o ../.ci_bin/remotesrv ./utils/remotesrv/. + - name: Setup Dolt Config + run: | + dolt config --global --add user.name 'Liquidata Actions' + dolt config --global --add user.email 'actions@liquidata.co' + - uses: actions/setup-java@v1 + with: + java-version: '9.0.4' # The JDK version to make available on the path. + java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk + architecture: x64 # (x64 or x86) - defaults to x64 + - name: Install mysql connector for java + run: | + curl -L -o /mysql-client-tests/java/mysql-connector-java-8.0.21.jar \ + https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.21/mysql-connector-java-8.0.21.jar + - name: Install python deps + run: | + pip3 install mysql-connector-python + pip3 install PyMySQL + - name: Install node deps + working-directory: ./mysql-client-tests/node + run: npm install + - name: Test all + run: | + bats --tap mysql-client-tests.bats + working-directory: ./mysql-client-tests diff --git a/mysql-client-tests/MySQLDockerfile b/MySQLDockerfile similarity index 94% rename from mysql-client-tests/MySQLDockerfile rename to MySQLDockerfile index d134320f5f..6e0cd720f8 100644 --- a/mysql-client-tests/MySQLDockerfile +++ b/MySQLDockerfile @@ -1,5 +1,6 @@ FROM ubuntu:18.04 -COPY . /mysql-client-tests +COPY ./mysql-client-tests /mysql-client-tests +COPY ./go /dolt-go # install python ENV DEBIAN_FRONTEND=noninteractive @@ -67,4 +68,4 @@ RUN npm install # run tests WORKDIR /mysql-client-tests - +#CMD ["bats", "mysql-client-tests.bats"] From 6f2b7667f4ba28126c8b90bc995d1a9e670dfc6d Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 27 Aug 2020 15:00:19 -0700 Subject: [PATCH 2/7] /{mysql-client-tests-entrypoint.sh, MySQLDockerfile, .github/}: Testing custom action --- .github/workflows/ci-mysql-client-tests.yaml | 17 +++++ .../workflows/ci-mysql-integration-tests.yaml | 68 ------------------- ACTIONS-README.md | 13 ++++ MySQLDockerfile | 29 +++++--- action.yaml | 8 +++ mysql-client-tests-entrypoint.sh | 13 ++++ 6 files changed, 69 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/ci-mysql-client-tests.yaml delete mode 100644 .github/workflows/ci-mysql-integration-tests.yaml create mode 100644 ACTIONS-README.md create mode 100644 action.yaml create mode 100755 mysql-client-tests-entrypoint.sh diff --git a/.github/workflows/ci-mysql-client-tests.yaml b/.github/workflows/ci-mysql-client-tests.yaml new file mode 100644 index 0000000000..7e5cd7f9a6 --- /dev/null +++ b/.github/workflows/ci-mysql-client-tests.yaml @@ -0,0 +1,17 @@ +on: [push] + +jobs: + mysql_client_integrations_job: + runs-on: ubuntu-latest + name: Run tests + steps: + # To use this repository's private action, + # you must check out the repository + - name: Checkout + uses: actions/checkout@v2 + - name: Test mysql client integrations + uses: ./ # Uses an action in the root directory + id: mysql-tests + # Use the output from the `hello` step + - name: Get the output results + run: echo "The results were ${{ steps.mysql-tests.outputs.results }}" \ No newline at end of file diff --git a/.github/workflows/ci-mysql-integration-tests.yaml b/.github/workflows/ci-mysql-integration-tests.yaml deleted file mode 100644 index 9b8e1a6b32..0000000000 --- a/.github/workflows/ci-mysql-integration-tests.yaml +++ /dev/null @@ -1,68 +0,0 @@ -name: Test MySQL Integrations - -on: - pull_request: - branches: [ master ] - -jobs: - test: - name: Bats tests for MySQL integrations - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - os: [ ubuntu-latest ] - steps: - - name: Setup Go 1.x - uses: actions/setup-go@v2 - with: - go-version: ^1.13 - id: go - - name: Setup Python 3.x - uses: actions/setup-python@v2 - with: - python-version: ^3.6 - - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: ^12 - - name: Create CI Bin - run: | - mkdir -p ./.ci_bin - echo "::add-path::$(pwd)/.ci_bin" - - name: Install Bats - run: | - npm i bats - echo "::add-path::$(pwd)/node_modules/.bin" - working-directory: ./.ci_bin - - name: Install Dolt - working-directory: ./go - run: | - go build -mod=readonly -o ../.ci_bin/dolt ./cmd/dolt/. - go build -mod=readonly -o ../.ci_bin/git-dolt ./cmd/git-dolt/. - go build -mod=readonly -o ../.ci_bin/git-dolt-smudge ./cmd/git-dolt-smudge/. - go build -mod=readonly -o ../.ci_bin/remotesrv ./utils/remotesrv/. - - name: Setup Dolt Config - run: | - dolt config --global --add user.name 'Liquidata Actions' - dolt config --global --add user.email 'actions@liquidata.co' - - uses: actions/setup-java@v1 - with: - java-version: '9.0.4' # The JDK version to make available on the path. - java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk - architecture: x64 # (x64 or x86) - defaults to x64 - - name: Install mysql connector for java - run: | - curl -L -o /mysql-client-tests/java/mysql-connector-java-8.0.21.jar \ - https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.21/mysql-connector-java-8.0.21.jar - - name: Install python deps - run: | - pip3 install mysql-connector-python - pip3 install PyMySQL - - name: Install node deps - working-directory: ./mysql-client-tests/node - run: npm install - - name: Test all - run: | - bats --tap mysql-client-tests.bats - working-directory: ./mysql-client-tests diff --git a/ACTIONS-README.md b/ACTIONS-README.md new file mode 100644 index 0000000000..7d858ba86b --- /dev/null +++ b/ACTIONS-README.md @@ -0,0 +1,13 @@ +# Dolt MySQL client integration docker action + +This action runs ./mysql-client-tests in docker. + +## Outputs + +### `results` + +The results of the tests. + +## Example usage + +uses: actions/mysql-client-test-action@v1 diff --git a/MySQLDockerfile b/MySQLDockerfile index 6e0cd720f8..68f6c78246 100644 --- a/MySQLDockerfile +++ b/MySQLDockerfile @@ -1,6 +1,15 @@ +FROM golang:1.15.0-buster as builder +WORKDIR /root/building/go +COPY ./go/ . + +# install dolt from source +ENV GOFLAGS="-mod=readonly" +RUN go build -o dolt ./cmd/dolt + FROM ubuntu:18.04 +COPY --from=builder /root/building/go/dolt /usr/local/bin/dolt COPY ./mysql-client-tests /mysql-client-tests -COPY ./go /dolt-go +COPY ./mysql-client-tests-entrypoint.sh /mysql-client-tests/entrypoint.sh # install python ENV DEBIAN_FRONTEND=noninteractive @@ -21,8 +30,7 @@ RUN apt-get -y install curl RUN pip3 install mysql-connector-python RUN pip3 install PyMySQL -# install java -# Install OpenJDK-8 +# install OpenJDK-8 RUN apt-get update && \ apt-get install -y openjdk-8-jdk && \ apt-get install -y ant && \ @@ -49,18 +57,16 @@ RUN cat /etc/apt/sources.list.d/nodesource.list RUN apt -y install nodejs # install bats -RUN apt-get update -RUN apt-get install bats +RUN apt-get -y update +RUN apt-get -y install bats -# install dolt TODO build from source -RUN curl -L https://github.com/liquidata-inc/dolt/releases/latest/download/install.sh | bash RUN dolt version # setup dolt config -RUN dolt config --global --add metrics.disabled true -RUN dolt config --global --add metrics.host localhost -RUN dolt config --global --add user.name mysql-test-runner -RUN dolt config --global --add user.email mysql-test-runner@liquidata.co +#RUN dolt config --global --add metrics.disabled true +#RUN dolt config --global --add metrics.host localhost +#RUN dolt config --global --add user.name mysql-test-runner +#RUN dolt config --global --add user.email mysql-test-runner@liquidata.co # install node deps WORKDIR /mysql-client-tests/node @@ -69,3 +75,4 @@ RUN npm install # run tests WORKDIR /mysql-client-tests #CMD ["bats", "mysql-client-tests.bats"] +ENTRYPOINT ["/mysql-client-tests/entrypoint.sh"] diff --git a/action.yaml b/action.yaml new file mode 100644 index 0000000000..9660678eec --- /dev/null +++ b/action.yaml @@ -0,0 +1,8 @@ +name: 'Dolt MySQL client integration tests' +description: 'Smoke tests for mysql client integrations' +outputs: + results: # logged test results output + description: 'The results of the tests' +runs: + using: 'docker' + image: 'MySQLDockerfile' diff --git a/mysql-client-tests-entrypoint.sh b/mysql-client-tests-entrypoint.sh new file mode 100755 index 0000000000..80bfa4155a --- /dev/null +++ b/mysql-client-tests-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +pwd +ls -a +dolt config --global --add metrics.disabled true +dolt config --global --add metrics.host localhost +dolt config --global --add user.name mysql-test-runner +dolt config --global --add user.email mysql-test-runner@liquidata.co + +bats -v + +results=$(bats /mysql-client-tests/mysql-client-tests.bats) +echo "::set-output name=results::$results" \ No newline at end of file From a634345b427a06911179b0939cedd217b69fe890 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 27 Aug 2020 15:53:45 -0700 Subject: [PATCH 3/7] /{mysql-client-tests-entrypoint.sh, .github/workflows/}: Iterating on cleanup --- .github/workflows/ci-mysql-client-tests.yaml | 8 ++++---- mysql-client-tests-entrypoint.sh | 7 +------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-mysql-client-tests.yaml b/.github/workflows/ci-mysql-client-tests.yaml index 7e5cd7f9a6..46f81b562f 100644 --- a/.github/workflows/ci-mysql-client-tests.yaml +++ b/.github/workflows/ci-mysql-client-tests.yaml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v2 - name: Test mysql client integrations uses: ./ # Uses an action in the root directory - id: mysql-tests - # Use the output from the `hello` step - - name: Get the output results - run: echo "The results were ${{ steps.mysql-tests.outputs.results }}" \ No newline at end of file +# id: mysql-tests +# # Use the output from the `hello` step +# - name: Get the output results +# run: echo "The results were ${{ steps.mysql-tests.outputs.results }}" \ No newline at end of file diff --git a/mysql-client-tests-entrypoint.sh b/mysql-client-tests-entrypoint.sh index 80bfa4155a..d0dc93ce4e 100755 --- a/mysql-client-tests-entrypoint.sh +++ b/mysql-client-tests-entrypoint.sh @@ -1,13 +1,8 @@ #!/bin/sh -pwd -ls -a dolt config --global --add metrics.disabled true dolt config --global --add metrics.host localhost dolt config --global --add user.name mysql-test-runner dolt config --global --add user.email mysql-test-runner@liquidata.co -bats -v - -results=$(bats /mysql-client-tests/mysql-client-tests.bats) -echo "::set-output name=results::$results" \ No newline at end of file +bats /mysql-client-tests/mysql-client-tests.bats From 66190b8f9e5ffa77e819fe033e1ac47b9573faab Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 27 Aug 2020 16:00:50 -0700 Subject: [PATCH 4/7] testing failed test --- .github/actions/mysql-client-tests/action.yaml | 8 ++++++++ .github/workflows/ci-mysql-client-tests.yaml | 8 +------- 2 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 .github/actions/mysql-client-tests/action.yaml diff --git a/.github/actions/mysql-client-tests/action.yaml b/.github/actions/mysql-client-tests/action.yaml new file mode 100644 index 0000000000..a28f3a2d7a --- /dev/null +++ b/.github/actions/mysql-client-tests/action.yaml @@ -0,0 +1,8 @@ +name: 'Dolt MySQL client integration tests' +description: 'Smoke tests for mysql client integrations' +outputs: + results: # logged test results output + description: 'The results of the tests' +runs: + using: 'docker' + image: '../../../MySQLDockerfile' diff --git a/.github/workflows/ci-mysql-client-tests.yaml b/.github/workflows/ci-mysql-client-tests.yaml index 46f81b562f..7b6711abab 100644 --- a/.github/workflows/ci-mysql-client-tests.yaml +++ b/.github/workflows/ci-mysql-client-tests.yaml @@ -5,13 +5,7 @@ jobs: runs-on: ubuntu-latest name: Run tests steps: - # To use this repository's private action, - # you must check out the repository - name: Checkout uses: actions/checkout@v2 - name: Test mysql client integrations - uses: ./ # Uses an action in the root directory -# id: mysql-tests -# # Use the output from the `hello` step -# - name: Get the output results -# run: echo "The results were ${{ steps.mysql-tests.outputs.results }}" \ No newline at end of file + uses: ./.github/actions/mysql-client-tests # Uses an action in the root directory From 2d4918ed964b084086f73ca4fbbcba30690b8620 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 27 Aug 2020 16:11:19 -0700 Subject: [PATCH 5/7] /action.yaml: mv action to .github/actions/mysql-client-test --- action.yaml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 action.yaml diff --git a/action.yaml b/action.yaml deleted file mode 100644 index 9660678eec..0000000000 --- a/action.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: 'Dolt MySQL client integration tests' -description: 'Smoke tests for mysql client integrations' -outputs: - results: # logged test results output - description: 'The results of the tests' -runs: - using: 'docker' - image: 'MySQLDockerfile' From 12f2857623d8ac2d36636307b450633b3dcb9db0 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 27 Aug 2020 16:19:44 -0700 Subject: [PATCH 6/7] /{.github/, MySQLDockerfile}: Add private actions for mysql client integration tests --- .github/actions/mysql-client-tests/action.yaml | 3 --- .github/workflows/ci-mysql-client-tests.yaml | 6 ++++-- ACTIONS-README.md | 13 ------------- MySQLDockerfile | 10 ---------- mysql-client-tests-entrypoint.sh | 2 ++ 5 files changed, 6 insertions(+), 28 deletions(-) delete mode 100644 ACTIONS-README.md diff --git a/.github/actions/mysql-client-tests/action.yaml b/.github/actions/mysql-client-tests/action.yaml index a28f3a2d7a..4d89a8ceda 100644 --- a/.github/actions/mysql-client-tests/action.yaml +++ b/.github/actions/mysql-client-tests/action.yaml @@ -1,8 +1,5 @@ name: 'Dolt MySQL client integration tests' description: 'Smoke tests for mysql client integrations' -outputs: - results: # logged test results output - description: 'The results of the tests' runs: using: 'docker' image: '../../../MySQLDockerfile' diff --git a/.github/workflows/ci-mysql-client-tests.yaml b/.github/workflows/ci-mysql-client-tests.yaml index 7b6711abab..adfe2ed423 100644 --- a/.github/workflows/ci-mysql-client-tests.yaml +++ b/.github/workflows/ci-mysql-client-tests.yaml @@ -1,4 +1,6 @@ -on: [push] +name: Test MySQL Client integrations + +on: [pull_request] jobs: mysql_client_integrations_job: @@ -8,4 +10,4 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Test mysql client integrations - uses: ./.github/actions/mysql-client-tests # Uses an action in the root directory + uses: ./.github/actions/mysql-client-tests diff --git a/ACTIONS-README.md b/ACTIONS-README.md deleted file mode 100644 index 7d858ba86b..0000000000 --- a/ACTIONS-README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Dolt MySQL client integration docker action - -This action runs ./mysql-client-tests in docker. - -## Outputs - -### `results` - -The results of the tests. - -## Example usage - -uses: actions/mysql-client-test-action@v1 diff --git a/MySQLDockerfile b/MySQLDockerfile index 68f6c78246..77abce6cf4 100644 --- a/MySQLDockerfile +++ b/MySQLDockerfile @@ -60,19 +60,9 @@ RUN apt -y install nodejs RUN apt-get -y update RUN apt-get -y install bats -RUN dolt version - -# setup dolt config -#RUN dolt config --global --add metrics.disabled true -#RUN dolt config --global --add metrics.host localhost -#RUN dolt config --global --add user.name mysql-test-runner -#RUN dolt config --global --add user.email mysql-test-runner@liquidata.co - # install node deps WORKDIR /mysql-client-tests/node RUN npm install -# run tests WORKDIR /mysql-client-tests -#CMD ["bats", "mysql-client-tests.bats"] ENTRYPOINT ["/mysql-client-tests/entrypoint.sh"] diff --git a/mysql-client-tests-entrypoint.sh b/mysql-client-tests-entrypoint.sh index d0dc93ce4e..ecaed346ba 100755 --- a/mysql-client-tests-entrypoint.sh +++ b/mysql-client-tests-entrypoint.sh @@ -1,8 +1,10 @@ #!/bin/sh +echo "updating dolt config for tests:" dolt config --global --add metrics.disabled true dolt config --global --add metrics.host localhost dolt config --global --add user.name mysql-test-runner dolt config --global --add user.email mysql-test-runner@liquidata.co +echo "Running mysql-client-tests:" bats /mysql-client-tests/mysql-client-tests.bats From e0e4e2fbfbf43a72d740f73b4d35f9e3f52f2bcc Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 27 Aug 2020 16:44:20 -0700 Subject: [PATCH 7/7] /{mysql-client-tests-entrypoint.sh, mysql-client-tests/README.md}: Small fixes, add readme --- mysql-client-tests-entrypoint.sh | 2 +- mysql-client-tests/README.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 mysql-client-tests/README.md diff --git a/mysql-client-tests-entrypoint.sh b/mysql-client-tests-entrypoint.sh index ecaed346ba..9ee3bc7e83 100755 --- a/mysql-client-tests-entrypoint.sh +++ b/mysql-client-tests-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh -echo "updating dolt config for tests:" +echo "Updating dolt config for tests:" dolt config --global --add metrics.disabled true dolt config --global --add metrics.host localhost dolt config --global --add user.name mysql-test-runner diff --git a/mysql-client-tests/README.md b/mysql-client-tests/README.md new file mode 100644 index 0000000000..071add29c5 --- /dev/null +++ b/mysql-client-tests/README.md @@ -0,0 +1,28 @@ +## MySQL Client Tests +We created smoke tests for Dolt's MySQL client integrations and we run these tests through Github Actions +on pull requests. + +These tests can be run locally using Docker. From the root directory of this repo, run: +```bash +$ docker build -t mysql-client-tests -f MySQLDockerfile . +$ docker run mysql-client-tests:latest +``` + +The `docker build` step will take a few minutes to complete as it needs to install all of the +dependencies in the image. + +Running the built container will produce output like: +```bash +$ docker run mysql-client-tests:latest +updating dolt config for tests: +Config successfully updated. +Config successfully updated. +Config successfully updated. +Config successfully updated. +Running mysql-client-tests: +1..4 +ok 1 python mysql.connector client +ok 2 python pymysql client +ok 3 mysql-connector-java client +ok 4 node mysql client +```