From 2a6c12bffbc21d5bbffd94d54113aa33044d6ae1 Mon Sep 17 00:00:00 2001 From: elianddb Date: Wed, 11 Feb 2026 17:35:12 -0800 Subject: [PATCH] fix ORM CI tests to run automatically, add new Prisma branch docker test using `@` delimiter, and amend Dockerfile docs # Conflicts: # integration-tests/orm-tests/Dockerfile --- .github/actions/orm-tests/action.yaml | 5 -- .github/workflows/ci-orm-tests.yaml | 29 ++++++-- integration-tests/ORMDockerfile | 64 ---------------- integration-tests/orm-tests/Dockerfile | 68 +++++++++++++++++ integration-tests/orm-tests/README.md | 27 ++----- .../main/java/com/dolt/hibernate/Util.java | 7 +- .../src/main/resources/hibernate.cfg.xml | 5 +- .../orm-tests/mikro-orm/src/index.ts | 7 +- .../orm-tests/orm-tests-entrypoint.sh | 2 +- integration-tests/orm-tests/orm-tests.bats | 74 ++++++++++++------- .../orm-tests/prisma/branch/package.json | 19 +++++ .../prisma/branch/prisma/schema.prisma | 49 ++++++++++++ .../orm-tests/prisma/prisma/schema.prisma | 2 +- .../orm-tests/typeorm/src/data-source.ts | 18 +++-- 14 files changed, 238 insertions(+), 138 deletions(-) delete mode 100644 .github/actions/orm-tests/action.yaml delete mode 100644 integration-tests/ORMDockerfile create mode 100644 integration-tests/orm-tests/Dockerfile create mode 100644 integration-tests/orm-tests/prisma/branch/package.json create mode 100644 integration-tests/orm-tests/prisma/branch/prisma/schema.prisma diff --git a/.github/actions/orm-tests/action.yaml b/.github/actions/orm-tests/action.yaml deleted file mode 100644 index cb33f83095..0000000000 --- a/.github/actions/orm-tests/action.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: 'Dolt ORM integration tests' -description: 'Smoke tests for ORM integrations' -runs: - using: 'docker' - image: '../../../integration-tests/ORMDockerfile' diff --git a/.github/workflows/ci-orm-tests.yaml b/.github/workflows/ci-orm-tests.yaml index 16ba22d98e..4e22a996e2 100644 --- a/.github/workflows/ci-orm-tests.yaml +++ b/.github/workflows/ci-orm-tests.yaml @@ -1,31 +1,48 @@ name: Test ORM integrations on: + pull_request: + paths: + - 'go/**' + - 'integration-tests/**' workflow_dispatch: repository_dispatch: types: [ test-orm-integrations ] +concurrency: + group: ci-orm-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: orm_integrations_job: runs-on: ubuntu-22.04 timeout-minutes: 30 - name: Run tests + name: ORM tests steps: - name: Checkout uses: actions/checkout@v4 - - name: Copy go package - run: cp -r ./go ./integration-tests/go + with: + path: dolt + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build ORM test image + uses: docker/build-push-action@v6 + with: + context: . + file: dolt/integration-tests/orm-tests/Dockerfile + tags: orm-tests:latest + load: true - name: Test ORM integrations - uses: ./.github/actions/orm-tests + run: docker run --rm orm-tests:latest - name: Configure AWS Credentials - if: ${{ failure() }} + if: ${{ failure() && !env.ACT }} uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-west-2 - name: Send Email - if: ${{ failure() }} + if: ${{ failure() && !env.ACT }} uses: ./.github/actions/ses-email-action with: template: 'OrmIntegrationFailureTemplate' diff --git a/integration-tests/ORMDockerfile b/integration-tests/ORMDockerfile deleted file mode 100644 index 5453faf1b8..0000000000 --- a/integration-tests/ORMDockerfile +++ /dev/null @@ -1,64 +0,0 @@ -FROM --platform=${BUILDPLATFORM} ubuntu:20.04 - -# install ORM tools and dependencies -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update -y && \ - apt install -y \ - curl \ - gnupg \ - software-properties-common && \ - curl -sL https://deb.nodesource.com/setup_22.x | bash - -RUN apt update -y && \ - apt install -y \ - nodejs \ - python3.9 \ - python3-pip \ - git \ - mysql-client \ - libmysqlclient-dev \ - # weird issue: installing openjdk-17-jdk errors if `maven` or possibly any other package is not installed after it - openjdk-17-jdk \ - # currently, `apt install maven` installs v3.6.0 which does not work with openjdk-17-jdk - maven \ - bats && \ - update-ca-certificates -f - -# install go -WORKDIR /root -ENV GO_VERSION=1.25.0 -ENV GOPATH=/go -ENV PATH=$PATH:$GOPATH/bin -ENV PATH=$PATH:$GOPATH/bin:/usr/local/go/bin -RUN curl -O "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" && \ - sha256sum "go${GO_VERSION}.linux-amd64.tar.gz" && \ - tar -xvf "go${GO_VERSION}.linux-amd64.tar.gz" -C /usr/local && \ - chown -R root:root /usr/local/go && \ - mkdir -p $HOME/go/{bin,src} && \ - go version - -# install mysql connector and pymsql -RUN pip3 install mysql-connector-python PyMySQL sqlalchemy - -# Setup JAVA_HOME -- useful for docker commandline -ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/ - -# install the current latest maven version, `v3.9.12`, because apt installed one does not work with jdk 17 -ADD https://dlcdn.apache.org/maven/maven-3/3.9.12/binaries/apache-maven-3.9.12-bin.tar.gz apache-maven-3.9.12-bin.tar.gz -RUN tar zxvf apache-maven-3.9.12-bin.tar.gz && \ - cp -r apache-maven-3.9.12 /opt && \ - rm -rf apache-maven-3.9.12 apache-maven-3.9.12-bin.tar.gz - -# add maven binary -ENV PATH=/opt/apache-maven-3.9.12/bin:$PATH - -# install dolt from source -WORKDIR /root/building -COPY ./go . -ENV GOFLAGS="-mod=readonly" -RUN go build -o /usr/local/bin/dolt ./cmd/dolt - -COPY orm-tests /orm-tests -COPY orm-tests/orm-tests-entrypoint.sh /orm-tests/entrypoint.sh - -WORKDIR /orm-tests -ENTRYPOINT ["/orm-tests/entrypoint.sh"] diff --git a/integration-tests/orm-tests/Dockerfile b/integration-tests/orm-tests/Dockerfile new file mode 100644 index 0000000000..43a9d840ab --- /dev/null +++ b/integration-tests/orm-tests/Dockerfile @@ -0,0 +1,68 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.25-alpine AS golang_cgo125 +ENV CGO_ENABLED=1 +ENV GO_LDFLAGS="-linkmode external -extldflags '-static'" +RUN apk add --no-cache build-base + +FROM golang_cgo125 AS dolt_build +RUN apk add --no-cache icu-dev icu-static +#COPY go-mysql-server /build/go-mysql-server +COPY dolt/go/go.mod /build/dolt/go/ +WORKDIR /build/dolt/go/ +RUN go mod download +COPY dolt/go/ /build/dolt/go/ +RUN go build -tags icu_static -ldflags "$GO_LDFLAGS" -o /build/bin/dolt ./cmd/dolt + +FROM --platform=${BUILDPLATFORM} ubuntu:20.04 AS runtime + +# install ORM tools and dependencies +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update -y && \ + apt install -y \ + ca-certificates \ + curl \ + gnupg \ + software-properties-common && \ + curl -sL https://deb.nodesource.com/setup_22.x | bash - +RUN apt update -y && \ + apt install -y \ + nodejs \ + python3.9 \ + python3-pip \ + git \ + mysql-client \ + libmysqlclient-dev \ + netcat-openbsd \ + # weird issue: installing openjdk-17-jdk errors if `maven` or possibly any other package is not installed after it + openjdk-17-jdk \ + # currently, `apt install maven` installs v3.6.0 which does not work with openjdk-17-jdk + maven && \ + update-ca-certificates -f + +RUN git clone --depth 1 --branch v1.13.0 https://github.com/bats-core/bats-core.git /tmp/bats-core && \ + /tmp/bats-core/install.sh /usr/local && \ + rm -rf /tmp/bats-core + +# install mysql connector and pymsql +RUN pip3 install mysql-connector-python PyMySQL sqlalchemy + +# Setup JAVA_HOME -- useful for docker commandline +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/ + +# install the current latest maven version, `v3.9.11`, because apt installed one does not work with jdk 17 +ADD https://archive.apache.org/dist/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz apache-maven-3.9.11-bin.tar.gz +RUN tar zxvf apache-maven-3.9.11-bin.tar.gz && \ + cp -r apache-maven-3.9.11 /opt && \ + rm -rf apache-maven-3.9.11 apache-maven-3.9.11-bin.tar.gz + +# add maven binary +ENV PATH=/opt/apache-maven-3.9.11/bin:$PATH + +COPY --from=dolt_build /build/bin/dolt /usr/local/bin/dolt +COPY dolt/integration-tests/orm-tests /orm-tests +COPY dolt/integration-tests/bats/helper /orm-tests/helper +COPY dolt/integration-tests/orm-tests/orm-tests-entrypoint.sh /orm-tests/entrypoint.sh +RUN chmod +x /orm-tests/entrypoint.sh + +WORKDIR /orm-tests +ENTRYPOINT ["/orm-tests/entrypoint.sh"] diff --git a/integration-tests/orm-tests/README.md b/integration-tests/orm-tests/README.md index 7d3e241cdd..bc843fea99 100644 --- a/integration-tests/orm-tests/README.md +++ b/integration-tests/orm-tests/README.md @@ -4,30 +4,19 @@ These tests verify that various ORM libraries that support MySQL are compatible we use the same test suite that the ORM provides to test their support with MySQL, but in some cases we may start with a smaller smoke test to get some quick, initial coverage. -These tests can be run locally using Docker. Before you can build the image, you also need to copy the go folder -into the integration-tests folder; unfortunately just symlinking doesn't seem to work. From the -integration-tests directory of the dolt repo, run: +These tests can be run locally using Docker from the root of the workspace directory that contains `dolt/`: ```bash -$ cp -r ../go . -$ docker build -t orm-tests -f ORMDockerfile . +$ docker build -t orm-tests -f dolt/integration-tests/orm-tests/Dockerfile . $ docker run orm-tests:latest ``` -Running the container should produce output like: +The `dolt` binary is built in a separate stage from the ORM test runtime. This speeds up first-time image builds with parallel stage building. Secondly, ORM-only changes will reuse the cached binary from the `dolt_build` stage. -```bash -Updating dolt config for tests: -Config successfully updated. -Config successfully updated. -Config successfully updated. -Config successfully updated. -Running orm-tests: -1..1 -not ok 1 peewee client test +You can also build other Dolt related dependencies from a local source into the Docker `dolt` binary. Copy the required source into the `dolt_build` stage's `build/` directory in the `orm-tests/Dockerfile`. The one caveat is that your repository must be placed in the same parent directory as `dolt`. +```dockerfile +COPY go-mysql-server /build/go-mysql-server ``` +The line above has been commented out in the actual Dockerfile so you know the correct placement. You are responsible for updating `go.mod` with the `replace` directive in your local `dolt/` directory. -### Future ORM Libraries to Test -- typeorm -- mikro-orm -- hibernate +To stop the build at a specific stage use [`--target `](https://docs.docker.com/build/building/multi-stage/#stop-at-a-specific-build-stage) option. You can then run the build as a normal image afterward, but it'll run within the target stage. Alternatively, use GoLand's bundled Docker plugin. You may have to add `*Dockerfile` as a pattern in your [File Types](https://www.jetbrains.com/help/go/creating-and-registering-file-types.html#register-new-association) settings. \ No newline at end of file diff --git a/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/java/com/dolt/hibernate/Util.java b/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/java/com/dolt/hibernate/Util.java index eefe82df20..c7417cdb47 100644 --- a/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/java/com/dolt/hibernate/Util.java +++ b/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/java/com/dolt/hibernate/Util.java @@ -10,8 +10,13 @@ public class Util { static{ try{ - sessionFactory = new Configuration().configure().buildSessionFactory(); + Configuration config = new Configuration(); + config.setProperty("hibernate.connection.url", "jdbc:" + System.getenv("DB_URL")); + config.setProperty("hibernate.connection.username", System.getenv("DB_USER")); + config.setProperty("hibernate.connection.password", System.getenv("DB_PASSWORD")); + + sessionFactory = config.configure().buildSessionFactory(); }catch (Throwable ex) { System.err.println("Session Factory could not be created." + ex); throw new ExceptionInInitializerError(ex); diff --git a/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/resources/hibernate.cfg.xml b/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/resources/hibernate.cfg.xml index 71cba18ccc..0a40ed6837 100644 --- a/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/resources/hibernate.cfg.xml +++ b/integration-tests/orm-tests/hibernate/DoltHibernateSmokeTest/src/main/resources/hibernate.cfg.xml @@ -5,10 +5,7 @@ org.hibernate.dialect.MySQLDialect - com.mysql.jdbc.Driver - dolt - - jdbc:mysql://localhost:3306/dolt + com.mysql.cj.jdbc.Driver true false diff --git a/integration-tests/orm-tests/mikro-orm/src/index.ts b/integration-tests/orm-tests/mikro-orm/src/index.ts index 83387db06f..99c6b5a760 100644 --- a/integration-tests/orm-tests/mikro-orm/src/index.ts +++ b/integration-tests/orm-tests/mikro-orm/src/index.ts @@ -3,13 +3,12 @@ import { MySqlDriver } from '@mikro-orm/mysql'; import { User } from "./entity/User"; async function connectAndGetOrm() { + const dbUrl = process.env.DB_URL ?? 'mysql://root@localhost:3306/dolt'; + const orm = await MikroORM.init({ entities: [User], type: "mysql", - clientUrl: "mysql://localhost:3306", - dbName: "dolt", - user: "dolt", - password: "", + clientUrl: dbUrl, persistOnCreate: true, }); diff --git a/integration-tests/orm-tests/orm-tests-entrypoint.sh b/integration-tests/orm-tests/orm-tests-entrypoint.sh index 51e3c19ae4..9632e02e43 100755 --- a/integration-tests/orm-tests/orm-tests-entrypoint.sh +++ b/integration-tests/orm-tests/orm-tests-entrypoint.sh @@ -7,4 +7,4 @@ dolt config --global --add user.name orm-test-runner dolt config --global --add user.email orm-test-runner@liquidata.co echo "Running orm-tests:" -bats /orm-tests/orm-tests.bats +bats /orm-tests/orm-tests.bats --show-output-of-passing-tests diff --git a/integration-tests/orm-tests/orm-tests.bats b/integration-tests/orm-tests/orm-tests.bats index 924ec57a20..f79edc2da3 100644 --- a/integration-tests/orm-tests/orm-tests.bats +++ b/integration-tests/orm-tests/orm-tests.bats @@ -1,27 +1,38 @@ #!/usr/bin/env bats +load helper/common + +setup_file() { + export BATS_TEST_RETRIES=5 +} setup() { - REPO_NAME="dolt_repo_$$" - mkdir $REPO_NAME - cd $REPO_NAME + setup_no_dolt_init + REPO_NAME="$(basename "$PWD")" + # TODO(elianddb): flaky dolt init and dolt sql-server lock race + dolt init && flock -w 30 .dolt/noms/LOCK true - USER="dolt" - dolt sql -q "CREATE USER dolt@'%'; GRANT ALL ON *.* TO dolt@'%';" - dolt sql-server --host 0.0.0.0 --loglevel=trace & - SERVER_PID=$! + PORT=$(definePORT) + start_sql_server_with_args_no_port --host 0.0.0.0 --port="$PORT" --loglevel=debug - # Give the server a chance to start - sleep 1 + export DB_HOST="127.0.0.1" + export DB_PORT="$PORT" + export DB_USER="root" + export DB_PASSWORD="" + export DB_NAME="$REPO_NAME" + export_DB_URL - export MYSQL_PWD="" - cd .. + export MYSQL_HOST="$DB_HOST" + export MYSQL_TCP_PORT="$DB_PORT" + export MYSQL_PWD="$DB_PASSWORD" +} + +export_DB_URL() { + export DB_URL="mysql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}" } teardown() { - cd .. - kill $SERVER_PID - rm -rf $REPO_NAME - rm -f /tmp/mysql.sock + stop_sql_server 1 + teardown_common } # Peewee is a lightweight ORM library for Python applications. This test checks performs a basic @@ -49,9 +60,7 @@ teardown() { # Prisma is an ORM for Node/TypeScript applications. This is a simple smoke test to make sure # Dolt can support the most basic Prisma operations. @test "Prisma ORM smoke test" { - mysql --protocol TCP -u dolt -e "create database dolt;" - - cd $BATS_TEST_DIRNAME/prisma + cd "$BATS_TEST_DIRNAME"/prisma npm install npx -c "prisma migrate dev --name init" } @@ -71,8 +80,6 @@ teardown() { # TypeORM is an ORM for Node/TypeScript applications. This is a simple smoke test to make sure # Dolt can support the most basic TypeORM operations. @test "TypeORM smoke test" { - mysql --protocol TCP -u dolt -e "create database dolt;" - cd $BATS_TEST_DIRNAME/typeorm npm install npm start @@ -81,8 +88,6 @@ teardown() { # MikroORM is an ORM for Node/TypeScript applications. This is a simple smoke test to make sure # Dolt can support the most basic MikroORM operations. @test "MikroORM smoke test" { - mysql --protocol TCP -u dolt -e "create database dolt;" - cd $BATS_TEST_DIRNAME/mikro-orm npm install npm start @@ -92,7 +97,7 @@ teardown() { # Dolt can support the most basic Hibernate operations. @test "Hibernate smoke test" { # need to create tables for it before running the test - mysql --protocol TCP -u dolt -e "create database dolt; use dolt; create table STUDENT (id INT NOT NULL auto_increment PRIMARY KEY, first_name VARCHAR(30) NOT NULL, last_name VARCHAR(30) NOT NULL, section VARCHAR(30) NOT NULL);" + mysql -D "$DB_NAME" -e "create table STUDENT (id INT NOT NULL auto_increment PRIMARY KEY, first_name VARCHAR(30) NOT NULL, last_name VARCHAR(30) NOT NULL, section VARCHAR(30) NOT NULL);" cd $BATS_TEST_DIRNAME/hibernate/DoltHibernateSmokeTest mvn clean install @@ -100,8 +105,21 @@ teardown() { mvn exec:java } -# Turn this test on to prevent the container from exiting if you need to exec a shell into -# the container to debug failed tests. -#@test "Pause container for an hour to debug failures" { -# sleep 3600 -#} \ No newline at end of file +# Prisma is an ORM for Node/TypeScript applications. This test reproduces a migration run against a branch-qualified connection string. +@test "Prisma ORM migration respects branch in connection string" { + dolt branch newbranch + + export DB_NAME="${REPO_NAME}@newbranch" + export_DB_URL + + cd "$BATS_TEST_DIRNAME"/prisma/branch + run npm install + log_status_eq 0 + + run npx prisma migrate dev --name init + log_status_eq 0 + + run mysql -D "$DB_NAME" -e "show tables like 'employees';" + log_status_eq 0 + [[ "$output" =~ "employees" ]] || false +} diff --git a/integration-tests/orm-tests/prisma/branch/package.json b/integration-tests/orm-tests/prisma/branch/package.json new file mode 100644 index 0000000000..69a7fbda93 --- /dev/null +++ b/integration-tests/orm-tests/prisma/branch/package.json @@ -0,0 +1,19 @@ +{ + "name": "prisma-debug", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "devDependencies": { + "@types/node": "^22.14.0", + "prisma": "^6.5.0", + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + }, + "dependencies": { + "@prisma/client": "^6.5.0", + "csv-parser": "^3.2.0" + }, + "prisma": { + "seed": "ts-node --transpile-only ./prisma/seed.ts" + } +} diff --git a/integration-tests/orm-tests/prisma/branch/prisma/schema.prisma b/integration-tests/orm-tests/prisma/branch/prisma/schema.prisma new file mode 100644 index 0000000000..9aee41bd7f --- /dev/null +++ b/integration-tests/orm-tests/prisma/branch/prisma/schema.prisma @@ -0,0 +1,49 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions? +// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init + +datasource db { + provider = "mysql" + url = env("DB_URL") +} + +generator client { + provider = "prisma-client-js" +} + +model Employee { + emp_no Int @id + birth_date DateTime + first_name String @db.VarChar(14) + last_name String @db.VarChar(16) + hire_date DateTime + dept_emps DeptEmp[] + department Department[] + col_to_drop String + + @@map("employees") +} + +model Department { + dept_no String @id @db.Char(4) + dept_name String @unique @db.VarChar(40) + manager_id Int? // Optional field to store manager's employee number + manager Employee? @relation(fields: [manager_id], references: [emp_no]) + dept_emps DeptEmp[] + + @@map("departments") +} + +model DeptEmp { + emp_no Int + dept_no String @db.Char(4) + from_date DateTime + to_date DateTime + employee Employee @relation(fields: [emp_no], references: [emp_no], onDelete: Cascade) + department Department @relation(fields: [dept_no], references: [dept_no], onDelete: Cascade) + + @@id([emp_no, dept_no]) + @@map("dept_emp") +} diff --git a/integration-tests/orm-tests/prisma/prisma/schema.prisma b/integration-tests/orm-tests/prisma/prisma/schema.prisma index aeb101dd98..61293f1cd4 100644 --- a/integration-tests/orm-tests/prisma/prisma/schema.prisma +++ b/integration-tests/orm-tests/prisma/prisma/schema.prisma @@ -1,6 +1,6 @@ datasource db { provider = "mysql" - url = "mysql://dolt@localhost:3306/dolt" + url = env("DB_URL") } model Sample { diff --git a/integration-tests/orm-tests/typeorm/src/data-source.ts b/integration-tests/orm-tests/typeorm/src/data-source.ts index c75a156e89..fc75c3a12f 100644 --- a/integration-tests/orm-tests/typeorm/src/data-source.ts +++ b/integration-tests/orm-tests/typeorm/src/data-source.ts @@ -2,13 +2,21 @@ import "reflect-metadata" import { DataSource } from "typeorm" import { User } from "./entity/User" +const host = process.env.DB_HOST ?? "localhost" +const portValue = process.env.DB_PORT ?? "" +const parsedPort = Number.parseInt(portValue, 10) +const port = Number.isNaN(parsedPort) ? 3306 : parsedPort +const username = process.env.DB_USER ?? "root" +const password = process.env.DB_PASSWORD ?? "" +const database = process.env.DB_NAME ?? "dolt" + export const AppDataSource = new DataSource({ type: "mysql", - host: "localhost", - port: 3306, - username: "dolt", - password: "", - database: "dolt", + host, + port, + username, + password, + database, synchronize: true, logging: false, entities: [User],