fix ORM CI tests to run automatically, add new Prisma branch docker test using @ delimiter, and amend Dockerfile docs

# Conflicts:
#	integration-tests/orm-tests/Dockerfile
This commit is contained in:
elianddb
2026-02-11 17:35:12 -08:00
parent 621a810361
commit 2a6c12bffb
14 changed files with 238 additions and 138 deletions

View File

@@ -1,5 +0,0 @@
name: 'Dolt ORM integration tests'
description: 'Smoke tests for ORM integrations'
runs:
using: 'docker'
image: '../../../integration-tests/ORMDockerfile'

View File

@@ -1,31 +1,48 @@
name: Test ORM integrations
on:
pull_request:
paths:
- 'go/**'
- 'integration-tests/**'
workflow_dispatch:
repository_dispatch:
types: [ test-orm-integrations ]
concurrency:
group: ci-orm-tests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
orm_integrations_job:
runs-on: ubuntu-22.04
timeout-minutes: 30
name: Run tests
name: ORM tests
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Copy go package
run: cp -r ./go ./integration-tests/go
with:
path: dolt
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build ORM test image
uses: docker/build-push-action@v6
with:
context: .
file: dolt/integration-tests/orm-tests/Dockerfile
tags: orm-tests:latest
load: true
- name: Test ORM integrations
uses: ./.github/actions/orm-tests
run: docker run --rm orm-tests:latest
- name: Configure AWS Credentials
if: ${{ failure() }}
if: ${{ failure() && !env.ACT }}
uses: aws-actions/configure-aws-credentials@v4
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Send Email
if: ${{ failure() }}
if: ${{ failure() && !env.ACT }}
uses: ./.github/actions/ses-email-action
with:
template: 'OrmIntegrationFailureTemplate'

View File

@@ -1,64 +0,0 @@
FROM --platform=${BUILDPLATFORM} ubuntu:20.04
# install ORM tools and dependencies
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update -y && \
apt install -y \
curl \
gnupg \
software-properties-common && \
curl -sL https://deb.nodesource.com/setup_22.x | bash -
RUN apt update -y && \
apt install -y \
nodejs \
python3.9 \
python3-pip \
git \
mysql-client \
libmysqlclient-dev \
# weird issue: installing openjdk-17-jdk errors if `maven` or possibly any other package is not installed after it
openjdk-17-jdk \
# currently, `apt install maven` installs v3.6.0 which does not work with openjdk-17-jdk
maven \
bats && \
update-ca-certificates -f
# install go
WORKDIR /root
ENV GO_VERSION=1.25.0
ENV GOPATH=/go
ENV PATH=$PATH:$GOPATH/bin
ENV PATH=$PATH:$GOPATH/bin:/usr/local/go/bin
RUN curl -O "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" && \
sha256sum "go${GO_VERSION}.linux-amd64.tar.gz" && \
tar -xvf "go${GO_VERSION}.linux-amd64.tar.gz" -C /usr/local && \
chown -R root:root /usr/local/go && \
mkdir -p $HOME/go/{bin,src} && \
go version
# install mysql connector and pymsql
RUN pip3 install mysql-connector-python PyMySQL sqlalchemy
# Setup JAVA_HOME -- useful for docker commandline
ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/
# install the current latest maven version, `v3.9.12`, because apt installed one does not work with jdk 17
ADD https://dlcdn.apache.org/maven/maven-3/3.9.12/binaries/apache-maven-3.9.12-bin.tar.gz apache-maven-3.9.12-bin.tar.gz
RUN tar zxvf apache-maven-3.9.12-bin.tar.gz && \
cp -r apache-maven-3.9.12 /opt && \
rm -rf apache-maven-3.9.12 apache-maven-3.9.12-bin.tar.gz
# add maven binary
ENV PATH=/opt/apache-maven-3.9.12/bin:$PATH
# install dolt from source
WORKDIR /root/building
COPY ./go .
ENV GOFLAGS="-mod=readonly"
RUN go build -o /usr/local/bin/dolt ./cmd/dolt
COPY orm-tests /orm-tests
COPY orm-tests/orm-tests-entrypoint.sh /orm-tests/entrypoint.sh
WORKDIR /orm-tests
ENTRYPOINT ["/orm-tests/entrypoint.sh"]

View File

@@ -0,0 +1,68 @@
# syntax=docker/dockerfile:1
FROM golang:1.25-alpine AS golang_cgo125
ENV CGO_ENABLED=1
ENV GO_LDFLAGS="-linkmode external -extldflags '-static'"
RUN apk add --no-cache build-base
FROM golang_cgo125 AS dolt_build
RUN apk add --no-cache icu-dev icu-static
#COPY go-mysql-server /build/go-mysql-server
COPY dolt/go/go.mod /build/dolt/go/
WORKDIR /build/dolt/go/
RUN go mod download
COPY dolt/go/ /build/dolt/go/
RUN go build -tags icu_static -ldflags "$GO_LDFLAGS" -o /build/bin/dolt ./cmd/dolt
FROM --platform=${BUILDPLATFORM} ubuntu:20.04 AS runtime
# install ORM tools and dependencies
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update -y && \
apt install -y \
ca-certificates \
curl \
gnupg \
software-properties-common && \
curl -sL https://deb.nodesource.com/setup_22.x | bash -
RUN apt update -y && \
apt install -y \
nodejs \
python3.9 \
python3-pip \
git \
mysql-client \
libmysqlclient-dev \
netcat-openbsd \
# weird issue: installing openjdk-17-jdk errors if `maven` or possibly any other package is not installed after it
openjdk-17-jdk \
# currently, `apt install maven` installs v3.6.0 which does not work with openjdk-17-jdk
maven && \
update-ca-certificates -f
RUN git clone --depth 1 --branch v1.13.0 https://github.com/bats-core/bats-core.git /tmp/bats-core && \
/tmp/bats-core/install.sh /usr/local && \
rm -rf /tmp/bats-core
# install mysql connector and pymsql
RUN pip3 install mysql-connector-python PyMySQL sqlalchemy
# Setup JAVA_HOME -- useful for docker commandline
ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64/
# install the current latest maven version, `v3.9.11`, because apt installed one does not work with jdk 17
ADD https://archive.apache.org/dist/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz apache-maven-3.9.11-bin.tar.gz
RUN tar zxvf apache-maven-3.9.11-bin.tar.gz && \
cp -r apache-maven-3.9.11 /opt && \
rm -rf apache-maven-3.9.11 apache-maven-3.9.11-bin.tar.gz
# add maven binary
ENV PATH=/opt/apache-maven-3.9.11/bin:$PATH
COPY --from=dolt_build /build/bin/dolt /usr/local/bin/dolt
COPY dolt/integration-tests/orm-tests /orm-tests
COPY dolt/integration-tests/bats/helper /orm-tests/helper
COPY dolt/integration-tests/orm-tests/orm-tests-entrypoint.sh /orm-tests/entrypoint.sh
RUN chmod +x /orm-tests/entrypoint.sh
WORKDIR /orm-tests
ENTRYPOINT ["/orm-tests/entrypoint.sh"]

View File

@@ -4,30 +4,19 @@ These tests verify that various ORM libraries that support MySQL are compatible
we use the same test suite that the ORM provides to test their support with MySQL, but in some
cases we may start with a smaller smoke test to get some quick, initial coverage.
These tests can be run locally using Docker. Before you can build the image, you also need to copy the go folder
into the integration-tests folder; unfortunately just symlinking doesn't seem to work. From the
integration-tests directory of the dolt repo, run:
These tests can be run locally using Docker from the root of the workspace directory that contains `dolt/`:
```bash
$ cp -r ../go .
$ docker build -t orm-tests -f ORMDockerfile .
$ docker build -t orm-tests -f dolt/integration-tests/orm-tests/Dockerfile .
$ docker run orm-tests:latest
```
Running the container should produce output like:
The `dolt` binary is built in a separate stage from the ORM test runtime. This speeds up first-time image builds with parallel stage building. Secondly, ORM-only changes will reuse the cached binary from the `dolt_build` stage.
```bash
Updating dolt config for tests:
Config successfully updated.
Config successfully updated.
Config successfully updated.
Config successfully updated.
Running orm-tests:
1..1
not ok 1 peewee client test
You can also build other Dolt related dependencies from a local source into the Docker `dolt` binary. Copy the required source into the `dolt_build` stage's `build/` directory in the `orm-tests/Dockerfile`. The one caveat is that your repository must be placed in the same parent directory as `dolt`.
```dockerfile
COPY go-mysql-server /build/go-mysql-server
```
The line above has been commented out in the actual Dockerfile so you know the correct placement. You are responsible for updating `go.mod` with the `replace` directive in your local `dolt/` directory.
### Future ORM Libraries to Test
- typeorm
- mikro-orm
- hibernate
To stop the build at a specific stage use [`--target <stage_name>`](https://docs.docker.com/build/building/multi-stage/#stop-at-a-specific-build-stage) option. You can then run the build as a normal image afterward, but it'll run within the target stage. Alternatively, use GoLand's bundled Docker plugin. You may have to add `*Dockerfile` as a pattern in your [File Types](https://www.jetbrains.com/help/go/creating-and-registering-file-types.html#register-new-association) settings.

View File

@@ -10,8 +10,13 @@ public class Util {
static{
try{
sessionFactory = new Configuration().configure().buildSessionFactory();
Configuration config = new Configuration();
config.setProperty("hibernate.connection.url", "jdbc:" + System.getenv("DB_URL"));
config.setProperty("hibernate.connection.username", System.getenv("DB_USER"));
config.setProperty("hibernate.connection.password", System.getenv("DB_PASSWORD"));
sessionFactory = config.configure().buildSessionFactory();
}catch (Throwable ex) {
System.err.println("Session Factory could not be created." + ex);
throw new ExceptionInInitializerError(ex);

View File

@@ -5,10 +5,7 @@
<hibernate-configuration>
<session-factory>
<property name="hibernate.dialect">org.hibernate.dialect.MySQLDialect</property>
<property name="hibernate.connection.driver_class">com.mysql.jdbc.Driver</property>
<property name="hibernate.connection.username">dolt</property>
<property name="hibernate.connection.password"></property>
<property name="hibernate.connection.url">jdbc:mysql://localhost:3306/dolt</property>
<property name="hibernate.connection.driver_class">com.mysql.cj.jdbc.Driver</property>
<property name="show_sql">true</property>
<property name="format_sql">false</property>
<mapping class="com.dolt.hibernate.model.Student"/>

View File

@@ -3,13 +3,12 @@ import { MySqlDriver } from '@mikro-orm/mysql';
import { User } from "./entity/User";
async function connectAndGetOrm() {
const dbUrl = process.env.DB_URL ?? 'mysql://root@localhost:3306/dolt';
const orm = await MikroORM.init<MySqlDriver>({
entities: [User],
type: "mysql",
clientUrl: "mysql://localhost:3306",
dbName: "dolt",
user: "dolt",
password: "",
clientUrl: dbUrl,
persistOnCreate: true,
});

View File

@@ -7,4 +7,4 @@ dolt config --global --add user.name orm-test-runner
dolt config --global --add user.email orm-test-runner@liquidata.co
echo "Running orm-tests:"
bats /orm-tests/orm-tests.bats
bats /orm-tests/orm-tests.bats --show-output-of-passing-tests

View File

@@ -1,27 +1,38 @@
#!/usr/bin/env bats
load helper/common
setup_file() {
export BATS_TEST_RETRIES=5
}
setup() {
REPO_NAME="dolt_repo_$$"
mkdir $REPO_NAME
cd $REPO_NAME
setup_no_dolt_init
REPO_NAME="$(basename "$PWD")"
# TODO(elianddb): flaky dolt init and dolt sql-server lock race
dolt init && flock -w 30 .dolt/noms/LOCK true
USER="dolt"
dolt sql -q "CREATE USER dolt@'%'; GRANT ALL ON *.* TO dolt@'%';"
dolt sql-server --host 0.0.0.0 --loglevel=trace &
SERVER_PID=$!
PORT=$(definePORT)
start_sql_server_with_args_no_port --host 0.0.0.0 --port="$PORT" --loglevel=debug
# Give the server a chance to start
sleep 1
export DB_HOST="127.0.0.1"
export DB_PORT="$PORT"
export DB_USER="root"
export DB_PASSWORD=""
export DB_NAME="$REPO_NAME"
export_DB_URL
export MYSQL_PWD=""
cd ..
export MYSQL_HOST="$DB_HOST"
export MYSQL_TCP_PORT="$DB_PORT"
export MYSQL_PWD="$DB_PASSWORD"
}
export_DB_URL() {
export DB_URL="mysql://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}"
}
teardown() {
cd ..
kill $SERVER_PID
rm -rf $REPO_NAME
rm -f /tmp/mysql.sock
stop_sql_server 1
teardown_common
}
# Peewee is a lightweight ORM library for Python applications. This test checks performs a basic
@@ -49,9 +60,7 @@ teardown() {
# Prisma is an ORM for Node/TypeScript applications. This is a simple smoke test to make sure
# Dolt can support the most basic Prisma operations.
@test "Prisma ORM smoke test" {
mysql --protocol TCP -u dolt -e "create database dolt;"
cd $BATS_TEST_DIRNAME/prisma
cd "$BATS_TEST_DIRNAME"/prisma
npm install
npx -c "prisma migrate dev --name init"
}
@@ -71,8 +80,6 @@ teardown() {
# TypeORM is an ORM for Node/TypeScript applications. This is a simple smoke test to make sure
# Dolt can support the most basic TypeORM operations.
@test "TypeORM smoke test" {
mysql --protocol TCP -u dolt -e "create database dolt;"
cd $BATS_TEST_DIRNAME/typeorm
npm install
npm start
@@ -81,8 +88,6 @@ teardown() {
# MikroORM is an ORM for Node/TypeScript applications. This is a simple smoke test to make sure
# Dolt can support the most basic MikroORM operations.
@test "MikroORM smoke test" {
mysql --protocol TCP -u dolt -e "create database dolt;"
cd $BATS_TEST_DIRNAME/mikro-orm
npm install
npm start
@@ -92,7 +97,7 @@ teardown() {
# Dolt can support the most basic Hibernate operations.
@test "Hibernate smoke test" {
# need to create tables for it before running the test
mysql --protocol TCP -u dolt -e "create database dolt; use dolt; create table STUDENT (id INT NOT NULL auto_increment PRIMARY KEY, first_name VARCHAR(30) NOT NULL, last_name VARCHAR(30) NOT NULL, section VARCHAR(30) NOT NULL);"
mysql -D "$DB_NAME" -e "create table STUDENT (id INT NOT NULL auto_increment PRIMARY KEY, first_name VARCHAR(30) NOT NULL, last_name VARCHAR(30) NOT NULL, section VARCHAR(30) NOT NULL);"
cd $BATS_TEST_DIRNAME/hibernate/DoltHibernateSmokeTest
mvn clean install
@@ -100,8 +105,21 @@ teardown() {
mvn exec:java
}
# Turn this test on to prevent the container from exiting if you need to exec a shell into
# the container to debug failed tests.
#@test "Pause container for an hour to debug failures" {
# sleep 3600
#}
# Prisma is an ORM for Node/TypeScript applications. This test reproduces a migration run against a branch-qualified connection string.
@test "Prisma ORM migration respects branch in connection string" {
dolt branch newbranch
export DB_NAME="${REPO_NAME}@newbranch"
export_DB_URL
cd "$BATS_TEST_DIRNAME"/prisma/branch
run npm install
log_status_eq 0
run npx prisma migrate dev --name init
log_status_eq 0
run mysql -D "$DB_NAME" -e "show tables like 'employees';"
log_status_eq 0
[[ "$output" =~ "employees" ]] || false
}

View File

@@ -0,0 +1,19 @@
{
"name": "prisma-debug",
"version": "1.0.0",
"main": "index.js",
"license": "MIT",
"devDependencies": {
"@types/node": "^22.14.0",
"prisma": "^6.5.0",
"ts-node": "^10.9.2",
"typescript": "^5.8.3"
},
"dependencies": {
"@prisma/client": "^6.5.0",
"csv-parser": "^3.2.0"
},
"prisma": {
"seed": "ts-node --transpile-only ./prisma/seed.ts"
}
}

View File

@@ -0,0 +1,49 @@
// This is your Prisma schema file,
// learn more about it in the docs: https://pris.ly/d/prisma-schema
// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions?
// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init
datasource db {
provider = "mysql"
url = env("DB_URL")
}
generator client {
provider = "prisma-client-js"
}
model Employee {
emp_no Int @id
birth_date DateTime
first_name String @db.VarChar(14)
last_name String @db.VarChar(16)
hire_date DateTime
dept_emps DeptEmp[]
department Department[]
col_to_drop String
@@map("employees")
}
model Department {
dept_no String @id @db.Char(4)
dept_name String @unique @db.VarChar(40)
manager_id Int? // Optional field to store manager's employee number
manager Employee? @relation(fields: [manager_id], references: [emp_no])
dept_emps DeptEmp[]
@@map("departments")
}
model DeptEmp {
emp_no Int
dept_no String @db.Char(4)
from_date DateTime
to_date DateTime
employee Employee @relation(fields: [emp_no], references: [emp_no], onDelete: Cascade)
department Department @relation(fields: [dept_no], references: [dept_no], onDelete: Cascade)
@@id([emp_no, dept_no])
@@map("dept_emp")
}

View File

@@ -1,6 +1,6 @@
datasource db {
provider = "mysql"
url = "mysql://dolt@localhost:3306/dolt"
url = env("DB_URL")
}
model Sample {

View File

@@ -2,13 +2,21 @@ import "reflect-metadata"
import { DataSource } from "typeorm"
import { User } from "./entity/User"
const host = process.env.DB_HOST ?? "localhost"
const portValue = process.env.DB_PORT ?? ""
const parsedPort = Number.parseInt(portValue, 10)
const port = Number.isNaN(parsedPort) ? 3306 : parsedPort
const username = process.env.DB_USER ?? "root"
const password = process.env.DB_PASSWORD ?? ""
const database = process.env.DB_NAME ?? "dolt"
export const AppDataSource = new DataSource({
type: "mysql",
host: "localhost",
port: 3306,
username: "dolt",
password: "",
database: "dolt",
host,
port,
username,
password,
database,
synchronize: true,
logging: false,
entities: [User],