mirror/formbricks
1
0
Fork 0
mirror of https://github.com/formbricks/formbricks.git synced 2026-02-22 10:08:42 -06:00
Code Issues Packages Projects Releases 132 Wiki Activity

fix: migrations speed (#4492)

Browse Source
This commit is contained in:
Anshuman Pandey
2024-12-18 14:25:57 +05:30
committed by GitHub
parent 5d1224e438
commit 28aec8852b
6 changed files with 85 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/database/migration/20241209104738_xm_user_identification/migration.ts
View File

@@ -3,9 +3,9 @@
/* eslint-disable @typescript-eslint/no-unnecessary-condition -- Required for a while loop here */
import { createId } from "@paralleldrive/cuid2";
import { Prisma } from "@prisma/client";
import type { DataMigrationScript } from "../../src/scripts/migration-runner";
import type { MigrationScript } from "../../src/scripts/migration-runner";
export const xmUserIdentification: DataMigrationScript = {
export const xmUserIdentification: MigrationScript = {
type: "data",
id: "n2u5d3wmcw1t2h8a4vgfu2y9",
name: "20241209104738_xm_user_identification",

4
packages/database/migration/20241209110456_xm_segment_migration/migration.ts
View File

@@ -5,7 +5,7 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access -- required for any type */
/* eslint-disable @typescript-eslint/no-explicit-any -- required for any type */
import type { DataMigrationScript } from "../../src/scripts/migration-runner";
import type { MigrationScript } from "../../src/scripts/migration-runner";
export const isResourceFilter = (resource: any): boolean => {
return resource.root !== undefined;
@@ -45,7 +45,7 @@ const findAndReplace = (filters: any): any => {
return newFilters;
};
export const xmSegmentMigration: DataMigrationScript = {
export const xmSegmentMigration: MigrationScript = {
type: "data",
id: "s644oyyqccstfdeejc4fluye",
name: "20241209110456_xm_segment_migration",

4
packages/database/migration/20241209111404_xm_attribute_removal/migration.ts
View File

@@ -1,8 +1,8 @@
/* eslint-disable @typescript-eslint/no-unnecessary-condition -- Required for a while loop here */
import { Prisma } from "@prisma/client";
import type { DataMigrationScript } from "../../src/scripts/migration-runner";
import type { MigrationScript } from "../../src/scripts/migration-runner";
export const xmAttributeRemoval: DataMigrationScript = {
export const xmAttributeRemoval: MigrationScript = {
type: "data",
id: "mq9x7rjdnq0saxoli9pl9b3o",
name: "20241209111404_xm_attribute_removal",

4
packages/database/migration/20241209111525_update_org_limits/migration.ts
View File

@@ -1,4 +1,4 @@
import type { DataMigrationScript } from "../../src/scripts/migration-runner";
import type { MigrationScript } from "../../src/scripts/migration-runner";
type Plan = "free" | "startup" | "scale";
@@ -30,7 +30,7 @@ export const BILLING_LIMITS = {
},
} as const;
export const updateOrgLimits: DataMigrationScript = {
export const updateOrgLimits: MigrationScript = {
type: "data",
id: "ax4otbz2f295rit6kn1jeu8l",
name: "20241209111525_update_org_limits",

4
packages/database/migration/20241209111725_product_revamp/migration.ts
View File

@@ -1,4 +1,4 @@
import type { DataMigrationScript } from "../../src/scripts/migration-runner";
import type { MigrationScript } from "../../src/scripts/migration-runner";
type Plan = "free" | "startup" | "scale" | "enterprise";
@@ -9,7 +9,7 @@ const projectsLimitByPlan: Record<Plan, number | null> = {
enterprise: null,
};
export const productRevamp: DataMigrationScript = {
export const productRevamp: MigrationScript = {
type: "data",
id: "wq3b8pvrvm70nzmsg2647olq",
name: "20241209111725_product_revamp",

154
packages/database/src/scripts/migration-runner.ts
View File

@@ -14,7 +14,7 @@ export interface DataMigrationContext {
>;
}
export interface DataMigrationScript {
export interface MigrationScript {
id?: string;
name: string;
run?: (context: DataMigrationContext) => Promise<void>;
@@ -24,20 +24,24 @@ export interface DataMigrationScript {
const prisma = new PrismaClient();
const TRANSACTION_TIMEOUT = 30 * 60 * 1000; // 30 minutes
const MIGRATIONS_DIR = path.resolve(__dirname, "../../migration");
const PRISMA_MIGRATIONS_DIR = path.resolve(__dirname, "../../migrations");
const runMigrations = async (dataMigrations: DataMigrationScript[]): Promise<void> => {
console.log(`Starting data migrations: ${dataMigrations.length.toString()} to run`);
const runMigrations = async (migrations: MigrationScript[]): Promise<void> => {
console.log(`Starting migrations: ${migrations.length.toString()} to run`);
const startTime = Date.now();
for (const dataMigration of dataMigrations) {
await runSingleMigration(dataMigration);
// empty the prisma migrations directory
await execAsync(`rm -rf ${PRISMA_MIGRATIONS_DIR}/*`);
for (let index = 0; index < migrations.length; index++) {
await runSingleMigration(migrations[index], index);
}
const endTime = Date.now();
console.log(`All data migrations completed in ${((endTime - startTime) / 1000).toFixed(2)}s`);
console.log(`All migrations completed in ${((endTime - startTime) / 1000).toFixed(2)}s`);
};
const runSingleMigration = async (migration: DataMigrationScript): Promise<void> => {
const runSingleMigration = async (migration: MigrationScript, index: number): Promise<void> => {
if (migration.type === "data") {
console.log(`Running data migration: ${migration.name}`);
@@ -105,71 +109,63 @@ const runSingleMigration = async (migration: DataMigrationScript): Promise<void>
throw error;
}
} else {
console.log(`Running schema migration: ${migration.name}`);
try {
console.log(`Running schema migration: ${migration.name}`);
// Original Prisma migrations directory
const originalMigrationsDir = path.resolve(__dirname, "../../migrations");
// Temporary migrations directory for controlled migration
const customMigrationsDir = path.resolve(__dirname, "../../migration");
let copyOnly = false;
// TODO: Check if this can be implemented
// // if the migration directory exists, we will check if the migration has already been applied
if (index > 0) {
const isApplied = await isSchemaMigrationApplied(migration.name, prisma);
// const migrationDir = path.join(originalMigrationsDir, migration.name);
// const hasAccess = await fs
// .access(migrationDir)
// .then(() => true)
// .catch(() => false);
if (isApplied) {
// schema migration is already applied, we can just copy the migration to the original migrations directory
copyOnly = true;
}
}
// if (hasAccess) {
// // Check if there is a migration.sql file in the directory
// const hasSchemaMigration = await fs
// .readdir(migrationDir)
// .then((files) => files.includes("migration.sql"));
// if (hasSchemaMigration) {
// // Check if the migration has already been applied in the database
// const isApplied = await isSchemaMigrationApplied(migration.name, prisma);
// if (isApplied) {
// console.log(`Schema migration ${migration.name} already applied. Skipping...`);
// return;
// }
// }
// }
const originalMigrationsDirExists = await fs
.access(PRISMA_MIGRATIONS_DIR)
.then(() => true)
.catch(() => false);
const originalMigrationsDirExists = await fs
.access(originalMigrationsDir)
.then(() => true)
.catch(() => false);
if (!originalMigrationsDirExists) {
await fs.mkdir(PRISMA_MIGRATIONS_DIR, { recursive: true });
}
if (!originalMigrationsDirExists) {
await fs.mkdir(originalMigrationsDir, { recursive: true });
// Copy specific schema migration from temp migrations directory to original migrations directory
const migrationToCopy = await fs
.readdir(MIGRATIONS_DIR)
.then((files) => files.find((dir) => dir.includes(migration.name)));
if (!migrationToCopy) {
console.error(`Schema migration not found: ${migration.name}`);
return;
}
const sourcePath = path.join(MIGRATIONS_DIR, migrationToCopy);
const destPath = path.join(PRISMA_MIGRATIONS_DIR, migrationToCopy);
// Copy migration folder
await fs.cp(sourcePath, destPath, { recursive: true });
if (copyOnly) {
console.log(`Schema migration ${migration.name} copied to migrations directory`);
return;
}
// Run Prisma migrate
// throws when migrate deploy fails
await execAsync("pnpm prisma migrate deploy");
console.log(`Successfully applied schema migration: ${migration.name}`);
} catch (err) {
console.error(`Schema migration ${migration.name} failed:`, err);
throw err;
}
// Copy specific schema migration from temp migrations directory to original migrations directory
const migrationToCopy = await fs
.readdir(customMigrationsDir)
.then((files) => files.find((dir) => dir.includes(migration.name)));
if (!migrationToCopy) {
console.error(`Schema migration not found: ${migration.name}`);
return;
}
const sourcePath = path.join(customMigrationsDir, migrationToCopy);
const destPath = path.join(originalMigrationsDir, migrationToCopy);
// Copy migration folder
await fs.cp(sourcePath, destPath, { recursive: true });
// Run Prisma migrate
// throws when migrate deploy fails
await execAsync("pnpm prisma migrate deploy");
console.log(`Successfully applied schema migration: ${migration.name}`);
}
};
const loadMigrations = async (): Promise<DataMigrationScript[]> => {
const migrations: DataMigrationScript[] = [];
const loadMigrations = async (): Promise<MigrationScript[]> => {
const migrations: MigrationScript[] = [];
const entries = await fs.readdir(MIGRATIONS_DIR, { withFileTypes: true });
@@ -221,7 +217,7 @@ const loadMigrations = async (): Promise<DataMigrationScript[]> => {
migrations.push({
type: "schema",
name: dirName,
} as DataMigrationScript);
} as MigrationScript);
} else if (hasDataMigration) {
// Check for duplicates among data migrations
if (dataMigrationNames.has(migrationName)) {
@@ -232,7 +228,7 @@ const loadMigrations = async (): Promise<DataMigrationScript[]> => {
// It's a data migration, dynamically import and extract the scripts
const modulePath = path.join(migrationPath, "migration.ts");
const mod = (await import(modulePath)) as Record<string, DataMigrationScript | undefined>;
const mod = (await import(modulePath)) as Record<string, MigrationScript | undefined>;
// Check each export in the module for a DataMigrationScript (type: "data")
for (const key of Object.keys(mod)) {
@@ -277,18 +273,18 @@ export async function applyMigrations(): Promise<void> {
}
}
// async function isSchemaMigrationApplied(migrationName: string, prismaClient: PrismaClient): Promise<boolean> {
// try {
// const applied: unknown[] = await prismaClient.$queryRaw`
// SELECT 1
// FROM _prisma_migrations
// WHERE migration_name = ${migrationName}
// AND finished_at IS NOT NULL
// LIMIT 1;
// `;
// return applied.length > 0;
// } catch (err) {
// console.log("Error: ", err);
// return false;
// }
// }
async function isSchemaMigrationApplied(migrationName: string, prismaClient: PrismaClient): Promise<boolean> {
try {
const applied: unknown[] = await prismaClient.$queryRaw`
SELECT 1
FROM _prisma_migrations
WHERE migration_name = ${migrationName}
AND finished_at IS NOT NULL
LIMIT 1;
`;
return applied.length > 0;
} catch (error: unknown) {
console.error(`Failed to check migration status: ${error as string}`);
throw new Error(`Could not verify migration status: ${error as string}`);
}
}