chore: Add a mutex lock around migrations to ensure in multi-instance deployments multiple machines don't attempt to run migrations at once (#10560)

server/utils/CacheHelper.ts

@@ -38,7 +38,7 @@ export class CacheHelper {
     const lockKey = `lock:${key}`;
     try {
       try {
-        lock = await MutexLock.lock.acquire([lockKey], lockTimeout);
+        lock = await MutexLock.acquire(lockKey, lockTimeout);
       } catch (err) {
         Logger.error(`Could not acquire lock for ${key}`, err);
       }
@@ -54,8 +54,8 @@ export class CacheHelper {
       }
       return value;
     } finally {
-      if (lock && lock.expiration > new Date().getTime()) {
-        await lock.release();
+      if (lock) {
+        await MutexLock.release(lock);
       }
     }
   }

server/utils/MutexLock.ts

@@ -1,4 +1,4 @@
-import Redlock from "redlock";
+import Redlock, { type Lock } from "redlock";
 import Redis from "@server/storage/redis";
 
 export class MutexLock {
@@ -18,5 +18,28 @@ export class MutexLock {
     return this.redlock;
   }
 
+  /**
+   * Acquire a Mutex lock
+   *
+   * @param resource The resource to lock
+   * @param timeout The duration to acquire the lock for if not released in milliseconds
+   * @returns A promise that resolves a to a Lock
+   */
+  public static acquire(resource: string, timeout: number) {
+    return this.lock.acquire([resource], timeout);
+  }
+
+  /**
+   * Safely release a lock
+   *
+   * @param lock The lock to release
+   */
+  public static release(lock: Lock) {
+    if (lock && lock.expiration > new Date().getTime()) {
+      return lock.release();
+    }
+    return false;
+  }
+
   private static redlock: Redlock;
 }

server/utils/startup.ts

@@ -6,9 +6,13 @@ import AuthenticationProvider from "@server/models/AuthenticationProvider";
 import Team from "@server/models/Team";
 import { migrations } from "@server/storage/database";
 import { getArg } from "./args";
+import { MutexLock } from "./MutexLock";
+import { Minute } from "@shared/utils/time";
 
 export async function checkPendingMigrations() {
+  let lock;
   try {
+    lock = await MutexLock.acquire("migrations", 10 * Minute.ms);
     const pending = await migrations.pending();
     if (!isEmpty(pending)) {
       if (getArg("no-migrate")) {
@@ -35,6 +39,10 @@ export async function checkPendingMigrations() {
       Logger.warn(chalk.red(err.message));
     }
     process.exit(1);
+  } finally {
+    if (lock) {
+      await MutexLock.release(lock);
+    }
   }
 }