From 935e24bd43bf812f102444566b4e2b43830a0d43 Mon Sep 17 00:00:00 2001 From: Victor Hugo dos Santos <115753265+victorvhs017@users.noreply.github.com> Date: Fri, 12 Sep 2025 08:16:13 -0300 Subject: [PATCH] chore: clean-up new cache package (#6532) --- .cursor/rules/cache-optimization.mdc | 19 +++--- .github/workflows/docker-build-validation.yml | 34 ++++++++++- .github/workflows/e2e.yml | 8 +-- apps/web/Dockerfile | 1 + apps/web/lib/constants.ts | 1 - apps/web/lib/env.ts | 7 ++- apps/web/lib/utils/logger-helpers.test.ts | 33 +++++------ apps/web/lib/utils/rate-limit.test.ts | 58 ------------------- apps/web/lib/utils/rate-limit.ts | 54 ----------------- apps/web/locales/de-DE.json | 1 - apps/web/locales/en-US.json | 1 - apps/web/locales/fr-FR.json | 1 - apps/web/locales/ja-JP.json | 1 - apps/web/locales/pt-BR.json | 1 - apps/web/locales/pt-PT.json | 1 - apps/web/locales/ro-RO.json | 1 - apps/web/locales/zh-Hans-CN.json | 1 - apps/web/locales/zh-Hant-TW.json | 1 - apps/web/package.json | 3 - apps/web/scripts/docker/read-secrets.sh | 13 ++++- docker-compose.dev.yml | 2 +- docker/docker-compose.yml | 28 +++++++-- .../configuration/environment-variables.mdx | 5 +- docs/self-hosting/setup/cluster-setup.mdx | 5 +- pnpm-lock.yaml | 50 ---------------- turbo.json | 1 - 26 files changed, 101 insertions(+), 230 deletions(-) delete mode 100644 apps/web/lib/utils/rate-limit.test.ts delete mode 100644 apps/web/lib/utils/rate-limit.ts diff --git a/.cursor/rules/cache-optimization.mdc b/.cursor/rules/cache-optimization.mdc index 06f780df73..3776348eef 100644 --- a/.cursor/rules/cache-optimization.mdc +++ b/.cursor/rules/cache-optimization.mdc @@ -16,9 +16,10 @@ Formbricks uses a **hybrid caching approach** optimized for enterprise scale: ## Key Files ### Core Cache Infrastructure -- [apps/web/modules/cache/lib/service.ts](mdc:apps/web/modules/cache/lib/service.ts) - Redis cache service -- [apps/web/modules/cache/lib/withCache.ts](mdc:apps/web/modules/cache/lib/withCache.ts) - Cache wrapper utilities -- [apps/web/modules/cache/lib/cacheKeys.ts](mdc:apps/web/modules/cache/lib/cacheKeys.ts) - Enterprise cache key patterns and utilities +- [packages/cache/src/service.ts](mdc:packages/cache/src/service.ts) - Redis cache service +- [packages/cache/src/client.ts](mdc:packages/cache/src/client.ts) - Cache client initialization and singleton management +- [apps/web/lib/cache/index.ts](mdc:apps/web/lib/cache/index.ts) - Cache service proxy for web app +- [packages/cache/src/index.ts](mdc:packages/cache/src/index.ts) - Cache package exports and utilities ### Environment State Caching (Critical Endpoint) - [apps/web/app/api/v1/client/[environmentId]/environment/route.ts](mdc:apps/web/app/api/v1/client/[environmentId]/environment/route.ts) - Main endpoint serving hundreds of thousands of SDK clients @@ -26,7 +27,7 @@ Formbricks uses a **hybrid caching approach** optimized for enterprise scale: ## Enterprise-Grade Cache Key Patterns -**Always use** the `createCacheKey` utilities from [cacheKeys.ts](mdc:apps/web/modules/cache/lib/cacheKeys.ts): +**Always use** the `createCacheKey` utilities from the cache package: ```typescript // ✅ Correct patterns @@ -50,14 +51,14 @@ export const getEnterpriseLicense = reactCache(async () => { }); ``` -### Use `withCache()` for Simple Database Queries +### Use `cache.withCache()` for Simple Database Queries ```typescript // ✅ Simple caching with automatic fallback (TTL in milliseconds) export const getActionClasses = (environmentId: string) => { - return withCache(() => fetchActionClassesFromDB(environmentId), { - key: createCacheKey.environment.actionClasses(environmentId), - ttl: 60 * 30 * 1000, // 30 minutes in milliseconds - })(); + return cache.withCache(() => fetchActionClassesFromDB(environmentId), + createCacheKey.environment.actionClasses(environmentId), + 60 * 30 * 1000 // 30 minutes in milliseconds + ); }; ``` diff --git a/.github/workflows/docker-build-validation.yml b/.github/workflows/docker-build-validation.yml index d2d6192dd0..8fc29e3a49 100644 --- a/.github/workflows/docker-build-validation.yml +++ b/.github/workflows/docker-build-validation.yml @@ -21,10 +21,10 @@ jobs: name: Validate Docker Build runs-on: ubuntu-latest - # Add PostgreSQL service container + # Add PostgreSQL and Redis service containers services: postgres: - image: pgvector/pgvector:pg17 + image: pgvector/pgvector@sha256:9ae02a756ba16a2d69dd78058e25915e36e189bb36ddf01ceae86390d7ed786a env: POSTGRES_USER: test POSTGRES_PASSWORD: test @@ -38,6 +38,11 @@ jobs: --health-timeout 5s --health-retries 5 + redis: + image: valkey/valkey@sha256:12ba4f45a7c3e1d0f076acd616cb230834e75a77e8516dde382720af32832d6d + ports: + - 6379:6379 + steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 @@ -67,6 +72,7 @@ jobs: secrets: | database_url=${{ secrets.DUMMY_DATABASE_URL }} encryption_key=${{ secrets.DUMMY_ENCRYPTION_KEY }} + redis_url=redis://localhost:6379 - name: Verify and Initialize PostgreSQL run: | @@ -96,6 +102,29 @@ jobs: echo "Network configuration:" netstat -tulpn | grep 5432 || echo "No process listening on port 5432" + - name: Verify Redis/Valkey Connection + run: | + echo "Verifying Redis/Valkey connection..." + # Install Redis client to test connection + sudo apt-get update && sudo apt-get install -y redis-tools + + # Test connection using redis-cli with timeout and proper error handling + echo "Testing Redis connection with 30 second timeout..." + if timeout 30 bash -c 'until redis-cli -h localhost -p 6379 ping >/dev/null 2>&1; do + echo "Waiting for Redis to be ready..." + sleep 2 + done'; then + echo "✅ Redis connection successful" + redis-cli -h localhost -p 6379 info server | head -5 + else + echo "❌ Redis connection failed after 30 seconds" + exit 1 + fi + + # Show network configuration for Redis + echo "Redis network configuration:" + netstat -tulpn | grep 6379 || echo "No process listening on port 6379" + - name: Test Docker Image with Health Check shell: bash env: @@ -113,6 +142,7 @@ jobs: -p 3000:3000 \ -e DATABASE_URL="postgresql://test:test@host.docker.internal:5432/formbricks" \ -e ENCRYPTION_KEY="$DUMMY_ENCRYPTION_KEY" \ + -e REDIS_URL="redis://host.docker.internal:6379" \ -d "formbricks-test:$GITHUB_SHA" # Start health check polling immediately (every 5 seconds for up to 5 minutes) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 85ac75424e..d9d5c4b389 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -46,15 +46,9 @@ jobs: --health-timeout=5s --health-retries=5 valkey: - image: valkey/valkey:8.1.1 + image: valkey/valkey@sha256:12ba4f45a7c3e1d0f076acd616cb230834e75a77e8516dde382720af32832d6d ports: - 6379:6379 - options: >- - --entrypoint "valkey-server" - --health-cmd="valkey-cli ping" - --health-interval=10s - --health-timeout=5s - --health-retries=5 minio: image: bitnami/minio:2025.7.23-debian-12-r5 env: diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 57852becc1..5af6d18de2 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -61,6 +61,7 @@ RUN pnpm build --filter=@formbricks/database # This mounts the secrets only during this build step without storing them in layers RUN --mount=type=secret,id=database_url \ --mount=type=secret,id=encryption_key \ + --mount=type=secret,id=redis_url \ --mount=type=secret,id=sentry_auth_token \ /tmp/read-secrets.sh pnpm build --filter=@formbricks/web... diff --git a/apps/web/lib/constants.ts b/apps/web/lib/constants.ts index 279c7fd221..7090d93292 100644 --- a/apps/web/lib/constants.ts +++ b/apps/web/lib/constants.ts @@ -151,7 +151,6 @@ export const DEBUG = env.DEBUG === "1"; export const ENTERPRISE_LICENSE_KEY = env.ENTERPRISE_LICENSE_KEY; export const REDIS_URL = env.REDIS_URL; -export const REDIS_HTTP_URL = env.REDIS_HTTP_URL; export const RATE_LIMITING_DISABLED = env.RATE_LIMITING_DISABLED === "1"; export const BREVO_API_KEY = env.BREVO_API_KEY; diff --git a/apps/web/lib/env.ts b/apps/web/lib/env.ts index 9d09784149..bbd6808a44 100644 --- a/apps/web/lib/env.ts +++ b/apps/web/lib/env.ts @@ -54,8 +54,10 @@ export const env = createEnv({ OIDC_ISSUER: z.string().optional(), OIDC_SIGNING_ALGORITHM: z.string().optional(), OPENTELEMETRY_LISTENER_URL: z.string().optional(), - REDIS_URL: z.string().optional(), - REDIS_HTTP_URL: z.string().optional(), + REDIS_URL: + process.env.NODE_ENV === "test" + ? z.string().optional() + : z.string().url("REDIS_URL is required for caching, rate limiting, and audit logging"), PASSWORD_RESET_DISABLED: z.enum(["1", "0"]).optional(), POSTHOG_API_HOST: z.string().optional(), POSTHOG_API_KEY: z.string().optional(), @@ -182,7 +184,6 @@ export const env = createEnv({ OIDC_ISSUER: process.env.OIDC_ISSUER, OIDC_SIGNING_ALGORITHM: process.env.OIDC_SIGNING_ALGORITHM, REDIS_URL: process.env.REDIS_URL, - REDIS_HTTP_URL: process.env.REDIS_HTTP_URL, PASSWORD_RESET_DISABLED: process.env.PASSWORD_RESET_DISABLED, PRIVACY_URL: process.env.PRIVACY_URL, RATE_LIMITING_DISABLED: process.env.RATE_LIMITING_DISABLED, diff --git a/apps/web/lib/utils/logger-helpers.test.ts b/apps/web/lib/utils/logger-helpers.test.ts index c7e4d99302..3348d92afd 100644 --- a/apps/web/lib/utils/logger-helpers.test.ts +++ b/apps/web/lib/utils/logger-helpers.test.ts @@ -1,15 +1,21 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { deepDiff, redactPII, sanitizeUrlForLogging } from "./logger-helpers"; -// Patch redis multi before any imports +// Patch cache before any imports beforeEach(async () => { - const redis = (await import("@/modules/cache/redis")).default; - if ((redis?.multi as any)?.mockReturnValue) { - (redis?.multi as any).mockReturnValue({ - set: vi.fn(), - exec: vi.fn().mockResolvedValue([["OK"]]), - }); - } + // Mock the cache service for tests + vi.doMock("@/lib/cache", () => ({ + cache: { + getRedisClient: vi.fn().mockResolvedValue({ + multi: vi.fn().mockReturnValue({ + set: vi.fn(), + exec: vi.fn().mockResolvedValue([["OK"]]), + }), + watch: vi.fn().mockResolvedValue("OK"), + get: vi.fn().mockResolvedValue(null), + }), + }, + })); }); vi.mock("@/modules/ee/license-check/lib/utils", () => ({ @@ -37,16 +43,7 @@ vi.mock("@/modules/ee/audit-logs/lib/service", () => ({ logAuditEvent: vi.fn().mockResolvedValue(undefined), })); -vi.mock("@/modules/cache/redis", () => ({ - default: { - watch: vi.fn().mockResolvedValue("OK"), - multi: vi.fn().mockReturnValue({ - set: vi.fn(), - exec: vi.fn().mockResolvedValue([["OK"]]), - }), - get: vi.fn().mockResolvedValue(null), - }, -})); +// Cache mock is handled in beforeEach above // Set ENCRYPTION_KEY for all tests unless explicitly testing its absence process.env.ENCRYPTION_KEY = "testsecret"; diff --git a/apps/web/lib/utils/rate-limit.test.ts b/apps/web/lib/utils/rate-limit.test.ts deleted file mode 100644 index 90c6bb1069..0000000000 --- a/apps/web/lib/utils/rate-limit.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from "vitest"; - -describe("in-memory rate limiter", () => { - test("allows requests within limit and throws after limit", async () => { - const { rateLimit } = await import("./rate-limit"); - const limiterFn = rateLimit({ interval: 1, allowedPerInterval: 2 }); - await expect(limiterFn("a")).resolves.toBeUndefined(); - await expect(limiterFn("a")).resolves.toBeUndefined(); - await expect(limiterFn("a")).rejects.toThrow("Rate limit exceeded"); - }); - - test("separate tokens have separate counts", async () => { - const { rateLimit } = await import("./rate-limit"); - const limiterFn = rateLimit({ interval: 1, allowedPerInterval: 2 }); - await expect(limiterFn("x")).resolves.toBeUndefined(); - await expect(limiterFn("y")).resolves.toBeUndefined(); - await expect(limiterFn("x")).resolves.toBeUndefined(); - await expect(limiterFn("y")).resolves.toBeUndefined(); - }); -}); - -describe("redis rate limiter", () => { - beforeEach(async () => { - vi.resetModules(); - const constants = await vi.importActual("@/lib/constants"); - vi.doMock("@/lib/constants", () => ({ - ...constants, - REDIS_HTTP_URL: "http://redis", - })); - }); - - test("sets expire on first use and does not throw", async () => { - global.fetch = vi - .fn() - .mockResolvedValueOnce({ ok: true, json: async () => ({ INCR: 1 }) }) - .mockResolvedValueOnce({ ok: true }); - const { rateLimit } = await import("./rate-limit"); - const limiter = rateLimit({ interval: 10, allowedPerInterval: 2 }); - await expect(limiter("t")).resolves.toBeUndefined(); - expect(fetch).toHaveBeenCalledTimes(2); - expect(fetch).toHaveBeenCalledWith("http://redis/INCR/t"); - expect(fetch).toHaveBeenCalledWith("http://redis/EXPIRE/t/10"); - }); - - test("does not throw when redis INCR response not ok", async () => { - global.fetch = vi.fn().mockResolvedValueOnce({ ok: false }); - const { rateLimit } = await import("./rate-limit"); - const limiter = rateLimit({ interval: 10, allowedPerInterval: 2 }); - await expect(limiter("t")).resolves.toBeUndefined(); - }); - - test("throws when INCR exceeds limit", async () => { - global.fetch = vi.fn().mockResolvedValueOnce({ ok: true, json: async () => ({ INCR: 3 }) }); - const { rateLimit } = await import("./rate-limit"); - const limiter = rateLimit({ interval: 10, allowedPerInterval: 2 }); - await expect(limiter("t")).rejects.toThrow("Rate limit exceeded for IP: t"); - }); -}); diff --git a/apps/web/lib/utils/rate-limit.ts b/apps/web/lib/utils/rate-limit.ts deleted file mode 100644 index 3d07fa4282..0000000000 --- a/apps/web/lib/utils/rate-limit.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { REDIS_HTTP_URL } from "@/lib/constants"; -import { LRUCache } from "lru-cache"; -import { logger } from "@formbricks/logger"; - -interface Options { - interval: number; - allowedPerInterval: number; -} - -const inMemoryRateLimiter = (options: Options) => { - const tokenCache = new LRUCache({ - max: 1000, - ttl: options.interval * 1000, // converts to expected input of milliseconds - }); - - return async (token: string) => { - const currentUsage = tokenCache.get(token) ?? 0; - if (currentUsage >= options.allowedPerInterval) { - throw new Error("Rate limit exceeded"); - } - tokenCache.set(token, currentUsage + 1); - }; -}; - -const redisRateLimiter = (options: Options) => async (token: string) => { - try { - if (!REDIS_HTTP_URL) { - throw new Error("Redis HTTP URL is not set"); - } - const tokenCountResponse = await fetch(`${REDIS_HTTP_URL}/INCR/${token}`); - if (!tokenCountResponse.ok) { - logger.error({ tokenCountResponse }, "Failed to increment token count in Redis"); - return; - } - - const { INCR } = await tokenCountResponse.json(); - if (INCR === 1) { - await fetch(`${REDIS_HTTP_URL}/EXPIRE/${token}/${options.interval.toString()}`); - } else if (INCR > options.allowedPerInterval) { - throw new Error(); - } - } catch (e) { - logger.error({ error: e }, "Rate limit exceeded"); - throw new Error("Rate limit exceeded for IP: " + token); - } -}; - -export const rateLimit = (options: Options) => { - if (REDIS_HTTP_URL) { - return redisRateLimiter(options); - } else { - return inMemoryRateLimiter(options); - } -}; diff --git a/apps/web/locales/de-DE.json b/apps/web/locales/de-DE.json index ceb3dc2331..fba8bbb9c3 100644 --- a/apps/web/locales/de-DE.json +++ b/apps/web/locales/de-DE.json @@ -748,7 +748,6 @@ "api_key_label": "API-Schlüssel Label", "api_key_security_warning": "Aus Sicherheitsgründen wird der API-Schlüssel nur einmal nach der Erstellung angezeigt. Bitte kopiere ihn sofort an einen sicheren Ort.", "api_key_updated": "API-Schlüssel aktualisiert", - "delete_permission": "Berechtigung löschen", "duplicate_access": "Doppelter Projektzugriff nicht erlaubt", "no_api_keys_yet": "Du hast noch keine API-Schlüssel", "no_env_permissions_found": "Keine Umgebungsberechtigungen gefunden", diff --git a/apps/web/locales/en-US.json b/apps/web/locales/en-US.json index b20ab6d9e9..cc8889776a 100644 --- a/apps/web/locales/en-US.json +++ b/apps/web/locales/en-US.json @@ -748,7 +748,6 @@ "api_key_label": "API Key Label", "api_key_security_warning": "For security reasons, the API key will only be shown once after creation. Please copy it to your destination right away.", "api_key_updated": "API Key updated", - "delete_permission": "Delete permission", "duplicate_access": "Duplicate project access not allowed", "no_api_keys_yet": "You don't have any API keys yet", "no_env_permissions_found": "No environment permissions found", diff --git a/apps/web/locales/fr-FR.json b/apps/web/locales/fr-FR.json index 4b34ae46c6..6872f221ea 100644 --- a/apps/web/locales/fr-FR.json +++ b/apps/web/locales/fr-FR.json @@ -748,7 +748,6 @@ "api_key_label": "Étiquette de clé API", "api_key_security_warning": "Pour des raisons de sécurité, la clé API ne sera affichée qu'une seule fois après sa création. Veuillez la copier immédiatement à votre destination.", "api_key_updated": "Clé API mise à jour", - "delete_permission": "Supprimer une permission", "duplicate_access": "L'accès en double au projet n'est pas autorisé", "no_api_keys_yet": "Vous n'avez pas encore de clés API.", "no_env_permissions_found": "Aucune autorisation d'environnement trouvée", diff --git a/apps/web/locales/ja-JP.json b/apps/web/locales/ja-JP.json index b645faebb7..0fad13b202 100644 --- a/apps/web/locales/ja-JP.json +++ b/apps/web/locales/ja-JP.json @@ -748,7 +748,6 @@ "api_key_label": "APIキーのラベル", "api_key_security_warning": "セキュリティ上の理由から、APIキーは作成時に一度だけ表示されます。すぐにコピーしてください。", "api_key_updated": "APIキーを更新しました", - "delete_permission": "削除権限", "duplicate_access": "重複したプロジェクトアクセスは許可されません", "no_api_keys_yet": "まだAPIキーがありません", "no_env_permissions_found": "環境権限が見つかりません", diff --git a/apps/web/locales/pt-BR.json b/apps/web/locales/pt-BR.json index 8ac988dc2a..fb9490b8cf 100644 --- a/apps/web/locales/pt-BR.json +++ b/apps/web/locales/pt-BR.json @@ -748,7 +748,6 @@ "api_key_label": "Rótulo da Chave API", "api_key_security_warning": "Por motivos de segurança, a chave da API será mostrada apenas uma vez após a criação. Por favor, copie-a para o seu destino imediatamente.", "api_key_updated": "Chave de API atualizada", - "delete_permission": "Remover permissão", "duplicate_access": "Acesso duplicado ao projeto não permitido", "no_api_keys_yet": "Você ainda não tem nenhuma chave de API", "no_env_permissions_found": "Nenhuma permissão de ambiente encontrada", diff --git a/apps/web/locales/pt-PT.json b/apps/web/locales/pt-PT.json index d34c86c403..86ed4aba2e 100644 --- a/apps/web/locales/pt-PT.json +++ b/apps/web/locales/pt-PT.json @@ -748,7 +748,6 @@ "api_key_label": "Etiqueta da Chave API", "api_key_security_warning": "Por razões de segurança, a chave API será mostrada apenas uma vez após a criação. Por favor, copie-a para o seu destino imediatamente.", "api_key_updated": "Chave API atualizada", - "delete_permission": "Eliminar permissão", "duplicate_access": "Acesso duplicado ao projeto não permitido", "no_api_keys_yet": "Ainda não tem nenhuma chave API", "no_env_permissions_found": "Nenhuma permissão de ambiente encontrada", diff --git a/apps/web/locales/ro-RO.json b/apps/web/locales/ro-RO.json index e28ece7e32..ee078bd32d 100644 --- a/apps/web/locales/ro-RO.json +++ b/apps/web/locales/ro-RO.json @@ -748,7 +748,6 @@ "api_key_label": "Etichetă Cheie API", "api_key_security_warning": "Din motive de securitate, cheia API va fi afișată o singură dată după creare. Vă rugăm să o copiați imediat la destinație.", "api_key_updated": "Cheie API actualizată", - "delete_permission": "Șterge permisiunea", "duplicate_access": "Accesul dublu la proiect nu este permis", "no_api_keys_yet": "Nu aveți încă chei API", "no_env_permissions_found": "Nu s-au găsit permisiuni pentru mediu", diff --git a/apps/web/locales/zh-Hans-CN.json b/apps/web/locales/zh-Hans-CN.json index cd0e20a161..24d4bc89f7 100644 --- a/apps/web/locales/zh-Hans-CN.json +++ b/apps/web/locales/zh-Hans-CN.json @@ -748,7 +748,6 @@ "api_key_label": "API Key Label", "api_key_security_warning": "出于安全原因，API 密钥在创建后只会显示一次。 请立即将其复制到您的目标位置。", "api_key_updated": "API Key 已更新", - "delete_permission": "删除 权限", "duplicate_access": "不允许 复制 项目 访问", "no_api_keys_yet": "您 还 没有 任 何 API key", "no_env_permissions_found": "未找到环境权限", diff --git a/apps/web/locales/zh-Hant-TW.json b/apps/web/locales/zh-Hant-TW.json index b1ace81b7d..1708defb6b 100644 --- a/apps/web/locales/zh-Hant-TW.json +++ b/apps/web/locales/zh-Hant-TW.json @@ -748,7 +748,6 @@ "api_key_label": "API 金鑰標籤", "api_key_security_warning": "為安全起見，API 金鑰僅在建立後顯示一次。請立即將其複製到您的目的地。", "api_key_updated": "API 金鑰已更新", - "delete_permission": "刪除 權限", "duplicate_access": "不允許重複的 project 存取", "no_api_keys_yet": "您還沒有任何 API 金鑰", "no_env_permissions_found": "找不到環境權限", diff --git a/apps/web/package.json b/apps/web/package.json index 5a8987c612..fb1a8ee326 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -38,7 +38,6 @@ "@hookform/resolvers": "5.0.1", "@intercom/messenger-js-sdk": "0.0.14", "@json2csv/node": "7.0.6", - "@keyv/redis": "4.4.0", "@lexical/code": "0.31.0", "@lexical/link": "0.31.0", "@lexical/list": "0.31.0", @@ -85,7 +84,6 @@ "@vercel/og": "0.8.5", "bcryptjs": "3.0.2", "boring-avatars": "2.0.1", - "cache-manager": "6.4.3", "class-variance-authority": "0.7.1", "clsx": "2.1.1", "cmdk": "1.1.1", @@ -98,7 +96,6 @@ "https-proxy-agent": "7.0.6", "jiti": "2.4.2", "jsonwebtoken": "9.0.2", - "keyv": "5.3.3", "lexical": "0.31.0", "lodash": "4.17.21", "lru-cache": "11.1.0", diff --git a/apps/web/scripts/docker/read-secrets.sh b/apps/web/scripts/docker/read-secrets.sh index 11396fd428..6a994a4fc8 100644 --- a/apps/web/scripts/docker/read-secrets.sh +++ b/apps/web/scripts/docker/read-secrets.sh @@ -6,7 +6,7 @@ if [ -f "/run/secrets/database_url" ]; then DATABASE_URL=${DATABASE_URL%$'\n'} export DATABASE_URL else - echo "DATABASE_URL secret not found. Build may fail if this is required." + echo "DATABASE_URL secret not found. Build will fail because it is required by the application." fi if [ -f "/run/secrets/encryption_key" ]; then @@ -14,7 +14,15 @@ if [ -f "/run/secrets/encryption_key" ]; then ENCRYPTION_KEY=${ENCRYPTION_KEY%$'\n'} export ENCRYPTION_KEY else - echo "ENCRYPTION_KEY secret not found. Build may fail if this is required." + echo "ENCRYPTION_KEY secret not found. Build will fail because it is required by the application." +fi + +if [ -f "/run/secrets/redis_url" ]; then + IFS= read -r REDIS_URL < /run/secrets/redis_url || true + REDIS_URL=${REDIS_URL%$'\n'} + export REDIS_URL +else + echo "REDIS_URL secret not found. Build will fail because it is required by the application." fi if [ -f "/run/secrets/sentry_auth_token" ]; then @@ -39,6 +47,7 @@ fi # Verify environment variables are set before starting build echo " DATABASE_URL: $([ -n "${DATABASE_URL:-}" ] && printf '[SET]' || printf '[NOT SET]')" echo " ENCRYPTION_KEY: $([ -n "${ENCRYPTION_KEY:-}" ] && printf '[SET]' || printf '[NOT SET]')" +echo " REDIS_URL: $([ -n "${REDIS_URL:-}" ] && printf '[SET]' || printf '[NOT SET]')" echo " SENTRY_AUTH_TOKEN: $([ -n "${SENTRY_AUTH_TOKEN:-}" ] && printf '[SET]' || printf '[NOT SET]')" echo " TARGETARCH: $([ -n "${TARGETARCH:-}" ] && printf '%s' "${TARGETARCH}" || printf '[NOT SET]')" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3ad8b40a97..cb454d512e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -17,7 +17,7 @@ services: - 1025:1025 valkey: - image: valkey/valkey:8.1.1 + image: valkey/valkey@sha256:12ba4f45a7c3e1d0f076acd616cb230834e75a77e8516dde382720af32832d6d command: "valkey-server" ports: - 6379:6379 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 43c94f8e61..67a08f4c2e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,6 +26,10 @@ x-environment: &environment # You can use: $(openssl rand -hex 32) to generate a secure one CRON_SECRET: + # Redis URL for caching, rate limiting, and audit logging + # To use external Redis/Valkey: remove the redis service below and update this URL + REDIS_URL: redis://redis:6379 + # Set the minimum log level(debug, info, warn, error, fatal) # LOG_LEVEL: info @@ -169,12 +173,6 @@ x-environment: &environment # Set the below to send OpenTelemetry data for tracing # OPENTELEMETRY_LISTENER_URL: http://localhost:4318/v1/traces - # Set the below to use Redis for Next Caching (default is In-Memory from Next Cache) - # REDIS_URL: - - # Set the below to use for Rate Limiting (default us In-Memory LRU Cache) - # REDIS_HTTP_URL: - ########################################## OPTIONAL (AUDIT LOGGING) ########################################### # Set the below to 1 to enable audit logging. @@ -208,17 +206,35 @@ services: # Replace the below with your own secure password & Make sure the password matches the password field in DATABASE_URL above - POSTGRES_PASSWORD=postgres + # Redis/Valkey service for caching, rate limiting, and audit logging + # Remove this service if you want to use an external Redis/Valkey instance + redis: + restart: always + image: valkey/valkey@sha256:12ba4f45a7c3e1d0f076acd616cb230834e75a77e8516dde382720af32832d6d + command: valkey-server --appendonly yes + volumes: + - redis:/data + ports: + - "6379:6379" + formbricks: restart: always image: ghcr.io/formbricks/formbricks:latest depends_on: - postgres + - redis ports: - 3000:3000 volumes: - ./saml-connection:/home/nextjs/apps/web/saml-connection <<: *environment + + volumes: postgres: driver: local + redis: + driver: local + uploads: + driver: local diff --git a/docs/self-hosting/configuration/environment-variables.mdx b/docs/self-hosting/configuration/environment-variables.mdx index 226705f21d..2267c017a3 100644 --- a/docs/self-hosting/configuration/environment-variables.mdx +++ b/docs/self-hosting/configuration/environment-variables.mdx @@ -53,7 +53,7 @@ These variables are present inside your machine's docker-compose file. Restart t | STRIPE_SECRET_KEY | Secret key for Stripe integration. | optional | | | STRIPE_WEBHOOK_SECRET | Webhook secret for Stripe integration. | optional | | | TELEMETRY_DISABLED | Disables telemetry if set to 1. | optional | | -| DEFAULT_BRAND_COLOR | Default brand color for your app (Can be overwritten from the UI as well). | optional | #64748b | +| DEFAULT_BRAND_COLOR | Default brand color for your app (Can be overwritten from the UI as well). | optional | #64748b | | DEFAULT_ORGANIZATION_ID | Automatically assign new users to a specific organization when joining | optional | | | OIDC_DISPLAY_NAME | Display name for Custom OpenID Connect Provider | optional | | | OIDC_CLIENT_ID | Client ID for Custom OpenID Connect Provider | optional (required if OIDC auth is enabled) | | @@ -69,8 +69,7 @@ These variables are present inside your machine's docker-compose file. Restart t | SENTRY_AUTH_TOKEN | Set this if you want to make errors more readable in Sentry. | optional | | | SESSION_MAX_AGE | Configure the maximum age for the session in seconds. | optional | 86400 (24 hours) | | USER_MANAGEMENT_MINIMUM_ROLE | Set this to control which roles can access user management features. Accepted values: "owner", "manager", "disabled" | optional | manager | -| REDIS_URL | Redis URL for caching and audit logging. Required for audit logging and optional for Next.js caching. | optional (required if audit logging is enabled) | | -| REDIS_HTTP_URL | Redis URL for rate limiting. If not set, rate limiting uses in-memory LRU cache. | optional | | +| REDIS_URL | Redis URL for caching, rate limiting, and audit logging. Application will not start without this. | required | redis://localhost:6379 | | AUDIT_LOG_ENABLED | Set this to 1 to enable audit logging. Requires Redis to be configured with the REDIS_URL env variable. | optional | 0 | | AUDIT_LOG_GET_USER_IP | Set to 1 to include user IP addresses in audit logs from request headers | optional | 0 | diff --git a/docs/self-hosting/setup/cluster-setup.mdx b/docs/self-hosting/setup/cluster-setup.mdx index 32e2dfddb9..496221bc85 100644 --- a/docs/self-hosting/setup/cluster-setup.mdx +++ b/docs/self-hosting/setup/cluster-setup.mdx @@ -120,13 +120,12 @@ graph TD ## Redis Configuration -Redis integration is an Enterprise Edition feature and requires an enterprise license key. +Redis is required for Formbricks to function. The application will not start without a Redis URL configured. -Configure Redis by adding the following environment variables to your instances: +Configure Redis by adding the following **required** environment variable to your instances: ```sh env REDIS_URL=redis://your-redis-host:6379 -REDIS_HTTP_URL=http://your-redis-host:8000 ``` ## S3 Configuration diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eb9d87a314..7dffe4ec41 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -159,9 +159,6 @@ importers: '@json2csv/node': specifier: 7.0.6 version: 7.0.6 - '@keyv/redis': - specifier: 4.4.0 - version: 4.4.0(keyv@5.3.3) '@lexical/code': specifier: 0.31.0 version: 0.31.0 @@ -300,9 +297,6 @@ importers: boring-avatars: specifier: 2.0.1 version: 2.0.1(react-dom@19.1.0(react@19.1.0))(react@19.1.0) - cache-manager: - specifier: 6.4.3 - version: 6.4.3 class-variance-authority: specifier: 0.7.1 version: 0.7.1 @@ -339,9 +333,6 @@ importers: jsonwebtoken: specifier: 9.0.2 version: 9.0.2 - keyv: - specifier: 5.3.3 - version: 5.3.3 lexical: specifier: 0.31.0 version: 0.31.0 @@ -2046,15 +2037,6 @@ packages: '@json2csv/plainjs@7.0.6': resolution: {integrity: sha512-4Md7RPDCSYpmW1HWIpWBOqCd4vWfIqm53S3e/uzQ62iGi7L3r34fK/8nhOMEe+/eVfCx8+gdSCt1d74SlacQHw==} - '@keyv/redis@4.4.0': - resolution: {integrity: sha512-n/KEj3S7crVkoykggqsMUtcjNGvjagGPlJYgO/r6m9hhGZfhp1txJElHxcdJ1ANi/LJoBuOSILj15g6HD2ucqQ==} - engines: {node: '>= 18'} - peerDependencies: - keyv: ^5.3.3 - - '@keyv/serialize@1.1.0': - resolution: {integrity: sha512-RlDgexML7Z63Q8BSaqhXdCYNBy/JQnqYIwxofUrNLGCblOMHp+xux2Q8nLMLlPpgHQPoU0Do8Z6btCpRBEqZ8g==} - '@lexical/clipboard@0.31.0': resolution: {integrity: sha512-oOZgDnzD68uZMy99H1dx6Le/uJNxmvXgbHb2Qp6Zrej7OUnCV/X9kzVSgLqWKytgrjsyUjTLjRhkQgwLhu61sA==} @@ -3356,10 +3338,6 @@ packages: resolution: {integrity: sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==} engines: {node: '>=14'} - '@redis/client@1.6.1': - resolution: {integrity: sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==} - engines: {node: '>=14'} - '@redis/client@5.8.1': resolution: {integrity: sha512-hD5Tvv7G0t8b3w8ao3kQ4jEPUmUUC6pqA18c8ciYF5xZGfUGBg0olQHW46v6qSt4O5bxOuB3uV7pM6H5wEjBwA==} engines: {node: '>= 18'} @@ -5429,9 +5407,6 @@ packages: resolution: {integrity: sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==} engines: {node: '>= 10'} - cache-manager@6.4.3: - resolution: {integrity: sha512-VV5eq/QQ5rIVix7/aICO4JyvSeEv9eIQuKL5iFwgM2BrcYoE0A/D1mNsAHJAsB0WEbNdBlKkn6Tjz6fKzh/cKQ==} - call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -7270,9 +7245,6 @@ packages: keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} - keyv@5.3.3: - resolution: {integrity: sha512-Rwu4+nXI9fqcxiEHtbkvoes2X+QfkTRo1TMkPfwzipGsJlJO/z69vqB4FNl9xJ3xCpAcbkvmEabZfPzrwN3+gQ==} - kolorist@1.8.0: resolution: {integrity: sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==} @@ -11980,14 +11952,6 @@ snapshots: '@json2csv/formatters': 7.0.6 '@streamparser/json': 0.0.20 - '@keyv/redis@4.4.0(keyv@5.3.3)': - dependencies: - '@redis/client': 1.6.1 - cluster-key-slot: 1.1.2 - keyv: 5.3.3 - - '@keyv/serialize@1.1.0': {} - '@lexical/clipboard@0.31.0': dependencies: '@lexical/html': 0.31.0 @@ -13487,12 +13451,6 @@ snapshots: generic-pool: 3.9.0 yallist: 4.0.0 - '@redis/client@1.6.1': - dependencies: - cluster-key-slot: 1.1.2 - generic-pool: 3.9.0 - yallist: 4.0.0 - '@redis/client@5.8.1': dependencies: cluster-key-slot: 1.1.2 @@ -16011,10 +15969,6 @@ snapshots: - bluebird optional: true - cache-manager@6.4.3: - dependencies: - keyv: 5.3.3 - call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -18097,10 +18051,6 @@ snapshots: dependencies: json-buffer: 3.0.1 - keyv@5.3.3: - dependencies: - '@keyv/serialize': 1.1.0 - kolorist@1.8.0: {} language-subtag-registry@0.3.23: {} diff --git a/turbo.json b/turbo.json index 4494b16699..356b7e0d38 100644 --- a/turbo.json +++ b/turbo.json @@ -176,7 +176,6 @@ "POSTHOG_API_KEY", "PRIVACY_URL", "RATE_LIMITING_DISABLED", - "REDIS_HTTP_URL", "REDIS_URL", "S3_ACCESS_KEY", "S3_BUCKET_NAME",