diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index addd11f5aa..51e18c51da 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -181,6 +181,12 @@ jobs: fi echo "License key length: ${#LICENSE_KEY}" + - name: Disable rate limiting for E2E tests + run: | + echo "RATE_LIMITING_DISABLED=1" >> .env + echo "Rate limiting disabled for E2E tests" + shell: bash + - name: Run App run: | echo "Starting app with enterprise license..." @@ -222,11 +228,14 @@ jobs: if: env.AZURE_ENABLED == 'true' env: PLAYWRIGHT_SERVICE_URL: ${{ secrets.PLAYWRIGHT_SERVICE_URL }} + CI: true run: | pnpm test-e2e:azure - name: Run E2E Tests (Local) if: env.AZURE_ENABLED == 'false' + env: + CI: true run: | pnpm test:e2e diff --git a/apps/web/locales/en-US.json b/apps/web/locales/en-US.json index 8a94221dc7..bdd443ef65 100644 --- a/apps/web/locales/en-US.json +++ b/apps/web/locales/en-US.json @@ -1301,8 +1301,8 @@ "contains": "Contains", "continue_to_settings": "Continue to Settings", "control_which_file_types_can_be_uploaded": "Control which file types can be uploaded.", - "convert_to_multiple_choice": "Convert to Multiple Choice", - "convert_to_single_choice": "Convert to Single Choice", + "convert_to_multiple_choice": "Convert to Multi-select", + "convert_to_single_choice": "Convert to Single-select", "country": "Country", "create_group": "Create group", "create_your_own_survey": "Create your own survey", diff --git a/apps/web/locales/pt-PT.json b/apps/web/locales/pt-PT.json index eba76d7364..db78e52c47 100644 --- a/apps/web/locales/pt-PT.json +++ b/apps/web/locales/pt-PT.json @@ -1301,8 +1301,8 @@ "contains": "Contém", "continue_to_settings": "Continuar para Definições", "control_which_file_types_can_be_uploaded": "Controlar quais tipos de ficheiros podem ser carregados.", - "convert_to_multiple_choice": "Converter para Escolha Múltipla", - "convert_to_single_choice": "Converter para Escolha Única", + "convert_to_multiple_choice": "Converter para Seleção Múltipla", + "convert_to_single_choice": "Converter para Seleção Única", "country": "País", "create_group": "Criar grupo", "create_your_own_survey": "Crie o seu próprio inquérito", diff --git a/apps/web/locales/ro-RO.json b/apps/web/locales/ro-RO.json index ebe96c66a0..74cd9b62fa 100644 --- a/apps/web/locales/ro-RO.json +++ b/apps/web/locales/ro-RO.json @@ -1301,8 +1301,8 @@ "contains": "Conține", "continue_to_settings": "Continuă către Setări", "control_which_file_types_can_be_uploaded": "Controlează ce tipuri de fișiere pot fi încărcate.", - "convert_to_multiple_choice": "Convertiți la alegere multiplă", - "convert_to_single_choice": "Convertiți la alegere unică", + "convert_to_multiple_choice": "Convertiți la selectare multiplă", + "convert_to_single_choice": "Convertiți la selectare unică", "country": "Țară", "create_group": "Creează grup", "create_your_own_survey": "Creează-ți propriul chestionar", diff --git a/apps/web/locales/zh-Hans-CN.json b/apps/web/locales/zh-Hans-CN.json index 24a5ee53c6..3990b1dd8e 100644 --- a/apps/web/locales/zh-Hans-CN.json +++ b/apps/web/locales/zh-Hans-CN.json @@ -1301,8 +1301,8 @@ "contains": "包含", "continue_to_settings": "继续 到 设置", "control_which_file_types_can_be_uploaded": "控制 可以 上传的 文件 类型", - "convert_to_multiple_choice": "转换为多选题", - "convert_to_single_choice": "转换为单选题", + "convert_to_multiple_choice": "转换为 多选", + "convert_to_single_choice": "转换为 单选", "country": "国家", "create_group": "创建 群组", "create_your_own_survey": "创建 你 的 调查", diff --git a/apps/web/playwright/action.spec.ts b/apps/web/playwright/action.spec.ts index f560bbe035..a86381b587 100644 --- a/apps/web/playwright/action.spec.ts +++ b/apps/web/playwright/action.spec.ts @@ -1,5 +1,5 @@ -import { actions } from "@/playwright/utils/mock"; import { Page, expect } from "@playwright/test"; +import { actions } from "@/playwright/utils/mock"; import { test } from "./lib/fixtures"; const createNoCodeClickAction = async ({ @@ -170,11 +170,14 @@ const createCodeAction = async ({ await page.getByRole("button", { name: "Create action", exact: true }).click(); - const successToast = await page.waitForSelector(".formbricks__toast__success"); + const successToast = await page.waitForSelector(".formbricks__toast__success", { timeout: 15000 }); expect(successToast).toBeTruthy(); + // Wait for the action to be fully created and committed to the database + await page.waitForLoadState("networkidle", { timeout: 15000 }); + const actionButton = page.getByTitle(name); - await expect(actionButton).toBeVisible(); + await expect(actionButton).toBeVisible({ timeout: 10000 }); }; const getActionButtonLocator = (page: Page, actionName: string) => { diff --git a/apps/web/playwright/api/auth/security.spec.ts b/apps/web/playwright/api/auth/security.spec.ts index eacac5f4e3..f5a650d96a 100644 --- a/apps/web/playwright/api/auth/security.spec.ts +++ b/apps/web/playwright/api/auth/security.spec.ts @@ -127,15 +127,64 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () => test.describe("Timing Attack Prevention - User Enumeration Protection", () => { test("should not reveal user existence through response timing differences", async ({ request }) => { - // Test multiple attempts to get reliable timing measurements - const attempts = 50; + // Helper functions for statistical analysis + const calculateMedian = (values: number[]): number => { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; + }; + + const calculateStdDev = (values: number[], mean: number): number => { + const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / values.length; + return Math.sqrt(variance); + }; + + logger.info("🔥 Phase 1: Warming up caches, DB connections, and JIT compilation..."); + + // Warm-up phase: Prime caches, database connections, JIT compilation + const warmupAttempts = 10; + for (let i = 0; i < warmupAttempts; i++) { + await request.post("/api/auth/callback/credentials", { + data: { + callbackUrl: "", + email: `warmup-nonexistent-${i}@example.com`, + password: "warmuppassword", + redirect: "false", + csrfToken: csrfToken, + json: "true", + }, + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + }); + + await request.post("/api/auth/callback/credentials", { + data: { + callbackUrl: "", + email: testUser.email, + password: "wrongwarmuppassword", + redirect: "false", + csrfToken: csrfToken, + json: "true", + }, + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + }); + } + + logger.info("✅ Warm-up complete. Starting actual measurements with 100 attempts per scenario..."); + + // Actual measurement phase with increased sample size + const attempts = 100; const nonExistentTimes: number[] = []; const existingUserTimes: number[] = []; - // Test non-existent user timing (multiple attempts for statistical reliability) + // Interleave tests to reduce impact of system load variations for (let i = 0; i < attempts; i++) { - const start = process.hrtime.bigint(); - const response = await request.post("/api/auth/callback/credentials", { + // Test non-existent user + const startNonExistent = process.hrtime.bigint(); + const responseNonExistent = await request.post("/api/auth/callback/credentials", { data: { callbackUrl: "", email: `nonexistent-timing-${i}@example.com`, @@ -148,17 +197,14 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () => "Content-Type": "application/x-www-form-urlencoded", }, }); - const end = process.hrtime.bigint(); - const responseTime = Number(end - start) / 1000000; // Convert to milliseconds + const endNonExistent = process.hrtime.bigint(); + const responseTimeNonExistent = Number(endNonExistent - startNonExistent) / 1000000; + nonExistentTimes.push(responseTimeNonExistent); + expect(responseNonExistent.status()).not.toBe(500); - nonExistentTimes.push(responseTime); - expect(response.status()).not.toBe(500); - } - - // Test existing user with wrong password timing (multiple attempts) - for (let i = 0; i < attempts; i++) { - const start = process.hrtime.bigint(); - const response = await request.post("/api/auth/callback/credentials", { + // Test existing user (interleaved) + const startExisting = process.hrtime.bigint(); + const responseExisting = await request.post("/api/auth/callback/credentials", { data: { callbackUrl: "", email: testUser.email, @@ -171,29 +217,43 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () => "Content-Type": "application/x-www-form-urlencoded", }, }); - const end = process.hrtime.bigint(); - const responseTime = Number(end - start) / 1000000; // Convert to milliseconds - - existingUserTimes.push(responseTime); - expect(response.status()).not.toBe(500); + const endExisting = process.hrtime.bigint(); + const responseTimeExisting = Number(endExisting - startExisting) / 1000000; + existingUserTimes.push(responseTimeExisting); + expect(responseExisting.status()).not.toBe(500); } - // Calculate averages + // Calculate statistics using median (more robust to outliers) + const medianNonExistent = calculateMedian(nonExistentTimes); + const medianExisting = calculateMedian(existingUserTimes); + + // Also calculate means for comparison const avgNonExistent = nonExistentTimes.reduce((a, b) => a + b, 0) / nonExistentTimes.length; const avgExisting = existingUserTimes.reduce((a, b) => a + b, 0) / existingUserTimes.length; - // Calculate the timing difference percentage - const timingDifference = Math.abs(avgExisting - avgNonExistent); - const timingDifferencePercent = (timingDifference / Math.max(avgExisting, avgNonExistent)) * 100; + // Calculate standard deviations + const stdDevNonExistent = calculateStdDev(nonExistentTimes, avgNonExistent); + const stdDevExisting = calculateStdDev(existingUserTimes, avgExisting); + // Calculate timing difference using MEDIAN (more reliable) + const timingDifference = Math.abs(medianExisting - medianNonExistent); + const timingDifferencePercent = (timingDifference / Math.max(medianExisting, medianNonExistent)) * 100; + + // Calculate coefficient of variation (CV) for reliability assessment + // CV = (StdDev / Mean) * 100 - measures relative variability + const cvNonExistent = (stdDevNonExistent / avgNonExistent) * 100; + const cvExisting = (stdDevExisting / avgExisting) * 100; + + // Log comprehensive statistics + logger.info("📊 Statistical Analysis:"); logger.info( - `Non-existent user avg: ${avgNonExistent.toFixed(2)}ms (${nonExistentTimes.map((t) => t.toFixed(0)).join(", ")})` + `Non-existent user - Mean: ${avgNonExistent.toFixed(2)}ms, Median: ${medianNonExistent.toFixed(2)}ms, StdDev: ${stdDevNonExistent.toFixed(2)}ms (CV: ${cvNonExistent.toFixed(1)}%)` ); logger.info( - `Existing user avg: ${avgExisting.toFixed(2)}ms (${existingUserTimes.map((t) => t.toFixed(0)).join(", ")})` + `Existing user - Mean: ${avgExisting.toFixed(2)}ms, Median: ${medianExisting.toFixed(2)}ms, StdDev: ${stdDevExisting.toFixed(2)}ms (CV: ${cvExisting.toFixed(1)}%)` ); logger.info( - `Timing difference: ${timingDifference.toFixed(2)}ms (${timingDifferencePercent.toFixed(1)}%)` + `Timing difference (median-based): ${timingDifference.toFixed(2)}ms (${timingDifferencePercent.toFixed(1)}%)` ); // CRITICAL SECURITY TEST: Timing difference should be minimal @@ -211,7 +271,34 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () => } // Fail the test if timing difference exceeds our security threshold - expect(timingDifferencePercent).toBeLessThan(20); // Fail at our actual security threshold + // Note: This uses MEDIAN-based comparison (more robust to outliers than mean) + expect(timingDifferencePercent).toBeLessThan(20); + + // Validate measurement reliability using coefficient of variation (CV) + // CV > 50% indicates high variability and unreliable measurements + const maxAcceptableCV = 50; // 50% is reasonable for network-based tests + + if (cvNonExistent > maxAcceptableCV) { + logger.warn( + `⚠️ High variability in non-existent user timing (CV: ${cvNonExistent.toFixed(1)}%). ` + + `Test measurements may be unreliable. Consider increasing warm-up or checking CI environment.` + ); + } + + if (cvExisting > maxAcceptableCV) { + logger.warn( + `⚠️ High variability in existing user timing (CV: ${cvExisting.toFixed(1)}%). ` + + `Test measurements may be unreliable. Consider increasing warm-up or checking CI environment.` + ); + } + + // These are soft checks - we warn but don't fail the test for high CV + // This allows for noisy CI environments while still alerting to potential issues + if (cvNonExistent <= maxAcceptableCV && cvExisting <= maxAcceptableCV) { + logger.info( + `✅ Measurement reliability good: CV ${cvNonExistent.toFixed(1)}% and ${cvExisting.toFixed(1)}% (threshold: ${maxAcceptableCV}%)` + ); + } }); test("should return consistent status codes regardless of user existence", async ({ request }) => { diff --git a/apps/web/playwright/fixtures/users.ts b/apps/web/playwright/fixtures/users.ts index 95b0b11046..da4cd5ff8b 100644 --- a/apps/web/playwright/fixtures/users.ts +++ b/apps/web/playwright/fixtures/users.ts @@ -72,6 +72,7 @@ export const createUsersFixture = (page: Page, workerInfo: TestInfo): UsersFixtu name: uname, email: userEmail, password: hashedPassword, + emailVerified: new Date(), locale: "en-US", memberships: { create: { diff --git a/apps/web/playwright/lib/utils.ts b/apps/web/playwright/lib/utils.ts index 6b2001b7b6..06aacab846 100644 --- a/apps/web/playwright/lib/utils.ts +++ b/apps/web/playwright/lib/utils.ts @@ -5,7 +5,7 @@ export async function loginAndGetApiKey(page: Page, users: UsersFixture) { const user = await users.create(); await user.login(); - await page.waitForURL(/\/environments\/[^/]+\/surveys/); + await page.waitForURL(/\/environments\/[^/]+\/surveys/, { timeout: 30000 }); const environmentId = /\/environments\/([^/]+)\/surveys/.exec(page.url())?.[1] ?? @@ -13,9 +13,9 @@ export async function loginAndGetApiKey(page: Page, users: UsersFixture) { throw new Error("Unable to parse environmentId from URL"); })(); - await page.goto(`/environments/${environmentId}/settings/api-keys`); + await page.goto(`/environments/${environmentId}/settings/api-keys`, { waitUntil: "domcontentloaded" }); - await page.getByRole("button", { name: "Add API Key" }).isVisible(); + await page.getByRole("button", { name: "Add API Key" }).waitFor({ state: "visible", timeout: 15000 }); await page.getByRole("button", { name: "Add API Key" }).click(); await page.getByPlaceholder("e.g. GitHub, PostHog, Slack").fill("E2E Test API Key"); await page.getByRole("button", { name: "+ Add permission" }).click(); @@ -26,7 +26,18 @@ export async function loginAndGetApiKey(page: Page, users: UsersFixture) { await page.getByTestId("organization-access-accessControl-read").click(); await page.getByTestId("organization-access-accessControl-write").click(); await page.getByRole("button", { name: "Add API Key" }).click(); - await page.locator(".copyApiKeyIcon").click(); + + // Wait for the API key creation to complete and appear in the list + // Use longer timeouts for cloud environments with high concurrency and network latency + // Wait for network idle to ensure the API key is fully committed to the database + await page.waitForLoadState("networkidle", { timeout: 30000 }); + await page.waitForSelector(".copyApiKeyIcon", { state: "visible", timeout: 30000 }); + + // Add a delay to ensure the API key is fully committed to the database + // This is especially important with high concurrency in cloud environments + await page.waitForTimeout(2000); + + await page.locator(".copyApiKeyIcon").first().click(); const apiKey = await page.evaluate("navigator.clipboard.readText()"); diff --git a/apps/web/playwright/survey.spec.ts b/apps/web/playwright/survey.spec.ts index 3a71abcbf3..80b60d211e 100644 --- a/apps/web/playwright/survey.spec.ts +++ b/apps/web/playwright/survey.spec.ts @@ -1,5 +1,5 @@ -import { surveys } from "@/playwright/utils/mock"; import { expect } from "@playwright/test"; +import { surveys } from "@/playwright/utils/mock"; import { test } from "./lib/fixtures"; import { createSurvey, createSurveyWithLogic, uploadFileForFileUploadQuestion } from "./utils/helper"; @@ -28,10 +28,13 @@ test.describe("Survey Create & Submit Response without logic", async () => { await expect(page.locator("#howToSendCardOption-link")).toBeVisible(); await page.locator("#howToSendCardOption-link").click(); + // Wait for any auto-save to complete before publishing + await page.waitForTimeout(2000); + await page.getByRole("button", { name: "Publish" }).click(); - // Get URL - await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/); + // Get URL - increase timeout for slower local environments + await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/, { timeout: 60000 }); await page.getByLabel("Copy survey link to clipboard").click(); url = await page.evaluate("navigator.clipboard.readText()"); }); @@ -291,7 +294,7 @@ test.describe("Multi Language Survey Create", async () => { .filter({ hasText: /^Add questionAdd a new question to your survey$/ }) .nth(1) .click(); - await page.getByRole("button", { name: "Multi-Select" }).click(); + await page.getByRole("button", { name: "Multi-Select Ask respondents" }).click(); await page.getByLabel("Question*").fill(surveys.createAndSubmit.multiSelectQuestion.question); await page.getByPlaceholder("Option 1").fill(surveys.createAndSubmit.multiSelectQuestion.options[0]); await page.getByPlaceholder("Option 2").fill(surveys.createAndSubmit.multiSelectQuestion.options[1]); @@ -446,7 +449,7 @@ test.describe("Multi Language Survey Create", async () => { await page.getByPlaceholder("Back").fill(surveys.germanCreate.back); // Fill Multi select question in german - await page.getByRole("main").getByText("Multi-Select").click(); + await page.getByRole("main").getByRole("heading", { name: "Multi-Select" }).click(); await page.getByPlaceholder("Your question here. Recall").click(); await page @@ -625,10 +628,13 @@ test.describe("Multi Language Survey Create", async () => { await expect(page.locator("#howToSendCardOption-link")).toBeVisible(); await page.locator("#howToSendCardOption-link").click(); + // Wait for any auto-save to complete before publishing + await page.waitForTimeout(2000); + await page.getByRole("button", { name: "Publish" }).click(); - await page.waitForTimeout(5000); - await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/); + await page.waitForTimeout(2000); + await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/, { timeout: 60000 }); await page.getByLabel("Select Language").click(); await page.getByText("German").click(); await page.getByLabel("Copy survey link to clipboard").click(); diff --git a/apps/web/playwright/utils/helper.ts b/apps/web/playwright/utils/helper.ts index fdb7e8b971..20c5315f98 100644 --- a/apps/web/playwright/utils/helper.ts +++ b/apps/web/playwright/utils/helper.ts @@ -1,9 +1,9 @@ -import { CreateSurveyParams, CreateSurveyWithLogicParams } from "@/playwright/utils/mock"; import { expect } from "@playwright/test"; import { readFileSync, writeFileSync } from "fs"; import { Page } from "playwright"; import { logger } from "@formbricks/logger"; import { TProjectConfigChannel } from "@formbricks/types/project"; +import { CreateSurveyParams, CreateSurveyWithLogicParams } from "@/playwright/utils/mock"; export const signUpAndLogin = async ( page: Page, @@ -203,7 +203,7 @@ export const createSurvey = async (page: Page, params: CreateSurveyParams) => { .filter({ hasText: new RegExp(`^${addQuestion}$`) }) .nth(1) .click(); - await page.getByRole("button", { name: "Multi-Select" }).click(); + await page.getByRole("button", { name: "Multi-Select Ask respondents" }).click(); await page.getByLabel("Question*").fill(params.multiSelectQuestion.question); await page.getByRole("button", { name: "Add description", exact: true }).click(); await page.locator('input[name="subheader"]').fill(params.multiSelectQuestion.description); @@ -416,7 +416,7 @@ export const createSurveyWithLogic = async (page: Page, params: CreateSurveyWith .filter({ hasText: new RegExp(`^${addQuestion}$`) }) .nth(1) .click(); - await page.getByRole("button", { name: "Multi-Select" }).click(); + await page.getByRole("button", { name: "Multi-Select Ask respondents" }).click(); await page.getByLabel("Question*").fill(params.multiSelectQuestion.question); await page.getByRole("button", { name: "Add description" }).click(); await page.locator('input[name="subheader"]').fill(params.multiSelectQuestion.description); diff --git a/package.json b/package.json index 3f3e6bd582..638244dcae 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "test": "turbo run test --no-cache", "test:coverage": "turbo run test:coverage --no-cache", "test:e2e": "playwright test", - "test-e2e:azure": "pnpm test:e2e -c playwright.service.config.ts --workers=20", + "test-e2e:azure": "pnpm test:e2e -c playwright.service.config.ts --workers=10", "prepare": "husky install", "storybook": "turbo run storybook", "fb-migrate-dev": "pnpm --filter @formbricks/database create-migration && pnpm prisma generate", diff --git a/playwright.config.ts b/playwright.config.ts index 28ce7f9308..18bddb935b 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -14,11 +14,11 @@ export default defineConfig({ /* Run tests in files in parallel */ fullyParallel: true, /* Retry on CI only */ - retries: 0, + retries: process.env.CI ? 2 : 0, /* Timeout for each test */ timeout: 120000, /* Fail the test run after the first failure */ - maxFailures: 1, // Stop execution after the first failed test + maxFailures: process.env.CI ? undefined : 1, // Allow more failures in CI to avoid cascading shutdowns /* Opt out of parallel tests on CI. */ // workers: os.cpus().length, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ diff --git a/playwright.service.config.ts b/playwright.service.config.ts index bcf1dd95bd..c805bee01a 100644 --- a/playwright.service.config.ts +++ b/playwright.service.config.ts @@ -7,7 +7,7 @@ export default defineConfig( config, getServiceConfig(config, { exposeNetwork: "", - timeout: 33000, + timeout: 120000, // Increased timeout for cloud environment with network latency os: ServiceOS.LINUX, useCloudHostedBrowsers: true, // Set to false if you want to only use reporting and not cloud hosted browsers }), @@ -18,5 +18,7 @@ export default defineConfig( If you are using more reporters, please update your configuration accordingly. */ reporter: [["list"], ["@azure/microsoft-playwright-testing/reporter"]], + retries: 2, // Always retry in cloud environment due to potential network/timing issues + maxFailures: undefined, // Don't stop on first failure to avoid cascading shutdowns with high parallelism } );