mirror/formbricks
1
0
Fork 0
mirror of https://github.com/formbricks/formbricks.git synced 2025-12-30 16:16:21 -06:00
Code Issues Packages Projects Releases 115 Wiki Activity

fix: improve E2E test reliability and security (#6653)

Browse Source
This commit is contained in:
Victor Hugo dos Santos
2025-10-06 02:02:51 -03:00
committed by GitHub
parent 5c9795cd23
commit ebf591a7e0
14 changed files with 176 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/e2e.yml vendored
View File

@@ -181,6 +181,12 @@ jobs:
fi
echo "License key length: ${#LICENSE_KEY}"
- name: Disable rate limiting for E2E tests
run: |
echo "RATE_LIMITING_DISABLED=1" >> .env
echo "Rate limiting disabled for E2E tests"
shell: bash
- name: Run App
run: |
echo "Starting app with enterprise license..."
@@ -222,11 +228,14 @@ jobs:
if: env.AZURE_ENABLED == 'true'
env:
PLAYWRIGHT_SERVICE_URL: ${{ secrets.PLAYWRIGHT_SERVICE_URL }}
CI: true
run: |
pnpm test-e2e:azure
- name: Run E2E Tests (Local)
if: env.AZURE_ENABLED == 'false'
env:
CI: true
run: |
pnpm test:e2e

4
apps/web/locales/en-US.json
View File

@@ -1301,8 +1301,8 @@
"contains": "Contains",
"continue_to_settings": "Continue to Settings",
"control_which_file_types_can_be_uploaded": "Control which file types can be uploaded.",
"convert_to_multiple_choice": "Convert to Multiple Choice",
"convert_to_single_choice": "Convert to Single Choice",
"convert_to_multiple_choice": "Convert to Multi-select",
"convert_to_single_choice": "Convert to Single-select",
"country": "Country",
"create_group": "Create group",
"create_your_own_survey": "Create your own survey",

4
apps/web/locales/pt-PT.json
View File

@@ -1301,8 +1301,8 @@
"contains": "Contém",
"continue_to_settings": "Continuar para Definições",
"control_which_file_types_can_be_uploaded": "Controlar quais tipos de ficheiros podem ser carregados.",
"convert_to_multiple_choice": "Converter para Escolha Múltipla",
"convert_to_single_choice": "Converter para Escolha Única",
"convert_to_multiple_choice": "Converter para Seleção Múltipla",
"convert_to_single_choice": "Converter para Seleção Única",
"country": "País",
"create_group": "Criar grupo",
"create_your_own_survey": "Crie o seu próprio inquérito",

4
apps/web/locales/ro-RO.json
View File

@@ -1301,8 +1301,8 @@
"contains": "Conține",
"continue_to_settings": "Continuă către Setări",
"control_which_file_types_can_be_uploaded": "Controlează ce tipuri de fișiere pot fi încărcate.",
"convert_to_multiple_choice": "Convertiți la alegere multiplă",
"convert_to_single_choice": "Convertiți la alegere unică",
"convert_to_multiple_choice": "Convertiți la selectare multiplă",
"convert_to_single_choice": "Convertiți la selectare unică",
"country": "Țară",
"create_group": "Creează grup",
"create_your_own_survey": "Creează-ți propriul chestionar",

4
apps/web/locales/zh-Hans-CN.json
View File

@@ -1301,8 +1301,8 @@
"contains": "包含",
"continue_to_settings": "继续 到 设置",
"control_which_file_types_can_be_uploaded": "控制 可以 上传的 文件 类型",
"convert_to_multiple_choice": "转换为多选题",
"convert_to_single_choice": "转换为单选题",
"convert_to_multiple_choice": "转换为 多选",
"convert_to_single_choice": "转换为 单选",
"country": "国家",
"create_group": "创建 群组",
"create_your_own_survey": "创建 你 的 调查",

9
apps/web/playwright/action.spec.ts
View File

@@ -1,5 +1,5 @@
import { actions } from "@/playwright/utils/mock";
import { Page, expect } from "@playwright/test";
import { actions } from "@/playwright/utils/mock";
import { test } from "./lib/fixtures";
const createNoCodeClickAction = async ({
@@ -170,11 +170,14 @@ const createCodeAction = async ({
await page.getByRole("button", { name: "Create action", exact: true }).click();
const successToast = await page.waitForSelector(".formbricks__toast__success");
const successToast = await page.waitForSelector(".formbricks__toast__success", { timeout: 15000 });
expect(successToast).toBeTruthy();
// Wait for the action to be fully created and committed to the database
await page.waitForLoadState("networkidle", { timeout: 15000 });
const actionButton = page.getByTitle(name);
await expect(actionButton).toBeVisible();
await expect(actionButton).toBeVisible({ timeout: 10000 });
};
const getActionButtonLocator = (page: Page, actionName: string) => {

143
apps/web/playwright/api/auth/security.spec.ts
View File

@@ -127,15 +127,64 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () =>
test.describe("Timing Attack Prevention - User Enumeration Protection", () => {
test("should not reveal user existence through response timing differences", async ({ request }) => {
// Test multiple attempts to get reliable timing measurements
const attempts = 50;
// Helper functions for statistical analysis
const calculateMedian = (values: number[]): number => {
const sorted = [...values].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
};
const calculateStdDev = (values: number[], mean: number): number => {
const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / values.length;
return Math.sqrt(variance);
};
logger.info("🔥 Phase 1: Warming up caches, DB connections, and JIT compilation...");
// Warm-up phase: Prime caches, database connections, JIT compilation
const warmupAttempts = 10;
for (let i = 0; i < warmupAttempts; i++) {
await request.post("/api/auth/callback/credentials", {
data: {
callbackUrl: "",
email: `warmup-nonexistent-${i}@example.com`,
password: "warmuppassword",
redirect: "false",
csrfToken: csrfToken,
json: "true",
},
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
});
await request.post("/api/auth/callback/credentials", {
data: {
callbackUrl: "",
email: testUser.email,
password: "wrongwarmuppassword",
redirect: "false",
csrfToken: csrfToken,
json: "true",
},
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
});
}
logger.info("✅ Warm-up complete. Starting actual measurements with 100 attempts per scenario...");
// Actual measurement phase with increased sample size
const attempts = 100;
const nonExistentTimes: number[] = [];
const existingUserTimes: number[] = [];
// Test non-existent user timing (multiple attempts for statistical reliability)
// Interleave tests to reduce impact of system load variations
for (let i = 0; i < attempts; i++) {
const start = process.hrtime.bigint();
const response = await request.post("/api/auth/callback/credentials", {
// Test non-existent user
const startNonExistent = process.hrtime.bigint();
const responseNonExistent = await request.post("/api/auth/callback/credentials", {
data: {
callbackUrl: "",
email: `nonexistent-timing-${i}@example.com`,
@@ -148,17 +197,14 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () =>
"Content-Type": "application/x-www-form-urlencoded",
},
});
const end = process.hrtime.bigint();
const responseTime = Number(end - start) / 1000000; // Convert to milliseconds
const endNonExistent = process.hrtime.bigint();
const responseTimeNonExistent = Number(endNonExistent - startNonExistent) / 1000000;
nonExistentTimes.push(responseTimeNonExistent);
expect(responseNonExistent.status()).not.toBe(500);
nonExistentTimes.push(responseTime);
expect(response.status()).not.toBe(500);
}
// Test existing user with wrong password timing (multiple attempts)
for (let i = 0; i < attempts; i++) {
const start = process.hrtime.bigint();
const response = await request.post("/api/auth/callback/credentials", {
// Test existing user (interleaved)
const startExisting = process.hrtime.bigint();
const responseExisting = await request.post("/api/auth/callback/credentials", {
data: {
callbackUrl: "",
email: testUser.email,
@@ -171,29 +217,43 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () =>
"Content-Type": "application/x-www-form-urlencoded",
},
});
const end = process.hrtime.bigint();
const responseTime = Number(end - start) / 1000000; // Convert to milliseconds
existingUserTimes.push(responseTime);
expect(response.status()).not.toBe(500);
const endExisting = process.hrtime.bigint();
const responseTimeExisting = Number(endExisting - startExisting) / 1000000;
existingUserTimes.push(responseTimeExisting);
expect(responseExisting.status()).not.toBe(500);
}
// Calculate averages
// Calculate statistics using median (more robust to outliers)
const medianNonExistent = calculateMedian(nonExistentTimes);
const medianExisting = calculateMedian(existingUserTimes);
// Also calculate means for comparison
const avgNonExistent = nonExistentTimes.reduce((a, b) => a + b, 0) / nonExistentTimes.length;
const avgExisting = existingUserTimes.reduce((a, b) => a + b, 0) / existingUserTimes.length;
// Calculate the timing difference percentage
const timingDifference = Math.abs(avgExisting - avgNonExistent);
const timingDifferencePercent = (timingDifference / Math.max(avgExisting, avgNonExistent)) * 100;
// Calculate standard deviations
const stdDevNonExistent = calculateStdDev(nonExistentTimes, avgNonExistent);
const stdDevExisting = calculateStdDev(existingUserTimes, avgExisting);
// Calculate timing difference using MEDIAN (more reliable)
const timingDifference = Math.abs(medianExisting - medianNonExistent);
const timingDifferencePercent = (timingDifference / Math.max(medianExisting, medianNonExistent)) * 100;
// Calculate coefficient of variation (CV) for reliability assessment
// CV = (StdDev / Mean) * 100 - measures relative variability
const cvNonExistent = (stdDevNonExistent / avgNonExistent) * 100;
const cvExisting = (stdDevExisting / avgExisting) * 100;
// Log comprehensive statistics
logger.info("📊 Statistical Analysis:");
logger.info(
`Non-existent user avg: ${avgNonExistent.toFixed(2)}ms (${nonExistentTimes.map((t) => t.toFixed(0)).join(", ")})`
`Non-existent user - Mean: ${avgNonExistent.toFixed(2)}ms, Median: ${medianNonExistent.toFixed(2)}ms, StdDev: ${stdDevNonExistent.toFixed(2)}ms (CV: ${cvNonExistent.toFixed(1)}%)`
);
logger.info(
`Existing user avg: ${avgExisting.toFixed(2)}ms (${existingUserTimes.map((t) => t.toFixed(0)).join(", ")})`
`Existing user - Mean: ${avgExisting.toFixed(2)}ms, Median: ${medianExisting.toFixed(2)}ms, StdDev: ${stdDevExisting.toFixed(2)}ms (CV: ${cvExisting.toFixed(1)}%)`
);
logger.info(
`Timing difference: ${timingDifference.toFixed(2)}ms (${timingDifferencePercent.toFixed(1)}%)`
`Timing difference (median-based): ${timingDifference.toFixed(2)}ms (${timingDifferencePercent.toFixed(1)}%)`
);
// CRITICAL SECURITY TEST: Timing difference should be minimal
@@ -211,7 +271,34 @@ test.describe("Authentication Security Tests - Vulnerability Prevention", () =>
}
// Fail the test if timing difference exceeds our security threshold
expect(timingDifferencePercent).toBeLessThan(20); // Fail at our actual security threshold
// Note: This uses MEDIAN-based comparison (more robust to outliers than mean)
expect(timingDifferencePercent).toBeLessThan(20);
// Validate measurement reliability using coefficient of variation (CV)
// CV > 50% indicates high variability and unreliable measurements
const maxAcceptableCV = 50; // 50% is reasonable for network-based tests
if (cvNonExistent > maxAcceptableCV) {
logger.warn(
`⚠️ High variability in non-existent user timing (CV: ${cvNonExistent.toFixed(1)}%). ` +
`Test measurements may be unreliable. Consider increasing warm-up or checking CI environment.`
);
}
if (cvExisting > maxAcceptableCV) {
logger.warn(
`⚠️ High variability in existing user timing (CV: ${cvExisting.toFixed(1)}%). ` +
`Test measurements may be unreliable. Consider increasing warm-up or checking CI environment.`
);
}
// These are soft checks - we warn but don't fail the test for high CV
// This allows for noisy CI environments while still alerting to potential issues
if (cvNonExistent <= maxAcceptableCV && cvExisting <= maxAcceptableCV) {
logger.info(
`✅ Measurement reliability good: CV ${cvNonExistent.toFixed(1)}% and ${cvExisting.toFixed(1)}% (threshold: ${maxAcceptableCV}%)`
);
}
});
test("should return consistent status codes regardless of user existence", async ({ request }) => {

1
apps/web/playwright/fixtures/users.ts
View File

@@ -72,6 +72,7 @@ export const createUsersFixture = (page: Page, workerInfo: TestInfo): UsersFixtu
name: uname,
email: userEmail,
password: hashedPassword,
emailVerified: new Date(),
locale: "en-US",
memberships: {
create: {

19
apps/web/playwright/lib/utils.ts
View File

@@ -5,7 +5,7 @@ export async function loginAndGetApiKey(page: Page, users: UsersFixture) {
const user = await users.create();
await user.login();
await page.waitForURL(/\/environments\/[^/]+\/surveys/);
await page.waitForURL(/\/environments\/[^/]+\/surveys/, { timeout: 30000 });
const environmentId =
/\/environments\/([^/]+)\/surveys/.exec(page.url())?.[1] ??
@@ -13,9 +13,9 @@ export async function loginAndGetApiKey(page: Page, users: UsersFixture) {
throw new Error("Unable to parse environmentId from URL");
})();
await page.goto(`/environments/${environmentId}/settings/api-keys`);
await page.goto(`/environments/${environmentId}/settings/api-keys`, { waitUntil: "domcontentloaded" });
await page.getByRole("button", { name: "Add API Key" }).isVisible();
await page.getByRole("button", { name: "Add API Key" }).waitFor({ state: "visible", timeout: 15000 });
await page.getByRole("button", { name: "Add API Key" }).click();
await page.getByPlaceholder("e.g. GitHub, PostHog, Slack").fill("E2E Test API Key");
await page.getByRole("button", { name: "+ Add permission" }).click();
@@ -26,7 +26,18 @@ export async function loginAndGetApiKey(page: Page, users: UsersFixture) {
await page.getByTestId("organization-access-accessControl-read").click();
await page.getByTestId("organization-access-accessControl-write").click();
await page.getByRole("button", { name: "Add API Key" }).click();
await page.locator(".copyApiKeyIcon").click();
// Wait for the API key creation to complete and appear in the list
// Use longer timeouts for cloud environments with high concurrency and network latency
// Wait for network idle to ensure the API key is fully committed to the database
await page.waitForLoadState("networkidle", { timeout: 30000 });
await page.waitForSelector(".copyApiKeyIcon", { state: "visible", timeout: 30000 });
// Add a delay to ensure the API key is fully committed to the database
// This is especially important with high concurrency in cloud environments
await page.waitForTimeout(2000);
await page.locator(".copyApiKeyIcon").first().click();
const apiKey = await page.evaluate("navigator.clipboard.readText()");

20
apps/web/playwright/survey.spec.ts
View File

@@ -1,5 +1,5 @@
import { surveys } from "@/playwright/utils/mock";
import { expect } from "@playwright/test";
import { surveys } from "@/playwright/utils/mock";
import { test } from "./lib/fixtures";
import { createSurvey, createSurveyWithLogic, uploadFileForFileUploadQuestion } from "./utils/helper";
@@ -28,10 +28,13 @@ test.describe("Survey Create & Submit Response without logic", async () => {
await expect(page.locator("#howToSendCardOption-link")).toBeVisible();
await page.locator("#howToSendCardOption-link").click();
// Wait for any auto-save to complete before publishing
await page.waitForTimeout(2000);
await page.getByRole("button", { name: "Publish" }).click();
// Get URL
await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/);
// Get URL - increase timeout for slower local environments
await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/, { timeout: 60000 });
await page.getByLabel("Copy survey link to clipboard").click();
url = await page.evaluate("navigator.clipboard.readText()");
});
@@ -291,7 +294,7 @@ test.describe("Multi Language Survey Create", async () => {
.filter({ hasText: /^Add questionAdd a new question to your survey$/ })
.nth(1)
.click();
await page.getByRole("button", { name: "Multi-Select" }).click();
await page.getByRole("button", { name: "Multi-Select Ask respondents" }).click();
await page.getByLabel("Question*").fill(surveys.createAndSubmit.multiSelectQuestion.question);
await page.getByPlaceholder("Option 1").fill(surveys.createAndSubmit.multiSelectQuestion.options[0]);
await page.getByPlaceholder("Option 2").fill(surveys.createAndSubmit.multiSelectQuestion.options[1]);
@@ -446,7 +449,7 @@ test.describe("Multi Language Survey Create", async () => {
await page.getByPlaceholder("Back").fill(surveys.germanCreate.back);
// Fill Multi select question in german
await page.getByRole("main").getByText("Multi-Select").click();
await page.getByRole("main").getByRole("heading", { name: "Multi-Select" }).click();
await page.getByPlaceholder("Your question here. Recall").click();
await page
@@ -625,10 +628,13 @@ test.describe("Multi Language Survey Create", async () => {
await expect(page.locator("#howToSendCardOption-link")).toBeVisible();
await page.locator("#howToSendCardOption-link").click();
// Wait for any auto-save to complete before publishing
await page.waitForTimeout(2000);
await page.getByRole("button", { name: "Publish" }).click();
await page.waitForTimeout(5000);
await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/);
await page.waitForTimeout(2000);
await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary(\?.*)?$/, { timeout: 60000 });
await page.getByLabel("Select Language").click();
await page.getByText("German").click();
await page.getByLabel("Copy survey link to clipboard").click();

6
apps/web/playwright/utils/helper.ts
View File

@@ -1,9 +1,9 @@
import { CreateSurveyParams, CreateSurveyWithLogicParams } from "@/playwright/utils/mock";
import { expect } from "@playwright/test";
import { readFileSync, writeFileSync } from "fs";
import { Page } from "playwright";
import { logger } from "@formbricks/logger";
import { TProjectConfigChannel } from "@formbricks/types/project";
import { CreateSurveyParams, CreateSurveyWithLogicParams } from "@/playwright/utils/mock";
export const signUpAndLogin = async (
page: Page,
@@ -203,7 +203,7 @@ export const createSurvey = async (page: Page, params: CreateSurveyParams) => {
.filter({ hasText: new RegExp(`^${addQuestion}$`) })
.nth(1)
.click();
await page.getByRole("button", { name: "Multi-Select" }).click();
await page.getByRole("button", { name: "Multi-Select Ask respondents" }).click();
await page.getByLabel("Question*").fill(params.multiSelectQuestion.question);
await page.getByRole("button", { name: "Add description", exact: true }).click();
await page.locator('input[name="subheader"]').fill(params.multiSelectQuestion.description);
@@ -416,7 +416,7 @@ export const createSurveyWithLogic = async (page: Page, params: CreateSurveyWith
.filter({ hasText: new RegExp(`^${addQuestion}$`) })
.nth(1)
.click();
await page.getByRole("button", { name: "Multi-Select" }).click();
await page.getByRole("button", { name: "Multi-Select Ask respondents" }).click();
await page.getByLabel("Question*").fill(params.multiSelectQuestion.question);
await page.getByRole("button", { name: "Add description" }).click();
await page.locator('input[name="subheader"]').fill(params.multiSelectQuestion.description);

2
package.json
View File

@@ -30,7 +30,7 @@
"test": "turbo run test --no-cache",
"test:coverage": "turbo run test:coverage --no-cache",
"test:e2e": "playwright test",
"test-e2e:azure": "pnpm test:e2e -c playwright.service.config.ts --workers=20",
"test-e2e:azure": "pnpm test:e2e -c playwright.service.config.ts --workers=10",
"prepare": "husky install",
"storybook": "turbo run storybook",
"fb-migrate-dev": "pnpm --filter @formbricks/database create-migration && pnpm prisma generate",

4
playwright.config.ts
View File

@@ -14,11 +14,11 @@ export default defineConfig({
/* Run tests in files in parallel */
fullyParallel: true,
/* Retry on CI only */
retries: 0,
retries: process.env.CI ? 2 : 0,
/* Timeout for each test */
timeout: 120000,
/* Fail the test run after the first failure */
maxFailures: 1, // Stop execution after the first failed test
maxFailures: process.env.CI ? undefined : 1, // Allow more failures in CI to avoid cascading shutdowns
/* Opt out of parallel tests on CI. */
// workers: os.cpus().length,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */

4
playwright.service.config.ts
View File

@@ -7,7 +7,7 @@ export default defineConfig(
config,
getServiceConfig(config, {
exposeNetwork: "<loopback>",
timeout: 33000,
timeout: 120000, // Increased timeout for cloud environment with network latency
os: ServiceOS.LINUX,
useCloudHostedBrowsers: true, // Set to false if you want to only use reporting and not cloud hosted browsers
}),
@@ -18,5 +18,7 @@ export default defineConfig(
If you are using more reporters, please update your configuration accordingly.
*/
reporter: [["list"], ["@azure/microsoft-playwright-testing/reporter"]],
retries: 2, // Always retry in cloud environment due to potential network/timing issues
maxFailures: undefined, // Don't stop on first failure to avoid cascading shutdowns with high parallelism
}
);