feat: add playwright e2e tests infrastructure (#1742)

Co-authored-by: ShubhamPalriwala <spalriwalau@gmail.com>

.github/workflows/playwright.yml

@@ -0,0 +1,37 @@
+name: E2E Tests
+on:
+  workflow_call:
+jobs:
+  build:
+    name: Run E2E Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Install Docker Compose
+        run: sudo apt-get update && sudo apt-get install -y docker-compose
+
+      - name: Install dependencies
+        run: npm install -g pnpm && pnpm install
+
+      - name: Build Formbricks Image & Run
+        run: docker-compose up -d
+
+      - name: Install Playwright Browsers
+        run: pnpm exec playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: pnpm test:e2e
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.github/workflows/pr.yml

@@ -27,8 +27,13 @@ jobs:
     uses: ./.github/workflows/build-web.yml
     secrets: inherit
 
+  e2e-test:
+    name: Run E2E Tests
+    uses: ./.github/workflows/playwright.yml
+    secrets: inherit
+
   required:
-    needs: [lint, test, build]
+    needs: [lint, test, build, e2e-test]
     if: always()
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -44,4 +44,10 @@ packages/database/zod
 # nixos stuff
 .direnv
 
-Zone.Identifier
+Zone.Identifier
+
+# Playwright
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

apps/web/playwright/lib/user.ts

@@ -0,0 +1,17 @@
+import { randomBytes } from "crypto";
+
+let user: {
+  name: string;
+  email: string;
+  password: string;
+} | null;
+
+export const getUser = () => {
+  if (!user) {
+    const name = randomBytes(4).toString("hex");
+    const email = `${name}@gmail.com`;
+    const password = "Test@123";
+    user = { name, email, password };
+  }
+  return user;
+};

apps/web/playwright/signup.spec.ts

@@ -0,0 +1,95 @@
+import { getUser } from "@/playwright/lib/user";
+import { test } from "@playwright/test";
+
+const { name, email, password } = getUser();
+
+test.describe("Signup Flow Test", async () => {
+  test.describe.configure({ mode: "serial" });
+
+  test("Valid User", async ({ page }) => {
+    await page.goto("/auth/signup");
+    await page.getByText("Continue with Email").click();
+
+    await page.waitForSelector('input[name="name"]');
+    await page.fill('input[name="name"]', name);
+    await page.press('input[name="name"]', "Tab");
+
+    await page.fill('input[name="email"]', email);
+    await page.press('input[name="email"]', "Tab");
+
+    await page.fill('input[name="password"]', password);
+    await page.press('input[name="password"]', "Enter");
+
+    await page.waitForURL("/auth/signup-without-verification-success");
+  });
+
+  test("Email is taken", async ({ page }) => {
+    await page.goto("/auth/signup");
+    await page.getByText("Continue with Email").click();
+
+    await page.waitForSelector('input[name="name"]');
+    await page.fill('input[name="name"]', name);
+    await page.press('input[name="name"]', "Tab");
+
+    await page.fill('input[name="email"]', email);
+    await page.press('input[name="email"]', "Tab");
+
+    await page.fill('input[name="password"]', password);
+    await page.press('input[name="password"]', "Enter");
+
+    let alertMessage = "user with this email address already exists";
+
+    await (await page.waitForSelector(`text=${alertMessage}`)).isVisible();
+  });
+
+  test("No Name", async ({ page }) => {
+    await page.goto("/auth/signup");
+    await page.getByText("Continue with Email").click();
+
+    await page.waitForSelector('input[name="name"]');
+    await page.fill('input[name="name"]', "");
+    await page.press('input[name="name"]', "Tab");
+
+    await page.fill('input[name="email"]', email);
+    await page.press('input[name="email"]', "Tab");
+
+    await page.fill('input[name="password"]', password);
+    await page.press('input[name="password"]', "Enter");
+
+    await page.getByText("Continue with Email").isDisabled();
+  });
+
+  test("Invalid Email", async ({ page }) => {
+    await page.goto("/auth/signup");
+    await page.getByText("Continue with Email").click();
+
+    await page.waitForSelector('input[name="name"]');
+    await page.fill('input[name="name"]', name);
+    await page.press('input[name="name"]', "Tab");
+
+    await page.fill('input[name="email"]', "invalid");
+    await page.press('input[name="email"]', "Tab");
+
+    await page.fill('input[name="password"]', password);
+    await page.press('input[name="password"]', "Enter");
+
+    await page.getByText("Continue with Email").isDisabled();
+  });
+
+  test("Invalid Password", async ({ page }) => {
+    await page.goto("/auth/signup");
+    await page.getByText("Continue with Email").click();
+
+    await page.waitForSelector('input[name="name"]');
+    await page.fill('input[name="name"]', name);
+    await page.press('input[name="name"]', "Tab");
+
+    await page.fill('input[name="email"]', email);
+    await page.press('input[name="email"]', "Tab");
+
+    await page.fill('input[name="password"]', "invalid");
+    await page.press('input[name="password"]', "Enter");
+
+    await page.getByText("Continue with Email").isDisabled();
+  });
+});

docker-compose.yml

@@ -24,18 +24,15 @@ x-encryption-key: &encryption_key 1b3d888592454d23b520040950654669
 
 x-mail-from: &mail_from
 x-smtp-host: &smtp_host
-x-smtp-port: &smtp_port # Enable SMTP_SECURE_ENABLED for TLS (port 465)
+x-smtp-port: &smtp_port
+x-smtp-secure-enabled: &smtp_secure_enabled # Enable SMTP_SECURE_ENABLED for TLS (port 465)
 
 
-x-smtp-secure-enabled: &smtp_secure_enabled
 x-smtp-user: &smtp_user
 x-smtp-password: &smtp_password
-  # Set the below value to your public-facing URL, e.g., https://example.com
-
-  # Set the below value if you have and want to share a shorter base URL than the x-survey-base-url
 
 x-short-url-base:
-  &short_url_base # Email Verification. If you enable Email Verification you have to setup SMTP-Settings, too.
+  &short_url_base # Set the below value if you have and want to share a shorter base URL than the x-survey-base-url
 
 
 x-email-verification-disabled: &email_verification_disabled 1
@@ -52,23 +49,21 @@ x-invite-disabled: &invite_disabled 0
 # Set the below values to display privacy policy, imprint and terms of service links in the footer of signup & public pages.
 x-privacy-url: &privacy_url
 x-terms-url: &terms_url
-x-imprint-url: &imprint_url # Configure Github Login
+x-imprint-url: &imprint_url
 
-
-x-github-auth-enabled: &github_auth_enabled 0
+x-github-auth-enabled: &github_auth_enabled 0 # Configure Github Login
 x-github-id: &github_id
-x-github-secret: &github_secret # Configure Google Login
+x-github-secret: &github_secret
 
-x-google-auth-enabled: &google_auth_enabled 0
+x-google-auth-enabled: &google_auth_enabled 0 # Configure Google Login
 x-google-client-id: &google_client_id
-x-google-client-secret: &google_client_secret # Disable Sentry warning
+x-google-client-secret: &google_client_secret
 
-x-sentry-ignore-api-resolution-error: &sentry_ignore_api_resolution_error # Enable Sentry Error Tracking
+x-sentry-ignore-api-resolution-error: &sentry_ignore_api_resolution_error # Disable Sentry warning
 
-x-next-public-sentry-dsn: &next_public_sentry_dsn # Cron Secret
+x-next-public-sentry-dsn: &next_public_sentry_dsn # Enable Sentry Error Tracking
 
-# Set this to a random string to secure your cron endpoints
-x-cron-secret: &cron_secret YOUR_CRON_SECRET
+x-cron-secret: &cron_secret YOUR_CRON_SECRET # Set this to a random string to secure your cron endpoints
 
 services:
   postgres:

package.json

@@ -26,10 +26,12 @@
     "lint": "turbo run lint",
     "release": "turbo run build --filter=js... && turbo run build --filter=n8n-node... && changeset publish",
     "test": "turbo run test",
+    "test:e2e": "playwright test",
     "prepare": "husky install"
   },
   "devDependencies": {
     "@changesets/cli": "^2.26.2",
+    "@playwright/test": "^1.40.1",
     "eslint-config-formbricks": "workspace:*",
     "husky": "^8.0.3",
     "lint-staged": "^15.1.0",
@@ -58,5 +60,8 @@
     "budgetPercentIncreaseRed": 20,
     "minimumChangeThreshold": 0,
     "showDetails": true
+  },
+  "dependencies": {
+    "playwright": "^1.40.1"
   }
 }

playwright.config.ts

@@ -0,0 +1,68 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: "./apps/web/playwright",
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Retry on CI only */
+  retries: 2,
+  /* Opt out of parallel tests on CI. */
+  workers: 1,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://localhost:3000",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+
+    // {
+    //   name: "firefox",
+    //   use: { ...devices["Desktop Firefox"] },
+    // },
+
+    // {
+    //   name: "webkit",
+    //   use: { ...devices["Desktop Safari"] },
+    // },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: "Mobile Chrome",
+    //   use: { ...devices["Pixel 5"] },
+    // },
+    // {
+    //   name: "Mobile Safari",
+    //   use: { ...devices["iPhone 12"] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: "Microsoft Edge",
+    //   use: { ...devices["Desktop Edge"], channel: "msedge" },
+    // },
+    // {
+    //   name: "Google Chrome",
+    //   use: { ...devices["Desktop Chrome"], channel: "chrome" },
+    // },
+  ],
+});

pnpm-lock.yaml

@@ -7,10 +7,17 @@ settings:
 importers:
 
   .:
+    dependencies:
+      playwright:
+        specifier: ^1.40.1
+        version: 1.40.1
     devDependencies:
       '@changesets/cli':
         specifier: ^2.26.2
         version: 2.26.2
+      '@playwright/test':
+        specifier: ^1.40.1
+        version: 1.40.1
       eslint-config-formbricks:
         specifier: workspace:*
         version: link:packages/eslint-config-formbricks
@@ -5147,6 +5154,14 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@playwright/test@1.40.1:
+    resolution: {integrity: sha512-EaaawMTOeEItCRvfmkI9v6rBkF1svM8wjl/YPRrg2N2Wmp+4qJYkWtJsbew1szfKKDm6fPLy4YAanBhIlf9dWw==}
+    engines: {node: '>=16'}
+    hasBin: true
+    dependencies:
+      playwright: 1.40.1
+    dev: true
+
   /@preact/preset-vite@2.7.0(@babel/core@7.23.5)(preact@10.19.2)(vite@5.0.6):
     resolution: {integrity: sha512-m5N0FVtxbCCDxNk55NGhsRpKJChYcupcuQHzMJc/Bll07IKZKn8amwYciyKFS9haU6AgzDAJ/ewvApr6Qg1DHw==}
     peerDependencies:
@@ -8808,7 +8823,7 @@ packages:
   /@types/jsonwebtoken@9.0.5:
     resolution: {integrity: sha512-VRLSGzik+Unrup6BsouBeHsf4d1hOEgYWTm/7Nmw1sXoN1+tRly/Gy/po3yeahnP4jfnQWWAhQAqcNfH7ngOkA==}
     dependencies:
-      '@types/node': 20.9.0
+      '@types/node': 20.10.3
     dev: true
 
   /@types/keyv@3.1.4:
@@ -8891,18 +8906,6 @@ packages:
     dependencies:
       undici-types: 5.26.5
 
-  /@types/node@20.8.6:
-    resolution: {integrity: sha512-eWO4K2Ji70QzKUqRy6oyJWUeB7+g2cRagT3T/nxYibYcT4y2BDL8lqolRXjTHmkZCdJfIPaY73KbJAZmcryxTQ==}
-    dependencies:
-      undici-types: 5.25.3
-    dev: true
-
-  /@types/node@20.9.0:
-    resolution: {integrity: sha512-nekiGu2NDb1BcVofVcEKMIwzlx4NjHlcjhoxxKBNLtz15Y1z7MYf549DFvkHSId02Ax6kGwWntIBPC3l/JZcmw==}
-    dependencies:
-      undici-types: 5.26.5
-    dev: true
-
   /@types/normalize-package-data@2.4.3:
     resolution: {integrity: sha512-ehPtgRgaULsFG8x0NeYJvmyH1hmlfsNLujHe9dQEia/7MAJYdzMSi19JtchUHjmBA6XC/75dK55mzZH+RyieSg==}
 
@@ -8919,7 +8922,7 @@ packages:
   /@types/qrcode@1.5.5:
     resolution: {integrity: sha512-CdfBi/e3Qk+3Z/fXYShipBT13OJ2fDO2Q2w5CIP5anLTLIndQG9z6P1cnm+8zCWSpm5dnxMFd/uREtb0EXuQzg==}
     dependencies:
-      '@types/node': 20.8.6
+      '@types/node': 20.10.3
     dev: true
 
   /@types/qs@6.9.9:
@@ -12804,6 +12807,13 @@ packages:
   /fs.realpath@1.0.0:
     resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
 
+  /fsevents@2.3.2:
+    resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
+    engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
+    os: [darwin]
+    requiresBuild: true
+    optional: true
+
   /fsevents@2.3.3:
     resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
     engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@@ -14359,7 +14369,7 @@ packages:
       '@jest/fake-timers': 29.7.0
       '@jest/types': 29.6.3
       '@types/jsdom': 20.0.1
-      '@types/node': 20.8.6
+      '@types/node': 20.10.3
       jest-mock: 29.7.0
       jest-util: 29.7.0
       jsdom: 20.0.3
@@ -17174,6 +17184,20 @@ packages:
     dependencies:
       find-up: 5.0.0
 
+  /playwright-core@1.40.1:
+    resolution: {integrity: sha512-+hkOycxPiV534c4HhpfX6yrlawqVUzITRKwHAmYfmsVreltEl6fAZJ3DPfLMOODw0H3s1Itd6MDCWmP1fl/QvQ==}
+    engines: {node: '>=16'}
+    hasBin: true
+
+  /playwright@1.40.1:
+    resolution: {integrity: sha512-2eHI7IioIpQ0bS1Ovg/HszsN/XKNwEG1kbzSDDmADpclKc7CyqkHw7Mg2JCz/bbCxg25QUPcjksoMW7JcIFQmw==}
+    engines: {node: '>=16'}
+    hasBin: true
+    dependencies:
+      playwright-core: 1.40.1
+    optionalDependencies:
+      fsevents: 2.3.2
+
   /pngjs@5.0.0:
     resolution: {integrity: sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==}
     engines: {node: '>=10.13.0'}
@@ -20133,10 +20157,6 @@ packages:
       which-boxed-primitive: 1.0.2
     dev: true
 
-  /undici-types@5.25.3:
-    resolution: {integrity: sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==}
-    dev: true
-
   /undici-types@5.26.5:
     resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
 

turbo.json

@@ -119,7 +119,8 @@
         "S3_SECRET_KEY",
         "S3_REGION",
         "S3_BUCKET_NAME",
-        "ENTERPRISE_LICENSE_KEY"
+        "ENTERPRISE_LICENSE_KEY",
+        "PLAYWRIGHT_CI"
       ]
     },
     "post-install": {