fix: always validate only responseData fields in client/management APIs (#7296) [Backport to release/4.7] (#7300)

Co-authored-by: Cursor <cursoragent@cursor.com>

apps/web/app/api/v1/client/[environmentId]/responses/[responseId]/route.ts

@@ -44,13 +44,10 @@ const validateResponse = (
     ...responseUpdateInput.data,
   };
 
-  const isFinished = responseUpdateInput.finished ?? false;
-
   const validationErrors = validateResponseData(
     survey.blocks,
     mergedData,
     responseUpdateInput.language ?? response.language ?? "en",
-    isFinished,
     survey.questions
   );
 

apps/web/app/api/v1/client/[environmentId]/responses/route.ts

@@ -41,7 +41,6 @@ const validateResponse = (responseInputData: TResponseInput, survey: TSurvey) =>
     survey.blocks,
     responseInputData.data,
     responseInputData.language ?? "en",
-    responseInputData.finished,
     survey.questions
   );
 

apps/web/app/api/v1/management/responses/[responseId]/route.ts

@@ -146,7 +146,6 @@ export const PUT = withV1ApiWrapper({
         result.survey.blocks,
         responseUpdate.data,
         responseUpdate.language ?? "en",
-        responseUpdate.finished,
         result.survey.questions
       );
 

apps/web/app/api/v1/management/responses/route.ts

@@ -155,7 +155,6 @@ export const POST = withV1ApiWrapper({
         surveyResult.survey.blocks,
         responseInput.data,
         responseInput.language ?? "en",
-        responseInput.finished,
         surveyResult.survey.questions
       );
 

apps/web/app/api/v2/client/[environmentId]/responses/route.ts

@@ -112,7 +112,6 @@ export const POST = async (request: Request, context: Context): Promise<Response
     survey.blocks,
     responseInputData.data,
     responseInputData.language ?? "en",
-    responseInputData.finished,
     survey.questions
   );
 

apps/web/modules/api/lib/validation.test.ts

@@ -95,7 +95,7 @@ describe("validateResponseData", () => {
     mockGetElementsFromBlocks.mockReturnValue(mockElements);
     mockValidateBlockResponses.mockReturnValue({});
 
-    validateResponseData([], mockResponseData, "en", true, mockQuestions);
+    validateResponseData([], mockResponseData, "en", mockQuestions);
 
     expect(mockTransformQuestionsToBlocks).toHaveBeenCalledWith(mockQuestions, []);
     expect(mockGetElementsFromBlocks).toHaveBeenCalledWith(transformedBlocks);
@@ -105,15 +105,15 @@ describe("validateResponseData", () => {
     mockGetElementsFromBlocks.mockReturnValue(mockElements);
     mockValidateBlockResponses.mockReturnValue({});
 
-    validateResponseData(mockBlocks, mockResponseData, "en", true, mockQuestions);
+    validateResponseData(mockBlocks, mockResponseData, "en", mockQuestions);
 
     expect(mockTransformQuestionsToBlocks).not.toHaveBeenCalled();
   });
 
   test("should return null when both blocks and questions are empty", () => {
-    expect(validateResponseData([], mockResponseData, "en", true, [])).toBeNull();
-    expect(validateResponseData(null, mockResponseData, "en", true, [])).toBeNull();
-    expect(validateResponseData(undefined, mockResponseData, "en", true, null)).toBeNull();
+    expect(validateResponseData([], mockResponseData, "en", [])).toBeNull();
+    expect(validateResponseData(null, mockResponseData, "en", [])).toBeNull();
+    expect(validateResponseData(undefined, mockResponseData, "en", null)).toBeNull();
   });
 
   test("should use default language code", () => {
@@ -125,25 +125,58 @@ describe("validateResponseData", () => {
     expect(mockValidateBlockResponses).toHaveBeenCalledWith(mockElements, mockResponseData, "en");
   });
 
-  test("should validate only present fields when finished is false", () => {
+  test("should validate only fields present in responseData", () => {
     const partialResponseData: TResponseData = { element1: "test" };
-    const partialElements = [mockElements[0]];
+    const elementsToValidate = [mockElements[0]];
     mockGetElementsFromBlocks.mockReturnValue(mockElements);
     mockValidateBlockResponses.mockReturnValue({});
 
-    validateResponseData(mockBlocks, partialResponseData, "en", false);
+    validateResponseData(mockBlocks, partialResponseData, "en");
 
-    expect(mockValidateBlockResponses).toHaveBeenCalledWith(partialElements, partialResponseData, "en");
+    expect(mockValidateBlockResponses).toHaveBeenCalledWith(elementsToValidate, partialResponseData, "en");
   });
 
-  test("should validate all fields when finished is true", () => {
-    const partialResponseData: TResponseData = { element1: "test" };
-    mockGetElementsFromBlocks.mockReturnValue(mockElements);
+  test("should never validate elements not in responseData", () => {
+    const blocksWithTwoElements: TSurveyBlock[] = [
+      ...mockBlocks,
+      {
+        id: "block2",
+        name: "Block 2",
+        elements: [
+          {
+            id: "element2",
+            type: TSurveyElementTypeEnum.OpenText,
+            headline: { default: "Q2" },
+            required: true,
+            inputType: "text",
+            charLimit: { enabled: false },
+          },
+        ],
+      },
+    ];
+    const allElements = [
+      ...mockElements,
+      {
+        id: "element2",
+        type: TSurveyElementTypeEnum.OpenText,
+        headline: { default: "Q2" },
+        required: true,
+        inputType: "text",
+        charLimit: { enabled: false },
+      },
+    ];
+    const responseDataWithOnlyElement1: TResponseData = { element1: "test" };
+    mockGetElementsFromBlocks.mockReturnValue(allElements);
     mockValidateBlockResponses.mockReturnValue({});
 
-    validateResponseData(mockBlocks, partialResponseData, "en", true);
+    validateResponseData(blocksWithTwoElements, responseDataWithOnlyElement1, "en");
 
-    expect(mockValidateBlockResponses).toHaveBeenCalledWith(mockElements, partialResponseData, "en");
+    // Only element1 should be validated, not element2 (even though it's required)
+    expect(mockValidateBlockResponses).toHaveBeenCalledWith(
+      [allElements[0]],
+      responseDataWithOnlyElement1,
+      "en"
+    );
   });
 });
 

apps/web/modules/api/lib/validation.ts

@@ -9,13 +9,13 @@ import { getElementsFromBlocks } from "@/lib/survey/utils";
 import { ApiErrorDetails } from "@/modules/api/v2/types/api-error";
 
 /**
- * Validates response data against survey validation rules
- * Handles partial responses (in-progress) by only validating present fields when finished is false
+ * Validates response data against survey validation rules.
+ * Only validates elements that have data in responseData - never validates
+ * all survey elements regardless of completion status.
  *
  * @param blocks - Survey blocks containing elements with validation rules (preferred)
  * @param responseData - Response data to validate (keyed by element ID)
  * @param languageCode - Language code for error messages (defaults to "en")
- * @param finished - Whether the response is finished (defaults to true for management APIs)
  * @param questions - Survey questions (legacy format, used as fallback if blocks are empty)
  * @returns Validation error map keyed by element ID, or null if validation passes
  */
@@ -23,7 +23,6 @@ export const validateResponseData = (
   blocks: TSurveyBlock[] | undefined | null,
   responseData: TResponseData,
   languageCode: string = "en",
-  finished: boolean = true,
   questions?: TSurveyQuestion[] | undefined | null
 ): TValidationErrorMap | null => {
   // Use blocks if available, otherwise transform questions to blocks
@@ -42,11 +41,8 @@ export const validateResponseData = (
   // Extract elements from blocks
   const allElements = getElementsFromBlocks(blocksToUse);
 
-  // If response is not finished, only validate elements that are present in the response data
-  // This prevents "required" errors for fields the user hasn't reached yet
-  const elementsToValidate = finished
-    ? allElements
-    : allElements.filter((element) => Object.keys(responseData).includes(element.id));
+  // Always validate only elements that are present in responseData
+  const elementsToValidate = allElements.filter((element) => Object.keys(responseData).includes(element.id));
 
   // Validate selected elements
   const errorMap = validateBlockResponses(elementsToValidate, responseData, languageCode);

apps/web/modules/api/v2/management/responses/[responseId]/route.ts

@@ -198,7 +198,6 @@ export const PUT = (request: Request, props: { params: Promise<{ responseId: str
         questionsResponse.data.blocks,
         body.data,
         body.language ?? "en",
-        body.finished,
         questionsResponse.data.questions
       );
 

apps/web/modules/api/v2/management/responses/route.ts

@@ -134,7 +134,6 @@ export const POST = async (request: Request) =>
         surveyQuestions.data.blocks,
         body.data,
         body.language ?? "en",
-        body.finished,
         surveyQuestions.data.questions
       );
 

apps/web/modules/ee/contacts/segments/lib/filter/prisma-query.test.ts

@@ -5,10 +5,14 @@ import { getSegment } from "../segments";
 import { segmentFilterToPrismaQuery } from "./prisma-query";
 
 const mockQueryRawUnsafe = vi.fn();
+const mockFindFirst = vi.fn();
 
 vi.mock("@formbricks/database", () => ({
   prisma: {
     $queryRawUnsafe: (...args: unknown[]) => mockQueryRawUnsafe(...args),
+    contactAttribute: {
+      findFirst: (...args: unknown[]) => mockFindFirst(...args),
+    },
   },
 }));
 
@@ -26,7 +30,9 @@ describe("segmentFilterToPrismaQuery", () => {
 
   beforeEach(() => {
     vi.clearAllMocks();
-    // Default mock: number filter raw SQL returns one matching contact
+    // Default: backfill is complete, no un-migrated rows
+    mockFindFirst.mockResolvedValue(null);
+    // Fallback path mock: raw SQL returns one matching contact when un-migrated rows exist
     mockQueryRawUnsafe.mockResolvedValue([{ contactId: "mock-contact-1" }]);
   });
 
@@ -145,7 +151,16 @@ describe("segmentFilterToPrismaQuery", () => {
             },
           },
         ],
-        OR: [{ id: { in: ["mock-contact-1"] } }],
+        OR: [
+          {
+            attributes: {
+              some: {
+                attributeKey: { key: "age" },
+                valueNumber: { gt: 30 },
+              },
+            },
+          },
+        ],
       });
     }
   });
@@ -757,7 +772,12 @@ describe("segmentFilterToPrismaQuery", () => {
       });
 
       expect(subgroup.AND[0].AND[2]).toStrictEqual({
-        id: { in: ["mock-contact-1"] },
+        attributes: {
+          some: {
+            attributeKey: { key: "age" },
+            valueNumber: { gte: 18 },
+          },
+        },
       });
 
       // Segment inclusion
@@ -1158,10 +1178,23 @@ describe("segmentFilterToPrismaQuery", () => {
         },
       });
 
-      // Second subgroup (numeric operators - now use raw SQL subquery returning contact IDs)
+      // Second subgroup (numeric operators - uses clean Prisma filter post-backfill)
       const secondSubgroup = whereClause.AND?.[0];
       expect(secondSubgroup.AND[1].AND).toContainEqual({
-        id: { in: ["mock-contact-1"] },
+        attributes: {
+          some: {
+            attributeKey: { key: "loginCount" },
+            valueNumber: { gt: 5 },
+          },
+        },
+      });
+      expect(secondSubgroup.AND[1].AND).toContainEqual({
+        attributes: {
+          some: {
+            attributeKey: { key: "purchaseAmount" },
+            valueNumber: { lte: 1000 },
+          },
+        },
       });
 
       // Third subgroup (negation operators in OR clause)
@@ -1232,7 +1265,7 @@ describe("segmentFilterToPrismaQuery", () => {
             {
               attributes: {
                 some: {
-                  attributeKey: { key: "purchaseDate" },
+                  attributeKey: { key: "purchaseDate", dataType: "date" },
                   OR: [
                     { valueDate: { lt: new Date(targetDate) } },
                     { valueDate: null, value: { lt: new Date(targetDate).toISOString() } },
@@ -1276,7 +1309,7 @@ describe("segmentFilterToPrismaQuery", () => {
             {
               attributes: {
                 some: {
-                  attributeKey: { key: "signupDate" },
+                  attributeKey: { key: "signupDate", dataType: "date" },
                   OR: [
                     { valueDate: { gt: new Date(targetDate) } },
                     { valueDate: null, value: { gt: new Date(targetDate).toISOString() } },
@@ -1321,7 +1354,7 @@ describe("segmentFilterToPrismaQuery", () => {
             {
               attributes: {
                 some: {
-                  attributeKey: { key: "lastActivityDate" },
+                  attributeKey: { key: "lastActivityDate", dataType: "date" },
                   OR: [
                     { valueDate: { gte: new Date(startDate), lte: new Date(endDate) } },
                     {
@@ -1638,8 +1671,15 @@ describe("segmentFilterToPrismaQuery", () => {
           mode: "insensitive",
         });
 
-        // Number filter uses raw SQL subquery (transition code) returning contact IDs
-        expect(andConditions[1]).toEqual({ id: { in: ["mock-contact-1"] } });
+        // Number filter uses clean Prisma filter post-backfill
+        expect(andConditions[1]).toEqual({
+          attributes: {
+            some: {
+              attributeKey: { key: "purchaseCount" },
+              valueNumber: { gt: 5 },
+            },
+          },
+        });
 
         // Date filter uses OR fallback with 'valueDate' and string 'value'
         expect((andConditions[2] as unknown as any).attributes.some.OR[0].valueDate).toHaveProperty("gte");

apps/web/modules/ee/contacts/segments/lib/filter/prisma-query.ts

@@ -107,7 +107,7 @@ const buildDateAttributeFilterWhereClause = (filter: TSegmentAttributeFilter): P
   return {
     attributes: {
       some: {
-        attributeKey: { key: contactAttributeKey },
+        attributeKey: { key: contactAttributeKey, dataType: "date" },
         OR: [{ valueDate: dateCondition }, { valueDate: null, value: stringDateCondition }],
       },
     },
@@ -116,59 +116,102 @@ const buildDateAttributeFilterWhereClause = (filter: TSegmentAttributeFilter): P
 
 /**
  * Builds a Prisma where clause for number attribute filters.
- * Uses a raw SQL subquery to handle both migrated rows (valueNumber populated)
- * and un-migrated rows (valueNumber NULL, value contains numeric string).
- * This is transition code for the deferred value backfill.
+ * Uses a clean Prisma query when all rows have valueNumber populated (post-backfill).
+ * Falls back to a raw SQL subquery for un-migrated rows (valueNumber NULL, value contains numeric string).
  *
  * TODO: After the backfill script has been run and all valueNumber columns are populated,
- * revert this to the clean Prisma-only version that queries valueNumber directly.
+ * remove the un-migrated fallback path entirely.
  */
 const buildNumberAttributeFilterWhereClause = async (
-  filter: TSegmentAttributeFilter
+  filter: TSegmentAttributeFilter,
+  environmentId: string
 ): Promise<Prisma.ContactWhereInput> => {
   const { root, qualifier, value } = filter;
   const { contactAttributeKey } = root;
   const { operator } = qualifier;
 
   const numericValue = typeof value === "number" ? value : Number(value);
-  const sqlOp = SQL_OPERATORS[operator];
 
-  if (!sqlOp) {
-    return {};
+  let valueNumberCondition: Prisma.FloatNullableFilter;
+
+  switch (operator) {
+    case "greaterThan":
+      valueNumberCondition = { gt: numericValue };
+      break;
+    case "greaterEqual":
+      valueNumberCondition = { gte: numericValue };
+      break;
+    case "lessThan":
+      valueNumberCondition = { lt: numericValue };
+      break;
+    case "lessEqual":
+      valueNumberCondition = { lte: numericValue };
+      break;
+    default:
+      return {};
   }
 
-  const matchingContactIds = await prisma.$queryRawUnsafe<{ contactId: string }[]>(
+  const migratedFilter: Prisma.ContactWhereInput = {
+    attributes: {
+      some: {
+        attributeKey: { key: contactAttributeKey },
+        valueNumber: valueNumberCondition,
+      },
+    },
+  };
+
+  const hasUnmigratedRows = await prisma.contactAttribute.findFirst({
+    where: {
+      attributeKey: {
+        key: contactAttributeKey,
+        environmentId,
+        dataType: "number",
+      },
+      valueNumber: null,
+    },
+    select: { id: true },
+  });
+
+  if (!hasUnmigratedRows) {
+    return migratedFilter;
+  }
+
+  const sqlOp = SQL_OPERATORS[operator];
+  const unmigratedMatchingIds = await prisma.$queryRawUnsafe<{ contactId: string }[]>(
     `
     SELECT DISTINCT ca."contactId"
     FROM "ContactAttribute" ca
     JOIN "ContactAttributeKey" cak ON ca."attributeKeyId" = cak.id
     WHERE cak.key = $1
-    AND (
-      (ca."valueNumber" IS NOT NULL AND ca."valueNumber" ${sqlOp} $2)
-      OR
-      (ca."valueNumber" IS NULL AND ca.value ~ $3 AND ca.value::double precision ${sqlOp} $2)
-    )
+    AND cak."environmentId" = $4
+    AND cak."dataType" = 'number'
+    AND ca."valueNumber" IS NULL
+    AND ca.value ~ $3
+    AND ca.value::double precision ${sqlOp} $2
     `,
     contactAttributeKey,
     numericValue,
-    NUMBER_PATTERN_SQL
+    NUMBER_PATTERN_SQL,
+    environmentId
   );
 
-  const contactIds = matchingContactIds.map((r) => r.contactId);
-
-  if (contactIds.length === 0) {
-    // Return an impossible condition so the filter correctly excludes all contacts
-    return { id: "__NUMBER_FILTER_NO_MATCH__" };
+  if (unmigratedMatchingIds.length === 0) {
+    return migratedFilter;
   }
 
-  return { id: { in: contactIds } };
+  const contactIds = unmigratedMatchingIds.map((r) => r.contactId);
+
+  return {
+    OR: [migratedFilter, { id: { in: contactIds } }],
+  };
 };
 
 /**
  * Builds a Prisma where clause from a segment attribute filter
  */
 const buildAttributeFilterWhereClause = async (
-  filter: TSegmentAttributeFilter
+  filter: TSegmentAttributeFilter,
+  environmentId: string
 ): Promise<Prisma.ContactWhereInput> => {
   const { root, qualifier, value } = filter;
   const { contactAttributeKey } = root;
@@ -215,7 +258,7 @@ const buildAttributeFilterWhereClause = async (
 
   // Handle number operators
   if (["greaterThan", "greaterEqual", "lessThan", "lessEqual"].includes(operator)) {
-    return await buildNumberAttributeFilterWhereClause(filter);
+    return await buildNumberAttributeFilterWhereClause(filter, environmentId);
   }
 
   // For string operators, ensure value is a primitive (not an object or array)
@@ -253,7 +296,8 @@ const buildAttributeFilterWhereClause = async (
  * Builds a Prisma where clause from a person filter
  */
 const buildPersonFilterWhereClause = async (
-  filter: TSegmentPersonFilter
+  filter: TSegmentPersonFilter,
+  environmentId: string
 ): Promise<Prisma.ContactWhereInput> => {
   const { personIdentifier } = filter.root;
 
@@ -265,7 +309,7 @@ const buildPersonFilterWhereClause = async (
         contactAttributeKey: personIdentifier,
       },
     };
-    return await buildAttributeFilterWhereClause(personFilter);
+    return await buildAttributeFilterWhereClause(personFilter, environmentId);
   }
 
   return {};
@@ -314,6 +358,7 @@ const buildDeviceFilterWhereClause = (
 const buildSegmentFilterWhereClause = async (
   filter: TSegmentSegmentFilter,
   segmentPath: Set<string>,
+  environmentId: string,
   deviceType?: "phone" | "desktop"
 ): Promise<Prisma.ContactWhereInput> => {
   const { root } = filter;
@@ -337,7 +382,7 @@ const buildSegmentFilterWhereClause = async (
   const newPath = new Set(segmentPath);
   newPath.add(segmentId);
 
-  return processFilters(segment.filters, newPath, deviceType);
+  return processFilters(segment.filters, newPath, environmentId, deviceType);
 };
 
 /**
@@ -346,19 +391,25 @@ const buildSegmentFilterWhereClause = async (
 const processSingleFilter = async (
   filter: TSegmentFilter,
   segmentPath: Set<string>,
+  environmentId: string,
   deviceType?: "phone" | "desktop"
 ): Promise<Prisma.ContactWhereInput> => {
   const { root } = filter;
 
   switch (root.type) {
     case "attribute":
-      return await buildAttributeFilterWhereClause(filter as TSegmentAttributeFilter);
+      return await buildAttributeFilterWhereClause(filter as TSegmentAttributeFilter, environmentId);
     case "person":
-      return await buildPersonFilterWhereClause(filter as TSegmentPersonFilter);
+      return await buildPersonFilterWhereClause(filter as TSegmentPersonFilter, environmentId);
     case "device":
       return buildDeviceFilterWhereClause(filter as TSegmentDeviceFilter, deviceType);
     case "segment":
-      return await buildSegmentFilterWhereClause(filter as TSegmentSegmentFilter, segmentPath, deviceType);
+      return await buildSegmentFilterWhereClause(
+        filter as TSegmentSegmentFilter,
+        segmentPath,
+        environmentId,
+        deviceType
+      );
     default:
       return {};
   }
@@ -370,6 +421,7 @@ const processSingleFilter = async (
 const processFilters = async (
   filters: TBaseFilters,
   segmentPath: Set<string>,
+  environmentId: string,
   deviceType?: "phone" | "desktop"
 ): Promise<Prisma.ContactWhereInput> => {
   if (filters.length === 0) return {};
@@ -386,10 +438,10 @@ const processFilters = async (
     // Process the resource based on its type
     if (isResourceFilter(resource)) {
       // If it's a single filter, process it directly
-      whereClause = await processSingleFilter(resource, segmentPath, deviceType);
+      whereClause = await processSingleFilter(resource, segmentPath, environmentId, deviceType);
     } else {
       // If it's a group of filters, process it recursively
-      whereClause = await processFilters(resource, segmentPath, deviceType);
+      whereClause = await processFilters(resource, segmentPath, environmentId, deviceType);
     }
 
     if (Object.keys(whereClause).length === 0) continue;
@@ -432,7 +484,7 @@ export const segmentFilterToPrismaQuery = reactCache(
 
       // Initialize an empty stack for tracking the current evaluation path
       const segmentPath = new Set<string>([segmentId]);
-      const filtersWhereClause = await processFilters(filters, segmentPath, deviceType);
+      const filtersWhereClause = await processFilters(filters, segmentPath, environmentId, deviceType);
 
       const whereClause = {
         AND: [baseWhereClause, filtersWhereClause],

apps/web/modules/ee/contacts/segments/lib/segments.ts

@@ -136,28 +136,48 @@ export const createSegment = async (segmentCreateInput: TSegmentCreateInput): Pr
 
   const { description, environmentId, filters, isPrivate, surveyId, title } = segmentCreateInput;
 
-  let data: Prisma.SegmentCreateArgs["data"] = {
-    environmentId,
-    title,
-    description,
-    isPrivate,
-    filters,
-  };
-
-  if (surveyId) {
-    data = {
-      ...data,
-      surveys: {
-        connect: {
-          id: surveyId,
-        },
-      },
-    };
-  }
+  const surveyConnect = surveyId ? { surveys: { connect: { id: surveyId } } } : {};
 
   try {
+    // Private segments use upsert because auto-save may have already created a
+    // default (empty-filter) segment via connectOrCreate before the user publishes.
+    // Without upsert the second create hits the (environmentId, title) unique constraint.
+    if (isPrivate) {
+      const segment = await prisma.segment.upsert({
+        where: {
+          environmentId_title: {
+            environmentId,
+            title,
+          },
+        },
+        create: {
+          environmentId,
+          title,
+          description,
+          isPrivate,
+          filters,
+          ...surveyConnect,
+        },
+        update: {
+          description,
+          filters,
+          ...surveyConnect,
+        },
+        select: selectSegment,
+      });
+
+      return transformPrismaSegment(segment);
+    }
+
     const segment = await prisma.segment.create({
-      data,
+      data: {
+        environmentId,
+        title,
+        description,
+        isPrivate,
+        filters,
+        ...surveyConnect,
+      },
       select: selectSegment,
     });
 

packages/surveys/src/lib/validation/evaluator.ts

@@ -137,6 +137,11 @@ const checkRequiredField = (
     return null;
   }
 
+  // CTA elements never block progression (informational only)
+  if (element.type === TSurveyElementTypeEnum.CTA) {
+    return null;
+  }
+
   if (element.type === TSurveyElementTypeEnum.Ranking) {
     return validateRequiredRanking(value, t);
   }