fix: recall parsing in emails and improvements in response pipeline (#2187)

Co-authored-by: Matti Nannt <mail@matthiasnannt.com>
Co-authored-by: Shubham Palriwala <spalriwalau@gmail.com>

apps/web/app/(app)/environments/[environmentId]/surveys/[surveyId]/components/CustomFilter.tsx

@@ -15,7 +15,7 @@ import { ChevronDown, ChevronUp, DownloadIcon } from "lucide-react";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import toast from "react-hot-toast";
 
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 import { TSurveyPersonAttributes } from "@formbricks/types/responses";
 import { TSurvey } from "@formbricks/types/surveys";
 import { TTag } from "@formbricks/types/tags";

apps/web/app/(app)/environments/[environmentId]/surveys/[surveyId]/components/QuestionFilterComboBox.tsx

@@ -6,7 +6,7 @@ import { ChevronDown, ChevronUp, X } from "lucide-react";
 import * as React from "react";
 
 import { getLocalizedValue } from "@formbricks/lib/i18n/utils";
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 import { TSurveyQuestionType } from "@formbricks/types/surveys";
 import { Command, CommandEmpty, CommandGroup, CommandItem, CommandList } from "@formbricks/ui/Command";
 import {

apps/web/app/(app)/environments/[environmentId]/surveys/[surveyId]/components/QuestionsComboBox.tsx

@@ -18,7 +18,7 @@ import {
 import * as React from "react";
 
 import { getLocalizedValue } from "@formbricks/lib/i18n/utils";
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 import { TSurveyQuestionType } from "@formbricks/types/surveys";
 import {
   Command,

apps/web/app/api/cron/weekly_summary/route.ts

@@ -4,6 +4,7 @@ import { headers } from "next/headers";
 import { prisma } from "@formbricks/database";
 import { CRON_SECRET } from "@formbricks/lib/constants";
 import { getLocalizedValue } from "@formbricks/lib/i18n/utils";
+import { checkForRecallInHeadline } from "@formbricks/lib/utils/recall";
 
 import { sendNoLiveSurveyNotificationEmail, sendWeeklySummaryNotificationEmail } from "./email";
 import { EnvironmentData, NotificationResponse, ProductData, Survey, SurveyResponse } from "./types";
@@ -175,21 +176,22 @@ const getNotificationResponse = (environment: EnvironmentData, productName: stri
   const surveys: Survey[] = [];
   // iterate through the surveys and calculate the overall insights
   for (const survey of environment.surveys) {
+    const parsedSurvey = checkForRecallInHeadline(survey, "default");
     const surveyData: Survey = {
-      id: survey.id,
-      name: survey.name,
-      status: survey.status,
-      responseCount: survey.responses.length,
+      id: parsedSurvey.id,
+      name: parsedSurvey.name,
+      status: parsedSurvey.status,
+      responseCount: parsedSurvey.responses.length,
       responses: [],
     };
     // iterate through the responses and calculate the survey insights
-    for (const response of survey.responses) {
+    for (const response of parsedSurvey.responses) {
       // only take the first 3 responses
       if (surveyData.responses.length >= 1) {
         break;
       }
       const surveyResponse: SurveyResponse = {};
-      for (const question of survey.questions) {
+      for (const question of parsedSurvey.questions) {
         const headline = question.headline;
         const answer = response.data[question.id]?.toString() || null;
         if (answer === null || answer === "" || answer?.length === 0) {

apps/web/app/api/pipeline/lib/handleIntegrations.ts

@@ -2,7 +2,6 @@ import { writeData as airtableWriteData } from "@formbricks/lib/airtable/service
 import { writeData } from "@formbricks/lib/googleSheet/service";
 import { getLocalizedValue } from "@formbricks/lib/i18n/utils";
 import { writeData as writeNotionData } from "@formbricks/lib/notion/service";
-import { getSurvey } from "@formbricks/lib/survey/service";
 import { TIntegration } from "@formbricks/types/integration";
 import { TIntegrationAirtable } from "@formbricks/types/integration/airtable";
 import { TIntegrationGoogleSheets } from "@formbricks/types/integration/googleSheet";
@@ -13,28 +12,32 @@ import { TSurvey, TSurveyQuestionType } from "@formbricks/types/surveys";
 export async function handleIntegrations(
   integrations: TIntegration[],
   data: TPipelineInput,
-  surveyData: TSurvey
+  survey: TSurvey
 ) {
   for (const integration of integrations) {
     switch (integration.type) {
       case "googleSheets":
-        await handleGoogleSheetsIntegration(integration as TIntegrationGoogleSheets, data);
+        await handleGoogleSheetsIntegration(integration as TIntegrationGoogleSheets, data, survey);
         break;
       case "airtable":
-        await handleAirtableIntegration(integration as TIntegrationAirtable, data);
+        await handleAirtableIntegration(integration as TIntegrationAirtable, data, survey);
         break;
       case "notion":
-        await handleNotionIntegration(integration as TIntegrationNotion, data, surveyData);
+        await handleNotionIntegration(integration as TIntegrationNotion, data, survey);
         break;
     }
   }
 }
 
-async function handleAirtableIntegration(integration: TIntegrationAirtable, data: TPipelineInput) {
+async function handleAirtableIntegration(
+  integration: TIntegrationAirtable,
+  data: TPipelineInput,
+  survey: TSurvey
+) {
   if (integration.config.data.length > 0) {
     for (const element of integration.config.data) {
       if (element.surveyId === data.surveyId) {
-        const values = await extractResponses(data, element.questionIds as string[]);
+        const values = await extractResponses(data, element.questionIds as string[], survey);
 
         await airtableWriteData(integration.config.key, element, values);
       }
@@ -42,21 +45,28 @@ async function handleAirtableIntegration(integration: TIntegrationAirtable, data
   }
 }
 
-async function handleGoogleSheetsIntegration(integration: TIntegrationGoogleSheets, data: TPipelineInput) {
+async function handleGoogleSheetsIntegration(
+  integration: TIntegrationGoogleSheets,
+  data: TPipelineInput,
+  survey: TSurvey
+) {
   if (integration.config.data.length > 0) {
     for (const element of integration.config.data) {
       if (element.surveyId === data.surveyId) {
-        const values = await extractResponses(data, element.questionIds as string[]);
+        const values = await extractResponses(data, element.questionIds as string[], survey);
         await writeData(integration.config.key, element.spreadsheetId, values);
       }
     }
   }
 }
 
-async function extractResponses(data: TPipelineInput, questionIds: string[]): Promise<string[][]> {
+async function extractResponses(
+  data: TPipelineInput,
+  questionIds: string[],
+  survey: TSurvey
+): Promise<string[][]> {
   const responses: string[] = [];
   const questions: string[] = [];
-  const survey = await getSurvey(data.surveyId);
 
   for (const questionId of questionIds) {
     const responseValue = data.response.data[questionId];
@@ -66,7 +76,6 @@ async function extractResponses(data: TPipelineInput, questionIds: string[]): Pr
     } else {
       responses.push("");
     }
-
     const question = survey?.questions.find((q) => q.id === questionId);
     questions.push(getLocalizedValue(question?.headline, "default") || "");
   }

apps/web/app/api/pipeline/route.ts

@@ -10,8 +10,8 @@ import { getProductByEnvironmentId } from "@formbricks/lib/product/service";
 import { getResponseCountBySurveyId } from "@formbricks/lib/response/service";
 import { getSurvey, updateSurvey } from "@formbricks/lib/survey/service";
 import { convertDatesInObject } from "@formbricks/lib/time";
+import { checkForRecallInHeadline } from "@formbricks/lib/utils/recall";
 import { ZPipelineInput } from "@formbricks/types/pipelines";
-import { TSurveyQuestion } from "@formbricks/types/surveys";
 import { TUserNotificationSettings } from "@formbricks/types/user";
 
 import { handleIntegrations } from "./lib/handleIntegrations";
@@ -102,22 +102,14 @@ export async function POST(request: Request) {
       },
     });
 
-    let surveyData;
+    const [integrations, surveyData] = await Promise.all([
+      getIntegrations(environmentId),
+      getSurvey(surveyId),
+    ]);
+    const survey = surveyData ? checkForRecallInHeadline(surveyData, "default") : undefined;
 
-    const integrations = await getIntegrations(environmentId);
-
-    if (integrations.length > 0) {
-      surveyData = await prisma.survey.findUnique({
-        where: {
-          id: surveyId,
-        },
-        select: {
-          id: true,
-          name: true,
-          questions: true,
-        },
-      });
-      handleIntegrations(integrations, inputValidation.data, surveyData);
+    if (integrations.length > 0 && survey) {
+      handleIntegrations(integrations, inputValidation.data, survey);
     }
     // filter all users that have email notifications enabled for this survey
     const usersWithNotifications = users.filter((user) => {
@@ -132,32 +124,12 @@ export async function POST(request: Request) {
     const responseCount = await getResponseCountBySurveyId(surveyId);
 
     if (usersWithNotifications.length > 0) {
-      // get survey
-      if (!surveyData) {
-        surveyData = await prisma.survey.findUnique({
-          where: {
-            id: surveyId,
-          },
-          select: {
-            id: true,
-            name: true,
-            questions: true,
-          },
-        });
-      }
-
-      if (!surveyData) {
+      if (!survey) {
         console.error(`Pipeline: Survey with id ${surveyId} not found`);
         return new Response("Survey not found", {
           status: 404,
         });
       }
-      // create survey object
-      const survey = {
-        id: surveyData.id,
-        name: surveyData.name,
-        questions: structuredClone(surveyData.questions) as TSurveyQuestion[],
-      };
       // send email to all users
       await Promise.all(
         usersWithNotifications.map(async (user) => {

apps/web/app/share/[sharingKey]/components/CustomFilter.tsx

@@ -10,7 +10,7 @@ import { differenceInDays, format, startOfDay, subDays } from "date-fns";
 import { ChevronDown, ChevronUp } from "lucide-react";
 import { useCallback, useEffect, useRef, useState } from "react";
 
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 import { TSurveyPersonAttributes } from "@formbricks/types/responses";
 import { TSurvey } from "@formbricks/types/surveys";
 import { TTag } from "@formbricks/types/tags";

packages/ee/multiLanguage/components/LanguageIndicator.tsx

@@ -1,7 +1,7 @@
 import { ChevronDown } from "lucide-react";
 import { useRef, useState } from "react";
 
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 import { TSurveyLanguage } from "@formbricks/types/surveys";
 
 import { getLanguageLabel } from "../lib/isoLanguages";

packages/ee/multiLanguage/components/LanguageSelect.tsx

@@ -1,7 +1,7 @@
 import { ChevronDown } from "lucide-react";
 import { useEffect, useRef, useState } from "react";
 
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 import { TLanguage } from "@formbricks/types/product";
 import { Button } from "@formbricks/ui/Button";
 import { Input } from "@formbricks/ui/Input";

packages/lib/emails/emails.ts

@@ -1,5 +1,5 @@
 import { TResponse } from "@formbricks/types/responses";
-import { TSurveyQuestion, TSurveyQuestionType } from "@formbricks/types/surveys";
+import { TSurvey, TSurveyQuestionType } from "@formbricks/types/surveys";
 
 import {
   DEBUG,
@@ -184,7 +184,7 @@ export const sendInviteAcceptedEmail = async (inviterName: string, inviteeName:
 export const sendResponseFinishedEmail = async (
   email: string,
   environmentId: string,
-  survey: { id: string; name: string; questions: TSurveyQuestion[] },
+  survey: TSurvey,
   response: TResponse,
   responseCount: number
 ) => {

packages/lib/response/service.ts

@@ -45,6 +45,7 @@ import { getSurvey } from "../survey/service";
 import { captureTelemetry } from "../telemetry";
 import { formatDateFields } from "../utils/datetime";
 import { convertToCsv, convertToXlsxBuffer } from "../utils/fileConversion";
+import { checkForRecallInHeadline } from "../utils/recall";
 import { validateInputs } from "../utils/validate";
 import { responseCache } from "./cache";
 
@@ -550,7 +551,10 @@ export const getSurveySummary = (
 
       const meta = getSurveySummaryMeta(responses, displayCount);
       const dropOff = getSurveySummaryDropOff(survey, responses, displayCount);
-      const questionWiseSummary = getQuestionWiseSummary(survey, responses);
+      const questionWiseSummary = getQuestionWiseSummary(
+        checkForRecallInHeadline(survey, "default"),
+        responses
+      );
 
       return { meta, dropOff, summary: questionWiseSummary };
     },

packages/lib/responses.ts

@@ -1,10 +1,10 @@
 import { TResponse } from "@formbricks/types/responses";
-import { TSurveyQuestion, TSurveyQuestionType } from "@formbricks/types/surveys";
+import { TSurvey, TSurveyQuestionType } from "@formbricks/types/surveys";
 
 import { getLocalizedValue } from "./i18n/utils";
 
 export const getQuestionResponseMapping = (
-  survey: { questions: TSurveyQuestion[] },
+  survey: TSurvey,
   response: TResponse
 ): { question: string; answer: string | string[]; type: TSurveyQuestionType }[] => {
   const questionResponseMapping: {

packages/lib/utils/hooks/useClickOutside.ts

@@ -1,7 +1,7 @@
 import { RefObject, useEffect } from "react";
 
 // Improved version of https://usehooks.com/useOnClickOutside/
-const useClickOutside = (
+export const useClickOutside = (
   ref: RefObject<HTMLElement>,
   handler: (event: MouseEvent | TouchEvent) => void
 ): void => {
@@ -34,5 +34,3 @@ const useClickOutside = (
     };
   }, [ref, handler]);
 };
-
-export default useClickOutside;

packages/lib/utils/hooks/useSyncScroll.ts

@@ -0,0 +1,26 @@
+import { RefObject, useEffect } from "react";
+
+// Custom hook to synchronize the horizontal scroll position of two elements.
+export const useSyncScroll = (
+  highlightContainerRef: RefObject<HTMLElement>,
+  inputRef: RefObject<HTMLElement>
+): void => {
+  useEffect(() => {
+    const syncScrollPosition = () => {
+      if (highlightContainerRef.current && inputRef.current) {
+        highlightContainerRef.current.scrollLeft = inputRef.current.scrollLeft;
+      }
+    };
+
+    const sourceElement = inputRef.current;
+    if (sourceElement) {
+      sourceElement.addEventListener("scroll", syncScrollPosition);
+    }
+
+    return () => {
+      if (sourceElement) {
+        sourceElement.removeEventListener("scroll", syncScrollPosition);
+      }
+    };
+  }, [inputRef, highlightContainerRef]);
+};

packages/lib/utils/recall.ts

@@ -1,6 +1,4 @@
-import { RefObject, useEffect } from "react";
-
-import { TI18nString, TSurvey, TSurveyQuestion } from "@formbricks/types/surveys";
+import { TI18nString, TSurvey, TSurveyQuestion, TSurveyQuestionsObject } from "@formbricks/types/surveys";
 
 import { getLocalizedValue } from "../i18n/utils";
 
@@ -52,9 +50,9 @@ export const findRecallInfoById = (text: string, id: string): string | null => {
 };
 
 // Converts recall information in a headline to a corresponding recall question headline, with or without a slash.
-export const recallToHeadline = (
+export const recallToHeadline = <T extends TSurveyQuestionsObject>(
   headline: TI18nString,
-  survey: TSurvey,
+  survey: T,
   withSlash: boolean,
   language: string
 ): TI18nString => {
@@ -120,8 +118,11 @@ export const checkForEmptyFallBackValue = (survey: TSurvey, langauge: string): T
 };
 
 // Processes each question in a survey to ensure headlines are formatted correctly for recall and return the modified survey.
-export const checkForRecallInHeadline = (survey: TSurvey, langauge: string): TSurvey => {
-  const modifiedSurvey: TSurvey = structuredClone(survey);
+export const checkForRecallInHeadline = <T extends TSurveyQuestionsObject>(
+  survey: T,
+  langauge: string
+): T => {
+  const modifiedSurvey: T = structuredClone(survey);
   modifiedSurvey.questions.forEach((question) => {
     question.headline = recallToHeadline(question.headline, modifiedSurvey, false, langauge);
   });
@@ -173,29 +174,3 @@ export const headlineToRecall = (
   });
   return text;
 };
-
-// Custom hook to synchronize the horizontal scroll position of two elements.
-export const useSyncScroll = (
-  highlightContainerRef: RefObject<HTMLElement>,
-  inputRef: RefObject<HTMLElement>,
-  text: string
-) => {
-  useEffect(() => {
-    const syncScrollPosition = () => {
-      if (highlightContainerRef.current && inputRef.current) {
-        highlightContainerRef.current.scrollLeft = inputRef.current.scrollLeft;
-      }
-    };
-
-    const sourceElement = inputRef.current;
-    if (sourceElement) {
-      sourceElement.addEventListener("scroll", syncScrollPosition);
-    }
-
-    return () => {
-      if (sourceElement) {
-        sourceElement.removeEventListener("scroll", syncScrollPosition);
-      }
-    };
-  }, [inputRef, highlightContainerRef, text]);
-};

packages/types/surveys.ts

@@ -392,6 +392,10 @@ export const ZSurveyQuestions = z.array(ZSurveyQuestion);
 
 export type TSurveyQuestions = z.infer<typeof ZSurveyQuestions>;
 
+export const ZSurveyQuestionsObject = z.object({ questions: ZSurveyQuestions });
+
+export type TSurveyQuestionsObject = z.infer<typeof ZSurveyQuestionsObject>;
+
 export const ZSurveyDisplayOption = z.enum(["displayOnce", "displayMultiple", "respondMultiple"]);
 
 export type TSurveyDisplayOption = z.infer<typeof ZSurveyDisplayOption>;

packages/types/tsconfig.json

@@ -5,6 +5,6 @@
   "compilerOptions": {
     "composite": true,
     "outDir": "./dist",
-    "rootDir": "./",
-  },
+    "rootDir": "./"
+  }
 }

packages/ui/ColorPicker/index.tsx

@@ -4,7 +4,7 @@
 import { useCallback, useRef, useState } from "react";
 import { HexColorInput, HexColorPicker } from "react-colorful";
 
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 
 export const ColorPicker = ({ color, onChange }: { color: string; onChange: (v: string) => void }) => {
   return (

packages/ui/QuestionFormInput/index.tsx

@@ -6,6 +6,7 @@ import { RefObject, useEffect, useMemo, useRef, useState } from "react";
 import { toast } from "react-hot-toast";
 
 import { extractLanguageCodes, getEnabledLanguages, getLocalizedValue } from "@formbricks/lib/i18n/utils";
+import { useSyncScroll } from "@formbricks/lib/utils/hooks/useSyncScroll";
 import {
   extractId,
   extractRecallInfo,
@@ -15,7 +16,6 @@ import {
   headlineToRecall,
   recallToHeadline,
   replaceRecallInfoWithUnderline,
-  useSyncScroll,
 } from "@formbricks/lib/utils/recall";
 import { TI18nString, TSurvey, TSurveyChoice, TSurveyQuestion } from "@formbricks/types/surveys";
 
@@ -135,7 +135,7 @@ export const QuestionFormInput = ({
   });
 
   // Hook to synchronize the horizontal scroll position of highlightContainerRef and inputRef.
-  useSyncScroll(highlightContainerRef, inputRef, getLocalizedValue(text, selectedLanguageCode));
+  useSyncScroll(highlightContainerRef, inputRef);
 
   useEffect(() => {
     if (!isWelcomeCard && (id === "headline" || id === "subheader")) {

packages/ui/ShareSurveyLink/components/LanguageDropdown.tsx

@@ -2,7 +2,7 @@ import { Languages } from "lucide-react";
 import { useRef, useState } from "react";
 
 import { getEnabledLanguages } from "@formbricks/lib/i18n/utils";
-import useClickOutside from "@formbricks/lib/useClickOutside";
+import { useClickOutside } from "@formbricks/lib/utils/hooks/useClickOutside";
 import { TSurvey } from "@formbricks/types/surveys";
 
 import { getLanguageLabel } from "../../../ee/multiLanguage/lib/isoLanguages";