fix: error handling in insight generation (#4075)

Co-authored-by: pandeymangg <anshuman.pandey9999@gmail.com>

apps/web/app/api/(internal)/insights/lib/document.ts

@@ -6,9 +6,8 @@ import { prisma } from "@formbricks/database";
 import { embeddingsModel, llmModel } from "@formbricks/lib/aiModels";
 import { validateInputs } from "@formbricks/lib/utils/validate";
 import {
-  TDocument,
+  TCreatedDocument,
   TDocumentCreateInput,
-  TGenerateDocumentObjectSchema,
   ZDocumentCreateInput,
   ZGenerateDocumentObjectSchema,
 } from "@formbricks/types/documents";
@@ -17,7 +16,7 @@ import { DatabaseError } from "@formbricks/types/errors";
 export const createDocument = async (
   surveyName: string,
   documentInput: TDocumentCreateInput
-): Promise<TDocument & { isSpam: Boolean; insights: TGenerateDocumentObjectSchema["insights"] }> => {
+): Promise<TCreatedDocument> => {
   validateInputs([surveyName, z.string()], [documentInput, ZDocumentCreateInput]);
 
   try {

apps/web/app/api/(internal)/insights/lib/insights.ts

@@ -9,6 +9,7 @@ import { embeddingsModel } from "@formbricks/lib/aiModels";
 import { getPromptText } from "@formbricks/lib/utils/ai";
 import { validateInputs } from "@formbricks/lib/utils/validate";
 import { ZId } from "@formbricks/types/common";
+import { TCreatedDocument } from "@formbricks/types/documents";
 import { DatabaseError } from "@formbricks/types/errors";
 import {
   TInsight,
@@ -135,8 +136,11 @@ export const generateInsightsForSurveyResponsesConcept = async (surveyData: {
         });
       });
 
-      const createDocumentResults = await Promise.all(createDocumentPromises);
-      const createdDocuments = createDocumentResults.filter(Boolean);
+      const createDocumentResults = await Promise.allSettled(createDocumentPromises);
+      const fullfilledCreateDocumentResults = createDocumentResults.filter(
+        (result) => result.status === "fulfilled"
+      ) as PromiseFulfilledResult<TCreatedDocument>[];
+      const createdDocuments = fullfilledCreateDocumentResults.filter(Boolean).map((result) => result.value);
 
       for (const document of createdDocuments) {
         if (document) {
@@ -151,7 +155,7 @@ export const generateInsightsForSurveyResponsesConcept = async (surveyData: {
               // Create or connect the insight
               insightPromises.push(handleInsightAssignments(environmentId, id, insight));
             }
-            await Promise.all(insightPromises);
+            await Promise.allSettled(insightPromises);
           }
         }
       }
@@ -175,121 +179,6 @@ export const generateInsightsForSurveyResponsesConcept = async (surveyData: {
   }
 };
 
-export const generateInsightsForSurveyResponses = async (surveyData: {
-  id: string;
-  name: string;
-  environmentId: string;
-  questions: TSurveyQuestions;
-}): Promise<void> => {
-  const { id: surveyId, name, environmentId, questions } = surveyData;
-
-  validateInputs([surveyId, ZId], [environmentId, ZId], [questions, ZSurveyQuestions]);
-  try {
-    const openTextQuestionsWithInsights = questions.filter(
-      (question) => question.type === TSurveyQuestionTypeEnum.OpenText && question.insightsEnabled
-    );
-
-    const openTextQuestionIds = openTextQuestionsWithInsights.map((question) => question.id);
-
-    if (openTextQuestionIds.length === 0) {
-      return;
-    }
-
-    // Fetching responses
-    const batchSize = 200;
-    let skip = 0;
-
-    const totalResponseCount = await prisma.response.count({
-      where: {
-        surveyId,
-        documents: {
-          none: {},
-        },
-        finished: true,
-      },
-    });
-
-    const pages = Math.ceil(totalResponseCount / batchSize);
-
-    for (let i = 0; i < pages; i++) {
-      const responses = await prisma.response.findMany({
-        where: {
-          surveyId,
-          documents: {
-            none: {},
-          },
-          finished: true,
-        },
-        select: {
-          id: true,
-          data: true,
-        },
-        take: batchSize,
-        skip,
-      });
-
-      const responsesWithOpenTextAnswers = responses.filter((response) =>
-        doesResponseHasAnyOpenTextAnswer(openTextQuestionIds, response.data)
-      );
-
-      skip += batchSize - responsesWithOpenTextAnswers.length;
-
-      const createDocumentPromises = responsesWithOpenTextAnswers.map((response) => {
-        return Promise.all(
-          openTextQuestionsWithInsights.map(async (question) => {
-            const responseText = response.data[question.id] as string;
-            if (!responseText) {
-              return;
-            }
-
-            const text = getPromptText(question.headline.default, responseText);
-
-            return await createDocument(name, {
-              environmentId,
-              surveyId,
-              responseId: response.id,
-              questionId: question.id,
-              text,
-            });
-          })
-        );
-      });
-
-      const createDocumentResults = await Promise.all(createDocumentPromises);
-      const createdDocuments = createDocumentResults.flat().filter(Boolean);
-
-      for (const document of createdDocuments) {
-        if (document) {
-          const insightPromises: Promise<void>[] = [];
-          const { insights, isSpam, id, environmentId } = document;
-          if (!isSpam) {
-            for (const insight of insights) {
-              if (typeof insight.title !== "string" || typeof insight.description !== "string") {
-                throw new Error("Insight title and description must be a string");
-              }
-
-              // create or connect the insight
-              insightPromises.push(handleInsightAssignments(environmentId, id, insight));
-            }
-            await Promise.all(insightPromises);
-          }
-        }
-      }
-      documentCache.revalidate({
-        environmentId: environmentId,
-        surveyId: surveyId,
-      });
-    }
-    return;
-  } catch (error) {
-    if (error instanceof Prisma.PrismaClientKnownRequestError) {
-      throw new DatabaseError(error.message);
-    }
-
-    throw error;
-  }
-};
-
 export const getQuestionResponseReferenceId = (surveyId: string, questionId: TSurveyQuestionId) => {
   return `${surveyId}-${questionId}`;
 };
@@ -342,45 +231,49 @@ export const handleInsightAssignments = async (
     category: TInsightCategory;
   }
 ) => {
-  // create embedding for insight
-  const { embedding } = await embed({
-    model: embeddingsModel,
-    value: getInsightVectorText(insight.title, insight.description),
-    experimental_telemetry: { isEnabled: true },
-  });
-  // find close insight to merge it with
-  const nearestInsights = await findNearestInsights(environmentId, embedding, 1, 0.2);
+  try {
+    // create embedding for insight
+    const { embedding } = await embed({
+      model: embeddingsModel,
+      value: getInsightVectorText(insight.title, insight.description),
+      experimental_telemetry: { isEnabled: true },
+    });
+    // find close insight to merge it with
+    const nearestInsights = await findNearestInsights(environmentId, embedding, 1, 0.2);
 
-  if (nearestInsights.length > 0) {
-    // create a documentInsight with this insight
-    await prisma.documentInsight.create({
-      data: {
-        documentId,
+    if (nearestInsights.length > 0) {
+      // create a documentInsight with this insight
+      await prisma.documentInsight.create({
+        data: {
+          documentId,
+          insightId: nearestInsights[0].id,
+        },
+      });
+      documentCache.revalidate({
         insightId: nearestInsights[0].id,
-      },
-    });
-    documentCache.revalidate({
-      insightId: nearestInsights[0].id,
-    });
-  } else {
-    // create new insight and documentInsight
-    const newInsight = await createInsight({
-      environmentId: environmentId,
-      title: insight.title,
-      description: insight.description,
-      category: insight.category ?? "other",
-      vector: embedding,
-    });
-    // create a documentInsight with this insight
-    await prisma.documentInsight.create({
-      data: {
-        documentId,
+      });
+    } else {
+      // create new insight and documentInsight
+      const newInsight = await createInsight({
+        environmentId: environmentId,
+        title: insight.title,
+        description: insight.description,
+        category: insight.category ?? "other",
+        vector: embedding,
+      });
+      // create a documentInsight with this insight
+      await prisma.documentInsight.create({
+        data: {
+          documentId,
+          insightId: newInsight.id,
+        },
+      });
+      documentCache.revalidate({
         insightId: newInsight.id,
-      },
-    });
-    documentCache.revalidate({
-      insightId: newInsight.id,
-    });
+      });
+    }
+  } catch (error) {
+    throw error;
   }
 };
 

apps/web/app/api/(internal)/insights/lib/utils.ts

@@ -72,7 +72,7 @@ export const generateInsightsEnabledForSurveyQuestions = async (
       return { success: false };
     }
 
-    const insightsEnabledValues = await Promise.all(
+    const insightsEnabledValues = await Promise.allSettled(
       openTextQuestions.map(async (question) => {
         const insightsEnabled = await getInsightsEnabled(question);
 
@@ -80,9 +80,13 @@ export const generateInsightsEnabledForSurveyQuestions = async (
       })
     );
 
-    const insightsEnabledQuestionIds = insightsEnabledValues
-      .filter((value) => value.insightsEnabled)
-      .map((value) => value.id);
+    const fullfilledInsightsEnabledValues = insightsEnabledValues.filter(
+      (value) => value.status === "fulfilled"
+    ) as PromiseFulfilledResult<{ id: string; insightsEnabled: boolean }>[];
+
+    const insightsEnabledQuestionIds = fullfilledInsightsEnabledValues
+      .filter((value) => value.value.insightsEnabled)
+      .map((value) => value.value.id);
 
     const updatedQuestions = survey.questions.map((question) => {
       if (question.type === "openText") {

apps/web/app/api/(internal)/insights/route.ts

@@ -1,8 +1,5 @@
 // This function can run for a maximum of 300 seconds
-import {
-  //  generateInsightsForSurveyResponses,
-  generateInsightsForSurveyResponsesConcept,
-} from "@/app/api/(internal)/insights/lib/insights";
+import { generateInsightsForSurveyResponsesConcept } from "@/app/api/(internal)/insights/lib/insights";
 import { responses } from "@/app/lib/api/response";
 import { transformErrorToDetails } from "@/app/lib/api/validator";
 import { headers } from "next/headers";
@@ -43,7 +40,6 @@ export const POST = async (request: Request) => {
       return responses.successResponse({ message: "No insights enabled questions found" });
     }
 
-    // await generateInsightsForSurveyResponses(data.survey);
     await generateInsightsForSurveyResponsesConcept(data.survey);
 
     return responses.successResponse({ message: "Insights generated successfully" });

apps/web/app/api/(internal)/pipeline/lib/documents.ts

@@ -86,7 +86,7 @@ export const createDocumentAndAssignInsight = async (
         // create or connect the insight
         insightPromises.push(handleInsightAssignments(documentInput.environmentId, document.id, insight));
       }
-      await Promise.all(insightPromises);
+      await Promise.allSettled(insightPromises);
     }
 
     documentCache.revalidate({

apps/web/app/api/(internal)/pipeline/route.ts

@@ -166,13 +166,17 @@ export const POST = async (request: Request) => {
               }
               const text = getPromptText(question.headline.default, response.data[question.id] as string);
               // TODO: check if subheadline gives more context and better embeddings
-              await createDocumentAndAssignInsight(survey.name, {
-                environmentId,
-                surveyId,
-                responseId: response.id,
-                questionId: question.id,
-                text,
-              });
+              try {
+                await createDocumentAndAssignInsight(survey.name, {
+                  environmentId,
+                  surveyId,
+                  responseId: response.id,
+                  questionId: question.id,
+                  text,
+                });
+              } catch (e) {
+                console.error(e);
+              }
             }
           }
         }

packages/lib/survey/utils.ts

@@ -100,14 +100,18 @@ export const doesSurveyHasOpenTextQuestion = (questions: TSurveyQuestions): bool
 };
 
 export const getInsightsEnabled = async (question: TSurveyQuestion): Promise<boolean> => {
-  const { object } = await generateObject({
-    model: llmModel,
-    schema: z.object({
-      insightsEnabled: z.boolean(),
-    }),
-    prompt: `We extract insights (e.g. feature requests, complaints, other) from survey questions. Can we find them in this question?: ${question.headline.default}`,
-    experimental_telemetry: { isEnabled: true },
-  });
+  try {
+    const { object } = await generateObject({
+      model: llmModel,
+      schema: z.object({
+        insightsEnabled: z.boolean(),
+      }),
+      prompt: `We extract insights (e.g. feature requests, complaints, other) from survey questions. Can we find them in this question?: ${question.headline.default}`,
+      experimental_telemetry: { isEnabled: true },
+    });
 
-  return object.insightsEnabled;
+    return object.insightsEnabled;
+  } catch (error) {
+    throw error;
+  }
 };

packages/types/documents.ts

@@ -54,3 +54,8 @@ export const ZGenerateDocumentObjectSchema = z.object({
 });
 
 export type TGenerateDocumentObjectSchema = z.infer<typeof ZGenerateDocumentObjectSchema>;
+
+export type TCreatedDocument = TDocument & {
+  isSpam: boolean;
+  insights: TGenerateDocumentObjectSchema["insights"];
+};