fix: uploads (#1449)

Co-authored-by: Matthias Nannt <mail@matthiasnannt.com>

apps/web/app/api/v1/client/storage/lib/uploadPrivateFile.ts

@@ -1,8 +1,5 @@
-import { env } from "@/env.mjs";
 import { responses } from "@/app/lib/api/response";
-import { WEBAPP_URL } from "@formbricks/lib/constants";
-import { getSignedUrlForS3Upload } from "@formbricks/lib/storage/service";
-import { generateLocalSignedUrl } from "@formbricks/lib/crypto";
+import { getUploadSignedUrl } from "@formbricks/lib/storage/service";
 
 const uploadPrivateFile = async (
   fileName: string,
@@ -13,40 +10,14 @@ const uploadPrivateFile = async (
   const accessType = "private"; // private files are only accessible by the user who has access to the environment
   // if s3 is not configured, we'll upload to a local folder named uploads
 
-  if (!env.S3_ACCESS_KEY || !env.S3_SECRET_KEY || !env.S3_REGION || !env.S3_BUCKET_NAME) {
-    try {
-      const { signature, timestamp, uuid } = generateLocalSignedUrl(fileName, environmentId, fileType);
-
-      return responses.successResponse({
-        signedUrl: `${WEBAPP_URL}/api/v1/client/storage/local`,
-        signingData: {
-          signature,
-          timestamp,
-          uuid,
-        },
-        fileUrl: new URL(`${WEBAPP_URL}/storage/${environmentId}/${accessType}/${fileName}`).href,
-      });
-    } catch (err) {
-      return responses.internalServerErrorResponse(err.message);
-    }
-  }
-
   try {
-    const signedUrl = await getSignedUrlForS3Upload(
-      fileName,
-      fileType,
-      accessType,
-      environmentId,
-      false,
-      plan
-    );
+    const signedUrlResponse = await getUploadSignedUrl(fileName, environmentId, fileType, accessType, plan);
 
     return responses.successResponse({
-      signedUrl,
-      fileUrl: new URL(`${WEBAPP_URL}/storage/${environmentId}/${accessType}/${fileName}`).href,
+      ...signedUrlResponse,
     });
   } catch (err) {
-    return responses.internalServerErrorResponse(err.message);
+    return responses.internalServerErrorResponse("Internal server error");
   }
 };
 

apps/web/app/api/v1/management/storage/lib/getSignedUrl.ts

@@ -1,7 +1,4 @@
-import { env } from "@/env.mjs";
-import { getSignedUrlForS3Upload } from "@formbricks/lib/storage/service";
-import { WEBAPP_URL } from "@formbricks/lib/constants";
-import { generateLocalSignedUrl } from "@formbricks/lib/crypto";
+import { getUploadSignedUrl } from "@formbricks/lib/storage/service";
 import { responses } from "@/app/lib/api/response";
 
 const getSignedUrlForPublicFile = async (fileName: string, environmentId: string, fileType: string) => {
@@ -9,41 +6,11 @@ const getSignedUrlForPublicFile = async (fileName: string, environmentId: string
 
   // if s3 is not configured, we'll upload to a local folder named uploads
 
-  if (!env.S3_ACCESS_KEY || !env.S3_SECRET_KEY || !env.S3_REGION || !env.S3_BUCKET_NAME) {
-    try {
-      const { signature, timestamp, uuid } = generateLocalSignedUrl(fileName, environmentId, fileType);
-
-      return responses.successResponse({
-        signedUrl: new URL(`${WEBAPP_URL}/api/v1/management/storage/local`).href,
-        signingData: {
-          signature,
-          timestamp,
-          uuid,
-        },
-        fileUrl: new URL(`${WEBAPP_URL}/storage/${environmentId}/${accessType}/${fileName}`).href,
-      });
-    } catch (err) {
-      if (err.name === "FileTooLargeError") {
-        return responses.badRequestResponse(err.message);
-      }
-
-      return responses.internalServerErrorResponse("Internal server error");
-    }
-  }
-
   try {
-    const { presignedFields, signedUrl } = await getSignedUrlForS3Upload(
-      fileName,
-      fileType,
-      accessType,
-      environmentId,
-      true
-    );
+    const signedUrlResponse = await getUploadSignedUrl(fileName, environmentId, fileType, accessType);
 
     return responses.successResponse({
-      signedUrl,
-      presignedFields,
-      fileUrl: new URL(`${WEBAPP_URL}/storage/${environmentId}/${accessType}/${fileName}`).href,
+      ...signedUrlResponse,
     });
   } catch (err) {
     return responses.internalServerErrorResponse("Internal server error");

apps/web/app/storage/[environmentId]/[accessType]/[fileName]/lib/deleteFile.ts

@@ -0,0 +1,24 @@
+import { responses } from "@/app/lib/api/response";
+import { storageCache } from "@formbricks/lib/storage/cache";
+import { deleteFile } from "@formbricks/lib/storage/service";
+import { TAccessType } from "@formbricks/types/storage";
+
+export const handleDeleteFile = async (environmentId: string, accessType: TAccessType, fileName: string) => {
+  try {
+    const { message, success, code } = await deleteFile(environmentId, accessType, fileName);
+
+    if (success) {
+      // revalidate cache
+      storageCache.revalidate({ fileKey: `${environmentId}/${accessType}/${fileName}` });
+      return responses.successResponse(message);
+    }
+
+    if (code === 404) {
+      return responses.notFoundResponse("File", "File not found");
+    }
+
+    return responses.internalServerErrorResponse(message);
+  } catch (err) {
+    return responses.internalServerErrorResponse("Something went wrong");
+  }
+};

apps/web/app/storage/[environmentId]/[accessType]/[fileName]/lib/getFile.ts

@@ -1,14 +1,14 @@
 import { env } from "@/env.mjs";
 import { responses } from "@/app/lib/api/response";
 import { UPLOADS_DIR } from "@formbricks/lib/constants";
-import { getFileFromLocalStorage, getFileFromS3 } from "@formbricks/lib/storage/service";
+import { getLocalFile, getS3File } from "@formbricks/lib/storage/service";
 import { notFound } from "next/navigation";
 import path from "path";
 
 const getFile = async (environmentId: string, accessType: string, fileName: string) => {
   if (!env.S3_ACCESS_KEY || !env.S3_SECRET_KEY || !env.S3_REGION || !env.S3_BUCKET_NAME) {
     try {
-      const { fileBuffer, metaData } = await getFileFromLocalStorage(
+      const { fileBuffer, metaData } = await getLocalFile(
         path.join(UPLOADS_DIR, environmentId, accessType, fileName)
       );
 
@@ -24,7 +24,7 @@ const getFile = async (environmentId: string, accessType: string, fileName: stri
   }
 
   try {
-    const signedUrl = await getFileFromS3(`${environmentId}/${accessType}/${fileName}`);
+    const signedUrl = await getS3File(`${environmentId}/${accessType}/${fileName}`);
 
     return new Response(null, {
       status: 302,

apps/web/app/storage/[environmentId]/[accessType]/[fileName]/route.ts

@@ -6,6 +6,7 @@ import { ZStorageRetrievalParams } from "@formbricks/types/storage";
 import { getServerSession } from "next-auth";
 import { NextRequest } from "next/server";
 import getFile from "./lib/getFile";
+import { handleDeleteFile } from "@/app/storage/[environmentId]/[accessType]/[fileName]/lib/deleteFile";
 
 export async function GET(
   _: NextRequest,
@@ -43,3 +44,44 @@ export async function GET(
 
   return await getFile(environmentId, accessType, fileName);
 }
+
+export async function DELETE(_: NextRequest, { params }: { params: { fileName: string } }) {
+  if (!params.fileName) {
+    return responses.badRequestResponse("Fields are missing or incorrectly formatted", {
+      fileName: "fileName is required",
+    });
+  }
+
+  const [environmentId, accessType, file] = params.fileName.split("/");
+
+  const paramValidation = ZStorageRetrievalParams.safeParse({ fileName: file, environmentId, accessType });
+
+  if (!paramValidation.success) {
+    return responses.badRequestResponse(
+      "Fields are missing or incorrectly formatted",
+      transformErrorToDetails(paramValidation.error),
+      true
+    );
+  }
+  // check if user is authenticated
+
+  const session = await getServerSession(authOptions);
+
+  if (!session || !session.user) {
+    return responses.notAuthenticatedResponse();
+  }
+
+  // check if the user has access to the environment
+
+  const isUserAuthorized = await hasUserEnvironmentAccess(session.user.id, environmentId);
+
+  if (!isUserAuthorized) {
+    return responses.unauthorizedResponse();
+  }
+
+  return await handleDeleteFile(
+    paramValidation.data.environmentId,
+    paramValidation.data.accessType,
+    paramValidation.data.fileName
+  );
+}

packages/lib/constants.ts

@@ -70,3 +70,9 @@ export const MAX_SIZES = {
   free: 1024 * 1024 * 10, // 10MB
   pro: 1024 * 1024 * 1024, // 1GB
 } as const;
+export const IS_S3_CONFIGURED: boolean =
+  env.S3_ACCESS_KEY && env.S3_SECRET_KEY && env.S3_REGION && env.S3_BUCKET_NAME ? true : false;
+export const LOCAL_UPLOAD_URL = {
+  public: new URL(`${WEBAPP_URL}/api/v1/management/storage/local`).href,
+  private: new URL(`${WEBAPP_URL}/api/v1/client/storage/local`).href,
+} as const;

packages/lib/product/service.ts

@@ -8,10 +8,11 @@ import { ZProduct, ZProductUpdateInput } from "@formbricks/types/product";
 import { Prisma } from "@prisma/client";
 import { revalidateTag, unstable_cache } from "next/cache";
 import { z } from "zod";
-import { SERVICES_REVALIDATION_INTERVAL, ITEMS_PER_PAGE } from "../constants";
+import { SERVICES_REVALIDATION_INTERVAL, ITEMS_PER_PAGE, IS_S3_CONFIGURED } from "../constants";
 import { validateInputs } from "../utils/validate";
 import { createEnvironment, getEnvironmentCacheTag, getEnvironmentsCacheTag } from "../environment/service";
 import { ZOptionalNumber } from "@formbricks/types/common";
+import { deleteLocalFilesByEnvironmentId, deleteS3FilesByEnvironmentId } from "../storage/service";
 
 export const getProductsCacheTag = (teamId: string): string => `teams-${teamId}-products`;
 export const getProductCacheTag = (environmentId: string): string => `environments-${environmentId}-product`;
@@ -176,6 +177,32 @@ export const deleteProduct = async (productId: string): Promise<TProduct> => {
   });
 
   if (product) {
+    // delete all files from storage related to this product
+
+    if (IS_S3_CONFIGURED) {
+      const s3FilesPromises = product.environments.map(async (environment) => {
+        return deleteS3FilesByEnvironmentId(environment.id);
+      });
+
+      try {
+        await Promise.all(s3FilesPromises);
+      } catch (err) {
+        // fail silently because we don't want to throw an error if the files are not deleted
+        console.error(err);
+      }
+    } else {
+      const localFilesPromises = product.environments.map(async (environment) => {
+        return deleteLocalFilesByEnvironmentId(environment.id);
+      });
+
+      try {
+        await Promise.all(localFilesPromises);
+      } catch (err) {
+        // fail silently because we don't want to throw an error if the files are not deleted
+        console.error(err);
+      }
+    }
+
     revalidateTag(getProductsCacheTag(product.teamId));
     revalidateTag(getEnvironmentsCacheTag(product.id));
     product.environments.forEach((environment) => {

packages/lib/storage/cache.ts

@@ -0,0 +1,16 @@
+import { revalidateTag } from "next/cache";
+
+interface RevalidateProps {
+  fileKey: string;
+}
+
+export const storageCache = {
+  tag: {
+    byFileKey(filekey: string): string {
+      return `storage-filekey-${filekey}`;
+    },
+  },
+  revalidate({ fileKey }: RevalidateProps): void {
+    revalidateTag(this.tag.byFileKey(fileKey));
+  },
+};

packages/lib/storage/service.ts

@@ -1,10 +1,22 @@
-import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
+import {
+  S3Client,
+  GetObjectCommand,
+  DeleteObjectCommand,
+  ListObjectsCommand,
+  DeleteObjectsCommand,
+} from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { createPresignedPost, PresignedPostOptions } from "@aws-sdk/s3-presigned-post";
-import { access, mkdir, writeFile, readFile } from "fs/promises";
+import { access, mkdir, writeFile, readFile, unlink, rmdir } from "fs/promises";
+import { join } from "path";
 import mime from "mime";
 import { env } from "@/env.mjs";
-import { MAX_SIZES } from "../constants";
+import { IS_S3_CONFIGURED, LOCAL_UPLOAD_URL, MAX_SIZES, UPLOADS_DIR, WEBAPP_URL } from "../constants";
+import { unstable_cache } from "next/cache";
+import { storageCache } from "./cache";
+import { TAccessType } from "@formbricks/types/storage";
+import { generateLocalSignedUrl } from "../crypto";
+import path from "path";
 
 // global variables
 
@@ -42,22 +54,47 @@ type TGetFileResponse = {
   };
 };
 
-export const getFileFromS3 = async (fileKey: string) => {
-  const getObjectCommand = new GetObjectCommand({
-    Bucket: AWS_BUCKET_NAME,
-    Key: fileKey,
-  });
+// discriminated union
+type TGetSignedUrlResponse =
+  | { signedUrl: string; fileUrl: string; presignedFields: Object }
+  | {
+      signedUrl: string;
+      fileUrl: string;
+      signingData: {
+        signature: string;
+        timestamp: number;
+        uuid: string;
+      };
+    };
 
-  try {
-    const signedUrl = await getSignedUrl(s3Client, getObjectCommand, { expiresIn: 3600 });
+export const getS3File = (fileKey: string): Promise<string> => {
+  const [_, accessType] = fileKey.split("/");
+  const expiresIn = accessType === "public" ? 60 * 60 : 10 * 60;
 
-    return signedUrl;
-  } catch (err) {
-    throw err;
-  }
+  const revalidateAfter = accessType === "public" ? expiresIn - 60 * 5 : expiresIn - 60 * 2;
+
+  return unstable_cache(
+    async () => {
+      const getObjectCommand = new GetObjectCommand({
+        Bucket: AWS_BUCKET_NAME,
+        Key: fileKey,
+      });
+
+      try {
+        return await getSignedUrl(s3Client, getObjectCommand, { expiresIn });
+      } catch (err) {
+        throw err;
+      }
+    },
+    [`getFileFromS3-${fileKey}`],
+    {
+      revalidate: revalidateAfter,
+      tags: [storageCache.tag.byFileKey(fileKey)],
+    }
+  )();
 };
 
-export const getFileFromLocalStorage = async (filePath: string): Promise<TGetFileResponse> => {
+export const getLocalFile = async (filePath: string): Promise<TGetFileResponse> => {
   try {
     const file = await readFile(filePath);
     let contentType = "";
@@ -79,7 +116,54 @@ export const getFileFromLocalStorage = async (filePath: string): Promise<TGetFil
   }
 };
 
-export const getSignedUrlForS3Upload = async (
+// a single service for generating a signed url based on user's environment variables
+export const getUploadSignedUrl = async (
+  fileName: string,
+  environmentId: string,
+  fileType: string,
+  accessType: TAccessType,
+  plan: "free" | "pro" = "free"
+): Promise<TGetSignedUrlResponse> => {
+  // handle the local storage case first
+  if (!IS_S3_CONFIGURED) {
+    try {
+      const { signature, timestamp, uuid } = generateLocalSignedUrl(fileName, environmentId, fileType);
+
+      return {
+        signedUrl: LOCAL_UPLOAD_URL[accessType],
+        signingData: {
+          signature,
+          timestamp,
+          uuid,
+        },
+        fileUrl: new URL(`${WEBAPP_URL}/storage/${environmentId}/${accessType}/${fileName}`).href,
+      };
+    } catch (err) {
+      throw err;
+    }
+  }
+
+  try {
+    const { presignedFields, signedUrl } = await getS3UploadSignedUrl(
+      fileName,
+      fileType,
+      accessType,
+      environmentId,
+      accessType === "public",
+      plan
+    );
+
+    return {
+      signedUrl,
+      presignedFields,
+      fileUrl: new URL(`${WEBAPP_URL}/storage/${environmentId}/${accessType}/${fileName}`).href,
+    };
+  } catch (err) {
+    throw err;
+  }
+};
+
+export const getS3UploadSignedUrl = async (
   fileName: string,
   contentType: string,
   accessType: string,
@@ -142,3 +226,99 @@ export const putFileToLocalStorage = async (
     throw err;
   }
 };
+
+export const deleteFile = async (environmentId: string, accessType: TAccessType, fileName: string) => {
+  if (!IS_S3_CONFIGURED) {
+    try {
+      await deleteLocalFile(path.join(UPLOADS_DIR, environmentId, accessType, fileName));
+      return { success: true, message: "File deleted" };
+    } catch (err: any) {
+      if (err.code !== "ENOENT") {
+        return { success: false, message: err.message ?? "Something went wrong" };
+      }
+
+      return { success: false, message: "File not found", code: 404 };
+    }
+  }
+
+  try {
+    await deleteS3File(`${environmentId}/${accessType}/${fileName}`);
+    return { success: true, message: "File deleted" };
+  } catch (err: any) {
+    if (err.name === "NoSuchKey") {
+      return { success: false, message: "File not found", code: 404 };
+    } else {
+      return { success: false, message: err.message ?? "Something went wrong" };
+    }
+  }
+};
+
+export const deleteLocalFile = async (filePath: string) => {
+  try {
+    await unlink(filePath);
+  } catch (err: any) {
+    throw err;
+  }
+};
+
+export const deleteS3File = async (fileKey: string) => {
+  const deleteObjectCommand = new DeleteObjectCommand({
+    Bucket: AWS_BUCKET_NAME,
+    Key: fileKey,
+  });
+
+  try {
+    await s3Client.send(deleteObjectCommand);
+  } catch (err) {
+    throw err;
+  }
+};
+
+export const deleteS3FilesByEnvironmentId = async (environmentId: string) => {
+  try {
+    // List all objects in the bucket with the prefix of environmentId
+    const listObjectsOutput = await s3Client.send(
+      new ListObjectsCommand({
+        Bucket: AWS_BUCKET_NAME,
+        Prefix: environmentId,
+      })
+    );
+
+    if (listObjectsOutput.Contents) {
+      const objectsToDelete = listObjectsOutput.Contents.map((obj) => {
+        return { Key: obj.Key };
+      });
+
+      if (!objectsToDelete.length) {
+        // no objects to delete
+        return null;
+      }
+
+      // Delete the objects
+      await s3Client.send(
+        new DeleteObjectsCommand({
+          Bucket: AWS_BUCKET_NAME,
+          Delete: {
+            Objects: objectsToDelete,
+          },
+        })
+      );
+    } else {
+      // no objects to delete
+      return null;
+    }
+  } catch (err) {
+    throw err;
+  }
+};
+
+export const deleteLocalFilesByEnvironmentId = async (environmentId: string) => {
+  const dirPath = join(UPLOADS_DIR, environmentId);
+
+  try {
+    await ensureDirectoryExists(dirPath);
+    await rmdir(dirPath, { recursive: true });
+  } catch (err) {
+    throw err;
+  }
+};

packages/types/storage.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 
 export const ZAccessType = z.enum(["public", "private"]);
+export type TAccessType = z.infer<typeof ZAccessType>;
 
 export const ZStorageRetrievalParams = z.object({
   fileName: z.string(),