mirror/formbricks-formbricks
1
0
Fork 0
mirror of https://github.com/formbricks/formbricks.git synced 2025-12-29 18:00:26 -06:00
Code Issues Packages Projects Releases 100 Wiki Activity

feat: improve configurability for upload storage (#2144)

Browse Source 
Co-authored-by: pandeymangg <anshuman.pandey9999@gmail.com>
Co-authored-by: Matti Nannt <mail@matthiasnannt.com>
This commit is contained in:
David Septimus
2024-03-13 08:57:13 -06:00
committed by GitHub
parent 6f78049c1f
commit 0cc365261e
8 changed files with 96 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.env.example
View File

@@ -56,11 +56,14 @@ SMTP_PASSWORD=smtpPassword
# Uncomment the variables you would like to use and customize the values.
# Custom local storage path for file uploads
#UPLOADS_DIR=
##############
# S3 STORAGE #
##############
# S3 Storage is required for the file uplaod in serverless environments like Vercel
# S3 Storage is required for the file upload in serverless environments like Vercel
S3_ACCESS_KEY=
S3_SECRET_KEY=
S3_REGION=

93
apps/formbricks-com/app/docs/self-hosting/external-auth-providers/page.mdx
View File

@@ -136,52 +136,53 @@ OIDC_SIGNING_ALGORITHM=HS256
These variables can be provided at the runtime i.e. in your docker-compose file.
| Variable | Description | Required | Default |
| --------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------------------------- | ----------------------- |
| WEBAPP_URL | Base URL of the site. | required | `http://localhost:3000` |
| DATABASE_URL | Database URL with credentials. | required | |
| NEXTAUTH_SECRET | Secret for NextAuth, used for session signing and encryption. | required | (Generated by the user) |
| ENCRYPTION_KEY | Secret for used by Formbricks for data encryption | required | (Generated by the user) |
| NEXTAUTH_URL | Location of the auth server. By default, this is the Formbricks docker instance itself. | required | `http://localhost:3000` |
| S3_ACCESS_KEY | Access key for S3. | optional (required if S3 is enabled) | |
| S3_SECRET_KEY | Secret key for S3. | optional (required if S3 is enabled) | |
| S3_REGION | Region for S3. | optional (required if S3 is enabled) | |
| S3_BUCKET | Bucket name for S3. | optional (required if S3 is enabled) | |
| S3_ENDPOINT | Endpoint for S3. | optional (required if S3 is enabled) | |
| PRIVACY_URL | URL for privacy policy. | optional | |
| TERMS_URL | URL for terms of service. | optional | |
| IMPRINT_URL | URL for imprint. | optional | |
| SIGNUP_DISABLED | Disables the ability for new users to create an account if set to `1`. | optional | |
| EMAIL_AUTH_DISABLED | Disables the ability for users to signup or login via email and password if set to `1`. | optional | |
| PASSWORD_RESET_DISABLED | Disables password reset functionality if set to `1`. | optional | |
| EMAIL_VERIFICATION_DISABLED | Disables email verification if set to `1`. | optional | |
| RATE_LIMITING_DISABLED | Disables rate limiting if set to `1`. | optional | |
| INVITE_DISABLED | Disables the ability for invited users to create an account if set to `1`. | optional | |
| MAIL_FROM | Email address to send emails from. | optional (required if email services are to be enabled) | |
| SMTP_HOST | Host URL of your SMTP server. | optional (required if email services are to be enabled) | |
| SMTP_PORT | Host Port of your SMTP server. | optional (required if email services are to be enabled) | |
| SMTP_USER | Username for your SMTP Server. | optional (required if email services are to be enabled) | |
| SMTP_PASSWORD | Password for your SMTP Server. | optional (required if email services are to be enabled) | |
| SMTP_SECURE_ENABLED | SMTP secure connection. For using TLS, set to `1` else to `0`. | optional (required if email services are to be enabled) | |
| GITHUB_ID | Client ID for GitHub. | optional (required if GitHub auth is enabled) | |
| GITHUB_SECRET | Secret for GitHub. | optional (required if GitHub auth is enabled) | |
| GOOGLE_CLIENT_ID | Client ID for Google. | optional (required if Google auth is enabled) | |
| GOOGLE_CLIENT_SECRET | Secret for Google. | optional (required if Google auth is enabled) | |
| CRON_SECRET | API Secret for running cron jobs. | optional | |
| STRIPE_SECRET_KEY | Secret key for Stripe integration. | optional | |
| STRIPE_WEBHOOK_SECRET | Webhook secret for Stripe integration. | optional | |
| TELEMETRY_DISABLED | Disables telemetry if set to `1`. | optional | |
| INSTANCE_ID | Instance ID for Formbricks Cloud to be sent to Telemetry. | optional | |
| INTERNAL_SECRET | Internal Secret (Currently we overwrite the value with a random value). | optional | |
| DEFAULT_BRAND_COLOR | Default brand color for your app (Can be overwritten from the UI as well). | optional | `#64748b` |
| DEFAULT_TEAM_ID | Automatically assign new users to a specific team when joining | optional | |
| DEFAULT_TEAM_ROLE | Role of the user in the default team. | optional | `admin` |
| ONBOARDING_DISABLED | Disables onboarding for new users if set to `1` | optional | |
| OIDC_DISPLAY_NAME | Display name for Custom OpenID Connect Provider | optional | |
| OIDC_CLIENT_ID | Client ID for Custom OpenID Connect Provider | optional (required if OIDC auth is enabled) | |
| OIDC_CLIENT_SECRET | Secret for Custom OpenID Connect Provider | optional (required if OIDC auth is enabled) | |
| OIDC_ISSUER | Issuer URL for Custom OpenID Connect Provider (should have `.well-known` configured at this) | optional (required if OIDC auth is enabled) | |
| OIDC_SIGNING_ALGORITHM | Signing Algorithm for Custom OpenID Connect Provider | optional | `RS256` |
| Variable | Description | Required | Default |
|-----------------------------|----------------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------|
| WEBAPP_URL | Base URL of the site. | required | `http://localhost:3000` |
| DATABASE_URL | Database URL with credentials. | required | |
| NEXTAUTH_SECRET | Secret for NextAuth, used for session signing and encryption. | required | (Generated by the user) |
| ENCRYPTION_KEY | Secret for used by Formbricks for data encryption | required | (Generated by the user) |
| NEXTAUTH_URL | Location of the auth server. By default, this is the Formbricks docker instance itself. | required | `http://localhost:3000` |
| UPLOADS_DIR | Local directory for storing uploads. | optional | `./uploads` |
| S3_ACCESS_KEY | Access key for S3. | optional | (resolved by the AWS SDK) |
| S3_SECRET_KEY | Secret key for S3. | optional | (resolved by the AWS SDK) |
| S3_REGION | Region for S3. | optional | (resolved by the AWS SDK) |
| S3_BUCKET | Bucket name for S3. | optional (required if S3 is enabled) | |
| S3_ENDPOINT | Endpoint for S3. | optional | (resolved by the AWS SDK) |
| PRIVACY_URL | URL for privacy policy. | optional | |
| TERMS_URL | URL for terms of service. | optional | |
| IMPRINT_URL | URL for imprint. | optional | |
| SIGNUP_DISABLED | Disables the ability for new users to create an account if set to `1`. | optional | |
| EMAIL_AUTH_DISABLED | Disables the ability for users to signup or login via email and password if set to `1`. | optional | |
| PASSWORD_RESET_DISABLED | Disables password reset functionality if set to `1`. | optional | |
| EMAIL_VERIFICATION_DISABLED | Disables email verification if set to `1`. | optional | |
| RATE_LIMITING_DISABLED | Disables rate limiting if set to `1`. | optional | |
| INVITE_DISABLED | Disables the ability for invited users to create an account if set to `1`. | optional | |
| MAIL_FROM | Email address to send emails from. | optional (required if email services are to be enabled) | |
| SMTP_HOST | Host URL of your SMTP server. | optional (required if email services are to be enabled) | |
| SMTP_PORT | Host Port of your SMTP server. | optional (required if email services are to be enabled) | |
| SMTP_USER | Username for your SMTP Server. | optional (required if email services are to be enabled) | |
| SMTP_PASSWORD | Password for your SMTP Server. | optional (required if email services are to be enabled) | |
| SMTP_SECURE_ENABLED | SMTP secure connection. For using TLS, set to `1` else to `0`. | optional (required if email services are to be enabled) | |
| GITHUB_ID | Client ID for GitHub. | optional (required if GitHub auth is enabled) | |
| GITHUB_SECRET | Secret for GitHub. | optional (required if GitHub auth is enabled) | |
| GOOGLE_CLIENT_ID | Client ID for Google. | optional (required if Google auth is enabled) | |
| GOOGLE_CLIENT_SECRET | Secret for Google. | optional (required if Google auth is enabled) | |
| CRON_SECRET | API Secret for running cron jobs. | optional | |
| STRIPE_SECRET_KEY | Secret key for Stripe integration. | optional | |
| STRIPE_WEBHOOK_SECRET | Webhook secret for Stripe integration. | optional | |
| TELEMETRY_DISABLED | Disables telemetry if set to `1`. | optional | |
| INSTANCE_ID | Instance ID for Formbricks Cloud to be sent to Telemetry. | optional | |
| INTERNAL_SECRET | Internal Secret (Currently we overwrite the value with a random value). | optional | |
| DEFAULT_BRAND_COLOR | Default brand color for your app (Can be overwritten from the UI as well). | optional | `#64748b` |
| DEFAULT_TEAM_ID | Automatically assign new users to a specific team when joining | optional | |
| DEFAULT_TEAM_ROLE | Role of the user in the default team. | optional | `admin` |
| ONBOARDING_DISABLED | Disables onboarding for new users if set to `1` | optional | |
| OIDC_DISPLAY_NAME | Display name for Custom OpenID Connect Provider | optional | |
| OIDC_CLIENT_ID | Client ID for Custom OpenID Connect Provider | optional (required if OIDC auth is enabled) | |
| OIDC_CLIENT_SECRET | Secret for Custom OpenID Connect Provider | optional (required if OIDC auth is enabled) | |
| OIDC_ISSUER | Issuer URL for Custom OpenID Connect Provider (should have `.well-known` configured at this) | optional (required if OIDC auth is enabled) | |
| OIDC_SIGNING_ALGORITHM | Signing Algorithm for Custom OpenID Connect Provider | optional | `RS256` |
## Build-time Variables

5
apps/web/app/storage/[environmentId]/[accessType]/[fileName]/lib/getFile.ts
View File

@@ -2,11 +2,12 @@ import { responses } from "@/app/lib/api/response";
import { notFound } from "next/navigation";
import path from "path";
import { IS_S3_CONFIGURED, UPLOADS_DIR } from "@formbricks/lib/constants";
import { UPLOADS_DIR } from "@formbricks/lib/constants";
import { isS3Configured } from "@formbricks/lib/constants";
import { getLocalFile, getS3File } from "@formbricks/lib/storage/service";
const getFile = async (environmentId: string, accessType: string, fileName: string) => {
if (!IS_S3_CONFIGURED) {
if (!isS3Configured()) {
try {
const { fileBuffer, metaData } = await getLocalFile(
path.join(UPLOADS_DIR, environmentId, accessType, fileName)

20
packages/lib/constants.ts
View File

@@ -83,19 +83,33 @@ export const DEFAULT_TEAM_ROLE = env.DEFAULT_TEAM_ROLE;
export const ONBOARDING_DISABLED = env.ONBOARDING_DISABLED;
// Storage constants
export const AWS_ACCESS_KEY_ID = env.AWS_ACCESS_KEY_ID;
export const AWS_SECRET_ACCESS_KEY = env.AWS_SECRET_ACCESS_KEY;
export const AWS_REGION = env.AWS_REGION;
export const S3_ACCESS_KEY = env.S3_ACCESS_KEY;
export const S3_SECRET_KEY = env.S3_SECRET_KEY;
export const S3_REGION = env.S3_REGION;
export const S3_ENDPOINT_URL = env.S3_ENDPOINT_URL;
export const S3_BUCKET_NAME = env.S3_BUCKET_NAME;
export const UPLOADS_DIR = "./uploads";
export const UPLOADS_DIR = env.UPLOADS_DIR || "./uploads";
export const MAX_SIZES = {
public: 1024 * 1024 * 10, // 10MB
free: 1024 * 1024 * 10, // 10MB
pro: 1024 * 1024 * 1024, // 1GB
} as const;
export const IS_S3_CONFIGURED: boolean =
env.S3_ACCESS_KEY && env.S3_SECRET_KEY && env.S3_REGION && env.S3_BUCKET_NAME ? true : false;
export const isS3Configured = () => {
// for aws sdk, it can pick up the creds for access key, secret key and the region from the environment variables
if (AWS_ACCESS_KEY_ID && AWS_SECRET_ACCESS_KEY && AWS_REGION) {
// so we only need to check if the bucket name is set
return !!S3_BUCKET_NAME;
}
// for other s3 compatible services, we need to provide the access key and secret key
return S3_ACCESS_KEY && S3_SECRET_KEY && (S3_ENDPOINT_URL ? S3_REGION : true) && S3_BUCKET_NAME
? true
: false;
};
// Pricing
export const PRICING_USERTARGETING_FREE_MTU = 2500;

12
packages/lib/env.ts
View File

@@ -8,8 +8,9 @@ export const env = createEnv({
*/
server: {
AIRTABLE_CLIENT_ID: z.string().optional(),
AWS_ACCESS_KEY: z.string().optional(),
AWS_SECRET_KEY: z.string().optional(),
AWS_ACCESS_KEY_ID: z.string().optional(),
AWS_SECRET_ACCESS_KEY: z.string().optional(),
AWS_REGION: z.string().optional(),
AZUREAD_CLIENT_ID: z.string().optional(),
AZUREAD_CLIENT_SECRET: z.string().optional(),
AZUREAD_TENANT_ID: z.string().optional(),
@@ -77,6 +78,7 @@ export const env = createEnv({
.url()
.optional()
.or(z.string().refine((str) => str === "")),
UPLOADS_DIR: z.string().min(1).optional(),
VERCEL_URL: z.string().optional(),
WEBAPP_URL: z.string().url().optional(),
},
@@ -106,8 +108,9 @@ export const env = createEnv({
*/
runtimeEnv: {
AIRTABLE_CLIENT_ID: process.env.AIRTABLE_CLIENT_ID,
AWS_ACCESS_KEY: process.env.AWS_ACCESS_KEY,
AWS_SECRET_KEY: process.env.AWS_SECRET_KEY,
AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY,
AWS_REGION: process.env.AWS_REGION,
AZUREAD_CLIENT_ID: process.env.AZUREAD_CLIENT_ID,
AZUREAD_CLIENT_SECRET: process.env.AZUREAD_CLIENT_SECRET,
AZUREAD_TENANT_ID: process.env.AZUREAD_TENANT_ID,
@@ -169,6 +172,7 @@ export const env = createEnv({
STRIPE_WEBHOOK_SECRET: process.env.STRIPE_WEBHOOK_SECRET,
TELEMETRY_DISABLED: process.env.TELEMETRY_DISABLED,
TERMS_URL: process.env.TERMS_URL,
UPLOADS_DIR: process.env.UPLOADS_DIR,
VERCEL_URL: process.env.VERCEL_URL,
WEBAPP_URL: process.env.WEBAPP_URL,
},

4
packages/lib/product/service.ts
View File

@@ -11,7 +11,7 @@ import { DatabaseError, ValidationError } from "@formbricks/types/errors";
import type { TProduct, TProductUpdateInput } from "@formbricks/types/product";
import { ZProduct, ZProductUpdateInput } from "@formbricks/types/product";
import { IS_S3_CONFIGURED, ITEMS_PER_PAGE, SERVICES_REVALIDATION_INTERVAL } from "../constants";
import { ITEMS_PER_PAGE, SERVICES_REVALIDATION_INTERVAL, isS3Configured } from "../constants";
import { environmentCache } from "../environment/cache";
import { createEnvironment } from "../environment/service";
import { deleteLocalFilesByEnvironmentId, deleteS3FilesByEnvironmentId } from "../storage/service";
@@ -196,7 +196,7 @@ export const deleteProduct = async (productId: string): Promise<TProduct> => {
if (product) {
// delete all files from storage related to this product
if (IS_S3_CONFIGURED) {
if (isS3Configured()) {
const s3FilesPromises = product.environments.map(async (environment) => {
return deleteS3FilesByEnvironmentId(environment.id);
});

19
packages/lib/storage/service.ts
View File

@@ -19,7 +19,6 @@ import path, { join } from "path";
import { TAccessType } from "@formbricks/types/storage";
import {
IS_S3_CONFIGURED,
MAX_SIZES,
S3_ACCESS_KEY,
S3_BUCKET_NAME,
@@ -28,6 +27,7 @@ import {
S3_SECRET_KEY,
UPLOADS_DIR,
WEBAPP_URL,
isS3Configured,
} from "../constants";
import { generateLocalSignedUrl } from "../crypto";
import { env } from "../env";
@@ -38,15 +38,18 @@ let s3ClientInstance: S3Client | null = null;
export const getS3Client = () => {
if (!s3ClientInstance) {
const credentials =
S3_ACCESS_KEY && S3_SECRET_KEY
? { accessKeyId: S3_ACCESS_KEY, secretAccessKey: S3_SECRET_KEY }
: undefined;
s3ClientInstance = new S3Client({
credentials: {
accessKeyId: S3_ACCESS_KEY!,
secretAccessKey: S3_SECRET_KEY!,
},
credentials,
region: S3_REGION,
endpoint: S3_ENDPOINT_URL,
});
}
return s3ClientInstance;
};
@@ -215,7 +218,7 @@ export const getUploadSignedUrl = async (
const updatedFileName = `${fileNameWithoutExtension}--fid--${randomUUID()}.${fileExtension}`;
// handle the local storage case first
if (!IS_S3_CONFIGURED) {
if (!isS3Configured()) {
try {
const { signature, timestamp, uuid } = generateLocalSignedUrl(updatedFileName, environmentId, fileType);
@@ -329,7 +332,7 @@ export const putFile = async (
environmentId: string
) => {
try {
if (!IS_S3_CONFIGURED) {
if (!isS3Configured()) {
await putFileToLocalStorage(fileName, fileBuffer, accessType, environmentId, UPLOADS_DIR);
return { success: true, message: "File uploaded" };
} else {
@@ -350,7 +353,7 @@ export const putFile = async (
};
export const deleteFile = async (environmentId: string, accessType: TAccessType, fileName: string) => {
if (!IS_S3_CONFIGURED) {
if (!isS3Configured()) {
try {
await deleteLocalFile(path.join(UPLOADS_DIR, environmentId, accessType, fileName));
return { success: true, message: "File deleted" };

6
turbo.json
View File

@@ -56,8 +56,9 @@
"env": [
"AIRTABLE_CLIENT_ID",
"ASSET_PREFIX_URL",
"AWS_ACCESS_KEY",
"AWS_SECRET_KEY",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_REGION",
"AZUREAD_CLIENT_ID",
"AZUREAD_CLIENT_SECRET",
"AZUREAD_TENANT_ID",
@@ -135,6 +136,7 @@
"SURVEYS_PACKAGE_BUILD",
"TELEMETRY_DISABLED",
"TERMS_URL",
"UPLOADS_DIR",
"VERCEL",
"VERCEL_URL",
"WEBAPP_URL"