feat(documents, storage): implement S3 storage driver

apps/app-server/package.json

@@ -24,6 +24,8 @@
     "dev:reset": "pnpm clean:all && pnpm migrate:push"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.722.0",
+    "@aws-sdk/lib-storage": "^3.722.0",
     "@corentinth/chisels": "^1.1.0",
     "@hono/node-server": "^1.13.7",
     "@hono/oauth-providers": "^0.6.2",

apps/app-server/src/modules/config/config.ts

@@ -107,7 +107,7 @@ export const configDefinition = {
         .toLowerCase()
         .transform(x => x === 'true')
         .pipe(z.boolean()),
-      default: true,
+      default: 'true',
       env: 'AUTH_IS_REGISTRATION_ENABLED',
     },
     jwtSecret: {

apps/app-server/src/modules/config/fragments/document-storage.config.ts

@@ -1,6 +1,7 @@
 import type { ConfigDefinition } from 'figue';
 import { z } from 'zod';
 import { FS_STORAGE_DRIVER_NAME } from '../../documents/storage/drivers/fs/fs.storage-driver';
+import { S3_STORAGE_DRIVER_NAME } from '../../documents/storage/drivers/s3/s3.storage-driver';
 
 export const documentStorageConfig = {
   maxUploadSize: {
@@ -11,7 +12,7 @@ export const documentStorageConfig = {
   },
   driver: {
     doc: 'The driver to use for document storage',
-    schema: z.enum([FS_STORAGE_DRIVER_NAME]),
+    schema: z.enum([FS_STORAGE_DRIVER_NAME, S3_STORAGE_DRIVER_NAME]),
     default: FS_STORAGE_DRIVER_NAME,
     env: 'DOCUMENT_STORAGE_DRIVER',
   },
@@ -24,5 +25,37 @@ export const documentStorageConfig = {
         env: 'DOCUMENT_STORAGE_FILESYSTEM_ROOT',
       },
     },
+    s3: {
+      accessKeyId: {
+        doc: 'The AWS access key ID for S3',
+        schema: z.string(),
+        default: '',
+        env: 'DOCUMENT_STORAGE_S3_ACCESS_KEY_ID',
+      },
+      secretAccessKey: {
+        doc: 'The AWS secret access key for S3',
+        schema: z.string(),
+        default: '',
+        env: 'DOCUMENT_STORAGE_S3_SECRET_ACCESS_KEY',
+      },
+      bucketName: {
+        doc: 'The S3 bucket name',
+        schema: z.string(),
+        default: '',
+        env: 'DOCUMENT_STORAGE_S3_BUCKET_NAME',
+      },
+      region: {
+        doc: 'The AWS region for S3',
+        schema: z.string(),
+        default: 'auto',
+        env: 'DOCUMENT_STORAGE_S3_REGION',
+      },
+      endpoint: {
+        doc: 'The S3 endpoint',
+        schema: z.string().optional(),
+        default: undefined,
+        env: 'DOCUMENT_STORAGE_S3_ENDPOINT',
+      },
+    },
   },
 } as const satisfies ConfigDefinition;

apps/app-server/src/modules/documents/documents.usecases.ts

@@ -14,17 +14,17 @@ export async function createDocument({
   documentsRepository: DocumentsRepository;
   documentsStorageService: DocumentStorageService;
 }) {
+  const { storageKey } = await documentsStorageService.saveFile({
+    file,
+    organizationId,
+  });
+
   const {
     name: fileName,
     size,
     type: mimeType,
   } = file;
 
-  const { storageKey } = await documentsStorageService.saveFile({
-    fileStream: file.stream(),
-    fileName,
-  });
-
   const { document } = await documentsRepository.saveOrganizationDocument({
     name: fileName,
     organizationId,

apps/app-server/src/modules/documents/storage/documents.storage.services.ts

@@ -3,9 +3,11 @@ import type { StorageDriver } from './drivers/drivers.models';
 import { injectArguments } from '@corentinth/chisels';
 import { createError } from '../../shared/errors/errors';
 import { FS_STORAGE_DRIVER_NAME, fsStorageDriverFactory } from './drivers/fs/fs.storage-driver';
+import { S3_STORAGE_DRIVER_NAME, s3StorageDriverFactory } from './drivers/s3/s3.storage-driver';
 
 const storageDriverFactories = {
   [FS_STORAGE_DRIVER_NAME]: fsStorageDriverFactory,
+  [S3_STORAGE_DRIVER_NAME]: s3StorageDriverFactory,
 };
 
 export type DocumentStorageService = Awaited<ReturnType<typeof createDocumentStorageService>>;
@@ -33,6 +35,14 @@ export async function createDocumentStorageService({ config }: { config: Config
   });
 }
 
-async function saveFile({ storageDriver, fileStream, fileName }: { storageDriver: StorageDriver; fileStream: ReadableStream; fileName: string }) {
-  return storageDriver.saveFile({ fileStream, fileName });
+async function saveFile({
+  file,
+  organizationId,
+  storageDriver,
+}: {
+  file: File;
+  organizationId: string;
+  storageDriver: StorageDriver;
+}) {
+  return storageDriver.saveFile({ file, organizationId });
 }

apps/app-server/src/modules/documents/storage/drivers/drivers.models.ts

@@ -2,7 +2,10 @@ import type { Config } from '../../../config/config.types';
 
 export type StorageDriver = {
   name: string;
-  saveFile: (args: { fileStream: ReadableStream; fileName: string }) => Promise<{ storageKey: string }>;
+  saveFile: (args: {
+    file: File;
+    organizationId: string;
+  }) => Promise<{ storageKey: string }>;
 };
 
 export type StorageDriverFactory = (args: { config: Config }) => Promise<StorageDriver>;

apps/app-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.test.ts

@@ -2,7 +2,6 @@ import type { Config } from '../../../../config/config.types';
 import fs from 'node:fs';
 import { tmpdir } from 'node:os';
 import path, { join } from 'node:path';
-import stream from 'node:stream';
 import { afterEach, beforeEach, describe, expect, test } from 'vitest';
 import { fsStorageDriverFactory } from './fs.storage-driver';
 import { createFileAlreadyExistsError } from './fs.storage-driver.errors';
@@ -32,16 +31,16 @@ describe('storage driver', () => {
         } as Config;
 
         const fsStorageDriver = await fsStorageDriverFactory({ config });
-        const fileStream = fs.createReadStream(path.join(__dirname, 'fixtures', 'text-file.txt'));
 
         const { storageKey } = await fsStorageDriver.saveFile({
-          fileStream: stream.Readable.toWeb(fileStream),
-          fileName: 'text-file.txt',
+          file: new File(['lorem ipsum'], 'text-file.txt', { type: 'text/plain' }),
+          organizationId: 'org_1',
         });
 
-        expect(storageKey).to.eql(`${tmpDirectory}/text-file.txt`);
+        expect(storageKey).to.eql(`org_1/text-file.txt`);
+        const storagePath = path.join(tmpDirectory, storageKey);
 
-        const fileExists = await fs.promises.access(storageKey, fs.constants.F_OK).then(() => true).catch(() => false);
+        const fileExists = await fs.promises.access(storagePath, fs.constants.F_OK).then(() => true).catch(() => false);
 
         expect(fileExists).to.eql(true);
       });
@@ -58,17 +57,16 @@ describe('storage driver', () => {
         } as Config;
 
         const fsStorageDriver = await fsStorageDriverFactory({ config });
-        const fileStream = fs.createReadStream(path.join(__dirname, 'fixtures', 'text-file.txt'));
 
         await fsStorageDriver.saveFile({
-          fileStream: stream.Readable.toWeb(fileStream),
-          fileName: 'text-file.txt',
+          file: new File(['lorem ipsum'], 'text-file.txt', { type: 'text/plain' }),
+          organizationId: 'org_1',
         });
 
         await expect(
           fsStorageDriver.saveFile({
-            fileStream: stream.Readable.toWeb(fileStream),
-            fileName: 'text-file.txt',
+            file: new File(['lorem ipsum'], 'text-file.txt', { type: 'text/plain' }),
+            organizationId: 'org_1',
           }),
         ).rejects.toThrow(createFileAlreadyExistsError());
       });

apps/app-server/src/modules/documents/storage/drivers/fs/fs.storage-driver.ts

@@ -24,20 +24,20 @@ export const fsStorageDriverFactory = defineStorageDriver(async ({ config }) =>
 
   return {
     name: FS_STORAGE_DRIVER_NAME,
-    saveFile: async ({ fileStream, fileName }) => {
-      // Save file to disk
-      const storageKey = `${root}/${fileName}`;
+    saveFile: async ({ file, organizationId }) => {
+      const storageKey = `${organizationId}/${file.name}`;
+      const storagePath = `${root}/${storageKey}`;
 
-      const fileExists = await checkFileExists({ path: storageKey });
+      const fileExists = await checkFileExists({ path: storagePath });
 
       if (fileExists) {
         throw createFileAlreadyExistsError();
       }
 
-      await ensureDirectoryExists({ path: storageKey });
+      await ensureDirectoryExists({ path: storagePath });
 
-      const writeStream = fs.createWriteStream(storageKey);
-      stream.Readable.fromWeb(fileStream).pipe(writeStream);
+      const writeStream = fs.createWriteStream(storagePath);
+      stream.Readable.fromWeb(file.stream()).pipe(writeStream);
 
       return new Promise((resolve, reject) => {
         writeStream.on('finish', () => {

apps/app-server/src/modules/documents/storage/drivers/s3/s3.storage-driver.ts

@@ -0,0 +1,40 @@
+import { S3Client } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+
+import { defineStorageDriver } from '../drivers.models';
+
+export const S3_STORAGE_DRIVER_NAME = 's3' as const;
+
+export const s3StorageDriverFactory = defineStorageDriver(async ({ config }) => {
+  const { accessKeyId, secretAccessKey, bucketName, region, endpoint } = config.documentsStorage.drivers.s3;
+
+  const s3Client = new S3Client({
+    region,
+    endpoint,
+    credentials: {
+      accessKeyId,
+      secretAccessKey,
+    },
+  });
+
+  return {
+    name: S3_STORAGE_DRIVER_NAME,
+    saveFile: async ({ file, organizationId }) => {
+      const storageKey = `${organizationId}/${file.name}`;
+
+      const upload = new Upload({
+        client: s3Client,
+        params: {
+          Bucket: bucketName,
+          Key: storageKey,
+          Body: file.stream(),
+          ContentLength: file.size,
+        },
+      });
+
+      await upload.done();
+
+      return { storageKey };
+    },
+  };
+});