Compare commits

..

4 Commits

Author SHA1 Message Date
Corentin Thomasset
f382397c0e chore(release): update versions (#489)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2025-09-10 15:38:36 +02:00
Corentin Thomasset
54514e15db fix(translations): update error messages for file size limits across multiple languages (#492) 2025-09-10 15:35:34 +02:00
Corentin Thomasset
bb9d5556d3 fix(upload): properly handle file-too-big errors (#491) 2025-09-10 14:57:46 +02:00
Corentin Thomasset
83e943c5b4 refactor(client): update favicons (#488) 2025-09-09 23:30:27 +02:00
58 changed files with 270 additions and 109 deletions

View File

@@ -1,5 +1,17 @@
# @papra/app-client
## 0.9.1
### Patch Changes
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added a client side guard for rejecting too-big files
- [#488](https://github.com/papra-hq/papra/pull/488) [`83e943c`](https://github.com/papra-hq/papra/commit/83e943c5b46432e55b6dfbaa587019a95ffab466) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix favicons display issues on firefox
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix i18n messages when a file-too-big error happens
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Clean all upload method to happen through the import status modal
## 0.9.0
### Patch Changes

View File

@@ -6,8 +6,7 @@ export default antfu({
},
ignores: [
// Generated file
'src/modules/i18n/locales.types.ts',
'public/manifest.json',
],
rules: {

View File

@@ -27,10 +27,23 @@
<meta property="twitter:image" content="https://papra.app/og-image.png">
<!-- Favicon and Icons -->
<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96" />
<link rel="shortcut icon" href="/favicon.ico" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
<link rel="manifest" href="/site.webmanifest" />
<link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
<link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
<link rel="icon" type="image/png" sizes="192x192" href="/android-icon-192x192.png">
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="96x96" href="/favicon-96x96.png">
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
<link rel="manifest" href="/manifest.json">
<meta name="msapplication-TileColor" content="#ffffff">
<meta name="msapplication-TileImage" content="/ms-icon-144x144.png">
<meta name="theme-color" content="#ffffff">
<!-- Structured Data (JSON-LD for rich snippets) -->
<script type="application/ld+json">

View File

@@ -1,7 +1,7 @@
{
"name": "@papra/app-client",
"type": "module",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"packageManager": "pnpm@10.12.3",
"description": "Papra frontend client",

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.0 KiB

View File

@@ -0,0 +1,2 @@
<?xml version="1.0" encoding="utf-8"?>
<browserconfig><msapplication><tile><square70x70logo src="/ms-icon-70x70.png"/><square150x150logo src="/ms-icon-150x150.png"/><square310x310logo src="/ms-icon-310x310.png"/><TileColor>#ffffff</TileColor></tile></msapplication></browserconfig>

Binary file not shown.

After

Width:  |  Height:  |  Size: 831 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.7 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

View File

@@ -0,0 +1,41 @@
{
"name": "Papra",
"icons": [
{
"src": "\/android-icon-36x36.png",
"sizes": "36x36",
"type": "image\/png",
"density": "0.75"
},
{
"src": "\/android-icon-48x48.png",
"sizes": "48x48",
"type": "image\/png",
"density": "1.0"
},
{
"src": "\/android-icon-72x72.png",
"sizes": "72x72",
"type": "image\/png",
"density": "1.5"
},
{
"src": "\/android-icon-96x96.png",
"sizes": "96x96",
"type": "image\/png",
"density": "2.0"
},
{
"src": "\/android-icon-144x144.png",
"sizes": "144x144",
"type": "image\/png",
"density": "3.0"
},
{
"src": "\/android-icon-192x192.png",
"sizes": "192x192",
"type": "image\/png",
"density": "4.0"
}
]
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.8 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'Das Dokument existiert bereits',
'api-errors.document.file_too_big': 'Die Dokumentdatei ist zu groß',
'api-errors.document.size_too_large': 'Die Datei ist zu groß',
'api-errors.intake_email.limit_reached': 'Die maximale Anzahl an Eingangse-Mails für diese Organisation wurde erreicht. Bitte aktualisieren Sie Ihren Plan, um weitere Eingangse-Mails zu erstellen.',
'api-errors.user.max_organization_count_reached': 'Sie haben die maximale Anzahl an Organisationen erreicht, die Sie erstellen können. Wenn Sie weitere erstellen möchten, kontaktieren Sie bitte den Support.',
'api-errors.default': 'Beim Verarbeiten Ihrer Anfrage ist ein Fehler aufgetreten.',

View File

@@ -538,7 +538,7 @@ export const translations = {
// API errors
'api-errors.document.already_exists': 'The document already exists',
'api-errors.document.file_too_big': 'The document file is too big',
'api-errors.document.size_too_large': 'The file size is too large',
'api-errors.intake_email.limit_reached': 'The maximum number of intake emails for this organization has been reached. Please upgrade your plan to create more intake emails.',
'api-errors.user.max_organization_count_reached': 'You have reached the maximum number of organizations you can create, if you need to create more, please contact support.',
'api-errors.default': 'An error occurred while processing your request.',

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'El documento ya existe',
'api-errors.document.file_too_big': 'El archivo del documento es demasiado grande',
'api-errors.document.size_too_large': 'El archivo es demasiado grande',
'api-errors.intake_email.limit_reached': 'Se ha alcanzado el número máximo de correos de ingreso para esta organización. Por favor, mejora tu plan para crear más correos de ingreso.',
'api-errors.user.max_organization_count_reached': 'Has alcanzado el número máximo de organizaciones que puedes crear, si necesitas crear más, contacta al soporte.',
'api-errors.default': 'Ocurrió un error al procesar tu solicitud.',

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'Le document existe déjà',
'api-errors.document.file_too_big': 'Le fichier du document est trop grand',
'api-errors.document.size_too_large': 'Le fichier est trop volumineux',
'api-errors.intake_email.limit_reached': 'Le nombre maximum d\'emails de réception pour cette organisation a été atteint. Veuillez mettre à niveau votre plan pour créer plus d\'emails de réception.',
'api-errors.user.max_organization_count_reached': 'Vous avez atteint le nombre maximum d\'organisations que vous pouvez créer, si vous avez besoin de créer plus, veuillez contacter le support.',
'api-errors.default': 'Une erreur est survenue lors du traitement de votre requête.',

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'Il documento esiste già',
'api-errors.document.file_too_big': 'Il file del documento è troppo grande',
'api-errors.document.size_too_large': 'Il file è troppo grande',
'api-errors.intake_email.limit_reached': 'È stato raggiunto il numero massimo di email di acquisizione per questa organizzazione. Aggiorna il tuo piano per crearne altre.',
'api-errors.user.max_organization_count_reached': 'Hai raggiunto il numero massimo di organizzazioni che puoi creare, se hai bisogno di crearne altre contatta il supporto.',
'api-errors.default': 'Si è verificato un errore durante l\'elaborazione della richiesta.',

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'Dokument już istnieje',
'api-errors.document.file_too_big': 'Plik dokumentu jest zbyt duży',
'api-errors.document.size_too_large': 'Plik jest zbyt duży',
'api-errors.intake_email.limit_reached': 'Osiągnięto maksymalną liczbę adresów e-mail do przyjęć dla tej organizacji. Aby utworzyć więcej adresów e-mail do przyjęć, zaktualizuj swój plan.',
'api-errors.user.max_organization_count_reached': 'Osiągnięto maksymalną liczbę organizacji, które możesz utworzyć. Jeśli potrzebujesz utworzyć więcej, skontaktuj się z pomocą techniczną.',
'api-errors.default': 'Wystąpił błąd podczas przetwarzania żądania.',

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'O documento já existe',
'api-errors.document.file_too_big': 'O arquivo do documento é muito grande',
'api-errors.document.size_too_large': 'O arquivo é muito grande',
'api-errors.intake_email.limit_reached': 'O número máximo de e-mails de entrada para esta organização foi atingido. Faça um upgrade no seu plano para criar mais e-mails de entrada.',
'api-errors.user.max_organization_count_reached': 'Você atingiu o número máximo de organizações que pode criar. Se precisar criar mais, entre em contato com o suporte.',
'api-errors.default': 'Ocorreu um erro ao processar sua solicitação.',

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'O documento já existe',
'api-errors.document.file_too_big': 'O arquivo do documento é muito grande',
'api-errors.document.size_too_large': 'O arquivo é muito grande',
'api-errors.intake_email.limit_reached': 'O número máximo de e-mails de entrada para esta organização foi atingido. Faça um upgrade no seu plano para criar mais e-mails de entrada.',
'api-errors.user.max_organization_count_reached': 'Atingiu o número máximo de organizações que pode criar. Se precisar de criar mais, entre em contato com o suporte.',
'api-errors.default': 'Ocorreu um erro ao processar a solicitação.',

View File

@@ -540,7 +540,7 @@ export const translations: Partial<TranslationsDictionary> = {
// API errors
'api-errors.document.already_exists': 'Documentul există deja',
'api-errors.document.file_too_big': 'Fișierul documentului este prea mare',
'api-errors.document.size_too_large': 'Fișierul este prea mare',
'api-errors.intake_email.limit_reached': 'Numărul maxim de email-uri de primire pentru această organizație a fost atins. Te rugăm să-ți îmbunătățești planul pentru a crea mai multe email-uri de primire.',
'api-errors.user.max_organization_count_reached': 'Ai atins numărul maxim de organizații pe care le poți crea. Dacă ai nevoie să creezi mai multe, te rugăm să contactezi asistența.',
'api-errors.default': 'A apărut o eroare la procesarea cererii.',

View File

@@ -38,6 +38,9 @@ export const buildTimeConfig = {
isEnabled: asBoolean(import.meta.env.VITE_INTAKE_EMAILS_IS_ENABLED, false),
},
isSubscriptionsEnabled: asBoolean(import.meta.env.VITE_IS_SUBSCRIPTIONS_ENABLED, false),
documentsStorage: {
maxUploadSize: asNumber(import.meta.env.VITE_DOCUMENTS_STORAGE_MAX_UPLOAD_SIZE, 10 * 1024 * 1024),
},
} as const;
export type Config = typeof buildTimeConfig;

View File

@@ -5,6 +5,7 @@ import { A } from '@solidjs/router';
import { throttle } from 'lodash-es';
import { createContext, createSignal, For, Match, Show, Switch, useContext } from 'solid-js';
import { Portal } from 'solid-js/web';
import { useConfig } from '@/modules/config/config.provider';
import { useI18n } from '@/modules/i18n/i18n.provider';
import { promptUploadFiles } from '@/modules/shared/files/upload';
import { useI18nApiErrors } from '@/modules/shared/http/composables/i18n-api-errors';
@@ -57,6 +58,7 @@ export const DocumentUploadProvider: ParentComponent = (props) => {
const throttledInvalidateOrganizationDocumentsQuery = throttle(invalidateOrganizationDocumentsQuery, 500);
const { getErrorMessage } = useI18nApiErrors();
const { t } = useI18n();
const { config } = useConfig();
const [getState, setState] = createSignal<'open' | 'closed' | 'collapsed'>('closed');
const [getTasks, setTasks] = createSignal<Task[]>([]);
@@ -72,6 +74,11 @@ export const DocumentUploadProvider: ParentComponent = (props) => {
await Promise.all(files.map(async (file) => {
updateTaskStatus({ file, status: 'uploading' });
if (file.size > config.documentsStorage.maxUploadSize) {
updateTaskStatus({ file, status: 'error', error: Object.assign(new Error('File too large'), { code: 'document.size_too_large' }) });
return;
}
const [result, error] = await safely(uploadDocument({ file, organizationId }));
if (error) {

View File

@@ -1,11 +1,9 @@
import type { Component } from 'solid-js';
import { useParams } from '@solidjs/router';
import { createSignal } from 'solid-js';
import { promptUploadFiles } from '@/modules/shared/files/upload';
import { queryClient } from '@/modules/shared/query/query-client';
import { cn } from '@/modules/shared/style/cn';
import { Button } from '@/modules/ui/components/button';
import { uploadDocument } from '../documents.services';
import { useDocumentUpload } from './document-import-status.component';
export const DocumentUploadArea: Component<{ organizationId?: string }> = (props) => {
const [isDragging, setIsDragging] = createSignal(false);
@@ -13,21 +11,7 @@ export const DocumentUploadArea: Component<{ organizationId?: string }> = (props
const getOrganizationId = () => props.organizationId ?? params.organizationId;
const uploadFiles = async ({ files }: { files: File[] }) => {
for (const file of files) {
await uploadDocument({ file, organizationId: getOrganizationId() });
}
await queryClient.invalidateQueries({
queryKey: ['organizations', getOrganizationId(), 'documents'],
refetchType: 'all',
});
};
const promptImport = async () => {
const { files } = await promptUploadFiles();
await uploadFiles({ files });
};
const { promptImport, uploadDocuments } = useDocumentUpload({ getOrganizationId });
const handleDragOver = (event: DragEvent) => {
event.preventDefault();
@@ -46,7 +30,7 @@ export const DocumentUploadArea: Component<{ organizationId?: string }> = (props
}
const files = [...event.dataTransfer.files].filter(file => file.type === 'application/pdf');
await uploadFiles({ files });
await uploadDocuments({ files });
};
return (

View File

@@ -1,13 +1,9 @@
import type { Document } from './documents.types';
import { safely } from '@corentinth/chisels';
import { throttle } from 'lodash-es';
import { createSignal } from 'solid-js';
import { useConfirmModal } from '../shared/confirm';
import { promptUploadFiles } from '../shared/files/upload';
import { isHttpErrorWithCode } from '../shared/http/http-errors';
import { queryClient } from '../shared/query/query-client';
import { createToast } from '../ui/components/sonner';
import { deleteDocument, restoreDocument, uploadDocument } from './documents.services';
import { deleteDocument, restoreDocument } from './documents.services';
export function invalidateOrganizationDocumentsQuery({ organizationId }: { organizationId: string }) {
return queryClient.invalidateQueries({
@@ -76,57 +72,3 @@ export function useRestoreDocument() {
},
};
}
function toastUploadError({ error, file }: { error: Error; file: File }) {
if (isHttpErrorWithCode({ error, code: 'document.already_exists' })) {
createToast({
type: 'error',
message: 'Document already exists',
description: `The document ${file.name} already exists, it has not been uploaded.`,
});
return;
}
if (isHttpErrorWithCode({ error, code: 'document.file_too_big' })) {
createToast({
type: 'error',
message: 'Document too big',
description: `The document ${file.name} is too big, it has not been uploaded.`,
});
return;
}
createToast({
type: 'error',
message: 'Failed to upload document',
description: error.message,
});
}
export function useUploadDocuments({ organizationId }: { organizationId: string }) {
const uploadDocuments = async ({ files }: { files: File[] }) => {
const throttledInvalidateOrganizationDocumentsQuery = throttle(invalidateOrganizationDocumentsQuery, 500);
await Promise.all(files.map(async (file) => {
const [, error] = await safely(uploadDocument({ file, organizationId }));
if (error) {
toastUploadError({ error, file });
}
await throttledInvalidateOrganizationDocumentsQuery({ organizationId });
}),
);
};
return {
uploadDocuments,
promptImport: async () => {
const { files } = await promptUploadFiles();
await uploadDocuments({ files });
},
};
}

View File

@@ -1,5 +1,4 @@
import { translations as defaultTranslations } from '@/locales/en.dictionary';
import type { translations as defaultTranslations } from '@/locales/en.dictionary';
export type TranslationKeys = keyof typeof defaultTranslations;
export type TranslationsDictionary = Record<TranslationKeys, string>;

View File

@@ -3,9 +3,9 @@ import { formatBytes } from '@corentinth/chisels';
import { useParams } from '@solidjs/router';
import { createQueries, keepPreviousData } from '@tanstack/solid-query';
import { createSignal, Show, Suspense } from 'solid-js';
import { useDocumentUpload } from '@/modules/documents/components/document-import-status.component';
import { DocumentUploadArea } from '@/modules/documents/components/document-upload-area.component';
import { createdAtColumn, DocumentsPaginatedList, standardActionsColumn, tagsColumn } from '@/modules/documents/components/documents-list.component';
import { useUploadDocuments } from '@/modules/documents/documents.composables';
import { fetchOrganizationDocuments, getOrganizationDocumentsStats } from '@/modules/documents/documents.services';
import { useI18n } from '@/modules/i18n/i18n.provider';
import { Button } from '@/modules/ui/components/button';
@@ -32,7 +32,7 @@ export const OrganizationPage: Component = () => {
],
}));
const { promptImport } = useUploadDocuments({ organizationId: params.organizationId });
const { promptImport } = useDocumentUpload({ getOrganizationId: () => params.organizationId });
return (
<div class="p-6 mt-4 pb-32 max-w-5xl mx-auto">

View File

@@ -1,5 +1,13 @@
# @papra/app-server
## 0.9.1
### Patch Changes
- [#492](https://github.com/papra-hq/papra/pull/492) [`54514e1`](https://github.com/papra-hq/papra/commit/54514e15db5deaffc59dcba34929b5e2e74282e1) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Added a client side guard for rejecting too-big files
- [#491](https://github.com/papra-hq/papra/pull/491) [`bb9d555`](https://github.com/papra-hq/papra/commit/bb9d5556d3f16225ae40ca4d39600999e819b2c4) Thanks [@CorentinTh](https://github.com/CorentinTh)! - Fix cleanup state when a too-big-file is uploaded
## 0.9.0
### Minor Changes

View File

@@ -1,7 +1,7 @@
{
"name": "@papra/app-server",
"type": "module",
"version": "0.9.0",
"version": "0.9.1",
"private": true,
"packageManager": "pnpm@10.12.3",
"description": "Papra app server",

View File

@@ -69,6 +69,9 @@ describe('config models', () => {
intakeEmails: {
isEnabled: true,
},
documentsStorage: {
maxUploadSize: 10485760,
},
},
});
});

View File

@@ -13,6 +13,7 @@ export function getPublicConfig({ config }: { config: Config }) {
'auth.providers.github.isEnabled',
'auth.providers.google.isEnabled',
'documents.deletedDocumentsRetentionDays',
'documentsStorage.maxUploadSize',
'intakeEmails.isEnabled',
]),
{

View File

@@ -0,0 +1,21 @@
import { Buffer } from 'node:buffer';
import { describe, expect, test } from 'vitest';
import { MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD } from './documents.constants';
const unusuallyLongFileName = 'an-unusually-long-file-name-in-order-to-test-the-content-length-header-with-the-metadata-that-are-included-in-the-form-data-so-lorem-ipsum-dolor-sit-amet-consectetur-adipiscing-elit-sed-do-eiusmod-tempor-incididunt-ut-labore-et-dolore-magna-aliqua-ut-enim-ad-minim-veniam-quis-nostrud-exercitation-ullamco-laboris-nisi-ut-aliquip-ex-ea-commodo-consequat-duis-aute-irure-dolor-in-reprehenderit-in-voluptate-velit-esse-cillum-dolore-eu-fugiat-nulla-pariatur-excepteur-sint-occaecat-proident-in-voluptate-velit-esse-cillum-dolore-eu-fugiat-nulla-pariatur-excepteur-sint-occaecat-proident-in-voluptate-velit-esse-cillum-dolore-eu-fugiat-nulla-pariatur-excepteur-sint-occaecat-proident.txt';
describe('documents constants', () => {
// eslint-disable-next-line test/prefer-lowercase-title
describe('MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD', () => {
test('when uploading a formdata multipart, the body has boundaries and other metadata, so the content length is greater than the file size', async () => {
const fileSize = 100;
const formData = new FormData();
formData.append('file', new File(['a'.repeat(fileSize)], unusuallyLongFileName, { type: 'text/plain' }));
const body = new Response(formData);
const contentLength = Buffer.from(await body.arrayBuffer()).length;
expect(contentLength).to.be.greaterThan(fileSize);
expect(contentLength).to.be.lessThan(fileSize + MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD);
});
});
});

View File

@@ -11,3 +11,6 @@ export const ORIGINAL_DOCUMENTS_STORAGE_KEY = 'originals';
// import { ocrLanguages } from '@papra/lecture';
// console.log(JSON.stringify(ocrLanguages));
export const OCR_LANGUAGES = ['afr', 'amh', 'ara', 'asm', 'aze', 'aze_cyrl', 'bel', 'ben', 'bod', 'bos', 'bul', 'cat', 'ceb', 'ces', 'chi_sim', 'chi_tra', 'chr', 'cym', 'dan', 'deu', 'dzo', 'ell', 'eng', 'enm', 'epo', 'est', 'eus', 'fas', 'fin', 'fra', 'frk', 'frm', 'gle', 'glg', 'grc', 'guj', 'hat', 'heb', 'hin', 'hrv', 'hun', 'iku', 'ind', 'isl', 'ita', 'ita_old', 'jav', 'jpn', 'kan', 'kat', 'kat_old', 'kaz', 'khm', 'kir', 'kor', 'kur', 'lao', 'lat', 'lav', 'lit', 'mal', 'mar', 'mkd', 'mlt', 'msa', 'mya', 'nep', 'nld', 'nor', 'ori', 'pan', 'pol', 'por', 'pus', 'ron', 'rus', 'san', 'sin', 'slk', 'slv', 'spa', 'spa_old', 'sqi', 'srp', 'srp_latn', 'swa', 'swe', 'syr', 'tam', 'tel', 'tgk', 'tgl', 'tha', 'tir', 'tur', 'uig', 'ukr', 'urd', 'uzb', 'uzb_cyrl', 'vie', 'yid'] as const;
// When uploading a formdata multipart, the body has boundaries and other metadata that need to be accounted for
export const MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD = 1024; // 1024 bytes

View File

@@ -13,7 +13,7 @@ import { deferTriggerWebhooks } from '../webhooks/webhook.usecases';
import { createDocumentActivityRepository } from './document-activity/document-activity.repository';
import { deferRegisterDocumentActivityLog } from './document-activity/document-activity.usecases';
import { createDocumentIsNotDeletedError } from './documents.errors';
import { formatDocumentForApi, formatDocumentsForApi } from './documents.models';
import { formatDocumentForApi, formatDocumentsForApi, isDocumentSizeLimitEnabled } from './documents.models';
import { createDocumentsRepository } from './documents.repository';
import { documentIdSchema } from './documents.schemas';
import { createDocumentCreationUsecase, deleteAllTrashDocuments, deleteTrashDocument, ensureDocumentExists, getDocumentOrThrow } from './documents.usecases';
@@ -34,6 +34,8 @@ export function registerDocumentsRoutes(context: RouteDefinitionContext) {
}
function setupCreateDocumentRoute({ app, ...deps }: RouteDefinitionContext) {
const { config } = deps;
app.post(
'/api/organizations/:organizationId/documents',
requireAuthentication({ apiKeyPermissions: ['documents:create'] }),
@@ -44,9 +46,12 @@ function setupCreateDocumentRoute({ app, ...deps }: RouteDefinitionContext) {
const { userId } = getUser({ context });
const { organizationId } = context.req.valid('param');
const { maxUploadSize } = config.documentsStorage;
const { fileStream, fileName, mimeType } = await getFileStreamFromMultipartForm({
body: context.req.raw.body,
headers: context.req.header(),
maxFileSize: isDocumentSizeLimitEnabled({ maxUploadSize }) ? maxUploadSize : undefined,
});
const createDocument = createDocumentCreationUsecase({ ...deps });

View File

@@ -14,6 +14,8 @@ import type { DocumentsRepository } from './documents.repository';
import type { Document } from './documents.types';
import type { DocumentStorageService } from './storage/documents.storage.services';
import type { EncryptionContext } from './storage/drivers/drivers.models';
import { PassThrough } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { safely } from '@corentinth/chisels';
import pLimit from 'p-limit';
import { createOrganizationDocumentStorageLimitReachedError } from '../organizations/organizations.errors';
@@ -101,17 +103,27 @@ export async function createDocument({
},
});
const outputStream = fileStream
.pipe(hashStream)
.pipe(byteCountStream);
// Create a PassThrough stream that will be used for saving the file
// This allows us to use pipeline for better error handling
const outputStream = new PassThrough();
const streamProcessingPromise = pipeline(
fileStream,
hashStream,
byteCountStream,
outputStream,
);
// We optimistically save the file to leverage streaming, if the file already exists, we will delete it
const newFileStorageContext = await documentsStorageService.saveFile({
fileStream: outputStream,
storageKey: originalDocumentStorageKey,
mimeType,
fileName,
});
const [newFileStorageContext] = await Promise.all([
documentsStorageService.saveFile({
fileStream: outputStream,
storageKey: originalDocumentStorageKey,
mimeType,
fileName,
}),
streamProcessingPromise,
]);
const hash = getHash();
const size = getByteCount();

View File

@@ -37,6 +37,14 @@ export const fsStorageDriverFactory = defineStorageDriver(({ documentStorageConf
writeStream.on('error', (error) => {
reject(error);
});
// Listen for errors on the input stream as well
fileStream.on('error', (error) => {
// Clean up the write stream and file
writeStream.destroy();
fs.unlink(storagePath, () => {}); // Ignore errors when cleaning up
reject(error);
});
});
},
getFileStream: async ({ storageKey }) => {

View File

@@ -1,4 +1,5 @@
import type { Context } from '../../app/server.types';
import { isNil } from '../utils';
export function getHeader({ context, name }: { context: Context; name: string }) {
return context.req.header(name);
@@ -15,3 +16,13 @@ export function getImpersonatedUserIdFromHeader({ context }: { context: Context
return { impersonatedUserId };
}
export function getContentLengthHeader({ headers }: { headers: Record<string, string> }): number | undefined {
const contentLengthHeaderValue = headers['content-length'] ?? headers['Content-Length'];
if (isNil(contentLengthHeaderValue)) {
return undefined;
}
return Number(contentLengthHeaderValue);
}

View File

@@ -0,0 +1,51 @@
import { describe, expect, test } from 'vitest';
import { isContentLengthPessimisticallyTooLarge } from './file-upload';
describe('file-upload', () => {
describe('isContentLengthPessimisticallyTooLarge', () => {
test(`a file upload request is considered pessimistically too large when
- a content length header is present
- a max file size limit is provided
- the content length is greater than the max file size limit plus an over-estimated overhead due to the multipart form data (boundaries, metadata, etc)`, () => {
expect(
isContentLengthPessimisticallyTooLarge({
contentLength: 1_000,
maxFileSize: 1_000,
overhead: 512,
}),
).to.eql(false);
expect(
isContentLengthPessimisticallyTooLarge({
contentLength: undefined,
maxFileSize: 1_000,
overhead: 512,
}),
).to.eql(false);
expect(
isContentLengthPessimisticallyTooLarge({
contentLength: 1_000,
maxFileSize: undefined,
overhead: 512,
}),
).to.eql(false);
expect(
isContentLengthPessimisticallyTooLarge({
contentLength: undefined,
maxFileSize: undefined,
overhead: 512,
}),
).to.eql(false);
expect(
isContentLengthPessimisticallyTooLarge({
contentLength: 1_513,
maxFileSize: 1_000,
overhead: 512,
}),
).to.eql(true);
});
});
});

View File

@@ -1,18 +1,41 @@
import type { Logger } from '../logger/logger';
import { Readable } from 'node:stream';
import createBusboy from 'busboy';
import { MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD } from '../../documents/documents.constants';
import { createDocumentSizeTooLargeError } from '../../documents/documents.errors';
import { createError } from '../errors/errors';
import { getContentLengthHeader } from '../headers/headers.models';
import { createLogger } from '../logger/logger';
import { isNil } from '../utils';
// Early check to avoid parsing the stream if the content length is set and too large
export function isContentLengthPessimisticallyTooLarge({
contentLength,
maxFileSize,
overhead = MULTIPART_FORM_DATA_SINGLE_FILE_CONTENT_LENGTH_OVERHEAD,
}: {
contentLength?: number;
maxFileSize?: number;
overhead?: number;
}) {
if (isNil(contentLength) || isNil(maxFileSize)) {
return false;
}
return contentLength > maxFileSize + overhead;
}
export async function getFileStreamFromMultipartForm({
body,
headers,
fieldName = 'file',
maxFileSize,
logger = createLogger({ namespace: 'file-upload' }),
}: {
body: ReadableStream | null | undefined;
headers: Record<string, string>;
fieldName?: string;
maxFileSize?: number;
logger?: Logger;
}) {
if (!body) {
@@ -23,12 +46,20 @@ export async function getFileStreamFromMultipartForm({
});
}
const contentLength = getContentLengthHeader({ headers });
if (isContentLengthPessimisticallyTooLarge({ contentLength, maxFileSize })) {
logger.debug({ contentLength, maxFileSize }, 'Content length is pessimistically too large');
throw createDocumentSizeTooLargeError();
}
const { promise, resolve, reject } = Promise.withResolvers<{ fileStream: Readable; fileName: string; mimeType: string }>();
const bb = createBusboy({
headers,
limits: {
files: 1, // Only allow one file
fileSize: maxFileSize,
},
})
.on('file', (formFieldname, fileStream, info) => {
@@ -44,6 +75,11 @@ export async function getFileStreamFromMultipartForm({
}));
}
fileStream.on('limit', () => {
logger.info({ contentLength, maxFileSize }, 'File stream limit reached');
fileStream.destroy(createDocumentSizeTooLargeError());
});
resolve({
fileStream,
fileName: info.filename,